@askalf/deepdive 0.13.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +136 -1
- package/dist/agent.d.ts +3 -1
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +48 -13
- package/dist/agent.js.map +1 -1
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js.map +1 -1
- package/dist/cache.d.ts.map +1 -1
- package/dist/cache.js +22 -2
- package/dist/cache.js.map +1 -1
- package/dist/citations.d.ts +1 -0
- package/dist/citations.d.ts.map +1 -1
- package/dist/citations.js +4 -1
- package/dist/citations.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +428 -19
- package/dist/cli.js.map +1 -1
- package/dist/completion.d.ts +3 -0
- package/dist/completion.d.ts.map +1 -0
- package/dist/completion.js +110 -0
- package/dist/completion.js.map +1 -0
- package/dist/confidence.d.ts +17 -0
- package/dist/confidence.d.ts.map +1 -0
- package/dist/confidence.js +41 -0
- package/dist/confidence.js.map +1 -0
- package/dist/config-file.d.ts +13 -0
- package/dist/config-file.d.ts.map +1 -0
- package/dist/config-file.js +150 -0
- package/dist/config-file.js.map +1 -0
- package/dist/config.d.ts +11 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +7 -0
- package/dist/config.js.map +1 -1
- package/dist/dates.d.ts +9 -0
- package/dist/dates.d.ts.map +1 -0
- package/dist/dates.js +218 -0
- package/dist/dates.js.map +1 -0
- package/dist/diff.d.ts +46 -0
- package/dist/diff.d.ts.map +1 -0
- package/dist/diff.js +227 -0
- package/dist/diff.js.map +1 -0
- package/dist/html-export.d.ts +6 -0
- package/dist/html-export.d.ts.map +1 -0
- package/dist/html-export.js +146 -0
- package/dist/html-export.js.map +1 -0
- package/dist/index.d.ts +11 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -1
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +7 -0
- package/dist/markdown.d.ts.map +1 -0
- package/dist/markdown.js +232 -0
- package/dist/markdown.js.map +1 -0
- package/dist/open.d.ts +6 -0
- package/dist/open.d.ts.map +1 -0
- package/dist/open.js +25 -0
- package/dist/open.js.map +1 -0
- package/dist/profiles.d.ts +5 -0
- package/dist/profiles.d.ts.map +1 -0
- package/dist/profiles.js +38 -0
- package/dist/profiles.js.map +1 -0
- package/dist/robots.d.ts +1 -0
- package/dist/robots.d.ts.map +1 -1
- package/dist/robots.js +4 -7
- package/dist/robots.js.map +1 -1
- package/dist/search/arxiv.d.ts +7 -0
- package/dist/search/arxiv.d.ts.map +1 -0
- package/dist/search/arxiv.js +81 -0
- package/dist/search/arxiv.js.map +1 -0
- package/dist/search/brave.d.ts +1 -1
- package/dist/search/brave.d.ts.map +1 -1
- package/dist/search/brave.js +2 -1
- package/dist/search/brave.js.map +1 -1
- package/dist/search/duckduckgo.d.ts +1 -1
- package/dist/search/duckduckgo.d.ts.map +1 -1
- package/dist/search/duckduckgo.js +2 -1
- package/dist/search/duckduckgo.js.map +1 -1
- package/dist/search/github.d.ts +16 -0
- package/dist/search/github.d.ts.map +1 -0
- package/dist/search/github.js +59 -0
- package/dist/search/github.js.map +1 -0
- package/dist/search/hackernews.d.ts +16 -0
- package/dist/search/hackernews.d.ts.map +1 -0
- package/dist/search/hackernews.js +47 -0
- package/dist/search/hackernews.js.map +1 -0
- package/dist/search/pubmed.d.ts +7 -0
- package/dist/search/pubmed.d.ts.map +1 -0
- package/dist/search/pubmed.js +75 -0
- package/dist/search/pubmed.js.map +1 -0
- package/dist/search/searxng.d.ts +1 -1
- package/dist/search/searxng.d.ts.map +1 -1
- package/dist/search/searxng.js +2 -1
- package/dist/search/searxng.js.map +1 -1
- package/dist/search/stackexchange.d.ts +17 -0
- package/dist/search/stackexchange.d.ts.map +1 -0
- package/dist/search/stackexchange.js +70 -0
- package/dist/search/stackexchange.js.map +1 -0
- package/dist/search/wikipedia.d.ts +16 -0
- package/dist/search/wikipedia.d.ts.map +1 -0
- package/dist/search/wikipedia.js +56 -0
- package/dist/search/wikipedia.js.map +1 -0
- package/dist/search.d.ts +1 -0
- package/dist/search.d.ts.map +1 -1
- package/dist/search.js +43 -0
- package/dist/search.js.map +1 -1
- package/dist/sessions.d.ts +15 -0
- package/dist/sessions.d.ts.map +1 -1
- package/dist/sessions.js +67 -0
- package/dist/sessions.js.map +1 -1
- package/dist/synthesize.d.ts +6 -1
- package/dist/synthesize.d.ts.map +1 -1
- package/dist/synthesize.js +14 -5
- package/dist/synthesize.js.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -9,7 +9,9 @@
|
|
|
9
9
|
// Prints the cited markdown report to stdout (or JSON with --json). Progress
|
|
10
10
|
// events go to stderr when --verbose is set or DEEPDIVE_VERBOSE=1.
|
|
11
11
|
import { writeFileSync } from "node:fs";
|
|
12
|
-
import { resolve } from "node:path";
|
|
12
|
+
import { resolve, join } from "node:path";
|
|
13
|
+
import { tmpdir } from "node:os";
|
|
14
|
+
import { spawn } from "node:child_process";
|
|
13
15
|
import { fileURLToPath } from "node:url";
|
|
14
16
|
import { resolveConfig } from "./config.js";
|
|
15
17
|
import { parseMaxCost, BudgetExceededError } from "./budget.js";
|
|
@@ -21,7 +23,15 @@ import { renderSourcesMarkdown, renderAnswerMarkdown } from "./citations.js";
|
|
|
21
23
|
import { synthesize } from "./synthesize.js";
|
|
22
24
|
import { verifyCitations as runVerify } from "./verify.js";
|
|
23
25
|
import { formatCostLine, looksLikeDario, estimateCost, } from "./pricing.js";
|
|
24
|
-
import { generateSessionId, saveSession, loadSession, listSessions, resolveSessionId, renderSessionsList, } from "./sessions.js";
|
|
26
|
+
import { generateSessionId, saveSession, loadSession, listSessions, resolveSessionId, renderSessionsList, deleteSession, pruneSessions, parseDuration, } from "./sessions.js";
|
|
27
|
+
import { renderHtmlReport } from "./html-export.js";
|
|
28
|
+
import { assessConfidence, formatConfidenceLine } from "./confidence.js";
|
|
29
|
+
import { loadConfigFile, fileConfigToEnv } from "./config-file.js";
|
|
30
|
+
import { resolveProfile } from "./profiles.js";
|
|
31
|
+
import { completionScript } from "./completion.js";
|
|
32
|
+
import { browserOpenCommand } from "./open.js";
|
|
33
|
+
import { diffSessions, renderDiffText, DIFF_NARRATE_SYSTEM, buildDiffNarrateUser, } from "./diff.js";
|
|
34
|
+
import { callLLM } from "./llm.js";
|
|
25
35
|
import { runDoctor, renderDoctorText, renderDoctorJson, exitCodeFor, scrubPath, } from "./doctor.js";
|
|
26
36
|
const USAGE = `deepdive — local research agent
|
|
27
37
|
|
|
@@ -35,6 +45,18 @@ Usage:
|
|
|
35
45
|
deepdive continue <id> [<question>] Full agent run seeded with the saved session's
|
|
36
46
|
sources (plans + searches + fetches new pages;
|
|
37
47
|
saved as a new session linked via parentId)
|
|
48
|
+
deepdive export <id> [--format=html|md] Render a saved session as a shareable artifact
|
|
49
|
+
(--out=report.html). Format inferred from --out.
|
|
50
|
+
deepdive diff <id-a> <id-b> [--narrate] Show how the answer + source set changed between
|
|
51
|
+
two saved runs. --narrate adds an LLM summary.
|
|
52
|
+
deepdive sessions rm <id> [<id>...] Delete one or more saved sessions
|
|
53
|
+
deepdive sessions prune --older-than=30d Delete old sessions (and/or --keep=<n> newest;
|
|
54
|
+
--dry-run to preview)
|
|
55
|
+
deepdive search "<query>" Run just the search adapter, print raw results
|
|
56
|
+
(no LLM/fetch). Honors --search / --json.
|
|
57
|
+
deepdive open <id> Render a session to HTML and open it in the
|
|
58
|
+
browser (--out to keep the file)
|
|
59
|
+
deepdive completion <bash|zsh|fish> Print a shell completion script
|
|
38
60
|
deepdive --help Show this help
|
|
39
61
|
|
|
40
62
|
Flags:
|
|
@@ -50,11 +72,13 @@ Flags:
|
|
|
50
72
|
dollar cap. e.g. --max-cost=$0.50 or --max-cost=5.
|
|
51
73
|
Env: DEEPDIVE_MAX_COST. Exit code 2 on cap-hit.
|
|
52
74
|
--max-tokens=<n> Output max tokens per LLM call. Default: 4096
|
|
53
|
-
--search=<adapter> Search adapter: duckduckgo | searxng | brave | tavily | exa |
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
75
|
+
--search=<adapter> Search adapter: duckduckgo | searxng | brave | tavily | exa |
|
|
76
|
+
auto | wikipedia | arxiv | github | hackernews |
|
|
77
|
+
stackexchange | pubmed
|
|
78
|
+
Default: duckduckgo (no key required). wikipedia, arxiv,
|
|
79
|
+
hackernews, stackexchange, and pubmed need no key; github
|
|
80
|
+
works keyless (DEEPDIVE_GITHUB_TOKEN raises the limit).
|
|
81
|
+
'auto' runs DDG first, Brave fallback (if DEEPDIVE_BRAVE_KEY).
|
|
58
82
|
--results-per-query=<n> Results per sub-query. Default: 5
|
|
59
83
|
--max-sources=<n> Total sources to fetch. Default: 12
|
|
60
84
|
--max-words-per-source=<n> Per-source content cap before synthesis. Default: 2000
|
|
@@ -68,6 +92,9 @@ Flags:
|
|
|
68
92
|
--deep[=<n>] Iterative research: run N additional critic-driven
|
|
69
93
|
rounds after the first synthesis. Default when
|
|
70
94
|
bare: 2. No deep pass when flag absent.
|
|
95
|
+
--profile=<name> Apply a named preset: deep | thorough | fast | cheap |
|
|
96
|
+
strict, or one defined in your config file. Layered
|
|
97
|
+
beneath env + flags. See ~/.deepdive/config.json.
|
|
71
98
|
--concurrency=<n> Parallel fetches. Default: 4
|
|
72
99
|
--no-cache Disable the on-disk page cache (default: enabled)
|
|
73
100
|
--cache-ttl-ms=<ms> Page cache TTL. Default: 3600000 (1 hour)
|
|
@@ -84,13 +111,23 @@ Flags:
|
|
|
84
111
|
exclusively (e.g. github.com,docs.anthropic.com).
|
|
85
112
|
--deny-domain=<list> Comma-separated hostname suffixes to drop
|
|
86
113
|
(e.g. pinterest.com,quora.com).
|
|
114
|
+
--since=<date|duration> Drop sources published before this — an absolute
|
|
115
|
+
date (2024, 2024-06, 2024-06-15) or a duration
|
|
116
|
+
(30d, 12h, 2w = that long ago). Sources with no
|
|
117
|
+
detectable date are kept. Env: DEEPDIVE_SINCE.
|
|
87
118
|
--api-format=<anthropic|openai>
|
|
88
119
|
Wire format for the LLM endpoint. Default:
|
|
89
120
|
auto-detected from --base-url (api.openai.com,
|
|
90
121
|
:11434 (Ollama), :8000 default to openai;
|
|
91
122
|
everything else to anthropic).
|
|
123
|
+
--tldr Lead the answer with a one-paragraph TL;DR (env: DEEPDIVE_TLDR)
|
|
92
124
|
--json Emit a JSON result to stdout instead of markdown
|
|
93
125
|
--out=<path> Write the output (markdown or json) to a file too
|
|
126
|
+
--format=<html|md> export: output format (default: inferred from --out, else html)
|
|
127
|
+
--narrate diff: add a one-shot LLM summary of what changed
|
|
128
|
+
--older-than=<dur> sessions prune: age cutoff — 30d, 12h, 90m, 2w
|
|
129
|
+
--keep=<n> sessions prune: always retain the newest <n> sessions
|
|
130
|
+
--dry-run sessions prune: report what would be deleted, delete nothing
|
|
94
131
|
--verbose, -v Stream progress events to stderr
|
|
95
132
|
--no-stream Buffer the final answer instead of streaming
|
|
96
133
|
tokens to stdout (auto-off for --json and
|
|
@@ -101,15 +138,22 @@ Flags:
|
|
|
101
138
|
Environment:
|
|
102
139
|
DEEPDIVE_BASE_URL, DEEPDIVE_API_KEY, DEEPDIVE_MODEL, DEEPDIVE_SEARCH,
|
|
103
140
|
DEEPDIVE_SEARXNG_URL, DEEPDIVE_BRAVE_KEY, DEEPDIVE_TAVILY_KEY, DEEPDIVE_EXA_KEY,
|
|
141
|
+
DEEPDIVE_WIKIPEDIA_LANG, DEEPDIVE_GITHUB_TOKEN, DEEPDIVE_STACKEXCHANGE_SITE,
|
|
104
142
|
DEEPDIVE_MAX_SOURCES, DEEPDIVE_FETCH_TIMEOUT_MS, DEEPDIVE_HEADED,
|
|
105
143
|
DEEPDIVE_DEEP_ROUNDS, DEEPDIVE_CONCURRENCY, DEEPDIVE_NO_CACHE,
|
|
106
|
-
DEEPDIVE_CACHE_DIR, DEEPDIVE_CACHE_TTL_MS, DEEPDIVE_JSON, DEEPDIVE_VERBOSE,
|
|
144
|
+
DEEPDIVE_CACHE_DIR, DEEPDIVE_CACHE_TTL_MS, DEEPDIVE_JSON, DEEPDIVE_VERBOSE, DEEPDIVE_TLDR,
|
|
107
145
|
DEEPDIVE_LLM_TIMEOUT_MS, DEEPDIVE_LLM_ATTEMPTS,
|
|
108
146
|
DEEPDIVE_NO_VERIFY_CITES, DEEPDIVE_STRICT_CITES, DEEPDIVE_CITE_MIN_RECALL,
|
|
109
147
|
DEEPDIVE_NO_COST, DEEPDIVE_PRICE_INPUT_PER_MTOK, DEEPDIVE_PRICE_OUTPUT_PER_MTOK,
|
|
110
148
|
DEEPDIVE_INCLUDE, DEEPDIVE_PDF_MAX_PAGES,
|
|
111
|
-
DEEPDIVE_ALLOW_DOMAIN, DEEPDIVE_DENY_DOMAIN, DEEPDIVE_API_FORMAT,
|
|
112
|
-
DEEPDIVE_NO_SESSIONS, DEEPDIVE_SESSIONS_DIR
|
|
149
|
+
DEEPDIVE_ALLOW_DOMAIN, DEEPDIVE_DENY_DOMAIN, DEEPDIVE_SINCE, DEEPDIVE_API_FORMAT,
|
|
150
|
+
DEEPDIVE_NO_SESSIONS, DEEPDIVE_SESSIONS_DIR, DEEPDIVE_CONFIG
|
|
151
|
+
|
|
152
|
+
Config file:
|
|
153
|
+
~/.deepdive/config.json (override path with DEEPDIVE_CONFIG) — JSON object of
|
|
154
|
+
default settings (friendly keys: model, search, deep, concurrency, …), an
|
|
155
|
+
optional "profiles" map, and an optional "defaultProfile". Precedence:
|
|
156
|
+
CLI flags > env vars > --profile > config-file base > built-in defaults.
|
|
113
157
|
`;
|
|
114
158
|
// Verbs that accept additional positional arguments. Anything else
|
|
115
159
|
// triggers the "wrap your question in quotes" error when more than one
|
|
@@ -120,6 +164,11 @@ const SUBCOMMAND_VERBS = new Set([
|
|
|
120
164
|
"show",
|
|
121
165
|
"resume",
|
|
122
166
|
"continue",
|
|
167
|
+
"export",
|
|
168
|
+
"diff",
|
|
169
|
+
"completion",
|
|
170
|
+
"search",
|
|
171
|
+
"open",
|
|
123
172
|
]);
|
|
124
173
|
// Exported for unit tests.
|
|
125
174
|
export function parseArgs(argv) {
|
|
@@ -169,6 +218,18 @@ export function parseArgs(argv) {
|
|
|
169
218
|
flags.noStream = true;
|
|
170
219
|
continue;
|
|
171
220
|
}
|
|
221
|
+
if (a === "--tldr") {
|
|
222
|
+
flags.tldr = true;
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
if (a === "--narrate") {
|
|
226
|
+
flags.narrate = true;
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
if (a === "--dry-run") {
|
|
230
|
+
flags.dryRun = true;
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
172
233
|
if (a === "--deep") {
|
|
173
234
|
flags.deepRounds = 2;
|
|
174
235
|
continue;
|
|
@@ -274,6 +335,21 @@ export function parseArgs(argv) {
|
|
|
274
335
|
case "out":
|
|
275
336
|
outPath = value;
|
|
276
337
|
break;
|
|
338
|
+
case "profile":
|
|
339
|
+
flags.profile = value;
|
|
340
|
+
break;
|
|
341
|
+
case "since":
|
|
342
|
+
flags.since = value;
|
|
343
|
+
break;
|
|
344
|
+
case "format":
|
|
345
|
+
flags.format = value.toLowerCase();
|
|
346
|
+
break;
|
|
347
|
+
case "older-than":
|
|
348
|
+
flags.olderThan = value;
|
|
349
|
+
break;
|
|
350
|
+
case "keep":
|
|
351
|
+
flags.keep = parseNonNegativeInt(value);
|
|
352
|
+
break;
|
|
277
353
|
default:
|
|
278
354
|
throw new Error(`unknown flag: --${key}`);
|
|
279
355
|
}
|
|
@@ -440,6 +516,17 @@ async function main(argv) {
|
|
|
440
516
|
process.stdout.write(USAGE);
|
|
441
517
|
return 0;
|
|
442
518
|
}
|
|
519
|
+
// `completion` needs no config; everything else gets config-file + profile
|
|
520
|
+
// defaults layered beneath env (real env wins). Do this before any
|
|
521
|
+
// resolveConfig so all subcommands see the same effective settings.
|
|
522
|
+
if (parsed.question === "completion") {
|
|
523
|
+
return completionCommand(parsed);
|
|
524
|
+
}
|
|
525
|
+
const cfgErr = applyConfigToEnv(parsed.flags);
|
|
526
|
+
if (cfgErr) {
|
|
527
|
+
process.stderr.write(`deepdive: ${cfgErr}\n`);
|
|
528
|
+
return 2;
|
|
529
|
+
}
|
|
443
530
|
if (parsed.question === "doctor") {
|
|
444
531
|
const config = resolveConfig(parsed.flags, process.env);
|
|
445
532
|
const useColor = process.stdout.isTTY && !process.env.NO_COLOR;
|
|
@@ -462,6 +549,18 @@ async function main(argv) {
|
|
|
462
549
|
if (parsed.question === "continue") {
|
|
463
550
|
return await continueCommand(parsed);
|
|
464
551
|
}
|
|
552
|
+
if (parsed.question === "export") {
|
|
553
|
+
return await exportCommand(parsed);
|
|
554
|
+
}
|
|
555
|
+
if (parsed.question === "diff") {
|
|
556
|
+
return await diffCommand(parsed);
|
|
557
|
+
}
|
|
558
|
+
if (parsed.question === "search") {
|
|
559
|
+
return await searchCommand(parsed);
|
|
560
|
+
}
|
|
561
|
+
if (parsed.question === "open") {
|
|
562
|
+
return await openCommand(parsed);
|
|
563
|
+
}
|
|
465
564
|
if (!parsed.question) {
|
|
466
565
|
process.stderr.write(`deepdive: missing question.\n\n${USAGE}`);
|
|
467
566
|
return 2;
|
|
@@ -469,8 +568,144 @@ async function main(argv) {
|
|
|
469
568
|
const config = resolveConfig(parsed.flags, process.env);
|
|
470
569
|
return await runResearch({ question: parsed.question, parsed, config });
|
|
471
570
|
}
|
|
571
|
+
// Layer config-file base + selected-profile settings into process.env, filling
|
|
572
|
+
// only keys the real environment hasn't already set — so the effective
|
|
573
|
+
// precedence is: CLI flags > env vars > profile > config-file base > defaults.
|
|
574
|
+
// Returns an error string for a fatal problem (unknown profile); a malformed
|
|
575
|
+
// config file is a non-fatal warning. Mutating process.env keeps every
|
|
576
|
+
// downstream resolveConfig(flags, process.env) call config-aware with no
|
|
577
|
+
// plumbing changes.
|
|
578
|
+
function applyConfigToEnv(flags) {
|
|
579
|
+
const loaded = loadConfigFile(process.env);
|
|
580
|
+
if (loaded.error) {
|
|
581
|
+
process.stderr.write(`deepdive: warning: ${loaded.error}; ignoring config file\n`);
|
|
582
|
+
}
|
|
583
|
+
const profileName = flags.profile ?? loaded.defaultProfile;
|
|
584
|
+
let profileEnv = {};
|
|
585
|
+
if (profileName) {
|
|
586
|
+
try {
|
|
587
|
+
profileEnv = fileConfigToEnv(resolveProfile(profileName, loaded.profiles));
|
|
588
|
+
}
|
|
589
|
+
catch (err) {
|
|
590
|
+
return safeErrorMessage(err);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
// Profile wins over the file base within the file layer.
|
|
594
|
+
const fileEnv = { ...fileConfigToEnv(loaded.base), ...profileEnv };
|
|
595
|
+
for (const [k, v] of Object.entries(fileEnv)) {
|
|
596
|
+
if (process.env[k] === undefined)
|
|
597
|
+
process.env[k] = v;
|
|
598
|
+
}
|
|
599
|
+
return undefined;
|
|
600
|
+
}
|
|
601
|
+
function completionCommand(parsed) {
|
|
602
|
+
const shell = parsed.extras[0];
|
|
603
|
+
if (shell !== "bash" && shell !== "zsh" && shell !== "fish") {
|
|
604
|
+
process.stderr.write(`deepdive: completion requires a shell: bash | zsh | fish\n` +
|
|
605
|
+
` e.g. source <(deepdive completion bash)\n`);
|
|
606
|
+
return 2;
|
|
607
|
+
}
|
|
608
|
+
process.stdout.write(completionScript(shell));
|
|
609
|
+
return 0;
|
|
610
|
+
}
|
|
611
|
+
// `deepdive search "<query>" [--search=<adapter>] [--json]` — run just the
|
|
612
|
+
// search adapter and print the raw candidate list. No LLM, no fetch, no
|
|
613
|
+
// browser — a cheap way to preview what a backend returns or debug an adapter.
|
|
614
|
+
async function searchCommand(parsed) {
|
|
615
|
+
const config = resolveConfig(parsed.flags, process.env);
|
|
616
|
+
const query = parsed.extras[0];
|
|
617
|
+
if (!query) {
|
|
618
|
+
process.stderr.write(`deepdive: search requires a query (e.g. deepdive search "rust async" --search=hackernews)\n`);
|
|
619
|
+
return 2;
|
|
620
|
+
}
|
|
621
|
+
let adapter;
|
|
622
|
+
try {
|
|
623
|
+
adapter = await resolveSearchAdapter(config.searchAdapter, process.env);
|
|
624
|
+
}
|
|
625
|
+
catch (err) {
|
|
626
|
+
process.stderr.write(`deepdive: ${safeErrorMessage(err)}\n`);
|
|
627
|
+
return 1;
|
|
628
|
+
}
|
|
629
|
+
const ac = new AbortController();
|
|
630
|
+
const sigint = () => ac.abort();
|
|
631
|
+
process.on("SIGINT", sigint);
|
|
632
|
+
process.on("SIGTERM", sigint);
|
|
633
|
+
try {
|
|
634
|
+
const count = parsed.flags.resultsPerQuery ?? 10;
|
|
635
|
+
const results = await adapter.search(query, count, ac.signal);
|
|
636
|
+
if (config.jsonOutput) {
|
|
637
|
+
process.stdout.write(JSON.stringify({ adapter: adapter.name, query, results }, null, 2) + "\n");
|
|
638
|
+
return 0;
|
|
639
|
+
}
|
|
640
|
+
if (results.length === 0) {
|
|
641
|
+
process.stdout.write(`(no results from ${adapter.name} for "${query}")\n`);
|
|
642
|
+
return 0;
|
|
643
|
+
}
|
|
644
|
+
const lines = results.map((r) => {
|
|
645
|
+
const snip = r.snippet
|
|
646
|
+
? `\n ${ellipsize(r.snippet.replace(/\s+/g, " "), 100)}`
|
|
647
|
+
: "";
|
|
648
|
+
return `${String(r.rank).padStart(2)}. ${r.title || r.url}\n ${r.url}${snip}`;
|
|
649
|
+
});
|
|
650
|
+
process.stdout.write(lines.join("\n") + "\n");
|
|
651
|
+
return 0;
|
|
652
|
+
}
|
|
653
|
+
catch (err) {
|
|
654
|
+
process.stderr.write(`deepdive: ${safeErrorMessage(err)}\n`);
|
|
655
|
+
return 1;
|
|
656
|
+
}
|
|
657
|
+
finally {
|
|
658
|
+
process.off("SIGINT", sigint);
|
|
659
|
+
process.off("SIGTERM", sigint);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
// `deepdive open <id> [--out=path]` — render a saved session to a self-
|
|
663
|
+
// contained HTML file (temp dir, or --out) and open it in the default browser.
|
|
664
|
+
// The browser spawn is best-effort; the file path is always printed so a
|
|
665
|
+
// headless box can open it manually.
|
|
666
|
+
async function openCommand(parsed) {
|
|
667
|
+
const config = resolveConfig(parsed.flags, process.env);
|
|
668
|
+
const idArg = parsed.extras[0];
|
|
669
|
+
if (!idArg) {
|
|
670
|
+
process.stderr.write(`deepdive: open requires a session id (try \`deepdive sessions ls\`)\n`);
|
|
671
|
+
return 2;
|
|
672
|
+
}
|
|
673
|
+
try {
|
|
674
|
+
const id = await resolveSessionId(idArg, { dir: config.sessions.dir });
|
|
675
|
+
const record = await loadSession(id, { dir: config.sessions.dir });
|
|
676
|
+
const file = parsed.outPath
|
|
677
|
+
? resolve(parsed.outPath)
|
|
678
|
+
: join(tmpdir(), `deepdive-${id}.html`);
|
|
679
|
+
writeFileSync(file, renderHtmlReport(record), "utf-8");
|
|
680
|
+
const { cmd, args } = browserOpenCommand(process.platform, file);
|
|
681
|
+
try {
|
|
682
|
+
const child = spawn(cmd, args, { stdio: "ignore", detached: true });
|
|
683
|
+
// Opener missing (e.g. headless box with no xdg-open) is non-fatal — the
|
|
684
|
+
// path is printed for manual opening.
|
|
685
|
+
child.on("error", () => undefined);
|
|
686
|
+
child.unref();
|
|
687
|
+
}
|
|
688
|
+
catch {
|
|
689
|
+
/* non-fatal */
|
|
690
|
+
}
|
|
691
|
+
process.stderr.write(`opened ${file}\n`);
|
|
692
|
+
process.stdout.write(file + "\n");
|
|
693
|
+
return 0;
|
|
694
|
+
}
|
|
695
|
+
catch (err) {
|
|
696
|
+
process.stderr.write(`deepdive: ${safeErrorMessage(err)}\n`);
|
|
697
|
+
return 1;
|
|
698
|
+
}
|
|
699
|
+
}
|
|
472
700
|
async function runResearch(opts) {
|
|
473
701
|
const { question, parsed, config, preKept, parentId } = opts;
|
|
702
|
+
// A --since value that was supplied but didn't parse is a user error — fail
|
|
703
|
+
// loud rather than silently running with no recency filter.
|
|
704
|
+
if (config.sinceRaw && config.sinceMs === undefined) {
|
|
705
|
+
process.stderr.write(`deepdive: --since must be a date (2024, 2024-06, 2024-06-15) or a duration ` +
|
|
706
|
+
`(30d, 12h, 2w); got: ${config.sinceRaw}\n`);
|
|
707
|
+
return 2;
|
|
708
|
+
}
|
|
474
709
|
const search = await resolveSearchAdapter(config.searchAdapter, process.env);
|
|
475
710
|
const cache = config.cache.enabled
|
|
476
711
|
? createCache({ dir: config.cache.dir, ttlMs: config.cache.ttlMs })
|
|
@@ -513,6 +748,8 @@ async function runResearch(opts) {
|
|
|
513
748
|
pdfMaxPages: config.pdfMaxPages,
|
|
514
749
|
include: config.include,
|
|
515
750
|
domainFilter: config.domainFilter,
|
|
751
|
+
tldr: config.tldr,
|
|
752
|
+
sinceMs: config.sinceMs,
|
|
516
753
|
env: process.env,
|
|
517
754
|
onEvent: (e) => {
|
|
518
755
|
if (config.verbose)
|
|
@@ -560,6 +797,14 @@ async function runResearch(opts) {
|
|
|
560
797
|
const costLine = config.costEnabled && !config.jsonOutput
|
|
561
798
|
? renderMultiModelCostSummary(result.cost, config.llm.baseUrl)
|
|
562
799
|
: "";
|
|
800
|
+
// Coverage/confidence signal — computed always (goes into --json), shown on
|
|
801
|
+
// stderr alongside the cost summary (suppressed by --no-cost / --json).
|
|
802
|
+
const confidence = assessConfidence({
|
|
803
|
+
sources: result.usage.kept,
|
|
804
|
+
citationsTotal: result.usage.citationsTotal,
|
|
805
|
+
citationsSupported: result.usage.citationsSupported,
|
|
806
|
+
});
|
|
807
|
+
const confidenceLine = config.costEnabled && !config.jsonOutput ? formatConfidenceLine(confidence) : "";
|
|
563
808
|
if (streaming && streamed) {
|
|
564
809
|
// Streaming mode already wrote the header + answer tokens. Close with
|
|
565
810
|
// the sources block, optional citation-health footer, and (if
|
|
@@ -575,6 +820,8 @@ async function runResearch(opts) {
|
|
|
575
820
|
}
|
|
576
821
|
if (costLine)
|
|
577
822
|
process.stderr.write(costLine + "\n");
|
|
823
|
+
if (confidenceLine)
|
|
824
|
+
process.stderr.write(confidenceLine + "\n");
|
|
578
825
|
if (sessionId)
|
|
579
826
|
writeSessionHint(sessionId);
|
|
580
827
|
return strictFail ? 1 : 0;
|
|
@@ -589,11 +836,13 @@ async function runResearch(opts) {
|
|
|
589
836
|
url: s.url,
|
|
590
837
|
title: s.title,
|
|
591
838
|
fetchedAt: s.fetchedAt,
|
|
839
|
+
publishedAt: s.publishedAt,
|
|
592
840
|
})),
|
|
593
841
|
answer: result.answer,
|
|
594
842
|
verification: result.verification,
|
|
595
843
|
cost: result.cost,
|
|
596
844
|
usage: result.usage,
|
|
845
|
+
confidence,
|
|
597
846
|
}, null, 2) + "\n"
|
|
598
847
|
: result.markdown +
|
|
599
848
|
(result.markdown.endsWith("\n") ? "" : "\n") +
|
|
@@ -606,6 +855,8 @@ async function runResearch(opts) {
|
|
|
606
855
|
}
|
|
607
856
|
if (costLine)
|
|
608
857
|
process.stderr.write(costLine + "\n");
|
|
858
|
+
if (confidenceLine)
|
|
859
|
+
process.stderr.write(confidenceLine + "\n");
|
|
609
860
|
if (sessionId)
|
|
610
861
|
writeSessionHint(sessionId);
|
|
611
862
|
return strictFail ? 1 : 0;
|
|
@@ -654,10 +905,19 @@ function writeSessionHint(id) {
|
|
|
654
905
|
async function sessionsCommand(parsed) {
|
|
655
906
|
const config = resolveConfig(parsed.flags, process.env);
|
|
656
907
|
const sub = parsed.extras[0] ?? "ls";
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
908
|
+
switch (sub) {
|
|
909
|
+
case "ls":
|
|
910
|
+
return await sessionsLs(config);
|
|
911
|
+
case "rm":
|
|
912
|
+
return await sessionsRm(parsed, config);
|
|
913
|
+
case "prune":
|
|
914
|
+
return await sessionsPrune(parsed, config);
|
|
915
|
+
default:
|
|
916
|
+
process.stderr.write(`deepdive: unknown sessions sub-command: ${sub} (try: ls | rm | prune)\n`);
|
|
917
|
+
return 2;
|
|
660
918
|
}
|
|
919
|
+
}
|
|
920
|
+
async function sessionsLs(config) {
|
|
661
921
|
const { sessions, bad } = await listSessions({ dir: config.sessions.dir });
|
|
662
922
|
if (config.jsonOutput) {
|
|
663
923
|
process.stdout.write(JSON.stringify({ sessions, bad }, null, 2) + "\n");
|
|
@@ -669,6 +929,151 @@ async function sessionsCommand(parsed) {
|
|
|
669
929
|
}
|
|
670
930
|
return 0;
|
|
671
931
|
}
|
|
932
|
+
async function sessionsRm(parsed, config) {
|
|
933
|
+
const idArgs = parsed.extras.slice(1);
|
|
934
|
+
if (idArgs.length === 0) {
|
|
935
|
+
process.stderr.write(`deepdive: sessions rm requires at least one session id (try \`deepdive sessions ls\`)\n`);
|
|
936
|
+
return 2;
|
|
937
|
+
}
|
|
938
|
+
let failed = 0;
|
|
939
|
+
for (const idArg of idArgs) {
|
|
940
|
+
try {
|
|
941
|
+
const id = await resolveSessionId(idArg, { dir: config.sessions.dir });
|
|
942
|
+
await deleteSession(id, { dir: config.sessions.dir });
|
|
943
|
+
process.stdout.write(`removed ${id}\n`);
|
|
944
|
+
}
|
|
945
|
+
catch (err) {
|
|
946
|
+
failed++;
|
|
947
|
+
process.stderr.write(`deepdive: ${safeErrorMessage(err)}\n`);
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
return failed > 0 ? 1 : 0;
|
|
951
|
+
}
|
|
952
|
+
async function sessionsPrune(parsed, config) {
|
|
953
|
+
const { olderThan, keep, dryRun } = parsed.flags;
|
|
954
|
+
if (olderThan === undefined && keep === undefined) {
|
|
955
|
+
process.stderr.write(`deepdive: sessions prune needs --older-than=<dur> and/or --keep=<n>\n` +
|
|
956
|
+
` e.g. deepdive sessions prune --older-than=30d\n` +
|
|
957
|
+
` deepdive sessions prune --keep=20\n` +
|
|
958
|
+
` deepdive sessions prune --older-than=7d --keep=5 --dry-run\n`);
|
|
959
|
+
return 2;
|
|
960
|
+
}
|
|
961
|
+
let olderThanMs;
|
|
962
|
+
if (olderThan !== undefined) {
|
|
963
|
+
olderThanMs = parseDuration(olderThan);
|
|
964
|
+
if (olderThanMs === undefined) {
|
|
965
|
+
process.stderr.write(`deepdive: --older-than must be a duration like 30d, 12h, 90m, 2w (got: ${olderThan})\n`);
|
|
966
|
+
return 2;
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
const { removed, remaining, bad } = await pruneSessions({ dir: config.sessions.dir }, { olderThanMs, keep, dryRun });
|
|
970
|
+
if (config.jsonOutput) {
|
|
971
|
+
process.stdout.write(JSON.stringify({ removed, remaining, bad, dryRun: !!dryRun }, null, 2) + "\n");
|
|
972
|
+
return 0;
|
|
973
|
+
}
|
|
974
|
+
const verb = dryRun ? "would remove" : "removed";
|
|
975
|
+
process.stdout.write(`${verb} ${removed.length} session${removed.length === 1 ? "" : "s"} · ${remaining} remaining\n`);
|
|
976
|
+
for (const m of removed) {
|
|
977
|
+
const q = m.question.length > 60 ? m.question.slice(0, 59) + "…" : m.question;
|
|
978
|
+
process.stdout.write(` ${dryRun ? "-" : "✓"} ${m.id} ${q}\n`);
|
|
979
|
+
}
|
|
980
|
+
if (bad.length > 0) {
|
|
981
|
+
process.stderr.write(`\n(${bad.length} unparsable session file${bad.length === 1 ? "" : "s"} left in place)\n`);
|
|
982
|
+
}
|
|
983
|
+
return 0;
|
|
984
|
+
}
|
|
985
|
+
// `deepdive export <id> [--format=html|md] [--out=path]` — render a saved
|
|
986
|
+
// session as a shareable artifact. HTML is a single self-contained document;
|
|
987
|
+
// md re-renders the original cited markdown. Format is inferred from --out's
|
|
988
|
+
// extension when not given, defaulting to html.
|
|
989
|
+
async function exportCommand(parsed) {
|
|
990
|
+
const config = resolveConfig(parsed.flags, process.env);
|
|
991
|
+
const idArg = parsed.extras[0];
|
|
992
|
+
if (!idArg) {
|
|
993
|
+
process.stderr.write(`deepdive: export requires a session id (try \`deepdive sessions ls\`)\n`);
|
|
994
|
+
return 2;
|
|
995
|
+
}
|
|
996
|
+
const format = parsed.flags.format ?? inferFormatFromPath(parsed.outPath) ?? "html";
|
|
997
|
+
if (format !== "html" && format !== "md" && format !== "markdown") {
|
|
998
|
+
process.stderr.write(`deepdive: --format must be html or md (got: ${format})\n`);
|
|
999
|
+
return 2;
|
|
1000
|
+
}
|
|
1001
|
+
try {
|
|
1002
|
+
const id = await resolveSessionId(idArg, { dir: config.sessions.dir });
|
|
1003
|
+
const record = await loadSession(id, { dir: config.sessions.dir });
|
|
1004
|
+
const output = format === "html"
|
|
1005
|
+
? renderHtmlReport(record)
|
|
1006
|
+
: renderAnswerMarkdown(record.question, record.answer, record.sources) +
|
|
1007
|
+
renderCitationHealthFooter(record.verification);
|
|
1008
|
+
if (parsed.outPath) {
|
|
1009
|
+
const path = resolve(parsed.outPath);
|
|
1010
|
+
writeFileSync(path, output, "utf-8");
|
|
1011
|
+
process.stderr.write(`wrote ${path}\n`);
|
|
1012
|
+
}
|
|
1013
|
+
else {
|
|
1014
|
+
process.stdout.write(output + (output.endsWith("\n") ? "" : "\n"));
|
|
1015
|
+
}
|
|
1016
|
+
return 0;
|
|
1017
|
+
}
|
|
1018
|
+
catch (err) {
|
|
1019
|
+
process.stderr.write(`deepdive: ${safeErrorMessage(err)}\n`);
|
|
1020
|
+
return 1;
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
function inferFormatFromPath(p) {
|
|
1024
|
+
if (!p)
|
|
1025
|
+
return undefined;
|
|
1026
|
+
const lower = p.toLowerCase();
|
|
1027
|
+
if (lower.endsWith(".html") || lower.endsWith(".htm"))
|
|
1028
|
+
return "html";
|
|
1029
|
+
if (lower.endsWith(".md") || lower.endsWith(".markdown"))
|
|
1030
|
+
return "md";
|
|
1031
|
+
return undefined;
|
|
1032
|
+
}
|
|
1033
|
+
// `deepdive diff <id-a> <id-b> [--narrate] [--json]` — show how the answer
|
|
1034
|
+
// (and the sources behind it) changed between two saved runs. The local-only
|
|
1035
|
+
// longitudinal view. `--narrate` adds a one-shot LLM summary of the change.
|
|
1036
|
+
async function diffCommand(parsed) {
|
|
1037
|
+
const config = resolveConfig(parsed.flags, process.env);
|
|
1038
|
+
const [idArgA, idArgB] = parsed.extras;
|
|
1039
|
+
if (!idArgA || !idArgB) {
|
|
1040
|
+
process.stderr.write(`deepdive: diff requires two session ids (try \`deepdive sessions ls\`)\n`);
|
|
1041
|
+
return 2;
|
|
1042
|
+
}
|
|
1043
|
+
const ac = new AbortController();
|
|
1044
|
+
const sigint = () => ac.abort();
|
|
1045
|
+
process.on("SIGINT", sigint);
|
|
1046
|
+
process.on("SIGTERM", sigint);
|
|
1047
|
+
try {
|
|
1048
|
+
const dir = { dir: config.sessions.dir };
|
|
1049
|
+
const a = await loadSession(await resolveSessionId(idArgA, dir), dir);
|
|
1050
|
+
const b = await loadSession(await resolveSessionId(idArgB, dir), dir);
|
|
1051
|
+
const diff = diffSessions(a, b);
|
|
1052
|
+
let narration;
|
|
1053
|
+
if (parsed.flags.narrate) {
|
|
1054
|
+
const { text } = await callLLM([{ role: "user", content: buildDiffNarrateUser(a, b) }], DIFF_NARRATE_SYSTEM, config.llm, ac.signal);
|
|
1055
|
+
narration = text.trim();
|
|
1056
|
+
}
|
|
1057
|
+
if (config.jsonOutput) {
|
|
1058
|
+
process.stdout.write(JSON.stringify({ diff, narration }, null, 2) + "\n");
|
|
1059
|
+
return 0;
|
|
1060
|
+
}
|
|
1061
|
+
const useColor = process.stdout.isTTY && !process.env.NO_COLOR;
|
|
1062
|
+
process.stdout.write(renderDiffText(diff, { color: useColor }) + "\n");
|
|
1063
|
+
if (narration) {
|
|
1064
|
+
process.stdout.write(`\n## What changed (narrated)\n\n${narration}\n`);
|
|
1065
|
+
}
|
|
1066
|
+
return 0;
|
|
1067
|
+
}
|
|
1068
|
+
catch (err) {
|
|
1069
|
+
process.stderr.write(`deepdive: ${safeErrorMessage(err)}\n`);
|
|
1070
|
+
return 1;
|
|
1071
|
+
}
|
|
1072
|
+
finally {
|
|
1073
|
+
process.off("SIGINT", sigint);
|
|
1074
|
+
process.off("SIGTERM", sigint);
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
672
1077
|
async function showCommand(parsed) {
|
|
673
1078
|
const config = resolveConfig(parsed.flags, process.env);
|
|
674
1079
|
const idArg = parsed.extras[0];
|
|
@@ -717,12 +1122,16 @@ async function resumeCommand(parsed) {
|
|
|
717
1122
|
// v0.10.0 — resume re-synthesizes against saved sources, so use the
|
|
718
1123
|
// synth-stage model (no plan / critic stages in resume mode).
|
|
719
1124
|
const synthLLM = { ...config.llm, model: config.models.synth };
|
|
720
|
-
const answer = await synthesize(newQuestion, record.sources, synthLLM, ac.signal,
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
1125
|
+
const answer = await synthesize(newQuestion, record.sources, synthLLM, ac.signal, {
|
|
1126
|
+
onToken: streaming
|
|
1127
|
+
? (chunk) => {
|
|
1128
|
+
streamed = true;
|
|
1129
|
+
process.stdout.write(chunk);
|
|
1130
|
+
}
|
|
1131
|
+
: undefined,
|
|
1132
|
+
onUsage,
|
|
1133
|
+
tldr: config.tldr,
|
|
1134
|
+
});
|
|
726
1135
|
// Cite verification (final only — no in-loop because there's no loop)
|
|
727
1136
|
let verification;
|
|
728
1137
|
if (config.verifyCitations !== false && answer) {
|