@askalf/deepdive 0.1.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +177 -52
  2. package/dist/agent.d.ts +48 -2
  3. package/dist/agent.d.ts.map +1 -1
  4. package/dist/agent.js +178 -65
  5. package/dist/agent.js.map +1 -1
  6. package/dist/cache.d.ts +16 -0
  7. package/dist/cache.d.ts.map +1 -0
  8. package/dist/cache.js +62 -0
  9. package/dist/cache.js.map +1 -0
  10. package/dist/cli.d.ts +1 -0
  11. package/dist/cli.d.ts.map +1 -1
  12. package/dist/cli.js +164 -18
  13. package/dist/cli.js.map +1 -1
  14. package/dist/concurrency.d.ts +2 -0
  15. package/dist/concurrency.d.ts.map +1 -0
  16. package/dist/concurrency.js +38 -0
  17. package/dist/concurrency.js.map +1 -0
  18. package/dist/config.d.ts +20 -0
  19. package/dist/config.d.ts.map +1 -1
  20. package/dist/config.js +64 -3
  21. package/dist/config.js.map +1 -1
  22. package/dist/doctor.d.ts +44 -0
  23. package/dist/doctor.d.ts.map +1 -0
  24. package/dist/doctor.js +533 -0
  25. package/dist/doctor.js.map +1 -0
  26. package/dist/index.d.ts +9 -3
  27. package/dist/index.d.ts.map +1 -1
  28. package/dist/index.js +9 -3
  29. package/dist/index.js.map +1 -1
  30. package/dist/llm-stream.d.ts +27 -0
  31. package/dist/llm-stream.d.ts.map +1 -0
  32. package/dist/llm-stream.js +173 -0
  33. package/dist/llm-stream.js.map +1 -0
  34. package/dist/llm.d.ts +10 -0
  35. package/dist/llm.d.ts.map +1 -1
  36. package/dist/llm.js +96 -20
  37. package/dist/llm.js.map +1 -1
  38. package/dist/plan.d.ts +7 -0
  39. package/dist/plan.d.ts.map +1 -1
  40. package/dist/plan.js +51 -0
  41. package/dist/plan.js.map +1 -1
  42. package/dist/retry.d.ts +18 -0
  43. package/dist/retry.d.ts.map +1 -0
  44. package/dist/retry.js +70 -0
  45. package/dist/retry.js.map +1 -0
  46. package/dist/robots.d.ts +26 -0
  47. package/dist/robots.d.ts.map +1 -0
  48. package/dist/robots.js +183 -0
  49. package/dist/robots.js.map +1 -0
  50. package/dist/search/duckduckgo.d.ts +2 -0
  51. package/dist/search/duckduckgo.d.ts.map +1 -1
  52. package/dist/search/duckduckgo.js +38 -13
  53. package/dist/search/duckduckgo.js.map +1 -1
  54. package/dist/search/exa.d.ts +17 -0
  55. package/dist/search/exa.d.ts.map +1 -0
  56. package/dist/search/exa.js +62 -0
  57. package/dist/search/exa.js.map +1 -0
  58. package/dist/search/searxng.d.ts.map +1 -1
  59. package/dist/search/searxng.js +2 -1
  60. package/dist/search/searxng.js.map +1 -1
  61. package/dist/search.d.ts.map +1 -1
  62. package/dist/search.js +9 -1
  63. package/dist/search.js.map +1 -1
  64. package/dist/synthesize.d.ts +1 -1
  65. package/dist/synthesize.d.ts.map +1 -1
  66. package/dist/synthesize.js +11 -2
  67. package/dist/synthesize.js.map +1 -1
  68. package/dist/url-util.d.ts +4 -0
  69. package/dist/url-util.d.ts.map +1 -0
  70. package/dist/url-util.js +24 -0
  71. package/dist/url-util.js.map +1 -0
  72. package/package.json +3 -2
package/dist/agent.js CHANGED
@@ -1,99 +1,212 @@
1
- // Main agent loop: plan → search → fetch → extract → synthesize.
1
+ // Main agent loop.
2
2
  //
3
- // Intentionally single-pass for v0.1.0. A follow-up loop (read the answer,
4
- // decide if more searches are needed) is the obvious next step and belongs
5
- // behind a --deep flag so v1 stays cheap and predictable.
3
+ // Single-pass mode (default): plan search fetch → extract → synthesize.
4
+ // Deep mode (--deep[=N]): after the first synthesis, a critic LLM reviews the
5
+ // draft, proposes follow-up queries, and the loop runs another round. Up to
6
+ // `maxRounds` extra rounds, bounded by the source cap.
7
+ //
8
+ // Fetching is concurrent (configurable via browser.concurrency) and optionally
9
+ // backed by an on-disk cache (src/cache.ts) so re-running a question is cheap.
6
10
  import { dedupeByUrl } from "./search.js";
7
- import { planQueries } from "./plan.js";
11
+ import { planQueries, critique } from "./plan.js";
8
12
  import { BrowserSession } from "./browser.js";
9
13
  import { extractContent } from "./extract.js";
10
14
  import { buildSourceTable, renderAnswerMarkdown } from "./citations.js";
11
15
  import { synthesize } from "./synthesize.js";
16
+ import { runConcurrent } from "./concurrency.js";
17
+ import { canFetch, DEFAULT_USER_AGENT, } from "./robots.js";
12
18
  export async function runAgent(question, config, signal) {
13
19
  emit(config, { type: "plan.start", question });
14
20
  const plan = await planQueries(question, config.llm, signal);
15
21
  emit(config, { type: "plan.done", plan });
16
22
  const seenUrls = new Set();
17
- const candidates = [];
18
- for (const query of plan.queries) {
19
- if (signal?.aborted)
20
- throw new Error("aborted");
21
- emit(config, { type: "search.start", query });
22
- const results = await config.search.search(query, config.resultsPerQuery, signal);
23
- const fresh = dedupeByUrl(results).filter((r) => !seenUrls.has(r.url));
24
- for (const r of fresh) {
25
- seenUrls.add(r.url);
26
- candidates.push({ url: r.url, title: r.title, snippet: r.snippet, query });
27
- }
28
- emit(config, { type: "search.done", query, count: fresh.length });
23
+ const allCandidates = [];
24
+ const allQueries = [];
25
+ const keptSources = [];
26
+ const rounds = [];
27
+ let browser = null;
28
+ let answer = "";
29
+ const makeBrowser = config.browserFactory ?? ((opts) => new BrowserSession(opts));
30
+ async function ensureBrowser() {
31
+ if (browser)
32
+ return browser;
33
+ browser = makeBrowser(config.browser);
34
+ await browser.start();
35
+ return browser;
29
36
  }
30
- const toFetch = candidates.slice(0, config.maxSources);
31
- const browser = new BrowserSession(config.browser);
32
- await browser.start();
33
- const fetched = [];
34
37
  try {
35
- for (const c of toFetch) {
38
+ const maxRoundsTotal = 1 + Math.max(0, config.deepRounds);
39
+ let queriesForRound = plan.queries;
40
+ for (let round = 0; round < maxRoundsTotal; round++) {
36
41
  if (signal?.aborted)
37
42
  throw new Error("aborted");
38
- emit(config, { type: "fetch.start", url: c.url });
39
- try {
40
- const page = await browser.fetch(c.url);
41
- const words = (page.text.match(/\S+/g) ?? []).length;
42
- emit(config, {
43
- type: "fetch.done",
44
- url: c.url,
45
- ok: page.status >= 200 && page.status < 400,
46
- status: page.status,
47
- words,
48
- });
49
- if (page.status >= 200 && page.status < 400 && words > 50) {
50
- fetched.push({ ...page, origTitle: c.title });
43
+ if (queriesForRound.length === 0)
44
+ break;
45
+ emit(config, { type: "round.start", round, queries: queriesForRound });
46
+ allQueries.push(...queriesForRound);
47
+ const candidatesBefore = allCandidates.length;
48
+ for (const query of queriesForRound) {
49
+ if (signal?.aborted)
50
+ throw new Error("aborted");
51
+ emit(config, { type: "search.start", query });
52
+ const results = await config.search.search(query, config.resultsPerQuery, signal);
53
+ const fresh = dedupeByUrl(results).filter((r) => !seenUrls.has(r.url));
54
+ for (const r of fresh) {
55
+ seenUrls.add(r.url);
56
+ allCandidates.push({
57
+ url: r.url,
58
+ title: r.title,
59
+ snippet: r.snippet,
60
+ query,
61
+ });
62
+ }
63
+ emit(config, { type: "search.done", query, count: fresh.length });
64
+ }
65
+ const candidatesFoundThisRound = allCandidates.length - candidatesBefore;
66
+ const headroom = Math.max(0, config.maxSources - keptSources.length);
67
+ const toFetch = allCandidates
68
+ .slice(candidatesBefore)
69
+ .slice(0, Math.max(headroom, candidatesFoundThisRound));
70
+ const fetched = await fetchMany(toFetch, config, ensureBrowser, signal);
71
+ for (const f of fetched) {
72
+ if (keptSources.length >= config.maxSources)
73
+ break;
74
+ if (f.page.status >= 200 && f.page.status < 400 && f.words > 50) {
75
+ const sourceId = keptSources.length + 1;
76
+ const extracted = extractContent(f.page.text, f.page.title || f.candidate.title, config.maxWordsPerSource);
77
+ if (extracted.text.length === 0)
78
+ continue;
79
+ keptSources.push({
80
+ id: sourceId,
81
+ url: f.page.finalUrl || f.page.url,
82
+ title: f.page.title || f.candidate.title,
83
+ fetchedAt: f.page.fetchedAt,
84
+ content: extracted.text,
85
+ });
51
86
  }
52
87
  }
53
- catch (err) {
54
- emit(config, {
55
- type: "fetch.done",
56
- url: c.url,
57
- ok: false,
58
- status: 0,
59
- words: 0,
60
- });
88
+ emit(config, {
89
+ type: "synthesize.start",
90
+ sourceCount: keptSources.length,
91
+ round,
92
+ });
93
+ const tokenSink = config.onSynthesizeToken
94
+ ? (chunk) => config.onSynthesizeToken(chunk, round)
95
+ : undefined;
96
+ answer = await synthesize(question, keptSources, config.llm, signal, tokenSink);
97
+ emit(config, { type: "synthesize.done", round });
98
+ const roundTrace = {
99
+ round,
100
+ queries: queriesForRound,
101
+ candidatesFound: candidatesFoundThisRound,
102
+ fetched: fetched.length,
103
+ kept: keptSources.length,
104
+ };
105
+ const isLastPossibleRound = round === maxRoundsTotal - 1;
106
+ if (!isLastPossibleRound && keptSources.length < config.maxSources) {
107
+ emit(config, { type: "critique.start", round });
108
+ const crit = await critique(question, answer, allQueries, config.llm, signal);
109
+ emit(config, { type: "critique.done", round, critique: crit });
110
+ roundTrace.critique = crit;
111
+ rounds.push(roundTrace);
112
+ if (crit.done || crit.queries.length === 0)
113
+ break;
114
+ queriesForRound = crit.queries;
115
+ }
116
+ else {
117
+ rounds.push(roundTrace);
118
+ break;
61
119
  }
62
120
  }
63
121
  }
64
122
  finally {
65
- await browser.close();
123
+ if (browser)
124
+ await browser.close().catch(() => undefined);
66
125
  }
67
- const extracted = [];
68
- const sourceRows = buildSourceTable(fetched.map((f) => ({
69
- url: f.finalUrl || f.url,
70
- title: f.title || f.origTitle,
71
- fetchedAt: f.fetchedAt,
72
- })));
73
- for (let i = 0; i < fetched.length; i++) {
74
- const f = fetched[i];
75
- const row = sourceRows[i];
76
- const ex = extractContent(f.text, f.title || f.origTitle, config.maxWordsPerSource);
77
- if (ex.text.length > 0) {
78
- extracted.push({ ...row, content: ex.text });
79
- }
126
+ const sourceTable = buildSourceTable(keptSources.map((s) => ({ url: s.url, title: s.title, fetchedAt: s.fetchedAt })));
127
+ // Rebind ids to the final table order (keptSources already assigned ids).
128
+ for (let i = 0; i < keptSources.length; i++) {
129
+ keptSources[i].id = sourceTable[i].id;
80
130
  }
81
- emit(config, { type: "synthesize.start", sourceCount: extracted.length });
82
- const answer = await synthesize(question, extracted, config.llm, signal);
83
- emit(config, { type: "synthesize.done" });
84
- const markdown = renderAnswerMarkdown(question, answer, extracted);
131
+ const markdown = renderAnswerMarkdown(question, answer, keptSources);
85
132
  return {
86
133
  question,
87
134
  plan,
88
- sources: extracted,
135
+ sources: keptSources,
136
+ answer,
89
137
  markdown,
138
+ rounds,
90
139
  usage: {
91
- queries: plan.queries.length,
92
- fetched: toFetch.length,
93
- kept: extracted.length,
140
+ queries: allQueries.length,
141
+ fetched: rounds.reduce((sum, r) => sum + r.fetched, 0),
142
+ kept: keptSources.length,
143
+ rounds: rounds.length,
144
+ cacheHits: config.cache?.hits ?? 0,
94
145
  },
95
146
  };
96
147
  }
148
+ async function fetchMany(candidates, config, ensureBrowser, signal) {
149
+ const results = await runConcurrent(candidates, config.concurrency, async (c) => fetchOne(c, config, ensureBrowser), signal);
150
+ return results.filter((r) => r !== null);
151
+ }
152
+ async function fetchOne(c, config, ensureBrowser) {
153
+ if (config.respectRobots !== false) {
154
+ const ua = config.robotsUserAgent ?? DEFAULT_USER_AGENT;
155
+ const result = await canFetch(c.url, {
156
+ userAgent: ua,
157
+ cache: config.robotsCache,
158
+ });
159
+ if (result === "deny") {
160
+ emit(config, { type: "fetch.skipped", url: c.url, reason: "robots" });
161
+ return null;
162
+ }
163
+ }
164
+ if (config.cache) {
165
+ const cached = await config.cache.get(c.url);
166
+ if (cached) {
167
+ const words = (cached.text.match(/\S+/g) ?? []).length;
168
+ emit(config, { type: "fetch.start", url: c.url, cached: true });
169
+ emit(config, {
170
+ type: "fetch.done",
171
+ url: c.url,
172
+ ok: cached.status >= 200 && cached.status < 400,
173
+ status: cached.status,
174
+ words,
175
+ cached: true,
176
+ });
177
+ return { candidate: c, page: cached, words, cached: true };
178
+ }
179
+ }
180
+ emit(config, { type: "fetch.start", url: c.url, cached: false });
181
+ try {
182
+ const browser = await ensureBrowser();
183
+ const page = await browser.fetch(c.url);
184
+ const words = (page.text.match(/\S+/g) ?? []).length;
185
+ if (config.cache && page.status >= 200 && page.status < 400) {
186
+ await config.cache.put(c.url, page).catch(() => undefined);
187
+ }
188
+ emit(config, {
189
+ type: "fetch.done",
190
+ url: c.url,
191
+ ok: page.status >= 200 && page.status < 400,
192
+ status: page.status,
193
+ words,
194
+ cached: false,
195
+ });
196
+ return { candidate: c, page, words, cached: false };
197
+ }
198
+ catch {
199
+ emit(config, {
200
+ type: "fetch.done",
201
+ url: c.url,
202
+ ok: false,
203
+ status: 0,
204
+ words: 0,
205
+ cached: false,
206
+ });
207
+ return null;
208
+ }
209
+ }
97
210
  function emit(config, event) {
98
211
  config.onEvent?.(event);
99
212
  }
package/dist/agent.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agent.js","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAAA,iEAAiE;AACjE,EAAE;AACF,2EAA2E;AAC3E,2EAA2E;AAC3E,0DAA0D;AAI1D,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAa,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,cAAc,EAAyC,MAAM,cAAc,CAAC;AACrF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAe,MAAM,gBAAgB,CAAC;AACrF,OAAO,EAAE,UAAU,EAA0B,MAAM,iBAAiB,CAAC;AA8BrE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,QAAgB,EAChB,MAAmB,EACnB,MAAoB;IAEpB,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC7D,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,UAAU,GAAqE,EAAE,CAAC;IAExF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACvE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACpB,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAEvD,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEtB,MAAM,OAAO,GAA4C,EAAE,CAAC;IAC5D,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;YAChD,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YAClD,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;gBACrD,IAAI,CAAC,MAAM,EAAE;oBACX,IAAI,EAAE,YAAY;oBAClB,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,EAAE,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;oBAC3C,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,KAAK;iBACN,CAAC,CAAC;gBACH,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;oBAC1D,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,EAAE;oBACX,IAAI,EAAE,YAAY;oBAClB,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,EAAE,EAAE,KAAK;oBACT,MAAM,EAAE,CAAC;oBACT,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,MAAM,SAAS,GAAwB,EAAE,CAAC;IAC1C,MAAM,UAAU,GAAG,gBAAgB,CACjC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClB,GAAG,EAAE,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG;QACxB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS;QAC7B,SAAS,EAAE,CAAC,CAAC,SAAS;KACvB,CAAC,CAAC,CACJ,CAAC;IACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC;QACpF,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;IAE1C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAEnE,OAAO;QACL,QAAQ;QACR,IAAI;QACJ,OAAO,EAAE,SAAS;QAClB,QAAQ;QACR,KAAK,EAAE;YACL,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;YAC5B,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,IAAI,EAAE,SAAS,CAAC,MAAM;SACvB;KACF,CAAC;AACJ,CAAC;AAED,SAAS,IAAI,CAAC,MAAmB,EAAE,KAAiB;IAClD,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC"}
1
+ {"version":3,"file":"agent.js","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAAA,mBAAmB;AACnB,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,4EAA4E;AAC5E,uDAAuD;AACvD,EAAE;AACF,+EAA+E;AAC/E,+EAA+E;AAI/E,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,QAAQ,EAA4B,MAAM,WAAW,CAAC;AAC5E,OAAO,EAAE,cAAc,EAAyC,MAAM,cAAc,CAAC;AACrF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAe,MAAM,gBAAgB,CAAC;AACrF,OAAO,EAAE,UAAU,EAA0B,MAAM,iBAAiB,CAAC;AAErE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EACL,QAAQ,EACR,kBAAkB,GAEnB,MAAM,aAAa,CAAC;AAmFrB,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,QAAgB,EAChB,MAAmB,EACnB,MAAoB;IAEpB,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC7D,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,aAAa,GAAgB,EAAE,CAAC;IACtC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,WAAW,GAAwB,EAAE,CAAC;IAC5C,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,IAAI,OAAO,GAAuB,IAAI,CAAC;IACvC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,WAAW,GACf,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC,IAAoB,EAAE,EAAE,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IAEhF,KAAK,UAAU,aAAa;QAC1B,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QAC5B,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QAC1D,IAAI,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC;QAEnC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,cAAc,EAAE,KAAK,EAAE,EAAE,CAAC;YACpD,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;YAChD,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAExC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;YACvE,UAAU,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAEpC,MAAM,gBAAgB,GAAG,aAAa,CAAC,MAAM,CAAC;YAC9C,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;gBACpC,IAAI,MAAM,EAAE,OAAO;oBAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC9C,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CACxC,KAAK,EACL,MAAM,CAAC,eAAe,EACtB,MAAM,CACP,CAAC;gBACF,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACvE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;oBACtB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBACpB,aAAa,CAAC,IAAI,CAAC;wBACjB,GAAG,EAAE,CAAC,CAAC,GAAG;wBACV,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,KAAK;qBACN,CAAC,CAAC;gBACL,CAAC;gBACD,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACpE,CAAC;YACD,MAAM,wBAAwB,GAAG,aAAa,CAAC,MAAM,GAAG,gBAAgB,CAAC;YAEzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;YACrE,MAAM,OAAO,GAAG,aAAa;iBAC1B,KAAK,CAAC,gBAAgB,CAAC;iBACvB,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAC,CAAC;YAE1D,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YAExE,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,IAAI,WAAW,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU;oBAAE,MAAM;gBACnD,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,KAAK,GAAG,EAAE,EAAE,CAAC;oBAChE,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;oBACxC,MAAM,SAAS,GAAG,cAAc,CAC9B,CAAC,CAAC,IAAI,CAAC,IAAI,EACX,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS,CAAC,KAAK,EACjC,MAAM,CAAC,iBAAiB,CACzB,CAAC;oBACF,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;wBAAE,SAAS;oBAC1C,WAAW,CAAC,IAAI,CAAC;wBACf,EAAE,EAAE,QAAQ;wBACZ,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG;wBAClC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS,CAAC,KAAK;wBACxC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS;wBAC3B,OAAO,EAAE,SAAS,CAAC,IAAI;qBACxB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,EAAE;gBACX,IAAI,EAAE,kBAAkB;gBACxB,WAAW,EAAE,WAAW,CAAC,MAAM;gBAC/B,KAAK;aACN,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,CAAC,iBAAiB;gBACxC,CAAC,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,iBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC;gBAC5D,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,GAAG,MAAM,UAAU,CACvB,QAAQ,EACR,WAAW,EACX,MAAM,CAAC,GAAG,EACV,MAAM,EACN,SAAS,CACV,CAAC;YACF,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC,CAAC;YAEjD,MAAM,UAAU,GAAe;gBAC7B,KAAK;gBACL,OAAO,EAAE,eAAe;gBACxB,eAAe,EAAE,wBAAwB;gBACzC,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,IAAI,EAAE,WAAW,CAAC,MAAM;aACzB,CAAC;YAEF,MAAM,mBAAmB,GAAG,KAAK,KAAK,cAAc,GAAG,CAAC,CAAC;YACzD,IAAI,CAAC,mBAAmB,IAAI,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;gBACnE,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC;gBAChD,MAAM,IAAI,GAAG,MAAM,QAAQ,CACzB,QAAQ,EACR,MAAM,EACN,UAAU,EACV,MAAM,CAAC,GAAG,EACV,MAAM,CACP,CAAC;gBACF,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC/D,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC;gBAC3B,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACxB,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;oBAAE,MAAM;gBAClD,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC;YACjC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACxB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,WAAW,GAAG,gBAAgB,CAClC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CACjF,CAAC;IACF,0EAA0E;IAC1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC;IAED,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAErE,OAAO;QACL,QAAQ;QACR,IAAI;QACJ,OAAO,EAAE,WAAW;QACpB,MAAM;QACN,QAAQ;QACR,MAAM;QACN,KAAK,EAAE;YACL,OAAO,EAAE,UAAU,CAAC,MAAM;YAC1B,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACtD,IAAI,EAAE,WAAW,CAAC,MAAM;YACxB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,SAAS,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC;SACnC;KACF,CAAC;AACJ,CAAC;AASD,KAAK,UAAU,SAAS,CACtB,UAAuB,EACvB,MAAmB,EACnB,aAAyC,EACzC,MAAoB;IAEpB,MAAM,OAAO,GAAG,MAAM,aAAa,CACjC,UAAU,EACV,MAAM,CAAC,WAAW,EAClB,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,CAAC,EAC/C,MAAM,CACP,CAAC;IACF,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,CAAY,EACZ,MAAmB,EACnB,aAAyC;IAEzC,IAAI,MAAM,CAAC,aAAa,KAAK,KAAK,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,MAAM,CAAC,eAAe,IAAI,kBAAkB,CAAC;QACxD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,CAAC,CAAC,GAAG,EAAE;YACnC,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,MAAM,CAAC,WAAW;SAC1B,CAAC,CAAC;QACH,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YACtE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YACvD,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,EAAE;gBACX,IAAI,EAAE,YAAY;gBAClB,GAAG,EAAE,CAAC,CAAC,GAAG;gBACV,EAAE,EAAE,MAAM,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG;gBAC/C,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK;gBACL,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;YACH,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC7D,CAAC;IACH,CAAC;IACD,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACjE,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC5D,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,CAAC,MAAM,EAAE;YACX,IAAI,EAAE,YAAY;YAClB,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,EAAE,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;YAC3C,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK;YACL,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QACH,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC,MAAM,EAAE;YACX,IAAI,EAAE,YAAY;YAClB,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,MAAmB,EAAE,KAAiB;IAClD,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC"}
@@ -0,0 +1,16 @@
1
+ import type { FetchedPage } from "./browser.js";
2
+ export interface CacheOptions {
3
+ dir: string;
4
+ ttlMs: number;
5
+ }
6
+ export interface PageCache {
7
+ readonly dir: string;
8
+ readonly ttlMs: number;
9
+ get(url: string): Promise<FetchedPage | null>;
10
+ put(url: string, page: FetchedPage): Promise<void>;
11
+ readonly hits: number;
12
+ readonly misses: number;
13
+ }
14
+ export declare function createCache(opts: CacheOptions): PageCache;
15
+ export declare function cacheKey(url: string): string;
16
+ //# sourceMappingURL=cache.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAC9C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,SAAS,CA+CzD;AAGD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAE5C"}
package/dist/cache.js ADDED
@@ -0,0 +1,62 @@
1
+ // Per-URL on-disk cache for fetched pages. Keyed by SHA-256(url). Stores
2
+ // the full FetchedPage JSON at <dir>/<hash>.json with mtime-based TTL.
3
+ //
4
+ // Why file-based: zero runtime deps, survives process restarts, inspectable,
5
+ // and good enough for a single-user local tool. The cache directory defaults
6
+ // to ~/.deepdive/cache/ but tests inject a tmpdir.
7
+ import { createHash } from "node:crypto";
8
+ import { promises as fs } from "node:fs";
9
+ import { join } from "node:path";
10
+ export function createCache(opts) {
11
+ const state = { hits: 0, misses: 0, mkdired: false };
12
+ async function ensureDir() {
13
+ if (state.mkdired)
14
+ return;
15
+ await fs.mkdir(opts.dir, { recursive: true });
16
+ state.mkdired = true;
17
+ }
18
+ return {
19
+ get dir() {
20
+ return opts.dir;
21
+ },
22
+ get ttlMs() {
23
+ return opts.ttlMs;
24
+ },
25
+ get hits() {
26
+ return state.hits;
27
+ },
28
+ get misses() {
29
+ return state.misses;
30
+ },
31
+ async get(url) {
32
+ const path = join(opts.dir, cacheKey(url) + ".json");
33
+ try {
34
+ const stat = await fs.stat(path);
35
+ if (Date.now() - stat.mtimeMs > opts.ttlMs) {
36
+ state.misses++;
37
+ return null;
38
+ }
39
+ const raw = await fs.readFile(path, "utf-8");
40
+ const page = JSON.parse(raw);
41
+ state.hits++;
42
+ return page;
43
+ }
44
+ catch {
45
+ state.misses++;
46
+ return null;
47
+ }
48
+ },
49
+ async put(url, page) {
50
+ await ensureDir();
51
+ const path = join(opts.dir, cacheKey(url) + ".json");
52
+ const tmp = path + ".tmp." + process.pid;
53
+ await fs.writeFile(tmp, JSON.stringify(page), "utf-8");
54
+ await fs.rename(tmp, path);
55
+ },
56
+ };
57
+ }
58
+ // Exported for unit tests.
59
+ export function cacheKey(url) {
60
+ return createHash("sha256").update(url).digest("hex").slice(0, 32);
61
+ }
62
+ //# sourceMappingURL=cache.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,uEAAuE;AACvE,EAAE;AACF,6EAA6E;AAC7E,6EAA6E;AAC7E,mDAAmD;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAiBjC,MAAM,UAAU,WAAW,CAAC,IAAkB;IAC5C,MAAM,KAAK,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAErD,KAAK,UAAU,SAAS;QACtB,IAAI,KAAK,CAAC,OAAO;YAAE,OAAO;QAC1B,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;IACvB,CAAC;IAED,OAAO;QACL,IAAI,GAAG;YACL,OAAO,IAAI,CAAC,GAAG,CAAC;QAClB,CAAC;QACD,IAAI,KAAK;YACP,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QACD,IAAI,IAAI;YACN,OAAO,KAAK,CAAC,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,MAAM;YACR,OAAO,KAAK,CAAC,MAAM,CAAC;QACtB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,GAAW;YACnB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;YACrD,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC3C,KAAK,CAAC,MAAM,EAAE,CAAC;oBACf,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAgB,CAAC;gBAC5C,KAAK,CAAC,IAAI,EAAE,CAAC;gBACb,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;gBACP,KAAK,CAAC,MAAM,EAAE,CAAC;gBACf,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,IAAiB;YACtC,MAAM,SAAS,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;YACrD,MAAM,GAAG,GAAG,IAAI,GAAG,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;YACzC,MAAM,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC7B,CAAC;KACF,CAAC;AACJ,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACrE,CAAC"}
package/dist/cli.d.ts CHANGED
@@ -7,5 +7,6 @@ interface ParsedArgs {
7
7
  help: boolean;
8
8
  }
9
9
  export declare function parseArgs(argv: string[]): ParsedArgs;
10
+ export declare function safeErrorMessage(err: unknown): string;
10
11
  export {};
11
12
  //# sourceMappingURL=cli.d.ts.map
package/dist/cli.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAaA,OAAO,EAAiB,KAAK,QAAQ,EAAE,MAAM,aAAa,CAAC;AA+B3D,UAAU,UAAU;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,QAAQ,CAAC;IAChB,IAAI,EAAE,OAAO,CAAC;CACf;AAGD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,UAAU,CAsEpD"}
1
+ {"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAcA,OAAO,EAAiB,KAAK,QAAQ,EAAE,MAAM,aAAa,CAAC;AA0D3D,UAAU,UAAU;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,QAAQ,CAAC;IAChB,IAAI,EAAE,OAAO,CAAC;CACf;AAGD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,UAAU,CAyGpD;AAoDD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAIrD"}