@askalf/deepdive 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +176 -52
- package/dist/agent.d.ts +48 -2
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +178 -65
- package/dist/agent.js.map +1 -1
- package/dist/cache.d.ts +16 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +62 -0
- package/dist/cache.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +162 -16
- package/dist/cli.js.map +1 -1
- package/dist/concurrency.d.ts +2 -0
- package/dist/concurrency.d.ts.map +1 -0
- package/dist/concurrency.js +38 -0
- package/dist/concurrency.js.map +1 -0
- package/dist/config.d.ts +20 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +64 -3
- package/dist/config.js.map +1 -1
- package/dist/doctor.d.ts +44 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +533 -0
- package/dist/doctor.js.map +1 -0
- package/dist/index.d.ts +9 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -3
- package/dist/index.js.map +1 -1
- package/dist/llm-stream.d.ts +27 -0
- package/dist/llm-stream.d.ts.map +1 -0
- package/dist/llm-stream.js +173 -0
- package/dist/llm-stream.js.map +1 -0
- package/dist/llm.d.ts +10 -0
- package/dist/llm.d.ts.map +1 -1
- package/dist/llm.js +96 -20
- package/dist/llm.js.map +1 -1
- package/dist/plan.d.ts +7 -0
- package/dist/plan.d.ts.map +1 -1
- package/dist/plan.js +51 -0
- package/dist/plan.js.map +1 -1
- package/dist/retry.d.ts +18 -0
- package/dist/retry.d.ts.map +1 -0
- package/dist/retry.js +70 -0
- package/dist/retry.js.map +1 -0
- package/dist/robots.d.ts +26 -0
- package/dist/robots.d.ts.map +1 -0
- package/dist/robots.js +183 -0
- package/dist/robots.js.map +1 -0
- package/dist/search/duckduckgo.d.ts +2 -0
- package/dist/search/duckduckgo.d.ts.map +1 -1
- package/dist/search/duckduckgo.js +38 -13
- package/dist/search/duckduckgo.js.map +1 -1
- package/dist/search/searxng.d.ts.map +1 -1
- package/dist/search/searxng.js +2 -1
- package/dist/search/searxng.js.map +1 -1
- package/dist/search.d.ts.map +1 -1
- package/dist/search.js +2 -1
- package/dist/search.js.map +1 -1
- package/dist/synthesize.d.ts +1 -1
- package/dist/synthesize.d.ts.map +1 -1
- package/dist/synthesize.js +11 -2
- package/dist/synthesize.js.map +1 -1
- package/dist/url-util.d.ts +4 -0
- package/dist/url-util.d.ts.map +1 -0
- package/dist/url-util.js +24 -0
- package/dist/url-util.js.map +1 -0
- package/package.json +2 -1
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
// Streaming Anthropic Messages client.
|
|
2
|
+
//
|
|
3
|
+
// Used by the synthesizer so tokens land on stdout as the model writes them
|
|
4
|
+
// instead of making the user stare at a blank terminal for 30+ seconds on a
|
|
5
|
+
// deep query. Retry applies to the initial connect only — mid-stream
|
|
6
|
+
// failures propagate because we've already emitted bytes to the caller.
|
|
7
|
+
import { trimTrailingSlashes } from "./url-util.js";
|
|
8
|
+
import { retry } from "./retry.js";
|
|
9
|
+
import { LLMError, DEFAULT_LLM_TIMEOUT_MS, DEFAULT_LLM_ATTEMPTS, } from "./llm.js";
|
|
10
|
+
export async function callLLMStream(messages, system, config, opts = {}, signal) {
|
|
11
|
+
const url = `${trimTrailingSlashes(config.baseUrl)}/v1/messages`;
|
|
12
|
+
const body = {
|
|
13
|
+
model: config.model,
|
|
14
|
+
max_tokens: config.maxTokens,
|
|
15
|
+
system,
|
|
16
|
+
messages,
|
|
17
|
+
stream: true,
|
|
18
|
+
};
|
|
19
|
+
const timeoutMs = config.timeoutMs ?? DEFAULT_LLM_TIMEOUT_MS;
|
|
20
|
+
const attempts = Math.max(1, config.maxAttempts ?? DEFAULT_LLM_ATTEMPTS);
|
|
21
|
+
// Retry wraps the initial connect only. Once we start emitting tokens
|
|
22
|
+
// through onToken, a mid-stream failure can't be undone, so we let it
|
|
23
|
+
// surface to the caller instead of silently retrying.
|
|
24
|
+
const res = await retry(async () => {
|
|
25
|
+
const combined = makeTimeoutSignal(timeoutMs, signal);
|
|
26
|
+
const r = await fetch(url, {
|
|
27
|
+
method: "POST",
|
|
28
|
+
headers: {
|
|
29
|
+
"content-type": "application/json",
|
|
30
|
+
"anthropic-version": "2023-06-01",
|
|
31
|
+
"x-api-key": config.apiKey,
|
|
32
|
+
accept: "text/event-stream",
|
|
33
|
+
},
|
|
34
|
+
body: JSON.stringify(body),
|
|
35
|
+
signal: combined,
|
|
36
|
+
});
|
|
37
|
+
if (!r.ok) {
|
|
38
|
+
const detail = await r.text().catch(() => "");
|
|
39
|
+
throw new LLMError(r.status, `LLM ${r.status} ${r.statusText}: ${clip(detail, 500)}`, detail);
|
|
40
|
+
}
|
|
41
|
+
return r;
|
|
42
|
+
}, {
|
|
43
|
+
attempts,
|
|
44
|
+
baseDelayMs: 500,
|
|
45
|
+
maxDelayMs: 8_000,
|
|
46
|
+
jitter: 0.25,
|
|
47
|
+
signal,
|
|
48
|
+
shouldRetry: (err) => {
|
|
49
|
+
if (err instanceof LLMError)
|
|
50
|
+
return err.retriable;
|
|
51
|
+
if (isUserAbort(err, signal))
|
|
52
|
+
return false;
|
|
53
|
+
return true;
|
|
54
|
+
},
|
|
55
|
+
});
|
|
56
|
+
if (!res.body) {
|
|
57
|
+
throw new Error("LLM response has no stream body");
|
|
58
|
+
}
|
|
59
|
+
let text = "";
|
|
60
|
+
let usage;
|
|
61
|
+
for await (const event of parseSSE(res.body, signal)) {
|
|
62
|
+
const type = event.type;
|
|
63
|
+
if (type === "content_block_delta" && event.delta?.type === "text_delta") {
|
|
64
|
+
const chunk = event.delta.text ?? "";
|
|
65
|
+
if (chunk) {
|
|
66
|
+
text += chunk;
|
|
67
|
+
opts.onToken?.(chunk);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
else if (type === "message_delta" && event.usage) {
|
|
71
|
+
// The streaming API reports final output_tokens in message_delta; the
|
|
72
|
+
// input_tokens arrived in message_start.
|
|
73
|
+
usage = { ...(usage ?? { input_tokens: 0, output_tokens: 0 }), ...event.usage };
|
|
74
|
+
}
|
|
75
|
+
else if (type === "message_start" && event.message?.usage) {
|
|
76
|
+
usage = {
|
|
77
|
+
input_tokens: event.message.usage.input_tokens ?? 0,
|
|
78
|
+
output_tokens: event.message.usage.output_tokens ?? 0,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return { text, usage };
|
|
83
|
+
}
|
|
84
|
+
// Exported for unit tests.
|
|
85
|
+
export async function* parseSSE(stream, signal) {
|
|
86
|
+
const reader = stream.getReader();
|
|
87
|
+
const decoder = new TextDecoder();
|
|
88
|
+
let buffer = "";
|
|
89
|
+
try {
|
|
90
|
+
while (true) {
|
|
91
|
+
if (signal?.aborted)
|
|
92
|
+
throw new Error("aborted");
|
|
93
|
+
const { done, value } = await reader.read();
|
|
94
|
+
if (done) {
|
|
95
|
+
// Flush any trailing event without blank-line terminator.
|
|
96
|
+
if (buffer.trim().length > 0) {
|
|
97
|
+
for (const ev of parseBlocks(buffer))
|
|
98
|
+
yield ev;
|
|
99
|
+
}
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
buffer += decoder.decode(value, { stream: true });
|
|
103
|
+
let eventStart = 0;
|
|
104
|
+
while (true) {
|
|
105
|
+
// SSE frames are terminated by \n\n (or \r\n\r\n).
|
|
106
|
+
const idx = indexOfBlankLine(buffer, eventStart);
|
|
107
|
+
if (idx === -1)
|
|
108
|
+
break;
|
|
109
|
+
const block = buffer.slice(eventStart, idx);
|
|
110
|
+
for (const ev of parseBlocks(block))
|
|
111
|
+
yield ev;
|
|
112
|
+
eventStart = idx + (buffer[idx + 1] === "\r" ? 4 : 2);
|
|
113
|
+
}
|
|
114
|
+
if (eventStart > 0)
|
|
115
|
+
buffer = buffer.slice(eventStart);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
reader.releaseLock();
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
function indexOfBlankLine(s, from) {
|
|
123
|
+
const a = s.indexOf("\n\n", from);
|
|
124
|
+
const b = s.indexOf("\r\n\r\n", from);
|
|
125
|
+
if (a === -1)
|
|
126
|
+
return b;
|
|
127
|
+
if (b === -1)
|
|
128
|
+
return a;
|
|
129
|
+
return Math.min(a, b);
|
|
130
|
+
}
|
|
131
|
+
// Exported for unit tests. Parses one SSE frame (which may be multi-line) and
|
|
132
|
+
// yields the decoded data payload(s). A frame with no `data:` line yields
|
|
133
|
+
// nothing.
|
|
134
|
+
export function* parseBlocks(block) {
|
|
135
|
+
const lines = block.split(/\r?\n/);
|
|
136
|
+
const dataLines = [];
|
|
137
|
+
for (const line of lines) {
|
|
138
|
+
if (line.startsWith("data:")) {
|
|
139
|
+
dataLines.push(line.slice(5).replace(/^ /, ""));
|
|
140
|
+
}
|
|
141
|
+
// `event:` and other fields are ignored; the `data` payload's `type`
|
|
142
|
+
// carries the semantic event name for our purposes.
|
|
143
|
+
}
|
|
144
|
+
if (dataLines.length === 0)
|
|
145
|
+
return;
|
|
146
|
+
const raw = dataLines.join("\n");
|
|
147
|
+
if (raw === "[DONE]")
|
|
148
|
+
return;
|
|
149
|
+
try {
|
|
150
|
+
yield JSON.parse(raw);
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
// Malformed frame — silently ignore. SSE allows heartbeat-style `:` lines
|
|
154
|
+
// and similar that shouldn't crash the stream.
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function makeTimeoutSignal(timeoutMs, userSignal) {
|
|
158
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
159
|
+
if (!userSignal)
|
|
160
|
+
return timeout;
|
|
161
|
+
return AbortSignal.any([userSignal, timeout]);
|
|
162
|
+
}
|
|
163
|
+
function isUserAbort(err, userSignal) {
|
|
164
|
+
if (!userSignal?.aborted)
|
|
165
|
+
return false;
|
|
166
|
+
const msg = err?.message ?? "";
|
|
167
|
+
const name = err?.name ?? "";
|
|
168
|
+
return name === "AbortError" || /abort/i.test(msg);
|
|
169
|
+
}
|
|
170
|
+
function clip(s, max) {
|
|
171
|
+
return s.length <= max ? s : s.slice(0, max - 1) + "…";
|
|
172
|
+
}
|
|
173
|
+
//# sourceMappingURL=llm-stream.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm-stream.js","sourceRoot":"","sources":["../src/llm-stream.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,EAAE;AACF,4EAA4E;AAC5E,4EAA4E;AAC5E,qEAAqE;AACrE,wEAAwE;AAExE,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EACL,QAAQ,EACR,sBAAsB,EACtB,oBAAoB,GAIrB,MAAM,UAAU,CAAC;AAMlB,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAsB,EACtB,MAAc,EACd,MAAiB,EACjB,OAAsB,EAAE,EACxB,MAAoB;IAEpB,MAAM,GAAG,GAAG,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC;IACjE,MAAM,IAAI,GAAG;QACX,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,MAAM,CAAC,SAAS;QAC5B,MAAM;QACN,QAAQ;QACR,MAAM,EAAE,IAAI;KACb,CAAC;IACF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,sBAAsB,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,oBAAoB,CAAC,CAAC;IAEzE,sEAAsE;IACtE,sEAAsE;IACtE,sDAAsD;IACtD,MAAM,GAAG,GAAG,MAAM,KAAK,CACrB,KAAK,IAAI,EAAE;QACT,MAAM,QAAQ,GAAG,iBAAiB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtD,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACzB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,mBAAmB,EAAE,YAAY;gBACjC,WAAW,EAAE,MAAM,CAAC,MAAM;gBAC1B,MAAM,EAAE,mBAAmB;aAC5B;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,QAAQ;SACjB,CAAC,CAAC;QACH,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,QAAQ,CAChB,CAAC,CAAC,MAAM,EACR,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,KAAK,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EACvD,MAAM,CACP,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC,EACD;QACE,QAAQ;QACR,WAAW,EAAE,GAAG;QAChB,UAAU,EAAE,KAAK;QACjB,MAAM,EAAE,IAAI;QACZ,MAAM;QACN,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE;YACnB,IAAI,GAAG,YAAY,QAAQ;gBAAE,OAAO,GAAG,CAAC,SAAS,CAAC;YAClD,IAAI,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC3C,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CACF,CAAC;IAEF,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,KAAyB,CAAC;IAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACxB,IAAI,IAAI,KAAK,qBAAqB,IAAI,KAAK,CAAC,KAAK,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;YACzE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;YACrC,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,IAAI,KAAK,CAAC;gBACd,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,KAAK,eAAe,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YACnD,sEAAsE;YACtE,yCAAyC;YACzC,KAAK,GAAG,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAClF,CAAC;aAAM,IAAI,IAAI,KAAK,eAAe,IAAI,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;YAC5D,KAAK,GAAG;gBACN,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC;gBACnD,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC;aACtD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACzB,CAAC;AAUD,2BAA2B;AAC3B,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,QAAQ,CAC7B,MAAkC,EAClC,MAAoB;IAEpB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;YAChD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI,EAAE,CAAC;gBACT,0DAA0D;gBAC1D,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7B,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,MAAM,CAAC;wBAAE,MAAM,EAAE,CAAC;gBACjD,CAAC;gBACD,OAAO;YACT,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,OAAO,IAAI,EAAE,CAAC;gBACZ,mDAAmD;gBACnD,MAAM,GAAG,GAAG,gBAAgB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBACjD,IAAI,GAAG,KAAK,CAAC,CAAC;oBAAE,MAAM;gBACtB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;gBAC5C,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,KAAK,CAAC;oBAAE,MAAM,EAAE,CAAC;gBAC9C,UAAU,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxD,CAAC;YACD,IAAI,UAAU,GAAG,CAAC;gBAAE,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAS,EAAE,IAAY;IAC/C,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAClC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACtC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACvB,IAAI,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACvB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxB,CAAC;AAED,8EAA8E;AAC9E,0EAA0E;AAC1E,WAAW;AACX,MAAM,SAAS,CAAC,CAAC,WAAW,CAAC,KAAa;IACxC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QAClD,CAAC;QACD,qEAAqE;QACrE,oDAAoD;IACtD,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACnC,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,GAAG,KAAK,QAAQ;QAAE,OAAO;IAC7B,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAa,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,0EAA0E;QAC1E,+CAA+C;IACjD,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,SAAiB,EACjB,UAAwB;IAExB,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,IAAI,CAAC,UAAU;QAAE,OAAO,OAAO,CAAC;IAChC,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,WAAW,CAAC,GAAY,EAAE,UAAwB;IACzD,IAAI,CAAC,UAAU,EAAE,OAAO;QAAE,OAAO,KAAK,CAAC;IACvC,MAAM,GAAG,GAAI,GAAa,EAAE,OAAO,IAAI,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAI,GAAa,EAAE,IAAI,IAAI,EAAE,CAAC;IACxC,OAAO,IAAI,KAAK,YAAY,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,IAAI,CAAC,CAAS,EAAE,GAAW;IAClC,OAAO,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;AACzD,CAAC"}
|
package/dist/llm.d.ts
CHANGED
|
@@ -3,6 +3,8 @@ export interface LLMConfig {
|
|
|
3
3
|
apiKey: string;
|
|
4
4
|
model: string;
|
|
5
5
|
maxTokens: number;
|
|
6
|
+
timeoutMs?: number;
|
|
7
|
+
maxAttempts?: number;
|
|
6
8
|
}
|
|
7
9
|
export interface LLMMessage {
|
|
8
10
|
role: "user" | "assistant";
|
|
@@ -15,5 +17,13 @@ export interface LLMResult {
|
|
|
15
17
|
output_tokens: number;
|
|
16
18
|
};
|
|
17
19
|
}
|
|
20
|
+
export declare class LLMError extends Error {
|
|
21
|
+
readonly status: number;
|
|
22
|
+
readonly detail: string;
|
|
23
|
+
constructor(status: number, message: string, detail?: string);
|
|
24
|
+
get retriable(): boolean;
|
|
25
|
+
}
|
|
26
|
+
export declare const DEFAULT_LLM_TIMEOUT_MS = 120000;
|
|
27
|
+
export declare const DEFAULT_LLM_ATTEMPTS = 3;
|
|
18
28
|
export declare function callLLM(messages: LLMMessage[], system: string, config: LLMConfig, signal?: AbortSignal): Promise<LLMResult>;
|
|
19
29
|
//# sourceMappingURL=llm.d.ts.map
|
package/dist/llm.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;CACzD;AAED,qBAAa,QAAS,SAAQ,KAAK;aAEf,MAAM,EAAE,MAAM;aAEd,MAAM,EAAE,MAAM;gBAFd,MAAM,EAAE,MAAM,EAC9B,OAAO,EAAE,MAAM,EACC,MAAM,GAAE,MAAW;IAKrC,IAAI,SAAS,IAAI,OAAO,CAEvB;CACF;AAED,eAAO,MAAM,sBAAsB,SAAU,CAAC;AAC9C,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC,wBAAsB,OAAO,CAC3B,QAAQ,EAAE,UAAU,EAAE,EACtB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,SAAS,CAAC,CAoEpB"}
|
package/dist/llm.js
CHANGED
|
@@ -1,33 +1,109 @@
|
|
|
1
|
-
// Minimal Anthropic Messages-compatible client.
|
|
1
|
+
// Minimal Anthropic Messages-compatible client with retry + per-call timeout.
|
|
2
|
+
//
|
|
2
3
|
// Default target is dario at http://localhost:3456, but any Anthropic-compat
|
|
3
|
-
// endpoint works.
|
|
4
|
+
// endpoint works. A single deep-research query can fire 30+ LLM calls; at a
|
|
5
|
+
// 1% per-call failure rate the overall success rate would be 74%. Retries
|
|
6
|
+
// with exponential backoff bring that above 99%.
|
|
7
|
+
//
|
|
8
|
+
// Retry policy: 3 attempts total, retries fetch-level errors (network,
|
|
9
|
+
// timeout) and HTTP 5xx / 429. Never retries 4xx (other than 429) — those
|
|
10
|
+
// indicate malformed requests, not transient failures.
|
|
11
|
+
import { trimTrailingSlashes } from "./url-util.js";
|
|
12
|
+
import { retry } from "./retry.js";
|
|
13
|
+
export class LLMError extends Error {
|
|
14
|
+
status;
|
|
15
|
+
detail;
|
|
16
|
+
constructor(status, message, detail = "") {
|
|
17
|
+
super(message);
|
|
18
|
+
this.status = status;
|
|
19
|
+
this.detail = detail;
|
|
20
|
+
this.name = "LLMError";
|
|
21
|
+
}
|
|
22
|
+
get retriable() {
|
|
23
|
+
return this.status === 429 || (this.status >= 500 && this.status <= 599);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export const DEFAULT_LLM_TIMEOUT_MS = 120_000;
|
|
27
|
+
export const DEFAULT_LLM_ATTEMPTS = 3;
|
|
4
28
|
export async function callLLM(messages, system, config, signal) {
|
|
5
|
-
const url = `${config.baseUrl
|
|
29
|
+
const url = `${trimTrailingSlashes(config.baseUrl)}/v1/messages`;
|
|
6
30
|
const body = {
|
|
7
31
|
model: config.model,
|
|
8
32
|
max_tokens: config.maxTokens,
|
|
9
33
|
system,
|
|
10
34
|
messages,
|
|
11
35
|
};
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
36
|
+
const timeoutMs = config.timeoutMs ?? DEFAULT_LLM_TIMEOUT_MS;
|
|
37
|
+
const attempts = Math.max(1, config.maxAttempts ?? DEFAULT_LLM_ATTEMPTS);
|
|
38
|
+
return retry(async () => {
|
|
39
|
+
const combined = makeTimeoutSignal(timeoutMs, signal);
|
|
40
|
+
let res;
|
|
41
|
+
try {
|
|
42
|
+
res = await fetch(url, {
|
|
43
|
+
method: "POST",
|
|
44
|
+
headers: {
|
|
45
|
+
"content-type": "application/json",
|
|
46
|
+
"anthropic-version": "2023-06-01",
|
|
47
|
+
"x-api-key": config.apiKey,
|
|
48
|
+
},
|
|
49
|
+
body: JSON.stringify(body),
|
|
50
|
+
signal: combined,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
// Fetch-level errors (network, DNS, timeout) bubble through retry.
|
|
55
|
+
throw err;
|
|
56
|
+
}
|
|
57
|
+
if (!res.ok) {
|
|
58
|
+
const detail = await safeReadText(res);
|
|
59
|
+
throw new LLMError(res.status, `LLM ${res.status} ${res.statusText}: ${clip(detail, 500)}`, detail);
|
|
60
|
+
}
|
|
61
|
+
const json = (await res.json());
|
|
62
|
+
const text = json.content
|
|
63
|
+
.filter((b) => b.type === "text" && typeof b.text === "string")
|
|
64
|
+
.map((b) => b.text)
|
|
65
|
+
.join("");
|
|
66
|
+
return { text, usage: json.usage };
|
|
67
|
+
}, {
|
|
68
|
+
attempts,
|
|
69
|
+
baseDelayMs: 500,
|
|
70
|
+
maxDelayMs: 8_000,
|
|
71
|
+
jitter: 0.25,
|
|
20
72
|
signal,
|
|
73
|
+
shouldRetry: (err) => {
|
|
74
|
+
if (err instanceof LLMError)
|
|
75
|
+
return err.retriable;
|
|
76
|
+
// Fetch errors (TypeError "fetch failed", AbortError from timeout) —
|
|
77
|
+
// retriable. User-initiated aborts (signal fires) raise "aborted"
|
|
78
|
+
// and we want those to NOT retry.
|
|
79
|
+
if (isUserAbort(err, signal))
|
|
80
|
+
return false;
|
|
81
|
+
return true;
|
|
82
|
+
},
|
|
21
83
|
});
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
84
|
+
}
|
|
85
|
+
function makeTimeoutSignal(timeoutMs, userSignal) {
|
|
86
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
87
|
+
if (!userSignal)
|
|
88
|
+
return timeout;
|
|
89
|
+
return AbortSignal.any([userSignal, timeout]);
|
|
90
|
+
}
|
|
91
|
+
function isUserAbort(err, userSignal) {
|
|
92
|
+
if (!userSignal?.aborted)
|
|
93
|
+
return false;
|
|
94
|
+
const msg = err?.message ?? "";
|
|
95
|
+
const name = err?.name ?? "";
|
|
96
|
+
return name === "AbortError" || /abort/i.test(msg);
|
|
97
|
+
}
|
|
98
|
+
async function safeReadText(res) {
|
|
99
|
+
try {
|
|
100
|
+
return await res.text();
|
|
25
101
|
}
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
return
|
|
102
|
+
catch {
|
|
103
|
+
return "";
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function clip(s, max) {
|
|
107
|
+
return s.length <= max ? s : s.slice(0, max - 1) + "…";
|
|
32
108
|
}
|
|
33
109
|
//# sourceMappingURL=llm.js.map
|
package/dist/llm.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,EAAE;AACF,6EAA6E;AAC7E,4EAA4E;AAC5E,0EAA0E;AAC1E,iDAAiD;AACjD,EAAE;AACF,uEAAuE;AACvE,0EAA0E;AAC1E,uDAAuD;AAEvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAqBnC,MAAM,OAAO,QAAS,SAAQ,KAAK;IAEf;IAEA;IAHlB,YACkB,MAAc,EAC9B,OAAe,EACC,SAAiB,EAAE;QAEnC,KAAK,CAAC,OAAO,CAAC,CAAC;QAJC,WAAM,GAAN,MAAM,CAAQ;QAEd,WAAM,GAAN,MAAM,CAAa;QAGnC,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC;IACzB,CAAC;IACD,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC;IAC3E,CAAC;CACF;AAED,MAAM,CAAC,MAAM,sBAAsB,GAAG,OAAO,CAAC;AAC9C,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAEtC,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,QAAsB,EACtB,MAAc,EACd,MAAiB,EACjB,MAAoB;IAEpB,MAAM,GAAG,GAAG,GAAG,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC;IACjE,MAAM,IAAI,GAAG;QACX,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,MAAM,CAAC,SAAS;QAC5B,MAAM;QACN,QAAQ;KACT,CAAC;IACF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,sBAAsB,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,oBAAoB,CAAC,CAAC;IAEzE,OAAO,KAAK,CACV,KAAK,IAAI,EAAE;QACT,MAAM,QAAQ,GAAG,iBAAiB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,GAAa,CAAC;QAClB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBACrB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,mBAAmB,EAAE,YAAY;oBACjC,WAAW,EAAE,MAAM,CAAC,MAAM;iBAC3B;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC1B,MAAM,EAAE,QAAQ;aACjB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,mEAAmE;YACnE,MAAM,GAAG,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,IAAI,QAAQ,CAChB,GAAG,CAAC,MAAM,EACV,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAC3D,MAAM,CACP,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAG7B,CAAC;QAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO;aACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;aAC9D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAK,CAAC;aACnB,IAAI,CAAC,EAAE,CAAC,CAAC;QAEZ,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;IACrC,CAAC,EACD;QACE,QAAQ;QACR,WAAW,EAAE,GAAG;QAChB,UAAU,EAAE,KAAK;QACjB,MAAM,EAAE,IAAI;QACZ,MAAM;QACN,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE;YACnB,IAAI,GAAG,YAAY,QAAQ;gBAAE,OAAO,GAAG,CAAC,SAAS,CAAC;YAClD,qEAAqE;YACrE,kEAAkE;YAClE,kCAAkC;YAClC,IAAI,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC3C,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CACF,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,SAAiB,EACjB,UAAwB;IAExB,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,IAAI,CAAC,UAAU;QAAE,OAAO,OAAO,CAAC;IAChC,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,WAAW,CAAC,GAAY,EAAE,UAAwB;IACzD,IAAI,CAAC,UAAU,EAAE,OAAO;QAAE,OAAO,KAAK,CAAC;IACvC,MAAM,GAAG,GAAI,GAAa,EAAE,OAAO,IAAI,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAI,GAAa,EAAE,IAAI,IAAI,EAAE,CAAC;IACxC,OAAO,IAAI,KAAK,YAAY,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,GAAa;IACvC,IAAI,CAAC;QACH,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,CAAS,EAAE,GAAW;IAClC,OAAO,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;AACzD,CAAC"}
|
package/dist/plan.d.ts
CHANGED
|
@@ -3,6 +3,13 @@ export interface Plan {
|
|
|
3
3
|
queries: string[];
|
|
4
4
|
reasoning: string;
|
|
5
5
|
}
|
|
6
|
+
export interface Critique {
|
|
7
|
+
done: boolean;
|
|
8
|
+
reasoning: string;
|
|
9
|
+
queries: string[];
|
|
10
|
+
}
|
|
6
11
|
export declare function planQueries(question: string, config: LLMConfig, signal?: AbortSignal): Promise<Plan>;
|
|
7
12
|
export declare function parsePlan(raw: string): Plan;
|
|
13
|
+
export declare function critique(question: string, draftAnswer: string, priorQueries: string[], config: LLMConfig, signal?: AbortSignal): Promise<Critique>;
|
|
14
|
+
export declare function parseCritique(raw: string): Critique;
|
|
8
15
|
//# sourceMappingURL=plan.d.ts.map
|
package/dist/plan.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plan.d.ts","sourceRoot":"","sources":["../src/plan.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"plan.d.ts","sourceRoot":"","sources":["../src/plan.ts"],"names":[],"mappings":"AAOA,OAAO,EAAW,KAAK,SAAS,EAAE,MAAM,UAAU,CAAC;AAEnD,MAAM,WAAW,IAAI;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAaD,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,SAAS,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,IAAI,CAAC,CAQf;AAGD,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiB3C;AAqBD,wBAAsB,QAAQ,CAC5B,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EAAE,EACtB,MAAM,EAAE,SAAS,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,QAAQ,CAAC,CAcnB;AAGD,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,CAsBnD"}
|
package/dist/plan.js
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
// Query planning — decompose a user question into 3–5 sub-queries suitable
|
|
2
2
|
// for web search. Pure over its inputs; wraps one LLM call.
|
|
3
|
+
//
|
|
4
|
+
// Also exports critique() for iterative deep research: given a draft answer
|
|
5
|
+
// and the queries already tried, the critic proposes 0–3 follow-up queries
|
|
6
|
+
// that would fill the gaps — or declares the answer done.
|
|
3
7
|
import { callLLM } from "./llm.js";
|
|
4
8
|
const PLANNER_SYSTEM = `You are a research planner. Given a user's question, produce 3-5 specific, searchable sub-queries that together will gather the evidence needed to answer it well.
|
|
5
9
|
|
|
@@ -36,6 +40,53 @@ export function parsePlan(raw) {
|
|
|
36
40
|
reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
|
|
37
41
|
};
|
|
38
42
|
}
|
|
43
|
+
const CRITIC_SYSTEM = `You are a research critic reviewing a draft answer to a user's question.
|
|
44
|
+
|
|
45
|
+
You will receive:
|
|
46
|
+
1. The original question.
|
|
47
|
+
2. The current draft answer with inline [N] citations.
|
|
48
|
+
3. The list of search queries already run.
|
|
49
|
+
|
|
50
|
+
Your job: identify the most important gaps in the draft answer — aspects of the question not answered, answered weakly, or answered with insufficient sourcing — and propose 0 to 3 additional search queries that would fill those gaps in the next round.
|
|
51
|
+
|
|
52
|
+
Rules:
|
|
53
|
+
- If the draft answer is already complete and well-sourced, set "done": true and return no queries.
|
|
54
|
+
- Do not repeat queries similar to ones already searched. The list of prior queries is provided.
|
|
55
|
+
- Queries must be concrete, search-engine-shaped phrases a user would type.
|
|
56
|
+
- Prefer queries targeting specific unanswered facts over generic re-searches.
|
|
57
|
+
- Keep the number of new queries minimal — fewer, sharper queries beat more, vaguer ones.
|
|
58
|
+
|
|
59
|
+
Output FORMAT (strict): one JSON object, no prose before or after, matching:
|
|
60
|
+
{"done": bool, "reasoning": "<1-2 sentences>", "queries": ["q1", "q2", ...]}`;
|
|
61
|
+
export async function critique(question, draftAnswer, priorQueries, config, signal) {
|
|
62
|
+
const userMessage = `Question: ${question}\n\n` +
|
|
63
|
+
`Draft answer:\n${draftAnswer}\n\n` +
|
|
64
|
+
`Queries already run (${priorQueries.length}):\n` +
|
|
65
|
+
priorQueries.map((q) => `- ${q}`).join("\n") +
|
|
66
|
+
`\n\nReview the draft and propose follow-up queries if needed.`;
|
|
67
|
+
const { text } = await callLLM([{ role: "user", content: userMessage }], CRITIC_SYSTEM, config, signal);
|
|
68
|
+
return parseCritique(text);
|
|
69
|
+
}
|
|
70
|
+
// Exported for unit tests.
|
|
71
|
+
export function parseCritique(raw) {
|
|
72
|
+
const json = extractFirstJsonObject(raw);
|
|
73
|
+
if (!json)
|
|
74
|
+
throw new Error(`critic did not return JSON: ${raw.slice(0, 200)}`);
|
|
75
|
+
const parsed = JSON.parse(json);
|
|
76
|
+
const queries = Array.isArray(parsed.queries)
|
|
77
|
+
? parsed.queries
|
|
78
|
+
.filter((q) => typeof q === "string")
|
|
79
|
+
.map((q) => q.trim())
|
|
80
|
+
.filter((q) => q.length > 0)
|
|
81
|
+
.slice(0, 3)
|
|
82
|
+
: [];
|
|
83
|
+
const done = typeof parsed.done === "boolean" ? parsed.done : queries.length === 0;
|
|
84
|
+
return {
|
|
85
|
+
done,
|
|
86
|
+
reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
|
|
87
|
+
queries,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
39
90
|
function extractFirstJsonObject(s) {
|
|
40
91
|
const start = s.indexOf("{");
|
|
41
92
|
if (start === -1)
|
package/dist/plan.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plan.js","sourceRoot":"","sources":["../src/plan.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,4DAA4D;
|
|
1
|
+
{"version":3,"file":"plan.js","sourceRoot":"","sources":["../src/plan.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,4DAA4D;AAC5D,EAAE;AACF,4EAA4E;AAC5E,2EAA2E;AAC3E,0DAA0D;AAE1D,OAAO,EAAE,OAAO,EAAkB,MAAM,UAAU,CAAC;AAanD,MAAM,cAAc,GAAG;;;;;;;;;qFAS8D,CAAC;AAEtF,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,MAAiB,EACjB,MAAoB;IAEpB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAC5B,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EACrC,cAAc,EACd,MAAM,EACN,MAAM,CACP,CAAC;IACF,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,MAAM,IAAI,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAChF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA8C,CAAC;IAC7E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,gCAAgC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO;SAC3B,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;SACjD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACf,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC5E,OAAO;QACL,OAAO;QACP,SAAS,EAAE,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;KACxE,CAAC;AACJ,CAAC;AAED,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;6EAiBuD,CAAC;AAE9E,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,QAAgB,EAChB,WAAmB,EACnB,YAAsB,EACtB,MAAiB,EACjB,MAAoB;IAEpB,MAAM,WAAW,GACf,aAAa,QAAQ,MAAM;QAC3B,kBAAkB,WAAW,MAAM;QACnC,wBAAwB,YAAY,CAAC,MAAM,MAAM;QACjD,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC5C,+DAA+D,CAAC;IAClE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAC5B,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EACxC,aAAa,EACb,MAAM,EACN,MAAM,CACP,CAAC;IACF,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,MAAM,IAAI,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/E,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAI7B,CAAC;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;QAC3C,CAAC,CAAC,MAAM,CAAC,OAAO;aACX,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;aACjD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,IAAI,GACR,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IACxE,OAAO;QACL,IAAI;QACJ,SAAS,EAAE,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;QACvE,OAAO;KACR,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,CAAS;IACvC,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,KAAK,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACf,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,GAAG,KAAK,CAAC;YACf,SAAS;QACX,CAAC;QACD,IAAI,CAAC,KAAK,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,GAAG,IAAI,CAAC;YACd,SAAS;QACX,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YACd,QAAQ,GAAG,CAAC,QAAQ,CAAC;YACrB,SAAS;QACX,CAAC;QACD,IAAI,QAAQ;YAAE,SAAS;QACvB,IAAI,CAAC,KAAK,GAAG;YAAE,KAAK,EAAE,CAAC;aAClB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YACnB,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/dist/retry.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export interface RetryOptions {
|
|
2
|
+
attempts: number;
|
|
3
|
+
baseDelayMs: number;
|
|
4
|
+
maxDelayMs: number;
|
|
5
|
+
jitter: number;
|
|
6
|
+
shouldRetry?: (err: unknown, attempt: number) => boolean;
|
|
7
|
+
onRetry?: (err: unknown, attempt: number, delayMs: number) => void;
|
|
8
|
+
signal?: AbortSignal;
|
|
9
|
+
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
10
|
+
random?: () => number;
|
|
11
|
+
}
|
|
12
|
+
export declare function retry<T>(fn: () => Promise<T>, opts?: Partial<RetryOptions>): Promise<T>;
|
|
13
|
+
export declare function backoffDelay(attempt: number, cfg: {
|
|
14
|
+
baseDelayMs: number;
|
|
15
|
+
maxDelayMs: number;
|
|
16
|
+
jitter: number;
|
|
17
|
+
}, random?: () => number): number;
|
|
18
|
+
//# sourceMappingURL=retry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../src/retry.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC;IACzD,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACnE,MAAM,CAAC,EAAE,WAAW,CAAC;IAErB,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5D,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;CACvB;AASD,wBAAsB,KAAK,CAAC,CAAC,EAC3B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,IAAI,GAAE,OAAO,CAAC,YAAY,CAAM,GAC/B,OAAO,CAAC,CAAC,CAAC,CAsBZ;AAGD,wBAAgB,YAAY,CAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAChE,MAAM,GAAE,MAAM,MAAoB,GACjC,MAAM,CAMR"}
|
package/dist/retry.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// Retry-with-backoff helper. Pure over its inputs — injectable `sleep` and
|
|
2
|
+
// `random` make it deterministic in tests.
|
|
3
|
+
//
|
|
4
|
+
// Default policy is "retry any thrown error with exponential backoff + jitter"
|
|
5
|
+
// — callers pass a `shouldRetry` predicate to narrow that (e.g. llm.ts only
|
|
6
|
+
// retries on 5xx / 429 / network errors, never on 4xx).
|
|
7
|
+
const DEFAULTS = {
|
|
8
|
+
attempts: 3,
|
|
9
|
+
baseDelayMs: 500,
|
|
10
|
+
maxDelayMs: 8000,
|
|
11
|
+
jitter: 0.25,
|
|
12
|
+
};
|
|
13
|
+
export async function retry(fn, opts = {}) {
|
|
14
|
+
const cfg = { ...DEFAULTS, ...opts };
|
|
15
|
+
const sleep = opts.sleep ?? defaultSleep;
|
|
16
|
+
const random = opts.random ?? Math.random;
|
|
17
|
+
let lastErr;
|
|
18
|
+
for (let attempt = 1; attempt <= cfg.attempts; attempt++) {
|
|
19
|
+
if (opts.signal?.aborted)
|
|
20
|
+
throw new Error("aborted");
|
|
21
|
+
try {
|
|
22
|
+
return await fn();
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
lastErr = err;
|
|
26
|
+
const isLast = attempt >= cfg.attempts;
|
|
27
|
+
if (isLast)
|
|
28
|
+
throw err;
|
|
29
|
+
if (opts.shouldRetry && !opts.shouldRetry(err, attempt))
|
|
30
|
+
throw err;
|
|
31
|
+
const delay = backoffDelay(attempt, cfg, random);
|
|
32
|
+
opts.onRetry?.(err, attempt, delay);
|
|
33
|
+
await sleep(delay, opts.signal);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
// Unreachable — loop either returns or throws — but TS wants a concrete value.
|
|
37
|
+
throw lastErr;
|
|
38
|
+
}
|
|
39
|
+
// Exported for unit tests.
|
|
40
|
+
export function backoffDelay(attempt, cfg, random = Math.random) {
|
|
41
|
+
const exp = Math.min(cfg.maxDelayMs, cfg.baseDelayMs * Math.pow(2, attempt - 1));
|
|
42
|
+
if (cfg.jitter <= 0)
|
|
43
|
+
return Math.floor(exp);
|
|
44
|
+
const spread = exp * cfg.jitter;
|
|
45
|
+
const delta = (random() * 2 - 1) * spread;
|
|
46
|
+
return Math.max(0, Math.floor(exp + delta));
|
|
47
|
+
}
|
|
48
|
+
async function defaultSleep(ms, signal) {
|
|
49
|
+
if (ms <= 0)
|
|
50
|
+
return;
|
|
51
|
+
await new Promise((resolve, reject) => {
|
|
52
|
+
const timer = setTimeout(() => {
|
|
53
|
+
signal?.removeEventListener("abort", onAbort);
|
|
54
|
+
resolve();
|
|
55
|
+
}, ms);
|
|
56
|
+
const onAbort = () => {
|
|
57
|
+
clearTimeout(timer);
|
|
58
|
+
reject(new Error("aborted"));
|
|
59
|
+
};
|
|
60
|
+
if (signal) {
|
|
61
|
+
if (signal.aborted) {
|
|
62
|
+
clearTimeout(timer);
|
|
63
|
+
reject(new Error("aborted"));
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=retry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry.js","sourceRoot":"","sources":["../src/retry.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,2CAA2C;AAC3C,EAAE;AACF,+EAA+E;AAC/E,4EAA4E;AAC5E,wDAAwD;AAexD,MAAM,QAAQ,GAAkF;IAC9F,QAAQ,EAAE,CAAC;IACX,WAAW,EAAE,GAAG;IAChB,UAAU,EAAE,IAAI;IAChB,MAAM,EAAE,IAAI;CACb,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,EAAoB,EACpB,OAA8B,EAAE;IAEhC,MAAM,GAAG,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,YAAY,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IAC1C,IAAI,OAAgB,CAAC;IAErB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC;QACzD,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;QACrD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,GAAG,GAAG,CAAC;YACd,MAAM,MAAM,GAAG,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC;YACvC,IAAI,MAAM;gBAAE,MAAM,GAAG,CAAC;YACtB,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC;gBAAE,MAAM,GAAG,CAAC;YACnE,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;YACjD,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YACpC,MAAM,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,+EAA+E;IAC/E,MAAM,OAAO,CAAC;AAChB,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,YAAY,CAC1B,OAAe,EACf,GAAgE,EAChE,SAAuB,IAAI,CAAC,MAAM;IAElC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;IAChC,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC;IAC1C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,EAAU,EAAE,MAAoB;IAC1D,IAAI,EAAE,IAAI,CAAC;QAAE,OAAO;IACpB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC9C,OAAO,EAAE,CAAC;QACZ,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/B,CAAC,CAAC;QACF,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC7B,OAAO;YACT,CAAC;YACD,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/robots.d.ts
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export interface RobotsRule {
|
|
2
|
+
allow: boolean;
|
|
3
|
+
path: string;
|
|
4
|
+
}
|
|
5
|
+
export interface ParsedRobots {
|
|
6
|
+
rules: RobotsRule[];
|
|
7
|
+
crawlDelaySec?: number;
|
|
8
|
+
}
|
|
9
|
+
export type RobotsCheckResult = "allow" | "deny" | "unknown";
|
|
10
|
+
export interface RobotsCache {
|
|
11
|
+
get(origin: string): ParsedRobots | null | undefined;
|
|
12
|
+
set(origin: string, parsed: ParsedRobots | null): void;
|
|
13
|
+
}
|
|
14
|
+
export interface CanFetchOptions {
|
|
15
|
+
userAgent: string;
|
|
16
|
+
cache?: RobotsCache;
|
|
17
|
+
fetchImpl?: typeof fetch;
|
|
18
|
+
timeoutMs?: number;
|
|
19
|
+
signal?: AbortSignal;
|
|
20
|
+
}
|
|
21
|
+
export declare const DEFAULT_USER_AGENT = "deepdive-bot";
|
|
22
|
+
export declare function createRobotsCache(): RobotsCache;
|
|
23
|
+
export declare function canFetch(url: string, opts: CanFetchOptions): Promise<RobotsCheckResult>;
|
|
24
|
+
export declare function parseRobotsTxt(text: string): ParsedRobots;
|
|
25
|
+
export declare function isPathAllowed(parsed: ParsedRobots, path: string, userAgent: string): boolean;
|
|
26
|
+
//# sourceMappingURL=robots.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAWA,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,OAAO,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAG3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;AAE7D,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,GAAG,SAAS,CAAC;IACrD,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,eAAO,MAAM,kBAAkB,iBAAiB,CAAC;AAEjD,wBAAgB,iBAAiB,IAAI,WAAW,CAM/C;AAED,wBAAsB,QAAQ,CAC5B,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAqB5B;AAiCD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAkDzD;AAGD,wBAAgB,aAAa,CAC3B,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,GAChB,OAAO,CA+BT"}
|