@mjasnikovs/pi-task 0.18.4 → 0.18.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/config.d.ts +18 -0
- package/dist/config/config.js +10 -1
- package/dist/config/register.js +32 -5
- package/dist/task/accept-debt.d.ts +52 -0
- package/dist/task/accept-debt.js +0 -0
- package/dist/task/auto-orchestrator.d.ts +2 -0
- package/dist/task/auto-orchestrator.js +20 -0
- package/dist/task/final-gate.d.ts +8 -0
- package/dist/task/final-gate.js +27 -7
- package/dist/task/frozen-path-guard.d.ts +39 -0
- package/dist/task/frozen-path-guard.js +116 -0
- package/dist/task/gate-deps.js +25 -0
- package/dist/task/phases.d.ts +6 -2
- package/dist/task/phases.js +7 -2
- package/dist/task/repo-health-check.d.ts +11 -0
- package/dist/task/repo-health-check.js +26 -3
- package/dist/task/service-blocks.d.ts +2 -2
- package/dist/task/service-blocks.js +3 -1
- package/dist/task/task-gates.d.ts +24 -0
- package/dist/task/task-gates.js +78 -8
- package/dist/workers/brave-search.d.ts +2 -5
- package/dist/workers/brave-warning.d.ts +4 -6
- package/dist/workers/brave-warning.js +11 -10
- package/dist/workers/ddg-search.d.ts +24 -0
- package/dist/workers/ddg-search.js +130 -0
- package/dist/workers/exa-search.d.ts +24 -0
- package/dist/workers/exa-search.js +164 -0
- package/dist/workers/pi-worker-docs.js +13 -1
- package/dist/workers/pi-worker-fetch.js +10 -1
- package/dist/workers/pi-worker-search.d.ts +7 -1
- package/dist/workers/pi-worker-search.js +18 -5
- package/dist/workers/research-cache.d.ts +39 -0
- package/dist/workers/research-cache.js +140 -0
- package/dist/workers/search-core.d.ts +14 -2
- package/dist/workers/search-core.js +34 -2
- package/dist/workers/search-types.d.ts +15 -0
- package/dist/workers/search-types.js +4 -0
- package/dist/workers/shared.d.ts +17 -0
- package/dist/workers/shared.js +0 -0
- package/package.json +1 -1
|
@@ -32,6 +32,23 @@
|
|
|
32
32
|
import { spawnSync } from 'node:child_process';
|
|
33
33
|
import { existsSync, readFileSync } from 'node:fs';
|
|
34
34
|
import * as path from 'node:path';
|
|
35
|
+
/** How much of a failing command's output to keep — bounded so a wedged tool that
|
|
36
|
+
* spews megabytes cannot bloat the trail. stderr leads (a crash trace lives there). */
|
|
37
|
+
const HEALTH_OUTPUT_MAX_LINES = 40;
|
|
38
|
+
const HEALTH_OUTPUT_MAX_CHARS = 4000;
|
|
39
|
+
/** Combine a failing command's stderr+stdout into a bounded, first-N-lines snippet. */
|
|
40
|
+
export function captureHealthOutput(stdout, stderr) {
|
|
41
|
+
const combined = [stderr, stdout]
|
|
42
|
+
.map(s => (s ?? '').trim())
|
|
43
|
+
.filter(s => s.length > 0)
|
|
44
|
+
.join('\n');
|
|
45
|
+
if (combined.length === 0)
|
|
46
|
+
return '';
|
|
47
|
+
let snippet = combined.split('\n').slice(0, HEALTH_OUTPUT_MAX_LINES).join('\n');
|
|
48
|
+
if (snippet.length > HEALTH_OUTPUT_MAX_CHARS)
|
|
49
|
+
snippet = `${snippet.slice(0, HEALTH_OUTPUT_MAX_CHARS)}…`;
|
|
50
|
+
return snippet;
|
|
51
|
+
}
|
|
35
52
|
function packageScripts(cwd) {
|
|
36
53
|
try {
|
|
37
54
|
const j = JSON.parse(readFileSync(path.join(cwd, 'package.json'), 'utf8'));
|
|
@@ -103,7 +120,12 @@ export function discoverHealthCommands(cwd) {
|
|
|
103
120
|
export function runRepoHealthCheck(cwd, timeoutMs = 600_000) {
|
|
104
121
|
const { ecosystem, cmds } = discoverHealthCommands(cwd);
|
|
105
122
|
if (!ecosystem || cmds.length === 0) {
|
|
106
|
-
return {
|
|
123
|
+
return {
|
|
124
|
+
ok: true,
|
|
125
|
+
reason: 'no repo-wide static-analysis command found',
|
|
126
|
+
ecosystem,
|
|
127
|
+
output: ''
|
|
128
|
+
};
|
|
107
129
|
}
|
|
108
130
|
for (const [bin, args] of cmds) {
|
|
109
131
|
const r = spawnSync(bin, args, { cwd, encoding: 'utf8', timeout: timeoutMs });
|
|
@@ -119,9 +141,10 @@ export function runRepoHealthCheck(cwd, timeoutMs = 600_000) {
|
|
|
119
141
|
return {
|
|
120
142
|
ok: false,
|
|
121
143
|
reason: `\`${bin} ${args.join(' ')}\` exited ${r.status}`,
|
|
122
|
-
ecosystem
|
|
144
|
+
ecosystem,
|
|
145
|
+
output: captureHealthOutput(r.stdout, r.stderr)
|
|
123
146
|
};
|
|
124
147
|
}
|
|
125
148
|
}
|
|
126
|
-
return { ok: true, reason: `${ecosystem}: static checks passed`, ecosystem };
|
|
149
|
+
return { ok: true, reason: `${ecosystem}: static checks passed`, ecosystem, output: '' };
|
|
127
150
|
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
export declare function formatServiceBlock(name: string, fullQuery: string, results:
|
|
1
|
+
import type { SearchResult } from '../workers/search-types.js';
|
|
2
|
+
export declare function formatServiceBlock(name: string, fullQuery: string, results: SearchResult[]): string;
|
|
3
3
|
export declare function formatFreshnessSkippedBlock(names: string[]): string;
|
|
@@ -5,6 +5,8 @@ export function formatServiceBlock(name, fullQuery, results) {
|
|
|
5
5
|
const bullets = results.map(r => `- **${r.title}** — ${r.url}\n ${r.description}`).join('\n');
|
|
6
6
|
return `${header}\n${bullets}`;
|
|
7
7
|
}
|
|
8
|
+
// Only the brave provider can be unconfigured (exa/ddg are keyless), so the
|
|
9
|
+
// skip reason names its missing key directly.
|
|
8
10
|
export function formatFreshnessSkippedBlock(names) {
|
|
9
|
-
return `### freshness-check skipped\nCould not verify external services (BRAVE_SEARCH_API_KEY not set):\n${names.map(n => `- ${n}`).join('\n')}`;
|
|
11
|
+
return `### freshness-check skipped\nCould not verify external services (search provider is brave but BRAVE_SEARCH_API_KEY is not set):\n${names.map(n => `- ${n}`).join('\n')}`;
|
|
10
12
|
}
|
|
@@ -101,6 +101,7 @@ export interface GateDeps {
|
|
|
101
101
|
repoHealth?: (cwd: string) => Promise<{
|
|
102
102
|
ok: boolean;
|
|
103
103
|
reason: string;
|
|
104
|
+
output?: string;
|
|
104
105
|
}>;
|
|
105
106
|
/** Does the working tree hold changes (excluding .pi-tasks)? Lets the pre-commit
|
|
106
107
|
* health check run only when the enforce pass actually edited something. */
|
|
@@ -118,6 +119,29 @@ export interface GateDeps {
|
|
|
118
119
|
* swallowed by the implementation, never by this sequence.
|
|
119
120
|
*/
|
|
120
121
|
record?: (cwd: string, taskId: string, line: string) => Promise<void>;
|
|
122
|
+
/**
|
|
123
|
+
* Record a durable ACCEPT-despite-verify-FAIL debt (task id + FAIL reason) to the
|
|
124
|
+
* run-level ledger (`.pi-tasks/accept-debt.md`, see accept-debt.ts). Called only on
|
|
125
|
+
* the picker's ACCEPT branch — the human blessed a failing artifact as-is, so the
|
|
126
|
+
* defect is real and recorded; the final integration gate re-checks it at run end
|
|
127
|
+
* and surfaces it if still open. Best-effort; absent in tests → no ledger written.
|
|
128
|
+
*/
|
|
129
|
+
recordAcceptDebt?: (cwd: string, taskId: string, reason: string) => Promise<void>;
|
|
130
|
+
/**
|
|
131
|
+
* The concrete paths this task's spec forbids modifying (its `Do NOT modify`
|
|
132
|
+
* CONSTRAINTS — see frozen-path-guard.ts / prohibition-probe.ts). Used to
|
|
133
|
+
* write-deny the enforce EDIT pass: a violating edit is reverted before it can
|
|
134
|
+
* be committed. Empty when the spec froze nothing → the guard is a no-op.
|
|
135
|
+
* Absent in tests / on a bare `/task` → the guard is skipped.
|
|
136
|
+
*/
|
|
137
|
+
frozenPaths?: (cwd: string, taskId: string) => Promise<string[]>;
|
|
138
|
+
/**
|
|
139
|
+
* Restore the given frozen paths to their committed (HEAD) state, discarding a
|
|
140
|
+
* gate child's edits to them, and return the files actually reverted. Prompt
|
|
141
|
+
* framing is A/B-proven insufficient for this class, so the deny is mechanical:
|
|
142
|
+
* the write is undone, not merely warned about. Absent → the guard warns only.
|
|
143
|
+
*/
|
|
144
|
+
revertFrozenPaths?: (cwd: string, paths: string[]) => Promise<string[]>;
|
|
121
145
|
}
|
|
122
146
|
/** Inputs the sequence needs that vary per caller. */
|
|
123
147
|
export interface GateParams {
|
package/dist/task/task-gates.js
CHANGED
|
@@ -10,6 +10,18 @@ import { SessionUI } from '../remote/bridge.js';
|
|
|
10
10
|
* regardless of this count (blessing an artifact as-is is a human's call).
|
|
11
11
|
*/
|
|
12
12
|
export const MAX_AUTO_AUTOFIX = 3;
|
|
13
|
+
/**
|
|
14
|
+
* Bound a captured health-check output before it is embedded in a gate-trail line.
|
|
15
|
+
* appendGateRecord flattens newlines to spaces, so the trail stays one line per
|
|
16
|
+
* entry; this just caps the volume (a wedged tool can emit megabytes). The health
|
|
17
|
+
* check already trims to its own first-N lines — this is the trail-side ceiling.
|
|
18
|
+
*/
|
|
19
|
+
const TRAIL_OUTPUT_MAX_CHARS = 1200;
|
|
20
|
+
function clampOutput(output) {
|
|
21
|
+
return output.length > TRAIL_OUTPUT_MAX_CHARS ?
|
|
22
|
+
`${output.slice(0, TRAIL_OUTPUT_MAX_CHARS)}…`
|
|
23
|
+
: output;
|
|
24
|
+
}
|
|
13
25
|
/**
|
|
14
26
|
* Show the boxed two-choice picker after a verify FAIL and return what the user
|
|
15
27
|
* decided. The model-recommended card is placed first so the renderer tints it
|
|
@@ -135,6 +147,16 @@ export async function runGatesForTask(ctxIn, deps, p) {
|
|
|
135
147
|
}
|
|
136
148
|
if (choice.action === 'accept') {
|
|
137
149
|
await rec('resolution: user ACCEPTED the work despite verify FAIL');
|
|
150
|
+
// Durable debt: the human blessed a FAILing artifact as-is, so the
|
|
151
|
+
// defect ships and nothing else revisits it (mx5 run 4 B3 / run 8
|
|
152
|
+
// TASK_0012). Record it to the run ledger; the final integration gate
|
|
153
|
+
// re-checks it at run end and surfaces it if still open. Best-effort.
|
|
154
|
+
try {
|
|
155
|
+
await deps.recordAcceptDebt?.(p.cwd, p.taskId, failReason);
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
// recording must never break the gate sequence
|
|
159
|
+
}
|
|
138
160
|
active.ui.notify(`${p.tag}: accepted "${p.title}" despite verify FAIL (${failReason.slice(0, 120)}) — proceeding.`, 'warning');
|
|
139
161
|
break;
|
|
140
162
|
}
|
|
@@ -202,13 +224,40 @@ export async function runGatesForTask(ctxIn, deps, p) {
|
|
|
202
224
|
// with no enforce dep.
|
|
203
225
|
if (deps.enforce && commit.committed) {
|
|
204
226
|
const mode = verifyCleanPass ? 'edit' : 'flag';
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
227
|
+
// BASELINE repo health, captured BEFORE the edit pass touches the tree, so the
|
|
228
|
+
// pre-commit gate below is DIFFERENTIAL: it can tell an enforce-CAUSED
|
|
229
|
+
// regression (was clean, now fails) from a repo that was ALREADY unhealthy
|
|
230
|
+
// (run-8 F8: five enforce passes were discarded on a lint that was already
|
|
231
|
+
// crashing — exit 2 — before enforce edited anything; the discard threw away
|
|
232
|
+
// good work for a fault it did not cause). Only meaningful in edit mode (flag
|
|
233
|
+
// makes no edits); the task's work is already committed so this reflects the
|
|
234
|
+
// committed state the pass is about to build on.
|
|
235
|
+
const healthBefore = mode === 'edit' && deps.repoHealth ? await deps.repoHealth(p.cwd) : undefined;
|
|
208
236
|
const verdict = await deps.enforce(active, p.cwd, p.title, mode);
|
|
237
|
+
// FROZEN-PATH WRITE-DENY (mechanical, not prompt — the "MUST NOT edit"
|
|
238
|
+
// instruction is A/B-proven ~0–1/5 reliable on the weak model): the enforce
|
|
239
|
+
// EDIT pass runs read,edit and has been seen mutating a path the spec froze.
|
|
240
|
+
// Before its edits are inspected/committed below, restore any frozen path it
|
|
241
|
+
// touched to the committed task state, so the violating write cannot land in
|
|
242
|
+
// the ENFORCE GUIDELINES commit regardless of what the model intended. The
|
|
243
|
+
// task's OWN frozen-path edits (if any) are already in HEAD and are the verify
|
|
244
|
+
// prohibition-probe's job — this only undoes the gate child's edits on top.
|
|
245
|
+
// No-op when the spec froze nothing or the deps are absent (bare /task, tests).
|
|
246
|
+
if (mode === 'edit' && deps.frozenPaths && deps.revertFrozenPaths) {
|
|
247
|
+
const frozen = await deps.frozenPaths(p.cwd, p.taskId);
|
|
248
|
+
if (frozen.length > 0) {
|
|
249
|
+
const reverted = await deps.revertFrozenPaths(p.cwd, frozen);
|
|
250
|
+
if (reverted.length > 0) {
|
|
251
|
+
await rec(`enforce: frozen-path write DENIED — reverted ${reverted.length} spec-frozen file(s) the edit pass modified: ${reverted.join(', ')}`);
|
|
252
|
+
active.ui.notify(`${p.tag}: guideline edits on "${p.title}" touched spec-frozen path(s) (${reverted.join(', ').slice(0, 120)}) — reverted before commit.`, 'warning');
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
209
256
|
// The child's verdict and its edits are independent facts: the pass has been
|
|
210
257
|
// observed declaring "clean" while having edited files (which then get
|
|
211
258
|
// committed as fixes) — record both so the trail cannot contradict itself.
|
|
259
|
+
// `editsMade` is read AFTER the frozen-path revert so a pass whose only edit
|
|
260
|
+
// was to a frozen path correctly shows a clean tree (nothing left to commit).
|
|
212
261
|
const editsMade = mode === 'edit' && deps.dirty ? await deps.dirty(p.cwd) : undefined;
|
|
213
262
|
await rec(`enforce(${mode}): ${verdict.ok ? `clean${verdict.reason ? ` (${verdict.reason})` : ''}` : (verdict.reason ?? 'not clean')}${editsMade ? ' — edits in tree' : ''}`);
|
|
214
263
|
if (!verdict.ok) {
|
|
@@ -220,19 +269,40 @@ export async function runGatesForTask(ctxIn, deps, p) {
|
|
|
220
269
|
// and discards the bad edits outright. Only runs when the tree is actually
|
|
221
270
|
// dirty (or dirtiness is unknowable); the differential guard below still
|
|
222
271
|
// catches behavioral regressions the static check cannot see.
|
|
272
|
+
//
|
|
273
|
+
// The gate is DIFFERENTIAL, not absolute (run-8 F8): discard the edits only
|
|
274
|
+
// when they REGRESSED the health signal — clean before, failing after. A repo
|
|
275
|
+
// that was already failing before enforce ran is not enforce's fault, so its
|
|
276
|
+
// edits are KEPT (and the pre-existing failure is recorded, to be caught by the
|
|
277
|
+
// final integration gate, not blamed on this pass). The failing command's
|
|
278
|
+
// output is captured into the trail so the discard is explainable — F8 was
|
|
279
|
+
// unreproducible precisely because only the exit code was recorded.
|
|
223
280
|
let enforceEditsBlocked = false;
|
|
224
281
|
if (mode === 'edit' && deps.repoHealth && editsMade !== false) {
|
|
225
|
-
const
|
|
226
|
-
|
|
282
|
+
const after = await deps.repoHealth(p.cwd);
|
|
283
|
+
// A regression needs a clean (or unknown) baseline turning to a fail. If
|
|
284
|
+
// healthBefore is undefined (repoHealth was absent at baseline time) treat
|
|
285
|
+
// the baseline as clean — the conservative absolute behavior.
|
|
286
|
+
const wasHealthyBefore = healthBefore?.ok ?? true;
|
|
287
|
+
const regressed = !after.ok && wasHealthyBefore;
|
|
288
|
+
if (regressed) {
|
|
227
289
|
enforceEditsBlocked = true;
|
|
290
|
+
const outputTail = after.output ? ` — output:\n${clampOutput(after.output)}` : '';
|
|
228
291
|
if (deps.discardEdits) {
|
|
229
292
|
await deps.discardEdits(p.cwd);
|
|
230
|
-
await rec(`enforce: edits discarded pre-commit
|
|
293
|
+
await rec(`enforce: edits discarded pre-commit — REGRESSED repo health (${after.reason})${outputTail}`);
|
|
231
294
|
}
|
|
232
295
|
else {
|
|
233
|
-
await rec(`enforce: edits
|
|
296
|
+
await rec(`enforce: edits REGRESSED repo health pre-commit (${after.reason}) — no discard available, left uncommitted${outputTail}`);
|
|
234
297
|
}
|
|
235
|
-
active.ui.notify(`${p.tag}: guideline edits on "${p.title}"
|
|
298
|
+
active.ui.notify(`${p.tag}: guideline edits on "${p.title}" regressed repo health (${after.reason.slice(0, 120)}) — discarded before commit.`, 'warning');
|
|
299
|
+
}
|
|
300
|
+
else if (!after.ok) {
|
|
301
|
+
// Failing both before and after → not enforce's fault. Keep the edits;
|
|
302
|
+
// record that the repo entered the gate already unhealthy so the trail
|
|
303
|
+
// explains why a still-failing repo did NOT trigger a discard here.
|
|
304
|
+
const outputTail = after.output ? ` — output:\n${clampOutput(after.output)}` : '';
|
|
305
|
+
await rec(`enforce: repo health still failing after edits but was ALREADY failing before the pass (${after.reason}) — pre-existing, edits kept${outputTail}`);
|
|
236
306
|
}
|
|
237
307
|
}
|
|
238
308
|
if (mode === 'edit' && !enforceEditsBlocked && editsMade === false) {
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* One-line startup hint shown when
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* start so the user knows search is disabled — it never blocks work and clears
|
|
7
|
-
* itself on the first interaction (any keystroke).
|
|
2
|
+
* One-line startup hint shown when Brave is the selected search provider but
|
|
3
|
+
* no key is configured (Brave is the only provider that needs one — exa and
|
|
4
|
+
* ddg are keyless, so no hint renders for them). It never blocks work and
|
|
5
|
+
* clears itself on the first interaction (any keystroke).
|
|
8
6
|
*/
|
|
9
7
|
import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
|
|
10
8
|
export declare function registerBraveKeyWarning(pi: ExtensionAPI): void;
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* One-line startup hint shown when
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* start so the user knows search is disabled — it never blocks work and clears
|
|
7
|
-
* itself on the first interaction (any keystroke).
|
|
2
|
+
* One-line startup hint shown when Brave is the selected search provider but
|
|
3
|
+
* no key is configured (Brave is the only provider that needs one — exa and
|
|
4
|
+
* ddg are keyless, so no hint renders for them). It never blocks work and
|
|
5
|
+
* clears itself on the first interaction (any keystroke).
|
|
8
6
|
*/
|
|
7
|
+
import { getConfig } from '../config/config.js';
|
|
9
8
|
const WIDGET_KEY = 'pi-task-brave-warning';
|
|
10
|
-
const WARNING = '⚠ pi-task: BRAVE_SEARCH_API_KEY not set — web search
|
|
11
|
-
+ 'Get a free key at https://api.search.brave.com/app/keys'
|
|
9
|
+
const WARNING = '⚠ pi-task: search provider is brave but BRAVE_SEARCH_API_KEY is not set — web search '
|
|
10
|
+
+ 'is disabled. Get a free key at https://api.search.brave.com/app/keys or switch '
|
|
11
|
+
+ 'provider in /task-config';
|
|
12
12
|
/** Mirrors the lookup in search-core so the hint matches what the worker reads. */
|
|
13
13
|
function hasBraveKey() {
|
|
14
14
|
return Boolean(process.env.BRAVE_SEARCH_API_KEY ?? process.env.BRAVE_API_KEY);
|
|
@@ -16,8 +16,9 @@ function hasBraveKey() {
|
|
|
16
16
|
export function registerBraveKeyWarning(pi) {
|
|
17
17
|
pi.on('session_start', (_event, ctx) => {
|
|
18
18
|
// Terminal-only hint: needs an interactive TUI to render and to catch the
|
|
19
|
-
// keystroke that dismisses it.
|
|
20
|
-
|
|
19
|
+
// keystroke that dismisses it. Only the brave provider can be misconfigured;
|
|
20
|
+
// skip whenever another provider is selected or a key is already present.
|
|
21
|
+
if (ctx.mode !== 'tui' || getConfig().searchProvider !== 'brave' || hasBraveKey())
|
|
21
22
|
return;
|
|
22
23
|
let unsubscribe = null;
|
|
23
24
|
const clear = () => {
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web search via DuckDuckGo's HTML endpoint — no API key required.
|
|
3
|
+
*
|
|
4
|
+
* DDG has no official web-results API; html.duckduckgo.com/html serves plain
|
|
5
|
+
* HTML result pages that parse cleanly with linkedom. Result links are wrapped
|
|
6
|
+
* in a `duckduckgo.com/l/?uddg=<encoded>` redirect that we unwrap so callers
|
|
7
|
+
* get the destination URL. Ad rows redirect through duckduckgo.com itself and
|
|
8
|
+
* are dropped.
|
|
9
|
+
*/
|
|
10
|
+
import type { FetchLike } from './exa-search.js';
|
|
11
|
+
import type { SearchResult } from './search-types.js';
|
|
12
|
+
export interface DdgSearchOpts {
|
|
13
|
+
count?: number;
|
|
14
|
+
timeoutMs?: number;
|
|
15
|
+
signal?: AbortSignal;
|
|
16
|
+
fetchImpl?: FetchLike;
|
|
17
|
+
}
|
|
18
|
+
export declare class DdgSearchError extends Error {
|
|
19
|
+
readonly kind: 'http' | 'network' | 'aborted' | 'rate-limit';
|
|
20
|
+
readonly status?: number | undefined;
|
|
21
|
+
constructor(message: string, kind: 'http' | 'network' | 'aborted' | 'rate-limit', status?: number | undefined);
|
|
22
|
+
}
|
|
23
|
+
export declare function ddgSearch(query: string, opts?: DdgSearchOpts): Promise<SearchResult[]>;
|
|
24
|
+
export declare function parseDdgHtml(html: string): SearchResult[];
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web search via DuckDuckGo's HTML endpoint — no API key required.
|
|
3
|
+
*
|
|
4
|
+
* DDG has no official web-results API; html.duckduckgo.com/html serves plain
|
|
5
|
+
* HTML result pages that parse cleanly with linkedom. Result links are wrapped
|
|
6
|
+
* in a `duckduckgo.com/l/?uddg=<encoded>` redirect that we unwrap so callers
|
|
7
|
+
* get the destination URL. Ad rows redirect through duckduckgo.com itself and
|
|
8
|
+
* are dropped.
|
|
9
|
+
*/
|
|
10
|
+
import { parseHTML } from 'linkedom';
|
|
11
|
+
const DDG_ENDPOINT = 'https://html.duckduckgo.com/html/';
|
|
12
|
+
const DEFAULT_COUNT = 10;
|
|
13
|
+
const MAX_COUNT = 20;
|
|
14
|
+
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
15
|
+
// DDG serves a bot-challenge page to clients without a browser-ish UA.
|
|
16
|
+
const USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0';
|
|
17
|
+
export class DdgSearchError extends Error {
|
|
18
|
+
kind;
|
|
19
|
+
status;
|
|
20
|
+
constructor(message, kind, status) {
|
|
21
|
+
super(message);
|
|
22
|
+
this.kind = kind;
|
|
23
|
+
this.status = status;
|
|
24
|
+
this.name = 'DdgSearchError';
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export async function ddgSearch(query, opts = {}) {
|
|
28
|
+
const count = Math.max(1, Math.min(MAX_COUNT, opts.count ?? DEFAULT_COUNT));
|
|
29
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
30
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
31
|
+
const url = `${DDG_ENDPOINT}?q=${encodeURIComponent(query)}`;
|
|
32
|
+
const internalController = new AbortController();
|
|
33
|
+
let userAborted = false;
|
|
34
|
+
const timeoutHandle = setTimeout(() => internalController.abort(), timeoutMs);
|
|
35
|
+
const onUserAbort = () => {
|
|
36
|
+
userAborted = true;
|
|
37
|
+
internalController.abort();
|
|
38
|
+
};
|
|
39
|
+
if (opts.signal) {
|
|
40
|
+
if (opts.signal.aborted)
|
|
41
|
+
onUserAbort();
|
|
42
|
+
else
|
|
43
|
+
opts.signal.addEventListener('abort', onUserAbort, { once: true });
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
let response;
|
|
47
|
+
try {
|
|
48
|
+
response = await fetchImpl(url, {
|
|
49
|
+
method: 'GET',
|
|
50
|
+
headers: {
|
|
51
|
+
'user-agent': USER_AGENT,
|
|
52
|
+
accept: 'text/html'
|
|
53
|
+
},
|
|
54
|
+
signal: internalController.signal
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
if (userAborted)
|
|
59
|
+
throw new DdgSearchError('Search aborted.', 'aborted');
|
|
60
|
+
throw new DdgSearchError(`DuckDuckGo request failed: ${describeError(err)}`, 'network');
|
|
61
|
+
}
|
|
62
|
+
if (response.status === 429 || response.status === 403) {
|
|
63
|
+
throw new DdgSearchError(`DuckDuckGo is rate-limiting this client (HTTP ${response.status}). Try again in a moment.`, 'rate-limit', response.status);
|
|
64
|
+
}
|
|
65
|
+
if (!response.ok) {
|
|
66
|
+
throw new DdgSearchError(`DuckDuckGo HTTP ${response.status} ${response.statusText}`, 'http', response.status);
|
|
67
|
+
}
|
|
68
|
+
return parseDdgHtml(await response.text()).slice(0, count);
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
clearTimeout(timeoutHandle);
|
|
72
|
+
if (opts.signal)
|
|
73
|
+
opts.signal.removeEventListener('abort', onUserAbort);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
export function parseDdgHtml(html) {
|
|
77
|
+
const { document } = parseHTML(html);
|
|
78
|
+
const doc = document;
|
|
79
|
+
const results = [];
|
|
80
|
+
for (const anchor of doc.querySelectorAll('a.result__a')) {
|
|
81
|
+
if (anchor.closest('.result--ad'))
|
|
82
|
+
continue;
|
|
83
|
+
const href = anchor.getAttribute('href');
|
|
84
|
+
const targetUrl = href === null ? null : unwrapDdgRedirect(href);
|
|
85
|
+
// A row whose link never leaves duckduckgo.com is an ad/module, not a hit.
|
|
86
|
+
if (targetUrl === null)
|
|
87
|
+
continue;
|
|
88
|
+
const title = collapse(anchor.textContent ?? '');
|
|
89
|
+
if (!title)
|
|
90
|
+
continue;
|
|
91
|
+
const row = anchor.closest('.result');
|
|
92
|
+
const snippet = row?.querySelector('.result__snippet')?.textContent ?? '';
|
|
93
|
+
results.push({ title, url: targetUrl, description: collapse(snippet) });
|
|
94
|
+
}
|
|
95
|
+
return results;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Result hrefs look like `//duckduckgo.com/l/?uddg=<encoded-destination>&rut=…`;
|
|
99
|
+
* return the decoded destination, a non-DDG href unchanged, or null for links
|
|
100
|
+
* that stay on duckduckgo.com (ad click-trackers).
|
|
101
|
+
*/
|
|
102
|
+
function unwrapDdgRedirect(href) {
|
|
103
|
+
let parsed;
|
|
104
|
+
try {
|
|
105
|
+
parsed = new URL(href, 'https://duckduckgo.com');
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
if (parsed.hostname.endsWith('duckduckgo.com')) {
|
|
111
|
+
const uddg = parsed.searchParams.get('uddg');
|
|
112
|
+
if (!uddg)
|
|
113
|
+
return null;
|
|
114
|
+
try {
|
|
115
|
+
return new URL(uddg).toString();
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return parsed.toString();
|
|
122
|
+
}
|
|
123
|
+
function collapse(text) {
|
|
124
|
+
return text.replace(/\s+/g, ' ').trim();
|
|
125
|
+
}
|
|
126
|
+
function describeError(err) {
|
|
127
|
+
if (err instanceof Error)
|
|
128
|
+
return err.message;
|
|
129
|
+
return String(err);
|
|
130
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web search via Exa's public MCP endpoint — no API key required.
|
|
3
|
+
*
|
|
4
|
+
* Exa hosts https://mcp.exa.ai/mcp as an intentionally keyless entry point: a
|
|
5
|
+
* single JSON-RPC `tools/call` of `web_search_exa` returns search results with
|
|
6
|
+
* content snippets. The response is SSE-framed (`data:` lines) or plain JSON,
|
|
7
|
+
* and the result payload is one text blob of `Title:`/`URL:`/`Text:` blocks
|
|
8
|
+
* separated by `---`, which we parse back into structured results.
|
|
9
|
+
*/
|
|
10
|
+
import type { SearchResult } from './search-types.js';
|
|
11
|
+
/** Injectable fetch — the narrow signature keeps test fakes free of Bun's extras. */
|
|
12
|
+
export type FetchLike = (url: string, init?: RequestInit) => Promise<Response>;
|
|
13
|
+
export interface ExaSearchOpts {
|
|
14
|
+
count?: number;
|
|
15
|
+
timeoutMs?: number;
|
|
16
|
+
signal?: AbortSignal;
|
|
17
|
+
fetchImpl?: FetchLike;
|
|
18
|
+
}
|
|
19
|
+
export declare class ExaSearchError extends Error {
|
|
20
|
+
readonly kind: 'http' | 'network' | 'aborted' | 'protocol';
|
|
21
|
+
readonly status?: number | undefined;
|
|
22
|
+
constructor(message: string, kind: 'http' | 'network' | 'aborted' | 'protocol', status?: number | undefined);
|
|
23
|
+
}
|
|
24
|
+
export declare function exaSearch(query: string, opts?: ExaSearchOpts): Promise<SearchResult[]>;
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web search via Exa's public MCP endpoint — no API key required.
|
|
3
|
+
*
|
|
4
|
+
* Exa hosts https://mcp.exa.ai/mcp as an intentionally keyless entry point: a
|
|
5
|
+
* single JSON-RPC `tools/call` of `web_search_exa` returns search results with
|
|
6
|
+
* content snippets. The response is SSE-framed (`data:` lines) or plain JSON,
|
|
7
|
+
* and the result payload is one text blob of `Title:`/`URL:`/`Text:` blocks
|
|
8
|
+
* separated by `---`, which we parse back into structured results.
|
|
9
|
+
*/
|
|
10
|
+
const EXA_MCP_ENDPOINT = 'https://mcp.exa.ai/mcp';
|
|
11
|
+
const DEFAULT_COUNT = 10;
|
|
12
|
+
const MAX_COUNT = 20;
|
|
13
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
14
|
+
const MAX_DESCRIPTION_CHARS = 400;
|
|
15
|
+
export class ExaSearchError extends Error {
|
|
16
|
+
kind;
|
|
17
|
+
status;
|
|
18
|
+
constructor(message, kind, status) {
|
|
19
|
+
super(message);
|
|
20
|
+
this.kind = kind;
|
|
21
|
+
this.status = status;
|
|
22
|
+
this.name = 'ExaSearchError';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
export async function exaSearch(query, opts = {}) {
|
|
26
|
+
const count = Math.max(1, Math.min(MAX_COUNT, opts.count ?? DEFAULT_COUNT));
|
|
27
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
28
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
29
|
+
const internalController = new AbortController();
|
|
30
|
+
let userAborted = false;
|
|
31
|
+
const timeoutHandle = setTimeout(() => internalController.abort(), timeoutMs);
|
|
32
|
+
const onUserAbort = () => {
|
|
33
|
+
userAborted = true;
|
|
34
|
+
internalController.abort();
|
|
35
|
+
};
|
|
36
|
+
if (opts.signal) {
|
|
37
|
+
if (opts.signal.aborted)
|
|
38
|
+
onUserAbort();
|
|
39
|
+
else
|
|
40
|
+
opts.signal.addEventListener('abort', onUserAbort, { once: true });
|
|
41
|
+
}
|
|
42
|
+
try {
|
|
43
|
+
let response;
|
|
44
|
+
try {
|
|
45
|
+
response = await fetchImpl(EXA_MCP_ENDPOINT, {
|
|
46
|
+
method: 'POST',
|
|
47
|
+
headers: {
|
|
48
|
+
'content-type': 'application/json',
|
|
49
|
+
accept: 'application/json, text/event-stream'
|
|
50
|
+
},
|
|
51
|
+
body: JSON.stringify({
|
|
52
|
+
jsonrpc: '2.0',
|
|
53
|
+
id: 1,
|
|
54
|
+
method: 'tools/call',
|
|
55
|
+
params: {
|
|
56
|
+
name: 'web_search_exa',
|
|
57
|
+
arguments: {
|
|
58
|
+
query,
|
|
59
|
+
numResults: count,
|
|
60
|
+
type: 'auto',
|
|
61
|
+
livecrawl: 'fallback',
|
|
62
|
+
contextMaxCharacters: 3000
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}),
|
|
66
|
+
signal: internalController.signal
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
catch (err) {
|
|
70
|
+
if (userAborted)
|
|
71
|
+
throw new ExaSearchError('Search aborted.', 'aborted');
|
|
72
|
+
throw new ExaSearchError(`Exa search request failed: ${describeError(err)}`, 'network');
|
|
73
|
+
}
|
|
74
|
+
if (!response.ok) {
|
|
75
|
+
throw new ExaSearchError(`Exa search HTTP ${response.status} ${response.statusText}`, 'http', response.status);
|
|
76
|
+
}
|
|
77
|
+
const rpc = parseRpcBody(await response.text());
|
|
78
|
+
if (rpc.error) {
|
|
79
|
+
throw new ExaSearchError(`Exa MCP error${rpc.error.code !== undefined ? ` ${rpc.error.code}` : ''}: ${rpc.error.message ?? 'unknown error'}`, 'protocol');
|
|
80
|
+
}
|
|
81
|
+
const text = rpc.result?.content?.find(c => c.type === 'text' && typeof c.text === 'string' && c.text.trim().length > 0)?.text;
|
|
82
|
+
if (rpc.result?.isError) {
|
|
83
|
+
throw new ExaSearchError(text?.trim() || 'Exa MCP returned an error result.', 'protocol');
|
|
84
|
+
}
|
|
85
|
+
if (!text)
|
|
86
|
+
throw new ExaSearchError('Exa MCP returned no text content.', 'protocol');
|
|
87
|
+
return parseResultBlocks(text).slice(0, count);
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
clearTimeout(timeoutHandle);
|
|
91
|
+
if (opts.signal)
|
|
92
|
+
opts.signal.removeEventListener('abort', onUserAbort);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* The endpoint answers either as a text/event-stream (`data: {json}` lines) or
|
|
97
|
+
* as a plain JSON body; accept both and take the first frame carrying a
|
|
98
|
+
* JSON-RPC result or error.
|
|
99
|
+
*/
|
|
100
|
+
function parseRpcBody(body) {
|
|
101
|
+
for (const line of body.split('\n')) {
|
|
102
|
+
if (!line.startsWith('data:'))
|
|
103
|
+
continue;
|
|
104
|
+
const payload = line.slice(5).trim();
|
|
105
|
+
if (!payload)
|
|
106
|
+
continue;
|
|
107
|
+
try {
|
|
108
|
+
const candidate = JSON.parse(payload);
|
|
109
|
+
if (candidate.result || candidate.error)
|
|
110
|
+
return candidate;
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
/* not this frame */
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
try {
|
|
117
|
+
const candidate = JSON.parse(body);
|
|
118
|
+
if (candidate.result || candidate.error)
|
|
119
|
+
return candidate;
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
/* fall through */
|
|
123
|
+
}
|
|
124
|
+
throw new ExaSearchError('Exa MCP returned an unparseable response.', 'protocol');
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Split the tool's text payload into `Title:`/`URL:` blocks. The per-result
|
|
128
|
+
* content lives after a `Text:` label (full text) or a `Highlights:` line
|
|
129
|
+
* (snippet mode); either becomes the description, whitespace-collapsed and
|
|
130
|
+
* capped so a result line stays a snippet, not a page dump.
|
|
131
|
+
*/
|
|
132
|
+
function parseResultBlocks(text) {
|
|
133
|
+
const blocks = text.split(/(?=^Title: )/m).filter(b => b.trim().length > 0);
|
|
134
|
+
const results = [];
|
|
135
|
+
for (const block of blocks) {
|
|
136
|
+
const title = block.match(/^Title: (.+)/m)?.[1]?.trim() ?? '';
|
|
137
|
+
const url = block.match(/^URL: (.+)/m)?.[1]?.trim() ?? '';
|
|
138
|
+
if (!url)
|
|
139
|
+
continue;
|
|
140
|
+
let content = '';
|
|
141
|
+
const textStart = block.indexOf('\nText: ');
|
|
142
|
+
if (textStart >= 0) {
|
|
143
|
+
content = block.slice(textStart + '\nText: '.length);
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
const highlights = block.match(/\nHighlights:[ \t]*\n/);
|
|
147
|
+
if (highlights?.index !== undefined) {
|
|
148
|
+
content = block.slice(highlights.index + highlights[0].length);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const description = content
|
|
152
|
+
.replace(/\n---\s*$/, '')
|
|
153
|
+
.replace(/\s+/g, ' ')
|
|
154
|
+
.trim()
|
|
155
|
+
.slice(0, MAX_DESCRIPTION_CHARS);
|
|
156
|
+
results.push({ title: title || url, url, description });
|
|
157
|
+
}
|
|
158
|
+
return results;
|
|
159
|
+
}
|
|
160
|
+
function describeError(err) {
|
|
161
|
+
if (err instanceof Error)
|
|
162
|
+
return err.message;
|
|
163
|
+
return String(err);
|
|
164
|
+
}
|