@mjasnikovs/pi-task 0.38.15 → 0.38.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/shared/child-process.js +9 -16
- package/dist/task/accept-debt.d.ts +7 -5
- package/dist/task/accept-debt.js +16 -13
- package/dist/task/auto-orchestrator.js +38 -36
- package/dist/task/autofix-ledger.d.ts +113 -0
- package/dist/task/autofix-ledger.js +152 -0
- package/dist/task/boot-probe.d.ts +63 -1
- package/dist/task/boot-probe.js +98 -2
- package/dist/task/child-runner.d.ts +50 -6
- package/dist/task/child-runner.js +48 -69
- package/dist/task/command-run.d.ts +49 -6
- package/dist/task/command-run.js +154 -18
- package/dist/task/external-context.d.ts +9 -12
- package/dist/task/external-context.js +5 -5
- package/dist/task/failure-classifier.d.ts +9 -1
- package/dist/task/failure-classifier.js +9 -0
- package/dist/task/final-gate-fix.d.ts +22 -26
- package/dist/task/final-gate-fix.js +2 -7
- package/dist/task/final-gate.d.ts +10 -2
- package/dist/task/final-gate.js +49 -88
- package/dist/task/gate-deps.js +20 -13
- package/dist/task/orchestrator.d.ts +33 -24
- package/dist/task/orchestrator.js +66 -44
- package/dist/task/phases.d.ts +58 -34
- package/dist/task/phases.js +140 -113
- package/dist/task/plan-orchestrator.js +2 -2
- package/dist/task/repo-health-check.d.ts +21 -21
- package/dist/task/repo-health-check.js +43 -112
- package/dist/task/run-end.d.ts +77 -0
- package/dist/task/run-end.js +37 -0
- package/dist/task/run-final-gate.js +71 -79
- package/dist/task/task-gates.d.ts +8 -0
- package/dist/task/task-gates.js +23 -4
- package/dist/task/terminal-outcome.d.ts +1 -1
- package/dist/task/terminal-outcome.js +12 -0
- package/dist/workers/brave-search.d.ts +7 -0
- package/dist/workers/brave-search.js +36 -55
- package/dist/workers/ddg-search.d.ts +1 -1
- package/dist/workers/ddg-search.js +27 -47
- package/dist/workers/exa-search.d.ts +2 -2
- package/dist/workers/exa-search.js +53 -68
- package/dist/workers/html-clean.js +67 -88
- package/dist/workers/http-request.d.ts +74 -0
- package/dist/workers/http-request.js +103 -0
- package/dist/workers/npm-version.js +37 -42
- package/dist/workers/pi-worker-core.d.ts +13 -2
- package/dist/workers/pi-worker-core.js +12 -17
- package/dist/workers/pi-worker-docs.d.ts +1 -1
- package/dist/workers/pi-worker-docs.js +49 -68
- package/dist/workers/pi-worker-fetch.d.ts +1 -1
- package/dist/workers/pi-worker-fetch.js +20 -21
- package/dist/workers/pi-worker-search.js +6 -4
- package/dist/workers/pi-worker.js +5 -4
- package/dist/workers/search-core.d.ts +1 -1
- package/dist/workers/search-core.js +36 -42
- package/dist/workers/search-types.d.ts +13 -0
- package/dist/workers/search-types.js +27 -0
- package/dist/workers/shared.d.ts +51 -11
- package/dist/workers/shared.js +0 -0
- package/dist/workers/worker-channels.d.ts +60 -0
- package/dist/workers/worker-channels.js +98 -0
- package/package.json +1 -1
package/dist/task/task-gates.js
CHANGED
|
@@ -319,12 +319,31 @@ export async function resolveVerifyGate(ctxIn, deps, p, rec, routeRootCause) {
|
|
|
319
319
|
fixInstruction
|
|
320
320
|
});
|
|
321
321
|
active = fixRes.ctx ?? active;
|
|
322
|
-
|
|
322
|
+
// The re-run's ending, mapped to this loop's own terminal kinds. A
|
|
323
|
+
// CANCEL lands on `interrupted` — the user stopped it, so the task is
|
|
324
|
+
// left resumable rather than reported as a fault.
|
|
325
|
+
if (fixRes.end.kind === 'no-session') {
|
|
323
326
|
return { stop: { kind: 'session-cancelled', ctx: active } };
|
|
324
|
-
|
|
327
|
+
}
|
|
328
|
+
// A CANCEL and an ESC-interrupt are NOT the same ending. Folding them
|
|
329
|
+
// together sent a cancelled re-run down the `interrupted` row, which
|
|
330
|
+
// demotes the task file — writing `failed` over the `cancelled` the
|
|
331
|
+
// cancel itself wrote.
|
|
332
|
+
if (fixRes.end.kind === 'cancelled') {
|
|
333
|
+
return { stop: { kind: 'cancelled', ctx: active } };
|
|
334
|
+
}
|
|
335
|
+
if (fixRes.end.kind === 'interrupted') {
|
|
325
336
|
return { stop: { kind: 'interrupted', ctx: active } };
|
|
326
|
-
|
|
327
|
-
|
|
337
|
+
}
|
|
338
|
+
if (fixRes.end.kind === 'failed') {
|
|
339
|
+
return {
|
|
340
|
+
stop: {
|
|
341
|
+
kind: 'failed',
|
|
342
|
+
ctx: active,
|
|
343
|
+
...(fixRes.end.reason === undefined ? {} : { reason: fixRes.end.reason })
|
|
344
|
+
}
|
|
345
|
+
};
|
|
346
|
+
}
|
|
328
347
|
// Resume reuses the same inner task id, so p.taskId is stable.
|
|
329
348
|
verified = await deps.verify(active, p.cwd, p.title, p.taskId);
|
|
330
349
|
await rec(verdictLine(verified));
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
* overrides, which is not a simplification.
|
|
26
26
|
*/
|
|
27
27
|
/** The gate outcomes a command has to act on. Mirrors runGatesForTask's union. */
|
|
28
|
-
export type TerminalOutcomeKind = 'done' | 'paused' | 'session-cancelled' | 'interrupted' | 'failed';
|
|
28
|
+
export type TerminalOutcomeKind = 'done' | 'paused' | 'session-cancelled' | 'cancelled' | 'interrupted' | 'failed';
|
|
29
29
|
/** What the message needs to name. */
|
|
30
30
|
export interface TerminalMessageContext {
|
|
31
31
|
/** The task or run id shown to the user (`TASK_0007`, `AUTO_0002`, `Task`). */
|
|
@@ -59,6 +59,18 @@ export const TERMINAL_OUTCOMES = {
|
|
|
59
59
|
message: c => `${c.tag} paused — could not start a session for autofix. `
|
|
60
60
|
+ `Run ${c.resumeCmd} to retry.`
|
|
61
61
|
},
|
|
62
|
+
cancelled: {
|
|
63
|
+
// The USER stopped the re-run. The task file already says `cancelled`, and
|
|
64
|
+
// `markResumable` writes `failed` — that both lies in the ledger and turns
|
|
65
|
+
// a deliberate stop into a red error. RUN_END_POLICY states this for the
|
|
66
|
+
// first implementation run; this row states the same thing for a re-run.
|
|
67
|
+
markResumable: false,
|
|
68
|
+
failParent: false,
|
|
69
|
+
level: 'warning',
|
|
70
|
+
// `cancelled` is already in RESUMABLE_STATES, so the file needs no demotion
|
|
71
|
+
// AND the resume works — naming it costs nothing and is true.
|
|
72
|
+
message: c => `${c.tag} cancelled${c.at} — resume with ${c.resumeCmd}.`
|
|
73
|
+
},
|
|
62
74
|
interrupted: {
|
|
63
75
|
markResumable: true,
|
|
64
76
|
// NOT a failure: the user stopped it. The parent run stays in_progress so
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type FetchLike } from './http-request.js';
|
|
1
2
|
import type { SearchResult } from './search-types.js';
|
|
2
3
|
export type BraveResult = SearchResult;
|
|
3
4
|
export interface BraveSearchOpts {
|
|
@@ -5,6 +6,12 @@ export interface BraveSearchOpts {
|
|
|
5
6
|
count?: number;
|
|
6
7
|
timeoutMs?: number;
|
|
7
8
|
signal?: AbortSignal;
|
|
9
|
+
/**
|
|
10
|
+
* Injectable fetch. Brave was the ONE provider without this: exa and ddg both
|
|
11
|
+
* took a `fetchImpl`, so brave's status ladder — the widest of the three — was
|
|
12
|
+
* the only one no test could drive at the request level.
|
|
13
|
+
*/
|
|
14
|
+
fetchImpl?: FetchLike;
|
|
8
15
|
}
|
|
9
16
|
export declare class BraveSearchError extends Error {
|
|
10
17
|
readonly kind: 'auth' | 'rate-limit' | 'http' | 'network' | 'aborted';
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { httpRequest, HttpRequestError } from './http-request.js';
|
|
1
2
|
const BRAVE_ENDPOINT = 'https://api.search.brave.com/res/v1/web/search';
|
|
2
3
|
const DEFAULT_COUNT = 10;
|
|
3
4
|
const MAX_COUNT = 20;
|
|
@@ -14,64 +15,44 @@ export class BraveSearchError extends Error {
|
|
|
14
15
|
}
|
|
15
16
|
export async function braveSearch(query, opts) {
|
|
16
17
|
const count = Math.max(1, Math.min(MAX_COUNT, opts.count ?? DEFAULT_COUNT));
|
|
17
|
-
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
18
18
|
const url = `${BRAVE_ENDPOINT}?q=${encodeURIComponent(query)}&count=${count}`;
|
|
19
|
-
const internalController = new AbortController();
|
|
20
|
-
let userAborted = false;
|
|
21
|
-
const timeoutHandle = setTimeout(() => internalController.abort(), timeoutMs);
|
|
22
|
-
const onUserAbort = () => {
|
|
23
|
-
userAborted = true;
|
|
24
|
-
internalController.abort();
|
|
25
|
-
};
|
|
26
|
-
if (opts.signal) {
|
|
27
|
-
if (opts.signal.aborted)
|
|
28
|
-
onUserAbort();
|
|
29
|
-
else
|
|
30
|
-
opts.signal.addEventListener('abort', onUserAbort, { once: true });
|
|
31
|
-
}
|
|
32
19
|
try {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
signal: internalController.signal
|
|
42
|
-
});
|
|
43
|
-
}
|
|
44
|
-
catch (err) {
|
|
45
|
-
if (userAborted) {
|
|
46
|
-
throw new BraveSearchError('Search aborted.', 'aborted');
|
|
20
|
+
return await httpRequest(url, {
|
|
21
|
+
timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
22
|
+
...(opts.signal === undefined ? {} : { signal: opts.signal }),
|
|
23
|
+
...(opts.fetchImpl === undefined ? {} : { fetchImpl: opts.fetchImpl }),
|
|
24
|
+
method: 'GET',
|
|
25
|
+
headers: {
|
|
26
|
+
accept: 'application/json',
|
|
27
|
+
'x-subscription-token': opts.apiKey
|
|
47
28
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
29
|
+
}, async (response) => {
|
|
30
|
+
// Brave's own status policy: a rejected key and a rate limit are
|
|
31
|
+
// different problems for the user, and neither is a plain HTTP fault.
|
|
32
|
+
if (response.status === 401 || response.status === 403) {
|
|
33
|
+
throw new BraveSearchError(`Brave Search rejected the key (HTTP ${response.status}). Check BRAVE_SEARCH_API_KEY.`, 'auth', response.status);
|
|
34
|
+
}
|
|
35
|
+
if (response.status === 429) {
|
|
36
|
+
throw new BraveSearchError('Brave Search rate limit hit (HTTP 429). Try again in a moment.', 'rate-limit', 429);
|
|
37
|
+
}
|
|
38
|
+
if (!response.ok) {
|
|
39
|
+
throw new BraveSearchError(`Brave Search HTTP ${response.status} ${response.statusText}`, 'http', response.status);
|
|
40
|
+
}
|
|
41
|
+
const body = (await response.json());
|
|
42
|
+
const rawResults = body.web?.results ?? [];
|
|
43
|
+
return rawResults
|
|
44
|
+
.filter((r) => typeof r.title === 'string'
|
|
45
|
+
&& typeof r.url === 'string'
|
|
46
|
+
&& typeof r.description === 'string')
|
|
47
|
+
.map(r => ({ title: r.title, url: r.url, description: r.description }));
|
|
48
|
+
});
|
|
66
49
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
50
|
+
catch (err) {
|
|
51
|
+
if (err instanceof HttpRequestError) {
|
|
52
|
+
throw err.kind === 'aborted' ?
|
|
53
|
+
new BraveSearchError('Search aborted.', 'aborted')
|
|
54
|
+
: new BraveSearchError(`Brave Search request failed: ${err.detail}`, 'network');
|
|
55
|
+
}
|
|
56
|
+
throw err;
|
|
71
57
|
}
|
|
72
58
|
}
|
|
73
|
-
function describeError(err) {
|
|
74
|
-
if (err instanceof Error)
|
|
75
|
-
return err.message;
|
|
76
|
-
return String(err);
|
|
77
|
-
}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* get the destination URL. Ad rows redirect through duckduckgo.com itself and
|
|
8
8
|
* are dropped.
|
|
9
9
|
*/
|
|
10
|
-
import type
|
|
10
|
+
import { type FetchLike } from './http-request.js';
|
|
11
11
|
import type { SearchResult } from './search-types.js';
|
|
12
12
|
export interface DdgSearchOpts {
|
|
13
13
|
count?: number;
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* are dropped.
|
|
9
9
|
*/
|
|
10
10
|
import { parseHTML } from 'linkedom';
|
|
11
|
+
import { httpRequest, HttpRequestError } from './http-request.js';
|
|
11
12
|
const DDG_ENDPOINT = 'https://html.duckduckgo.com/html/';
|
|
12
13
|
const DEFAULT_COUNT = 10;
|
|
13
14
|
const MAX_COUNT = 20;
|
|
@@ -26,51 +27,35 @@ export class DdgSearchError extends Error {
|
|
|
26
27
|
}
|
|
27
28
|
export async function ddgSearch(query, opts = {}) {
|
|
28
29
|
const count = Math.max(1, Math.min(MAX_COUNT, opts.count ?? DEFAULT_COUNT));
|
|
29
|
-
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
30
|
-
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
31
30
|
const url = `${DDG_ENDPOINT}?q=${encodeURIComponent(query)}`;
|
|
32
|
-
const internalController = new AbortController();
|
|
33
|
-
let userAborted = false;
|
|
34
|
-
const timeoutHandle = setTimeout(() => internalController.abort(), timeoutMs);
|
|
35
|
-
const onUserAbort = () => {
|
|
36
|
-
userAborted = true;
|
|
37
|
-
internalController.abort();
|
|
38
|
-
};
|
|
39
|
-
if (opts.signal) {
|
|
40
|
-
if (opts.signal.aborted)
|
|
41
|
-
onUserAbort();
|
|
42
|
-
else
|
|
43
|
-
opts.signal.addEventListener('abort', onUserAbort, { once: true });
|
|
44
|
-
}
|
|
45
31
|
try {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
}
|
|
65
|
-
if (!response.ok) {
|
|
66
|
-
throw new DdgSearchError(`DuckDuckGo HTTP ${response.status} ${response.statusText}`, 'http', response.status);
|
|
67
|
-
}
|
|
68
|
-
return parseDdgHtml(await response.text()).slice(0, count);
|
|
32
|
+
return await httpRequest(url, {
|
|
33
|
+
timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
34
|
+
...(opts.signal === undefined ? {} : { signal: opts.signal }),
|
|
35
|
+
...(opts.fetchImpl === undefined ? {} : { fetchImpl: opts.fetchImpl }),
|
|
36
|
+
method: 'GET',
|
|
37
|
+
headers: {
|
|
38
|
+
'user-agent': USER_AGENT,
|
|
39
|
+
accept: 'text/html'
|
|
40
|
+
}
|
|
41
|
+
}, async (response) => {
|
|
42
|
+
// DDG's own status policy: 429/403 is throttling, not a plain HTTP fault.
|
|
43
|
+
if (response.status === 429 || response.status === 403) {
|
|
44
|
+
throw new DdgSearchError(`DuckDuckGo is rate-limiting this client (HTTP ${response.status}). Try again in a moment.`, 'rate-limit', response.status);
|
|
45
|
+
}
|
|
46
|
+
if (!response.ok) {
|
|
47
|
+
throw new DdgSearchError(`DuckDuckGo HTTP ${response.status} ${response.statusText}`, 'http', response.status);
|
|
48
|
+
}
|
|
49
|
+
return parseDdgHtml(await response.text()).slice(0, count);
|
|
50
|
+
});
|
|
69
51
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
52
|
+
catch (err) {
|
|
53
|
+
if (err instanceof HttpRequestError) {
|
|
54
|
+
throw err.kind === 'aborted' ?
|
|
55
|
+
new DdgSearchError('Search aborted.', 'aborted')
|
|
56
|
+
: new DdgSearchError(`DuckDuckGo request failed: ${err.detail}`, 'network');
|
|
57
|
+
}
|
|
58
|
+
throw err;
|
|
74
59
|
}
|
|
75
60
|
}
|
|
76
61
|
export function parseDdgHtml(html) {
|
|
@@ -123,8 +108,3 @@ function unwrapDdgRedirect(href) {
|
|
|
123
108
|
function collapse(text) {
|
|
124
109
|
return text.replace(/\s+/g, ' ').trim();
|
|
125
110
|
}
|
|
126
|
-
function describeError(err) {
|
|
127
|
-
if (err instanceof Error)
|
|
128
|
-
return err.message;
|
|
129
|
-
return String(err);
|
|
130
|
-
}
|
|
@@ -7,9 +7,9 @@
|
|
|
7
7
|
* and the result payload is one text blob of `Title:`/`URL:`/`Text:` blocks
|
|
8
8
|
* separated by `---`, which we parse back into structured results.
|
|
9
9
|
*/
|
|
10
|
+
import { type FetchLike } from './http-request.js';
|
|
10
11
|
import type { SearchResult } from './search-types.js';
|
|
11
|
-
|
|
12
|
-
export type FetchLike = (url: string, init?: RequestInit) => Promise<Response>;
|
|
12
|
+
export type { FetchLike };
|
|
13
13
|
export interface ExaSearchOpts {
|
|
14
14
|
count?: number;
|
|
15
15
|
timeoutMs?: number;
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* and the result payload is one text blob of `Title:`/`URL:`/`Text:` blocks
|
|
8
8
|
* separated by `---`, which we parse back into structured results.
|
|
9
9
|
*/
|
|
10
|
+
import { httpRequest, HttpRequestError } from './http-request.js';
|
|
10
11
|
const EXA_MCP_ENDPOINT = 'https://mcp.exa.ai/mcp';
|
|
11
12
|
const DEFAULT_COUNT = 10;
|
|
12
13
|
const MAX_COUNT = 20;
|
|
@@ -24,72 +25,61 @@ export class ExaSearchError extends Error {
|
|
|
24
25
|
}
|
|
25
26
|
export async function exaSearch(query, opts = {}) {
|
|
26
27
|
const count = Math.max(1, Math.min(MAX_COUNT, opts.count ?? DEFAULT_COUNT));
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
};
|
|
36
|
-
if (opts.signal) {
|
|
37
|
-
if (opts.signal.aborted)
|
|
38
|
-
onUserAbort();
|
|
39
|
-
else
|
|
40
|
-
opts.signal.addEventListener('abort', onUserAbort, { once: true });
|
|
41
|
-
}
|
|
28
|
+
return await request({
|
|
29
|
+
timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
30
|
+
...(opts.signal === undefined ? {} : { signal: opts.signal }),
|
|
31
|
+
...(opts.fetchImpl === undefined ? {} : { fetchImpl: opts.fetchImpl })
|
|
32
|
+
}, count, query);
|
|
33
|
+
}
|
|
34
|
+
/** The Exa-specific half: its argv, its status policy, its error type. */
|
|
35
|
+
async function request(bounds, count, query) {
|
|
42
36
|
try {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
contextMaxCharacters: 3000
|
|
63
|
-
}
|
|
37
|
+
return await httpRequest(EXA_MCP_ENDPOINT, {
|
|
38
|
+
...bounds,
|
|
39
|
+
method: 'POST',
|
|
40
|
+
headers: {
|
|
41
|
+
'content-type': 'application/json',
|
|
42
|
+
accept: 'application/json, text/event-stream'
|
|
43
|
+
},
|
|
44
|
+
body: JSON.stringify({
|
|
45
|
+
jsonrpc: '2.0',
|
|
46
|
+
id: 1,
|
|
47
|
+
method: 'tools/call',
|
|
48
|
+
params: {
|
|
49
|
+
name: 'web_search_exa',
|
|
50
|
+
arguments: {
|
|
51
|
+
query,
|
|
52
|
+
numResults: count,
|
|
53
|
+
type: 'auto',
|
|
54
|
+
livecrawl: 'fallback',
|
|
55
|
+
contextMaxCharacters: 3000
|
|
64
56
|
}
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
throw new ExaSearchError(text?.trim() || 'Exa MCP returned an error result.', 'protocol');
|
|
84
|
-
}
|
|
85
|
-
if (!text)
|
|
86
|
-
throw new ExaSearchError('Exa MCP returned no text content.', 'protocol');
|
|
87
|
-
return parseResultBlocks(text).slice(0, count);
|
|
57
|
+
}
|
|
58
|
+
})
|
|
59
|
+
}, async (response) => {
|
|
60
|
+
if (!response.ok) {
|
|
61
|
+
throw new ExaSearchError(`Exa search HTTP ${response.status} ${response.statusText}`, 'http', response.status);
|
|
62
|
+
}
|
|
63
|
+
const rpc = parseRpcBody(await response.text());
|
|
64
|
+
if (rpc.error) {
|
|
65
|
+
throw new ExaSearchError(`Exa MCP error${rpc.error.code !== undefined ? ` ${rpc.error.code}` : ''}: ${rpc.error.message ?? 'unknown error'}`, 'protocol');
|
|
66
|
+
}
|
|
67
|
+
const text = rpc.result?.content?.find(c => c.type === 'text' && typeof c.text === 'string' && c.text.trim().length > 0)?.text;
|
|
68
|
+
if (rpc.result?.isError) {
|
|
69
|
+
throw new ExaSearchError(text?.trim() || 'Exa MCP returned an error result.', 'protocol');
|
|
70
|
+
}
|
|
71
|
+
if (!text)
|
|
72
|
+
throw new ExaSearchError('Exa MCP returned no text content.', 'protocol');
|
|
73
|
+
return parseResultBlocks(text).slice(0, count);
|
|
74
|
+
});
|
|
88
75
|
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
76
|
+
catch (err) {
|
|
77
|
+
if (err instanceof HttpRequestError) {
|
|
78
|
+
throw err.kind === 'aborted' ?
|
|
79
|
+
new ExaSearchError('Search aborted.', 'aborted')
|
|
80
|
+
: new ExaSearchError(`Exa search request failed: ${err.detail}`, 'network');
|
|
81
|
+
}
|
|
82
|
+
throw err;
|
|
93
83
|
}
|
|
94
84
|
}
|
|
95
85
|
/**
|
|
@@ -157,8 +147,3 @@ function parseResultBlocks(text) {
|
|
|
157
147
|
}
|
|
158
148
|
return results;
|
|
159
149
|
}
|
|
160
|
-
function describeError(err) {
|
|
161
|
-
if (err instanceof Error)
|
|
162
|
-
return err.message;
|
|
163
|
-
return String(err);
|
|
164
|
-
}
|
|
@@ -2,6 +2,7 @@ import { parseHTML } from 'linkedom';
|
|
|
2
2
|
import { Readability } from '@mozilla/readability';
|
|
3
3
|
import TurndownService from 'turndown';
|
|
4
4
|
import { readPkgVersion } from '../shared/pkg-version.js';
|
|
5
|
+
import { httpRequest, HttpRequestError, describeError } from './http-request.js';
|
|
5
6
|
const turndown = new TurndownService({
|
|
6
7
|
codeBlockStyle: 'fenced',
|
|
7
8
|
headingStyle: 'atx',
|
|
@@ -86,100 +87,83 @@ export class FetchAndCleanError extends Error {
|
|
|
86
87
|
}
|
|
87
88
|
}
|
|
88
89
|
export async function fetchAndClean(url, opts = {}) {
|
|
89
|
-
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
90
90
|
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
91
|
-
const internalController = new AbortController();
|
|
92
91
|
let sizeExceeded = false;
|
|
93
|
-
let userAborted = false;
|
|
94
|
-
const timeoutHandle = setTimeout(() => internalController.abort(), timeoutMs);
|
|
95
|
-
const onUserAbort = () => {
|
|
96
|
-
userAborted = true;
|
|
97
|
-
internalController.abort();
|
|
98
|
-
};
|
|
99
|
-
if (opts.signal) {
|
|
100
|
-
if (opts.signal.aborted)
|
|
101
|
-
onUserAbort();
|
|
102
|
-
else
|
|
103
|
-
opts.signal.addEventListener('abort', onUserAbort, { once: true });
|
|
104
|
-
}
|
|
105
92
|
try {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
catch (err) {
|
|
115
|
-
if (userAborted) {
|
|
116
|
-
throw new FetchAndCleanError('Fetch aborted.', 'aborted', err);
|
|
93
|
+
return await httpRequest(url, {
|
|
94
|
+
timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
95
|
+
...(opts.signal === undefined ? {} : { signal: opts.signal }),
|
|
96
|
+
headers: { 'user-agent': USER_AGENT },
|
|
97
|
+
redirect: 'follow'
|
|
98
|
+
}, async (response, ctl) => {
|
|
99
|
+
if (!response.ok) {
|
|
100
|
+
throw new FetchAndCleanError(`Fetch failed: HTTP ${response.status} ${response.statusText} for ${url}`, 'http-error');
|
|
117
101
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
while (true) {
|
|
137
|
-
// response.body's stream type doesn't resolve here, so the chunk
|
|
138
|
-
// surfaces as `any`; pin it to the Uint8Array the reader yields.
|
|
139
|
-
const { value, done } = (await reader.read());
|
|
140
|
-
if (done)
|
|
141
|
-
break;
|
|
142
|
-
if (value) {
|
|
143
|
-
bytesRead += value.byteLength;
|
|
144
|
-
if (bytesRead > maxBytes) {
|
|
145
|
-
sizeExceeded = true;
|
|
146
|
-
internalController.abort();
|
|
102
|
+
const contentType = response.headers.get('content-type') ?? '';
|
|
103
|
+
const kind = classifyContentType(contentType);
|
|
104
|
+
if (kind === 'reject') {
|
|
105
|
+
throw new FetchAndCleanError(`${url} is ${contentType || 'unknown content type'}, not a text or HTML page that pi-worker-fetch can read.`, 'not-html');
|
|
106
|
+
}
|
|
107
|
+
const reader = response.body?.getReader();
|
|
108
|
+
if (!reader) {
|
|
109
|
+
throw new FetchAndCleanError(`Could not fetch ${url}: empty response body`, 'network');
|
|
110
|
+
}
|
|
111
|
+
const decoder = decoderFor(contentType);
|
|
112
|
+
let text = '';
|
|
113
|
+
let bytesRead = 0;
|
|
114
|
+
try {
|
|
115
|
+
while (true) {
|
|
116
|
+
// response.body's stream type doesn't resolve here, so the chunk
|
|
117
|
+
// surfaces as `any`; pin it to the Uint8Array the reader yields.
|
|
118
|
+
const { value, done } = (await reader.read());
|
|
119
|
+
if (done)
|
|
147
120
|
break;
|
|
121
|
+
if (value) {
|
|
122
|
+
bytesRead += value.byteLength;
|
|
123
|
+
if (bytesRead > maxBytes) {
|
|
124
|
+
// OUR abort, told apart from the user's and the clock's
|
|
125
|
+
// by the seam — all three abort the same signal.
|
|
126
|
+
sizeExceeded = true;
|
|
127
|
+
ctl.abort();
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
text += decoder.decode(value, { stream: true });
|
|
148
131
|
}
|
|
149
|
-
text += decoder.decode(value, { stream: true });
|
|
150
132
|
}
|
|
133
|
+
text += decoder.decode();
|
|
151
134
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
135
|
+
catch (err) {
|
|
136
|
+
if (sizeExceeded) {
|
|
137
|
+
// fall through to throw outside the catch
|
|
138
|
+
}
|
|
139
|
+
else if (ctl.userAborted()) {
|
|
140
|
+
throw new FetchAndCleanError('Fetch aborted.', 'aborted', err);
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
throw new FetchAndCleanError(`Could not fetch ${url}: ${describeError(err)}`, 'network', err);
|
|
144
|
+
}
|
|
160
145
|
}
|
|
161
|
-
|
|
162
|
-
throw new FetchAndCleanError(
|
|
146
|
+
if (sizeExceeded) {
|
|
147
|
+
throw new FetchAndCleanError(`${url} exceeds ${formatBytes(maxBytes)} size cap. Try a more specific URL.`, 'too-large');
|
|
163
148
|
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
title: hostnameOf(finalUrl),
|
|
175
|
-
markdown: text.trim(),
|
|
176
|
-
finalUrl
|
|
177
|
-
};
|
|
149
|
+
const finalUrl = response.url || url;
|
|
150
|
+
if (kind === 'html')
|
|
151
|
+
return cleanHtml(text, finalUrl);
|
|
152
|
+
// text-ish formats are already clean — return them verbatim.
|
|
153
|
+
return {
|
|
154
|
+
title: hostnameOf(finalUrl),
|
|
155
|
+
markdown: text.trim(),
|
|
156
|
+
finalUrl
|
|
157
|
+
};
|
|
158
|
+
});
|
|
178
159
|
}
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
160
|
+
catch (err) {
|
|
161
|
+
if (err instanceof HttpRequestError) {
|
|
162
|
+
throw err.kind === 'aborted' ?
|
|
163
|
+
new FetchAndCleanError('Fetch aborted.', 'aborted', err.cause)
|
|
164
|
+
: new FetchAndCleanError(`Could not fetch ${url}: ${err.detail}`, 'network', err.cause);
|
|
165
|
+
}
|
|
166
|
+
throw err;
|
|
183
167
|
}
|
|
184
168
|
}
|
|
185
169
|
function hostnameOf(url) {
|
|
@@ -190,11 +174,6 @@ function hostnameOf(url) {
|
|
|
190
174
|
return url;
|
|
191
175
|
}
|
|
192
176
|
}
|
|
193
|
-
function describeError(err) {
|
|
194
|
-
if (err instanceof Error)
|
|
195
|
-
return err.message;
|
|
196
|
-
return String(err);
|
|
197
|
-
}
|
|
198
177
|
function formatBytes(n) {
|
|
199
178
|
if (n >= 1024 * 1024)
|
|
200
179
|
return `${(n / 1024 / 1024).toFixed(1)} MB`;
|