@mjasnikovs/pi-task 0.38.14 → 0.38.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/shared/child-process.js +9 -16
  2. package/dist/task/accept-debt.d.ts +7 -5
  3. package/dist/task/accept-debt.js +16 -13
  4. package/dist/task/auto-orchestrator.js +38 -36
  5. package/dist/task/autofix-ledger.d.ts +113 -0
  6. package/dist/task/autofix-ledger.js +152 -0
  7. package/dist/task/boot-probe.d.ts +63 -1
  8. package/dist/task/boot-probe.js +98 -2
  9. package/dist/task/child-runner.d.ts +50 -6
  10. package/dist/task/child-runner.js +48 -69
  11. package/dist/task/command-run.d.ts +49 -6
  12. package/dist/task/command-run.js +154 -18
  13. package/dist/task/external-context.d.ts +9 -12
  14. package/dist/task/external-context.js +5 -5
  15. package/dist/task/failure-classifier.d.ts +9 -1
  16. package/dist/task/failure-classifier.js +9 -0
  17. package/dist/task/final-gate-fix.d.ts +22 -26
  18. package/dist/task/final-gate-fix.js +2 -7
  19. package/dist/task/final-gate.d.ts +10 -2
  20. package/dist/task/final-gate.js +49 -88
  21. package/dist/task/gate-deps.js +20 -13
  22. package/dist/task/orchestrator.d.ts +33 -24
  23. package/dist/task/orchestrator.js +66 -44
  24. package/dist/task/phases.d.ts +58 -34
  25. package/dist/task/phases.js +140 -113
  26. package/dist/task/plan-orchestrator.js +2 -2
  27. package/dist/task/repo-health-check.d.ts +21 -21
  28. package/dist/task/repo-health-check.js +43 -112
  29. package/dist/task/run-end.d.ts +77 -0
  30. package/dist/task/run-end.js +37 -0
  31. package/dist/task/run-final-gate.js +71 -79
  32. package/dist/task/task-gates.d.ts +8 -0
  33. package/dist/task/task-gates.js +23 -4
  34. package/dist/task/terminal-outcome.d.ts +1 -1
  35. package/dist/task/terminal-outcome.js +12 -0
  36. package/dist/workers/brave-search.d.ts +7 -0
  37. package/dist/workers/brave-search.js +36 -55
  38. package/dist/workers/ddg-search.d.ts +1 -1
  39. package/dist/workers/ddg-search.js +27 -47
  40. package/dist/workers/exa-search.d.ts +2 -2
  41. package/dist/workers/exa-search.js +53 -68
  42. package/dist/workers/html-clean.js +67 -88
  43. package/dist/workers/http-request.d.ts +74 -0
  44. package/dist/workers/http-request.js +103 -0
  45. package/dist/workers/npm-version.js +37 -42
  46. package/dist/workers/pi-worker-core.d.ts +13 -2
  47. package/dist/workers/pi-worker-core.js +12 -17
  48. package/dist/workers/pi-worker-docs.d.ts +1 -1
  49. package/dist/workers/pi-worker-docs.js +49 -68
  50. package/dist/workers/pi-worker-fetch.d.ts +1 -1
  51. package/dist/workers/pi-worker-fetch.js +20 -21
  52. package/dist/workers/pi-worker-search.js +6 -4
  53. package/dist/workers/pi-worker.js +5 -4
  54. package/dist/workers/search-core.d.ts +1 -1
  55. package/dist/workers/search-core.js +36 -42
  56. package/dist/workers/search-types.d.ts +13 -0
  57. package/dist/workers/search-types.js +27 -0
  58. package/dist/workers/shared.d.ts +51 -11
  59. package/dist/workers/shared.js +0 -0
  60. package/dist/workers/worker-channels.d.ts +60 -0
  61. package/dist/workers/worker-channels.js +98 -0
  62. package/package.json +1 -1
@@ -319,12 +319,31 @@ export async function resolveVerifyGate(ctxIn, deps, p, rec, routeRootCause) {
319
319
  fixInstruction
320
320
  });
321
321
  active = fixRes.ctx ?? active;
322
- if (fixRes.sessionCancelled)
322
+ // The re-run's ending, mapped to this loop's own terminal kinds. A
323
+ // CANCEL lands on `interrupted` — the user stopped it, so the task is
324
+ // left resumable rather than reported as a fault.
325
+ if (fixRes.end.kind === 'no-session') {
323
326
  return { stop: { kind: 'session-cancelled', ctx: active } };
324
- if (fixRes.interrupted)
327
+ }
328
+ // A CANCEL and an ESC-interrupt are NOT the same ending. Folding them
329
+ // together sent a cancelled re-run down the `interrupted` row, which
330
+ // demotes the task file — writing `failed` over the `cancelled` the
331
+ // cancel itself wrote.
332
+ if (fixRes.end.kind === 'cancelled') {
333
+ return { stop: { kind: 'cancelled', ctx: active } };
334
+ }
335
+ if (fixRes.end.kind === 'interrupted') {
325
336
  return { stop: { kind: 'interrupted', ctx: active } };
326
- if (!fixRes.ok)
327
- return { stop: { kind: 'failed', ctx: active, reason: fixRes.reason } };
337
+ }
338
+ if (fixRes.end.kind === 'failed') {
339
+ return {
340
+ stop: {
341
+ kind: 'failed',
342
+ ctx: active,
343
+ ...(fixRes.end.reason === undefined ? {} : { reason: fixRes.end.reason })
344
+ }
345
+ };
346
+ }
328
347
  // Resume reuses the same inner task id, so p.taskId is stable.
329
348
  verified = await deps.verify(active, p.cwd, p.title, p.taskId);
330
349
  await rec(verdictLine(verified));
@@ -25,7 +25,7 @@
25
25
  * overrides, which is not a simplification.
26
26
  */
27
27
  /** The gate outcomes a command has to act on. Mirrors runGatesForTask's union. */
28
- export type TerminalOutcomeKind = 'done' | 'paused' | 'session-cancelled' | 'interrupted' | 'failed';
28
+ export type TerminalOutcomeKind = 'done' | 'paused' | 'session-cancelled' | 'cancelled' | 'interrupted' | 'failed';
29
29
  /** What the message needs to name. */
30
30
  export interface TerminalMessageContext {
31
31
  /** The task or run id shown to the user (`TASK_0007`, `AUTO_0002`, `Task`). */
@@ -59,6 +59,18 @@ export const TERMINAL_OUTCOMES = {
59
59
  message: c => `${c.tag} paused — could not start a session for autofix. `
60
60
  + `Run ${c.resumeCmd} to retry.`
61
61
  },
62
+ cancelled: {
63
+ // The USER stopped the re-run. The task file already says `cancelled`, and
64
+ // `markResumable` writes `failed` — that both lies in the ledger and turns
65
+ // a deliberate stop into a red error. RUN_END_POLICY states this for the
66
+ // first implementation run; this row states the same thing for a re-run.
67
+ markResumable: false,
68
+ failParent: false,
69
+ level: 'warning',
70
+ // `cancelled` is already in RESUMABLE_STATES, so the file needs no demotion
71
+ // AND the resume works — naming it costs nothing and is true.
72
+ message: c => `${c.tag} cancelled${c.at} — resume with ${c.resumeCmd}.`
73
+ },
62
74
  interrupted: {
63
75
  markResumable: true,
64
76
  // NOT a failure: the user stopped it. The parent run stays in_progress so
@@ -1,3 +1,4 @@
1
+ import { type FetchLike } from './http-request.js';
1
2
  import type { SearchResult } from './search-types.js';
2
3
  export type BraveResult = SearchResult;
3
4
  export interface BraveSearchOpts {
@@ -5,6 +6,12 @@ export interface BraveSearchOpts {
5
6
  count?: number;
6
7
  timeoutMs?: number;
7
8
  signal?: AbortSignal;
9
+ /**
10
+ * Injectable fetch. Brave was the ONE provider without this: exa and ddg both
11
+ * took a `fetchImpl`, so brave's status ladder — the widest of the three — was
12
+ * the only one no test could drive at the request level.
13
+ */
14
+ fetchImpl?: FetchLike;
8
15
  }
9
16
  export declare class BraveSearchError extends Error {
10
17
  readonly kind: 'auth' | 'rate-limit' | 'http' | 'network' | 'aborted';
@@ -1,3 +1,4 @@
1
+ import { httpRequest, HttpRequestError } from './http-request.js';
1
2
  const BRAVE_ENDPOINT = 'https://api.search.brave.com/res/v1/web/search';
2
3
  const DEFAULT_COUNT = 10;
3
4
  const MAX_COUNT = 20;
@@ -14,64 +15,44 @@ export class BraveSearchError extends Error {
14
15
  }
15
16
  export async function braveSearch(query, opts) {
16
17
  const count = Math.max(1, Math.min(MAX_COUNT, opts.count ?? DEFAULT_COUNT));
17
- const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
18
18
  const url = `${BRAVE_ENDPOINT}?q=${encodeURIComponent(query)}&count=${count}`;
19
- const internalController = new AbortController();
20
- let userAborted = false;
21
- const timeoutHandle = setTimeout(() => internalController.abort(), timeoutMs);
22
- const onUserAbort = () => {
23
- userAborted = true;
24
- internalController.abort();
25
- };
26
- if (opts.signal) {
27
- if (opts.signal.aborted)
28
- onUserAbort();
29
- else
30
- opts.signal.addEventListener('abort', onUserAbort, { once: true });
31
- }
32
19
  try {
33
- let response;
34
- try {
35
- response = await fetch(url, {
36
- method: 'GET',
37
- headers: {
38
- accept: 'application/json',
39
- 'x-subscription-token': opts.apiKey
40
- },
41
- signal: internalController.signal
42
- });
43
- }
44
- catch (err) {
45
- if (userAborted) {
46
- throw new BraveSearchError('Search aborted.', 'aborted');
20
+ return await httpRequest(url, {
21
+ timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
22
+ ...(opts.signal === undefined ? {} : { signal: opts.signal }),
23
+ ...(opts.fetchImpl === undefined ? {} : { fetchImpl: opts.fetchImpl }),
24
+ method: 'GET',
25
+ headers: {
26
+ accept: 'application/json',
27
+ 'x-subscription-token': opts.apiKey
47
28
  }
48
- throw new BraveSearchError(`Brave Search request failed: ${describeError(err)}`, 'network');
49
- }
50
- if (response.status === 401 || response.status === 403) {
51
- throw new BraveSearchError(`Brave Search rejected the key (HTTP ${response.status}). Check BRAVE_SEARCH_API_KEY.`, 'auth', response.status);
52
- }
53
- if (response.status === 429) {
54
- throw new BraveSearchError('Brave Search rate limit hit (HTTP 429). Try again in a moment.', 'rate-limit', 429);
55
- }
56
- if (!response.ok) {
57
- throw new BraveSearchError(`Brave Search HTTP ${response.status} ${response.statusText}`, 'http', response.status);
58
- }
59
- const body = (await response.json());
60
- const rawResults = body.web?.results ?? [];
61
- return rawResults
62
- .filter((r) => typeof r.title === 'string'
63
- && typeof r.url === 'string'
64
- && typeof r.description === 'string')
65
- .map(r => ({ title: r.title, url: r.url, description: r.description }));
29
+ }, async (response) => {
30
+ // Brave's own status policy: a rejected key and a rate limit are
31
+ // different problems for the user, and neither is a plain HTTP fault.
32
+ if (response.status === 401 || response.status === 403) {
33
+ throw new BraveSearchError(`Brave Search rejected the key (HTTP ${response.status}). Check BRAVE_SEARCH_API_KEY.`, 'auth', response.status);
34
+ }
35
+ if (response.status === 429) {
36
+ throw new BraveSearchError('Brave Search rate limit hit (HTTP 429). Try again in a moment.', 'rate-limit', 429);
37
+ }
38
+ if (!response.ok) {
39
+ throw new BraveSearchError(`Brave Search HTTP ${response.status} ${response.statusText}`, 'http', response.status);
40
+ }
41
+ const body = (await response.json());
42
+ const rawResults = body.web?.results ?? [];
43
+ return rawResults
44
+ .filter((r) => typeof r.title === 'string'
45
+ && typeof r.url === 'string'
46
+ && typeof r.description === 'string')
47
+ .map(r => ({ title: r.title, url: r.url, description: r.description }));
48
+ });
66
49
  }
67
- finally {
68
- clearTimeout(timeoutHandle);
69
- if (opts.signal)
70
- opts.signal.removeEventListener('abort', onUserAbort);
50
+ catch (err) {
51
+ if (err instanceof HttpRequestError) {
52
+ throw err.kind === 'aborted' ?
53
+ new BraveSearchError('Search aborted.', 'aborted')
54
+ : new BraveSearchError(`Brave Search request failed: ${err.detail}`, 'network');
55
+ }
56
+ throw err;
71
57
  }
72
58
  }
73
- function describeError(err) {
74
- if (err instanceof Error)
75
- return err.message;
76
- return String(err);
77
- }
@@ -7,7 +7,7 @@
7
7
  * get the destination URL. Ad rows redirect through duckduckgo.com itself and
8
8
  * are dropped.
9
9
  */
10
- import type { FetchLike } from './exa-search.js';
10
+ import { type FetchLike } from './http-request.js';
11
11
  import type { SearchResult } from './search-types.js';
12
12
  export interface DdgSearchOpts {
13
13
  count?: number;
@@ -8,6 +8,7 @@
8
8
  * are dropped.
9
9
  */
10
10
  import { parseHTML } from 'linkedom';
11
+ import { httpRequest, HttpRequestError } from './http-request.js';
11
12
  const DDG_ENDPOINT = 'https://html.duckduckgo.com/html/';
12
13
  const DEFAULT_COUNT = 10;
13
14
  const MAX_COUNT = 20;
@@ -26,51 +27,35 @@ export class DdgSearchError extends Error {
26
27
  }
27
28
  export async function ddgSearch(query, opts = {}) {
28
29
  const count = Math.max(1, Math.min(MAX_COUNT, opts.count ?? DEFAULT_COUNT));
29
- const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
30
- const fetchImpl = opts.fetchImpl ?? fetch;
31
30
  const url = `${DDG_ENDPOINT}?q=${encodeURIComponent(query)}`;
32
- const internalController = new AbortController();
33
- let userAborted = false;
34
- const timeoutHandle = setTimeout(() => internalController.abort(), timeoutMs);
35
- const onUserAbort = () => {
36
- userAborted = true;
37
- internalController.abort();
38
- };
39
- if (opts.signal) {
40
- if (opts.signal.aborted)
41
- onUserAbort();
42
- else
43
- opts.signal.addEventListener('abort', onUserAbort, { once: true });
44
- }
45
31
  try {
46
- let response;
47
- try {
48
- response = await fetchImpl(url, {
49
- method: 'GET',
50
- headers: {
51
- 'user-agent': USER_AGENT,
52
- accept: 'text/html'
53
- },
54
- signal: internalController.signal
55
- });
56
- }
57
- catch (err) {
58
- if (userAborted)
59
- throw new DdgSearchError('Search aborted.', 'aborted');
60
- throw new DdgSearchError(`DuckDuckGo request failed: ${describeError(err)}`, 'network');
61
- }
62
- if (response.status === 429 || response.status === 403) {
63
- throw new DdgSearchError(`DuckDuckGo is rate-limiting this client (HTTP ${response.status}). Try again in a moment.`, 'rate-limit', response.status);
64
- }
65
- if (!response.ok) {
66
- throw new DdgSearchError(`DuckDuckGo HTTP ${response.status} ${response.statusText}`, 'http', response.status);
67
- }
68
- return parseDdgHtml(await response.text()).slice(0, count);
32
+ return await httpRequest(url, {
33
+ timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
34
+ ...(opts.signal === undefined ? {} : { signal: opts.signal }),
35
+ ...(opts.fetchImpl === undefined ? {} : { fetchImpl: opts.fetchImpl }),
36
+ method: 'GET',
37
+ headers: {
38
+ 'user-agent': USER_AGENT,
39
+ accept: 'text/html'
40
+ }
41
+ }, async (response) => {
42
+ // DDG's own status policy: 429/403 is throttling, not a plain HTTP fault.
43
+ if (response.status === 429 || response.status === 403) {
44
+ throw new DdgSearchError(`DuckDuckGo is rate-limiting this client (HTTP ${response.status}). Try again in a moment.`, 'rate-limit', response.status);
45
+ }
46
+ if (!response.ok) {
47
+ throw new DdgSearchError(`DuckDuckGo HTTP ${response.status} ${response.statusText}`, 'http', response.status);
48
+ }
49
+ return parseDdgHtml(await response.text()).slice(0, count);
50
+ });
69
51
  }
70
- finally {
71
- clearTimeout(timeoutHandle);
72
- if (opts.signal)
73
- opts.signal.removeEventListener('abort', onUserAbort);
52
+ catch (err) {
53
+ if (err instanceof HttpRequestError) {
54
+ throw err.kind === 'aborted' ?
55
+ new DdgSearchError('Search aborted.', 'aborted')
56
+ : new DdgSearchError(`DuckDuckGo request failed: ${err.detail}`, 'network');
57
+ }
58
+ throw err;
74
59
  }
75
60
  }
76
61
  export function parseDdgHtml(html) {
@@ -123,8 +108,3 @@ function unwrapDdgRedirect(href) {
123
108
  function collapse(text) {
124
109
  return text.replace(/\s+/g, ' ').trim();
125
110
  }
126
- function describeError(err) {
127
- if (err instanceof Error)
128
- return err.message;
129
- return String(err);
130
- }
@@ -7,9 +7,9 @@
7
7
  * and the result payload is one text blob of `Title:`/`URL:`/`Text:` blocks
8
8
  * separated by `---`, which we parse back into structured results.
9
9
  */
10
+ import { type FetchLike } from './http-request.js';
10
11
  import type { SearchResult } from './search-types.js';
11
- /** Injectable fetch the narrow signature keeps test fakes free of Bun's extras. */
12
- export type FetchLike = (url: string, init?: RequestInit) => Promise<Response>;
12
+ export type { FetchLike };
13
13
  export interface ExaSearchOpts {
14
14
  count?: number;
15
15
  timeoutMs?: number;
@@ -7,6 +7,7 @@
7
7
  * and the result payload is one text blob of `Title:`/`URL:`/`Text:` blocks
8
8
  * separated by `---`, which we parse back into structured results.
9
9
  */
10
+ import { httpRequest, HttpRequestError } from './http-request.js';
10
11
  const EXA_MCP_ENDPOINT = 'https://mcp.exa.ai/mcp';
11
12
  const DEFAULT_COUNT = 10;
12
13
  const MAX_COUNT = 20;
@@ -24,72 +25,61 @@ export class ExaSearchError extends Error {
24
25
  }
25
26
  export async function exaSearch(query, opts = {}) {
26
27
  const count = Math.max(1, Math.min(MAX_COUNT, opts.count ?? DEFAULT_COUNT));
27
- const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
28
- const fetchImpl = opts.fetchImpl ?? fetch;
29
- const internalController = new AbortController();
30
- let userAborted = false;
31
- const timeoutHandle = setTimeout(() => internalController.abort(), timeoutMs);
32
- const onUserAbort = () => {
33
- userAborted = true;
34
- internalController.abort();
35
- };
36
- if (opts.signal) {
37
- if (opts.signal.aborted)
38
- onUserAbort();
39
- else
40
- opts.signal.addEventListener('abort', onUserAbort, { once: true });
41
- }
28
+ return await request({
29
+ timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
30
+ ...(opts.signal === undefined ? {} : { signal: opts.signal }),
31
+ ...(opts.fetchImpl === undefined ? {} : { fetchImpl: opts.fetchImpl })
32
+ }, count, query);
33
+ }
34
+ /** The Exa-specific half: its argv, its status policy, its error type. */
35
+ async function request(bounds, count, query) {
42
36
  try {
43
- let response;
44
- try {
45
- response = await fetchImpl(EXA_MCP_ENDPOINT, {
46
- method: 'POST',
47
- headers: {
48
- 'content-type': 'application/json',
49
- accept: 'application/json, text/event-stream'
50
- },
51
- body: JSON.stringify({
52
- jsonrpc: '2.0',
53
- id: 1,
54
- method: 'tools/call',
55
- params: {
56
- name: 'web_search_exa',
57
- arguments: {
58
- query,
59
- numResults: count,
60
- type: 'auto',
61
- livecrawl: 'fallback',
62
- contextMaxCharacters: 3000
63
- }
37
+ return await httpRequest(EXA_MCP_ENDPOINT, {
38
+ ...bounds,
39
+ method: 'POST',
40
+ headers: {
41
+ 'content-type': 'application/json',
42
+ accept: 'application/json, text/event-stream'
43
+ },
44
+ body: JSON.stringify({
45
+ jsonrpc: '2.0',
46
+ id: 1,
47
+ method: 'tools/call',
48
+ params: {
49
+ name: 'web_search_exa',
50
+ arguments: {
51
+ query,
52
+ numResults: count,
53
+ type: 'auto',
54
+ livecrawl: 'fallback',
55
+ contextMaxCharacters: 3000
64
56
  }
65
- }),
66
- signal: internalController.signal
67
- });
68
- }
69
- catch (err) {
70
- if (userAborted)
71
- throw new ExaSearchError('Search aborted.', 'aborted');
72
- throw new ExaSearchError(`Exa search request failed: ${describeError(err)}`, 'network');
73
- }
74
- if (!response.ok) {
75
- throw new ExaSearchError(`Exa search HTTP ${response.status} ${response.statusText}`, 'http', response.status);
76
- }
77
- const rpc = parseRpcBody(await response.text());
78
- if (rpc.error) {
79
- throw new ExaSearchError(`Exa MCP error${rpc.error.code !== undefined ? ` ${rpc.error.code}` : ''}: ${rpc.error.message ?? 'unknown error'}`, 'protocol');
80
- }
81
- const text = rpc.result?.content?.find(c => c.type === 'text' && typeof c.text === 'string' && c.text.trim().length > 0)?.text;
82
- if (rpc.result?.isError) {
83
- throw new ExaSearchError(text?.trim() || 'Exa MCP returned an error result.', 'protocol');
84
- }
85
- if (!text)
86
- throw new ExaSearchError('Exa MCP returned no text content.', 'protocol');
87
- return parseResultBlocks(text).slice(0, count);
57
+ }
58
+ })
59
+ }, async (response) => {
60
+ if (!response.ok) {
61
+ throw new ExaSearchError(`Exa search HTTP ${response.status} ${response.statusText}`, 'http', response.status);
62
+ }
63
+ const rpc = parseRpcBody(await response.text());
64
+ if (rpc.error) {
65
+ throw new ExaSearchError(`Exa MCP error${rpc.error.code !== undefined ? ` ${rpc.error.code}` : ''}: ${rpc.error.message ?? 'unknown error'}`, 'protocol');
66
+ }
67
+ const text = rpc.result?.content?.find(c => c.type === 'text' && typeof c.text === 'string' && c.text.trim().length > 0)?.text;
68
+ if (rpc.result?.isError) {
69
+ throw new ExaSearchError(text?.trim() || 'Exa MCP returned an error result.', 'protocol');
70
+ }
71
+ if (!text)
72
+ throw new ExaSearchError('Exa MCP returned no text content.', 'protocol');
73
+ return parseResultBlocks(text).slice(0, count);
74
+ });
88
75
  }
89
- finally {
90
- clearTimeout(timeoutHandle);
91
- if (opts.signal)
92
- opts.signal.removeEventListener('abort', onUserAbort);
76
+ catch (err) {
77
+ if (err instanceof HttpRequestError) {
78
+ throw err.kind === 'aborted' ?
79
+ new ExaSearchError('Search aborted.', 'aborted')
80
+ : new ExaSearchError(`Exa search request failed: ${err.detail}`, 'network');
81
+ }
82
+ throw err;
93
83
  }
94
84
  }
95
85
  /**
@@ -157,8 +147,3 @@ function parseResultBlocks(text) {
157
147
  }
158
148
  return results;
159
149
  }
160
- function describeError(err) {
161
- if (err instanceof Error)
162
- return err.message;
163
- return String(err);
164
- }
@@ -2,6 +2,7 @@ import { parseHTML } from 'linkedom';
2
2
  import { Readability } from '@mozilla/readability';
3
3
  import TurndownService from 'turndown';
4
4
  import { readPkgVersion } from '../shared/pkg-version.js';
5
+ import { httpRequest, HttpRequestError, describeError } from './http-request.js';
5
6
  const turndown = new TurndownService({
6
7
  codeBlockStyle: 'fenced',
7
8
  headingStyle: 'atx',
@@ -86,100 +87,83 @@ export class FetchAndCleanError extends Error {
86
87
  }
87
88
  }
88
89
  export async function fetchAndClean(url, opts = {}) {
89
- const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
90
90
  const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
91
- const internalController = new AbortController();
92
91
  let sizeExceeded = false;
93
- let userAborted = false;
94
- const timeoutHandle = setTimeout(() => internalController.abort(), timeoutMs);
95
- const onUserAbort = () => {
96
- userAborted = true;
97
- internalController.abort();
98
- };
99
- if (opts.signal) {
100
- if (opts.signal.aborted)
101
- onUserAbort();
102
- else
103
- opts.signal.addEventListener('abort', onUserAbort, { once: true });
104
- }
105
92
  try {
106
- let response;
107
- try {
108
- response = await fetch(url, {
109
- headers: { 'user-agent': USER_AGENT },
110
- redirect: 'follow',
111
- signal: internalController.signal
112
- });
113
- }
114
- catch (err) {
115
- if (userAborted) {
116
- throw new FetchAndCleanError('Fetch aborted.', 'aborted', err);
93
+ return await httpRequest(url, {
94
+ timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
95
+ ...(opts.signal === undefined ? {} : { signal: opts.signal }),
96
+ headers: { 'user-agent': USER_AGENT },
97
+ redirect: 'follow'
98
+ }, async (response, ctl) => {
99
+ if (!response.ok) {
100
+ throw new FetchAndCleanError(`Fetch failed: HTTP ${response.status} ${response.statusText} for ${url}`, 'http-error');
117
101
  }
118
- throw new FetchAndCleanError(`Could not fetch ${url}: ${describeError(err)}`, 'network', err);
119
- }
120
- if (!response.ok) {
121
- throw new FetchAndCleanError(`Fetch failed: HTTP ${response.status} ${response.statusText} for ${url}`, 'http-error');
122
- }
123
- const contentType = response.headers.get('content-type') ?? '';
124
- const kind = classifyContentType(contentType);
125
- if (kind === 'reject') {
126
- throw new FetchAndCleanError(`${url} is ${contentType || 'unknown content type'}, not a text or HTML page that pi-worker-fetch can read.`, 'not-html');
127
- }
128
- const reader = response.body?.getReader();
129
- if (!reader) {
130
- throw new FetchAndCleanError(`Could not fetch ${url}: empty response body`, 'network');
131
- }
132
- const decoder = decoderFor(contentType);
133
- let text = '';
134
- let bytesRead = 0;
135
- try {
136
- while (true) {
137
- // response.body's stream type doesn't resolve here, so the chunk
138
- // surfaces as `any`; pin it to the Uint8Array the reader yields.
139
- const { value, done } = (await reader.read());
140
- if (done)
141
- break;
142
- if (value) {
143
- bytesRead += value.byteLength;
144
- if (bytesRead > maxBytes) {
145
- sizeExceeded = true;
146
- internalController.abort();
102
+ const contentType = response.headers.get('content-type') ?? '';
103
+ const kind = classifyContentType(contentType);
104
+ if (kind === 'reject') {
105
+ throw new FetchAndCleanError(`${url} is ${contentType || 'unknown content type'}, not a text or HTML page that pi-worker-fetch can read.`, 'not-html');
106
+ }
107
+ const reader = response.body?.getReader();
108
+ if (!reader) {
109
+ throw new FetchAndCleanError(`Could not fetch ${url}: empty response body`, 'network');
110
+ }
111
+ const decoder = decoderFor(contentType);
112
+ let text = '';
113
+ let bytesRead = 0;
114
+ try {
115
+ while (true) {
116
+ // response.body's stream type doesn't resolve here, so the chunk
117
+ // surfaces as `any`; pin it to the Uint8Array the reader yields.
118
+ const { value, done } = (await reader.read());
119
+ if (done)
147
120
  break;
121
+ if (value) {
122
+ bytesRead += value.byteLength;
123
+ if (bytesRead > maxBytes) {
124
+ // OUR abort, told apart from the user's and the clock's
125
+ // by the seam — all three abort the same signal.
126
+ sizeExceeded = true;
127
+ ctl.abort();
128
+ break;
129
+ }
130
+ text += decoder.decode(value, { stream: true });
148
131
  }
149
- text += decoder.decode(value, { stream: true });
150
132
  }
133
+ text += decoder.decode();
151
134
  }
152
- text += decoder.decode();
153
- }
154
- catch (err) {
155
- if (sizeExceeded) {
156
- // fall through to throw outside the catch
157
- }
158
- else if (userAborted) {
159
- throw new FetchAndCleanError('Fetch aborted.', 'aborted', err);
135
+ catch (err) {
136
+ if (sizeExceeded) {
137
+ // fall through to throw outside the catch
138
+ }
139
+ else if (ctl.userAborted()) {
140
+ throw new FetchAndCleanError('Fetch aborted.', 'aborted', err);
141
+ }
142
+ else {
143
+ throw new FetchAndCleanError(`Could not fetch ${url}: ${describeError(err)}`, 'network', err);
144
+ }
160
145
  }
161
- else {
162
- throw new FetchAndCleanError(`Could not fetch ${url}: ${describeError(err)}`, 'network', err);
146
+ if (sizeExceeded) {
147
+ throw new FetchAndCleanError(`${url} exceeds ${formatBytes(maxBytes)} size cap. Try a more specific URL.`, 'too-large');
163
148
  }
164
- }
165
- if (sizeExceeded) {
166
- throw new FetchAndCleanError(`${url} exceeds ${formatBytes(maxBytes)} size cap. Try a more specific URL.`, 'too-large');
167
- }
168
- const finalUrl = response.url || url;
169
- if (kind === 'html') {
170
- return cleanHtml(text, finalUrl);
171
- }
172
- // text-ish formats are already clean — return them verbatim.
173
- return {
174
- title: hostnameOf(finalUrl),
175
- markdown: text.trim(),
176
- finalUrl
177
- };
149
+ const finalUrl = response.url || url;
150
+ if (kind === 'html')
151
+ return cleanHtml(text, finalUrl);
152
+ // text-ish formats are already clean — return them verbatim.
153
+ return {
154
+ title: hostnameOf(finalUrl),
155
+ markdown: text.trim(),
156
+ finalUrl
157
+ };
158
+ });
178
159
  }
179
- finally {
180
- clearTimeout(timeoutHandle);
181
- if (opts.signal)
182
- opts.signal.removeEventListener('abort', onUserAbort);
160
+ catch (err) {
161
+ if (err instanceof HttpRequestError) {
162
+ throw err.kind === 'aborted' ?
163
+ new FetchAndCleanError('Fetch aborted.', 'aborted', err.cause)
164
+ : new FetchAndCleanError(`Could not fetch ${url}: ${err.detail}`, 'network', err.cause);
165
+ }
166
+ throw err;
183
167
  }
184
168
  }
185
169
  function hostnameOf(url) {
@@ -190,11 +174,6 @@ function hostnameOf(url) {
190
174
  return url;
191
175
  }
192
176
  }
193
- function describeError(err) {
194
- if (err instanceof Error)
195
- return err.message;
196
- return String(err);
197
- }
198
177
  function formatBytes(n) {
199
178
  if (n >= 1024 * 1024)
200
179
  return `${(n / 1024 / 1024).toFixed(1)} MB`;