@mjasnikovs/pi-task 0.38.9 → 0.38.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/task/auto-orchestrator.js +2 -2
- package/dist/task/boot-probe.d.ts +298 -0
- package/dist/task/boot-probe.js +806 -0
- package/dist/task/child-runner.d.ts +17 -0
- package/dist/task/child-runner.js +6 -0
- package/dist/task/final-gate.d.ts +33 -281
- package/dist/task/final-gate.js +24 -834
- package/dist/task/launch-manifest.d.ts +5 -0
- package/dist/task/launch-manifest.js +21 -0
- package/dist/task/orchestrator.js +3 -3
- package/dist/task/phases.d.ts +18 -0
- package/dist/task/phases.js +4 -3
- package/dist/task/task-gates.d.ts +69 -0
- package/dist/task/task-gates.js +114 -90
- package/dist/workers/docs-core.d.ts +0 -4
- package/dist/workers/docs-core.js +10 -34
- package/dist/workers/docs-project.js +3 -3
- package/dist/workers/docs-resolve.d.ts +18 -0
- package/dist/workers/docs-resolve.js +39 -0
- package/dist/workers/docs-retrieve.d.ts +13 -0
- package/dist/workers/docs-retrieve.js +17 -2
- package/dist/workers/fetch-core.d.ts +0 -4
- package/dist/workers/fetch-core.js +2 -5
- package/dist/workers/phantom-imports.d.ts +3 -3
- package/dist/workers/phantom-imports.js +16 -29
- package/dist/workers/pi-worker-docs.d.ts +49 -0
- package/dist/workers/pi-worker-docs.js +33 -9
- package/dist/workers/pi-worker-fetch.d.ts +18 -0
- package/dist/workers/pi-worker-fetch.js +19 -4
- package/dist/workers/research-cache.d.ts +2 -0
- package/dist/workers/research-cache.js +65 -3
- package/package.json +1 -1
|
@@ -12,4 +12,17 @@ export interface RetrieveOptions {
|
|
|
12
12
|
limit?: number;
|
|
13
13
|
contentBudget?: number;
|
|
14
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* How many chunks a retrieval returns, per corpus.
|
|
17
|
+
*
|
|
18
|
+
* The two values differ and always have; this is the first place they sit side by
|
|
19
|
+
* side, and no comment in the history explains WHY an npm package gets 8 and
|
|
20
|
+
* project source gets 50. Recorded as-is rather than harmonised: changing either is
|
|
21
|
+
* a retrieval-policy change with its own A/B, not a tidy-up. Before this they were
|
|
22
|
+
* three declarations across three files, so the divergence was invisible.
|
|
23
|
+
*/
|
|
24
|
+
export declare const PACKAGE_RETRIEVE_LIMIT = 8;
|
|
25
|
+
export declare const PROJECT_RETRIEVE_LIMIT = 50;
|
|
26
|
+
/** Character budget for the assembled chunk text. The same for both corpora. */
|
|
27
|
+
export declare const RETRIEVE_CONTENT_BUDGET = 24000;
|
|
15
28
|
export declare function retrieveChunks(cache: CacheHandle, opts: RetrieveOptions): RetrievedChunk[];
|
|
@@ -1,5 +1,20 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* How many chunks a retrieval returns, per corpus.
|
|
3
|
+
*
|
|
4
|
+
* The two values differ and always have; this is the first place they sit side by
|
|
5
|
+
* side, and no comment in the history explains WHY an npm package gets 8 and
|
|
6
|
+
* project source gets 50. Recorded as-is rather than harmonised: changing either is
|
|
7
|
+
* a retrieval-policy change with its own A/B, not a tidy-up. Before this they were
|
|
8
|
+
* three declarations across three files, so the divergence was invisible.
|
|
9
|
+
*/
|
|
10
|
+
export const PACKAGE_RETRIEVE_LIMIT = 8;
|
|
11
|
+
export const PROJECT_RETRIEVE_LIMIT = 50;
|
|
12
|
+
/** Character budget for the assembled chunk text. The same for both corpora. */
|
|
13
|
+
export const RETRIEVE_CONTENT_BUDGET = 24_000;
|
|
14
|
+
// Both callers always pass `limit`/`contentBudget` explicitly, so these defaults are
|
|
15
|
+
// only a backstop for a third caller that does not.
|
|
16
|
+
const DEFAULT_LIMIT = PROJECT_RETRIEVE_LIMIT;
|
|
17
|
+
const DEFAULT_BUDGET = RETRIEVE_CONTENT_BUDGET;
|
|
3
18
|
const MIN_TOKEN_LEN = 2;
|
|
4
19
|
const FALLBACK_DTS_CHARS = 12_000;
|
|
5
20
|
const FALLBACK_README_CHARS = 4_000;
|
|
@@ -104,7 +104,3 @@ export interface SelectedContent {
|
|
|
104
104
|
export declare function selectContent(markdown: string, requestedUrl: string): SelectedContent;
|
|
105
105
|
/** The shipped strategy: fragment-aware selection + the recalibrated prompt. */
|
|
106
106
|
export declare const shippedStrategy: PromptStrategy;
|
|
107
|
-
export declare function formatResultText(parsed: {
|
|
108
|
-
answer: string;
|
|
109
|
-
excerpt?: string;
|
|
110
|
-
}, verified: boolean | undefined): string;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { fetchAndClean as defaultFetchAndClean } from './html-clean.js';
|
|
2
2
|
import { runFocusedExtraction } from './focused-extractor.js';
|
|
3
3
|
import { abstentionSentence } from './abstention.js';
|
|
4
|
-
import { formatResultText as formatResultTextShared } from '../shared/child-output.js';
|
|
5
4
|
const CONTENT_BUDGET = 30_000;
|
|
6
5
|
const HEAD_CHARS = 25_000;
|
|
7
6
|
const TAIL_CHARS = 5_000;
|
|
@@ -243,7 +242,5 @@ function buildPrompt(args) {
|
|
|
243
242
|
}
|
|
244
243
|
/** The shipped strategy: fragment-aware selection + the recalibrated prompt. */
|
|
245
244
|
export const shippedStrategy = { selectContent, buildPrompt };
|
|
246
|
-
//
|
|
247
|
-
|
|
248
|
-
return formatResultTextShared('', parsed, verified);
|
|
249
|
-
}
|
|
245
|
+
// A fetch answer carries no package header, so the shared formatter is bound with an
|
|
246
|
+
// empty one at the single call site rather than behind a wrapper of its own.
|
|
@@ -26,14 +26,14 @@ export interface RuntimeImportVerdict {
|
|
|
26
26
|
*/
|
|
27
27
|
export declare function classifyRuntimeImport(spec: string, runtime: string, sub: string, typeText: string): RuntimeImportVerdict;
|
|
28
28
|
/** Default loader: resolve the runtime's installed types and read their .d.ts. */
|
|
29
|
-
export declare function loadRuntimeTypeText(runtime: string, cwd: string): string | null
|
|
29
|
+
export declare function loadRuntimeTypeText(runtime: string, cwd: string): Promise<string | null>;
|
|
30
30
|
/**
|
|
31
31
|
* Scan `text` for runtime-namespace specifiers and return the ones the installed
|
|
32
32
|
* types do not declare. `loadText` is injectable for tests; in production it reads
|
|
33
33
|
* the runtime's type package. A runtime whose types can't be loaded is skipped
|
|
34
34
|
* (we never flag what we can't verify), so this is silent when types are absent.
|
|
35
35
|
*/
|
|
36
|
-
export declare function findPhantomImports(text: string, cwd: string, loadText?: (runtime: string, cwd: string) => string | null): PhantomImport[]
|
|
36
|
+
export declare function findPhantomImports(text: string, cwd: string, loadText?: (runtime: string, cwd: string) => string | null | Promise<string | null>): Promise<PhantomImport[]>;
|
|
37
37
|
/** Render flagged phantoms as an authoritative research section, or '' if none. */
|
|
38
38
|
export declare function formatApiCorrections(phantoms: PhantomImport[]): string;
|
|
39
39
|
/**
|
|
@@ -54,7 +54,7 @@ export declare function formatApiOverrideBanner(phantoms: PhantomImport[]): stri
|
|
|
54
54
|
* Unreadable mentions are skipped; silent when types are absent or nothing is flagged.
|
|
55
55
|
* Drives the impl-handoff override banner (Layer B).
|
|
56
56
|
*/
|
|
57
|
-
export declare function findDeliveryPhantoms(spec: string, cwd: string): PhantomImport[]
|
|
57
|
+
export declare function findDeliveryPhantoms(spec: string, cwd: string): Promise<PhantomImport[]>;
|
|
58
58
|
/**
|
|
59
59
|
* Subtractively rewrite every flagged phantom specifier in `text` to the canonical
|
|
60
60
|
* import the installed types prove, so NO affirmative occurrence of the non-existent
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import * as fs from 'node:fs';
|
|
15
15
|
import * as path from 'node:path';
|
|
16
|
-
import { resolvePackage, splitRuntimeNamespace,
|
|
16
|
+
import { resolvePackage, splitRuntimeNamespace, hasTypeFiles, isDtsFile, resolveTypeSource, ResolveError } from './docs-resolve.js';
|
|
17
17
|
// Runtime builtin specifiers as they appear in prose/code: `bun:sql`,
|
|
18
18
|
// `node:fs/promises`. Bounded to the runtimes splitRuntimeNamespace accepts.
|
|
19
19
|
const SPEC_RE = /\b(?:bun|node|deno):[a-z0-9][a-z0-9/_-]*/gi;
|
|
@@ -67,40 +67,27 @@ function suggestionFor(v, runtime) {
|
|
|
67
67
|
return (`\`${v.spec}\` is NOT a real module.${list} Import the needed symbol from `
|
|
68
68
|
+ `"${runtime}" or verify the correct specifier with pi-worker-docs; do not declare a module for it.`);
|
|
69
69
|
}
|
|
70
|
-
/**
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
let cur;
|
|
70
|
+
/** The phantom-import checker's adapter over the shared redirect walk: hops resolve
|
|
71
|
+
* SYNCHRONOUSLY and never install. A runtime whose types aren't on disk simply
|
|
72
|
+
* can't be verified (null), so we never flag what we can't prove. */
|
|
73
|
+
async function resolveRuntimeTypesRoot(runtime, cwd) {
|
|
74
|
+
let start;
|
|
76
75
|
try {
|
|
77
|
-
|
|
76
|
+
start = resolvePackage(runtime, cwd);
|
|
78
77
|
}
|
|
79
78
|
catch (err) {
|
|
80
79
|
if (err instanceof ResolveError)
|
|
81
80
|
return null;
|
|
82
81
|
throw err;
|
|
83
82
|
}
|
|
84
|
-
const
|
|
85
|
-
for (let hop = 0; hop < 3; hop++) {
|
|
86
|
-
let next = detectTypesRedirect(cur);
|
|
87
|
-
if (next && visited.has(next))
|
|
88
|
-
next = null;
|
|
89
|
-
if (!next && !hasTypeFiles(cur.root)) {
|
|
90
|
-
const types = typesPackageName(cur.name);
|
|
91
|
-
if (types && !visited.has(types))
|
|
92
|
-
next = types;
|
|
93
|
-
}
|
|
94
|
-
if (!next)
|
|
95
|
-
break;
|
|
96
|
-
visited.add(next);
|
|
83
|
+
const cur = await resolveTypeSource(start, runtime, next => {
|
|
97
84
|
try {
|
|
98
|
-
|
|
85
|
+
return Promise.resolve(resolvePackage(next, cwd));
|
|
99
86
|
}
|
|
100
87
|
catch {
|
|
101
|
-
|
|
88
|
+
return Promise.resolve(null);
|
|
102
89
|
}
|
|
103
|
-
}
|
|
90
|
+
});
|
|
104
91
|
return hasTypeFiles(cur.root) ? cur.root : null;
|
|
105
92
|
}
|
|
106
93
|
const MAX_TYPE_BYTES = 4_000_000;
|
|
@@ -141,8 +128,8 @@ function readRuntimeTypeText(root) {
|
|
|
141
128
|
return parts.join('\n');
|
|
142
129
|
}
|
|
143
130
|
/** Default loader: resolve the runtime's installed types and read their .d.ts. */
|
|
144
|
-
export function loadRuntimeTypeText(runtime, cwd) {
|
|
145
|
-
const root = resolveRuntimeTypesRoot(runtime, cwd);
|
|
131
|
+
export async function loadRuntimeTypeText(runtime, cwd) {
|
|
132
|
+
const root = await resolveRuntimeTypesRoot(runtime, cwd);
|
|
146
133
|
if (!root)
|
|
147
134
|
return null;
|
|
148
135
|
const text = readRuntimeTypeText(root);
|
|
@@ -154,7 +141,7 @@ export function loadRuntimeTypeText(runtime, cwd) {
|
|
|
154
141
|
* the runtime's type package. A runtime whose types can't be loaded is skipped
|
|
155
142
|
* (we never flag what we can't verify), so this is silent when types are absent.
|
|
156
143
|
*/
|
|
157
|
-
export function findPhantomImports(text, cwd, loadText = loadRuntimeTypeText) {
|
|
144
|
+
export async function findPhantomImports(text, cwd, loadText = loadRuntimeTypeText) {
|
|
158
145
|
const out = [];
|
|
159
146
|
const typeTextByRuntime = new Map();
|
|
160
147
|
for (const spec of extractRuntimeSpecifiers(text)) {
|
|
@@ -162,7 +149,7 @@ export function findPhantomImports(text, cwd, loadText = loadRuntimeTypeText) {
|
|
|
162
149
|
if (!ns)
|
|
163
150
|
continue;
|
|
164
151
|
if (!typeTextByRuntime.has(ns.runtime)) {
|
|
165
|
-
typeTextByRuntime.set(ns.runtime, loadText(ns.runtime, cwd));
|
|
152
|
+
typeTextByRuntime.set(ns.runtime, await loadText(ns.runtime, cwd));
|
|
166
153
|
}
|
|
167
154
|
const typeText = typeTextByRuntime.get(ns.runtime);
|
|
168
155
|
if (!typeText)
|
|
@@ -219,7 +206,7 @@ const MENTION_TRAILING_PUNCT = /[.,;:!?)\]}>"']+$/;
|
|
|
219
206
|
* Unreadable mentions are skipped; silent when types are absent or nothing is flagged.
|
|
220
207
|
* Drives the impl-handoff override banner (Layer B).
|
|
221
208
|
*/
|
|
222
|
-
export function findDeliveryPhantoms(spec, cwd) {
|
|
209
|
+
export async function findDeliveryPhantoms(spec, cwd) {
|
|
223
210
|
let text = spec;
|
|
224
211
|
const seen = new Set();
|
|
225
212
|
for (const m of spec.matchAll(MENTION_RE)) {
|
|
@@ -5,6 +5,32 @@ import { resolvePackage as defaultResolvePackage } from './docs-resolve.js';
|
|
|
5
5
|
import { retrieveChunks as defaultRetrieveChunks } from './docs-retrieve.js';
|
|
6
6
|
import { npmVersionLookup as defaultNpmVersionLookup } from './npm-version.js';
|
|
7
7
|
import type { SpawnFn } from '../shared/child-process.js';
|
|
8
|
+
interface DocsDetails {
|
|
9
|
+
version?: string;
|
|
10
|
+
hitCache?: boolean;
|
|
11
|
+
chunksRetrieved?: number;
|
|
12
|
+
excerptVerified?: boolean;
|
|
13
|
+
childExitCode?: number;
|
|
14
|
+
indexingMs?: number;
|
|
15
|
+
indexedFiles?: number;
|
|
16
|
+
resolveError?: 'not_installed' | 'invalid_name';
|
|
17
|
+
cacheError?: string;
|
|
18
|
+
aborted?: boolean;
|
|
19
|
+
autoInstalled?: boolean;
|
|
20
|
+
installError?: string;
|
|
21
|
+
npmLatest?: string;
|
|
22
|
+
npmPublishedAt?: string;
|
|
23
|
+
versionSource?: 'declared-range' | 'npm-latest';
|
|
24
|
+
declaredRange?: string;
|
|
25
|
+
/**
|
|
26
|
+
* The answer restated a declaration for a question that needed usage semantics, so it
|
|
27
|
+
* is UNANSWERED (F-2). Set by isTypeOnlyAnswer; read by `cacheable` so a non-answer is
|
|
28
|
+
* never memoised and re-served to a later sibling task.
|
|
29
|
+
*/
|
|
30
|
+
typeOnly?: boolean;
|
|
31
|
+
/** The 5B CAP arm refused this call: the attempt's project-lookup budget is spent. */
|
|
32
|
+
budgetSpent?: boolean;
|
|
33
|
+
}
|
|
8
34
|
/**
|
|
9
35
|
* Pull `@see {@link https://…}` pointers out of retrieved .d.ts/README text.
|
|
10
36
|
*
|
|
@@ -30,3 +56,26 @@ export interface PiWorkerDocsInternals {
|
|
|
30
56
|
npmVersionLookup?: typeof defaultNpmVersionLookup;
|
|
31
57
|
}
|
|
32
58
|
export declare function registerPiWorkerDocs(pi: ExtensionAPI, internals?: PiWorkerDocsInternals): void;
|
|
59
|
+
/**
|
|
60
|
+
* The F-2(e) cache rule for the docs channel, as a NAMED export rather than an
|
|
61
|
+
* anonymous property of an adapter literal.
|
|
62
|
+
*
|
|
63
|
+
* It was reachable only through `registerTool → execute()`, so
|
|
64
|
+
* pi-worker-docs-typeonly.test.ts gave up and hand-retyped it under a
|
|
65
|
+
* "keep in sync" comment — six tests asserting against a copy that a change to the
|
|
66
|
+
* shipped rule would leave green. That is the same drift class the rule itself
|
|
67
|
+
* exists to prevent: four regexes matching three phrasings, documented at length in
|
|
68
|
+
* abstention.ts, which cost a real bug.
|
|
69
|
+
*/
|
|
70
|
+
export declare function docsCacheable(d: Pick<DocsDetails, 'childExitCode' | 'typeOnly' | 'excerptVerified'>, text: string): boolean;
|
|
71
|
+
/** The docs cache key: a package's answer is per (module, question). A project-source
|
|
72
|
+
* `.` lookup is never cached — the working tree mutates as tasks implement. */
|
|
73
|
+
export declare function docsCacheKey(params: {
|
|
74
|
+
module: string;
|
|
75
|
+
query: string;
|
|
76
|
+
}): string | null;
|
|
77
|
+
/** Package provenance for per-entry resume invalidation. */
|
|
78
|
+
export declare function docsCachePkg(params: {
|
|
79
|
+
module: string;
|
|
80
|
+
}): string | undefined;
|
|
81
|
+
export {};
|
|
@@ -3,7 +3,8 @@ import { Type } from '@sinclair/typebox';
|
|
|
3
3
|
import { Text } from '@earendil-works/pi-tui';
|
|
4
4
|
import { openCache as defaultOpenCache } from './docs-cache.js';
|
|
5
5
|
import { retrieveChunks as defaultRetrieveChunks } from './docs-retrieve.js';
|
|
6
|
-
import {
|
|
6
|
+
import { formatResultText } from '../shared/child-output.js';
|
|
7
|
+
import { docsRaw, packageHeader, buildPrompt, buildVersionBanner } from './docs-core.js';
|
|
7
8
|
import { formatNpmVersionSection } from './npm-version.js';
|
|
8
9
|
import { runFocusedExtraction } from './focused-extractor.js';
|
|
9
10
|
import { makeWorkerTool } from './shared.js';
|
|
@@ -374,15 +375,13 @@ export function registerPiWorkerDocs(pi, internals = {}) {
|
|
|
374
375
|
// version do not change within a run). A project-source `.` lookup is NOT cached:
|
|
375
376
|
// the working tree mutates as tasks implement, so its answer can go stale mid-run
|
|
376
377
|
// (the docs SQLite index already keys those on file mtime).
|
|
377
|
-
cacheKey:
|
|
378
|
-
null
|
|
379
|
-
: `${normalizeQuery(params.module)}::${normalizeQuery(params.query)}`,
|
|
378
|
+
cacheKey: docsCacheKey,
|
|
380
379
|
// Package provenance for per-entry resume invalidation: a docs digest describes
|
|
381
380
|
// one package at one declared version, so a resume drops it only when THAT
|
|
382
381
|
// package moves — an unrelated install no longer discards it. Package names are
|
|
383
382
|
// matched against package.json verbatim (npm names are case-sensitive), unlike
|
|
384
383
|
// the cache key, which normalises for phrasing collisions.
|
|
385
|
-
cachePkg:
|
|
384
|
+
cachePkg: docsCachePkg,
|
|
386
385
|
// Only a completed lookup (child exited 0) is a real answer; not-installed,
|
|
387
386
|
// no-chunks, resolve/cache errors, and aborts omit childExitCode:0 and fall
|
|
388
387
|
// through to a live retry next time.
|
|
@@ -396,9 +395,34 @@ export function registerPiWorkerDocs(pi, internals = {}) {
|
|
|
396
395
|
//
|
|
397
396
|
// `text` is supplied by makeWorkerTool (shared.ts) alongside details, so the
|
|
398
397
|
// content check needs no new plumbing.
|
|
399
|
-
cacheable:
|
|
400
|
-
&& d.typeOnly !== true
|
|
401
|
-
&& d.excerptVerified !== false
|
|
402
|
-
&& !isAbstention(text)
|
|
398
|
+
cacheable: docsCacheable
|
|
403
399
|
});
|
|
404
400
|
}
|
|
401
|
+
/**
|
|
402
|
+
* The F-2(e) cache rule for the docs channel, as a NAMED export rather than an
|
|
403
|
+
* anonymous property of an adapter literal.
|
|
404
|
+
*
|
|
405
|
+
* It was reachable only through `registerTool → execute()`, so
|
|
406
|
+
* pi-worker-docs-typeonly.test.ts gave up and hand-retyped it under a
|
|
407
|
+
* "keep in sync" comment — six tests asserting against a copy that a change to the
|
|
408
|
+
* shipped rule would leave green. That is the same drift class the rule itself
|
|
409
|
+
* exists to prevent: four regexes matching three phrasings, documented at length in
|
|
410
|
+
* abstention.ts, which cost a real bug.
|
|
411
|
+
*/
|
|
412
|
+
export function docsCacheable(d, text) {
|
|
413
|
+
return (d.childExitCode === 0
|
|
414
|
+
&& d.typeOnly !== true
|
|
415
|
+
&& d.excerptVerified !== false
|
|
416
|
+
&& !isAbstention(text));
|
|
417
|
+
}
|
|
418
|
+
/** The docs cache key: a package's answer is per (module, question). A project-source
|
|
419
|
+
* `.` lookup is never cached — the working tree mutates as tasks implement. */
|
|
420
|
+
export function docsCacheKey(params) {
|
|
421
|
+
return params.module === '.' ?
|
|
422
|
+
null
|
|
423
|
+
: `${normalizeQuery(params.module)}::${normalizeQuery(params.query)}`;
|
|
424
|
+
}
|
|
425
|
+
/** Package provenance for per-entry resume invalidation. */
|
|
426
|
+
export function docsCachePkg(params) {
|
|
427
|
+
return params.module === '.' ? undefined : packageRootOf(params.module);
|
|
428
|
+
}
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import type { EventEmitter } from 'node:events';
|
|
2
2
|
import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
|
|
3
3
|
import { fetchAndClean as defaultFetchAndClean } from './html-clean.js';
|
|
4
|
+
interface FetchDetails {
|
|
5
|
+
childExitCode?: number;
|
|
6
|
+
answer?: string;
|
|
7
|
+
excerpt?: string;
|
|
8
|
+
excerptVerified?: boolean;
|
|
9
|
+
}
|
|
4
10
|
interface ProcLike extends EventEmitter {
|
|
5
11
|
stdout: EventEmitter | null;
|
|
6
12
|
stderr: EventEmitter | null;
|
|
@@ -17,4 +23,16 @@ export interface PiWorkerFetchInternals {
|
|
|
17
23
|
spawn?: SpawnFn;
|
|
18
24
|
}
|
|
19
25
|
export declare function registerPiWorkerFetch(pi: ExtensionAPI, internals?: PiWorkerFetchInternals): void;
|
|
26
|
+
/**
|
|
27
|
+
* The F-2(e) cache rule for the fetch channel, named for the same reason as
|
|
28
|
+
* `docsCacheable`: pi-worker-fetch.test.ts carried a hand-retyped copy driving four
|
|
29
|
+
* tests, which a change to the shipped rule would leave green.
|
|
30
|
+
*/
|
|
31
|
+
export declare function fetchCacheable(d: Pick<FetchDetails, 'childExitCode'>, text: string): boolean;
|
|
32
|
+
/** The fetch cache key. URL verbatim (path case can matter), question normalised —
|
|
33
|
+
* same page, different question is a different answer. */
|
|
34
|
+
export declare function fetchCacheKey(params: {
|
|
35
|
+
url: string;
|
|
36
|
+
query: string;
|
|
37
|
+
}): string;
|
|
20
38
|
export {};
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { Type } from '@sinclair/typebox';
|
|
2
2
|
import { Text } from '@earendil-works/pi-tui';
|
|
3
3
|
import { FetchAndCleanError } from './html-clean.js';
|
|
4
|
-
import { fetchFocused
|
|
4
|
+
import { fetchFocused } from './fetch-core.js';
|
|
5
|
+
import { formatResultText } from '../shared/child-output.js';
|
|
5
6
|
import { makeWorkerTool } from './shared.js';
|
|
6
7
|
import { normalizeQuery } from './research-cache.js';
|
|
7
8
|
import { isAbstention } from './abstention.js';
|
|
@@ -51,7 +52,8 @@ export function registerPiWorkerFetch(pi, internals = {}) {
|
|
|
51
52
|
if (result.failure !== undefined) {
|
|
52
53
|
return { text: result.failure, details: { childExitCode: result.childExitCode } };
|
|
53
54
|
}
|
|
54
|
-
const body = formatResultText(
|
|
55
|
+
const body = formatResultText('', // a fetched page answer carries no package header
|
|
56
|
+
{ answer: result.answer, excerpt: result.excerpt }, result.excerptVerified) || '(no output)';
|
|
55
57
|
// The coverage miss is the one outcome that carries an instruction. It goes
|
|
56
58
|
// in the TEXT, not only in details: details are for the harness, and the
|
|
57
59
|
// worker acts on what it reads.
|
|
@@ -90,7 +92,7 @@ export function registerPiWorkerFetch(pi, internals = {}) {
|
|
|
90
92
|
// otherwise). The URL is kept verbatim (path case can matter); the query is
|
|
91
93
|
// normalised. Both parts key the entry — same page, different question is a
|
|
92
94
|
// different answer.
|
|
93
|
-
cacheKey:
|
|
95
|
+
cacheKey: fetchCacheKey,
|
|
94
96
|
// Only a completed fetch (child exited 0) is a real answer; invalid-URL,
|
|
95
97
|
// fetch failures, and aborts omit childExitCode:0 and fall through.
|
|
96
98
|
// F-2(e), on the fetch channel. A child that ran fine and answered
|
|
@@ -99,6 +101,19 @@ export function registerPiWorkerFetch(pi, internals = {}) {
|
|
|
99
101
|
// the same dead-end-paid-many-times shape pi-worker-docs already closed
|
|
100
102
|
// for packages, with escalation unable to re-fire because the miss never
|
|
101
103
|
// recurred. One predicate now covers every corpus (workers/abstention.ts).
|
|
102
|
-
cacheable:
|
|
104
|
+
cacheable: fetchCacheable
|
|
103
105
|
});
|
|
104
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* The F-2(e) cache rule for the fetch channel, named for the same reason as
|
|
109
|
+
* `docsCacheable`: pi-worker-fetch.test.ts carried a hand-retyped copy driving four
|
|
110
|
+
* tests, which a change to the shipped rule would leave green.
|
|
111
|
+
*/
|
|
112
|
+
export function fetchCacheable(d, text) {
|
|
113
|
+
return d.childExitCode === 0 && !isAbstention(text);
|
|
114
|
+
}
|
|
115
|
+
/** The fetch cache key. URL verbatim (path case can matter), question normalised —
|
|
116
|
+
* same page, different question is a different answer. */
|
|
117
|
+
export function fetchCacheKey(params) {
|
|
118
|
+
return `${params.url.trim()}::${normalizeQuery(params.query)}`;
|
|
119
|
+
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** The env var the orchestrator stamps with the per-run id children inherit. */
|
|
2
2
|
export declare const RESEARCH_RUN_ID_ENV = "PI_TASK_RUN_ID";
|
|
3
|
+
/** Is this a transient filesystem error worth retrying inside the caller's deadline? */
|
|
4
|
+
export declare function isRetryableFsError(err: unknown): boolean;
|
|
3
5
|
export declare function researchCacheFile(cwd: string): string;
|
|
4
6
|
/**
|
|
5
7
|
* The current run's id, or undefined when caching is off (the orchestrator did not
|
|
@@ -117,6 +117,12 @@ const LOCK_SUFFIX = '.lock';
|
|
|
117
117
|
const LOCK_TIMEOUT_MS = 2_000;
|
|
118
118
|
/** Poll interval while the lock is held by someone else. */
|
|
119
119
|
const LOCK_POLL_MS = 10;
|
|
120
|
+
/**
|
|
121
|
+
* How long the atomic replace retries a transient refusal. Short: the lock is HELD for
|
|
122
|
+
* every one of these milliseconds, so this trades a bounded stall for not losing a
|
|
123
|
+
* write the lock was taken to protect.
|
|
124
|
+
*/
|
|
125
|
+
const RENAME_TIMEOUT_MS = 500;
|
|
120
126
|
/**
|
|
121
127
|
* A lock older than this is treated as abandoned and removed. Two writers can both
|
|
122
128
|
* decide that and both proceed, which degrades exactly to the pre-lock behaviour (one
|
|
@@ -124,6 +130,32 @@ const LOCK_POLL_MS = 10;
|
|
|
124
130
|
* the run. Sized far above the critical section, so a live holder is never stolen from.
|
|
125
131
|
*/
|
|
126
132
|
const LOCK_STALE_MS = 30_000;
|
|
133
|
+
/**
|
|
134
|
+
* Filesystem errors that mean "try again in a moment", not "this will never work".
|
|
135
|
+
*
|
|
136
|
+
* POSIX gives `mkdir` two clean answers when someone else holds the lock: it succeeds,
|
|
137
|
+
* or it is EEXIST. Windows has a third. A directory that another process is removing
|
|
138
|
+
* enters a DELETE-PENDING state, and a create against it fails with EPERM/EACCES/EBUSY
|
|
139
|
+
* instead of EEXIST — so the exact moment the previous writer released the lock is a
|
|
140
|
+
* window in which the next writer's `mkdir` fails with a code that used to be read as
|
|
141
|
+
* fatal. The store was then silently skipped: no throw, no log, exit code 0, one entry
|
|
142
|
+
* missing. That is the whole of CI's `39 of 40` on windows-latest; the same run's
|
|
143
|
+
* ubuntu half is green because POSIX never produces the code.
|
|
144
|
+
*
|
|
145
|
+
* `rename` over an existing file has the same shape on Windows — it fails while any
|
|
146
|
+
* other handle is open on the target, including a scanner's — so the cache write
|
|
147
|
+
* retries on this set too.
|
|
148
|
+
*/
|
|
149
|
+
const RETRYABLE_FS_CODES = new Set(['EEXIST', 'EPERM', 'EACCES', 'EBUSY', 'ENOTEMPTY']);
|
|
150
|
+
/** Is this a transient filesystem error worth retrying inside the caller's deadline? */
|
|
151
|
+
export function isRetryableFsError(err) {
|
|
152
|
+
const code = err?.code;
|
|
153
|
+
return code !== undefined && RETRYABLE_FS_CODES.has(code);
|
|
154
|
+
}
|
|
155
|
+
/** Does this error mean the lock directory is genuinely held right now? */
|
|
156
|
+
function isHeldError(err) {
|
|
157
|
+
return err?.code === 'EEXIST';
|
|
158
|
+
}
|
|
127
159
|
/**
|
|
128
160
|
* Schema marker for per-entry package provenance. A file without it was written by a
|
|
129
161
|
* version that stored no `pkg` on its entries, so its docs entries are indistinguishable
|
|
@@ -307,15 +339,35 @@ export async function lookupResearch(cwd, runId, key) {
|
|
|
307
339
|
}
|
|
308
340
|
/** Write the cache file atomic-ish, so a concurrent reader never sees it half-written. */
|
|
309
341
|
async function writeCacheFile(cwd, out) {
|
|
342
|
+
const tmp = `${researchCacheFile(cwd)}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
310
343
|
try {
|
|
311
344
|
await fsp.mkdir(tasksDir(cwd), { recursive: true });
|
|
312
|
-
const tmp = `${researchCacheFile(cwd)}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
313
345
|
await fsp.writeFile(tmp, JSON.stringify(out), 'utf8');
|
|
314
|
-
|
|
346
|
+
// The replace, not the write, is the part Windows can transiently refuse — any
|
|
347
|
+
// other open handle on the target (a reader, a scanner) fails it with EPERM.
|
|
348
|
+
// The whole point of the lock above is that this write is not lost, so a
|
|
349
|
+
// transient refusal is retried inside the same bounded budget rather than
|
|
350
|
+
// swallowed. The lock is still held throughout.
|
|
351
|
+
const deadline = Date.now() + RENAME_TIMEOUT_MS;
|
|
352
|
+
for (;;) {
|
|
353
|
+
try {
|
|
354
|
+
await fsp.rename(tmp, researchCacheFile(cwd));
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
catch (err) {
|
|
358
|
+
if (!isRetryableFsError(err) || Date.now() >= deadline)
|
|
359
|
+
throw err;
|
|
360
|
+
await new Promise(resolve => setTimeout(resolve, LOCK_POLL_MS));
|
|
361
|
+
}
|
|
362
|
+
}
|
|
315
363
|
}
|
|
316
364
|
catch {
|
|
317
365
|
// best-effort cache
|
|
318
366
|
}
|
|
367
|
+
finally {
|
|
368
|
+
// Never leave a .tmp behind for a replace that never happened.
|
|
369
|
+
await fsp.rm(tmp, { force: true }).catch(() => { });
|
|
370
|
+
}
|
|
319
371
|
}
|
|
320
372
|
/**
|
|
321
373
|
* Serialises this process's own writers per cache file, so the 4-6 parallel tool calls
|
|
@@ -337,8 +389,18 @@ async function acquireLock(lockPath, deadline) {
|
|
|
337
389
|
return true;
|
|
338
390
|
}
|
|
339
391
|
catch (err) {
|
|
340
|
-
if (err
|
|
392
|
+
if (!isRetryableFsError(err))
|
|
341
393
|
return false;
|
|
394
|
+
// Not EEXIST but still retryable ⇒ Windows delete-pending (see
|
|
395
|
+
// RETRYABLE_FS_CODES). There is no lock to inspect for staleness: the
|
|
396
|
+
// directory is on its way out, so wait one poll and try to create it again
|
|
397
|
+
// rather than reporting a hold that nobody has.
|
|
398
|
+
if (!isHeldError(err)) {
|
|
399
|
+
if (Date.now() >= deadline)
|
|
400
|
+
return false;
|
|
401
|
+
await new Promise(resolve => setTimeout(resolve, LOCK_POLL_MS));
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
342
404
|
try {
|
|
343
405
|
const st = await fsp.stat(lockPath);
|
|
344
406
|
if (Date.now() - st.mtimeMs > LOCK_STALE_MS) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mjasnikovs/pi-task",
|
|
3
|
-
"version": "0.38.
|
|
3
|
+
"version": "0.38.11",
|
|
4
4
|
"description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|