@lincoln504/pi-research 1.6.5 → 1.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -9
- package/agent-skill/pi-research/README.md +5 -4
- package/agent-skill/pi-research/SKILL.md +1 -1
- package/dist/cli.mjs +616 -153
- package/dist/pdf-extract-worker.mjs +1289 -0
- package/dist/prompts/research-tool-usage.md +2 -1
- package/dist/prompts/researcher.md +1 -1
- package/dist/prompts/system-coordinator.md +2 -2
- package/dist/prompts/system-lead-router.md +1 -1
- package/docs/AGENT-SKILL.md +6 -6
- package/docs/ARCHITECTURE.md +24 -22
- package/docs/CHANGELOG.md +73 -0
- package/docs/CONFIGURATION.md +3 -2
- package/docs/SDK.md +7 -12
- package/package.json +8 -5
- package/scripts/build.cjs +35 -0
- package/scripts/setup.cjs +12 -10
- package/src/cleanup/research-cleanup.ts +11 -5
- package/src/cli.ts +7 -5
- package/src/config.ts +9 -3
- package/src/core/interfaces/knowledge-interfaces.ts +8 -0
- package/src/core/interfaces/orchestration-interfaces.ts +1 -1
- package/src/core/interfaces/research-plan-types.ts +6 -0
- package/src/core/llm/agentic-repair.ts +40 -3
- package/src/core/llm/llm-utils.ts +93 -5
- package/src/core/llm/model-registry-factory.ts +2 -15
- package/src/core/pi-version.ts +1 -1
- package/src/core/planning-service.ts +110 -15
- package/src/core/planning-utils.ts +3 -0
- package/src/core/service-registry.ts +8 -6
- package/src/index.ts +8 -5
- package/src/infrastructure/browser/config.ts +6 -7
- package/src/infrastructure/browser/ensure-browser.ts +14 -7
- package/src/infrastructure/browser/thread-worker-browser.ts +18 -19
- package/src/knowledge/chunker.ts +22 -4
- package/src/knowledge/embedder-init.ts +41 -1
- package/src/knowledge/embedder-utils.ts +7 -1
- package/src/knowledge/store.ts +68 -1
- package/src/knowledge/transformers-loader.ts +58 -1
- package/src/observers/research-observer-impl.ts +4 -1
- package/src/orchestration/deep-research-orchestrator.ts +3 -3
- package/src/orchestration/headless-observer.ts +33 -0
- package/src/orchestration/quick-research-orchestrator.ts +17 -3
- package/src/orchestration/research-orchestration-service.ts +27 -4
- package/src/orchestration/session-state.ts +36 -7
- package/src/prompts/research-tool-usage.md +2 -1
- package/src/prompts/researcher.md +1 -1
- package/src/prompts/system-coordinator.md +2 -2
- package/src/prompts/system-lead-router.md +1 -1
- package/src/sdk.ts +12 -4
- package/src/tools/research-knowledge-search.ts +23 -7
- package/src/tools/research-tool-definition.ts +1 -1
- package/src/tools/scrape.ts +12 -9
- package/src/tools/search.ts +12 -6
- package/src/tools/security.ts +6 -1
- package/src/tools/stackexchange.ts +6 -1
- package/src/utils/cancellation.ts +34 -1
- package/src/utils/llm-usage.ts +8 -0
- package/src/utils/log-utils.ts +1 -1
- package/src/web-research/browser-search.ts +6 -5
- package/src/web-research/pdf-extract-worker.mjs +1289 -0
- package/src/web-research/pdf-extract-worker.ts +42 -0
- package/src/web-research/pdf-extraction.ts +392 -0
- package/src/web-research/web-scraper.ts +3 -111
|
@@ -136,18 +136,18 @@ export async function initBrowser(): Promise<void> {
|
|
|
136
136
|
// (consumers install without our lockfile, so a floating range resolves
|
|
137
137
|
// to a newer, broken version):
|
|
138
138
|
//
|
|
139
|
-
// • camoufox-js ^0.
|
|
140
|
-
//
|
|
141
|
-
// 2026-07-16;
|
|
142
|
-
// 0.13.1/0.14.0
|
|
143
|
-
//
|
|
144
|
-
//
|
|
145
|
-
//
|
|
146
|
-
//
|
|
147
|
-
//
|
|
148
|
-
//
|
|
149
|
-
//
|
|
150
|
-
//
|
|
139
|
+
// • camoufox-js ^0.12.0 (2026-08-30 refresh) → the 0.10.x pin survived
|
|
140
|
+
// two refresh cycles on three since-lapsed blockers (Windows assets
|
|
141
|
+
// returned in v152.0.4-beta.26, 2026-07-16; impit's `only-allow pnpm`
|
|
142
|
+
// guard existed only in 0.13.1/0.14.0; and better-sqlite3 13's changed
|
|
143
|
+
// install model). The refresh accepted better-sqlite3 13. MEASURED on
|
|
144
|
+
// 13.0.3: `prebuilds/` for all eight platform/arch pairs ship INSIDE
|
|
145
|
+
// the tarball and load at runtime via node-gyp-build — no install
|
|
146
|
+
// script, no toolchain needed on modern npm (≥11.19 skips the
|
|
147
|
+
// injected `node-gyp rebuild` that a binding.gyp without an install
|
|
148
|
+
// script triggers; it was that needless rebuild, not a missing
|
|
149
|
+
// binary, that broke toolchain-less Windows installs). A future bump
|
|
150
|
+
// must still check better-sqlite3 first, not camoufox.
|
|
151
151
|
// • playwright-core 1.60.0 (exact) → playwright-core and the camoufox
|
|
152
152
|
// binary are one matched Juggler-protocol pair. 1.61 added a
|
|
153
153
|
// viewport.isMobile field the build REJECTS ("property
|
|
@@ -155,13 +155,12 @@ export async function initBrowser(): Promise<void> {
|
|
|
155
155
|
// launch below. Corroborated upstream: camoufox-js 0.12.0 declares
|
|
156
156
|
// `peerDependencies: playwright-core "<1.61.0"`, the same bound this
|
|
157
157
|
// pin has held by hand.
|
|
158
|
-
// • impit 0.
|
|
159
|
-
//
|
|
160
|
-
//
|
|
161
|
-
//
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
// don't propagate to installers of a published package.
|
|
158
|
+
// • impit 0.14.4 (exact, a direct dep though only camoufox-js uses it;
|
|
159
|
+
// 2026-08-30 refresh from 0.13.0) → kept EXACT because it is the only
|
|
160
|
+
// way to force a version for consumers: npm `overrides` don't
|
|
161
|
+
// propagate to installers of a published package, and impit's
|
|
162
|
+
// mid-stream `only-allow pnpm` guard incident (0.13.1/0.14.0) showed
|
|
163
|
+
// floating ranges breaking fresh global installs our lockfile masks.
|
|
165
164
|
//
|
|
166
165
|
// What is NOT pinned: the browser BINARY. `camoufox-js fetch` takes no
|
|
167
166
|
// version argument — it walks the GitHub releases newest-first and takes
|
package/src/knowledge/chunker.ts
CHANGED
|
@@ -40,6 +40,14 @@ export class Chunker {
|
|
|
40
40
|
let sentenceExtensions = 0;
|
|
41
41
|
let newlineExtensions = 0;
|
|
42
42
|
let spaceExtensions = 0;
|
|
43
|
+
// Running count of ``` fences in text[0..start) — fence parity for the
|
|
44
|
+
// code-block-atomicity check. Maintained incrementally at the bottom of
|
|
45
|
+
// the loop (each iteration scans only the span of text newly consumed);
|
|
46
|
+
// the previous text.slice(0, start) rescan per iteration made chunking
|
|
47
|
+
// O(n²): ~350MB of allocation+regex scanning for a 1MB document, on the
|
|
48
|
+
// ingest hot path, worst exactly on the largest pages.
|
|
49
|
+
let fencesBefore = 0;
|
|
50
|
+
let lastFenceScan = 0;
|
|
43
51
|
|
|
44
52
|
while (start < text.length) {
|
|
45
53
|
const iterationStart = start;
|
|
@@ -49,10 +57,9 @@ export class Chunker {
|
|
|
49
57
|
let slice = text.slice(start, end);
|
|
50
58
|
let extendedForCodeBlock = false;
|
|
51
59
|
|
|
52
|
-
// 0. Code blocks atomicity
|
|
53
|
-
|
|
54
|
-
const
|
|
55
|
-
const startsInCodeBlock = codeBlockMatchesBefore && codeBlockMatchesBefore.length % 2 !== 0;
|
|
60
|
+
// 0. Code blocks atomicity (fence parity from the running counter —
|
|
61
|
+
// see fencesBefore above).
|
|
62
|
+
const startsInCodeBlock = fencesBefore % 2 !== 0;
|
|
56
63
|
|
|
57
64
|
// Hard cap: never extend a chunk beyond 4x targetSize (min 2000 chars) for code blocks.
|
|
58
65
|
// A very large code block (e.g. a package compatibility table) can otherwise
|
|
@@ -232,6 +239,17 @@ export class Chunker {
|
|
|
232
239
|
}
|
|
233
240
|
}
|
|
234
241
|
}
|
|
242
|
+
|
|
243
|
+
// Advance the running fence counter over the span this iteration newly
|
|
244
|
+
// consumed. start only ever moves forward (start-side forward-progress
|
|
245
|
+
// guard), so scanning [lastFenceScan, start) once keeps fencesBefore
|
|
246
|
+
// exactly equal to the fence count in text[0..start) that the next
|
|
247
|
+
// iteration's parity check needs — amortized O(n) across the whole loop.
|
|
248
|
+
if (start > lastFenceScan) {
|
|
249
|
+
const newlyConsumed = text.slice(lastFenceScan, start).match(/```/g);
|
|
250
|
+
if (newlyConsumed) fencesBefore += newlyConsumed.length;
|
|
251
|
+
lastFenceScan = start;
|
|
252
|
+
}
|
|
235
253
|
}
|
|
236
254
|
|
|
237
255
|
const duration = Date.now() - startTime;
|
|
@@ -208,6 +208,40 @@ export async function isModelCached(model: string): Promise<boolean> {
|
|
|
208
208
|
|
|
209
209
|
const entries = await readdir(onnxDir).catch(() => [] as string[]);
|
|
210
210
|
|
|
211
|
+
// External-weights integrity: transformers.js exports most embedding models as a
|
|
212
|
+
// small graph stub (model.onnx) that references its real weights in a SIBLING
|
|
213
|
+
// model.onnx_data. An interrupted download can leave EXACTLY the stub without the
|
|
214
|
+
// weights — every check above passes (the stub is a valid protobuf, config and
|
|
215
|
+
// tokenizer are present), so this function reported "cached", the loader ran with
|
|
216
|
+
// allowRemoteModels=false, and transformers failed with "model.onnx_data ... not
|
|
217
|
+
// found locally" — an error isCorruptModelError did not classify, so no purge and
|
|
218
|
+
// no re-download: init retried into the same wall until the caller's budget died.
|
|
219
|
+
// Detect the reference in the graph bytes and require the weights file when it is
|
|
220
|
+
// present. The scan is bounded to 1MB: a self-contained (internal-weights) model.onnx
|
|
221
|
+
// is far larger than that, and the reference string lives in the small external-data
|
|
222
|
+
// layout's protobuf.
|
|
223
|
+
const onnxPath = path.default.join(onnxDir, 'model.onnx');
|
|
224
|
+
const onnxStat = await stat(onnxPath).catch(() => null);
|
|
225
|
+
if (onnxStat && onnxStat.size > 0 && onnxStat.size <= 1024 * 1024) {
|
|
226
|
+
try {
|
|
227
|
+
const handle = await import('node:fs/promises');
|
|
228
|
+
const fh = await handle.default.open(onnxPath, 'r');
|
|
229
|
+
try {
|
|
230
|
+
const { buffer, bytesRead } = await fh.read(Buffer.alloc(Math.min(onnxStat.size, 1024 * 1024)), 0, Math.min(onnxStat.size, 1024 * 1024), 0);
|
|
231
|
+
const referencesExternalWeights = buffer.subarray(0, bytesRead).includes('model.onnx_data');
|
|
232
|
+
if (referencesExternalWeights && !entries.includes('model.onnx_data')) {
|
|
233
|
+
return false;
|
|
234
|
+
}
|
|
235
|
+
} finally {
|
|
236
|
+
await fh.close();
|
|
237
|
+
}
|
|
238
|
+
} catch {
|
|
239
|
+
// A read failure here is NOT evidence of a poisoned cache — treat the
|
|
240
|
+
// verdict as inconclusive and fall through to the historical checks
|
|
241
|
+
// (the loader-side corrupt classification remains the backstop).
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
211
245
|
// If an external weights file is listed, it must be non-empty — an interrupted
|
|
212
246
|
// download can leave a zero-length model.onnx_data that crashes the ONNX loader.
|
|
213
247
|
const hasExternalWeights = entries.includes('model.onnx_data');
|
|
@@ -259,7 +293,13 @@ export function isCorruptModelError(err: unknown): boolean {
|
|
|
259
293
|
msg.includes('deserialize') ||
|
|
260
294
|
msg.includes('file_length') ||
|
|
261
295
|
msg.includes('protobuf parsing failed') ||
|
|
262
|
-
msg.includes('invalid protobuf')
|
|
296
|
+
msg.includes('invalid protobuf') ||
|
|
297
|
+
// transformers.js local-file-missing: the cache LOOKED complete to isModelCached
|
|
298
|
+
// but a referenced file (typically external weights, model.onnx_data) is absent —
|
|
299
|
+
// an interrupted download. Classifying it here routes to purge + one remote
|
|
300
|
+
// re-download instead of init retrying into the same wall.
|
|
301
|
+
msg.includes('not found locally') ||
|
|
302
|
+
msg.includes('local_files_only=true')
|
|
263
303
|
);
|
|
264
304
|
}
|
|
265
305
|
|
|
@@ -73,6 +73,9 @@ export function isEmbedderUnreachable(err: unknown): boolean {
|
|
|
73
73
|
* Matched signatures (observed in the wild):
|
|
74
74
|
* - onnxruntime-node: "Cannot find module '.../bin/napi-v6/darwin/x64/onnxruntime_binding.node'"
|
|
75
75
|
* - @lancedb/lancedb: "Cannot find native binding. npm has a bug related to optional dependencies"
|
|
76
|
+
* - @huggingface/transformers (optional dep skipped at install time, #10): the typed
|
|
77
|
+
* TRANSFORMERS_UNAVAILABLE error from transformers-loader.ts, or its
|
|
78
|
+
* raw form "Cannot find package '@huggingface/transformers'"
|
|
76
79
|
* - jiti-masked module-eval failure of the above surfaces as:
|
|
77
80
|
* "KnowledgeStoreService is not a constructor"
|
|
78
81
|
*/
|
|
@@ -89,8 +92,10 @@ export function isNativeStackUnavailableError(err: unknown): boolean {
|
|
|
89
92
|
// `err` is already known truthy (guarded above), so no `err &&` here.
|
|
90
93
|
const code = (typeof err === 'object' && 'code' in err) ? String((err as { code?: unknown }).code) : '';
|
|
91
94
|
if (code === 'ERR_DLOPEN_FAILED') return true;
|
|
95
|
+
// The typed error from transformers-loader.ts — unambiguous by construction.
|
|
96
|
+
if (code === 'TRANSFORMERS_UNAVAILABLE') return true;
|
|
92
97
|
if ((code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND') &&
|
|
93
|
-
(msg.includes('onnxruntime') || msg.includes('lancedb'))) {
|
|
98
|
+
(msg.includes('onnxruntime') || msg.includes('lancedb') || msg.includes('@huggingface/transformers'))) {
|
|
94
99
|
return true;
|
|
95
100
|
}
|
|
96
101
|
|
|
@@ -99,6 +104,7 @@ export function isNativeStackUnavailableError(err: unknown): boolean {
|
|
|
99
104
|
msg.includes('onnxruntime_binding.node') ||
|
|
100
105
|
/cannot find module '[^']*onnxruntime/.test(msg) ||
|
|
101
106
|
/cannot find module '[^']*lancedb/.test(msg) ||
|
|
107
|
+
/cannot find (?:package|module) ['"]@huggingface[/\\]transformers['"]/i.test(msg) ||
|
|
102
108
|
msg.includes('knowledgestoreservice is not a constructor')
|
|
103
109
|
);
|
|
104
110
|
}
|
package/src/knowledge/store.ts
CHANGED
|
@@ -87,6 +87,14 @@ export class KnowledgeStore implements IKnowledgeStore {
|
|
|
87
87
|
private manifestPath: string;
|
|
88
88
|
private isClosing = false;
|
|
89
89
|
private activeWrites = new Set<Promise<any>>();
|
|
90
|
+
// Last recorded model/schema migration (persisted in the store manifest).
|
|
91
|
+
// Compared against the incoming migration target to detect alternating-model
|
|
92
|
+
// thrash against a shared dbDir (see the alternation warn in openInternal).
|
|
93
|
+
private lastMigration: { model: string; ts: number } | null = null;
|
|
94
|
+
// The migration BEFORE lastMigration — detects a RETURN to an earlier model
|
|
95
|
+
// (A→B→A, the true ping-pong signature) while leaving a deliberate onward
|
|
96
|
+
// change (A→B→C) quiet. Persisted alongside lastMigration.
|
|
97
|
+
private prevMigration: { model: string; ts: number } | null = null;
|
|
90
98
|
// LanceDB table version at the last successful FTS rebuild. rebuildFtsIndex() skips
|
|
91
99
|
// when the version is unchanged: a research run that committed no new transactions
|
|
92
100
|
// would otherwise create a fresh FTS index (and a new table version) on every cleanup,
|
|
@@ -141,6 +149,12 @@ export class KnowledgeStore implements IKnowledgeStore {
|
|
|
141
149
|
this.tableName = manifest.activeTableName;
|
|
142
150
|
logger.debug(`[store] Loaded active table name from manifest: ${this.tableName}`);
|
|
143
151
|
}
|
|
152
|
+
if (manifest.lastMigration && typeof manifest.lastMigration.model === 'string') {
|
|
153
|
+
this.lastMigration = { model: manifest.lastMigration.model, ts: Number(manifest.lastMigration.ts) || 0 };
|
|
154
|
+
}
|
|
155
|
+
if (manifest.prevMigration && typeof manifest.prevMigration.model === 'string') {
|
|
156
|
+
this.prevMigration = { model: manifest.prevMigration.model, ts: Number(manifest.prevMigration.ts) || 0 };
|
|
157
|
+
}
|
|
144
158
|
}
|
|
145
159
|
} catch (err) {
|
|
146
160
|
logger.warn('[store] Failed to load manifest, using default table name:', err);
|
|
@@ -151,7 +165,16 @@ export class KnowledgeStore implements IKnowledgeStore {
|
|
|
151
165
|
private async saveManifest(): Promise<boolean> {
|
|
152
166
|
const tempPath = `${this.manifestPath}.tmp`;
|
|
153
167
|
try {
|
|
154
|
-
const content = JSON.stringify({
|
|
168
|
+
const content = JSON.stringify({
|
|
169
|
+
activeTableName: this.tableName,
|
|
170
|
+
// Record of the two most recent model/schema migrations. Read back on open
|
|
171
|
+
// to detect ALTERNATION (two configurations sharing this dbDir each
|
|
172
|
+
// migrating the table back and forth) — previously that thrash was
|
|
173
|
+
// invisible outside a per-open WARN line, while every alternation
|
|
174
|
+
// served an empty store.
|
|
175
|
+
lastMigration: this.lastMigration,
|
|
176
|
+
prevMigration: this.prevMigration,
|
|
177
|
+
}, null, 2);
|
|
155
178
|
await fsPromises.writeFile(tempPath, content, { encoding: 'utf-8', mode: 0o600 });
|
|
156
179
|
await fsPromises.rename(tempPath, this.manifestPath);
|
|
157
180
|
return true;
|
|
@@ -166,6 +189,14 @@ export class KnowledgeStore implements IKnowledgeStore {
|
|
|
166
189
|
|
|
167
190
|
async open(): Promise<void> {
|
|
168
191
|
if (this.db) return;
|
|
192
|
+
// close() is irreversible in this class (isClosing latches the operation
|
|
193
|
+
// no-op path), so an open() after it must not "succeed" and hand back a
|
|
194
|
+
// zombie whose every operation silently no-ops — the service layer always
|
|
195
|
+
// constructs FRESH components after dispose, and this makes any deviation
|
|
196
|
+
// fail loud at the call site instead of as mysterious empty results.
|
|
197
|
+
if (this.isClosing) {
|
|
198
|
+
throw new Error('KnowledgeStore.open() called after close() — construct a fresh store instead.');
|
|
199
|
+
}
|
|
169
200
|
|
|
170
201
|
// Coalesce concurrent opens in-process. withLock (when provided) is a
|
|
171
202
|
// cross-process FILE lock, not an in-process mutex — and when it's absent there
|
|
@@ -286,6 +317,42 @@ export class KnowledgeStore implements IKnowledgeStore {
|
|
|
286
317
|
throw new Error(errorMsg, { cause: err });
|
|
287
318
|
}
|
|
288
319
|
}
|
|
320
|
+
|
|
321
|
+
// Alternation detector: the PREVIOUS migration moved this shared dbDir
|
|
322
|
+
// to a model that is NOT the one being migrated to now — the A→B→A
|
|
323
|
+
// thrash signature of two configurations (EMBEDDING_MODEL is
|
|
324
|
+
// machine-scoped; one shell with an override is enough) fighting over
|
|
325
|
+
// one table. Each alternation serves an EMPTY store and leans on the
|
|
326
|
+
// newest backup for the other model's data; make that LOUD and
|
|
327
|
+
// countable instead of a per-open WARN nobody correlates.
|
|
328
|
+
const prev = this.lastMigration;
|
|
329
|
+
const twoAgo = this.prevMigration;
|
|
330
|
+
// Thrash = RETURN to the model from two migrations ago (A→B→A):
|
|
331
|
+
// the ping-pong of two configurations sharing this dbDir. A first
|
|
332
|
+
// migration (no history) and a deliberate onward change (A→B→C)
|
|
333
|
+
// stay quiet; with two configs fighting, every leg after the first
|
|
334
|
+
// round-trip IS a bounce-back, so the real case still fires on
|
|
335
|
+
// every subsequent leg. (The prev.model check is kept even though a
|
|
336
|
+
// real migration to a different target implies it — defense in
|
|
337
|
+
// depth against a hand-edited manifest.)
|
|
338
|
+
const isThrash =
|
|
339
|
+
prev !== null && prev.model !== this.options.modelName &&
|
|
340
|
+
twoAgo !== null && twoAgo.model === this.options.modelName;
|
|
341
|
+
if (isThrash) {
|
|
342
|
+
const ageHours = Math.max(0, Math.round((Date.now() - twoAgo!.ts) / 3_600_000));
|
|
343
|
+
logger.warn(
|
|
344
|
+
`[store] MODEL ALTERNATION on the shared table: '${this.options.modelName}' was already here ~${ageHours}h ago ` +
|
|
345
|
+
`(migrations: '${twoAgo!.model}' → '${prev!.model}' → back to '${this.options.modelName}'). ` +
|
|
346
|
+
`Two configurations sharing this dbDir are thrashing it — each open serves an empty store until re-ingest. ` +
|
|
347
|
+
`Fix the EMBEDDING_MODEL drift, or give each model its own dbDir.`,
|
|
348
|
+
);
|
|
349
|
+
metrics.increment('knowledge_store_model_migration_total', 1, { alternation: 'true' });
|
|
350
|
+
} else {
|
|
351
|
+
metrics.increment('knowledge_store_model_migration_total', 1, { alternation: 'false' });
|
|
352
|
+
}
|
|
353
|
+
this.prevMigration = prev;
|
|
354
|
+
this.lastMigration = { model: this.options.modelName, ts: Date.now() };
|
|
355
|
+
// Persisted by the saveManifest() later in openInternal.
|
|
289
356
|
}
|
|
290
357
|
} catch (schemaErr) {
|
|
291
358
|
// A migration failure is fatal — re-throw so the caller learns the store
|
|
@@ -18,14 +18,71 @@
|
|
|
18
18
|
* (isNativeStackUnavailableError) and degrades the store to DISABLED. Help, version,
|
|
19
19
|
* status, and browser-only research then work with no binding present.
|
|
20
20
|
*
|
|
21
|
+
* The package itself is an OPTIONAL dependency (it can be skipped entirely at
|
|
22
|
+
* install time when its native chain cannot build — sharp's platform binary is
|
|
23
|
+
* the historic offender, #10). A genuinely absent module is therefore a SUPPORTED
|
|
24
|
+
* state, not a crash: getTransformers() reclassifies it as a typed
|
|
25
|
+
* TRANSFORMERS_UNAVAILABLE error carrying remediation, which the knowledge store's
|
|
26
|
+
* native-unavailable classifier memoizes as DISABLED instead of re-running the
|
|
27
|
+
* init + backoff storm on every store touch. A transformers-INTERNAL load failure
|
|
28
|
+
* (its own onnxruntime import, a corrupt install) is re-thrown untouched — that is
|
|
29
|
+
* the pre-existing native-stack classification's job.
|
|
30
|
+
*
|
|
21
31
|
* Modules that need transformers *types* should keep `import type { … } from
|
|
22
32
|
* '@huggingface/transformers'` (type imports are erased and never hoisted); only
|
|
23
33
|
* runtime value access (pipeline, env, …) must go through getTransformers().
|
|
24
34
|
*/
|
|
25
35
|
|
|
36
|
+
/** Error.code on the typed error thrown when the optional module is not installed. */
|
|
37
|
+
export const TRANSFORMERS_UNAVAILABLE = 'TRANSFORMERS_UNAVAILABLE';
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* True when `err` is Node's "the module itself is not installed" rejection for
|
|
41
|
+
* OUR specifier — i.e. the optional dependency was skipped at install time.
|
|
42
|
+
* Deliberately narrow: a transformers-INTERNAL missing module (onnxruntime
|
|
43
|
+
* binding, corrupt install) resolves to a DIFFERENT file path and must stay
|
|
44
|
+
* unclassified here so the native-stack classifier handles it instead.
|
|
45
|
+
*/
|
|
46
|
+
export function isTransformersMissingError(err: unknown): boolean {
|
|
47
|
+
if (!err || typeof err !== 'object') return false;
|
|
48
|
+
const code = 'code' in err ? String((err as { code?: unknown }).code) : '';
|
|
49
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
50
|
+
// Node's ESM form is "Cannot find package '@huggingface/transformers' imported
|
|
51
|
+
// from …"; the CJS/child-process form is "Cannot find module '…'".
|
|
52
|
+
return (
|
|
53
|
+
code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND'
|
|
54
|
+
) && /@huggingface[/\\]transformers/.test(msg);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Build the typed, remediation-carrying error for an absent optional package.
|
|
59
|
+
* Exported for tests; the code is what isNativeStackUnavailableError() keys on.
|
|
60
|
+
*/
|
|
61
|
+
export function transformersUnavailableError(cause: unknown): Error {
|
|
62
|
+
const err = new Error(
|
|
63
|
+
"Optional dependency '@huggingface/transformers' is not installed, so local embeddings are unavailable. "
|
|
64
|
+
+ 'npm skips this package when its native image chain cannot install (most often a failing sharp platform binary). '
|
|
65
|
+
+ 'The knowledge store runs without embeddings until it is present — reinstall this package to re-resolve its '
|
|
66
|
+
+ 'optional dependencies (pi install npm:@lincoln504/pi-research), or install it directly '
|
|
67
|
+
+ '(npm install @huggingface/transformers).',
|
|
68
|
+
{ cause },
|
|
69
|
+
);
|
|
70
|
+
(err as { code?: string }).code = TRANSFORMERS_UNAVAILABLE;
|
|
71
|
+
return err;
|
|
72
|
+
}
|
|
73
|
+
|
|
26
74
|
let _transformers: typeof import('@huggingface/transformers') | undefined;
|
|
27
75
|
|
|
28
76
|
/** Resolve the transformers module, memoizing after the first load. */
|
|
29
77
|
export async function getTransformers(): Promise<typeof import('@huggingface/transformers')> {
|
|
30
|
-
|
|
78
|
+
if (_transformers) return _transformers;
|
|
79
|
+
try {
|
|
80
|
+
_transformers = await import('@huggingface/transformers');
|
|
81
|
+
} catch (err) {
|
|
82
|
+
if (isTransformersMissingError(err)) {
|
|
83
|
+
throw transformersUnavailableError(err);
|
|
84
|
+
}
|
|
85
|
+
throw err;
|
|
86
|
+
}
|
|
87
|
+
return _transformers;
|
|
31
88
|
}
|
|
@@ -158,7 +158,10 @@ export function createResearchObserver(
|
|
|
158
158
|
},
|
|
159
159
|
|
|
160
160
|
onPlanningSuccess: (plan) => {
|
|
161
|
-
|
|
161
|
+
// A fallback plan means the model's planning output was unusable and the engine
|
|
162
|
+
// degraded to a single researcher — the user asked for deep research and got a
|
|
163
|
+
// shallow one. Say so in the status line instead of presenting it as 'ready'.
|
|
164
|
+
updateSliceStatus(panelState, 'coord', plan.fallback ? 'fallback — planning failed' : 'ready', debouncedRefresh);
|
|
162
165
|
completeSlice(panelState, 'coord');
|
|
163
166
|
const unitsPerResearcher = getUnitsPerResearcher(undefined, ctx.cachingActive);
|
|
164
167
|
const count = plan.researchers?.length || 0;
|
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
import type {
|
|
18
18
|
ResearchObserver,
|
|
19
19
|
} from '../core/interfaces/observer-interfaces.ts';
|
|
20
|
-
import { HeadlessObserver, makeSafeObserver, type HeadlessObserverOptions } from './headless-observer.ts';
|
|
20
|
+
import { HeadlessObserver, makeSafeObserver, isHeadlessObserverBag, type HeadlessObserverOptions } from './headless-observer.ts';
|
|
21
21
|
import {
|
|
22
22
|
ServiceNames,
|
|
23
23
|
type IResearchOrchestration,
|
|
@@ -63,7 +63,7 @@ export class DeepResearchOrchestrator {
|
|
|
63
63
|
|
|
64
64
|
// Resolve observer: if options were provided instead of an instance, create the instance.
|
|
65
65
|
// Either way, wrap it so a throwing observer callback can never fail the run.
|
|
66
|
-
if (options.observer &&
|
|
66
|
+
if (options.observer && isHeadlessObserverBag(options.observer)) {
|
|
67
67
|
this.observer = makeSafeObserver(new HeadlessObserver(options.observer as HeadlessObserverOptions));
|
|
68
68
|
} else {
|
|
69
69
|
this.observer = options.observer ? makeSafeObserver(options.observer as ResearchObserver) : undefined;
|
|
@@ -237,7 +237,7 @@ export class DeepResearchOrchestrator {
|
|
|
237
237
|
// Once per round, not per loop entry: a 'wait' retry re-enters the same
|
|
238
238
|
// round and must not re-probe the same infrastructure it just probed.
|
|
239
239
|
if (firstEntryThisRound) {
|
|
240
|
-
await orchestrationService.checkHealth(this.currentRound, researchId, ctx);
|
|
240
|
+
await orchestrationService.checkHealth(this.currentRound, researchId, ctx, signal);
|
|
241
241
|
}
|
|
242
242
|
|
|
243
243
|
// 1. Update/Generate Plan
|
|
@@ -75,6 +75,39 @@ export function makeSafeObserver<T extends object>(observer: T): T {
|
|
|
75
75
|
}) as T;
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
+
/**
|
|
79
|
+
* True when `observer` is a HeadlessObserverOptions BAG — a bare
|
|
80
|
+
* `{ onProgress }` callback object — rather than a full ResearchObserver.
|
|
81
|
+
*
|
|
82
|
+
* Detection is by ABSENCE of a typed lifecycle method, not merely presence of
|
|
83
|
+
* onProgress: a full observer that ALSO exposes onProgress (a subclass, an
|
|
84
|
+
* EventEmitter-style adapter, a proxy) would otherwise be silently demoted to
|
|
85
|
+
* a bag, funnelling every event through onProgress and losing every typed
|
|
86
|
+
* callback. Any typed method (onStart/onRoundStart/onResearcherStart/…)
|
|
87
|
+
* marks the object a full observer; HeadlessObserver itself is exempt (it
|
|
88
|
+
* emits through onProgress but is already the wrapped form).
|
|
89
|
+
*/
|
|
90
|
+
export function isHeadlessObserverBag(observer: unknown): boolean {
|
|
91
|
+
if (!observer || typeof observer !== 'object') return false;
|
|
92
|
+
if (observer instanceof HeadlessObserver) return false;
|
|
93
|
+
const o = observer as Record<string, unknown>;
|
|
94
|
+
if (typeof o['onProgress'] !== 'function') return false;
|
|
95
|
+
// The full optional surface of ResearchObserver (observer-interfaces.ts).
|
|
96
|
+
// Anything typed marks the object a full observer; only a bare { onProgress }
|
|
97
|
+
// bag falls through to demotion.
|
|
98
|
+
const lifecycleMethods = [
|
|
99
|
+
'onStart', 'onRunQueued',
|
|
100
|
+
'onPlanningStart', 'onPlanningProgress', 'onPlanningTokens', 'onPlanningSuccess',
|
|
101
|
+
'onRoundStart',
|
|
102
|
+
'onSearchStart', 'onSearchProgress', 'onSearchComplete',
|
|
103
|
+
'onResearcherStart', 'onResearcherProgress', 'onResearcherTokensHint', 'onResearcherComplete', 'onResearcherFailure',
|
|
104
|
+
'onToolResult',
|
|
105
|
+
'onEvaluationStart', 'onEvaluationProgress', 'onEvaluationTokens', 'onEvaluationDecision',
|
|
106
|
+
'onSynthesisStart', 'onComplete', 'onError', 'onTokensConsumed',
|
|
107
|
+
];
|
|
108
|
+
return !lifecycleMethods.some(m => typeof o[m] === 'function');
|
|
109
|
+
}
|
|
110
|
+
|
|
78
111
|
export class HeadlessObserver implements ResearchObserver {
|
|
79
112
|
constructor(private options: HeadlessObserverOptions = {}) {}
|
|
80
113
|
|
|
@@ -21,8 +21,9 @@ import { ensureAssistantResponse, parseCitations, isAnalysisFreeSynthesis, synth
|
|
|
21
21
|
import { getMaxScrapeBatches, resolveExcludedTools, QUICK_THIN_REPORT_NOTICE } from '../constants.ts';
|
|
22
22
|
import { resolveResearchModel, isPromptCachingActiveForModel } from '../core/llm/research-model-resolver.ts';
|
|
23
23
|
import type { ResearchObserver } from '../core/interfaces/observer-interfaces.ts';
|
|
24
|
-
import { HeadlessObserver, makeSafeObserver, type HeadlessObserverOptions } from './headless-observer.ts';
|
|
24
|
+
import { HeadlessObserver, makeSafeObserver, isHeadlessObserverBag, type HeadlessObserverOptions } from './headless-observer.ts';
|
|
25
25
|
import { getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
|
|
26
|
+
import { raceWithSignal } from '../utils/cancellation.ts';
|
|
26
27
|
import { ServiceNames, type IKnowledgeStoreService, type IResearchSynthesisService, type IResearchOrchestration } from '../core/service-interfaces.ts';
|
|
27
28
|
import type { ResearchSessionService } from './research-session-service.ts';
|
|
28
29
|
import { normalizeUrl, registerScrapedLinks, getCachedScrapedContent } from '../utils/shared-links.ts';
|
|
@@ -54,7 +55,7 @@ export class QuickResearchOrchestrator {
|
|
|
54
55
|
|
|
55
56
|
// Resolve observer: if options were provided instead of an instance, create the instance.
|
|
56
57
|
// Either way, wrap it so a throwing observer callback can never fail the run.
|
|
57
|
-
if (options.observer &&
|
|
58
|
+
if (options.observer && isHeadlessObserverBag(options.observer)) {
|
|
58
59
|
this.observer = makeSafeObserver(new HeadlessObserver(options.observer as HeadlessObserverOptions));
|
|
59
60
|
} else {
|
|
60
61
|
this.observer = options.observer ? makeSafeObserver(options.observer as ResearchObserver) : undefined;
|
|
@@ -86,7 +87,15 @@ export class QuickResearchOrchestrator {
|
|
|
86
87
|
process.env['PI_RESEARCH_SKIP_HEALTHCHECK'] === 'true';
|
|
87
88
|
const health = skipHealthcheck
|
|
88
89
|
? { success: true as boolean, error: undefined as string | undefined, components: [] as Array<{ component?: string; healthy?: boolean; error?: string }> }
|
|
89
|
-
:
|
|
90
|
+
// Abort-aware: without the race, a cancel landing inside the probe
|
|
91
|
+
// waits out the FULL healthcheck budget before the run notices.
|
|
92
|
+
// The abandoned probe drains in the background (its own timeouts
|
|
93
|
+
// bound it); the throw lands in the outer catch, which classifies
|
|
94
|
+
// via signal.aborted as a cancellation, not a fault.
|
|
95
|
+
: await raceWithSignal(runHealthCheck({ ctx }), signal);
|
|
96
|
+
if (health === undefined) {
|
|
97
|
+
throw new Error('Research aborted');
|
|
98
|
+
}
|
|
90
99
|
if (skipHealthcheck) {
|
|
91
100
|
logger.debug('[QuickOrchestrator] Pre-flight healthcheck skipped (PI_RESEARCH_SKIP_HEALTHCHECK); relying on per-task timeouts.');
|
|
92
101
|
}
|
|
@@ -512,6 +521,11 @@ export class QuickResearchOrchestrator {
|
|
|
512
521
|
// exactly once (the inner catch sets the flag for its own scope).
|
|
513
522
|
if (!terminalCallbackFired) {
|
|
514
523
|
const aborted = signal?.aborted === true;
|
|
524
|
+
// Audit L4: the early-throw path (healthcheck/session-registration) fired
|
|
525
|
+
// the terminal callbacks but recorded no run metric — the session counter
|
|
526
|
+
// under-counted exactly these failures. Mirrors the inner catch's labels.
|
|
527
|
+
metrics.observe('research_session_duration_ms', Date.now() - sessionStart, { mode: 'quick', complexity: '0', status: aborted ? 'cancelled' : 'error' });
|
|
528
|
+
metrics.increment('research_sessions_total', 1, { mode: 'quick', complexity: '0', status: aborted ? 'cancelled' : 'error' });
|
|
515
529
|
if (!aborted) {
|
|
516
530
|
observer?.onResearcherFailure?.('quick', error instanceof Error ? error.message : String(error));
|
|
517
531
|
}
|
|
@@ -38,6 +38,7 @@ import { getCachedScrapedContent, normalizeUrl, cleanupSharedLinks } from '../ut
|
|
|
38
38
|
import { runResearcher } from './researcher-executor.ts';
|
|
39
39
|
import { ResearchRunSemaphore, ResearchRunCapacityError } from '../infrastructure/research-run-semaphore.ts';
|
|
40
40
|
import { recordResearcherFailure, shouldStopResearch, createResearchStopError } from './session-state.ts';
|
|
41
|
+
import { raceWithSignal } from '../utils/cancellation.ts';
|
|
41
42
|
import { isAbortSentinel, boundSessionAbort } from './abort-utils.ts';
|
|
42
43
|
import type { ResearchSessionService } from './research-session-service.ts';
|
|
43
44
|
import { QuickResearchOrchestrator } from './quick-research-orchestrator.ts';
|
|
@@ -163,10 +164,18 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
163
164
|
(observer as { onError?: (e: Error) => void } | undefined)?.onError?.(err as Error);
|
|
164
165
|
} catch { /* observer isolation */ }
|
|
165
166
|
}
|
|
166
|
-
if (err instanceof ResearchRunCapacityError)
|
|
167
|
+
if (err instanceof ResearchRunCapacityError) {
|
|
168
|
+
// Audit L1: a capacity-refused run was invisible to run-level metrics —
|
|
169
|
+
// the rethrow happened before the success/error accounting ran.
|
|
170
|
+
metrics.increment('research_manager_requests_total', 1, { depth: String(depth), status: 'capacity_refused', source: 'extension' });
|
|
171
|
+
throw err; // capacity exhausted → fail fast
|
|
172
|
+
}
|
|
167
173
|
// A cancel while queueing for a slot must stay cancelled. Fail-open here would
|
|
168
174
|
// start the very run the user just aborted.
|
|
169
|
-
if (err instanceof Error && err.name === 'AbortError')
|
|
175
|
+
if (err instanceof Error && err.name === 'AbortError') {
|
|
176
|
+
metrics.increment('research_manager_requests_total', 1, { depth: String(depth), status: 'cancelled', source: 'extension' });
|
|
177
|
+
throw err;
|
|
178
|
+
}
|
|
170
179
|
logger.warn(`[ResearchOrchestrationService] Run-cap unavailable, proceeding without it: ${err instanceof Error ? err.message : String(err)}`);
|
|
171
180
|
}
|
|
172
181
|
|
|
@@ -682,7 +691,7 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
682
691
|
* @param round - Current round number
|
|
683
692
|
* @param ctx - Optional extension context for container isolation
|
|
684
693
|
*/
|
|
685
|
-
async checkHealth(round: number, _researchId?: string, ctx?: any): Promise<void> {
|
|
694
|
+
async checkHealth(round: number, _researchId?: string, ctx?: any, signal?: AbortSignal): Promise<void> {
|
|
686
695
|
// Round 1 is skipped because there is nothing to diagnose yet — the pool was
|
|
687
696
|
// built moments earlier and no work has run through it. A consequence worth
|
|
688
697
|
// stating: a complexity-1 run has exactly ONE research round, so it now never
|
|
@@ -692,6 +701,10 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
692
701
|
// does not use. This check is advisory logging; a quick run loses nothing but
|
|
693
702
|
// that cost.
|
|
694
703
|
if (round <= 1) return;
|
|
704
|
+
if (signal?.aborted) {
|
|
705
|
+
logger.debug(`[ResearchOrchestrationService] Health check at Round ${round} skipped (run already aborted).`);
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
695
708
|
|
|
696
709
|
try {
|
|
697
710
|
const container = tryGetServiceContainerFromCtx(ctx);
|
|
@@ -702,7 +715,17 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
|
|
|
702
715
|
registry = healthRegistry;
|
|
703
716
|
}
|
|
704
717
|
|
|
705
|
-
|
|
718
|
+
// Abort-aware: a user cancel landing inside a probe previously waited out
|
|
719
|
+
// the FULL probe budget (105s observed) — the one await on this path that
|
|
720
|
+
// ignored the run signal. The abandoned probe keeps draining in the
|
|
721
|
+
// background (probes carry their own timeouts and are safe to leave
|
|
722
|
+
// running); its dropped result costs nothing because the next abort-aware
|
|
723
|
+
// await in the run loop raises the cancel anyway.
|
|
724
|
+
const health = await raceWithSignal(registry.runAll(), signal);
|
|
725
|
+
if (health === undefined) {
|
|
726
|
+
logger.debug(`[ResearchOrchestrationService] Health check at Round ${round} abandoned (run signal fired).`);
|
|
727
|
+
return;
|
|
728
|
+
}
|
|
706
729
|
|
|
707
730
|
if (health.status === 'healthy') {
|
|
708
731
|
logger.debug(`[ResearchOrchestrationService] Health status at Round ${round}: [OK] All systems operational`);
|