@lincoln504/pi-research 1.1.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/cli.mjs +2599 -1268
- package/dist/thread-worker.mjs +224 -147
- package/docs/AGENT-SKILL.md +11 -4
- package/docs/ARCHITECTURE.md +20 -10
- package/docs/CHANGELOG.md +195 -17
- package/docs/CONFIGURATION.md +9 -7
- package/docs/KNOWLEDGE-STORE.md +4 -3
- package/docs/PI-EXTENSION.md +1 -1
- package/docs/SDK.md +55 -0
- package/docs/THIRD-PARTY-NOTICES.md +1 -1
- package/package.json +5 -4
- package/scripts/cleanup.cjs +114 -12
- package/scripts/prepare.cjs +15 -5
- package/scripts/setup.cjs +24 -4
- package/skills/pi-research/README.md +9 -2
- package/skills/pi-research/SKILL.md +6 -3
- package/skills/pi-research/references/configuration.md +6 -2
- package/skills/pi-research/scripts/run.mjs +24 -2
- package/src/cleanup/research-cleanup.ts +12 -4
- package/src/cli.ts +372 -86
- package/src/config.ts +134 -17
- package/src/constants.ts +31 -0
- package/src/core/interfaces/health-check-interfaces.ts +3 -39
- package/src/core/interfaces/knowledge-interfaces.ts +6 -1
- package/src/core/interfaces/planning-interfaces.ts +8 -0
- package/src/core/interfaces/service-names.ts +0 -1
- package/src/core/interfaces/state-manager-interfaces.ts +11 -3
- package/src/core/llm/agentic-repair.ts +14 -0
- package/src/core/pi-version.ts +136 -0
- package/src/core/planning-service.ts +43 -1
- package/src/core/service-initialization.ts +6 -14
- package/src/core/service-interfaces.ts +1 -1
- package/src/core/service-registry.ts +117 -8
- package/src/healthcheck/index.ts +7 -2
- package/src/index.ts +11 -18
- package/src/infrastructure/browser/browser-client.ts +80 -13
- package/src/infrastructure/browser/browser-error-utils.ts +6 -1
- package/src/infrastructure/browser/browser-server.ts +41 -12
- package/src/infrastructure/browser/browser-task-scheduler.ts +159 -35
- package/src/infrastructure/browser/cleanup-utils.ts +95 -9
- package/src/infrastructure/browser/client-agent.ts +6 -5
- package/src/infrastructure/browser/config.ts +14 -3
- package/src/infrastructure/browser/priority-task-queue.ts +10 -3
- package/src/infrastructure/browser/scheduler-factory.ts +44 -4
- package/src/infrastructure/browser/task-execution-service.ts +92 -28
- package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
- package/src/infrastructure/browser/thread-worker-messaging.ts +54 -2
- package/src/infrastructure/browser/thread-worker-types.ts +3 -1
- package/src/infrastructure/browser/thread-worker.mjs +224 -147
- package/src/infrastructure/browser/worker-pool-manager.ts +18 -5
- package/src/infrastructure/embedding/embedding-client.ts +22 -8
- package/src/infrastructure/embedding/embedding-factory.ts +61 -15
- package/src/infrastructure/embedding/embedding-server.ts +105 -11
- package/src/infrastructure/file-lock-service.ts +295 -28
- package/src/infrastructure/knowledge-store-service.ts +48 -3
- package/src/infrastructure/process-lifecycle-service.ts +134 -34
- package/src/infrastructure/research-run-semaphore.ts +37 -3
- package/src/infrastructure/state/state-backup-manager.ts +5 -7
- package/src/infrastructure/state/state-browser-api.ts +5 -2
- package/src/infrastructure/state/state-browser-manager.ts +15 -1
- package/src/infrastructure/state/state-manager-service.ts +5 -5
- package/src/infrastructure/state/state-manager.ts +34 -23
- package/src/infrastructure/types/state-types.ts +9 -0
- package/src/knowledge/chunker.ts +88 -1
- package/src/knowledge/embedder-init.ts +1 -1
- package/src/knowledge/embedder.ts +188 -46
- package/src/knowledge/index.ts +1 -1
- package/src/knowledge/store.ts +101 -21
- package/src/knowledge/webgpu-viability.ts +9 -4
- package/src/knowledge/writer-queue.ts +88 -46
- package/src/logger.ts +5 -1
- package/src/observers/research-observer-impl.ts +31 -0
- package/src/orchestration/abort-utils.ts +60 -0
- package/src/orchestration/deep-research-orchestrator.ts +25 -4
- package/src/orchestration/headless-observer.ts +8 -0
- package/src/orchestration/quick-research-orchestrator.ts +2 -2
- package/src/orchestration/research-orchestration-service.ts +65 -15
- package/src/orchestration/research-session-service.ts +37 -11
- package/src/orchestration/research-synthesis-service.ts +33 -11
- package/src/orchestration/researcher-executor.ts +100 -20
- package/src/orchestration/researcher.ts +39 -20
- package/src/orchestration/session-state.ts +11 -0
- package/src/sdk.ts +125 -19
- package/src/security/cisa-kev.ts +10 -3
- package/src/security/github-advisories.ts +18 -7
- package/src/security/index.ts +51 -30
- package/src/security/nvd-client.ts +24 -8
- package/src/security/osv-client.ts +14 -4
- package/src/skill-install/skill-installer.ts +153 -11
- package/src/stackexchange/index.ts +20 -5
- package/src/stackexchange/output/table.ts +7 -2
- package/src/stackexchange/rest-client.ts +41 -3
- package/src/tools/research-knowledge-search.ts +15 -0
- package/src/tools/research-tool-definition.ts +8 -2
- package/src/tools/scrape.ts +48 -25
- package/src/tools/search.ts +16 -2
- package/src/tools/security.ts +68 -7
- package/src/tui/research-health.ts +18 -2
- package/src/utils/atomic-replace.ts +101 -0
- package/src/utils/circuit-breaker.ts +9 -2
- package/src/utils/citation-utils.ts +3 -1
- package/src/utils/disk-space-checker.ts +11 -6
- package/src/utils/http-body.ts +176 -0
- package/src/utils/json-utils.ts +116 -76
- package/src/utils/llm-usage.ts +17 -1
- package/src/utils/log-rotation.ts +42 -10
- package/src/utils/log-utils.ts +40 -0
- package/src/utils/metrics-summary.ts +27 -9
- package/src/utils/stdio-capture.ts +63 -22
- package/src/utils/text-utils.ts +8 -1
- package/src/utils/tool-usage-tracker.ts +9 -2
- package/src/web-research/browser-search.ts +70 -5
- package/src/web-research/scraper-utils.ts +13 -0
- package/src/web-research/search.ts +25 -6
- package/src/web-research/types.ts +19 -4
- package/src/web-research/web-scraper.ts +72 -54
- package/src/youtube/transcript-client.ts +5 -1
- package/src/core/health-check-service.ts +0 -265
|
@@ -9,7 +9,7 @@ export type { SearchResult, ScrapeResult, IScheduler, SchedulerMetadata, ISchedu
|
|
|
9
9
|
export type { ISchedulerFactory } from './scheduler-factory.ts';
|
|
10
10
|
|
|
11
11
|
// Health check interfaces
|
|
12
|
-
export type {
|
|
12
|
+
export type { IHealthRegistryService } from './interfaces/health-check-interfaces.ts';
|
|
13
13
|
|
|
14
14
|
// Knowledge store interfaces
|
|
15
15
|
export type {
|
|
@@ -17,6 +17,13 @@ import { logger } from '../logger.ts';
|
|
|
17
17
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
18
18
|
|
|
19
19
|
|
|
20
|
+
/**
|
|
21
|
+
* How long teardown waits for an initialization that is still in flight. Long
|
|
22
|
+
* enough to cover a normal service coming up, short enough that a service blocked
|
|
23
|
+
* on something slow (a model download) cannot hold shutdown open indefinitely.
|
|
24
|
+
*/
|
|
25
|
+
const DISPOSE_INFLIGHT_INIT_TIMEOUT_MS = 5_000;
|
|
26
|
+
|
|
20
27
|
/**
|
|
21
28
|
* Service lifecycle stages
|
|
22
29
|
*/
|
|
@@ -209,15 +216,52 @@ export class ServiceContainer {
|
|
|
209
216
|
throw new Error(`Service '${name}' is not registered`);
|
|
210
217
|
}
|
|
211
218
|
|
|
212
|
-
// Return existing instance if already initialized
|
|
219
|
+
// Return existing instance if already initialized.
|
|
213
220
|
if (registration.instance) {
|
|
221
|
+
// `_initializeService` publishes `registration.instance` BEFORE awaiting the
|
|
222
|
+
// service's own initialize(), deliberately: a service whose initialize()
|
|
223
|
+
// resolves a peer that resolves it back would otherwise deadlock, and the
|
|
224
|
+
// partial instance is what breaks that cycle. But the same early publish
|
|
225
|
+
// hands a HALF-BUILT service to unrelated concurrent callers — e.g.
|
|
226
|
+
// StateManagerService only assigns its inner manager after nine awaited
|
|
227
|
+
// getService() calls, so every method on it throws "not initialized" until
|
|
228
|
+
// then, and the embedding factory resolves it and immediately calls
|
|
229
|
+
// updateState(). Worse, the ctx branch below would re-run initialize() on an
|
|
230
|
+
// instance that is still running it, constructing a second StateManager and
|
|
231
|
+
// discarding the first.
|
|
232
|
+
//
|
|
233
|
+
// So: if it is still initializing, wait for that to finish. The cycle case is
|
|
234
|
+
// exactly the case where `caller` is set (we are inside some service's
|
|
235
|
+
// initialize()), and there we must NOT wait — that is the deadlock the early
|
|
236
|
+
// publish exists to avoid. Outside an initialization there is no cycle to
|
|
237
|
+
// break and waiting is always correct.
|
|
238
|
+
if (registration.initializationPromise && caller === undefined) {
|
|
239
|
+
await registration.initializationPromise.catch(() => { /* surfaced below */ });
|
|
240
|
+
if (!registration.instance) {
|
|
241
|
+
// Initialization failed and cleaned up after itself — fall through and
|
|
242
|
+
// retry from scratch rather than returning a torn-down instance.
|
|
243
|
+
return this.get<T>(name, ctx);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
214
246
|
// Re-initialize if ctx is provided and service supports it
|
|
215
247
|
if (ctx && registration.instance.initialize) {
|
|
216
248
|
// Wrap in initializationContext so nested get() calls are tracked
|
|
217
|
-
|
|
249
|
+
const reinit = initializationContext.run(name, async () => {
|
|
218
250
|
await registration.instance!.initialize!(ctx);
|
|
219
251
|
return registration.instance as T;
|
|
220
252
|
});
|
|
253
|
+
// Recorded like a first init: a re-init is just as in-flight. Without
|
|
254
|
+
// this, disposeAll's settle-wait cannot see it (it would dispose the
|
|
255
|
+
// instance under a running initialize()), and concurrent callers
|
|
256
|
+
// received the instance mid-re-initialization.
|
|
257
|
+
registration.initializationPromise = reinit as Promise<IService>;
|
|
258
|
+
const clearIfCurrent = () => {
|
|
259
|
+
if (registration.initializationPromise === (reinit as Promise<IService>)) {
|
|
260
|
+
registration.initializationPromise = null;
|
|
261
|
+
}
|
|
262
|
+
};
|
|
263
|
+
reinit.then(clearIfCurrent, clearIfCurrent);
|
|
264
|
+
return reinit;
|
|
221
265
|
}
|
|
222
266
|
return registration.instance as T;
|
|
223
267
|
}
|
|
@@ -243,7 +287,21 @@ export class ServiceContainer {
|
|
|
243
287
|
}
|
|
244
288
|
|
|
245
289
|
/**
|
|
246
|
-
* Get a service instance synchronously (returns null if not
|
|
290
|
+
* Get a service instance synchronously (returns null if not registered or not
|
|
291
|
+
* yet constructed).
|
|
292
|
+
*
|
|
293
|
+
* Deliberately returns instances in NON-settled lifecycles too. This is a
|
|
294
|
+
* lifecycle *inspector* as much as an accessor: the health check resolves the
|
|
295
|
+
* knowledge store through it precisely so it can report `initializing` /
|
|
296
|
+
* `disabled` / `disposed (not running)` WITHOUT forcing initialization (forcing
|
|
297
|
+
* it was a past cause of 45 s CI stalls). Filtering those states out here would
|
|
298
|
+
* make every one of them indistinguishable from "absent".
|
|
299
|
+
*
|
|
300
|
+
* The trade-off: because `_initializeService` publishes the instance before
|
|
301
|
+
* awaiting its `initialize()` (to break dependency cycles), a caller that
|
|
302
|
+
* immediately *uses* the result can catch a half-built service. Callers that
|
|
303
|
+
* need a ready service must use the async `get()`, which waits — see the note
|
|
304
|
+
* there. Callers that only inspect `lifecycle` are safe by construction.
|
|
247
305
|
*/
|
|
248
306
|
tryGet<T extends IService>(name: string): T | null {
|
|
249
307
|
if (this.isDisposing) {
|
|
@@ -286,6 +344,15 @@ export class ServiceContainer {
|
|
|
286
344
|
throw new Error(`Service '${name}' is not registered`);
|
|
287
345
|
}
|
|
288
346
|
|
|
347
|
+
// Settle an in-flight initialization first. Disposing the early-published
|
|
348
|
+
// instance while its initialize() is still running hands the awaiting get()
|
|
349
|
+
// caller a service that was disposed under it — observed as store-closed
|
|
350
|
+
// errors when /research-config cleared the knowledge store during a
|
|
351
|
+
// concurrent first search.
|
|
352
|
+
if (registration.initializationPromise) {
|
|
353
|
+
await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
|
|
354
|
+
}
|
|
355
|
+
|
|
289
356
|
if (registration.instance && registration.instance.dispose) {
|
|
290
357
|
await registration.instance.dispose().catch((err: unknown) => {
|
|
291
358
|
logger.warn(`[ServiceContainer] Error disposing service '${name}':`, err);
|
|
@@ -309,6 +376,11 @@ export class ServiceContainer {
|
|
|
309
376
|
throw new Error(`Service '${name}' is not registered`);
|
|
310
377
|
}
|
|
311
378
|
|
|
379
|
+
// Settle an in-flight initialization before swapping — see clear().
|
|
380
|
+
if (registration.initializationPromise) {
|
|
381
|
+
await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
|
|
382
|
+
}
|
|
383
|
+
|
|
312
384
|
// Dispose old instance if present
|
|
313
385
|
if (registration.instance && registration.instance.dispose) {
|
|
314
386
|
await registration.instance.dispose().catch((err: unknown) => {
|
|
@@ -347,7 +419,41 @@ export class ServiceContainer {
|
|
|
347
419
|
}
|
|
348
420
|
|
|
349
421
|
try {
|
|
350
|
-
|
|
422
|
+
// Settle any initialization still in flight BEFORE snapshotting what to tear
|
|
423
|
+
// down. `activeServices` is "instances that are non-null right now", so a
|
|
424
|
+
// get() whose factory has not resolved yet is invisible to it — and nothing
|
|
425
|
+
// else awaits it. Such a service installs itself into the registry AFTER
|
|
426
|
+
// teardown reports completion and is never disposed: for BrowserTaskScheduler
|
|
427
|
+
// that is a listening HTTP server plus a cluster pool and Camoufox children
|
|
428
|
+
// outliving shutdown; for KnowledgeStoreService an ONNX session and LanceDB
|
|
429
|
+
// handles. (SchedulerService.dispose hand-rolls this same guard for itself,
|
|
430
|
+
// which does not help when the leaked service IS SchedulerService.)
|
|
431
|
+
//
|
|
432
|
+
// Bounded: a service blocked on something slow (a model download) must not
|
|
433
|
+
// hold teardown open indefinitely. On timeout we proceed — the snapshot below
|
|
434
|
+
// still catches anything that landed in the meantime.
|
|
435
|
+
const inFlight = Array.from(this.services.values())
|
|
436
|
+
.map((r) => r.initializationPromise)
|
|
437
|
+
.filter((p): p is Promise<IService> => p != null);
|
|
438
|
+
if (inFlight.length > 0) {
|
|
439
|
+
if (this.defaultOptions.enableLogging) {
|
|
440
|
+
logger.debug(`[ServiceContainer] Waiting for ${inFlight.length} in-flight initialization(s) before teardown`);
|
|
441
|
+
}
|
|
442
|
+
let settleTimer: ReturnType<typeof setTimeout> | undefined;
|
|
443
|
+
await Promise.race([
|
|
444
|
+
Promise.allSettled(inFlight),
|
|
445
|
+
new Promise<void>((resolve) => {
|
|
446
|
+
settleTimer = setTimeout(() => {
|
|
447
|
+
logger.warn('[ServiceContainer] Timed out waiting for in-flight service initialization; proceeding with teardown');
|
|
448
|
+
resolve();
|
|
449
|
+
}, DISPOSE_INFLIGHT_INIT_TIMEOUT_MS);
|
|
450
|
+
settleTimer.unref?.();
|
|
451
|
+
}),
|
|
452
|
+
]);
|
|
453
|
+
if (settleTimer) clearTimeout(settleTimer);
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
const activeServices = Array.from(this.services.keys()).filter(name =>
|
|
351
457
|
this.services.get(name)?.instance !== null
|
|
352
458
|
);
|
|
353
459
|
|
|
@@ -443,14 +549,17 @@ export class ServiceContainer {
|
|
|
443
549
|
return null;
|
|
444
550
|
}
|
|
445
551
|
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
552
|
+
// Promise first: the instance is published BEFORE its initialize() completes
|
|
553
|
+
// (the early publish that lets dependency cycles resolve), so instance-first
|
|
554
|
+
// ordering reported a mid-initialization service as INITIALIZED.
|
|
450
555
|
if (registration.initializationPromise) {
|
|
451
556
|
return ServiceLifecycle.INITIALIZING;
|
|
452
557
|
}
|
|
453
558
|
|
|
559
|
+
if (registration.instance) {
|
|
560
|
+
return ServiceLifecycle.INITIALIZED;
|
|
561
|
+
}
|
|
562
|
+
|
|
454
563
|
return ServiceLifecycle.UNINITIALIZED;
|
|
455
564
|
}
|
|
456
565
|
|
package/src/healthcheck/index.ts
CHANGED
|
@@ -80,7 +80,12 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
|
|
|
80
80
|
// respected (same pattern as the KnowledgeStore check's Math.max floor).
|
|
81
81
|
}, { timeoutMs: Math.max(healthTimeoutMs, 150000), critical: true });
|
|
82
82
|
|
|
83
|
-
// Register Knowledge Store Check
|
|
83
|
+
// Register Knowledge Store Check — NON-critical: the store is the optional cache
|
|
84
|
+
// research must run without (see the darwin-x64 comment below). A failure here
|
|
85
|
+
// (e.g. a dead embedding server, ECONNREFUSED) must degrade health, not flip the
|
|
86
|
+
// aggregate to unhealthy: as a critical check it aborted runs, and the raw store
|
|
87
|
+
// error then fed the TUI's browser-oriented formatter, which misattributed it as
|
|
88
|
+
// "the browser hit a network error / check your internet connection".
|
|
84
89
|
registry.register('KnowledgeStore', async (options) => {
|
|
85
90
|
// Idle fast-path — mirrors the BrowserRuntime check above. Read the ALREADY
|
|
86
91
|
// resolved instance via tryGet(), which does NOT initialize the service;
|
|
@@ -222,7 +227,7 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
|
|
|
222
227
|
}
|
|
223
228
|
return { healthy: false, error: `Knowledge store healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
|
|
224
229
|
}
|
|
225
|
-
}, { timeoutMs: Math.max(healthTimeoutMs, 45000) });
|
|
230
|
+
}, { timeoutMs: Math.max(healthTimeoutMs, 45000), critical: false });
|
|
226
231
|
|
|
227
232
|
// Register State Manager Check
|
|
228
233
|
registry.register('StateManager', async () => {
|
package/src/index.ts
CHANGED
|
@@ -11,6 +11,7 @@ import type { ResearchResultDetails } from './types/index.ts';
|
|
|
11
11
|
import { createResearchTool, createHealthTool } from './tool.ts';
|
|
12
12
|
import { createResearchKnowledgeSearchTool } from './tools/research-knowledge-search.ts';
|
|
13
13
|
import { logger } from './logger.ts';
|
|
14
|
+
import { checkPiCompatibility } from './core/pi-version.ts';
|
|
14
15
|
import { randomUUID } from 'node:crypto';
|
|
15
16
|
import { shutdownManager } from './utils/shutdown-manager.ts';
|
|
16
17
|
import { healthRegistry } from './healthcheck/index.ts';
|
|
@@ -98,25 +99,17 @@ export default async function (pi: ExtensionAPI) {
|
|
|
98
99
|
// 0.80.8+ host auto-building its ModelRuntime. This in-host check is the real
|
|
99
100
|
// enforcement point: when running as an extension the host's version is NOT
|
|
100
101
|
// constrained by npm's resolution of our own package.json range.
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
);
|
|
102
|
+
// Two thresholds, not one — see src/core/pi-version.ts. Below the floor the APIs
|
|
103
|
+
// we call do not exist, so we refuse. Above the last TESTED line we warn and
|
|
104
|
+
// continue: pi is pre-1.0, a minor bump can break anything under semver (0.83.0
|
|
105
|
+
// already extended the ResourceLoader contract), and our own dependency range
|
|
106
|
+
// cannot constrain the host at all when running as an extension.
|
|
107
|
+
const compat = checkPiCompatibility(PI_VERSION);
|
|
108
|
+
if (compat.fatal) {
|
|
109
|
+
throw new Error(compat.message ?? `[pi-research] Unsupported pi-coding-agent version "${PI_VERSION}".`);
|
|
110
110
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|| (major === minMajor && minor < minMinor)
|
|
114
|
-
|| (major === minMajor && minor === minMinor && patch < minPatch);
|
|
115
|
-
if (tooOld) {
|
|
116
|
-
throw new Error(
|
|
117
|
-
`[pi-research] pi-coding-agent v${PI_VERSION} is too old. ` +
|
|
118
|
-
`Requires v${minMajor}.${minMinor}.${minPatch}+. Please update pi-coding-agent.`,
|
|
119
|
-
);
|
|
111
|
+
if (compat.message) {
|
|
112
|
+
logger.warn(compat.message);
|
|
120
113
|
}
|
|
121
114
|
|
|
122
115
|
// Re-register the beforeExit safety net (deactivate() strips event listeners during reload).
|
|
@@ -7,13 +7,69 @@
|
|
|
7
7
|
|
|
8
8
|
import * as http from 'node:http';
|
|
9
9
|
import type { Config } from '../../config.ts';
|
|
10
|
+
import { getConfig } from '../../config.ts';
|
|
10
11
|
import type { SearchResult } from '../../web-research/types.ts';
|
|
11
12
|
import type { IScheduler } from '../../core/interfaces/scheduler-interfaces.ts';
|
|
12
13
|
import { logger } from '../../logger.ts';
|
|
13
14
|
import { errorTracker } from '../../utils/error-tracker.ts';
|
|
15
|
+
import { Utf8Body } from '../../utils/http-body.ts';
|
|
14
16
|
import { getClientAgent } from './client-agent.ts';
|
|
15
17
|
import type { NodeError } from '../../types/index.ts';
|
|
16
18
|
|
|
19
|
+
/**
|
|
20
|
+
* Client-side patience margin on top of the leader's own task budget. The leader
|
|
21
|
+
* already bounds each task (nav budget + queue-wait margin) and answers with the
|
|
22
|
+
* more informative in-band timeout; the follower only needs headroom for the HTTP
|
|
23
|
+
* hop and leader-side dispatch, so its own timer never answers first.
|
|
24
|
+
*/
|
|
25
|
+
const CLIENT_PATIENCE_MARGIN_MS = 15_000;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Absolute ceiling on any follower request, so a wedged-but-listening leader
|
|
29
|
+
* still fails in bounded time. Sits above the largest configurable leader budget
|
|
30
|
+
* (120s nav + 120s task margin) plus the patience margin, so it only binds when
|
|
31
|
+
* config is out of range. A DEAD leader is not this timer's job — its socket
|
|
32
|
+
* fails fast with ECONNREFUSED/ECONNRESET.
|
|
33
|
+
*/
|
|
34
|
+
const CLIENT_TIMEOUT_CAP_MS = 300_000;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Mirror of the leader's fixed healthcheck budget in
|
|
38
|
+
* BrowserTaskScheduler.runHealthCheck (45s probe deadline + 60s queue-wait
|
|
39
|
+
* margin) — not config-derived there either.
|
|
40
|
+
*/
|
|
41
|
+
const HEALTHCHECK_LEADER_BUDGET_MS = 45_000 + 60_000;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Per-operation follower timeout: the leader's own budget for that task kind
|
|
45
|
+
* plus a patience margin, bounded by an absolute cap.
|
|
46
|
+
*
|
|
47
|
+
* Invariant: the follower is at least as patient as the leader. The old flat 60s
|
|
48
|
+
* undercut the leader's healthcheck budget (105s) and config-driven search/scrape
|
|
49
|
+
* budgets (up to 130s+ within schema bounds), and the client-timeout text matches
|
|
50
|
+
* isTaskTimeoutError, so retry gates skipped it — long tasks failed client-side,
|
|
51
|
+
* unretried, while the leader was still working on them.
|
|
52
|
+
*/
|
|
53
|
+
export function resolveClientRequestTimeoutMs(operation: string, config?: Config): number {
|
|
54
|
+
const cfg = config ?? getConfig();
|
|
55
|
+
let leaderBudgetMs: number;
|
|
56
|
+
switch (operation) {
|
|
57
|
+
case 'search':
|
|
58
|
+
// BrowserTaskScheduler.runSearch: SEARCH_TIMEOUT_MS + BROWSER_TASK_TIMEOUT_MS.
|
|
59
|
+
leaderBudgetMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS;
|
|
60
|
+
break;
|
|
61
|
+
case 'browser-task':
|
|
62
|
+
// BrowserTaskScheduler.runScrape: SCRAPE_TIMEOUT_MS + BROWSER_TASK_TIMEOUT_MS.
|
|
63
|
+
leaderBudgetMs = cfg.SCRAPE_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS;
|
|
64
|
+
break;
|
|
65
|
+
default:
|
|
66
|
+
// healthcheck, plus any future/unknown path: the largest fixed budget.
|
|
67
|
+
leaderBudgetMs = HEALTHCHECK_LEADER_BUDGET_MS;
|
|
68
|
+
break;
|
|
69
|
+
}
|
|
70
|
+
return Math.min(CLIENT_TIMEOUT_CAP_MS, leaderBudgetMs + CLIENT_PATIENCE_MARGIN_MS);
|
|
71
|
+
}
|
|
72
|
+
|
|
17
73
|
/**
|
|
18
74
|
* HTTP client that communicates with a remote scheduler.
|
|
19
75
|
* Used when this process is not the leader.
|
|
@@ -21,12 +77,19 @@ import type { NodeError } from '../../types/index.ts';
|
|
|
21
77
|
export class BrowserClient implements IScheduler {
|
|
22
78
|
private readonly authSecret: string;
|
|
23
79
|
|
|
80
|
+
/** The leader port this client was built for — lets the herd-guard cooldown
|
|
81
|
+
* verify the cached handle still targets the REGISTERED leader, not a
|
|
82
|
+
* predecessor that died inside the cooldown window. */
|
|
83
|
+
get targetPort(): number {
|
|
84
|
+
return this.port;
|
|
85
|
+
}
|
|
86
|
+
|
|
24
87
|
constructor(private readonly port: number, authSecret?: string) {
|
|
25
88
|
this.authSecret = authSecret ?? process.env['PI_BROWSER_AUTH_SECRET'] ?? '';
|
|
26
89
|
logger.log(`[BrowserClient] Connecting to global scheduler at http://127.0.0.1:${port}`);
|
|
27
90
|
}
|
|
28
91
|
|
|
29
|
-
private async request<T>(path: string, data: any, signal?: AbortSignal): Promise<T> {
|
|
92
|
+
private async request<T>(path: string, data: any, signal?: AbortSignal, config?: Config): Promise<T> {
|
|
30
93
|
const start = Date.now();
|
|
31
94
|
// Extract operation from path for error tracking
|
|
32
95
|
const operation = path.includes('/search') ? 'search' :
|
|
@@ -35,8 +98,8 @@ export class BrowserClient implements IScheduler {
|
|
|
35
98
|
|
|
36
99
|
return new Promise((resolve, reject) => {
|
|
37
100
|
const agent = getClientAgent();
|
|
38
|
-
//
|
|
39
|
-
const timeoutMs =
|
|
101
|
+
// Config-derived per task kind: leader budget + margin, capped.
|
|
102
|
+
const timeoutMs = resolveClientRequestTimeoutMs(operation, config);
|
|
40
103
|
let resolved = false;
|
|
41
104
|
const controller = new AbortController();
|
|
42
105
|
|
|
@@ -93,8 +156,11 @@ export class BrowserClient implements IScheduler {
|
|
|
93
156
|
// when headers arrive, but the body is still streaming — clearing
|
|
94
157
|
// now would let a stalled body hang forever and make an external
|
|
95
158
|
// abort a no-op. Keep both armed until 'end' or 'error'.
|
|
96
|
-
|
|
97
|
-
|
|
159
|
+
// This response carries scraped page content, so per-chunk decoding
|
|
160
|
+
// would corrupt every non-ASCII page that exceeds one chunk. Collect
|
|
161
|
+
// bytes and decode once. See Utf8Body.
|
|
162
|
+
const collected = new Utf8Body();
|
|
163
|
+
res.on('data', (chunk: Buffer) => collected.push(chunk));
|
|
98
164
|
res.on('end', () => {
|
|
99
165
|
if (resolved) return;
|
|
100
166
|
resolved = true;
|
|
@@ -102,7 +168,7 @@ export class BrowserClient implements IScheduler {
|
|
|
102
168
|
abortCleanup?.();
|
|
103
169
|
const duration = Date.now() - start;
|
|
104
170
|
try {
|
|
105
|
-
const parsed = JSON.parse(
|
|
171
|
+
const parsed = JSON.parse(collected.toString());
|
|
106
172
|
if (res.statusCode === 503 && parsed?.error === 'draining') {
|
|
107
173
|
// Planned leadership handover, not a fault. Distinct message so
|
|
108
174
|
// isServerDrainingError can classify it: retry paths treat it as
|
|
@@ -135,7 +201,8 @@ export class BrowserClient implements IScheduler {
|
|
|
135
201
|
}
|
|
136
202
|
} catch (_e) {
|
|
137
203
|
// FIX (#23): Truncate response body in error to prevent data leakage
|
|
138
|
-
const
|
|
204
|
+
const raw = collected.toString();
|
|
205
|
+
const preview = raw.length > 200 ? raw.slice(0, 200) + '...' : raw;
|
|
139
206
|
const error = new Error(`Failed to parse response (status ${res.statusCode}): ${preview}`);
|
|
140
207
|
errorTracker.trackError(error, {
|
|
141
208
|
component: 'browser-manager',
|
|
@@ -199,16 +266,16 @@ export class BrowserClient implements IScheduler {
|
|
|
199
266
|
});
|
|
200
267
|
}
|
|
201
268
|
|
|
202
|
-
async runSearch(query: string,
|
|
203
|
-
return this.request('/search', { query }, signal);
|
|
269
|
+
async runSearch(query: string, config?: Config, signal?: AbortSignal): Promise<SearchResult[]> {
|
|
270
|
+
return this.request('/search', { query }, signal, config);
|
|
204
271
|
}
|
|
205
272
|
|
|
206
|
-
async runScrape(url: string,
|
|
207
|
-
return this.request('/scrape', { url }, signal);
|
|
273
|
+
async runScrape(url: string, config?: Config, signal?: AbortSignal): Promise<any> {
|
|
274
|
+
return this.request('/scrape', { url }, signal, config);
|
|
208
275
|
}
|
|
209
276
|
|
|
210
|
-
async runHealthCheck(
|
|
211
|
-
return this.request('/healthcheck', {}, signal);
|
|
277
|
+
async runHealthCheck(config?: Config, signal?: AbortSignal): Promise<{ success: boolean }> {
|
|
278
|
+
return this.request('/healthcheck', {}, signal, config);
|
|
212
279
|
}
|
|
213
280
|
|
|
214
281
|
async shutdown() {
|
|
@@ -115,7 +115,12 @@ export function isPoolShutdownError(error: unknown): boolean {
|
|
|
115
115
|
// classifier matched it, so the first attempt rethrew instead of retrying —
|
|
116
116
|
// and auto-recovery fires only after 3 consecutive worker failures, i.e.
|
|
117
117
|
// exactly when a burst of tasks is in flight and would all die together.
|
|
118
|
-
err.message.includes('Worker pool is being reset')
|
|
118
|
+
err.message.includes('Worker pool is being reset') ||
|
|
119
|
+
// poolifier's wording once destroy() has COMPLETED (vs "destroying pool"
|
|
120
|
+
// while it is in progress). A queued closure holding a pool captured before
|
|
121
|
+
// an auto-recovery destroy dispatches into exactly this window; it is the
|
|
122
|
+
// same transient swap the "being reset" string covers, not a hard fault.
|
|
123
|
+
err.message.includes('Cannot execute a task on not started pool')
|
|
119
124
|
);
|
|
120
125
|
}
|
|
121
126
|
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import * as http from 'node:http';
|
|
2
2
|
import * as crypto from 'node:crypto';
|
|
3
3
|
import { logger } from '../../logger.ts';
|
|
4
|
+
import { Utf8Body } from '../../utils/http-body.ts';
|
|
4
5
|
import type { SearchResult } from '../../web-research/types.ts';
|
|
5
6
|
import { isCloudflareBlockError, isPoolShutdownError } from './browser-error-utils.ts';
|
|
6
7
|
|
|
7
8
|
export interface BrowserServerOptions {
|
|
8
|
-
onSearch: (query: string) => Promise<SearchResult[]>;
|
|
9
|
-
onScrape: (url: string) => Promise<any>;
|
|
10
|
-
onHealthCheck: () => Promise<{ success: boolean }>;
|
|
9
|
+
onSearch: (query: string, signal?: AbortSignal) => Promise<SearchResult[]>;
|
|
10
|
+
onScrape: (url: string, signal?: AbortSignal) => Promise<any>;
|
|
11
|
+
onHealthCheck: (signal?: AbortSignal) => Promise<{ success: boolean }>;
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
/**
|
|
@@ -92,6 +93,12 @@ export class BrowserServer {
|
|
|
92
93
|
const expected = Buffer.from(getOrCreateAuthSecret(), 'utf8');
|
|
93
94
|
const actual = Buffer.from(authHeader ?? '', 'utf8');
|
|
94
95
|
if (actual.length !== expected.length || !crypto.timingSafeEqual(actual, expected)) {
|
|
96
|
+
// Drain the request before answering (mirrors the embedding
|
|
97
|
+
// server's 403 path): without it a still-arriving body leaves
|
|
98
|
+
// the request half-read and Node destroys the socket after the
|
|
99
|
+
// response flush, so the client can see ECONNRESET instead of
|
|
100
|
+
// the status.
|
|
101
|
+
req.resume();
|
|
95
102
|
res.writeHead(403, { 'Content-Type': 'application/json' });
|
|
96
103
|
res.end(JSON.stringify({ error: 'Unauthorized' }));
|
|
97
104
|
return;
|
|
@@ -103,6 +110,7 @@ export class BrowserServer {
|
|
|
103
110
|
// stops the client reusing this socket for a retry that would
|
|
104
111
|
// only 503 again.
|
|
105
112
|
if (this.draining) {
|
|
113
|
+
req.resume(); // drain before answering — see the 403 path
|
|
106
114
|
res.writeHead(503, {
|
|
107
115
|
'Content-Type': 'application/json',
|
|
108
116
|
'Connection': 'close',
|
|
@@ -112,18 +120,34 @@ export class BrowserServer {
|
|
|
112
120
|
}
|
|
113
121
|
|
|
114
122
|
if (req.method !== 'POST') {
|
|
123
|
+
req.resume(); // drain before answering — see the 403 path
|
|
115
124
|
res.writeHead(405);
|
|
116
125
|
res.end('Method Not Allowed');
|
|
117
126
|
return;
|
|
118
127
|
}
|
|
119
128
|
|
|
120
|
-
|
|
129
|
+
// Follower cancellation must cross this HTTP hop: when the
|
|
130
|
+
// client's socket dies before the response completes (the
|
|
131
|
+
// follower aborted its request), the leader-side task previously
|
|
132
|
+
// kept its queue slot and worker page until its full timeout.
|
|
133
|
+
// Abort the task the moment the connection tears down. 'close'
|
|
134
|
+
// also fires after a normally completed response, so gate on
|
|
135
|
+
// writableEnded to abort only a premature teardown.
|
|
136
|
+
const requestAbort = new AbortController();
|
|
137
|
+
res.on('close', () => {
|
|
138
|
+
if (!res.writableEnded) requestAbort.abort();
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
// Decoded once at 'end' rather than per chunk: `body += chunk`
|
|
142
|
+
// mangles any multi-byte character that straddles a chunk boundary.
|
|
143
|
+
// See Utf8Body.
|
|
144
|
+
const collected = new Utf8Body();
|
|
121
145
|
let oversized = false;
|
|
122
146
|
const MAX_BODY_SIZE = 10 * 1024 * 1024; // 10MB limit
|
|
123
147
|
|
|
124
|
-
req.on('data', chunk => {
|
|
125
|
-
|
|
126
|
-
if (
|
|
148
|
+
req.on('data', (chunk: Buffer) => {
|
|
149
|
+
collected.push(chunk);
|
|
150
|
+
if (collected.byteLength > MAX_BODY_SIZE) {
|
|
127
151
|
oversized = true;
|
|
128
152
|
req.destroy(new Error('Payload too large'));
|
|
129
153
|
}
|
|
@@ -144,18 +168,18 @@ export class BrowserServer {
|
|
|
144
168
|
|
|
145
169
|
req.on('end', async () => {
|
|
146
170
|
try {
|
|
147
|
-
const data = JSON.parse(
|
|
171
|
+
const data = JSON.parse(collected.toString());
|
|
148
172
|
let result: any;
|
|
149
173
|
|
|
150
174
|
switch (req.url) {
|
|
151
175
|
case '/search':
|
|
152
|
-
result = await this.options.onSearch(data.query);
|
|
176
|
+
result = await this.options.onSearch(data.query, requestAbort.signal);
|
|
153
177
|
break;
|
|
154
178
|
case '/scrape':
|
|
155
|
-
result = await this.options.onScrape(data.url);
|
|
179
|
+
result = await this.options.onScrape(data.url, requestAbort.signal);
|
|
156
180
|
break;
|
|
157
181
|
case '/healthcheck':
|
|
158
|
-
result = await this.options.onHealthCheck();
|
|
182
|
+
result = await this.options.onHealthCheck(requestAbort.signal);
|
|
159
183
|
break;
|
|
160
184
|
default:
|
|
161
185
|
res.writeHead(404);
|
|
@@ -169,7 +193,12 @@ export class BrowserServer {
|
|
|
169
193
|
// A Cloudflare/bot challenge is an EXPECTED scrape outcome, not a
|
|
170
194
|
// server fault — log it at WARN so it doesn't inflate ERROR counts.
|
|
171
195
|
// Everything else is a genuine request-handling error.
|
|
172
|
-
if (
|
|
196
|
+
if (requestAbort.signal.aborted) {
|
|
197
|
+
// The client disconnected before completion; the abort above
|
|
198
|
+
// cancelled the leader-side task and nobody is waiting for
|
|
199
|
+
// this reply — expected teardown, not a server fault.
|
|
200
|
+
logger.debug('[BrowserServer] Client disconnected mid-request; task aborted:', error instanceof Error ? error.message : String(error));
|
|
201
|
+
} else if (isCloudflareBlockError(error)) {
|
|
173
202
|
logger.warn('[BrowserServer] Request blocked by bot protection:', error instanceof Error ? error.message : String(error));
|
|
174
203
|
} else if (isPoolShutdownError(error)) {
|
|
175
204
|
// Pool destroyed mid-request during quit/SIGTERM teardown — expected,
|