@lincoln504/pi-research 1.1.2 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/README.md +3 -3
  2. package/dist/cli.mjs +2599 -1268
  3. package/dist/thread-worker.mjs +224 -147
  4. package/docs/AGENT-SKILL.md +11 -4
  5. package/docs/ARCHITECTURE.md +20 -10
  6. package/docs/CHANGELOG.md +195 -17
  7. package/docs/CONFIGURATION.md +9 -7
  8. package/docs/KNOWLEDGE-STORE.md +4 -3
  9. package/docs/PI-EXTENSION.md +1 -1
  10. package/docs/SDK.md +55 -0
  11. package/docs/THIRD-PARTY-NOTICES.md +1 -1
  12. package/package.json +5 -4
  13. package/scripts/cleanup.cjs +114 -12
  14. package/scripts/prepare.cjs +15 -5
  15. package/scripts/setup.cjs +24 -4
  16. package/skills/pi-research/README.md +9 -2
  17. package/skills/pi-research/SKILL.md +6 -3
  18. package/skills/pi-research/references/configuration.md +6 -2
  19. package/skills/pi-research/scripts/run.mjs +24 -2
  20. package/src/cleanup/research-cleanup.ts +12 -4
  21. package/src/cli.ts +372 -86
  22. package/src/config.ts +134 -17
  23. package/src/constants.ts +31 -0
  24. package/src/core/interfaces/health-check-interfaces.ts +3 -39
  25. package/src/core/interfaces/knowledge-interfaces.ts +6 -1
  26. package/src/core/interfaces/planning-interfaces.ts +8 -0
  27. package/src/core/interfaces/service-names.ts +0 -1
  28. package/src/core/interfaces/state-manager-interfaces.ts +11 -3
  29. package/src/core/llm/agentic-repair.ts +14 -0
  30. package/src/core/pi-version.ts +136 -0
  31. package/src/core/planning-service.ts +43 -1
  32. package/src/core/service-initialization.ts +6 -14
  33. package/src/core/service-interfaces.ts +1 -1
  34. package/src/core/service-registry.ts +117 -8
  35. package/src/healthcheck/index.ts +7 -2
  36. package/src/index.ts +11 -18
  37. package/src/infrastructure/browser/browser-client.ts +80 -13
  38. package/src/infrastructure/browser/browser-error-utils.ts +6 -1
  39. package/src/infrastructure/browser/browser-server.ts +41 -12
  40. package/src/infrastructure/browser/browser-task-scheduler.ts +159 -35
  41. package/src/infrastructure/browser/cleanup-utils.ts +95 -9
  42. package/src/infrastructure/browser/client-agent.ts +6 -5
  43. package/src/infrastructure/browser/config.ts +14 -3
  44. package/src/infrastructure/browser/priority-task-queue.ts +10 -3
  45. package/src/infrastructure/browser/scheduler-factory.ts +44 -4
  46. package/src/infrastructure/browser/task-execution-service.ts +92 -28
  47. package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
  48. package/src/infrastructure/browser/thread-worker-messaging.ts +54 -2
  49. package/src/infrastructure/browser/thread-worker-types.ts +3 -1
  50. package/src/infrastructure/browser/thread-worker.mjs +224 -147
  51. package/src/infrastructure/browser/worker-pool-manager.ts +18 -5
  52. package/src/infrastructure/embedding/embedding-client.ts +22 -8
  53. package/src/infrastructure/embedding/embedding-factory.ts +61 -15
  54. package/src/infrastructure/embedding/embedding-server.ts +105 -11
  55. package/src/infrastructure/file-lock-service.ts +295 -28
  56. package/src/infrastructure/knowledge-store-service.ts +48 -3
  57. package/src/infrastructure/process-lifecycle-service.ts +134 -34
  58. package/src/infrastructure/research-run-semaphore.ts +37 -3
  59. package/src/infrastructure/state/state-backup-manager.ts +5 -7
  60. package/src/infrastructure/state/state-browser-api.ts +5 -2
  61. package/src/infrastructure/state/state-browser-manager.ts +15 -1
  62. package/src/infrastructure/state/state-manager-service.ts +5 -5
  63. package/src/infrastructure/state/state-manager.ts +34 -23
  64. package/src/infrastructure/types/state-types.ts +9 -0
  65. package/src/knowledge/chunker.ts +88 -1
  66. package/src/knowledge/embedder-init.ts +1 -1
  67. package/src/knowledge/embedder.ts +188 -46
  68. package/src/knowledge/index.ts +1 -1
  69. package/src/knowledge/store.ts +101 -21
  70. package/src/knowledge/webgpu-viability.ts +9 -4
  71. package/src/knowledge/writer-queue.ts +88 -46
  72. package/src/logger.ts +5 -1
  73. package/src/observers/research-observer-impl.ts +31 -0
  74. package/src/orchestration/abort-utils.ts +60 -0
  75. package/src/orchestration/deep-research-orchestrator.ts +25 -4
  76. package/src/orchestration/headless-observer.ts +8 -0
  77. package/src/orchestration/quick-research-orchestrator.ts +2 -2
  78. package/src/orchestration/research-orchestration-service.ts +65 -15
  79. package/src/orchestration/research-session-service.ts +37 -11
  80. package/src/orchestration/research-synthesis-service.ts +33 -11
  81. package/src/orchestration/researcher-executor.ts +100 -20
  82. package/src/orchestration/researcher.ts +39 -20
  83. package/src/orchestration/session-state.ts +11 -0
  84. package/src/sdk.ts +125 -19
  85. package/src/security/cisa-kev.ts +10 -3
  86. package/src/security/github-advisories.ts +18 -7
  87. package/src/security/index.ts +51 -30
  88. package/src/security/nvd-client.ts +24 -8
  89. package/src/security/osv-client.ts +14 -4
  90. package/src/skill-install/skill-installer.ts +153 -11
  91. package/src/stackexchange/index.ts +20 -5
  92. package/src/stackexchange/output/table.ts +7 -2
  93. package/src/stackexchange/rest-client.ts +41 -3
  94. package/src/tools/research-knowledge-search.ts +15 -0
  95. package/src/tools/research-tool-definition.ts +8 -2
  96. package/src/tools/scrape.ts +48 -25
  97. package/src/tools/search.ts +16 -2
  98. package/src/tools/security.ts +68 -7
  99. package/src/tui/research-health.ts +18 -2
  100. package/src/utils/atomic-replace.ts +101 -0
  101. package/src/utils/circuit-breaker.ts +9 -2
  102. package/src/utils/citation-utils.ts +3 -1
  103. package/src/utils/disk-space-checker.ts +11 -6
  104. package/src/utils/http-body.ts +176 -0
  105. package/src/utils/json-utils.ts +116 -76
  106. package/src/utils/llm-usage.ts +17 -1
  107. package/src/utils/log-rotation.ts +42 -10
  108. package/src/utils/log-utils.ts +40 -0
  109. package/src/utils/metrics-summary.ts +27 -9
  110. package/src/utils/stdio-capture.ts +63 -22
  111. package/src/utils/text-utils.ts +8 -1
  112. package/src/utils/tool-usage-tracker.ts +9 -2
  113. package/src/web-research/browser-search.ts +70 -5
  114. package/src/web-research/scraper-utils.ts +13 -0
  115. package/src/web-research/search.ts +25 -6
  116. package/src/web-research/types.ts +19 -4
  117. package/src/web-research/web-scraper.ts +72 -54
  118. package/src/youtube/transcript-client.ts +5 -1
  119. package/src/core/health-check-service.ts +0 -265
@@ -9,7 +9,7 @@ export type { SearchResult, ScrapeResult, IScheduler, SchedulerMetadata, ISchedu
9
9
  export type { ISchedulerFactory } from './scheduler-factory.ts';
10
10
 
11
11
  // Health check interfaces
12
- export type { HealthCheckResult, HealthCheckCache, IHealthCheckService, IHealthRegistryService } from './interfaces/health-check-interfaces.ts';
12
+ export type { IHealthRegistryService } from './interfaces/health-check-interfaces.ts';
13
13
 
14
14
  // Knowledge store interfaces
15
15
  export type {
@@ -17,6 +17,13 @@ import { logger } from '../logger.ts';
17
17
  import { AsyncLocalStorage } from 'node:async_hooks';
18
18
 
19
19
 
20
+ /**
21
+ * How long teardown waits for an initialization that is still in flight. Long
22
+ * enough to cover a normal service coming up, short enough that a service blocked
23
+ * on something slow (a model download) cannot hold shutdown open indefinitely.
24
+ */
25
+ const DISPOSE_INFLIGHT_INIT_TIMEOUT_MS = 5_000;
26
+
20
27
  /**
21
28
  * Service lifecycle stages
22
29
  */
@@ -209,15 +216,52 @@ export class ServiceContainer {
209
216
  throw new Error(`Service '${name}' is not registered`);
210
217
  }
211
218
 
212
- // Return existing instance if already initialized
219
+ // Return existing instance if already initialized.
213
220
  if (registration.instance) {
221
+ // `_initializeService` publishes `registration.instance` BEFORE awaiting the
222
+ // service's own initialize(), deliberately: a service whose initialize()
223
+ // resolves a peer that resolves it back would otherwise deadlock, and the
224
+ // partial instance is what breaks that cycle. But the same early publish
225
+ // hands a HALF-BUILT service to unrelated concurrent callers — e.g.
226
+ // StateManagerService only assigns its inner manager after nine awaited
227
+ // getService() calls, so every method on it throws "not initialized" until
228
+ // then, and the embedding factory resolves it and immediately calls
229
+ // updateState(). Worse, the ctx branch below would re-run initialize() on an
230
+ // instance that is still running it, constructing a second StateManager and
231
+ // discarding the first.
232
+ //
233
+ // So: if it is still initializing, wait for that to finish. The cycle case is
234
+ // exactly the case where `caller` is set (we are inside some service's
235
+ // initialize()), and there we must NOT wait — that is the deadlock the early
236
+ // publish exists to avoid. Outside an initialization there is no cycle to
237
+ // break and waiting is always correct.
238
+ if (registration.initializationPromise && caller === undefined) {
239
+ await registration.initializationPromise.catch(() => { /* surfaced below */ });
240
+ if (!registration.instance) {
241
+ // Initialization failed and cleaned up after itself — fall through and
242
+ // retry from scratch rather than returning a torn-down instance.
243
+ return this.get<T>(name, ctx);
244
+ }
245
+ }
214
246
  // Re-initialize if ctx is provided and service supports it
215
247
  if (ctx && registration.instance.initialize) {
216
248
  // Wrap in initializationContext so nested get() calls are tracked
217
- return initializationContext.run(name, async () => {
249
+ const reinit = initializationContext.run(name, async () => {
218
250
  await registration.instance!.initialize!(ctx);
219
251
  return registration.instance as T;
220
252
  });
253
+ // Recorded like a first init: a re-init is just as in-flight. Without
254
+ // this, disposeAll's settle-wait cannot see it (it would dispose the
255
+ // instance under a running initialize()), and concurrent callers
256
+ // received the instance mid-re-initialization.
257
+ registration.initializationPromise = reinit as Promise<IService>;
258
+ const clearIfCurrent = () => {
259
+ if (registration.initializationPromise === (reinit as Promise<IService>)) {
260
+ registration.initializationPromise = null;
261
+ }
262
+ };
263
+ reinit.then(clearIfCurrent, clearIfCurrent);
264
+ return reinit;
221
265
  }
222
266
  return registration.instance as T;
223
267
  }
@@ -243,7 +287,21 @@ export class ServiceContainer {
243
287
  }
244
288
 
245
289
  /**
246
- * Get a service instance synchronously (returns null if not initialized)
290
+ * Get a service instance synchronously (returns null if not registered or not
291
+ * yet constructed).
292
+ *
293
+ * Deliberately returns instances in NON-settled lifecycles too. This is a
294
+ * lifecycle *inspector* as much as an accessor: the health check resolves the
295
+ * knowledge store through it precisely so it can report `initializing` /
296
+ * `disabled` / `disposed (not running)` WITHOUT forcing initialization (forcing
297
+ * it was a past cause of 45 s CI stalls). Filtering those states out here would
298
+ * make every one of them indistinguishable from "absent".
299
+ *
300
+ * The trade-off: because `_initializeService` publishes the instance before
301
+ * awaiting its `initialize()` (to break dependency cycles), a caller that
302
+ * immediately *uses* the result can catch a half-built service. Callers that
303
+ * need a ready service must use the async `get()`, which waits — see the note
304
+ * there. Callers that only inspect `lifecycle` are safe by construction.
247
305
  */
248
306
  tryGet<T extends IService>(name: string): T | null {
249
307
  if (this.isDisposing) {
@@ -286,6 +344,15 @@ export class ServiceContainer {
286
344
  throw new Error(`Service '${name}' is not registered`);
287
345
  }
288
346
 
347
+ // Settle an in-flight initialization first. Disposing the early-published
348
+ // instance while its initialize() is still running hands the awaiting get()
349
+ // caller a service that was disposed under it — observed as store-closed
350
+ // errors when /research-config cleared the knowledge store during a
351
+ // concurrent first search.
352
+ if (registration.initializationPromise) {
353
+ await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
354
+ }
355
+
289
356
  if (registration.instance && registration.instance.dispose) {
290
357
  await registration.instance.dispose().catch((err: unknown) => {
291
358
  logger.warn(`[ServiceContainer] Error disposing service '${name}':`, err);
@@ -309,6 +376,11 @@ export class ServiceContainer {
309
376
  throw new Error(`Service '${name}' is not registered`);
310
377
  }
311
378
 
379
+ // Settle an in-flight initialization before swapping — see clear().
380
+ if (registration.initializationPromise) {
381
+ await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
382
+ }
383
+
312
384
  // Dispose old instance if present
313
385
  if (registration.instance && registration.instance.dispose) {
314
386
  await registration.instance.dispose().catch((err: unknown) => {
@@ -347,7 +419,41 @@ export class ServiceContainer {
347
419
  }
348
420
 
349
421
  try {
350
- const activeServices = Array.from(this.services.keys()).filter(name =>
422
+ // Settle any initialization still in flight BEFORE snapshotting what to tear
423
+ // down. `activeServices` is "instances that are non-null right now", so a
424
+ // get() whose factory has not resolved yet is invisible to it — and nothing
425
+ // else awaits it. Such a service installs itself into the registry AFTER
426
+ // teardown reports completion and is never disposed: for BrowserTaskScheduler
427
+ // that is a listening HTTP server plus a cluster pool and Camoufox children
428
+ // outliving shutdown; for KnowledgeStoreService an ONNX session and LanceDB
429
+ // handles. (SchedulerService.dispose hand-rolls this same guard for itself,
430
+ // which does not help when the leaked service IS SchedulerService.)
431
+ //
432
+ // Bounded: a service blocked on something slow (a model download) must not
433
+ // hold teardown open indefinitely. On timeout we proceed — the snapshot below
434
+ // still catches anything that landed in the meantime.
435
+ const inFlight = Array.from(this.services.values())
436
+ .map((r) => r.initializationPromise)
437
+ .filter((p): p is Promise<IService> => p != null);
438
+ if (inFlight.length > 0) {
439
+ if (this.defaultOptions.enableLogging) {
440
+ logger.debug(`[ServiceContainer] Waiting for ${inFlight.length} in-flight initialization(s) before teardown`);
441
+ }
442
+ let settleTimer: ReturnType<typeof setTimeout> | undefined;
443
+ await Promise.race([
444
+ Promise.allSettled(inFlight),
445
+ new Promise<void>((resolve) => {
446
+ settleTimer = setTimeout(() => {
447
+ logger.warn('[ServiceContainer] Timed out waiting for in-flight service initialization; proceeding with teardown');
448
+ resolve();
449
+ }, DISPOSE_INFLIGHT_INIT_TIMEOUT_MS);
450
+ settleTimer.unref?.();
451
+ }),
452
+ ]);
453
+ if (settleTimer) clearTimeout(settleTimer);
454
+ }
455
+
456
+ const activeServices = Array.from(this.services.keys()).filter(name =>
351
457
  this.services.get(name)?.instance !== null
352
458
  );
353
459
 
@@ -443,14 +549,17 @@ export class ServiceContainer {
443
549
  return null;
444
550
  }
445
551
 
446
- if (registration.instance) {
447
- return ServiceLifecycle.INITIALIZED;
448
- }
449
-
552
+ // Promise first: the instance is published BEFORE its initialize() completes
553
+ // (the early publish that lets dependency cycles resolve), so instance-first
554
+ // ordering reported a mid-initialization service as INITIALIZED.
450
555
  if (registration.initializationPromise) {
451
556
  return ServiceLifecycle.INITIALIZING;
452
557
  }
453
558
 
559
+ if (registration.instance) {
560
+ return ServiceLifecycle.INITIALIZED;
561
+ }
562
+
454
563
  return ServiceLifecycle.UNINITIALIZED;
455
564
  }
456
565
 
@@ -80,7 +80,12 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
80
80
  // respected (same pattern as the KnowledgeStore check's Math.max floor).
81
81
  }, { timeoutMs: Math.max(healthTimeoutMs, 150000), critical: true });
82
82
 
83
- // Register Knowledge Store Check
83
+ // Register Knowledge Store Check — NON-critical: the store is the optional cache
84
+ // research must run without (see the darwin-x64 comment below). A failure here
85
+ // (e.g. a dead embedding server, ECONNREFUSED) must degrade health, not flip the
86
+ // aggregate to unhealthy: as a critical check it aborted runs, and the raw store
87
+ // error then fed the TUI's browser-oriented formatter, which misattributed it as
88
+ // "the browser hit a network error / check your internet connection".
84
89
  registry.register('KnowledgeStore', async (options) => {
85
90
  // Idle fast-path — mirrors the BrowserRuntime check above. Read the ALREADY
86
91
  // resolved instance via tryGet(), which does NOT initialize the service;
@@ -222,7 +227,7 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
222
227
  }
223
228
  return { healthy: false, error: `Knowledge store healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
224
229
  }
225
- }, { timeoutMs: Math.max(healthTimeoutMs, 45000) });
230
+ }, { timeoutMs: Math.max(healthTimeoutMs, 45000), critical: false });
226
231
 
227
232
  // Register State Manager Check
228
233
  registry.register('StateManager', async () => {
package/src/index.ts CHANGED
@@ -11,6 +11,7 @@ import type { ResearchResultDetails } from './types/index.ts';
11
11
  import { createResearchTool, createHealthTool } from './tool.ts';
12
12
  import { createResearchKnowledgeSearchTool } from './tools/research-knowledge-search.ts';
13
13
  import { logger } from './logger.ts';
14
+ import { checkPiCompatibility } from './core/pi-version.ts';
14
15
  import { randomUUID } from 'node:crypto';
15
16
  import { shutdownManager } from './utils/shutdown-manager.ts';
16
17
  import { healthRegistry } from './healthcheck/index.ts';
@@ -98,25 +99,17 @@ export default async function (pi: ExtensionAPI) {
98
99
  // 0.80.8+ host auto-building its ModelRuntime. This in-host check is the real
99
100
  // enforcement point: when running as an extension the host's version is NOT
100
101
  // constrained by npm's resolution of our own package.json range.
101
- const versionParts = PI_VERSION.split('.').map(Number);
102
- const major = versionParts[0] ?? 0;
103
- const minor = versionParts[1] ?? 0;
104
- const patch = versionParts[2] ?? 0;
105
- if (isNaN(major) || isNaN(minor) || isNaN(patch)) {
106
- throw new Error(
107
- `[pi-research] Cannot parse pi-coding-agent version "${PI_VERSION}". ` +
108
- `Please ensure pi-coding-agent is installed correctly.`,
109
- );
102
+ // Two thresholds, not one — see src/core/pi-version.ts. Below the floor the APIs
103
+ // we call do not exist, so we refuse. Above the last TESTED line we warn and
104
+ // continue: pi is pre-1.0, a minor bump can break anything under semver (0.83.0
105
+ // already extended the ResourceLoader contract), and our own dependency range
106
+ // cannot constrain the host at all when running as an extension.
107
+ const compat = checkPiCompatibility(PI_VERSION);
108
+ if (compat.fatal) {
109
+ throw new Error(compat.message ?? `[pi-research] Unsupported pi-coding-agent version "${PI_VERSION}".`);
110
110
  }
111
- const minMajor = 0, minMinor = 80, minPatch = 8;
112
- const tooOld = major < minMajor
113
- || (major === minMajor && minor < minMinor)
114
- || (major === minMajor && minor === minMinor && patch < minPatch);
115
- if (tooOld) {
116
- throw new Error(
117
- `[pi-research] pi-coding-agent v${PI_VERSION} is too old. ` +
118
- `Requires v${minMajor}.${minMinor}.${minPatch}+. Please update pi-coding-agent.`,
119
- );
111
+ if (compat.message) {
112
+ logger.warn(compat.message);
120
113
  }
121
114
 
122
115
  // Re-register the beforeExit safety net (deactivate() strips event listeners during reload).
@@ -7,13 +7,69 @@
7
7
 
8
8
  import * as http from 'node:http';
9
9
  import type { Config } from '../../config.ts';
10
+ import { getConfig } from '../../config.ts';
10
11
  import type { SearchResult } from '../../web-research/types.ts';
11
12
  import type { IScheduler } from '../../core/interfaces/scheduler-interfaces.ts';
12
13
  import { logger } from '../../logger.ts';
13
14
  import { errorTracker } from '../../utils/error-tracker.ts';
15
+ import { Utf8Body } from '../../utils/http-body.ts';
14
16
  import { getClientAgent } from './client-agent.ts';
15
17
  import type { NodeError } from '../../types/index.ts';
16
18
 
19
+ /**
20
+ * Client-side patience margin on top of the leader's own task budget. The leader
21
+ * already bounds each task (nav budget + queue-wait margin) and answers with the
22
+ * more informative in-band timeout; the follower only needs headroom for the HTTP
23
+ * hop and leader-side dispatch, so its own timer never answers first.
24
+ */
25
+ const CLIENT_PATIENCE_MARGIN_MS = 15_000;
26
+
27
+ /**
28
+ * Absolute ceiling on any follower request, so a wedged-but-listening leader
29
+ * still fails in bounded time. Sits above the largest configurable leader budget
30
+ * (120s nav + 120s task margin) plus the patience margin, so it only binds when
31
+ * config is out of range. A DEAD leader is not this timer's job — its socket
32
+ * fails fast with ECONNREFUSED/ECONNRESET.
33
+ */
34
+ const CLIENT_TIMEOUT_CAP_MS = 300_000;
35
+
36
+ /**
37
+ * Mirror of the leader's fixed healthcheck budget in
38
+ * BrowserTaskScheduler.runHealthCheck (45s probe deadline + 60s queue-wait
39
+ * margin) — not config-derived there either.
40
+ */
41
+ const HEALTHCHECK_LEADER_BUDGET_MS = 45_000 + 60_000;
42
+
43
+ /**
44
+ * Per-operation follower timeout: the leader's own budget for that task kind
45
+ * plus a patience margin, bounded by an absolute cap.
46
+ *
47
+ * Invariant: the follower is at least as patient as the leader. The old flat 60s
48
+ * undercut the leader's healthcheck budget (105s) and config-driven search/scrape
49
+ * budgets (up to 130s+ within schema bounds), and the client-timeout text matches
50
+ * isTaskTimeoutError, so retry gates skipped it — long tasks failed client-side,
51
+ * unretried, while the leader was still working on them.
52
+ */
53
+ export function resolveClientRequestTimeoutMs(operation: string, config?: Config): number {
54
+ const cfg = config ?? getConfig();
55
+ let leaderBudgetMs: number;
56
+ switch (operation) {
57
+ case 'search':
58
+ // BrowserTaskScheduler.runSearch: SEARCH_TIMEOUT_MS + BROWSER_TASK_TIMEOUT_MS.
59
+ leaderBudgetMs = cfg.SEARCH_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS;
60
+ break;
61
+ case 'browser-task':
62
+ // BrowserTaskScheduler.runScrape: SCRAPE_TIMEOUT_MS + BROWSER_TASK_TIMEOUT_MS.
63
+ leaderBudgetMs = cfg.SCRAPE_TIMEOUT_MS + cfg.BROWSER_TASK_TIMEOUT_MS;
64
+ break;
65
+ default:
66
+ // healthcheck, plus any future/unknown path: the largest fixed budget.
67
+ leaderBudgetMs = HEALTHCHECK_LEADER_BUDGET_MS;
68
+ break;
69
+ }
70
+ return Math.min(CLIENT_TIMEOUT_CAP_MS, leaderBudgetMs + CLIENT_PATIENCE_MARGIN_MS);
71
+ }
72
+
17
73
  /**
18
74
  * HTTP client that communicates with a remote scheduler.
19
75
  * Used when this process is not the leader.
@@ -21,12 +77,19 @@ import type { NodeError } from '../../types/index.ts';
21
77
  export class BrowserClient implements IScheduler {
22
78
  private readonly authSecret: string;
23
79
 
80
+ /** The leader port this client was built for — lets the herd-guard cooldown
81
+ * verify the cached handle still targets the REGISTERED leader, not a
82
+ * predecessor that died inside the cooldown window. */
83
+ get targetPort(): number {
84
+ return this.port;
85
+ }
86
+
24
87
  constructor(private readonly port: number, authSecret?: string) {
25
88
  this.authSecret = authSecret ?? process.env['PI_BROWSER_AUTH_SECRET'] ?? '';
26
89
  logger.log(`[BrowserClient] Connecting to global scheduler at http://127.0.0.1:${port}`);
27
90
  }
28
91
 
29
- private async request<T>(path: string, data: any, signal?: AbortSignal): Promise<T> {
92
+ private async request<T>(path: string, data: any, signal?: AbortSignal, config?: Config): Promise<T> {
30
93
  const start = Date.now();
31
94
  // Extract operation from path for error tracking
32
95
  const operation = path.includes('/search') ? 'search' :
@@ -35,8 +98,8 @@ export class BrowserClient implements IScheduler {
35
98
 
36
99
  return new Promise((resolve, reject) => {
37
100
  const agent = getClientAgent();
38
- // 60s timeout for client requests (1/3 of original 180s)
39
- const timeoutMs = 60000;
101
+ // Config-derived per task kind: leader budget + margin, capped.
102
+ const timeoutMs = resolveClientRequestTimeoutMs(operation, config);
40
103
  let resolved = false;
41
104
  const controller = new AbortController();
42
105
 
@@ -93,8 +156,11 @@ export class BrowserClient implements IScheduler {
93
156
  // when headers arrive, but the body is still streaming — clearing
94
157
  // now would let a stalled body hang forever and make an external
95
158
  // abort a no-op. Keep both armed until 'end' or 'error'.
96
- let body = '';
97
- res.on('data', chunk => body += chunk);
159
+ // This response carries scraped page content, so per-chunk decoding
160
+ // would corrupt every non-ASCII page that exceeds one chunk. Collect
161
+ // bytes and decode once. See Utf8Body.
162
+ const collected = new Utf8Body();
163
+ res.on('data', (chunk: Buffer) => collected.push(chunk));
98
164
  res.on('end', () => {
99
165
  if (resolved) return;
100
166
  resolved = true;
@@ -102,7 +168,7 @@ export class BrowserClient implements IScheduler {
102
168
  abortCleanup?.();
103
169
  const duration = Date.now() - start;
104
170
  try {
105
- const parsed = JSON.parse(body);
171
+ const parsed = JSON.parse(collected.toString());
106
172
  if (res.statusCode === 503 && parsed?.error === 'draining') {
107
173
  // Planned leadership handover, not a fault. Distinct message so
108
174
  // isServerDrainingError can classify it: retry paths treat it as
@@ -135,7 +201,8 @@ export class BrowserClient implements IScheduler {
135
201
  }
136
202
  } catch (_e) {
137
203
  // FIX (#23): Truncate response body in error to prevent data leakage
138
- const preview = body.length > 200 ? body.slice(0, 200) + '...' : body;
204
+ const raw = collected.toString();
205
+ const preview = raw.length > 200 ? raw.slice(0, 200) + '...' : raw;
139
206
  const error = new Error(`Failed to parse response (status ${res.statusCode}): ${preview}`);
140
207
  errorTracker.trackError(error, {
141
208
  component: 'browser-manager',
@@ -199,16 +266,16 @@ export class BrowserClient implements IScheduler {
199
266
  });
200
267
  }
201
268
 
202
- async runSearch(query: string, _config?: Config, signal?: AbortSignal): Promise<SearchResult[]> {
203
- return this.request('/search', { query }, signal);
269
+ async runSearch(query: string, config?: Config, signal?: AbortSignal): Promise<SearchResult[]> {
270
+ return this.request('/search', { query }, signal, config);
204
271
  }
205
272
 
206
- async runScrape(url: string, _config?: Config, signal?: AbortSignal): Promise<any> {
207
- return this.request('/scrape', { url }, signal);
273
+ async runScrape(url: string, config?: Config, signal?: AbortSignal): Promise<any> {
274
+ return this.request('/scrape', { url }, signal, config);
208
275
  }
209
276
 
210
- async runHealthCheck(_config?: Config, signal?: AbortSignal): Promise<{ success: boolean }> {
211
- return this.request('/healthcheck', {}, signal);
277
+ async runHealthCheck(config?: Config, signal?: AbortSignal): Promise<{ success: boolean }> {
278
+ return this.request('/healthcheck', {}, signal, config);
212
279
  }
213
280
 
214
281
  async shutdown() {
@@ -115,7 +115,12 @@ export function isPoolShutdownError(error: unknown): boolean {
115
115
  // classifier matched it, so the first attempt rethrew instead of retrying —
116
116
  // and auto-recovery fires only after 3 consecutive worker failures, i.e.
117
117
  // exactly when a burst of tasks is in flight and would all die together.
118
- err.message.includes('Worker pool is being reset')
118
+ err.message.includes('Worker pool is being reset') ||
119
+ // poolifier's wording once destroy() has COMPLETED (vs "destroying pool"
120
+ // while it is in progress). A queued closure holding a pool captured before
121
+ // an auto-recovery destroy dispatches into exactly this window; it is the
122
+ // same transient swap the "being reset" string covers, not a hard fault.
123
+ err.message.includes('Cannot execute a task on not started pool')
119
124
  );
120
125
  }
121
126
 
@@ -1,13 +1,14 @@
1
1
  import * as http from 'node:http';
2
2
  import * as crypto from 'node:crypto';
3
3
  import { logger } from '../../logger.ts';
4
+ import { Utf8Body } from '../../utils/http-body.ts';
4
5
  import type { SearchResult } from '../../web-research/types.ts';
5
6
  import { isCloudflareBlockError, isPoolShutdownError } from './browser-error-utils.ts';
6
7
 
7
8
  export interface BrowserServerOptions {
8
- onSearch: (query: string) => Promise<SearchResult[]>;
9
- onScrape: (url: string) => Promise<any>;
10
- onHealthCheck: () => Promise<{ success: boolean }>;
9
+ onSearch: (query: string, signal?: AbortSignal) => Promise<SearchResult[]>;
10
+ onScrape: (url: string, signal?: AbortSignal) => Promise<any>;
11
+ onHealthCheck: (signal?: AbortSignal) => Promise<{ success: boolean }>;
11
12
  }
12
13
 
13
14
  /**
@@ -92,6 +93,12 @@ export class BrowserServer {
92
93
  const expected = Buffer.from(getOrCreateAuthSecret(), 'utf8');
93
94
  const actual = Buffer.from(authHeader ?? '', 'utf8');
94
95
  if (actual.length !== expected.length || !crypto.timingSafeEqual(actual, expected)) {
96
+ // Drain the request before answering (mirrors the embedding
97
+ // server's 403 path): without it a still-arriving body leaves
98
+ // the request half-read and Node destroys the socket after the
99
+ // response flush, so the client can see ECONNRESET instead of
100
+ // the status.
101
+ req.resume();
95
102
  res.writeHead(403, { 'Content-Type': 'application/json' });
96
103
  res.end(JSON.stringify({ error: 'Unauthorized' }));
97
104
  return;
@@ -103,6 +110,7 @@ export class BrowserServer {
103
110
  // stops the client reusing this socket for a retry that would
104
111
  // only 503 again.
105
112
  if (this.draining) {
113
+ req.resume(); // drain before answering — see the 403 path
106
114
  res.writeHead(503, {
107
115
  'Content-Type': 'application/json',
108
116
  'Connection': 'close',
@@ -112,18 +120,34 @@ export class BrowserServer {
112
120
  }
113
121
 
114
122
  if (req.method !== 'POST') {
123
+ req.resume(); // drain before answering — see the 403 path
115
124
  res.writeHead(405);
116
125
  res.end('Method Not Allowed');
117
126
  return;
118
127
  }
119
128
 
120
- let body = '';
129
+ // Follower cancellation must cross this HTTP hop: when the
130
+ // client's socket dies before the response completes (the
131
+ // follower aborted its request), the leader-side task previously
132
+ // kept its queue slot and worker page until its full timeout.
133
+ // Abort the task the moment the connection tears down. 'close'
134
+ // also fires after a normally completed response, so gate on
135
+ // writableEnded to abort only a premature teardown.
136
+ const requestAbort = new AbortController();
137
+ res.on('close', () => {
138
+ if (!res.writableEnded) requestAbort.abort();
139
+ });
140
+
141
+ // Decoded once at 'end' rather than per chunk: `body += chunk`
142
+ // mangles any multi-byte character that straddles a chunk boundary.
143
+ // See Utf8Body.
144
+ const collected = new Utf8Body();
121
145
  let oversized = false;
122
146
  const MAX_BODY_SIZE = 10 * 1024 * 1024; // 10MB limit
123
147
 
124
- req.on('data', chunk => {
125
- body += chunk;
126
- if (body.length > MAX_BODY_SIZE) {
148
+ req.on('data', (chunk: Buffer) => {
149
+ collected.push(chunk);
150
+ if (collected.byteLength > MAX_BODY_SIZE) {
127
151
  oversized = true;
128
152
  req.destroy(new Error('Payload too large'));
129
153
  }
@@ -144,18 +168,18 @@ export class BrowserServer {
144
168
 
145
169
  req.on('end', async () => {
146
170
  try {
147
- const data = JSON.parse(body);
171
+ const data = JSON.parse(collected.toString());
148
172
  let result: any;
149
173
 
150
174
  switch (req.url) {
151
175
  case '/search':
152
- result = await this.options.onSearch(data.query);
176
+ result = await this.options.onSearch(data.query, requestAbort.signal);
153
177
  break;
154
178
  case '/scrape':
155
- result = await this.options.onScrape(data.url);
179
+ result = await this.options.onScrape(data.url, requestAbort.signal);
156
180
  break;
157
181
  case '/healthcheck':
158
- result = await this.options.onHealthCheck();
182
+ result = await this.options.onHealthCheck(requestAbort.signal);
159
183
  break;
160
184
  default:
161
185
  res.writeHead(404);
@@ -169,7 +193,12 @@ export class BrowserServer {
169
193
  // A Cloudflare/bot challenge is an EXPECTED scrape outcome, not a
170
194
  // server fault — log it at WARN so it doesn't inflate ERROR counts.
171
195
  // Everything else is a genuine request-handling error.
172
- if (isCloudflareBlockError(error)) {
196
+ if (requestAbort.signal.aborted) {
197
+ // The client disconnected before completion; the abort above
198
+ // cancelled the leader-side task and nobody is waiting for
199
+ // this reply — expected teardown, not a server fault.
200
+ logger.debug('[BrowserServer] Client disconnected mid-request; task aborted:', error instanceof Error ? error.message : String(error));
201
+ } else if (isCloudflareBlockError(error)) {
173
202
  logger.warn('[BrowserServer] Request blocked by bot protection:', error instanceof Error ? error.message : String(error));
174
203
  } else if (isPoolShutdownError(error)) {
175
204
  // Pool destroyed mid-request during quit/SIGTERM teardown — expected,