@lincoln504/pi-research 1.1.0 → 1.2.0

package/src/core/pi-version.ts

@@ -0,0 +1,136 @@
+/**
+ * Host (pi) version compatibility.
+ *
+ * Extracted from the extension entry point so the policy is one testable unit
+ * rather than inline arithmetic, because the range this package declares cannot
+ * itself enforce compatibility:
+ *
+ *   - As a pi EXTENSION, the host supplies `@earendil-works/*`. Our package.json
+ *     range never constrains it, so an in-process check is the only enforcement.
+ *   - As a standalone CLI/SDK, the range is `>=0.80.8 <1` and published tarballs
+ *     carry no lockfile, so every fresh install resolves the newest 0.x at that
+ *     instant. pi is pre-1.0: a minor bump may break anything under semver, and
+ *     already has (0.83.0 extended the ResourceLoader contract).
+ *
+ * Hence two thresholds, not one. Below the FLOOR we throw — those APIs are gone
+ * and nothing will work. Above the TESTED ceiling we warn once and continue: a
+ * newer pi usually works, refusing it would strand users on every pi release, but
+ * silently pretending it is verified is how "works on my machine" ships.
+ */
+
+export interface SemverParts {
+  major: number;
+  minor: number;
+  patch: number;
+  /**
+   * True when the version carried a pre-release tag (`-rc.1`). Semver orders a
+   * pre-release BELOW its release, so `0.80.8-rc.1` predates the `0.80.8` floor
+   * and may lack the APIs the floor exists for. Only the floor comparison uses
+   * this; the tested-ceiling check compares lines and ignores it.
+   */
+  prerelease?: boolean;
+}
+
+/**
+ * Parse a semver-ish version string. Tolerates a `v` prefix and pre-release /
+ * build metadata suffixes (`0.84.0-rc.1`, `0.84.0+abc`), which pi has used.
+ * Returns null when the numeric core cannot be read, so callers degrade to an
+ * explicit "cannot determine" rather than to a silently wrong comparison.
+ */
+export function parsePiVersion(version: string): SemverParts | null {
+  const m = /^v?(\d+)\.(\d+)\.(\d+)(-)?/.exec(String(version ?? '').trim());
+  if (!m) return null;
+  const major = Number(m[1]);
+  const minor = Number(m[2]);
+  const patch = Number(m[3]);
+  if ([major, minor, patch].some((n) => !Number.isFinite(n))) return null;
+  // The flag is emitted only when set, so a plain release parses to the exact
+  // {major, minor, patch} shape existing comparisons and tests expect.
+  return m[4] === '-' ? { major, minor, patch, prerelease: true } : { major, minor, patch };
+}
+
+/** Ordering comparison: negative if a < b, 0 if equal, positive if a > b. */
+export function compareVersions(a: SemverParts, b: SemverParts): number {
+  return (a.major - b.major) || (a.minor - b.minor) || (a.patch - b.patch);
+}
+
+/**
+ * Oldest host this package works against.
+ *
+ * 0.80.8 introduced `ModelRuntime` and removed `AuthStorage` /
+ * `ModelRegistry.create()`. `buildModelRegistry` calls `ModelRuntime.create()`
+ * unconditionally, and `createAgentSession()` is invoked without the removed
+ * `modelRegistry` option, relying on the 0.80.8+ host to build its own.
+ */
+export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 80, patch: 8 };
+
+/**
+ * Newest host line this release was actually exercised against (CI + a real run).
+ * Bump this — deliberately — when a new pi is verified, not automatically.
+ * Compared on MAJOR.MINOR only: a patch bump within a tested line is not a new
+ * surface, and warning on it would be noise.
+ */
+export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor: 83, patch: 0 };
+
+export type PiCompatibilityLevel = 'ok' | 'unparseable' | 'too-old' | 'untested';
+
+export interface PiCompatibility {
+  level: PiCompatibilityLevel;
+  /** True when the host must be rejected outright. */
+  fatal: boolean;
+  message: string | null;
+}
+
+/**
+ * Classify a host version against the supported window.
+ *
+ * Pure: no logging, no throwing. The caller decides what to do with each level,
+ * which is what makes the policy testable without stubbing a logger.
+ */
+export function checkPiCompatibility(
+  version: string,
+  opts: { min?: SemverParts; testedMax?: SemverParts } = {},
+): PiCompatibility {
+  const min = opts.min ?? PI_MIN_VERSION;
+  const testedMax = opts.testedMax ?? PI_TESTED_MAX_VERSION;
+
+  const parsed = parsePiVersion(version);
+  if (!parsed) {
+    return {
+      level: 'unparseable',
+      fatal: true,
+      message:
+        `[pi-research] Cannot parse pi-coding-agent version "${version}". ` +
+        `Please ensure pi-coding-agent is installed correctly.`,
+    };
+  }
+
+  // A pre-release OF the floor version orders below the floor (0.80.8-rc.1 <
+  // 0.80.8) and may predate the very APIs the floor guards.
+  if (compareVersions(parsed, min) < 0 || (compareVersions(parsed, min) === 0 && parsed.prerelease === true)) {
+    return {
+      level: 'too-old',
+      fatal: true,
+      message:
+        `[pi-research] pi-coding-agent v${version} is too old. ` +
+        `Requires v${min.major}.${min.minor}.${min.patch}+. Please update pi-coding-agent.`,
+    };
+  }
+
+  // Compare on major.minor only — a patch release inside a tested line is fine.
+  const line: SemverParts = { major: parsed.major, minor: parsed.minor, patch: 0 };
+  const testedLine: SemverParts = { major: testedMax.major, minor: testedMax.minor, patch: 0 };
+  if (compareVersions(line, testedLine) > 0) {
+    return {
+      level: 'untested',
+      fatal: false,
+      message:
+        `[pi-research] pi-coding-agent v${version} is newer than the last version this ` +
+        `release of pi-research was tested against (v${testedMax.major}.${testedMax.minor}.x). ` +
+        `Continuing — it will most likely work — but if you hit anything odd, that mismatch ` +
+        `is the first thing to suspect. Updating pi-research usually resolves it.`,
+    };
+  }
+
+  return { level: 'ok', fatal: false, message: null };
+}

package/src/core/service-registry.ts

@@ -17,6 +17,13 @@ import { logger } from '../logger.ts';
 import { AsyncLocalStorage } from 'node:async_hooks';
 
 
+/**
+ * How long teardown waits for an initialization that is still in flight. Long
+ * enough to cover a normal service coming up, short enough that a service blocked
+ * on something slow (a model download) cannot hold shutdown open indefinitely.
+ */
+const DISPOSE_INFLIGHT_INIT_TIMEOUT_MS = 5_000;
+
 /**
  * Service lifecycle stages
  */
@@ -209,15 +216,52 @@ export class ServiceContainer {
       throw new Error(`Service '${name}' is not registered`);
     }
 
-    // Return existing instance if already initialized
+    // Return existing instance if already initialized.
     if (registration.instance) {
+      // `_initializeService` publishes `registration.instance` BEFORE awaiting the
+      // service's own initialize(), deliberately: a service whose initialize()
+      // resolves a peer that resolves it back would otherwise deadlock, and the
+      // partial instance is what breaks that cycle. But the same early publish
+      // hands a HALF-BUILT service to unrelated concurrent callers — e.g.
+      // StateManagerService only assigns its inner manager after nine awaited
+      // getService() calls, so every method on it throws "not initialized" until
+      // then, and the embedding factory resolves it and immediately calls
+      // updateState(). Worse, the ctx branch below would re-run initialize() on an
+      // instance that is still running it, constructing a second StateManager and
+      // discarding the first.
+      //
+      // So: if it is still initializing, wait for that to finish. The cycle case is
+      // exactly the case where `caller` is set (we are inside some service's
+      // initialize()), and there we must NOT wait — that is the deadlock the early
+      // publish exists to avoid. Outside an initialization there is no cycle to
+      // break and waiting is always correct.
+      if (registration.initializationPromise && caller === undefined) {
+        await registration.initializationPromise.catch(() => { /* surfaced below */ });
+        if (!registration.instance) {
+          // Initialization failed and cleaned up after itself — fall through and
+          // retry from scratch rather than returning a torn-down instance.
+          return this.get<T>(name, ctx);
+        }
+      }
       // Re-initialize if ctx is provided and service supports it
       if (ctx && registration.instance.initialize) {
         // Wrap in initializationContext so nested get() calls are tracked
-        return initializationContext.run(name, async () => {
+        const reinit = initializationContext.run(name, async () => {
           await registration.instance!.initialize!(ctx);
           return registration.instance as T;
         });
+        // Recorded like a first init: a re-init is just as in-flight. Without
+        // this, disposeAll's settle-wait cannot see it (it would dispose the
+        // instance under a running initialize()), and concurrent callers
+        // received the instance mid-re-initialization.
+        registration.initializationPromise = reinit as Promise<IService>;
+        const clearIfCurrent = () => {
+          if (registration.initializationPromise === (reinit as Promise<IService>)) {
+            registration.initializationPromise = null;
+          }
+        };
+        reinit.then(clearIfCurrent, clearIfCurrent);
+        return reinit;
       }
       return registration.instance as T;
     }
@@ -243,7 +287,21 @@ export class ServiceContainer {
   }
 
   /**
-   * Get a service instance synchronously (returns null if not initialized)
+   * Get a service instance synchronously (returns null if not registered or not
+   * yet constructed).
+   *
+   * Deliberately returns instances in NON-settled lifecycles too. This is a
+   * lifecycle *inspector* as much as an accessor: the health check resolves the
+   * knowledge store through it precisely so it can report `initializing` /
+   * `disabled` / `disposed (not running)` WITHOUT forcing initialization (forcing
+   * it was a past cause of 45 s CI stalls). Filtering those states out here would
+   * make every one of them indistinguishable from "absent".
+   *
+   * The trade-off: because `_initializeService` publishes the instance before
+   * awaiting its `initialize()` (to break dependency cycles), a caller that
+   * immediately *uses* the result can catch a half-built service. Callers that
+   * need a ready service must use the async `get()`, which waits — see the note
+   * there. Callers that only inspect `lifecycle` are safe by construction.
    */
   tryGet<T extends IService>(name: string): T | null {
     if (this.isDisposing) {
@@ -286,6 +344,15 @@ export class ServiceContainer {
       throw new Error(`Service '${name}' is not registered`);
     }
 
+    // Settle an in-flight initialization first. Disposing the early-published
+    // instance while its initialize() is still running hands the awaiting get()
+    // caller a service that was disposed under it — observed as store-closed
+    // errors when /research-config cleared the knowledge store during a
+    // concurrent first search.
+    if (registration.initializationPromise) {
+      await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
+    }
+
     if (registration.instance && registration.instance.dispose) {
       await registration.instance.dispose().catch((err: unknown) => {
         logger.warn(`[ServiceContainer] Error disposing service '${name}':`, err);
@@ -309,6 +376,11 @@ export class ServiceContainer {
       throw new Error(`Service '${name}' is not registered`);
     }
 
+    // Settle an in-flight initialization before swapping — see clear().
+    if (registration.initializationPromise) {
+      await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
+    }
+
     // Dispose old instance if present
     if (registration.instance && registration.instance.dispose) {
       await registration.instance.dispose().catch((err: unknown) => {
@@ -347,7 +419,41 @@ export class ServiceContainer {
     }
 
     try {
-      const activeServices = Array.from(this.services.keys()).filter(name => 
+      // Settle any initialization still in flight BEFORE snapshotting what to tear
+      // down. `activeServices` is "instances that are non-null right now", so a
+      // get() whose factory has not resolved yet is invisible to it — and nothing
+      // else awaits it. Such a service installs itself into the registry AFTER
+      // teardown reports completion and is never disposed: for BrowserTaskScheduler
+      // that is a listening HTTP server plus a cluster pool and Camoufox children
+      // outliving shutdown; for KnowledgeStoreService an ONNX session and LanceDB
+      // handles. (SchedulerService.dispose hand-rolls this same guard for itself,
+      // which does not help when the leaked service IS SchedulerService.)
+      //
+      // Bounded: a service blocked on something slow (a model download) must not
+      // hold teardown open indefinitely. On timeout we proceed — the snapshot below
+      // still catches anything that landed in the meantime.
+      const inFlight = Array.from(this.services.values())
+        .map((r) => r.initializationPromise)
+        .filter((p): p is Promise<IService> => p != null);
+      if (inFlight.length > 0) {
+        if (this.defaultOptions.enableLogging) {
+          logger.debug(`[ServiceContainer] Waiting for ${inFlight.length} in-flight initialization(s) before teardown`);
+        }
+        let settleTimer: ReturnType<typeof setTimeout> | undefined;
+        await Promise.race([
+          Promise.allSettled(inFlight),
+          new Promise<void>((resolve) => {
+            settleTimer = setTimeout(() => {
+              logger.warn('[ServiceContainer] Timed out waiting for in-flight service initialization; proceeding with teardown');
+              resolve();
+            }, DISPOSE_INFLIGHT_INIT_TIMEOUT_MS);
+            settleTimer.unref?.();
+          }),
+        ]);
+        if (settleTimer) clearTimeout(settleTimer);
+      }
+
+      const activeServices = Array.from(this.services.keys()).filter(name =>
         this.services.get(name)?.instance !== null
       );
       
@@ -443,14 +549,17 @@ export class ServiceContainer {
       return null;
     }
 
-    if (registration.instance) {
-      return ServiceLifecycle.INITIALIZED;
-    }
-
+    // Promise first: the instance is published BEFORE its initialize() completes
+    // (the early publish that lets dependency cycles resolve), so instance-first
+    // ordering reported a mid-initialization service as INITIALIZED.
     if (registration.initializationPromise) {
       return ServiceLifecycle.INITIALIZING;
     }
 
+    if (registration.instance) {
+      return ServiceLifecycle.INITIALIZED;
+    }
+
     return ServiceLifecycle.UNINITIALIZED;
   }
 

package/src/index.ts

@@ -11,6 +11,7 @@ import type { ResearchResultDetails } from './types/index.ts';
 import { createResearchTool, createHealthTool } from './tool.ts';
 import { createResearchKnowledgeSearchTool } from './tools/research-knowledge-search.ts';
 import { logger } from './logger.ts';
+import { checkPiCompatibility } from './core/pi-version.ts';
 import { randomUUID } from 'node:crypto';
 import { shutdownManager } from './utils/shutdown-manager.ts';
 import { healthRegistry } from './healthcheck/index.ts';
@@ -98,25 +99,17 @@ export default async function (pi: ExtensionAPI) {
   // 0.80.8+ host auto-building its ModelRuntime. This in-host check is the real
   // enforcement point: when running as an extension the host's version is NOT
   // constrained by npm's resolution of our own package.json range.
-  const versionParts = PI_VERSION.split('.').map(Number);
-  const major = versionParts[0] ?? 0;
-  const minor = versionParts[1] ?? 0;
-  const patch = versionParts[2] ?? 0;
-  if (isNaN(major) || isNaN(minor) || isNaN(patch)) {
-    throw new Error(
-      `[pi-research] Cannot parse pi-coding-agent version "${PI_VERSION}". ` +
-      `Please ensure pi-coding-agent is installed correctly.`,
-    );
+  // Two thresholds, not one — see src/core/pi-version.ts. Below the floor the APIs
+  // we call do not exist, so we refuse. Above the last TESTED line we warn and
+  // continue: pi is pre-1.0, a minor bump can break anything under semver (0.83.0
+  // already extended the ResourceLoader contract), and our own dependency range
+  // cannot constrain the host at all when running as an extension.
+  const compat = checkPiCompatibility(PI_VERSION);
+  if (compat.fatal) {
+    throw new Error(compat.message ?? `[pi-research] Unsupported pi-coding-agent version "${PI_VERSION}".`);
   }
-  const minMajor = 0, minMinor = 80, minPatch = 8;
-  const tooOld = major < minMajor
-    || (major === minMajor && minor < minMinor)
-    || (major === minMajor && minor === minMinor && patch < minPatch);
-  if (tooOld) {
-    throw new Error(
-      `[pi-research] pi-coding-agent v${PI_VERSION} is too old. ` +
-      `Requires v${minMajor}.${minMinor}.${minPatch}+. Please update pi-coding-agent.`,
-    );
+  if (compat.message) {
+    logger.warn(compat.message);
   }
 
   // Re-register the beforeExit safety net (deactivate() strips event listeners during reload).

package/src/infrastructure/browser/browser-client.ts

@@ -11,6 +11,7 @@ import type { SearchResult } from '../../web-research/types.ts';
 import type { IScheduler } from '../../core/interfaces/scheduler-interfaces.ts';
 import { logger } from '../../logger.ts';
 import { errorTracker } from '../../utils/error-tracker.ts';
+import { Utf8Body } from '../../utils/http-body.ts';
 import { getClientAgent } from './client-agent.ts';
 import type { NodeError } from '../../types/index.ts';
 
@@ -21,6 +22,13 @@ import type { NodeError } from '../../types/index.ts';
 export class BrowserClient implements IScheduler {
     private readonly authSecret: string;
 
+    /** The leader port this client was built for — lets the herd-guard cooldown
+     * verify the cached handle still targets the REGISTERED leader, not a
+     * predecessor that died inside the cooldown window. */
+    get targetPort(): number {
+        return this.port;
+    }
+
     constructor(private readonly port: number, authSecret?: string) {
         this.authSecret = authSecret ?? process.env['PI_BROWSER_AUTH_SECRET'] ?? '';
         logger.log(`[BrowserClient] Connecting to global scheduler at http://127.0.0.1:${port}`);
@@ -93,8 +101,11 @@ export class BrowserClient implements IScheduler {
                 // when headers arrive, but the body is still streaming — clearing
                 // now would let a stalled body hang forever and make an external
                 // abort a no-op. Keep both armed until 'end' or 'error'.
-                let body = '';
-                res.on('data', chunk => body += chunk);
+                // This response carries scraped page content, so per-chunk decoding
+                // would corrupt every non-ASCII page that exceeds one chunk. Collect
+                // bytes and decode once. See Utf8Body.
+                const collected = new Utf8Body();
+                res.on('data', (chunk: Buffer) => collected.push(chunk));
                 res.on('end', () => {
                     if (resolved) return;
                     resolved = true;
@@ -102,7 +113,7 @@ export class BrowserClient implements IScheduler {
                     abortCleanup?.();
                     const duration = Date.now() - start;
                     try {
-                        const parsed = JSON.parse(body);
+                        const parsed = JSON.parse(collected.toString());
                         if (res.statusCode === 503 && parsed?.error === 'draining') {
                             // Planned leadership handover, not a fault. Distinct message so
                             // isServerDrainingError can classify it: retry paths treat it as
@@ -135,7 +146,8 @@ export class BrowserClient implements IScheduler {
                         }
                     } catch (_e) {
                         // FIX (#23): Truncate response body in error to prevent data leakage
-                        const preview = body.length > 200 ? body.slice(0, 200) + '...' : body;
+                        const raw = collected.toString();
+                        const preview = raw.length > 200 ? raw.slice(0, 200) + '...' : raw;
                         const error = new Error(`Failed to parse response (status ${res.statusCode}): ${preview}`);
                         errorTracker.trackError(error, {
                             component: 'browser-manager',

package/src/infrastructure/browser/browser-error-utils.ts

@@ -109,7 +109,18 @@ export function isPoolShutdownError(error: unknown): boolean {
     return typeof err.message === 'string' && (
         err.message.includes('Worker pool is shutting down') ||
         err.message.includes('Cannot execute a task on destroying pool') ||
-        err.message.includes('destroying pool')
+        err.message.includes('destroying pool') ||
+        // worker-pool-manager throws this when auto-recovery is still swapping the
+        // pool. Its own text says "please retry", but until it was listed here no
+        // classifier matched it, so the first attempt rethrew instead of retrying —
+        // and auto-recovery fires only after 3 consecutive worker failures, i.e.
+        // exactly when a burst of tasks is in flight and would all die together.
+        err.message.includes('Worker pool is being reset') ||
+        // poolifier's wording once destroy() has COMPLETED (vs "destroying pool"
+        // while it is in progress). A queued closure holding a pool captured before
+        // an auto-recovery destroy dispatches into exactly this window; it is the
+        // same transient swap the "being reset" string covers, not a hard fault.
+        err.message.includes('Cannot execute a task on not started pool')
     );
 }
 

package/src/infrastructure/browser/browser-server.ts

@@ -1,6 +1,7 @@
 import * as http from 'node:http';
 import * as crypto from 'node:crypto';
 import { logger } from '../../logger.ts';
+import { Utf8Body } from '../../utils/http-body.ts';
 import type { SearchResult } from '../../web-research/types.ts';
 import { isCloudflareBlockError, isPoolShutdownError } from './browser-error-utils.ts';
 
@@ -92,6 +93,12 @@ export class BrowserServer {
                 const expected = Buffer.from(getOrCreateAuthSecret(), 'utf8');
                 const actual = Buffer.from(authHeader ?? '', 'utf8');
                 if (actual.length !== expected.length || !crypto.timingSafeEqual(actual, expected)) {
+                    // Drain the request before answering (mirrors the embedding
+                    // server's 403 path): without it a still-arriving body leaves
+                    // the request half-read and Node destroys the socket after the
+                    // response flush, so the client can see ECONNRESET instead of
+                    // the status.
+                    req.resume();
                     res.writeHead(403, { 'Content-Type': 'application/json' });
                     res.end(JSON.stringify({ error: 'Unauthorized' }));
                     return;
@@ -103,6 +110,7 @@ export class BrowserServer {
                 // stops the client reusing this socket for a retry that would
                 // only 503 again.
                 if (this.draining) {
+                    req.resume(); // drain before answering — see the 403 path
                     res.writeHead(503, {
                         'Content-Type': 'application/json',
                         'Connection': 'close',
@@ -112,18 +120,22 @@ export class BrowserServer {
                 }
 
                 if (req.method !== 'POST') {
+                    req.resume(); // drain before answering — see the 403 path
                     res.writeHead(405);
                     res.end('Method Not Allowed');
                     return;
                 }
 
-                let body = '';
+                // Decoded once at 'end' rather than per chunk: `body += chunk`
+                // mangles any multi-byte character that straddles a chunk boundary.
+                // See Utf8Body.
+                const collected = new Utf8Body();
                 let oversized = false;
                 const MAX_BODY_SIZE = 10 * 1024 * 1024; // 10MB limit
 
-                req.on('data', chunk => {
-                    body += chunk;
-                    if (body.length > MAX_BODY_SIZE) {
+                req.on('data', (chunk: Buffer) => {
+                    collected.push(chunk);
+                    if (collected.byteLength > MAX_BODY_SIZE) {
                         oversized = true;
                         req.destroy(new Error('Payload too large'));
                     }
@@ -144,7 +156,7 @@ export class BrowserServer {
 
                 req.on('end', async () => {
                     try {
-                        const data = JSON.parse(body);
+                        const data = JSON.parse(collected.toString());
                         let result: any;
 
                         switch (req.url) {