@lincoln504/pi-research 1.1.0 → 1.2.0

package/src/infrastructure/browser/thread-worker.mjs

@@ -1284,6 +1284,11 @@ var init_circuit_breaker = __esm({
           metrics.observe("circuit_breaker_call_duration_ms", duration, { breaker: this.options.name, status: "error" });
           if (this.options.isTransientError(error)) {
             this.onFailure(error);
+          } else if (admittedAsProbe && this.state === "HALF_OPEN") {
+            logger.warn(
+              `[CircuitBreaker] '${this.options.name}' HALF_OPEN probe failed with a non-counted error; treating as inconclusive and re-opening: ${error instanceof Error ? error.message : String(error)}`
+            );
+            this.transitionTo("OPEN");
           }
           throw error;
         } finally {
@@ -1379,7 +1384,16 @@ function isServerDrainingError(error) {
 function isPoolShutdownError(error) {
   if (!error || typeof error !== "object") return false;
   const err = error;
-  return typeof err.message === "string" && (err.message.includes("Worker pool is shutting down") || err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool"));
+  return typeof err.message === "string" && (err.message.includes("Worker pool is shutting down") || err.message.includes("Cannot execute a task on destroying pool") || err.message.includes("destroying pool") || // worker-pool-manager throws this when auto-recovery is still swapping the
+  // pool. Its own text says "please retry", but until it was listed here no
+  // classifier matched it, so the first attempt rethrew instead of retrying —
+  // and auto-recovery fires only after 3 consecutive worker failures, i.e.
+  // exactly when a burst of tasks is in flight and would all die together.
+  err.message.includes("Worker pool is being reset") || // poolifier's wording once destroy() has COMPLETED (vs "destroying pool"
+  // while it is in progress). A queued closure holding a pool captured before
+  // an auto-recovery destroy dispatches into exactly this window; it is the
+  // same transient swap the "being reset" string covers, not a hard fault.
+  err.message.includes("Cannot execute a task on not started pool"));
 }
 var DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
 var init_browser_error_utils = __esm({
@@ -1743,12 +1757,34 @@ function setupOrphanProtection() {
   if (!cluster.isWorker) {
     return;
   }
-  if (!process2.ppid) {
+  const originalPpid = process2.ppid;
+  if (!originalPpid) {
     return;
   }
+  const exitAsOrphan = async () => {
+    if (cleanupBrowser) {
+      await Promise.race([
+        cleanupBrowser().catch((err) => {
+          logToDebugFile("WARN", `[Worker-${workerId}] Browser cleanup failed during orphan exit:`, err);
+        }),
+        new Promise((resolve) => {
+          const t = setTimeout(resolve, 1e4);
+          if (t.unref) t.unref();
+        })
+      ]);
+    }
+    cleanupOrphanProtection();
+    process2.env["PI_PROCESS_EXITING"] = "1";
+    process2.exit(1);
+  };
   const checkOrphan = async () => {
     try {
-      process2.kill(process2.ppid, 0);
+      if (process2.ppid !== originalPpid) {
+        logToDebugFile("WARN", `[Worker-${workerId}] Re-parented (${originalPpid} -> ${process2.ppid}): parent died, shutting down...`);
+        await exitAsOrphan();
+        return;
+      }
+      process2.kill(originalPpid, 0);
       orphanCheckTimer = setTimeout(checkOrphan, 1e4);
       if (orphanCheckTimer) {
         orphanCheckTimer.unref();
@@ -1760,14 +1796,7 @@ function setupOrphanProtection() {
         return;
       }
       logToDebugFile("WARN", `[Worker-${workerId}] Parent process died or unreachable (orphaned), shutting down...`);
-      if (cleanupBrowser) {
-        await cleanupBrowser().catch((err) => {
-          logToDebugFile("WARN", `[Worker-${workerId}] Browser cleanup failed during orphan exit:`, err);
-        });
-      }
-      cleanupOrphanProtection();
-      process2.env["PI_PROCESS_EXITING"] = "1";
-      process2.exit(1);
+      await exitAsOrphan();
     }
   };
   orphanCheckTimer = setTimeout(checkOrphan, 1e4);

package/src/infrastructure/browser/worker-pool-manager.ts

@@ -112,6 +112,7 @@ export class WorkerPoolManager implements IService {
             try {
                 if (this.pool && this.currentWorkerCount !== maxWorkers) {
                     logger.log(`[WorkerPoolManager] Worker count changed from ${this.currentWorkerCount} to ${maxWorkers}, recreating pool...`);
+                    this.poolEpoch++; // intentional destroy: silence the dying pool's exit events
                     await this.pool.destroy();
                     this.pool = null;
                 }
@@ -211,9 +212,13 @@ export class WorkerPoolManager implements IService {
                         }
                     },
                     exitHandler: (code: number) => {
-                        // null exit code means the worker was killed intentionally (e.g. pool.destroy()).
-                        // Ignore exits from a superseded pool instance (see errorHandler).
-                        if (code !== 0 && code !== null && this.poolEpoch === myEpoch) {
+                        // A worker we destroy on purpose exits with code null — but so does one
+                        // the OOM killer SIGKILLs, and that is exactly the death mode that leaves
+                        // in-flight execute() promises stranded (poolifier never settles them).
+                        // Every intentional destroy bumps poolEpoch first, so a null-code exit
+                        // that still matches the live epoch is an unexpected death and must count
+                        // toward auto-recovery like any other abnormal exit.
+                        if (code !== 0 && this.poolEpoch === myEpoch) {
                             logger.error(`[WorkerPoolManager] Worker exited with code ${code}`);
                             this.consecutiveErrors++;
                             if (this.consecutiveErrors >= 3) {
@@ -245,6 +250,7 @@ export class WorkerPoolManager implements IService {
                     // Only null the shared field if it still points at the pool we built —
                     // a concurrent path may already have moved on.
                     if (this.pool === built) this.pool = null;
+                    this.poolEpoch++; // intentional destroy: silence the dying pool's exit events
                     await built.destroy().catch((err: any) => logger.debug('Swallowed pool destroy error:', err));
                     throw new Error('Worker pool is shutting down');
                 }
@@ -337,6 +343,7 @@ export class WorkerPoolManager implements IService {
                 // pool (and bumped the generation), so skip — destroying again is spurious
                 // and nulling shared state would clobber a pool rebuilt after the shutdown.
                 if (this.isShuttingDown || this.generation !== myGen) return;
+                this.poolEpoch++; // intentional destroy: silence the dying pool's exit events
                 if (deadPool) await deadPool.destroy();
                 logger.info('[WorkerPoolManager] Auto-recovery: old pool destroyed.');
             } catch (err) {
@@ -399,6 +406,7 @@ export class WorkerPoolManager implements IService {
                 // Use a timeout for pool destruction. Workers run async browser teardown
                 // in their killHandler (context.close / browser.close via Playwright), so
                 // allow enough time for those to complete before the IPC channel closes.
+                this.poolEpoch++; // intentional destroy: silence the dying pool's exit events
                 const destroyPromise = this.pool.destroy();
                 destroyPromise.catch((err: Error) => logger.debug(`[WorkerPoolManager] Background pool destroy rejection: ${err.message}`));
                 await raceWithDeadline(destroyPromise, 5000);

package/src/infrastructure/embedding/embedding-client.ts

@@ -8,6 +8,7 @@
 import * as http from 'node:http';
 import { logger } from '../../logger.ts';
 import type { IEmbedder } from '../../core/interfaces/knowledge-interfaces.ts';
+import { Utf8Body } from '../../utils/http-body.ts';
 
 export class EmbeddingClient implements IEmbedder {
   // Public so store.ts can set the dimension from existing table schema
@@ -17,10 +18,21 @@ export class EmbeddingClient implements IEmbedder {
   constructor(
     private readonly port: number,
     private _device: string = 'unknown',
+    /**
+     * Shared secret published by the leader in state. Undefined when the leader
+     * predates auth (it does not check a header either), so the request simply
+     * carries none — an older leader keeps working rather than 403-ing.
+     */
+    private readonly authSecret?: string,
   ) {
     logger.log(`[EmbeddingClient] Connecting to embedding server at http://127.0.0.1:${port}`);
   }
 
+  /** Auth header for the leader, or nothing when connecting to a pre-auth leader. */
+  private authHeaders(): Record<string, string> {
+    return this.authSecret ? { 'x-embedding-auth': this.authSecret } : {};
+  }
+
   // ---- IEmbedder ----
 
   getDevice(): string {
@@ -93,25 +105,26 @@ export class EmbeddingClient implements IEmbedder {
           port: this.port,
           path,
           method: 'GET',
+          headers: this.authHeaders(),
         },
         (res) => {
           // Keep the timeout armed through the body read — clearing it on headers
           // would let a stalled body hang for the full process lifetime.
-          let body = '';
-          res.on('data', (chunk: Buffer) => { body += chunk; });
+          const collected = new Utf8Body();
+          res.on('data', (chunk: Buffer) => { collected.push(chunk); });
           res.on('end', () => {
             if (resolved) return;
             resolved = true;
             clearTimeout(timer);
             try {
-              const parsed = JSON.parse(body) as T;
+              const parsed = JSON.parse(collected.toString()) as T;
               if (res.statusCode !== 200) {
                 reject(new Error(`[EmbeddingClient] GET ${path} returned HTTP ${res.statusCode}`));
               } else {
                 resolve(parsed);
               }
             } catch (_e) {
-              reject(new Error(`[EmbeddingClient] Failed to parse GET ${path} response: ${body}`));
+              reject(new Error(`[EmbeddingClient] Failed to parse GET ${path} response: ${collected.toString()}`));
             }
           });
           res.on('error', (err) => {
@@ -160,18 +173,19 @@ export class EmbeddingClient implements IEmbedder {
           headers: {
             'Content-Type': 'application/json',
             'Content-Length': Buffer.byteLength(body),
+            ...this.authHeaders(),
           },
         },
         (res) => {
           // Keep the timeout armed through the body read (see getRequest).
-          let responseBody = '';
-          res.on('data', (chunk: Buffer) => { responseBody += chunk; });
+          const collected = new Utf8Body();
+          res.on('data', (chunk: Buffer) => { collected.push(chunk); });
           res.on('end', () => {
             if (resolved) return;
             resolved = true;
             clearTimeout(timer);
             try {
-              const parsed = JSON.parse(responseBody) as T;
+              const parsed = JSON.parse(collected.toString()) as T;
               if (res.statusCode !== 200) {
                 const msg = (parsed as Record<string, unknown>)?.['error'];
                 reject(new Error(typeof msg === 'string' ? msg : `HTTP ${res.statusCode}`));
@@ -179,7 +193,7 @@ export class EmbeddingClient implements IEmbedder {
                 resolve(parsed);
               }
             } catch (_e) {
-              reject(new Error(`[EmbeddingClient] Failed to parse POST ${path} response: ${responseBody}`));
+              reject(new Error(`[EmbeddingClient] Failed to parse POST ${path} response: ${collected.toString()}`));
             }
           });
           res.on('error', (err) => {

package/src/infrastructure/embedding/embedding-factory.ts

@@ -19,7 +19,7 @@ import type { IEmbedder } from '../../core/interfaces/knowledge-interfaces.ts';
 import { Embedder } from '../../knowledge/embedder.ts';
 import { getModelEmbedderConfig } from '../../knowledge/model-config.ts';
 
-import { EmbeddingServer } from './embedding-server.ts';
+import { EmbeddingServer, getEmbeddingServerAuthSecret } from './embedding-server.ts';
 import { EmbeddingClient } from './embedding-client.ts';
 
 // ---------------------------------------------------------------------------
@@ -119,11 +119,19 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
       return _embeddingInstance;
     }
     logger.info(`[EmbeddingFactory] Configuration change detected (${_cachedModel} on ${_cachedDevice} -> ${cfg.EMBEDDING_MODEL} on ${cfg.EMBEDDING_DEVICE}). Disposing stale instance.`);
-    await _embeddingInstance.dispose?.();
+    // Claim the instance SYNCHRONOUSLY, before the first await. Two callers that
+    // resolve different EMBEDDING_MODEL/EMBEDDING_DEVICE values would otherwise both
+    // observe it, both await dispose() on the same live EmbeddingServer (double
+    // close(), double embedder dispose) and both proceed to init().
+    const stale = _embeddingInstance;
     _embeddingInstance = null;
+    await stale.dispose?.();
   }
 
-  if (_embeddingInitPromise) return _embeddingInitPromise;
+  // An internal retry (_attempt > 0) is re-entering from INSIDE the promise stored
+  // here, so handing that promise back would make it await itself. Only a fresh
+  // external call may join an in-flight init.
+  if (_embeddingInitPromise && _attempt === 0) return _embeddingInitPromise;
 
   let p: Promise<IEmbedder>;
 
@@ -166,6 +174,9 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
         pid: process.pid,
         serverId,
         model: cfg.EMBEDDING_MODEL,
+        // Published with the claim, not with the port, so a follower that adopts
+        // this leader always has the secret available whenever the port is.
+        authSecret: getEmbeddingServerAuthSecret(),
         ...(myStartTime !== null ? { startTime: myStartTime } : {}),
       };
       iAmCandidate = true;
@@ -223,7 +234,7 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
           const portOk = await isPortListening(info.port);
           if (portOk) {
             logger.info(`[EmbeddingFactory] Connecting to embedding server on port ${info.port}`);
-            const client = new EmbeddingClient(info.port);
+            const client = new EmbeddingClient(info.port, cfg.EMBEDDING_DEVICE, info.authSecret);
             await client.fetchHealth();
             _embeddingInstance = client;
             _cachedModel = cfg.EMBEDDING_MODEL;
@@ -241,8 +252,12 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
         }
       }
 
-      // Timed out or candidate died — attempt to claim candidacy ourselves (tail-call)
-      _embeddingInitPromise = null;
+      // Timed out or candidate died — attempt to claim candidacy ourselves (tail-call).
+      // The in-flight marker is deliberately LEFT in place: clearing it here, while
+      // this very promise is still running, let a concurrent getEmbedder() see no
+      // instance and no init in flight and start a second, competing election in the
+      // same process. The retry re-enters with _attempt > 0, which skips the join
+      // above, so leaving it set cannot deadlock.
       if (_attempt + 1 >= MAX_LEADER_WAIT_RETRIES) {
         throw new Error(
           `[EmbeddingFactory] Could not obtain an embedding server after ${MAX_LEADER_WAIT_RETRIES} attempts ` +
@@ -297,11 +312,13 @@ export async function getEmbedder(config?: Config, _attempt = 0): Promise<IEmbed
   // between init resolving and this microtask, re-assigning would resurrect the
   // disposed instance and hand it to the next getEmbedder() caller. Only clear
   // the in-flight promise marker.
-  p.then(() => {
-    _embeddingInitPromise = null;
-  }).catch(() => {
-    _embeddingInitPromise = null;
-  });
+  // Clear the marker only if it is STILL ours. An unconditional clear let a settling
+  // promise wipe a newer caller's in-flight marker — admitting yet another election,
+  // and cascading. Same guard as scheduler-factory's initialization promise.
+  const clearIfCurrent = (): void => {
+    if (_embeddingInitPromise === p) _embeddingInitPromise = null;
+  };
+  p.then(clearIfCurrent).catch(clearIfCurrent);
   return p;
 }
 

package/src/infrastructure/embedding/embedding-server.ts

@@ -6,15 +6,49 @@
  */
 
 import * as http from 'node:http';
+import * as crypto from 'node:crypto';
 import * as path from 'node:path';
 import { logger } from '../../logger.ts';
 import { captureStdio } from '../../utils/stdio-capture.ts';
 import { buildDefaultDebugLogPath } from '../../utils/log-utils.ts';
 import { DiskSpaceChecker } from '../../utils/disk-space-checker.ts';
+import { Utf8Body } from '../../utils/http-body.ts';
 import type { IEmbedder } from '../../core/interfaces/knowledge-interfaces.ts';
 import type { IStateManager } from '../../core/interfaces/state-manager-interfaces.ts';
 import type { Embedder } from '../../knowledge/embedder.ts';
 
+/**
+ * Per-process shared secret every embed request must present.
+ *
+ * The server binds 127.0.0.1, but loopback is not an authorization boundary —
+ * any local process could otherwise enqueue embedding work. That matters more
+ * than raw GPU cost: the serial queue steps the leader down when it poisons, so
+ * sustained abuse denies the knowledge store cluster-wide. The browser server has
+ * had this since #21; the embedding server was the asymmetric gap.
+ *
+ * Generated lazily and held in memory only — it is published to followers through
+ * the state file, which is already 0o600 and already carries the browser secret.
+ */
+let _embeddingAuthSecret: string | null = null;
+
+export function getEmbeddingServerAuthSecret(): string {
+  if (!_embeddingAuthSecret) {
+    _embeddingAuthSecret = crypto.randomBytes(32).toString('hex');
+  }
+  return _embeddingAuthSecret;
+}
+
+/**
+ * Constant-time comparison of a presented header against the expected secret.
+ * Length is compared first because timingSafeEqual throws on a length mismatch.
+ */
+function isAuthorized(presented: string | string[] | undefined): boolean {
+  const header = Array.isArray(presented) ? presented[0] : presented;
+  const expected = Buffer.from(getEmbeddingServerAuthSecret(), 'utf8');
+  const actual = Buffer.from(header ?? '', 'utf8');
+  return actual.length === expected.length && crypto.timingSafeEqual(actual, expected);
+}
+
 // ---------------------------------------------------------------------------
 // SerialQueue — ensures embed/embedMany never run concurrently on the GPU
 // ---------------------------------------------------------------------------
@@ -428,13 +462,27 @@ export class EmbeddingServer implements IEmbedder {
   // ---- HTTP request handler ----
 
   private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    // Authorize BEFORE reading the body, so an unauthenticated caller cannot make
+    // us buffer up to MAX_BODY_SIZE, and before routing, so it cannot probe
+    // leadership/health state either. Mirrors the browser server.
+    if (!isAuthorized(req.headers['x-embedding-auth'])) {
+      res.writeHead(403, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: 'Unauthorized' }));
+      req.resume(); // drain so the socket is released rather than left half-read
+      return;
+    }
+
     const MAX_BODY_SIZE = 50 * 1024 * 1024; // 50 MB
 
-    let body = '';
+    // Buffered, then decoded once: `body += chunk` would split multi-byte UTF-8
+    // across chunk boundaries and silently replace both halves with U+FFFD — and
+    // the text arriving here is exactly what gets embedded, so that corruption
+    // would be baked into the vectors. See Utf8Body.
+    const collected = new Utf8Body();
     let oversized = false;
     req.on('data', (chunk: Buffer) => {
-      body += chunk;
-      if (body.length > MAX_BODY_SIZE) {
+      collected.push(chunk);
+      if (collected.byteLength > MAX_BODY_SIZE) {
         oversized = true;
         req.destroy(new Error('Payload too large'));
       }
@@ -459,6 +507,19 @@ export class EmbeddingServer implements IEmbedder {
         finish();
       });
 
+      // A client that goes away mid-request emits 'aborted'/'close' and does NOT
+      // reliably emit 'error', so neither handler above would fire and this promise
+      // would dangle forever — retaining req, res and the buffered body for the rest
+      // of the process's life. That is not an exotic case: EmbeddingClient destroys
+      // the request on its own 120s timeout, which is the same as this server's
+      // SerialQueue soft timeout, so a slow embed reliably produces one. handleRequest
+      // is invoked with `void`, so nothing else would ever observe the leak.
+      // On a normal request this fires after 'end' and finish() is already a no-op;
+      // nothing follows this await (the caller invokes handleRequest with `void`), so
+      // an early resolve is inert either way — the 'end' handler still writes the
+      // response on its own.
+      req.on('close', finish);
+
       req.on('end', async () => {
         try {
           if (req.method === 'GET' && req.url === '/health') {
@@ -480,7 +541,7 @@ export class EmbeddingServer implements IEmbedder {
             return;
           }
 
-          const data = JSON.parse(body);
+          const data = JSON.parse(collected.toString());
 
           switch (req.url) {
             case '/embed': {