@remnic/core 9.3.765 → 9.3.766

package/src/access-http-cancellation.test.ts

@@ -0,0 +1,240 @@
+import assert from "node:assert/strict";
+import { request as httpRequest } from "node:http";
+import test from "node:test";
+
+import { EngramAccessHttpServer } from "./access-http.js";
+import { EngramAccessService, type EngramAccessRecallRequest } from "./access-service.js";
+
+function deferred<T = void>(): {
+  promise: Promise<T>;
+  resolve: (value: T | PromiseLike<T>) => void;
+  reject: (reason?: unknown) => void;
+} {
+  let resolve!: (value: T | PromiseLike<T>) => void;
+  let reject!: (reason?: unknown) => void;
+  const promise = new Promise<T>((resolvePromise, rejectPromise) => {
+    resolve = resolvePromise;
+    reject = rejectPromise;
+  });
+  return { promise, resolve, reject };
+}
+
+function waitFor<T>(promise: Promise<T>, timeoutMs = 1_000): Promise<T> {
+  return Promise.race([
+    promise,
+    new Promise<never>((_resolve, reject) => {
+      const timer = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs);
+      timer.unref?.();
+    }),
+  ]);
+}
+
+test("HTTP recall aborts in-flight work when the client disconnects", async () => {
+  const recallStarted = deferred<void>();
+  const recallSettled = deferred<void>();
+  const forceRelease = deferred<void>();
+  let observedSignal: AbortSignal | undefined;
+  const service = {
+    recall: async (input: EngramAccessRecallRequest) => {
+      observedSignal = input.abortSignal;
+      recallStarted.resolve();
+      try {
+        await Promise.race([
+          forceRelease.promise,
+          new Promise<void>((_resolve, reject) => {
+            input.abortSignal?.addEventListener(
+              "abort",
+              () => reject(input.abortSignal?.reason),
+              { once: true },
+            );
+          }),
+        ]);
+      } finally {
+        recallSettled.resolve();
+      }
+      if (input.abortSignal?.aborted) {
+        throw input.abortSignal.reason;
+      }
+      return {} as never;
+    },
+  } as unknown as EngramAccessService;
+  const server = new EngramAccessHttpServer({
+    service,
+    port: 0,
+    authToken: "test-token",
+    adminConsoleEnabled: false,
+  });
+  const status = await server.start();
+
+  try {
+    const client = httpRequest({
+      host: "127.0.0.1",
+      port: status.port,
+      path: "/engram/v1/recall",
+      method: "POST",
+      headers: {
+        authorization: "Bearer test-token",
+        "content-type": "application/json",
+      },
+    });
+    client.on("error", () => {});
+    client.end(JSON.stringify({ query: "slow recall" }));
+
+    await waitFor(recallStarted.promise);
+    assert.ok(observedSignal, "the HTTP request signal must reach the recall service");
+    client.destroy();
+
+    await waitFor(recallSettled.promise);
+    assert.equal(observedSignal.aborted, true);
+  } finally {
+    forceRelease.resolve();
+    await server.stop();
+  }
+});
+
+test("a disconnected recall queued on the principal budget lock never starts", async () => {
+  const firstStarted = deferred<void>();
+  const releaseFirst = deferred<void>();
+  let secondStarted = false;
+  const service = Object.create(EngramAccessService.prototype) as EngramAccessService;
+  const lockHost = service as unknown as {
+    budgetLocks: Map<string, Promise<void>>;
+    withBudgetLock<T>(
+      principal: string,
+      abortSignal: AbortSignal | undefined,
+      operation: () => Promise<T>,
+    ): Promise<T>;
+  };
+  lockHost.budgetLocks = new Map();
+
+  const first = lockHost.withBudgetLock("principal", undefined, async () => {
+    firstStarted.resolve();
+    await releaseFirst.promise;
+  });
+  await firstStarted.promise;
+
+  const controller = new AbortController();
+  const second = lockHost.withBudgetLock("principal", controller.signal, async () => {
+    secondStarted = true;
+  });
+  controller.abort();
+  releaseFirst.resolve();
+
+  await first;
+  await assert.rejects(waitFor(second), (error: Error) => error.name === "AbortError");
+  assert.equal(secondStarted, false);
+});
+
+test("an aborted queued recall does not poison the principal budget lock", async () => {
+  const firstStarted = deferred<void>();
+  const releaseFirst = deferred<void>();
+  let secondStarted = false;
+  let thirdStarted = false;
+  const service = Object.create(EngramAccessService.prototype) as EngramAccessService;
+  const lockHost = service as unknown as {
+    budgetLocks: Map<string, Promise<void>>;
+    withBudgetLock<T>(
+      principal: string,
+      abortSignal: AbortSignal | undefined,
+      operation: () => Promise<T>,
+    ): Promise<T>;
+  };
+  lockHost.budgetLocks = new Map();
+
+  const first = lockHost.withBudgetLock("principal", undefined, async () => {
+    firstStarted.resolve();
+    await releaseFirst.promise;
+  });
+  await firstStarted.promise;
+
+  const controller = new AbortController();
+  const second = lockHost.withBudgetLock("principal", controller.signal, async () => {
+    secondStarted = true;
+  });
+  const third = lockHost.withBudgetLock("principal", undefined, async () => {
+    thirdStarted = true;
+  });
+  controller.abort();
+  releaseFirst.resolve();
+
+  await first;
+  await assert.rejects(waitFor(second), (error: Error) => error.name === "AbortError");
+  await waitFor(third);
+  assert.equal(secondStarted, false);
+  assert.equal(thirdStarted, true);
+});
+
+function makeRecallServiceProbe(): {
+  service: EngramAccessService;
+  setExecuteRecall: (execute: () => Promise<void>) => void;
+  requestFingerprint: () => unknown;
+  stored: () => boolean;
+} {
+  const service = Object.create(EngramAccessService.prototype) as EngramAccessService;
+  let executeRecall = async () => {};
+  let capturedFingerprint: unknown;
+  let didStore = false;
+  const host = service as unknown as {
+    budgetLocks: Map<string, Promise<void>>;
+    orchestrator: { config: Record<string, unknown> };
+    resolveRequestPrincipal: () => string;
+    executeRecall: () => Promise<{ response: Record<string, never>; budgetRecordPrincipal: null }>;
+    handleIdempotentRead: (options: {
+      requestFingerprint: unknown;
+      execute: () => Promise<Record<string, never>>;
+    }) => Promise<Record<string, never>>;
+  };
+  host.budgetLocks = new Map();
+  host.orchestrator = { config: {} };
+  host.resolveRequestPrincipal = () => "principal";
+  host.executeRecall = async () => {
+    await executeRecall();
+    return { response: {}, budgetRecordPrincipal: null };
+  };
+  host.handleIdempotentRead = async (options) => {
+    capturedFingerprint = options.requestFingerprint;
+    const response = await options.execute();
+    didStore = true;
+    return response;
+  };
+  return {
+    service,
+    setExecuteRecall: (execute) => {
+      executeRecall = execute;
+    },
+    requestFingerprint: () => capturedFingerprint,
+    stored: () => didStore,
+  };
+}
+
+test("recall excludes its abort signal from the idempotency fingerprint", async () => {
+  const probe = makeRecallServiceProbe();
+  await probe.service.recall({
+    query: "same payload",
+    idempotencyKey: "same-key",
+    abortSignal: new AbortController().signal,
+  });
+
+  assert.equal(
+    Object.hasOwn(probe.requestFingerprint() as object, "abortSignal"),
+    false,
+  );
+});
+
+test("recall does not store a result when the pipeline consumes an abort", async () => {
+  const probe = makeRecallServiceProbe();
+  const controller = new AbortController();
+  probe.setExecuteRecall(async () => {
+    controller.abort();
+  });
+
+  await assert.rejects(
+    probe.service.recall({
+      query: "cancelled",
+      idempotencyKey: "cancelled-key",
+      abortSignal: controller.signal,
+    }),
+    (error: Error) => error.name === "AbortError",
+  );
+  assert.equal(probe.stored(), false);
+});

package/src/access-http.ts

@@ -7,6 +7,7 @@ import path from "node:path";
 import { fileURLToPath, URL } from "node:url";
 import { gunzipSync } from "node:zlib";
 import { log } from "./logger.js";
+import { abortError, isAbortError } from "./abort-error.js";
 import { EngramAccessInputError, type EngramAccessService, type EngramAccessMemoryResponse, type EngramAccessWriteResponse } from "./access-service.js";
 import { CorrectionContractError } from "./correction/correction-contract.js";
 import { WearablesInputError } from "./wearables/errors.js";
@@ -38,6 +39,12 @@ import { handleChatMessage, handleChatEventsSSE } from "./chat/chat-http.js";
 // server lifecycle (issue #1685 item 1 / #1687 Thread 21).
 import { cleanupExpiredChatSessions } from "./chat/chat-session.js";
 
+export interface AccessHttpReadinessState {
+  ready: boolean;
+  warmupAttempts: number;
+  lastError?: string | null;
+}
+
 export interface EngramAccessHttpServerOptions {
   service: EngramAccessService;
   host?: string;
@@ -66,6 +73,11 @@ export interface EngramAccessHttpServerOptions {
   emitLegacyTools?: boolean;
   /** Optional authenticated admin dashboard/config controls supplied by the host server. */
   adminControls?: RemnicAdminControls;
+  /**
+   * Standalone readiness state. Defaults to ready so embedded hosts keep their
+   * existing health behavior.
+   */
+  readiness?: () => AccessHttpReadinessState;
 }
 
 export interface EngramAccessHttpServerStatus {
@@ -279,6 +291,7 @@ export class EngramAccessHttpServer {
   private readonly adminControls?: RemnicAdminControls;
   private readonly trustPrincipalHeader: boolean;
   private readonly adapterRegistry: AdapterRegistry | null;
+  private readonly readiness: () => AccessHttpReadinessState;
   private readonly writeRequestTimestamps: number[] = [];
   private readonly mcpServer: EngramMcpServer;
   private server: Server | null = null;
@@ -319,6 +332,7 @@ export class EngramAccessHttpServer {
     this.adminConsolePrefillToken = options.adminConsolePrefillToken === true ? this.authToken : undefined;
     this.adminControls = options.adminControls;
     this.trustPrincipalHeader = options.trustPrincipalHeader === true;
+    this.readiness = options.readiness ?? (() => ({ ready: true, warmupAttempts: 0 }));
     this.adapterRegistry = options.enableAdapters !== false
       ? (options.adapterRegistry ?? new AdapterRegistry())
       : null;
@@ -344,8 +358,22 @@ export class EngramAccessHttpServer {
 
     const server = createServer((req, res) => {
       const correlationId = randomUUID();
+      const abortController = new AbortController();
+      const abortDisconnectedRequest = () => {
+        if (!res.writableFinished && !abortController.signal.aborted) {
+          abortController.abort(abortError("HTTP client disconnected"));
+        }
+      };
+      req.once("aborted", abortDisconnectedRequest);
+      res.once("close", abortDisconnectedRequest);
       correlationIdStore.run(correlationId, () => {
-        void this.handle(req, res, correlationId).catch((err) => {
+        void this.handle(req, res, correlationId, abortController.signal).catch((err) => {
+          if (isAbortError(err)) {
+            if (!res.destroyed && !res.writableEnded) {
+              res.end();
+            }
+            return;
+          }
           log.debug(`engram access HTTP request failed [${correlationId}]: ${err}`);
           if (err instanceof HttpError) {
             const payload: Record<string, unknown> = { error: err.message, code: err.code };
@@ -370,6 +398,9 @@ export class EngramAccessHttpServer {
             err,
           );
           this.respondJson(res, 500, { error: "internal_error", code: "internal_error" });
+        }).finally(() => {
+          req.off("aborted", abortDisconnectedRequest);
+          res.off("close", abortDisconnectedRequest);
         });
       });
     });
@@ -583,7 +614,12 @@ export class EngramAccessHttpServer {
     return queryDisclosure;
   }
 
-  private async handle(req: IncomingMessage, res: ServerResponse, correlationId: string): Promise<void> {
+  private async handle(
+    req: IncomingMessage,
+    res: ServerResponse,
+    correlationId: string,
+    abortSignal: AbortSignal,
+  ): Promise<void> {
     const parsed = new URL(req.url ?? "/", `http://${hostToUrlAuthority(this.host)}`);
     const pathname = parsed.pathname;
 
@@ -591,6 +627,21 @@ export class EngramAccessHttpServer {
       return;
     }
 
+    if (req.method === "GET" && pathname === "/engram/v1/health") {
+      const readiness = this.readiness();
+      if (!readiness.ready) {
+        this.respondJson(res, 503, {
+          ok: false,
+          ready: false,
+          warmupAttempts: readiness.warmupAttempts,
+          lastError: readiness.lastError ?? null,
+          code: "not_ready",
+        });
+        return;
+      }
+
+    }
+
     if (!this.isAuthorized(req, pathname)) {
       const body = JSON.stringify({ error: "unauthorized", code: "unauthorized" });
       res.writeHead(401, {
@@ -767,6 +818,7 @@ export class EngramAccessHttpServer {
         ...(tags !== undefined ? { tags } : {}),
         ...(tagMatch !== undefined ? { tagMatch } : {}),
         ...(includeLowConfidence ? { includeLowConfidence: true } : {}),
+        abortSignal,
       });
       this.respondJson(res, 200, response);
       return;

package/src/access-recall-surface.ts

@@ -823,6 +823,7 @@ export class AccessRecallSurface {
       ...(authenticatedPrincipal ? { principalOverride: authenticatedPrincipal } : {}),
       ...(asOf !== undefined ? { asOf } : {}),
       ...(request.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
+      ...(request.abortSignal ? { abortSignal: request.abortSignal } : {}),
     };
     const startedAt = Date.now();
     const context = await this.deps.orchestrator.recall(query, request.sessionKey, recallOptions);

package/src/access-service.ts

@@ -5,6 +5,7 @@ import { readdirSync, unlinkSync } from "node:fs";
 import { createHash } from "node:crypto";
 import { ZodError } from "zod";
 import { AccessIdempotencyStore, hashAccessIdempotencyPayload } from "./access-idempotency.js";
+import { throwIfAborted } from "./abort-error.js";
 import { resolveNamespaceCapabilities,
   resolveMemoryLifecycleCapabilities,
   resolveQmdCapabilities,
@@ -401,12 +402,10 @@ export interface EngramAccessRecallRequest {
   topK?: number;
   mode?: RecallPlanMode | "auto";
   includeDebug?: boolean;
+  abortSignal?: AbortSignal;
   /**
-   * Recall disclosure depth (issue #677).  Selects how much content each
-   * result returns: `"chunk"` (default), `"section"`, or `"raw"`.  Omitting
-   * this field is equivalent to passing `"chunk"` and preserves pre-#677
-   * behavior.  Surfaces (CLI / HTTP / MCP) and per-level token telemetry
-   * are wired in subsequent PRs of #677.
+   * Recall disclosure depth. Omitting it preserves the `"chunk"` default.
+   * Other accepted values are `"section"` and `"raw"`.
    */
   disclosure?: RecallDisclosure;
   /**
@@ -2421,7 +2420,7 @@ export class EngramAccessService {
     }
   }
 
-  private async withBudgetLock<T>(principal: string, fn: () => Promise<T>): Promise<T> {
+  private async withBudgetLock<T>(principal: string, abortSignal: AbortSignal | undefined, fn: () => Promise<T>): Promise<T> {
     const key = principal || "__anonymous__";
     const previous = this.budgetLocks.get(key) ?? Promise.resolve();
     let release!: () => void;
@@ -2430,9 +2429,9 @@ export class EngramAccessService {
     });
     const queued = previous.then(() => current, () => current);
     this.budgetLocks.set(key, queued);
-
     await previous.catch(() => {});
     try {
+      throwIfAborted(abortSignal);
       return await fn();
     } finally {
       release();
@@ -2736,6 +2735,7 @@ export class EngramAccessService {
       throw new EngramAccessInputError("query is required");
     }
     const normalizedRequest = { ...request, query };
+    const { abortSignal: _abortSignal, ...requestFingerprint } = normalizedRequest;
     const authenticatedPrincipal = request.authenticatedPrincipal?.trim();
     const principal = this.resolveRequestPrincipal(request.sessionKey, authenticatedPrincipal);
     if (resolveNamespaceCapabilities(this.orchestrator.config).namespaces && !principal) {
@@ -2743,15 +2743,15 @@ export class EngramAccessService {
         "authentication required: namespaces are enabled and no principal was supplied",
       );
     }
-    const budgetLockPrincipal = principal ?? "default";
-    return this.withBudgetLock(budgetLockPrincipal, async () => {
+    return this.withBudgetLock(principal ?? "default", request.abortSignal, async () => {
       let budgetRecordPrincipal: string | null = null;
       const response = await this.handleIdempotentRead({
         operation: "recall",
         idempotencyKey: request.idempotencyKey,
-        requestFingerprint: normalizedRequest,
+        requestFingerprint,
         execute: async () => {
           const result = await this.executeRecall(normalizedRequest);
+          throwIfAborted(request.abortSignal);
           budgetRecordPrincipal = result.budgetRecordPrincipal;
           return result.response;
         },

package/src/search/lancedb-backend.ts

@@ -1,5 +1,6 @@
 import { log } from "../logger.js";
 import {
+  reportSearchDegradation,
   resolveEnsureCollectionArgs,
   type SearchBackend,
   type SearchExecutionOptions,
@@ -491,6 +492,13 @@ export class LanceDbBackend implements SearchBackend {
       }
     } catch (err) {
       log.debug(`LanceDbBackend search (${mode}) failed: ${err}`);
+      if (!isSearchAborted(execution)) {
+        reportSearchDegradation(execution, {
+          backend: "lancedb",
+          code: "backend_error",
+          detail: err instanceof Error ? err.name : typeof err,
+        });
+      }
       return [];
     }
   }

package/src/search/meilisearch-backend.ts

@@ -1,5 +1,6 @@
 import { log } from "../logger.js";
 import {
+  reportSearchDegradation,
   resolveEnsureCollectionArgs,
   type SearchBackend,
   type SearchExecutionOptions,
@@ -282,6 +283,13 @@ export class MeilisearchBackend implements SearchBackend {
     } catch (err) {
       log.debug(`MeilisearchBackend search failed: ${err}`);
       if (rethrow) throw err;
+      if (!isSearchAborted(execution)) {
+        reportSearchDegradation(execution, {
+          backend: "meilisearch",
+          code: "backend_error",
+          detail: err instanceof Error ? err.name : typeof err,
+        });
+      }
       return [];
     }
   }

package/src/search/orama-backend.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import { mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises";
 import { log } from "../logger.js";
 import {
+  reportSearchDegradation,
   resolveEnsureCollectionArgs,
   type SearchBackend,
   type SearchExecutionOptions,
@@ -581,6 +582,13 @@ export class OramaBackend implements SearchBackend {
       }));
     } catch (err) {
       log.debug(`OramaBackend search (${mode}) failed: ${err}`);
+      if (!isSearchAborted(execution)) {
+        reportSearchDegradation(execution, {
+          backend: "orama",
+          code: "backend_error",
+          detail: err instanceof Error ? err.name : typeof err,
+        });
+      }
       return [];
     }
   }

package/src/search/port.ts

@@ -19,13 +19,15 @@ export interface SearchQueryOptions {
  * rule 34).
  */
 export interface SearchDegradation {
-  backend: "qmd";
+  backend: "qmd" | "remote" | "meilisearch" | "orama" | "lancedb";
   code:
     | "backend_unavailable"
+    | "backend_error"
     | "daemon_timeout"
     | "daemon_loading"
     | "subprocess_error"
-    | "deadline_exceeded";
+    | "deadline_exceeded"
+    | "remote_error";
   detail?: string;
 }
 
@@ -41,6 +43,17 @@ export interface SearchExecutionOptions {
   onDegradation?: (degradation: SearchDegradation) => void;
 }
 
+export function reportSearchDegradation(
+  execution: SearchExecutionOptions | undefined,
+  degradation: SearchDegradation,
+): void {
+  try {
+    execution?.onDegradation?.(degradation);
+  } catch {
+    // Observability must never break search.
+  }
+}
+
 export function resolveEnsureCollectionArgs(
   collectionOrExecution?: string | SearchExecutionOptions,
   execution?: SearchExecutionOptions,

package/src/search/remote-backend.ts

@@ -1,5 +1,11 @@
 import { log } from "../logger.js";
-import type { SearchBackend, SearchExecutionOptions, SearchQueryOptions, SearchResult } from "./port.js";
+import {
+  reportSearchDegradation,
+  type SearchBackend,
+  type SearchExecutionOptions,
+  type SearchQueryOptions,
+  type SearchResult,
+} from "./port.js";
 
 export interface RemoteSearchBackendOptions {
   baseUrl: string;
@@ -103,7 +109,13 @@ export class RemoteSearchBackend implements SearchBackend {
     body: Record<string, unknown>,
     execution?: SearchExecutionOptions,
   ): Promise<SearchResult[]> {
-    if (!this.available) return [];
+    if (!this.available) {
+      reportSearchDegradation(execution, {
+        backend: "remote",
+        code: "backend_unavailable",
+      });
+      return [];
+    }
     try {
       const res = await fetch(`${this.baseUrl}${endpoint}`, {
         method: "POST",
@@ -115,13 +127,30 @@ export class RemoteSearchBackend implements SearchBackend {
       });
       if (!res.ok) {
         log.debug(`RemoteSearchBackend ${endpoint} returned ${res.status}`);
+        reportSearchDegradation(execution, {
+          backend: "remote",
+          code: "remote_error",
+          detail: `HTTP ${res.status}`,
+        });
         return [];
       }
       const data = await res.json();
-      if (!Array.isArray(data)) return [];
+      if (!Array.isArray(data)) {
+        reportSearchDegradation(execution, {
+          backend: "remote",
+          code: "remote_error",
+          detail: "invalid response body",
+        });
+        return [];
+      }
       return data as SearchResult[];
     } catch (err) {
       log.debug(`RemoteSearchBackend ${endpoint} failed: ${err}`);
+      reportSearchDegradation(execution, {
+        backend: "remote",
+        code: "remote_error",
+        detail: err instanceof Error ? err.name : typeof err,
+      });
       return [];
     }
   }