@remnic/core 9.7.15 → 9.9.0

package/src/access-observe-write-surface.ts

@@ -11,6 +11,7 @@
  */
 
 import { createHash } from "node:crypto";
+import { buildAccessWriteRequestFingerprint } from "./write-envelope.js";
 import { FileCalendarSource, buildBriefing, parseBriefingFocus, parseBriefingWindow } from "./briefing.js";
 import {
   resolveCompressionCapabilities,
@@ -31,9 +32,11 @@ import type { MemoryActionOutcome, MemoryActionType } from "./types.js";
 import { exportWorkBoardMarkdown, exportWorkBoardSnapshot, importWorkBoardSnapshot } from "./work/board.js";
 import { wrapWorkLayerContext } from "./work/boundary.js";
 import { WorkStorage } from "./work/storage.js";
+import { WriteQuarantineStore, type QuarantineOperation } from "./write-quarantine.js";
 import {
   ENGRAM_ACCESS_WRITE_SCHEMA_VERSION,
   EngramAccessInputError,
+  NamespaceNotWritableError,
   type CodingScopedWriteInput,
   type EngramAccessBriefingRequest,
   type EngramAccessBriefingResponse,
@@ -117,8 +120,50 @@ export interface AccessObserveWriteSurfaceDeps {
 }
 
 export class AccessObserveWriteSurface {
+  private quarantineStoreInstance?: WriteQuarantineStore;
+
   constructor(private readonly deps: AccessObserveWriteSurfaceDeps) {}
 
+  private quarantineStore(): WriteQuarantineStore {
+    if (!this.quarantineStoreInstance) {
+      this.quarantineStoreInstance = new WriteQuarantineStore(
+        this.deps.orchestrator.config.memoryDir,
+      );
+    }
+    return this.quarantineStoreInstance;
+  }
+
+  /**
+   * Dead-letter a write the namespace ACL just rejected (issue #1888). Only
+   * acts on {@link NamespaceNotWritableError}; other input errors pass through
+   * untouched. Parking is best-effort — a quarantine failure is logged and
+   * swallowed so it never masks or replaces the original loud rejection, which
+   * the caller re-throws. The ACL placement is unchanged; the payload simply
+   * stops being destroyed.
+   */
+  private async parkRejectedWrite(
+    err: unknown,
+    operation: QuarantineOperation,
+    payload: unknown,
+  ): Promise<void> {
+    if (!(err instanceof NamespaceNotWritableError)) return;
+    try {
+      await this.quarantineStore().quarantine({
+        operation,
+        principal: err.principal,
+        attemptedNamespace: err.attemptedNamespace,
+        payload,
+      });
+      log.warn(
+        `quarantine: parked rejected ${operation} write for principal=${err.principal ?? "-"} attemptedNamespace=${err.attemptedNamespace} (namespace not writable); replay after fixing config`,
+      );
+    } catch (quarantineErr) {
+      log.warn(
+        `quarantine: failed to park rejected ${operation} write: ${quarantineErr instanceof Error ? quarantineErr.message : String(quarantineErr)}`,
+      );
+    }
+  }
+
   async qmdHealth(
     searchBackend: string,
     qmdEnabled: boolean,
@@ -139,6 +184,9 @@ export class AccessObserveWriteSurface {
         upgradeAvailable: null,
         doctorAvailable: null,
         debugStatus: searchBackend !== "qmd" ? `backend=${searchBackend}` : "backend=disabled",
+        pendingEmbeddings: null,
+        oldestPendingAgeMs: null,
+        embeddingBacklogThreshold: this.deps.orchestrator.config.qmdEmbeddingBacklogThreshold,
       };
     }
 
@@ -162,6 +210,9 @@ export class AccessObserveWriteSurface {
         upgradeAvailable: null,
         doctorAvailable: null,
         debugStatus: "backend=unavailable",
+        pendingEmbeddings: null,
+        oldestPendingAgeMs: null,
+        embeddingBacklogThreshold: this.deps.orchestrator.config.qmdEmbeddingBacklogThreshold,
       };
     }
     const diagnosticAvailable = await this.deps.qmdProbeAvailable(searchBackend, qmdEnabled);
@@ -170,7 +221,7 @@ export class AccessObserveWriteSurface {
       ? await this.deps.qmdCollectionState(searchBackend, qmdEnabled, collection)
       : "unknown";
     const active = operationalAvailable && collectionState !== "missing";
-    const degraded =
+    let degraded =
       searchBackend === "qmd" && qmdEnabled && (!active || !diagnosticAvailable || collectionState === "unknown");
     const debugStatus = qmd.debugStatus();
     const versionStatus =
@@ -188,6 +239,27 @@ export class AccessObserveWriteSurface {
               ? "daemon"
               : "cli";
 
+    const threshold = this.deps.orchestrator.config.qmdEmbeddingBacklogThreshold;
+    let pendingEmbeddings: number | null = null;
+    let oldestPendingAgeMs: number | null = null;
+    let degradedReason: string | undefined;
+    if (typeof (qmd as { status?: unknown }).status === "function") {
+      try {
+        const statusReport = await Promise.race([
+          (qmd as unknown as { status: () => Promise<{ pendingEmbeddings: number | null; oldestPendingAgeMs: number | null }> }).status(),
+          new Promise<null>((resolve) => setTimeout(() => resolve(null), 5_000).unref?.()),
+        ]);
+        if (statusReport) {
+          pendingEmbeddings = statusReport.pendingEmbeddings;
+          oldestPendingAgeMs = statusReport.oldestPendingAgeMs;
+        }
+      } catch { /* status probe failed — non-fatal */ }
+    }
+    if (threshold > 0 && pendingEmbeddings !== null && pendingEmbeddings > threshold) {
+      degraded = true;
+      degradedReason = `embedding-backlog: ${pendingEmbeddings} pending > threshold ${threshold}`;
+    }
+
     return {
       enabled: qmdEnabled,
       active,
@@ -201,6 +273,10 @@ export class AccessObserveWriteSurface {
       upgradeAvailable: versionStatus?.upgradeAvailable ?? null,
       doctorAvailable: versionStatus?.capabilities?.doctor ?? null,
       debugStatus,
+      pendingEmbeddings,
+      oldestPendingAgeMs,
+      embeddingBacklogThreshold: threshold,
+      ...(degradedReason ? { degradedReason } : {}),
     };
   }
 
@@ -227,6 +303,8 @@ export class AccessObserveWriteSurface {
           upgradeAvailable: boolean | null;
           doctorAvailable: boolean | null;
           daemonMode: boolean | null;
+          pendingEmbeddings?: number | null;
+          oldestPendingAgeMs?: number | null;
         }>;
       }
     ).searchHealthForNamespace;
@@ -240,7 +318,11 @@ export class AccessObserveWriteSurface {
         signal: controller.signal,
       });
       const active = health.available && health.collectionState !== "missing";
-      const degraded = !active || health.collectionState === "unknown";
+      const pending = health.pendingEmbeddings ?? null;
+      const oldest = health.oldestPendingAgeMs ?? null;
+      const threshold = this.deps.orchestrator.config.qmdEmbeddingBacklogThreshold;
+      const backlogDegraded = threshold > 0 && pending !== null && pending > threshold;
+      const degraded = !active || health.collectionState === "unknown" || backlogDegraded;
       const mode = !active ? "fallback" : health.daemonMode === true ? "daemon" : "cli";
 
       return {
@@ -256,6 +338,12 @@ export class AccessObserveWriteSurface {
         upgradeAvailable: health.upgradeAvailable,
         doctorAvailable: health.doctorAvailable,
         debugStatus: health.debugStatus,
+        pendingEmbeddings: pending,
+        oldestPendingAgeMs: oldest,
+        embeddingBacklogThreshold: threshold,
+        ...(backlogDegraded && pending !== null
+          ? { degradedReason: `embedding backlog ${pending} exceeds threshold ${threshold}` }
+          : {}),
       };
     } catch (error) {
       const detail = displayErrorDetail(error) || "unknown";
@@ -272,6 +360,9 @@ export class AccessObserveWriteSurface {
         upgradeAvailable: null,
         doctorAvailable: null,
         debugStatus: `backend=namespace-unavailable error=${detail}`,
+        pendingEmbeddings: null,
+        oldestPendingAgeMs: null,
+        embeddingBacklogThreshold: this.deps.orchestrator.config.qmdEmbeddingBacklogThreshold,
       };
     } finally {
       clearTimeout(timer);
@@ -368,7 +459,18 @@ export class AccessObserveWriteSurface {
     request: EngramAccessMemoryStoreRequest,
     hooks?: { enforceWriteQuota?: () => void | Promise<void> }
   ): Promise<EngramAccessWriteResponse> {
-    const namespace = await this.deps.resolveCodingScopedWriteNamespace(request);
+    let namespace: string;
+    try {
+      namespace = await this.deps.resolveCodingScopedWriteNamespace(request);
+    } catch (err) {
+      // A dry run is a no-persist validation, and a replay re-submit sets
+      // suppressQuarantine so a still-unwritable target propagates instead of
+      // re-parking (issue #1888): never dead-letter either.
+      if (request.dryRun !== true && request.suppressQuarantine !== true) {
+        await this.parkRejectedWrite(err, "memory_store", request);
+      }
+      throw err;
+    }
     const schemaVersion = request.schemaVersion ?? ENGRAM_ACCESS_WRITE_SCHEMA_VERSION;
     if (schemaVersion !== ENGRAM_ACCESS_WRITE_SCHEMA_VERSION) {
       throw new EngramAccessInputError(`unsupported schemaVersion: ${schemaVersion}`);
@@ -420,18 +522,20 @@ export class AccessObserveWriteSurface {
     return this.deps.handleIdempotentWrite({
       operation: "memory_store",
       idempotencyKey: request.idempotencyKey,
-      requestFingerprint: {
+      // Shared builder (issue #1989 PR3): byte-parity with the historical
+      // inline literal is asserted by access-fingerprint-parity.test.ts.
+      requestFingerprint: buildAccessWriteRequestFingerprint({
         schemaVersion,
+        namespace,
         content: request.content,
         category: request.category,
         confidence: request.confidence,
-        namespace,
         tags: request.tags,
         entityRef: request.entityRef,
         ttl: request.ttl,
         sourceReason: request.sourceReason,
         sourceConnector: request.sourceConnector,
-      },
+      }),
       skip: request.dryRun === true,
       beforeExecute: hooks?.enforceWriteQuota,
       execute,
@@ -442,7 +546,17 @@ export class AccessObserveWriteSurface {
     request: EngramAccessSuggestionSubmitRequest,
     hooks?: { enforceWriteQuota?: () => void | Promise<void> }
   ): Promise<EngramAccessWriteResponse> {
-    const namespace = await this.deps.resolveCodingScopedWriteNamespace(request);
+    let namespace: string;
+    try {
+      namespace = await this.deps.resolveCodingScopedWriteNamespace(request);
+    } catch (err) {
+      // A dry run never persists, and a replay re-submit sets suppressQuarantine
+      // so a still-unwritable target propagates instead of re-parking (#1888).
+      if (request.dryRun !== true && request.suppressQuarantine !== true) {
+        await this.parkRejectedWrite(err, "suggestion_submit", request);
+      }
+      throw err;
+    }
     const schemaVersion = request.schemaVersion ?? ENGRAM_ACCESS_WRITE_SCHEMA_VERSION;
     if (schemaVersion !== ENGRAM_ACCESS_WRITE_SCHEMA_VERSION) {
       throw new EngramAccessInputError(`unsupported schemaVersion: ${schemaVersion}`);
@@ -491,18 +605,20 @@ export class AccessObserveWriteSurface {
     return this.deps.handleIdempotentWrite({
       operation: "suggestion_submit",
       idempotencyKey: request.idempotencyKey,
-      requestFingerprint: {
+      // Shared builder (issue #1989 PR3): byte-parity with the historical
+      // inline literal is asserted by access-fingerprint-parity.test.ts.
+      requestFingerprint: buildAccessWriteRequestFingerprint({
         schemaVersion,
+        namespace,
         content: request.content,
         category: request.category,
         confidence: request.confidence,
-        namespace,
         tags: request.tags,
         entityRef: request.entityRef,
         ttl: request.ttl,
         sourceReason: request.sourceReason,
         sourceConnector: request.sourceConnector,
-      },
+      }),
       skip: request.dryRun === true,
       beforeExecute: hooks?.enforceWriteQuota,
       execute,
@@ -538,7 +654,17 @@ export class AccessObserveWriteSurface {
     //    self base leaves NO coding context bound to the session, matching how
     //    `memory_store` resolves its full scoped write namespace before any
     //    session mutation.
-    const scope = await this.deps.resolveMemoryScopePlan(request);
+    let scope: MemoryScopePlan;
+    try {
+      scope = await this.deps.resolveMemoryScopePlan(request);
+    } catch (err) {
+      // A replay re-submit sets suppressQuarantine so a still-unwritable target
+      // propagates instead of re-parking (#1888); observe has no dryRun path.
+      if (request.suppressQuarantine !== true) {
+        await this.parkRejectedWrite(err, "observe", request);
+      }
+      throw err;
+    }
     const writeNamespace = scope.writeNamespace;
 
     // Backward-compatible BASE writable namespace (pre-#1495 response semantics)

package/src/access-operations-batch.ts

@@ -18,6 +18,7 @@ import { EngramAccessForbiddenError } from "./access-errors.js";
 import { EngramAccessInputError, type EngramAccessService } from "./access-service.js";
 import { enforceNamespaceAllowList, tokenCapabilityStore } from "./access-token-capabilities.js";
 import { projectTagProjectId } from "./coding/coding-namespace.js";
+import { resolveNamespacePreflight } from "./scopes/scope-plan.js";
 import { expandTildePath } from "./utils/path.js";
 import { resolvePrincipal } from "./namespaces/principal.js";
 import { getRecallTimingStatus, isRecallTimingsOperator } from "./recall-timings.js";
@@ -113,6 +114,7 @@ defineOperation({ name: "recall_xray", description: "X-ray recall.", schema: str
     return { result: await ctx.service.recallXray({ query: defStr(input.query, ""), sessionKey: optStr(input.sessionKey), namespace: optStr(input.namespace), budget, authenticatedPrincipal: ctx.authenticatedPrincipal, ...(dr && dr !== "" ? { disclosure: dr as RecallDisclosure } : {}), ...(ctx.abortSignal ? { abortSignal: ctx.abortSignal } : {}) }) };
   },
 });
+defineOperation({ name: "namespace_writable", description: "Read-only preflight: is a namespace writable for the caller's principal?", allowedByOps: ["namespace_writable", "observe", "memory_store"], schema: strictSchema({ namespace: S.str, sessionKey: S.str, op: S.str }), handler: async (input, ctx) => { const ns = optStr(input.namespace); const sk = optStr(input.sessionKey); const writeOp = optStr(input.op) === "memory_store" ? "memory_store" : "observe"; return { result: resolveNamespacePreflight(tokenCapabilityStore.getStore(), ns && ns.length > 0 ? ns : undefined, sk && sk.length > 0 ? sk : undefined, ctx.authenticatedPrincipal, ctx.service.configRef, writeOp) }; } });
 
 // === WEARABLES ===
 defineOperation({ name: "wearables_status", description: "Wearables status.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.wearablesStatus() }) });