@remnic/core 10.0.37 → 10.0.39

package/src/orchestration/extraction-run.ts

@@ -38,6 +38,7 @@ export interface ExtractionRunResult {
   persistedCount: number;
   durableOutputCount: number;
   postPersistMetadataFailed?: boolean;
+  retainedBuffer?: boolean; failureClass?: ExtractionFailureClass; // retained: the run left the window in the buffer for a retry (a recorded failure) of this class
 }
 
 export class ExtractionDeadlineError extends Error {
@@ -118,7 +119,7 @@ import {
   commitEmptySuccessExtraction,
   computeExtractionRetryNextEligibleMs,
   deriveSourceConnector,
-  deriveTopicsFromExtraction,
+  deriveTopicsFromExtraction, emptyExtractionResult,
   runExtractionPostPersist,
   runExtractionPostPersistBestEffort,
   EXTRACTION_RETRY_STATE_MAX_ENTRIES,
@@ -439,7 +440,7 @@ export class ExtractionRunCoordinator {
     turns: BufferTurn[],
     options: {
       clearBufferAfterExtraction?: boolean;
-      clearMatchingTurns?: boolean;
+      clearMatchingTurns?: boolean; retentionWindow?: readonly BufferTurn[];
       skipCharThreshold?: boolean;
       skipUserTurnThreshold?: boolean;
       deadlineMs?: number;
@@ -1044,7 +1045,7 @@ export class ExtractionRunCoordinator {
       if (!shouldPersistProcessedFingerprint) {
         await commitEmptySuccessExtraction(emptySuccessCommit);
       }
-      return { status: "skipped", reason: "empty_extraction_result", persistedCount: 0, durableOutputCount: 0 };
+      return emptyExtractionResult(recordedRetryFailure, result.extractionFailureClass);
     }
 
     let threadIdForExtraction: string | null = null;
@@ -1108,7 +1109,7 @@ export class ExtractionRunCoordinator {
       normalizedTurns: normalizedTurns as BufferTurn[],
       bufferKey,
       retainDeferredTurns: (key, retainedTurns, max) =>
-        this.deps.getBuffer().retainDeferredTurns(key, retainedTurns, max, turns),
+        this.deps.getBuffer().retainDeferredTurns(key, retainedTurns, max, options.retentionWindow ?? turns),
       clearBuffer,
       failOnExtractionFailure: options.failOnExtractionFailure === true,
     });

package/src/orchestration/turn-ingestion.ts

@@ -42,6 +42,7 @@ import { TranscriptManager } from "../transcript.js";
 import type { BufferTurn, PluginConfig, SourceConnectorProvenance } from "../types.js";
 
 import { groupTurnsByExtractionScope, bufferTurnsEqual } from "../buffer-turn-helpers.js";
+import { ExtractionDrain } from "./extraction-drain.js";
 import { resolveScopeProfilePlan, type ResolvedScopeProfilePlan, hasExplicitNamespacePolicy } from "../namespaces/scope-profiles.js";
 import { resolveCodingNamespaceOverlay, combineNamespaces } from "../coding/coding-namespace.js";
 import { canWriteNamespace, defaultNamespaceForPrincipal } from "../namespaces/principal.js";
@@ -114,6 +115,8 @@ export interface TurnIngestionDeps {
     },
   ): Promise<void>;
   resolveMemoryIdOrHandle(ref: string, sessionKey?: string): string;
+  /** Max conversation chars the extraction model accepts; absent on legacy hosts (single-run behavior). */
+  extractionWindowBudgetChars?(): number;
   runExtraction(
     ...args: Parameters<ExtractionRunCoordinator["runExtraction"]>
   ): Promise<ExtractionRunResult>;
@@ -156,9 +159,10 @@ function isFlushPlanRecoveryTurn(
 }
 
 export class TurnIngestionCoordinator {
-  constructor(
-    private readonly deps: TurnIngestionDeps,
-  ) {}
+  private readonly drain: ExtractionDrain; // chunked trigger-drain (extraction-drain.ts)
+  constructor(private readonly deps: TurnIngestionDeps) {
+    this.drain = new ExtractionDrain(deps);
+  }
 
   async processTurn(
     role: "user" | "assistant",
@@ -1067,7 +1071,7 @@ export class TurnIngestionCoordinator {
             effectiveScopePlan = fresh;
           }
         }
-        const result = await this.deps.runExtraction(effectiveTurns, {
+        const result = await this.drain.run(effectiveTurns, {
           clearBufferAfterExtraction:
             options.clearBufferAfterExtraction ?? true,
           clearMatchingTurns: options.clearMatchingTurns === true,
@@ -1083,7 +1087,7 @@ export class TurnIngestionCoordinator {
           principalOverride: options.principalOverride,
           scopeProfileWritePlan: effectiveScopePlan,
           forceExtractionAttempt: options.forceExtractionAttempt === true,
-        });
+        }, preemptable);
         settleTask(undefined, result);
       } catch (err) {
         const reason = abortSignal.reason;

package/src/orchestrator.ts

@@ -238,6 +238,7 @@ import {
 } from "./maintenance/pattern-reinforcement.js";
 import { refreshConvergedNamespaces } from "./convergence-refresh.js";
 import { ModelRegistry } from "./model-registry.js";
+import { extractionWindowBudgetChars } from "./extraction-window-drain.js";
 import { applyRuntimeRetrievalPolicy, expandQuery } from "./retrieval.js";
 import {
   mergeWithAgentResults,
@@ -3062,6 +3063,26 @@ export class Orchestrator {
     );
   }
 
+  /**
+   * Max conversation characters the extraction model can take as input
+   * (context window the ModelRegistry learned, capped by
+   * `localLlmMaxContext` when set). Exposed for the trigger-drain chunking
+   * in the turn-ingestion coordinator so an oversized buffer window is
+   * drained in fitting chunks instead of failing whole.
+   */
+  private extractionWindowBudgetChars(): number {
+    // Object.create(Orchestrator.prototype) test doubles skip the constructor,
+    // so neither modelRegistry nor config exists there: report "no budget".
+    if (!this.modelRegistry) return Number.POSITIVE_INFINITY;
+    return extractionWindowBudgetChars({
+      registry: this.modelRegistry,
+      localExtractionActive:
+        this.config.modelSource !== "gateway" && resolveLocalLlmCapabilities(this.config).localLlm,
+      model: this.config.localLlmModel,
+      maxContext: this.config.localLlmMaxContext,
+    });
+  }
+
   private normalizeExtractionFingerprintTurns(turns: BufferTurn[]): string[] {
     return this.extractionRunCoordinator.normalizeExtractionFingerprintTurns(turns);
   }

package/src/secure-store/secure-fs.ts

@@ -902,7 +902,9 @@ const ENCRYPTABLE_STATE_SIDECARS = new Set([
 ]);
 
 function isEncryptableStateSidecar(normalized: string): boolean {
-  return ENCRYPTABLE_STATE_SIDECARS.has(normalized);
+  // state/dropped/ holds copies of extraction-buffer turns dropped after
+  // repeated extraction failures — same content sensitivity as buffer.json.
+  return ENCRYPTABLE_STATE_SIDECARS.has(normalized) || normalized.startsWith("state/dropped/");
 }
 
 /**

package/src/storage/dropped-extraction-chunk.ts

@@ -0,0 +1,27 @@
+import { randomUUID } from "node:crypto";
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import type { BufferTurn } from "../types.js";
+
+/**
+ * Persist a copy of extraction-buffer turns that are being DROPPED after repeated extraction failures (trigger-drain drop
+ * path), so a destructive drop never loses content silently. Written under `<stateDir>/dropped/` through the store's secure
+ * writer, so encrypted stores keep it encrypted (secure-fs encrypts the whole `state/dropped/` prefix). Returns the path.
+ */
+export async function saveDroppedExtractionChunk(
+  stateDir: string,
+  writeSecureFile: (filePath: string, content: string) => Promise<void>,
+  bufferKey: string,
+  reason: string,
+  turns: readonly BufferTurn[],
+): Promise<string> {
+  const droppedDir = path.join(stateDir, "dropped");
+  await mkdir(droppedDir, { recursive: true });
+  const safeKey = bufferKey.replace(/[^a-zA-Z0-9._-]/g, "-").slice(0, 80) || "default";
+  // The random suffix keeps copies written in the same millisecond (or for keys that truncate to the same name) from
+  // overwriting each other.
+  const filePath = path.join(droppedDir, `${Date.now()}-${randomUUID().slice(0, 8)}-${safeKey}.json`);
+  const payload = { bufferKey, reason, droppedAt: new Date().toISOString(), turns };
+  await writeSecureFile(filePath, JSON.stringify(payload, null, 2));
+  return filePath;
+}

package/src/storage-dropped-chunk-filenames.test.ts

@@ -0,0 +1,30 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readdir, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { StorageManager } from "./storage.js";
+import type { BufferTurn } from "./types.js";
+
+function makeTurn(content: string): BufferTurn {
+  return { role: "user", content, timestamp: "2026-10-29T00:00:00.000Z", sessionKey: "s1" };
+}
+
+test("dropped-chunk recovery files never overwrite each other, even within one millisecond", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-dropped-chunks-"));
+  const realNow = Date.now;
+  try {
+    Date.now = () => 1_793_000_000_000;
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const first = await storage.saveDroppedExtractionChunk("same-key", "first", [makeTurn("one")]);
+    const second = await storage.saveDroppedExtractionChunk("same-key", "second", [makeTurn("two")]);
+
+    assert.notEqual(first, second);
+    const files = await readdir(path.dirname(first));
+    assert.equal(files.length, 2, "both safety copies exist");
+  } finally {
+    Date.now = realNow;
+    await rm(dir, { recursive: true, force: true });
+  }
+});

package/src/storage.ts

@@ -144,6 +144,7 @@ import {
 import { supersessionKeysForFact } from "./temporal-supersession.js";
 import { BufferSessionFiles } from "./storage/buffer-session-files.js";
 import { runSupersessionSideEffects, type SupersessionAuditOptions } from "./storage/supersession-side-effects.js";
+import { saveDroppedExtractionChunk as saveDroppedChunk } from "./storage/dropped-extraction-chunk.js";
 import type {
   AccessTrackingEntry,
   BufferEntryState,
@@ -4799,7 +4800,6 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     return results.filter((filePath): filePath is string => filePath !== null);
   }
 
-
   private orderWindowPaths(filePaths: string[]): string[] {
     const correctionPaths: string[] = [];
     const factPaths: string[] = [];
@@ -5520,6 +5520,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   async saveBuffer(state: BufferState): Promise<void> { await this.bufferSessionFiles().reconcile(state); }
   async saveBufferSession(key: string, entry: BufferEntryState): Promise<void> { await this.bufferSessionFiles().save(key, entry); }
   async deleteBufferSession(key: string): Promise<void> { await this.bufferSessionFiles().remove(key); }
+  saveDroppedExtractionChunk(bufferKey: string, reason: string, turns: readonly import("./types.js").BufferTurn[]): Promise<string> { // storage/dropped-extraction-chunk.ts
+    return saveDroppedChunk(this.stateDir, (file, content) => this.writeStorageSecureFile(file, content), bufferKey, reason, turns);
+  }
 
   async loadMeta(): Promise<MetaState> {
     const metaPath = path.join(this.stateDir, "meta.json");
@@ -5564,8 +5567,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
 
   async saveMeta(state: MetaState): Promise<void> {
     await this.ensureDirectories();
-    const metaPath = path.join(this.stateDir, "meta.json");
-    await this.writeStorageSecureFile(metaPath, JSON.stringify(state, null, 2));
+    await this.writeStorageSecureFile(path.join(this.stateDir, "meta.json"), JSON.stringify(state, null, 2));
   }
 
   async appendMemoryActionEvents(events: MemoryActionEvent[]): Promise<number> {
@@ -7208,10 +7210,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   async saveTopics(topics: TopicScore[]): Promise<void> {
     const metaPath = path.join(this.stateDir, "topics.json");
     await mkdir(this.stateDir, { recursive: true });
-    await this.writeStorageSecureFile(
-      metaPath,
-      JSON.stringify({ topics, updatedAt: new Date().toISOString() }, null, 2)
-    );
+    await this.writeStorageSecureFile(metaPath, JSON.stringify({ topics, updatedAt: new Date().toISOString() }, null, 2));
     log.debug(`saved ${topics.length} topic scores`);
   }
 

package/src/types.ts

@@ -1299,6 +1299,18 @@ export interface PluginConfig
   extractionJudgeModel: string;
   /** Maximum number of candidate facts per judge LLM batch call. */
   extractionJudgeBatchSize: number;
+  /**
+   * Per-call timeout for the judge LLM (ms). Default 20000: measured 2.1-2.5 s
+   * per compact verdict call (completion budget 24 tokens per candidate, up
+   * to 2048) on a small fast judge model; the big shared core model measured
+   * 13-22 s per compact call, 41 s verbose — raise it there. The earlier 150s
+   * default only papered over 59-305s verbose replies; sub-second budgets
+   * abort every call. On the local LLM path each attempt lasts
+   * min(localLlmTimeoutMs, extractionJudgeTimeoutMs) (localLlmTimeoutMs
+   * defaults to 180000), so a value above localLlmTimeoutMs does not lengthen
+   * an attempt: raise both settings for a slower judge model.
+   */
+  extractionJudgeTimeoutMs: number;
   /** Shadow mode: log judge verdicts but do not filter facts. Default false. */
   extractionJudgeShadow: boolean;
   /**
@@ -1685,6 +1697,13 @@ export interface PluginConfig
   extractionBreakerCooldownMs: number;
   /** Breaker open cooldown (ms) for auth/config failures (longer — provider is misconfigured). */
   extractionBreakerAuthCooldownMs: number;
+  /**
+   * Consecutive failed extraction attempts after which the failing
+   * trigger-drain chunk is dropped (with a side-file copy under
+   * `state/dropped/`) so a session whose window never extracts cannot stall
+   * the buffer forever. Default: `5`.
+   */
+  extractionChunkDropAfterFailures: number;
   // Local LLM fast tier (v9.1) — smaller model for quick ops
   localLlmFastEnabled: boolean;
   localLlmFastModel: string;