@remnic/core 10.0.38 → 10.0.40

package/src/storage/dropped-extraction-chunk.ts

@@ -0,0 +1,27 @@
+import { randomUUID } from "node:crypto";
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import type { BufferTurn } from "../types.js";
+
+/**
+ * Persist a copy of extraction-buffer turns that are being DROPPED after repeated extraction failures (trigger-drain drop
+ * path), so a destructive drop never loses content silently. Written under `<stateDir>/dropped/` through the store's secure
+ * writer, so encrypted stores keep it encrypted (secure-fs encrypts the whole `state/dropped/` prefix). Returns the path.
+ */
+export async function saveDroppedExtractionChunk(
+  stateDir: string,
+  writeSecureFile: (filePath: string, content: string) => Promise<void>,
+  bufferKey: string,
+  reason: string,
+  turns: readonly BufferTurn[],
+): Promise<string> {
+  const droppedDir = path.join(stateDir, "dropped");
+  await mkdir(droppedDir, { recursive: true });
+  const safeKey = bufferKey.replace(/[^a-zA-Z0-9._-]/g, "-").slice(0, 80) || "default";
+  // The random suffix keeps copies written in the same millisecond (or for keys that truncate to the same name) from
+  // overwriting each other.
+  const filePath = path.join(droppedDir, `${Date.now()}-${randomUUID().slice(0, 8)}-${safeKey}.json`);
+  const payload = { bufferKey, reason, droppedAt: new Date().toISOString(), turns };
+  await writeSecureFile(filePath, JSON.stringify(payload, null, 2));
+  return filePath;
+}

package/src/storage-dropped-chunk-filenames.test.ts

@@ -0,0 +1,30 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readdir, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { StorageManager } from "./storage.js";
+import type { BufferTurn } from "./types.js";
+
+function makeTurn(content: string): BufferTurn {
+  return { role: "user", content, timestamp: "2026-10-29T00:00:00.000Z", sessionKey: "s1" };
+}
+
+test("dropped-chunk recovery files never overwrite each other, even within one millisecond", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-dropped-chunks-"));
+  const realNow = Date.now;
+  try {
+    Date.now = () => 1_793_000_000_000;
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const first = await storage.saveDroppedExtractionChunk("same-key", "first", [makeTurn("one")]);
+    const second = await storage.saveDroppedExtractionChunk("same-key", "second", [makeTurn("two")]);
+
+    assert.notEqual(first, second);
+    const files = await readdir(path.dirname(first));
+    assert.equal(files.length, 2, "both safety copies exist");
+  } finally {
+    Date.now = realNow;
+    await rm(dir, { recursive: true, force: true });
+  }
+});

package/src/storage.ts

@@ -144,6 +144,7 @@ import {
 import { supersessionKeysForFact } from "./temporal-supersession.js";
 import { BufferSessionFiles } from "./storage/buffer-session-files.js";
 import { runSupersessionSideEffects, type SupersessionAuditOptions } from "./storage/supersession-side-effects.js";
+import { saveDroppedExtractionChunk as saveDroppedChunk } from "./storage/dropped-extraction-chunk.js";
 import type {
   AccessTrackingEntry,
   BufferEntryState,
@@ -4799,7 +4800,6 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     return results.filter((filePath): filePath is string => filePath !== null);
   }
 
-
   private orderWindowPaths(filePaths: string[]): string[] {
     const correctionPaths: string[] = [];
     const factPaths: string[] = [];
@@ -5520,6 +5520,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   async saveBuffer(state: BufferState): Promise<void> { await this.bufferSessionFiles().reconcile(state); }
   async saveBufferSession(key: string, entry: BufferEntryState): Promise<void> { await this.bufferSessionFiles().save(key, entry); }
   async deleteBufferSession(key: string): Promise<void> { await this.bufferSessionFiles().remove(key); }
+  saveDroppedExtractionChunk(bufferKey: string, reason: string, turns: readonly import("./types.js").BufferTurn[]): Promise<string> { // storage/dropped-extraction-chunk.ts
+    return saveDroppedChunk(this.stateDir, (file, content) => this.writeStorageSecureFile(file, content), bufferKey, reason, turns);
+  }
 
   async loadMeta(): Promise<MetaState> {
     const metaPath = path.join(this.stateDir, "meta.json");
@@ -5564,8 +5567,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
 
   async saveMeta(state: MetaState): Promise<void> {
     await this.ensureDirectories();
-    const metaPath = path.join(this.stateDir, "meta.json");
-    await this.writeStorageSecureFile(metaPath, JSON.stringify(state, null, 2));
+    await this.writeStorageSecureFile(path.join(this.stateDir, "meta.json"), JSON.stringify(state, null, 2));
   }
 
   async appendMemoryActionEvents(events: MemoryActionEvent[]): Promise<number> {
@@ -7208,10 +7210,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   async saveTopics(topics: TopicScore[]): Promise<void> {
     const metaPath = path.join(this.stateDir, "topics.json");
     await mkdir(this.stateDir, { recursive: true });
-    await this.writeStorageSecureFile(
-      metaPath,
-      JSON.stringify({ topics, updatedAt: new Date().toISOString() }, null, 2)
-    );
+    await this.writeStorageSecureFile(metaPath, JSON.stringify({ topics, updatedAt: new Date().toISOString() }, null, 2));
     log.debug(`saved ${topics.length} topic scores`);
   }
 

package/src/types.ts

@@ -1697,6 +1697,13 @@ export interface PluginConfig
   extractionBreakerCooldownMs: number;
   /** Breaker open cooldown (ms) for auth/config failures (longer — provider is misconfigured). */
   extractionBreakerAuthCooldownMs: number;
+  /**
+   * Consecutive failed extraction attempts after which the failing
+   * trigger-drain chunk is dropped (with a side-file copy under
+   * `state/dropped/`) so a session whose window never extracts cannot stall
+   * the buffer forever. Default: `5`.
+   */
+  extractionChunkDropAfterFailures: number;
   // Local LLM fast tier (v9.1) — smaller model for quick ops
   localLlmFastEnabled: boolean;
   localLlmFastModel: string;