@remnic/core 9.69.87 → 9.69.89

package/src/orchestrator.ts

@@ -296,6 +296,7 @@ import { isDisagreementPrompt } from "./signal.js";
 import { lintWorkspaceFiles, rotateMarkdownFileToArchive } from "./hygiene.js";
 import { isPathInsideStorageRoot } from "./storage-paths.js";
 import { EmbeddingFallback } from "./embedding-fallback.js";
+import { storageBackedIndexStoreIo } from "./offline-sync-embedding-generation.js";
 import {
   decideSemanticDedup,
   type SemanticDedupDecision,
@@ -1635,7 +1636,7 @@ export class Orchestrator {
       debounceMs: config.sessionObserverDebounceMs ?? 120_000,
       bands: config.sessionObserverBands ?? [],
     });
-    this.embeddingFallback = new EmbeddingFallback(config);
+    this.embeddingFallback = new EmbeddingFallback(config, storageBackedIndexStoreIo(this.storage));
     this.policyRuntime = new PolicyRuntimeManager(config.memoryDir, config);
     this.summarizer = new HourlySummarizer(
       config,

package/src/secure-store/secure-fs.ts

@@ -288,7 +288,8 @@ export function filePathAad(filePath: string, memoryDir?: string): Buffer {
 export async function readMaybeEncryptedFileBuffer(
   filePath: string,
   key: Buffer | null,
-  memoryDir?: string
+  memoryDir?: string,
+  opts?: { aadFilePath?: string }
 ): Promise<Buffer> {
   const buf = await readFile(filePath);
   if (!isEncryptedFile(buf)) {
@@ -301,7 +302,7 @@ export async function readMaybeEncryptedFileBuffer(
       `secure-store is locked — cannot read encrypted file at ${filePath}. Run \`remnic secure-store unlock\` to decrypt.`
     );
   }
-  return decryptFileBodyForPath(buf, key, filePath, memoryDir);
+  return decryptFileBodyForPath(buf, key, filePath, memoryDir, opts?.aadFilePath);
 }
 
 export async function readMaybeEncryptedFile(
@@ -323,6 +324,17 @@ export interface WriteMaybeEncryptedFileOptions {
    * Default true.
    */
   atomic?: boolean;
+  /**
+   * ABSOLUTE final path whose canonical AAD (via {@link filePathAad})
+   * replaces the physical `filePath`'s. Used by the offline-sync generation
+   * transaction: staged ciphertext physically lives under a staging
+   * directory but must decrypt at its FINAL published path after the
+   * staging directory is swapped into place by rename. Deriving the AAD
+   * from the absolute final path through the SAME filePathAad logic as
+   * ordinary reads keeps write and read derivations identical on every
+   * platform (POSIX and Windows separators alike). (issue #3148, round 4/5)
+   */
+  aadFilePath?: string;
 }
 
 /**
@@ -342,12 +354,12 @@ export async function writeMaybeEncryptedFile(
   options: WriteMaybeEncryptedFileOptions = {},
   memoryDir?: string
 ): Promise<void> {
-  const { mode = 0o600, atomic = true } = options;
+  const { mode = 0o600, atomic = true, aadFilePath } = options;
   await mkdir(path.dirname(filePath), { recursive: true });
 
   let data: Buffer | string;
   if (key !== null) {
-    const aad = filePathAad(filePath, memoryDir);
+    const aad = aadFilePath !== undefined ? filePathAad(aadFilePath, memoryDir) : filePathAad(filePath, memoryDir);
     data = encryptFileBody(content, key, aad);
   } else {
     data = content;
@@ -653,8 +665,16 @@ function uniqueAtomicTempPath(filePath: string, label: string): string {
   return `${filePath}.${label}-${process.pid}-${Date.now()}-${randomUUID()}`;
 }
 
-function decryptFileBodyForPath(buf: Buffer, key: Buffer, filePath: string, memoryDir?: string): Buffer {
-  const aad = filePathAad(filePath, memoryDir);
+function decryptFileBodyForPath(
+  buf: Buffer,
+  key: Buffer,
+  filePath: string,
+  memoryDir?: string,
+  aadFilePath?: string
+): Buffer {
+  // A canonical-AAD stage read (offline-sync generation transaction) binds
+  // the FINAL published path, not the physical staging path.
+  const aad = aadFilePath !== undefined ? filePathAad(aadFilePath, memoryDir) : filePathAad(filePath, memoryDir);
   try {
     return decryptFileBody(buf, key, aad);
   } catch (err) {

package/src/storage.ts

@@ -2682,10 +2682,15 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     );
     return this.writeStorageSecureFile(target, content);
   }
-  protected writeStorageSecureFile(filePath: string, content: string | Buffer, forceEncrypt = false): Promise<void> {
+  protected writeStorageSecureFile(
+    filePath: string,
+    content: string | Buffer,
+    forceEncrypt = false,
+    secureOptions?: { aadFilePath?: string }
+  ): Promise<void> {
     assertNotOkfReservedBasename(filePath);
     const writeKey = this.resolveWriteKey(forceEncrypt);
-    return writeMaybeEncryptedFile(filePath, content, writeKey, {}, this.baseDir).then(() => {
+    return writeMaybeEncryptedFile(filePath, content, writeKey, secureOptions ?? {}, this.baseDir).then(() => {
       // No manual sniff-cache update needed (issue #1909 round 10): this rewrite
       // changes the file's (size, mtime), so isEncryptedFileHeader re-sniffs on
       // the next append when its identity check misses. Identity validation also
@@ -2733,9 +2738,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     return this.deletionRevisionStore.deleteManagedStorageFile(filePath, deletionMtimeMs);
   }
 
-  async readOfflineSyncFile(filePath: string): Promise<Buffer> {
+  async readOfflineSyncFile(filePath: string, opts?: { aadFilePath?: string }): Promise<Buffer> {
     const target = this.assertManagedStoragePath(filePath, "storage.readOfflineSyncFile");
-    return readMaybeEncryptedFileBuffer(target, this._secureStoreKey, this.baseDir);
+    return readMaybeEncryptedFileBuffer(target, this._secureStoreKey, this.baseDir, opts);
   }
 
   async digestOfflineSyncFile(filePath: string): Promise<{ sha256: string; bytes: number }> {
@@ -2895,9 +2900,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     }
   }
 
-  async writeOfflineSyncStagingFile(filePath: string, content: Buffer): Promise<void> {
+  async writeOfflineSyncStagingFile(filePath: string, content: Buffer, opts?: { aadFilePath?: string }): Promise<void> {
     const target = this.assertManagedStoragePath(filePath, "storage.writeOfflineSyncStagingFile");
-    await this.writeStorageSecureFile(target, content);
+    await this.writeStorageSecureFile(target, content, false, opts);
   }
 
   createContentHashIndex(): ContentHashIndex {

package/src/support-passport/offline-sync-guard.ts

@@ -26,10 +26,14 @@ type FrontmatterDecision = "allow" | "wait";
 interface SupportPassportOfflineSyncStorage {
   dir: string;
   readMemoryByPath(filePath: string): Promise<MemoryFile | null>;
-  readOfflineSyncFile(filePath: string): Promise<Buffer>;
+  readOfflineSyncFile(filePath: string, opts?: { aadFilePath?: string }): Promise<Buffer>;
   digestOfflineSyncFile(filePath: string): Promise<{ sha256: string; bytes: number }>;
   writeOfflineSyncFile(filePath: string, content: Buffer): Promise<void>;
-  writeOfflineSyncStagingFile(filePath: string, content: Buffer): Promise<void>;
+  writeOfflineSyncStagingFile(
+    filePath: string,
+    content: Buffer,
+    opts?: { aadFilePath?: string },
+  ): Promise<void>;
   writeOfflineSyncFileChunks(filePath: string, chunks: AsyncIterable<Buffer>): Promise<void>;
   deleteOfflineSyncFile(filePath: string, deletionMtimeMs?: number | null): Promise<void>;
   recordReplicatedDeletionRevision(filePath: string, mtimeMs: number): Promise<void>;
@@ -219,6 +223,16 @@ export async function applySupportPassportOfflineSyncChangeset(
       guard.assertContentAllowed(target.path, target.content);
       await storage.writeOfflineSyncFile(target.filePath, target.content);
     },
+    // Secure staging for embedding-generation replacement (#3150): staged
+    // ciphertext binds the FINAL published path, matching the snapshot apply.
+    writeStagingFile: async ({ filePath, content, finalAadFilePath }) =>
+      storage.writeOfflineSyncStagingFile(
+        filePath,
+        content,
+        finalAadFilePath === undefined ? undefined : { aadFilePath: finalAadFilePath },
+      ),
+    readStagingFile: async ({ filePath, finalAadFilePath }) =>
+      storage.readOfflineSyncFile(filePath, finalAadFilePath === undefined ? undefined : { aadFilePath: finalAadFilePath }),
     deleteFile: async (target) => {
       await guard.assertTargetAllowed(target);
       await storage.deleteOfflineSyncFile(target.filePath, target.mtimeMs ?? null);

package/dist/chunk-3BTFBQLO.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/offline-sync-file-io.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { createReadStream } from \"node:fs\";\nimport { open } from \"node:fs/promises\";\nimport { throwIfAborted } from \"./abort-error.js\";\nimport { MAGIC_HEADER_SIZE, isEncryptedFile } from \"./secure-store/secure-fs.js\";\n\nexport interface OfflineSyncFileTarget {\n  root: string;\n  path: string;\n  filePath: string;\n}\n\nexport type OfflineSyncExcludeFile = (target: OfflineSyncFileTarget) => boolean | Promise<boolean>;\nexport async function shouldExcludeOfflineSyncFile(\n  excludeFile: OfflineSyncExcludeFile | undefined,\n  target: OfflineSyncFileTarget\n): Promise<boolean> {\n  return excludeFile ? await excludeFile(target) : false;\n}\n\nexport async function sha256OfflineSyncFile(\n  filePath: string,\n  signal?: AbortSignal\n): Promise<{ sha256: string; bytes: number }> {\n  const hash = createHash(\"sha256\");\n  let bytes = 0;\n  for await (const chunk of createReadStream(filePath)) {\n    throwIfAborted(signal, \"offline sync request aborted\");\n    const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);\n    hash.update(buffer);\n    bytes += buffer.length;\n  }\n  throwIfAborted(signal, \"offline sync request aborted\");\n  return { sha256: hash.digest(\"hex\"), bytes };\n}\n\nexport async function isEncryptedOfflineSyncFile(filePath: string): Promise<boolean> {\n  const handle = await open(filePath, \"r\");\n  try {\n    const header = Buffer.alloc(MAGIC_HEADER_SIZE);\n    const { bytesRead } = await handle.read(header, 0, header.length, 0);\n    return bytesRead >= MAGIC_HEADER_SIZE && isEncryptedFile(header);\n  } finally {\n    await handle.close();\n  }\n}\n\nexport async function readPlainOfflineSyncFileChunk(options: {\n  filePath: string;\n  offset: number;\n  length: number;\n  bytes: number;\n}): Promise<Buffer> {\n  const chunkBytes = Math.min(options.length, options.bytes - options.offset);\n  const chunk = Buffer.alloc(chunkBytes);\n  if (chunkBytes === 0) return chunk;\n  const handle = await open(options.filePath, \"r\");\n  try {\n    const { bytesRead } = await handle.read(chunk, 0, chunk.length, options.offset);\n    return bytesRead === chunk.length ? chunk : chunk.subarray(0, bytesRead);\n  } finally {\n    await handle.close();\n  }\n}\n\n/** Streamed whole-file sha256 for plain-file chunk reads — the value the\n * file-content response contract exposes as x-remnic-file-sha256. Deliberately\n * UNCACHED: identity keys like (path, size, mtimeMs) are spoofable (a rewrite\n * that preserves size and mtime would serve a stale digest, defeating the\n * uploader's verify-before-idempotent-skip check). One bounded-memory hash\n * pass per chunk request; content reads stay windowed. */\nexport async function plainFileDigest(filePath: string): Promise<string> {\n  const hash = createHash(\"sha256\");\n  for await (const piece of createReadStream(filePath)) {\n    hash.update(piece as Buffer);\n  }\n  return hash.digest(\"hex\");\n}\n"],"mappings":";;;;;;;;;AAAA,SAAS,kBAAkB;AAC3B,SAAS,wBAAwB;AACjC,SAAS,YAAY;AAWrB,eAAsB,6BACpB,aACA,QACkB;AAClB,SAAO,cAAc,MAAM,YAAY,MAAM,IAAI;AACnD;AAEA,eAAsB,sBACpB,UACA,QAC4C;AAC5C,QAAM,OAAO,WAAW,QAAQ;AAChC,MAAI,QAAQ;AACZ,mBAAiB,SAAS,iBAAiB,QAAQ,GAAG;AACpD,mBAAe,QAAQ,8BAA8B;AACrD,UAAM,SAAS,OAAO,SAAS,KAAK,IAAI,QAAQ,OAAO,KAAK,KAAK;AACjE,SAAK,OAAO,MAAM;AAClB,aAAS,OAAO;AAAA,EAClB;AACA,iBAAe,QAAQ,8BAA8B;AACrD,SAAO,EAAE,QAAQ,KAAK,OAAO,KAAK,GAAG,MAAM;AAC7C;AAEA,eAAsB,2BAA2B,UAAoC;AACnF,QAAM,SAAS,MAAM,KAAK,UAAU,GAAG;AACvC,MAAI;AACF,UAAM,SAAS,OAAO,MAAM,iBAAiB;AAC7C,UAAM,EAAE,UAAU,IAAI,MAAM,OAAO,KAAK,QAAQ,GAAG,OAAO,QAAQ,CAAC;AACnE,WAAO,aAAa,qBAAqB,gBAAgB,MAAM;AAAA,EACjE,UAAE;AACA,UAAM,OAAO,MAAM;AAAA,EACrB;AACF;AAEA,eAAsB,8BAA8B,SAKhC;AAClB,QAAM,aAAa,KAAK,IAAI,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,MAAM;AAC1E,QAAM,QAAQ,OAAO,MAAM,UAAU;AACrC,MAAI,eAAe,EAAG,QAAO;AAC7B,QAAM,SAAS,MAAM,KAAK,QAAQ,UAAU,GAAG;AAC/C,MAAI;AACF,UAAM,EAAE,UAAU,IAAI,MAAM,OAAO,KAAK,OAAO,GAAG,MAAM,QAAQ,QAAQ,MAAM;AAC9E,WAAO,cAAc,MAAM,SAAS,QAAQ,MAAM,SAAS,GAAG,SAAS;AAAA,EACzE,UAAE;AACA,UAAM,OAAO,MAAM;AAAA,EACrB;AACF;AAQA,eAAsB,gBAAgB,UAAmC;AACvE,QAAM,OAAO,WAAW,QAAQ;AAChC,mBAAiB,SAAS,iBAAiB,QAAQ,GAAG;AACpD,SAAK,OAAO,KAAe;AAAA,EAC7B;AACA,SAAO,KAAK,OAAO,KAAK;AAC1B;","names":[]}

package/dist/chunk-AREGMX2J.js

@@ -1,50 +0,0 @@
-// src/offline-sync-exclude-globs.ts
-var OFFLINE_DECRYPT_STAGING_DIR_PREFIX = ".remnic-offline-decrypt-";
-var DEFAULT_OFFLINE_SYNC_EXCLUDE_GLOBS = [
-  // Leading `**/` matches zero or more segments, so each pattern covers both
-  // the root `state/` dir AND per-namespace `namespaces/<ns>/state/` dirs
-  // (Cursor review on PR #1793: multi-namespace deployments previously kept
-  // pushing their namespaced live sqlite files).
-  "**/state/*.sqlite",
-  "**/state/*.sqlite-*",
-  "**/state/index_tags.json",
-  "**/state/entity-mention-index.json",
-  "**/state/memory-governance/runs/**",
-  // Rotated recall-impression archives (issue #1910). The active
-  // recall_impressions.jsonl stays remote-authoritative; only the .1..N
-  // archives and the .lock are node-local and never pushed/hashed.
-  "**/state/recall_impressions.jsonl.*",
-  // Durable recall-impression pending spill directory (issue #2033). Node-local:
-  // an impression spills a per-event file here when its rotation lock cannot be
-  // acquired, and the next lock holder folds them back into the synced active
-  // recall_impressions.jsonl. The `.*` glob above matches the directory name but
-  // not its children, so exclude the contents explicitly.
-  "**/state/recall_impressions.jsonl.pending.d/**",
-  // Durable lifecycle-append pending spill directory (issue #2033). Node-local:
-  // an append spills a per-event file here when it cannot get the ledger lock,
-  // and the next lock holder folds them back into the synced
-  // memory-lifecycle-ledger.jsonl. Pushing them would duplicate rows remotely.
-  // The offline-sync snapshot entrypoints drain this queue first (access-service
-  // `drainPendingLifecycleForSync` -> `StorageManager.drainPendingMemoryLifecycleEventsForSync`),
-  // aborting the snapshot when durable rows cannot be folded, so excluding the
-  // dir never silently drops append-only lifecycle history (promotions/imports).
-  "**/state/memory-lifecycle-ledger.jsonl.pending.d/**",
-  // The active lifecycle ledger lock is node-local and must never be
-  // transferred to another node during an offline snapshot.
-  "**/state/memory-lifecycle-ledger.jsonl.lock",
-  // Transient plaintext decrypt staging (issue #2033 P1). The CLI offline
-  // decrypt path materializes a decrypted secure-store file under an owner-only
-  // `<OFFLINE_DECRYPT_STAGING_DIR_PREFIX>*` dir before hashing/pushing, removing
-  // it in a `finally`. A hard crash between create and cleanup would otherwise
-  // let a later snapshot walk `<prefix>*/content` and push decrypted secure-store
-  // plaintext to the remote. Exclude the dir and its contents unconditionally so
-  // a crash-orphan can never be enumerated into a snapshot; the leading `**/`
-  // matches the root-level dir (zero segments) and any nested placement.
-  "**/.remnic-offline-decrypt-*/**"
-];
-
-export {
-  OFFLINE_DECRYPT_STAGING_DIR_PREFIX,
-  DEFAULT_OFFLINE_SYNC_EXCLUDE_GLOBS
-};
-//# sourceMappingURL=chunk-AREGMX2J.js.map

package/dist/chunk-AREGMX2J.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/offline-sync-exclude-globs.ts"],"sourcesContent":["// Issue #1786: node-local runtime state that each node rebuilds from synced\n// records. Pushing these wastes bandwidth, corrupts the remote's live state\n// dir, and trips the large-file push retry loop on a live SQLite database.\n// Extracted from offline-sync.ts (issue #1995) so the god-file line-count\n// ratchet does not grow when the list gains entries; behavior is unchanged.\n/**\n * Directory-name prefix for the transient plaintext staging dir the CLI offline\n * decrypt path (`offline-storage-io.ts`) creates under the memory root when it\n * must materialize a decrypted secure-store file before hashing/pushing (#2033\n * P1). Shared here so the hard-exclude glob below and the CLI's `mkdtemp` prefix\n * can never drift apart: a crash-orphaned staging dir must NEVER be enumerated\n * into an offline snapshot, or decrypted secure-store plaintext would leak into\n * the remote push.\n */\nexport const OFFLINE_DECRYPT_STAGING_DIR_PREFIX = \".remnic-offline-decrypt-\";\n\nexport const DEFAULT_OFFLINE_SYNC_EXCLUDE_GLOBS: readonly string[] = [\n  // Leading `**/` matches zero or more segments, so each pattern covers both\n  // the root `state/` dir AND per-namespace `namespaces/<ns>/state/` dirs\n  // (Cursor review on PR #1793: multi-namespace deployments previously kept\n  // pushing their namespaced live sqlite files).\n  \"**/state/*.sqlite\",\n  \"**/state/*.sqlite-*\",\n  \"**/state/index_tags.json\",\n  \"**/state/entity-mention-index.json\",\n  \"**/state/memory-governance/runs/**\",\n  // Rotated recall-impression archives (issue #1910). The active\n  // recall_impressions.jsonl stays remote-authoritative; only the .1..N\n  // archives and the .lock are node-local and never pushed/hashed.\n  \"**/state/recall_impressions.jsonl.*\",\n  // Durable recall-impression pending spill directory (issue #2033). Node-local:\n  // an impression spills a per-event file here when its rotation lock cannot be\n  // acquired, and the next lock holder folds them back into the synced active\n  // recall_impressions.jsonl. The `.*` glob above matches the directory name but\n  // not its children, so exclude the contents explicitly.\n  \"**/state/recall_impressions.jsonl.pending.d/**\",\n  // Durable lifecycle-append pending spill directory (issue #2033). Node-local:\n  // an append spills a per-event file here when it cannot get the ledger lock,\n  // and the next lock holder folds them back into the synced\n  // memory-lifecycle-ledger.jsonl. Pushing them would duplicate rows remotely.\n  // The offline-sync snapshot entrypoints drain this queue first (access-service\n  // `drainPendingLifecycleForSync` -> `StorageManager.drainPendingMemoryLifecycleEventsForSync`),\n  // aborting the snapshot when durable rows cannot be folded, so excluding the\n  // dir never silently drops append-only lifecycle history (promotions/imports).\n  \"**/state/memory-lifecycle-ledger.jsonl.pending.d/**\",\n  // The active lifecycle ledger lock is node-local and must never be\n  // transferred to another node during an offline snapshot.\n  \"**/state/memory-lifecycle-ledger.jsonl.lock\",\n  // Transient plaintext decrypt staging (issue #2033 P1). The CLI offline\n  // decrypt path materializes a decrypted secure-store file under an owner-only\n  // `<OFFLINE_DECRYPT_STAGING_DIR_PREFIX>*` dir before hashing/pushing, removing\n  // it in a `finally`. A hard crash between create and cleanup would otherwise\n  // let a later snapshot walk `<prefix>*/content` and push decrypted secure-store\n  // plaintext to the remote. Exclude the dir and its contents unconditionally so\n  // a crash-orphan can never be enumerated into a snapshot; the leading `**/`\n  // matches the root-level dir (zero segments) and any nested placement.\n  \"**/.remnic-offline-decrypt-*/**\",\n];\n"],"mappings":";AAcO,IAAM,qCAAqC;AAE3C,IAAM,qCAAwD;AAAA;AAAA;AAAA;AAAA;AAAA,EAKnE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA;AAAA;AAAA,EAIA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA;AAAA;AAAA;AAAA,EAGA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA;AACF;","names":[]}