@remnic/core 9.69.103 → 9.69.105

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/dist/access-admin-ops-surface.js +10 -10
  2. package/dist/access-authorization-probe.js +11 -11
  3. package/dist/access-boundary.js +10 -10
  4. package/dist/access-cli.js +30 -30
  5. package/dist/access-extraction-force-flush.js +10 -10
  6. package/dist/access-http-briefing.js +11 -11
  7. package/dist/access-http-query.js +10 -10
  8. package/dist/access-http.js +23 -23
  9. package/dist/access-identity-continuity-surface.js +10 -10
  10. package/dist/access-lcm-surface.js +10 -10
  11. package/dist/access-mcp.js +17 -17
  12. package/dist/access-namespace-preflight.js +10 -10
  13. package/dist/access-observe-write-category.js +7 -7
  14. package/dist/access-observe-write-surface.js +10 -10
  15. package/dist/access-offline-manifest.js +6 -6
  16. package/dist/access-operations-batch.js +11 -11
  17. package/dist/access-operations.js +16 -16
  18. package/dist/access-recall-concurrency.js +10 -10
  19. package/dist/access-recall-response.js +10 -10
  20. package/dist/access-recall-surface.js +10 -10
  21. package/dist/access-schema.js +6 -6
  22. package/dist/access-service.js +10 -10
  23. package/dist/access-surface-catalog.js +12 -12
  24. package/dist/active-recall.js +6 -6
  25. package/dist/{chunk-47LTKAGL.js → chunk-2COXDQ6C.js} +21 -6
  26. package/dist/chunk-2COXDQ6C.js.map +1 -0
  27. package/dist/{chunk-KN4GNQH7.js → chunk-2TCNEJ36.js} +4 -4
  28. package/dist/{chunk-5FHH4NJR.js → chunk-5R4YPRAR.js} +17 -4
  29. package/dist/chunk-5R4YPRAR.js.map +1 -0
  30. package/dist/{chunk-ANPRNTPT.js → chunk-63LFM77Z.js} +15 -2
  31. package/dist/chunk-63LFM77Z.js.map +1 -0
  32. package/dist/{chunk-JHZOY4CJ.js → chunk-6WRHNA4I.js} +7 -7
  33. package/dist/{chunk-HTXTVY64.js → chunk-7NBUE5VY.js} +2 -2
  34. package/dist/{chunk-4QHWRWCF.js → chunk-7YQR6XHY.js} +8 -8
  35. package/dist/{chunk-2JJD7GIW.js → chunk-ATFFWIKL.js} +2 -2
  36. package/dist/{chunk-2GSO4FEC.js → chunk-AU43D3IT.js} +3 -3
  37. package/dist/{chunk-OC6ATPIP.js → chunk-CGBE3N7T.js} +2 -2
  38. package/dist/{chunk-NN62MUZA.js → chunk-DB6XFMOF.js} +3 -3
  39. package/dist/{chunk-5GKRQBPQ.js → chunk-FYX2LXIX.js} +2 -2
  40. package/dist/{chunk-6RWPKWCZ.js → chunk-GZ2KV2RO.js} +3 -3
  41. package/dist/{chunk-D4MXH276.js → chunk-JC7QQ2DJ.js} +2 -2
  42. package/dist/{chunk-GWWHV7F3.js → chunk-KFHEZJQZ.js} +2 -2
  43. package/dist/{chunk-B44TVSEI.js → chunk-KV7LFOPF.js} +2 -2
  44. package/dist/{chunk-L6BUCPGI.js → chunk-LHDSCWDK.js} +11 -11
  45. package/dist/{chunk-5PEBJUYD.js → chunk-MZOQ26LH.js} +2 -2
  46. package/dist/{chunk-NBYSCXOY.js → chunk-OFGTRDWX.js} +2 -2
  47. package/dist/{chunk-26DJ7FG6.js → chunk-PZ2QQULY.js} +2 -2
  48. package/dist/{chunk-LP4UW5WI.js → chunk-QAIVTLL2.js} +2 -2
  49. package/dist/{chunk-HYFVZJLX.js → chunk-QUNCLKHM.js} +2 -2
  50. package/dist/{chunk-Q3LWW7WS.js → chunk-QX3K3INR.js} +2 -2
  51. package/dist/{chunk-K5ABKRXK.js → chunk-SFDBLCZV.js} +7 -7
  52. package/dist/{chunk-XM32D6YZ.js → chunk-STQC2FYQ.js} +4 -4
  53. package/dist/{chunk-PNZAFABF.js → chunk-TXTRHF3B.js} +2 -2
  54. package/dist/{chunk-SOVRTUD3.js → chunk-VDVHIRCK.js} +2 -2
  55. package/dist/{chunk-D42DLA4D.js → chunk-WR66CTVU.js} +3 -3
  56. package/dist/{chunk-UCPGOUDZ.js → chunk-Y57GJDG5.js} +108 -3
  57. package/dist/chunk-Y57GJDG5.js.map +1 -0
  58. package/dist/{chunk-NIGZ3LE7.js → chunk-YI6XIFQV.js} +2 -2
  59. package/dist/cli.js +27 -27
  60. package/dist/config.js +6 -6
  61. package/dist/deep-recall-http-glue.js +7 -7
  62. package/dist/embedding-fallback.d.ts +2 -1
  63. package/dist/embedding-fallback.d.ts.map +1 -1
  64. package/dist/embedding-fallback.js +2 -2
  65. package/dist/embedding-index-storage.d.ts +28 -1
  66. package/dist/embedding-index-storage.d.ts.map +1 -1
  67. package/dist/embedding-index-storage.js +1 -1
  68. package/dist/external-wiki-access.js +11 -11
  69. package/dist/index.js +30 -30
  70. package/dist/memory-browse-operations.js +11 -11
  71. package/dist/offline-sync-changeset-generations.js +4 -4
  72. package/dist/offline-sync-embedding-generation.js +3 -3
  73. package/dist/offline-sync-exclude-globs.d.ts +8 -0
  74. package/dist/offline-sync-exclude-globs.d.ts.map +1 -1
  75. package/dist/offline-sync-exclude-globs.js +3 -1
  76. package/dist/offline-sync-snapshot-stream.js +6 -6
  77. package/dist/offline-sync.d.ts.map +1 -1
  78. package/dist/offline-sync.js +5 -5
  79. package/dist/operator-toolkit.js +7 -7
  80. package/dist/orchestrator.js +30 -30
  81. package/dist/recall-navigation-operations.js +11 -11
  82. package/dist/recall-navigation.js +10 -10
  83. package/dist/resume-bundles.js +7 -7
  84. package/dist/support-passport/index.js +13 -13
  85. package/package.json +2 -2
  86. package/src/embedding-fallback-cross-process.test.ts +533 -0
  87. package/src/embedding-fallback.ts +19 -7
  88. package/src/embedding-index-storage.test.ts +482 -2
  89. package/src/embedding-index-storage.ts +131 -2
  90. package/src/offline-sync-exclude-globs.ts +23 -0
  91. package/src/offline-sync.test.ts +325 -0
  92. package/src/offline-sync.ts +17 -1
  93. package/dist/chunk-47LTKAGL.js.map +0 -1
  94. package/dist/chunk-5FHH4NJR.js.map +0 -1
  95. package/dist/chunk-ANPRNTPT.js.map +0 -1
  96. package/dist/chunk-UCPGOUDZ.js.map +0 -1
  97. /package/dist/{chunk-KN4GNQH7.js.map → chunk-2TCNEJ36.js.map} +0 -0
  98. /package/dist/{chunk-JHZOY4CJ.js.map → chunk-6WRHNA4I.js.map} +0 -0
  99. /package/dist/{chunk-HTXTVY64.js.map → chunk-7NBUE5VY.js.map} +0 -0
  100. /package/dist/{chunk-4QHWRWCF.js.map → chunk-7YQR6XHY.js.map} +0 -0
  101. /package/dist/{chunk-2JJD7GIW.js.map → chunk-ATFFWIKL.js.map} +0 -0
  102. /package/dist/{chunk-2GSO4FEC.js.map → chunk-AU43D3IT.js.map} +0 -0
  103. /package/dist/{chunk-OC6ATPIP.js.map → chunk-CGBE3N7T.js.map} +0 -0
  104. /package/dist/{chunk-NN62MUZA.js.map → chunk-DB6XFMOF.js.map} +0 -0
  105. /package/dist/{chunk-5GKRQBPQ.js.map → chunk-FYX2LXIX.js.map} +0 -0
  106. /package/dist/{chunk-6RWPKWCZ.js.map → chunk-GZ2KV2RO.js.map} +0 -0
  107. /package/dist/{chunk-D4MXH276.js.map → chunk-JC7QQ2DJ.js.map} +0 -0
  108. /package/dist/{chunk-GWWHV7F3.js.map → chunk-KFHEZJQZ.js.map} +0 -0
  109. /package/dist/{chunk-B44TVSEI.js.map → chunk-KV7LFOPF.js.map} +0 -0
  110. /package/dist/{chunk-L6BUCPGI.js.map → chunk-LHDSCWDK.js.map} +0 -0
  111. /package/dist/{chunk-5PEBJUYD.js.map → chunk-MZOQ26LH.js.map} +0 -0
  112. /package/dist/{chunk-NBYSCXOY.js.map → chunk-OFGTRDWX.js.map} +0 -0
  113. /package/dist/{chunk-26DJ7FG6.js.map → chunk-PZ2QQULY.js.map} +0 -0
  114. /package/dist/{chunk-LP4UW5WI.js.map → chunk-QAIVTLL2.js.map} +0 -0
  115. /package/dist/{chunk-HYFVZJLX.js.map → chunk-QUNCLKHM.js.map} +0 -0
  116. /package/dist/{chunk-Q3LWW7WS.js.map → chunk-QX3K3INR.js.map} +0 -0
  117. /package/dist/{chunk-K5ABKRXK.js.map → chunk-SFDBLCZV.js.map} +0 -0
  118. /package/dist/{chunk-XM32D6YZ.js.map → chunk-STQC2FYQ.js.map} +0 -0
  119. /package/dist/{chunk-PNZAFABF.js.map → chunk-TXTRHF3B.js.map} +0 -0
  120. /package/dist/{chunk-SOVRTUD3.js.map → chunk-VDVHIRCK.js.map} +0 -0
  121. /package/dist/{chunk-D42DLA4D.js.map → chunk-WR66CTVU.js.map} +0 -0
  122. /package/dist/{chunk-NIGZ3LE7.js.map → chunk-YI6XIFQV.js.map} +0 -0
@@ -14,6 +14,7 @@
14
14
  * caching or provider logic (that lives in `embedding-fallback.ts`).
15
15
  */
16
16
  import path from "node:path";
17
+ import { randomUUID } from "node:crypto";
17
18
  import { constants as bufferConstants } from "node:buffer";
18
19
  import { lstat, mkdir, mkdtemp, realpath, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
19
20
  import { log } from "./logger.js";
@@ -165,6 +166,21 @@ function replacementBackupPath(shardDir: string): string {
165
166
  return path.join(path.dirname(shardDir), "embeddings.pre-replace.tmp");
166
167
  }
167
168
 
169
+ /**
170
+ * Warm-cache generation marker (`<stateDir>/embeddings.generation`).
171
+ * Two-phase publication barrier: in-flight before a publication's first
172
+ * destructive write, fresh unique value after its last write. A stamp is
173
+ * therefore stable only for a completed generation — readers can never
174
+ * cache a stamp that stays stable while generation bytes change, and an
175
+ * in-flight stamp never repeats, so probes cached during a failed
176
+ * completion keep revalidating (PR #3176 codex P2). Plain fs like the
177
+ * status file.
178
+ */
179
+ const GENERATION_MARKER_BASENAME = "embeddings.generation";
180
+
181
+ /** Reserved in-flight marker value; never equals a completion marker. */
182
+ const GENERATION_MARKER_IN_FLIGHT = "in-flight";
183
+
168
184
  /**
169
185
  * Stable FNV-1a shard assignment: an entry always hashes to the same shard,
170
186
  * so an update or removal never needs a manifest to find its file.
@@ -469,7 +485,9 @@ export class EmbeddingIndexFileStore {
469
485
  * the published position after an interrupted replacement. Returns true
470
486
  * when a rollback happened. Only fires in the actual rename gap
471
487
  * (published directory absent); callers must invalidate any cached index
472
- * view when this returns true.
488
+ * view when this returns true. A successful rollback finalizes the
489
+ * generation marker strictly: a failed completion write fails the
490
+ * recovery instead of leaving every search reloading forever.
473
491
  */
474
492
  async recoverIfInterrupted(fence?: EmbeddingGenerationFence): Promise<boolean> {
475
493
  // Only the rename gap (published directory absent) is recoverable. When
@@ -492,6 +510,7 @@ export class EmbeddingIndexFileStore {
492
510
  // The rollback rename is destructive: reassert lock ownership first.
493
511
  await fence?.();
494
512
  await rename(backupPath, this.shardDir);
513
+ await this.writeGenerationMarker(randomUUID());
495
514
  log.warn(
496
515
  `embedding index: recovered interrupted replacement; former generation restored from ${backupPath}`,
497
516
  );
@@ -514,6 +533,97 @@ export class EmbeddingIndexFileStore {
514
533
  }
515
534
  }
516
535
 
536
+ /** Barrier first half: strict, so bytes can never change under a stable stamp. */
537
+ private async beginGenerationMarker(): Promise<void> {
538
+ await this.writeGenerationMarker(GENERATION_MARKER_IN_FLIGHT);
539
+ }
540
+
541
+ /**
542
+ * Barrier second half: written after the last content write. Best-effort:
543
+ * a failure leaves the in-flight marker, whose stamp identityStamp()
544
+ * never repeats, so cached probes revalidate on every search until a
545
+ * publication completes.
546
+ */
547
+ private async completeGenerationMarker(): Promise<void> {
548
+ await this.writeGenerationMarker(randomUUID()).catch((err) => {
549
+ log.warn(`embedding index: could not write generation completion marker: ${err instanceof Error ? err.message : String(err)}`);
550
+ });
551
+ }
552
+
553
+ private async writeGenerationMarker(value: string): Promise<void> {
554
+ await this.writeAtomicFile(
555
+ path.join(path.dirname(this.shardDir), GENERATION_MARKER_BASENAME),
556
+ value,
557
+ { bypassSecureIo: true },
558
+ );
559
+ }
560
+
561
+ /** Marker value, or null when absent/unreadable (pre-marker stores keep the stat stamp). */
562
+ private async readGenerationMarker(): Promise<string | null> {
563
+ try {
564
+ const raw = await readFile(
565
+ path.join(path.dirname(this.shardDir), GENERATION_MARKER_BASENAME),
566
+ "utf-8",
567
+ );
568
+ const value = raw.trim();
569
+ return value.length > 0 ? value : null;
570
+ } catch {
571
+ return null;
572
+ }
573
+ }
574
+
575
+ /**
576
+ * Warm-cache stamp. A completed generation marker alone would freeze the
577
+ * stamp for a pre-marker writer in a rolling deployment (it publishes
578
+ * shard/index changes without updating the marker), so the marker is
579
+ * combined with the watched path's stat: the marker keeps consecutive
580
+ * publications distinct on coarse-tick filesystems, the stat keeps
581
+ * legacy writers visible. In-flight never repeats, forcing revalidation
582
+ * until a publication completes. Stores never published through this
583
+ * module keep the plain stat stamp — a directory stat alone aliases
584
+ * consecutive publications on coarse-tick filesystems and across inode
585
+ * reuse.
586
+ */
587
+ async identityStamp(): Promise<string> {
588
+ const layout = await this.detectLayout();
589
+ if (layout === "sharded") {
590
+ const info = await lstat(this.shardDir).catch((err) => {
591
+ // SAFETY: only the errno code is inspected; any other rejection shape rethrows uninterpreted.
592
+ if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
593
+ throw new EmbeddingIndexStorageError(
594
+ `cannot stat embedding index shard directory ${this.shardDir}: ${err instanceof Error ? err.message : String(err)}`,
595
+ );
596
+ });
597
+ if (!info) return "rename-gap";
598
+ const marker = await this.readGenerationMarker();
599
+ // An in-flight marker must never produce a repeatable stamp: a probe
600
+ // that cached one (completion write failed) must revalidate on every
601
+ // later probe until a publication completes.
602
+ if (marker) {
603
+ if (marker === GENERATION_MARKER_IN_FLIGHT) return `gen:${randomUUID()}`;
604
+ return `gen:${marker}:${info.ino}:${info.mtimeMs}`;
605
+ }
606
+ return `shard:${info.ino}:${info.mtimeMs}`;
607
+ }
608
+ if (layout === "legacy") {
609
+ const info = await stat(this.indexPath).catch((err) => {
610
+ // SAFETY: only the errno code is inspected; any other rejection shape rethrows uninterpreted.
611
+ if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
612
+ throw new EmbeddingIndexStorageError(
613
+ `cannot stat embedding index file ${this.indexPath}: ${err instanceof Error ? err.message : String(err)}`,
614
+ );
615
+ });
616
+ if (!info) return "rename-gap";
617
+ const marker = await this.readGenerationMarker();
618
+ if (marker) {
619
+ if (marker === GENERATION_MARKER_IN_FLIGHT) return `gen:${randomUUID()}`;
620
+ return `gen:${marker}:${info.ino}:${info.mtimeMs}`;
621
+ }
622
+ return `legacy:${info.ino}:${info.mtimeMs}`;
623
+ }
624
+ return "empty";
625
+ }
626
+
517
627
  /** Header identity of the authoritative generation, read leniently. */
518
628
  async identityFromDisk(): Promise<EmbeddingIndexIdentity | null> {
519
629
  const layout = await this.detectLayout();
@@ -751,6 +861,8 @@ export class EmbeddingIndexFileStore {
751
861
  // Serialization is done: reassert lock ownership immediately before
752
862
  // the destructive live-generation writes.
753
863
  await opts.fence?.();
864
+ await this.beginGenerationMarker();
865
+ await opts.fence?.();
754
866
  for (const payload of payloads) {
755
867
  await this.writeAtomicFile(
756
868
  path.join(this.shardDir, shardFileName(payload.shardIndex)),
@@ -764,6 +876,7 @@ export class EmbeddingIndexFileStore {
764
876
  await rm(path.join(this.shardDir, shardFileName(shardIndex)), { force: true });
765
877
  }
766
878
  }
879
+ await this.completeGenerationMarker();
767
880
  return;
768
881
  }
769
882
 
@@ -779,7 +892,10 @@ export class EmbeddingIndexFileStore {
779
892
  // Serialized above: reassert lock ownership immediately before the
780
893
  // destructive atomic write.
781
894
  await opts.fence?.();
895
+ await this.beginGenerationMarker();
896
+ await opts.fence?.();
782
897
  await this.writeAtomicFile(this.indexPath, whole);
898
+ await this.completeGenerationMarker();
783
899
  return;
784
900
  }
785
901
 
@@ -920,6 +1036,9 @@ export class EmbeddingIndexFileStore {
920
1036
  // delay between the cleanup and the rename still allows a stale-break,
921
1037
  // and publishing now would clobber the peer's write.
922
1038
  await fence?.();
1039
+ // Publication barrier: in-flight before the renames, final at commit.
1040
+ await this.beginGenerationMarker();
1041
+ await fence?.();
923
1042
  let demoted = false;
924
1043
  try {
925
1044
  await stat(this.shardDir);
@@ -931,9 +1050,19 @@ export class EmbeddingIndexFileStore {
931
1050
  try {
932
1051
  await rename(stagingDir, this.shardDir);
933
1052
  } catch (err) {
934
- if (demoted) await rename(backupPath, this.shardDir).catch(() => undefined);
1053
+ if (demoted) {
1054
+ try {
1055
+ await rename(backupPath, this.shardDir);
1056
+ await this.completeGenerationMarker();
1057
+ } catch (rollbackErr) {
1058
+ log.warn(`embedding index: publication rollback did not restore the generation; marker stays in-flight: ${rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)}`);
1059
+ }
1060
+ } else {
1061
+ await this.completeGenerationMarker();
1062
+ }
935
1063
  throw err;
936
1064
  }
1065
+ await this.completeGenerationMarker();
937
1066
  await rm(backupPath, { recursive: true, force: true }).catch((err) => {
938
1067
  log.warn(`embedding index: could not remove replacement backup ${backupPath}: ${err}`);
939
1068
  });
@@ -67,12 +67,35 @@ export const DEFAULT_OFFLINE_SYNC_EXCLUDE_GLOBS: readonly string[] = [
67
67
  "**/namespaces/*/state/embeddings.pre-replace.tmp/**",
68
68
  "**/namespaces/*/state/embeddings.staging.tmp-*/**",
69
69
  "**/namespaces/*/state/embeddings.json.pre-migration.tmp-*",
70
+ // The warm-cache generation marker (PR #3176) is node-local coherence
71
+ // state: a synced copy — worse, a captured `in-flight` value — would make
72
+ // the receiving node's stamp probes unique forever and force a full index
73
+ // reload on every search. Exclude on both sides: never snapshotted, and
74
+ // an incoming (old or hostile) marker never overwrites or deletes the
75
+ // local one.
76
+ "**/state/embeddings.generation",
77
+ "**/namespaces/*/state/embeddings.generation",
70
78
  ];
71
79
 
72
80
 
73
81
  const DEFAULT_OFFLINE_SYNC_EXCLUDE_REGEXPS: readonly RegExp[] =
74
82
  DEFAULT_OFFLINE_SYNC_EXCLUDE_GLOBS.map((glob) => globToRegExp(glob));
75
83
 
84
+ /**
85
+ * The warm-cache generation marker is the one default-excluded path that
86
+ * must ALSO be refused on apply (push-side default excludes like the live
87
+ * LCM sqlite are deliberately apply-accepted for first-sync bootstrapping).
88
+ * Node-local on both sides: never snapshotted, never overwritten by an
89
+ * incoming old or hostile copy.
90
+ */
91
+ export function isEmbeddingGenerationMarkerPath(relPosix: string): boolean {
92
+ const normalized = relPosix.includes("\\") ? relPosix.replaceAll("\\", "/") : relPosix;
93
+ // Case-insensitive: on APFS/NTFS a differently cased path resolves to the
94
+ // same file, so `state/Embeddings.Generation` must be refused like the
95
+ // canonical spelling.
96
+ return /(?:^|\/)state\/embeddings\.generation$/i.test(normalized);
97
+ }
98
+
76
99
  /**
77
100
  * Precompiled-once default-exclude check for the snapshot enumeration hot
78
101
  * path (Kilo review, PR #1793). Moved here from offline-sync.ts with the
@@ -26,7 +26,15 @@ import {
26
26
  summarizeOfflineSyncPendingFiles,
27
27
  } from "./offline-sync.js";
28
28
  import { isEncryptedFile } from "./secure-store/secure-fs.js";
29
+ import { isEmbeddingGenerationMarkerPath } from "./offline-sync-exclude-globs.js";
29
30
  import { StorageManager } from "./storage.js";
31
+ import { parseConfig } from "./config.js";
32
+ import { EmbeddingFallback } from "./embedding-fallback.js";
33
+ import {
34
+ clearHostEmbeddingProvidersForTest,
35
+ registerHostEmbeddingProvider,
36
+ } from "./host-embedding-provider.js";
37
+ import type { EmbeddingIndexStoreIo } from "./embedding-index-storage.js";
30
38
 
31
39
  async function tempDir(name: string): Promise<string> {
32
40
  return mkdtemp(path.join(os.tmpdir(), `${name}-`));
@@ -551,6 +559,21 @@ test("offline sync push-side default excludes live LCM sqlite but apply-side sti
551
559
  assert.equal(await readUtf8(root, "state/lcm.sqlite-shm"), "live shm");
552
560
  assert.equal(await readUtf8(root, "state/lcm.sqlite-wal"), "live wal");
553
561
 
562
+ // The apply-side acceptance is real, not vacuous: a node with NO local
563
+ // sqlite must bootstrap it from the incoming snapshot.
564
+ const freshRoot = await tempDir("remnic-offline-lcm-sqlite-bootstrap");
565
+ try {
566
+ const bootstrap = await applyOfflineSyncSnapshot({
567
+ root: freshRoot,
568
+ snapshot: remoteSnapshot,
569
+ baseFiles: [],
570
+ });
571
+ assert.equal(await readUtf8(freshRoot, "state/lcm.sqlite"), "live db", "first sync bootstraps the LCM DB from the incoming snapshot");
572
+ assert.equal(bootstrap.upserted, 1);
573
+ } finally {
574
+ await rm(freshRoot, { recursive: true, force: true });
575
+ }
576
+
554
577
  // The push-side changeset excludes lcm.sqlite and friends so the
555
578
  // local node never pushes the live sqlite state up.
556
579
  const changeset = await buildOfflineSyncChangeset({
@@ -569,6 +592,308 @@ test("offline sync push-side default excludes live LCM sqlite but apply-side sti
569
592
  }
570
593
  });
571
594
 
595
+ test("offline sync excludes the node-local embedding generation marker", async () => {
596
+ const root = await tempDir("remnic-offline-emb3176-marker");
597
+ try {
598
+ await write(root, "facts/a.md", "alpha");
599
+ await write(root, "state/embeddings.generation", "in-flight");
600
+ await write(root, "namespaces/team/state/embeddings.generation", "gen-team-uuid");
601
+
602
+ const snapshot = await buildOfflineSyncSnapshot({
603
+ root,
604
+ sourceId: "remote",
605
+ includeContent: true,
606
+ });
607
+ assert.deepEqual(snapshot.files.map((file) => file.path), ["facts/a.md"]);
608
+
609
+ const changeset = await buildOfflineSyncChangeset({
610
+ root,
611
+ sourceId: "laptop",
612
+ baseFiles: [{
613
+ path: "facts/a.md",
614
+ sha256: createHash("sha256").update("alpha").digest("hex"),
615
+ bytes: Buffer.byteLength("alpha"),
616
+ mtimeMs: 0,
617
+ }],
618
+ });
619
+ assert.deepEqual(changeset.changes.map((change) => change.path), []);
620
+ } finally {
621
+ await rm(root, { recursive: true, force: true });
622
+ }
623
+ });
624
+
625
+ test("offline sync apply never lets an incoming marker overwrite the local one", async () => {
626
+ const root = await tempDir("remnic-offline-emb3176-marker-apply");
627
+ try {
628
+ const localMarker = "0d9d2c11-5f6e-4a0e-8f1a-2b3c4d5e6f70";
629
+ await write(root, "facts/a.md", "alpha");
630
+ await write(root, "state/embeddings.generation", localMarker);
631
+
632
+ // A hostile or pre-exclusion remote snapshot carries the marker as an
633
+ // ordinary file; the pinned-`in-flight` payload would make every local
634
+ // stamp probe unique and force a full reload on every search.
635
+ const hostile = "in-flight";
636
+ const snapshot = {
637
+ format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
638
+ schemaVersion: 1 as const,
639
+ createdAt: new Date().toISOString(),
640
+ sourceId: "remote",
641
+ includeTranscripts: true,
642
+ files: [
643
+ {
644
+ path: "facts/b.md",
645
+ sha256: createHash("sha256").update("beta").digest("hex"),
646
+ bytes: Buffer.byteLength("beta"),
647
+ mtimeMs: 0,
648
+ contentBase64: Buffer.from("beta").toString("base64"),
649
+ },
650
+ {
651
+ path: "state/embeddings.generation",
652
+ sha256: createHash("sha256").update(hostile).digest("hex"),
653
+ bytes: Buffer.byteLength(hostile),
654
+ mtimeMs: 0,
655
+ contentBase64: Buffer.from(hostile).toString("base64"),
656
+ },
657
+ ],
658
+ };
659
+ const result = await applyOfflineSyncSnapshot({ root, snapshot, baseFiles: [] });
660
+
661
+ assert.equal(await readUtf8(root, "state/embeddings.generation"), localMarker, "the local marker is node-local: an incoming marker must never overwrite it");
662
+ assert.equal(await readUtf8(root, "facts/b.md"), "beta", "unrelated incoming files still apply");
663
+ assert.equal(result.deleted, 0);
664
+ assert.deepEqual(result.conflicts, [], "an incoming marker must be skipped node-local, not turned into a conflict");
665
+
666
+ // A stale base entry for the marker plus its absence from the incoming
667
+ // snapshot is NOT a delete instruction either.
668
+ const staleBase = [{
669
+ path: "state/embeddings.generation",
670
+ sha256: createHash("sha256").update("stale").digest("hex"),
671
+ bytes: Buffer.byteLength("stale"),
672
+ mtimeMs: 0,
673
+ }];
674
+ const deletionProbe = await applyOfflineSyncSnapshot({
675
+ root,
676
+ snapshot: {
677
+ ...snapshot,
678
+ files: snapshot.files.filter((file) => file.path !== "state/embeddings.generation"),
679
+ },
680
+ baseFiles: staleBase,
681
+ });
682
+ assert.equal(await exists(root, "state/embeddings.generation"), true);
683
+ assert.equal(await readUtf8(root, "state/embeddings.generation"), localMarker);
684
+ assert.equal(deletionProbe.deleted, 0);
685
+ assert.deepEqual(deletionProbe.conflicts, [], "a marker absence is node-local, never a delete instruction or conflict");
686
+ } finally {
687
+ await rm(root, { recursive: true, force: true });
688
+ }
689
+ });
690
+
691
+ test("offline sync apply refuses differently cased marker paths on case-insensitive filesystems", async () => {
692
+ const root = await tempDir("remnic-offline-emb3176-marker-case");
693
+ try {
694
+ const localMarker = "0d9d2c11-5f6e-4a0e-8f1a-2b3c4d5e6f70";
695
+ await write(root, "facts/a.md", "alpha");
696
+ await write(root, "state/embeddings.generation", localMarker);
697
+
698
+ assert.equal(isEmbeddingGenerationMarkerPath("state/embeddings.generation"), true);
699
+ assert.equal(isEmbeddingGenerationMarkerPath("state/Embeddings.Generation"), true, "APFS/NTFS resolve this to the same file");
700
+ assert.equal(isEmbeddingGenerationMarkerPath("State/Embeddings.Generation"), true);
701
+ assert.equal(isEmbeddingGenerationMarkerPath("state/embeddings.json"), false);
702
+
703
+ const hostile = "in-flight";
704
+ const markerEntry = (relPath: string) => ({
705
+ path: relPath,
706
+ sha256: createHash("sha256").update(hostile).digest("hex"),
707
+ bytes: Buffer.byteLength(hostile),
708
+ mtimeMs: 0,
709
+ contentBase64: Buffer.from(hostile).toString("base64"),
710
+ });
711
+ const result = await applyOfflineSyncSnapshot({
712
+ root,
713
+ snapshot: {
714
+ format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
715
+ schemaVersion: 1 as const,
716
+ createdAt: new Date().toISOString(),
717
+ sourceId: "remote",
718
+ includeTranscripts: true,
719
+ files: [
720
+ {
721
+ path: "facts/b.md",
722
+ sha256: createHash("sha256").update("beta").digest("hex"),
723
+ bytes: Buffer.byteLength("beta"),
724
+ mtimeMs: 0,
725
+ contentBase64: Buffer.from("beta").toString("base64"),
726
+ },
727
+ markerEntry("state/Embeddings.Generation"),
728
+ ],
729
+ },
730
+ baseFiles: [],
731
+ });
732
+ assert.equal(await readUtf8(root, "state/embeddings.generation"), localMarker, "the canonical local marker survives a cased incoming copy");
733
+ assert.equal(await exists(root, "state/Embeddings.Generation"), false, "a cased marker path is refused node-local, never written");
734
+ assert.equal(await exists(root, "facts/b.md"), true, "unrelated incoming files still apply");
735
+ assert.deepEqual(result.conflicts, []);
736
+ } finally {
737
+ await rm(root, { recursive: true, force: true });
738
+ }
739
+ });
740
+
741
+ test("a hostile incoming marker leaves the local warm embedding cache untouched", async () => {
742
+ const root = await tempDir("remnic-offline-emb3176-marker-warm");
743
+ try {
744
+ const vectorFor = (text: string): number[] => {
745
+ const vec = [0, 0, 0, 0];
746
+ for (let i = 0; i < text.length; i++) vec[i % 4] += text.charCodeAt(i) % 13;
747
+ return vec;
748
+ };
749
+ const unregister = registerHostEmbeddingProvider(root, {
750
+ id: "host-model-a",
751
+ model: "host-model-a",
752
+ embed: async (text: string) => vectorFor(text),
753
+ });
754
+ let reads = 0;
755
+ const countingIo: EmbeddingIndexStoreIo = {
756
+ readUtf8: async (filePath) => {
757
+ reads += 1;
758
+ return readFile(filePath, "utf-8");
759
+ },
760
+ writeUtf8: async (filePath, contents) => {
761
+ await mkdir(path.dirname(filePath), { recursive: true });
762
+ await writeFile(filePath, contents, "utf-8");
763
+ },
764
+ };
765
+ const fallback = new EmbeddingFallback(
766
+ parseConfig({
767
+ memoryDir: root,
768
+ embeddingFallbackEnabled: true,
769
+ embeddingFallbackProvider: "openai",
770
+ openaiApiKey: false,
771
+ }),
772
+ countingIo,
773
+ );
774
+ try {
775
+ await fallback.indexFile("mem-a", "parent fact about database indexes", "facts/a.md");
776
+ assert.equal((await fallback.search("parent fact about database indexes", 5))[0]?.id, "mem-a");
777
+ const readsBeforeApply = reads;
778
+
779
+ const hostile = "in-flight";
780
+ await applyOfflineSyncSnapshot({
781
+ root,
782
+ snapshot: {
783
+ format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
784
+ schemaVersion: 1 as const,
785
+ createdAt: new Date().toISOString(),
786
+ sourceId: "remote",
787
+ includeTranscripts: true,
788
+ files: [{
789
+ path: "state/embeddings.generation",
790
+ sha256: createHash("sha256").update(hostile).digest("hex"),
791
+ bytes: Buffer.byteLength(hostile),
792
+ mtimeMs: 0,
793
+ contentBase64: Buffer.from(hostile).toString("base64"),
794
+ }],
795
+ },
796
+ baseFiles: [],
797
+ });
798
+
799
+ const warm = await fallback.search("parent fact about database indexes", 5);
800
+ assert.equal(warm[0]?.id, "mem-a", "the local generation is still served");
801
+ assert.equal(
802
+ reads,
803
+ readsBeforeApply,
804
+ "the local marker survived the apply: the warm cache keeps its stable stamp and performs no index reads",
805
+ );
806
+ } finally {
807
+ unregister();
808
+ }
809
+ } finally {
810
+ await rm(root, { recursive: true, force: true });
811
+ }
812
+ });
813
+
814
+ test("offline sync changeset apply refuses marker upserts and deletes, continues ordinary changes", async () => {
815
+ const root = await tempDir("remnic-offline-emb3176-marker-changeset");
816
+ try {
817
+ const localMarker = "7f6c1e64-9b2a-4c7d-8e0f-1a2b3c4d5e6f";
818
+ const hostile = "in-flight";
819
+ const sha = (value: string) => createHash("sha256").update(value).digest("hex");
820
+ await write(root, "facts/a.md", "alpha");
821
+ await write(root, "state/embeddings.generation", localMarker);
822
+
823
+ const result = await applyOfflineSyncChangeset({
824
+ root,
825
+ currentFiles: [
826
+ { path: "facts/a.md", sha256: sha("alpha"), bytes: 5, mtimeMs: 0 },
827
+ { path: "state/embeddings.generation", sha256: sha(localMarker), bytes: localMarker.length, mtimeMs: 0 },
828
+ ],
829
+ changeset: {
830
+ format: "remnic.offline-sync.changeset.v1",
831
+ schemaVersion: 1,
832
+ createdAt: new Date().toISOString(),
833
+ sourceId: "peer",
834
+ includeTranscripts: false,
835
+ changes: [
836
+ { type: "upsert", path: "facts/b.md", file: { path: "facts/b.md", sha256: sha("beta"), bytes: 4, mtimeMs: 0, contentBase64: Buffer.from("beta").toString("base64") } },
837
+ { type: "upsert", path: "state/embeddings.generation", file: { path: "state/embeddings.generation", sha256: sha(hostile), bytes: hostile.length, mtimeMs: 0, contentBase64: Buffer.from(hostile).toString("base64") } },
838
+ ],
839
+ } as never,
840
+ });
841
+ const deletionResult = await applyOfflineSyncChangeset({
842
+ root,
843
+ currentFiles: [
844
+ { path: "facts/a.md", sha256: sha("alpha"), bytes: 5, mtimeMs: 0 },
845
+ { path: "facts/b.md", sha256: sha("beta"), bytes: 4, mtimeMs: 0 },
846
+ { path: "state/embeddings.generation", sha256: sha(localMarker), bytes: localMarker.length, mtimeMs: 0 },
847
+ ],
848
+ changeset: {
849
+ format: "remnic.offline-sync.changeset.v1",
850
+ schemaVersion: 1,
851
+ createdAt: new Date().toISOString(),
852
+ sourceId: "peer",
853
+ includeTranscripts: false,
854
+ changes: [
855
+ { type: "delete", path: "facts/a.md", baseSha256: sha("alpha"), mtimeMs: 0 },
856
+ { type: "delete", path: "state/embeddings.generation", baseSha256: sha(localMarker), mtimeMs: 0 },
857
+ ],
858
+ } as never,
859
+ });
860
+
861
+ assert.equal(await readUtf8(root, "state/embeddings.generation"), localMarker, "node-local marker: neither an incoming upsert nor a delete may touch it");
862
+ assert.equal(await exists(root, "state/embeddings.generation"), true);
863
+ assert.equal(await readUtf8(root, "facts/b.md"), "beta", "ordinary upserts continue");
864
+ assert.equal(await exists(root, "facts/a.md"), false, "ordinary deletes continue");
865
+ assert.equal(deletionResult.appliedDeletes, 1);
866
+ assert.deepEqual(result.conflicts, [], "a marker upsert must be skipped node-local, not turned into a conflict");
867
+ assert.deepEqual(deletionResult.conflicts, [], "a marker delete must be skipped node-local, not turned into a conflict");
868
+ } finally {
869
+ await rm(root, { recursive: true, force: true });
870
+ }
871
+ });
872
+
873
+ test("offline sync file content chunks refuse the generation marker", async () => {
874
+ const root = await tempDir("remnic-offline-emb3176-marker-chunk");
875
+ try {
876
+ await write(root, "state/embeddings.generation", "local-uuid");
877
+ const content = Buffer.from("in-flight");
878
+ await assert.rejects(
879
+ () =>
880
+ applyOfflineSyncFileContentChunk({
881
+ root,
882
+ sourceId: "peer",
883
+ path: "state/embeddings.generation",
884
+ sha256: createHash("sha256").update(content).digest("hex"),
885
+ bytes: content.length,
886
+ mtimeMs: 0,
887
+ content,
888
+ }),
889
+ /node-local/,
890
+ );
891
+ assert.equal(await readUtf8(root, "state/embeddings.generation"), "local-uuid", "the chunked hostile payload never lands");
892
+ } finally {
893
+ await rm(root, { recursive: true, force: true });
894
+ }
895
+ });
896
+
572
897
  test("offline sync includes durable runtime state and excludes only transient sync temp files", async () => {
573
898
  const root = await tempDir("remnic-offline-runtime-state");
574
899
  try {
@@ -25,7 +25,7 @@ import {
25
25
  } from "./transfer/fs-utils.js";
26
26
  import { parseFlexibleIsoTimestamp } from "./utils/iso-timestamp.js";
27
27
  import { EmbeddingIndexStorageError } from "./embedding-index-storage.js";
28
- import { matchesOfflineSyncDefaultExclude } from "./offline-sync-exclude-globs.js";
28
+ import { isEmbeddingGenerationMarkerPath, matchesOfflineSyncDefaultExclude } from "./offline-sync-exclude-globs.js";
29
29
  import {
30
30
  applyIncomingEmbeddingGenerations,
31
31
  assertEmbeddingGenerationStillIncluded,
@@ -1696,6 +1696,15 @@ export async function applyOfflineSyncSnapshot(options: {
1696
1696
  }
1697
1697
 
1698
1698
  if (incoming) {
1699
+ // The generation marker (PR #3176) is node-local on BOTH sides: an
1700
+ // incoming copy — old or hostile — never overwrites the local file
1701
+ // and never enters the base set. Other default excludes stay
1702
+ // apply-accepted (#1786: first-sync LCM sqlite bootstrapping).
1703
+ if (isEmbeddingGenerationMarkerPath(relPath)) {
1704
+ nextBase.delete(relPath);
1705
+ skipped += 1;
1706
+ continue;
1707
+ }
1699
1708
  if (currentEntry?.sha256 === incoming.sha256) {
1700
1709
  if (await setSafeFileMtime(root, relPath, incoming.mtimeMs)) {
1701
1710
  nextBase.set(relPath, toFileState(incoming));
@@ -1933,6 +1942,10 @@ export async function applyOfflineSyncChangeset(options: {
1933
1942
  for (const change of changeset.changes) {
1934
1943
  if (generationRouting.conflictedPaths.has(change.path)) continue;
1935
1944
  if (generationRouting.transactionHandled.has(change.path)) continue;
1945
+ if (isEmbeddingGenerationMarkerPath(change.path)) {
1946
+ skipped += 1;
1947
+ continue;
1948
+ }
1936
1949
  const currentEntry = currentMap.get(change.path);
1937
1950
  if (change.type === "upsert") {
1938
1951
  if (currentEntry?.sha256 === change.file.sha256) {
@@ -2225,6 +2238,9 @@ export async function applyOfflineSyncFileContentChunk(options: {
2225
2238
  if (shouldExcludeRelPath(relPath, includeTranscripts)) {
2226
2239
  throw new Error(`offline sync file content path is excluded: ${relPath}`);
2227
2240
  }
2241
+ if (isEmbeddingGenerationMarkerPath(relPath)) {
2242
+ throw new Error(`offline sync file content path is node-local: ${relPath}`);
2243
+ }
2228
2244
  const sha256 = assertSha256(options.sha256, "sha256");
2229
2245
  const bytes = assertNonNegativeInteger(options.bytes, "bytes");
2230
2246
  const mtimeMs = assertOfflineSyncMtimeMs(options.mtimeMs, "mtimeMs");