@indigoai-us/hq-cloud 6.16.65 → 6.16.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/dist/bin/sync-runner-company.d.ts +14 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-company.js +42 -8
  4. package/dist/bin/sync-runner-company.js.map +1 -1
  5. package/dist/bin/sync-runner-company.test.js +62 -7
  6. package/dist/bin/sync-runner-company.test.js.map +1 -1
  7. package/dist/bin/sync-runner-planning.d.ts +12 -3
  8. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  9. package/dist/bin/sync-runner-planning.js +65 -3
  10. package/dist/bin/sync-runner-planning.js.map +1 -1
  11. package/dist/bin/sync-runner-planning.test.js +172 -1
  12. package/dist/bin/sync-runner-planning.test.js.map +1 -1
  13. package/dist/bin/sync-runner-watch-loop.d.ts +51 -2
  14. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  15. package/dist/bin/sync-runner-watch-loop.js +409 -165
  16. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  17. package/dist/bin/sync-runner-watch-routes.d.ts +19 -2
  18. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
  19. package/dist/bin/sync-runner-watch-routes.js +40 -5
  20. package/dist/bin/sync-runner-watch-routes.js.map +1 -1
  21. package/dist/bin/sync-runner-watch-routes.test.js +47 -0
  22. package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
  23. package/dist/bin/sync-runner.d.ts +59 -5
  24. package/dist/bin/sync-runner.d.ts.map +1 -1
  25. package/dist/bin/sync-runner.js +48 -4
  26. package/dist/bin/sync-runner.js.map +1 -1
  27. package/dist/bin/sync-runner.test.js +1553 -192
  28. package/dist/bin/sync-runner.test.js.map +1 -1
  29. package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
  30. package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
  31. package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
  32. package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
  33. package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
  34. package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
  35. package/dist/cli/onepass-pull-planner.test.js +266 -0
  36. package/dist/cli/onepass-pull-planner.test.js.map +1 -0
  37. package/dist/cli/push-scan-exclusions.test.js +4 -4
  38. package/dist/cli/push-scan-exclusions.test.js.map +1 -1
  39. package/dist/cli/share.d.ts +31 -1
  40. package/dist/cli/share.d.ts.map +1 -1
  41. package/dist/cli/share.js +143 -8
  42. package/dist/cli/share.js.map +1 -1
  43. package/dist/cli/share.test.js +350 -5
  44. package/dist/cli/share.test.js.map +1 -1
  45. package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
  46. package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
  47. package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
  48. package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
  49. package/dist/cli/sync-scope.test.js +8 -0
  50. package/dist/cli/sync-scope.test.js.map +1 -1
  51. package/dist/cli/sync.d.ts +12 -0
  52. package/dist/cli/sync.d.ts.map +1 -1
  53. package/dist/cli/sync.js +297 -125
  54. package/dist/cli/sync.js.map +1 -1
  55. package/dist/cli/sync.test.js +255 -2
  56. package/dist/cli/sync.test.js.map +1 -1
  57. package/dist/journal-row-store.d.ts +40 -5
  58. package/dist/journal-row-store.d.ts.map +1 -1
  59. package/dist/journal-row-store.js +240 -21
  60. package/dist/journal-row-store.js.map +1 -1
  61. package/dist/journal-row-store.test.js +54 -0
  62. package/dist/journal-row-store.test.js.map +1 -1
  63. package/dist/journal-snapshot-v4.d.ts.map +1 -1
  64. package/dist/journal-snapshot-v4.js +11 -6
  65. package/dist/journal-snapshot-v4.js.map +1 -1
  66. package/dist/journal-store.test.js +324 -7
  67. package/dist/journal-store.test.js.map +1 -1
  68. package/dist/journal.d.ts +51 -16
  69. package/dist/journal.d.ts.map +1 -1
  70. package/dist/journal.js +303 -79
  71. package/dist/journal.js.map +1 -1
  72. package/dist/lib/exit-codes.d.ts +15 -0
  73. package/dist/lib/exit-codes.d.ts.map +1 -1
  74. package/dist/lib/exit-codes.js +15 -0
  75. package/dist/lib/exit-codes.js.map +1 -1
  76. package/dist/personal-vault.d.ts.map +1 -1
  77. package/dist/personal-vault.js +10 -3
  78. package/dist/personal-vault.js.map +1 -1
  79. package/dist/personal-vault.test.js +35 -0
  80. package/dist/personal-vault.test.js.map +1 -1
  81. package/dist/remote-pull.d.ts.map +1 -1
  82. package/dist/remote-pull.js +9 -2
  83. package/dist/remote-pull.js.map +1 -1
  84. package/dist/remote-pull.test.js +59 -2
  85. package/dist/remote-pull.test.js.map +1 -1
  86. package/dist/s3.d.ts +1 -0
  87. package/dist/s3.d.ts.map +1 -1
  88. package/dist/s3.js +20 -2
  89. package/dist/s3.js.map +1 -1
  90. package/dist/s3.list-cost.test.d.ts +2 -0
  91. package/dist/s3.list-cost.test.d.ts.map +1 -0
  92. package/dist/s3.list-cost.test.js +51 -0
  93. package/dist/s3.list-cost.test.js.map +1 -0
  94. package/dist/sync/area-ledger.d.ts +27 -3
  95. package/dist/sync/area-ledger.d.ts.map +1 -1
  96. package/dist/sync/area-ledger.js +149 -39
  97. package/dist/sync/area-ledger.js.map +1 -1
  98. package/dist/sync/area-ledger.test.js +192 -5
  99. package/dist/sync/area-ledger.test.js.map +1 -1
  100. package/dist/sync/parallel-sync-coordinator.test.js +29 -0
  101. package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
  102. package/dist/sync/realtime-telemetry.d.ts +1 -1
  103. package/dist/sync/realtime-telemetry.d.ts.map +1 -1
  104. package/dist/sync/realtime-telemetry.js +14 -6
  105. package/dist/sync/realtime-telemetry.js.map +1 -1
  106. package/dist/sync/realtime-telemetry.test.js +136 -16
  107. package/dist/sync/realtime-telemetry.test.js.map +1 -1
  108. package/dist/sync/remote-file-spool.d.ts +16 -0
  109. package/dist/sync/remote-file-spool.d.ts.map +1 -0
  110. package/dist/sync/remote-file-spool.js +91 -0
  111. package/dist/sync/remote-file-spool.js.map +1 -0
  112. package/dist/sync/remote-file-spool.test.d.ts +2 -0
  113. package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
  114. package/dist/sync/remote-file-spool.test.js +77 -0
  115. package/dist/sync/remote-file-spool.test.js.map +1 -0
  116. package/dist/sync/state-store.d.ts +6 -0
  117. package/dist/sync/state-store.d.ts.map +1 -1
  118. package/dist/sync/state-store.js +14 -2
  119. package/dist/sync/state-store.js.map +1 -1
  120. package/dist/watcher.d.ts +12 -0
  121. package/dist/watcher.d.ts.map +1 -1
  122. package/dist/watcher.js +131 -0
  123. package/dist/watcher.js.map +1 -1
  124. package/dist/watcher.test.js +63 -0
  125. package/dist/watcher.test.js.map +1 -1
  126. package/package.json +2 -1
package/dist/journal.js CHANGED
@@ -17,7 +17,7 @@ import * as os from "os";
17
17
  import * as path from "path";
18
18
  import * as crypto from "crypto";
19
19
  import { JournalRowOverlay, JournalRowStore, journalEntryFingerprint, } from "./journal-row-store.js";
20
- import { createJournalRecordProxy } from "./journal-record-proxy.js";
20
+ import { createJournalRecordProxy, journalRecordRows } from "./journal-record-proxy.js";
21
21
  import { decodeJournalSnapshot, decodedJournalRows } from "./journal-snapshot-decoder.js";
22
22
  import { forgetSharedAreaJournalRows } from "./journal-snapshot-rows.js";
23
23
  import { encodeV4JournalSnapshotChunks } from "./journal-snapshot-v4.js";
@@ -114,6 +114,9 @@ export function journalRows(journal) {
114
114
  const backing = storeBackedJournals.get(journal);
115
115
  if (backing)
116
116
  return backing.store;
117
+ const proxied = journalRecordRows(journal.files);
118
+ if (proxied)
119
+ return proxied;
117
120
  const decoded = decodedJournalRows(journal);
118
121
  if (decoded)
119
122
  return decoded.rows;
@@ -146,9 +149,12 @@ export function closeCachedAreaJournalMigration(slug) {
146
149
  closeCachedAreaMigration(getStateDir(), slug);
147
150
  }
148
151
  /**
149
- * Release all process-local area bridges after a completed runner pass.
150
- * They hold lazy StateStore/aggregate caches but no file descriptors; durable
151
- * state is authoritative, so a later pass can safely construct a fresh bridge.
152
+ * Release all process-local area bridges at runner shutdown.
153
+ *
154
+ * A bridge owns one packed aggregate per authoritative journal. It is retained
155
+ * through ordinary watch passes so a later scoped callback cannot re-pack the
156
+ * whole journal synchronously. Durable layout/phase changes use the narrower
157
+ * close helper above; shutdown is the terminal memory boundary.
152
158
  */
153
159
  export function releaseCachedAreaJournalMigrations() {
154
160
  for (const [cacheKey, cached] of cachedAreaMigrations) {
@@ -191,7 +197,8 @@ function readStoreBackedJournal(journal, options = {}) {
191
197
  existing.pullsJson = JSON.stringify(journal.pulls ?? []);
192
198
  return journal;
193
199
  }
194
- const decoded = decodedJournalRows(journal);
200
+ const proxied = journalRecordRows(journal.files);
201
+ const decoded = proxied ?? decodedJournalRows(journal);
195
202
  if (decoded) {
196
203
  if (options.privateOverlay) {
197
204
  const store = new JournalRowOverlay(decoded);
@@ -207,7 +214,22 @@ function readStoreBackedJournal(journal, options = {}) {
207
214
  });
208
215
  return result;
209
216
  }
210
- decoded.enableChangeTracking();
217
+ if (decoded instanceof JournalRowStore)
218
+ decoded.enableChangeTracking();
219
+ else {
220
+ const store = new JournalRowOverlay(decoded);
221
+ const result = { ...journal, files: createJournalRecordProxy(store) };
222
+ if (Object.hasOwn(journal, "pulls"))
223
+ result.pulls = structuredClone(journal.pulls);
224
+ storeBackedJournals.set(result, {
225
+ store,
226
+ version: result.version,
227
+ lastSync: result.lastSync,
228
+ pullsJson: JSON.stringify(result.pulls ?? []),
229
+ keysNormalized: false,
230
+ });
231
+ return result;
232
+ }
211
233
  storeBackedJournals.set(journal, {
212
234
  store: decoded,
213
235
  version: journal.version,
@@ -279,11 +301,27 @@ function journalDeltaFromStoreDirtyLog(backing, next) {
279
301
  }
280
302
  function checkpointStoreBackedJournal(backing, journal) {
281
303
  backing.store.drainDirtyLog();
282
- backing.store.clearCommittedChanges?.();
283
304
  backing.version = journal.version;
284
305
  backing.lastSync = journal.lastSync;
285
306
  backing.pullsJson = JSON.stringify(journal.pulls ?? []);
286
307
  }
308
+ /**
309
+ * Move one full-pass facade onto the immutable durable revision it just
310
+ * published. Its prior private overlay contains the now-committed rows; merely
311
+ * clearing that overlay would make the caller's journal roll back to the old
312
+ * packed base. The StateStore reducer publishes a fresh overlay for every
313
+ * delta, so later scoped writes replace the aggregate instead of mutating this
314
+ * pass's newly established base.
315
+ */
316
+ function rebaseStoreBackedJournal(backing, journal, durable) {
317
+ const rows = journalRecordRows(durable.files) ?? decodedJournalRows(durable);
318
+ const store = new JournalRowOverlay(rows ?? new RecordJournalRows(durable.files));
319
+ journal.version = durable.version;
320
+ journal.lastSync = durable.lastSync;
321
+ journal.pulls = structuredClone(durable.pulls ?? []);
322
+ journal.files = createJournalRecordProxy(store);
323
+ backing.store = store;
324
+ }
287
325
  function storeBackedJournalNeedsMigration(backing, journal) {
288
326
  return journal.version !== JOURNAL_VERSION_CURRENT ||
289
327
  !Array.isArray(journal.pulls) ||
@@ -357,16 +395,15 @@ function entryFingerprint(entry) {
357
395
  */
358
396
  export function journalDeltaFromBaseline(baseline, next) {
359
397
  const upserts = {};
360
- const files = next.files;
361
- for (const key of Object.keys(files)) {
362
- const entry = files[key];
398
+ const nextRows = journalRows(next);
399
+ for (const [key, entry] of nextRows.entries()) {
363
400
  if (baseline.rows.get(key) === entryFingerprint(entry))
364
401
  continue;
365
402
  upserts[key] = entry;
366
403
  }
367
404
  const deletes = [];
368
405
  for (const key of baseline.rows.keys()) {
369
- if (!Object.hasOwn(files, key))
406
+ if (!nextRows.has(key))
370
407
  deletes.push(key);
371
408
  }
372
409
  const delta = {
@@ -510,7 +547,11 @@ export function readJournal(slug) {
510
547
  // mutable without deep-cloning the aggregate before every pass.
511
548
  return readStoreBackedJournal(areaMigration.readAreaJournalCached().journal, { privateOverlay: true });
512
549
  }
513
- return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration));
550
+ // The retained keyed projection and this full-pass facade can borrow the
551
+ // same immutable packed rows. Each owns only its own overlay, so an
552
+ // uncommitted full pass is never visible to a scoped reader (and a scoped
553
+ // delta cannot dirty the full pass).
554
+ return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration, true), { privateOverlay: true });
514
555
  }
515
556
  /**
516
557
  * Read the mutable outer shell required by a full pull without detaching every
@@ -521,9 +562,8 @@ export function readJournal(slug) {
521
562
  * own changes in a private overlay. `writeJournal()` drains that overlay's
522
563
  * dirty log, avoiding a full-ledger fingerprint at every download checkpoint.
523
564
  *
524
- * Legacy journals deliberately keep `readJournal()`'s packed-record boundary.
525
- * That path has different ownership and dirty-log semantics, and does not make
526
- * the area-authoritative structured-clone copy that this reader avoids.
565
+ * Legacy and area-authoritative journals both borrow immutable packed rows and
566
+ * put their caller-owned mutations in a private overlay.
527
567
  *
528
568
  * @internal Full-pull boundary; not a public journal record API.
529
569
  */
@@ -559,49 +599,55 @@ export function readJournalForPass(slug) {
559
599
  function readJournalAggregate(slug) {
560
600
  return readJournalAggregateWith(slug, openAreaMigrationIfAuthoritative(slug));
561
601
  }
562
- function readJournalAggregateWith(slug, areaMigration) {
602
+ function readJournalAggregateWith(slug, areaMigration, retainScopedProjection = false) {
563
603
  if (areaMigration)
564
604
  return areaMigration.readAreaJournal();
565
- if (StateStore.exists(getStateDir(), journalScopeId(slug))) {
566
- return openJournalStore(slug, emptyJournal()).getState().journal;
567
- }
568
- return openJournalStore(slug, readLegacyJournal(slug)).getState().journal;
605
+ const store = StateStore.exists(getStateDir(), journalScopeId(slug))
606
+ ? openJournalStore(slug, emptyJournal())
607
+ : openJournalStore(slug, readLegacyJournal(slug));
608
+ if (retainScopedProjection) {
609
+ // A full pass has already paid to recover this packed snapshot. Retain its
610
+ // projection for the next scoped pass instead of making that pass decode
611
+ // the same vault before it looks up even one named row. Read-only
612
+ // enumeration deliberately does not retain: watcher warm-up may inspect
613
+ // every locator and must not evict the hot personal projection.
614
+ retainScopedJournalStore(slug, new JournalStore(slug, store));
615
+ }
616
+ return store.getState().journal;
569
617
  }
570
618
  /**
571
619
  * Read a mutable, detached snapshot for named rows in a scoped pass.
572
620
  *
573
- * Area-authoritative journals use the ledger's keyed reader so the work is
574
- * proportional to the named keys rather than the full cached aggregate. The
575
- * legacy store has no keyed persistence surface, so it reads the named keys
576
- * straight out of the aggregate (O(K)) and, only when delete roots are named,
577
- * makes one linear pass over the rows with an ancestor-set lookup per row. It
578
- * must never be rows x roots: a watch-mode batch names one delete root per
579
- * changed path, and a warm-up catch-up hands the pass ~10k of them against a
580
- * 70k-row vault journal.
621
+ * Area-authoritative journals use the ledger's keyed reader. Other journals
622
+ * retain one refreshed packed projection after the preceding full pass, so a
623
+ * scoped read is proportional to named keys rather than a fresh snapshot
624
+ * recovery. Only named delete roots require a linear row scan, with an
625
+ * ancestor-set lookup per row; it must never be rows x roots.
581
626
  */
582
627
  export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
583
628
  const areaMigration = openAreaMigrationIfAuthoritative(slug);
584
629
  if (areaMigration) {
585
630
  return rememberJournalBaseline(areaMigration.readAreaJournalScoped(keys, deleteScopeRoots));
586
631
  }
587
- const full = readJournalAggregate(slug);
588
- const fullRows = journalRows(full);
632
+ const store = scopedJournalStore(slug);
633
+ store.refresh();
634
+ const metadata = store.metadata();
589
635
  const files = {};
590
636
  const roots = new Set(deleteScopeRoots);
591
637
  if (roots.has("")) {
592
- for (const [key, entry] of fullRows.entries()) {
638
+ for (const [key, entry] of store.scanPrefix("")) {
593
639
  files[key] = structuredClone(entry);
594
640
  }
595
641
  }
596
642
  else {
597
643
  for (const key of new Set(keys)) {
598
- const entry = fullRows.get(key);
644
+ const entry = store.get(key);
599
645
  if (entry === undefined)
600
646
  continue;
601
647
  files[key] = structuredClone(entry);
602
648
  }
603
649
  if (roots.size > 0) {
604
- for (const [key, entry] of fullRows.entries()) {
650
+ for (const [key, entry] of store.scanPrefix("")) {
605
651
  if (Object.hasOwn(files, key))
606
652
  continue;
607
653
  if (isUnderAnyRoot(key, roots))
@@ -610,10 +656,10 @@ export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
610
656
  }
611
657
  }
612
658
  return rememberJournalBaseline({
613
- version: full.version,
614
- lastSync: full.lastSync,
659
+ version: metadata.version,
660
+ lastSync: metadata.lastSync,
615
661
  files,
616
- pulls: structuredClone(full.pulls ?? []),
662
+ pulls: structuredClone(metadata.pulls),
617
663
  });
618
664
  }
619
665
  /**
@@ -694,9 +740,22 @@ function journalStoreOptions(slug, initialJournal) {
694
740
  }
695
741
  if (record.type === JOURNAL_DELTA_RECORD_TYPE) {
696
742
  const delta = record.payload;
697
- for (const key of delta.deletes)
698
- delete state.journal.files[key];
699
- Object.assign(state.journal.files, delta.upserts);
743
+ const existingRows = journalRecordRows(state.journal.files) ?? decodedJournalRows(state.journal);
744
+ if (existingRows) {
745
+ // StateStore's aggregate is also the immutable source for any
746
+ // full-pass facade opened before this append. Once a first delta is
747
+ // observed, put durable changes in a separate overlay rather than
748
+ // mutating that shared packed row store.
749
+ const rows = existingRows instanceof JournalRowOverlay
750
+ ? existingRows.withCommittedDelta(delta.upserts, delta.deletes)
751
+ : new JournalRowOverlay(existingRows).withCommittedDelta(delta.upserts, delta.deletes);
752
+ state.journal.files = createJournalRecordProxy(rows);
753
+ }
754
+ else {
755
+ for (const key of delta.deletes)
756
+ delete state.journal.files[key];
757
+ Object.assign(state.journal.files, delta.upserts);
758
+ }
700
759
  state.journal.version = delta.version;
701
760
  state.journal.lastSync = delta.lastSync ?? state.journal.lastSync;
702
761
  state.journal.pulls = delta.pulls ?? state.journal.pulls;
@@ -735,6 +794,10 @@ function openAreaMigrationIfAuthoritative(slug) {
735
794
  closeCachedAreaMigration(stateDir, slug);
736
795
  return undefined;
737
796
  }
797
+ // Area authority replaces the legacy v3 aggregate. Drop any retained legacy
798
+ // projection before a scoped reader can accidentally keep both generations
799
+ // resident (or use the obsolete writer after cutover).
800
+ releaseScopedJournalStore(slug);
738
801
  const resolverPath = path.join(areaLayoutDirectory(stateDir, slug), "area-resolver.json");
739
802
  let resolverJson;
740
803
  try {
@@ -799,19 +862,20 @@ export function openJournalStateStoreForMigration(slug) {
799
862
  async function openJournalStoreAsync(slug, initialJournal) {
800
863
  return StateStore.openAsync(journalStoreOptions(slug, initialJournal));
801
864
  }
802
- /**
803
- * Long-lived keyed projection over the v3 snapshot/WAL authority.
804
- *
805
- * The first open recovers once. Subsequent refreshes consume only authenticated
806
- * complete WAL frames, rebuilding the projection solely after a generation
807
- * publication or invalid-tail fallback.
808
- */
809
865
  export class JournalStore {
810
866
  slug;
811
867
  store;
812
868
  areaMigration;
813
869
  files = new JournalRowStore({ trackChanges: false });
814
870
  usesSharedRows = false;
871
+ /**
872
+ * Durable deltas this keyed session has observed after its base snapshot.
873
+ * The base can be shared with a full-pass facade, so publishing a scoped
874
+ * delta here must never mutate it in place.
875
+ */
876
+ committedOverlay = new Map();
877
+ /** Base keys deleted then reinserted by durable records move to the tail. */
878
+ committedMoved = new Set();
815
879
  /**
816
880
  * Pass-local rows that have not reached the shared projection yet. A missing
817
881
  * value represents a staged delete; every other value is a frozen copy of
@@ -841,27 +905,56 @@ export class JournalStore {
841
905
  this.assertOpen();
842
906
  if (this.overlay.has(key))
843
907
  return this.overlay.get(key);
908
+ if (this.committedOverlay.has(key))
909
+ return this.committedOverlay.get(key);
844
910
  return this.files.get(key);
845
911
  }
846
912
  scanPrefix(prefix) {
847
913
  this.assertOpen();
848
- const rows = new Map(this.files.entriesForPrefix(prefix));
849
914
  const normalizedPrefix = prefix.replace(/\/+$/, "");
850
- for (const [key, entry] of this.overlay) {
851
- if (!isJournalStorePrefixMatch(key, normalizedPrefix))
852
- continue;
853
- if (entry === undefined)
854
- rows.delete(key);
855
- else
856
- rows.set(key, entry);
857
- }
858
- return Array.from(rows.entries());
915
+ const files = this.files;
916
+ const committedOverlay = this.committedOverlay;
917
+ const committedMoved = this.committedMoved;
918
+ const overlay = this.overlay;
919
+ const iterate = function* () {
920
+ for (const [key, entry] of files.entriesForPrefix(normalizedPrefix)) {
921
+ if (committedMoved.has(key))
922
+ continue;
923
+ const replacement = overlay.has(key)
924
+ ? overlay.get(key)
925
+ : committedOverlay.has(key)
926
+ ? committedOverlay.get(key)
927
+ : entry;
928
+ if (replacement !== undefined)
929
+ yield [key, replacement];
930
+ }
931
+ // A Map replacement preserves the base row's position and appends newly
932
+ // introduced overlay rows. Preserve that observable order without holding
933
+ // a materialized JournalEntry for every base row at once.
934
+ const additions = new Map(committedOverlay);
935
+ for (const [key, entry] of overlay)
936
+ additions.set(key, entry);
937
+ for (const [key, entry] of additions) {
938
+ if (entry === undefined || (files.has(key) && !committedMoved.has(key)))
939
+ continue;
940
+ if (isJournalStorePrefixMatch(key, normalizedPrefix))
941
+ yield [key, entry];
942
+ }
943
+ };
944
+ const scan = {
945
+ [Symbol.iterator]: iterate,
946
+ filter: ((predicate, thisArg) => Array.from(iterate()).filter(predicate, thisArg)),
947
+ };
948
+ return scan;
859
949
  }
860
950
  rowCount() {
861
951
  if (this.disposed)
862
952
  return 0;
953
+ const replacements = new Map(this.committedOverlay);
954
+ for (const [key, entry] of this.overlay)
955
+ replacements.set(key, entry);
863
956
  let count = this.files.size;
864
- for (const [key, entry] of this.overlay) {
957
+ for (const [key, entry] of replacements) {
865
958
  const existsInSharedStore = this.files.has(key);
866
959
  if (entry === undefined) {
867
960
  if (existsInSharedStore)
@@ -877,7 +970,9 @@ export class JournalStore {
877
970
  set(key, entry) {
878
971
  this.assertOpen();
879
972
  const ownedEntry = immutableCopy(entry);
880
- const sharedEntry = this.files.get(key);
973
+ const sharedEntry = this.committedOverlay.has(key)
974
+ ? this.committedOverlay.get(key)
975
+ : this.files.get(key);
881
976
  if (sharedEntry !== undefined && sameEntry(sharedEntry, ownedEntry)) {
882
977
  this.overlay.delete(key);
883
978
  return;
@@ -890,12 +985,12 @@ export class JournalStore {
890
985
  if (this.overlay.has(key)) {
891
986
  if (this.overlay.get(key) === undefined)
892
987
  return false;
893
- if (!this.files.has(key)) {
988
+ if (!this.committedOverlay.has(key) && !this.files.has(key)) {
894
989
  this.overlay.delete(key);
895
990
  return true;
896
991
  }
897
992
  }
898
- else if (!this.files.has(key)) {
993
+ else if (this.get(key) === undefined) {
899
994
  return false;
900
995
  }
901
996
  this.overlay.set(key, undefined);
@@ -934,6 +1029,17 @@ export class JournalStore {
934
1029
  pulls: this.pulls,
935
1030
  });
936
1031
  }
1032
+ /**
1033
+ * The StateStore aggregate after a durable append. Its row view is immutable
1034
+ * across later deltas, which lets a full-pass facade rebase without exposing
1035
+ * its completed checkpoint to subsequent scoped writers.
1036
+ */
1037
+ durableJournalForRebase() {
1038
+ this.assertOpen();
1039
+ if (this.areaMigration)
1040
+ return this.areaMigration.readAreaJournalCached().journal;
1041
+ return this.store.getState().journal;
1042
+ }
937
1043
  refresh() {
938
1044
  this.assertOpen();
939
1045
  if (this.areaMigration) {
@@ -960,12 +1066,10 @@ export class JournalStore {
960
1066
  this.applyRefresh(result);
961
1067
  }
962
1068
  applyRefresh(result) {
963
- if (result.recovered) {
1069
+ if (result.recovered || result.records.length > 0) {
964
1070
  this.replaceFromJournal(this.store.getState().journal);
965
1071
  return;
966
1072
  }
967
- for (const record of result.records)
968
- this.applyRecord(record.type, record.payload);
969
1073
  }
970
1074
  applyDelta(delta) {
971
1075
  this.assertOpen();
@@ -1025,13 +1129,11 @@ export class JournalStore {
1025
1129
  this.applyCommitted(appended);
1026
1130
  ensureJournalLocator(this.slug);
1027
1131
  }
1028
- applyCommitted(appended) {
1029
- if (appended.refresh.recovered)
1030
- this.replaceFromJournal(this.store.getState().journal);
1031
- else
1032
- for (const record of appended.refresh.records)
1033
- this.applyRecord(record.type, record.payload);
1034
- this.applyRecord(appended.type, appended.payload);
1132
+ applyCommitted(_appended) {
1133
+ // StateStore has reduced the just-appended durable frame already. Repoint
1134
+ // this cache at that immutable aggregate rather than replaying the frame
1135
+ // into a second mutable overlay beside it.
1136
+ this.replaceFromJournal(this.store.getState().journal);
1035
1137
  }
1036
1138
  /**
1037
1139
  * Packing copies every source row into typed arrays, so this projection never
@@ -1040,21 +1142,30 @@ export class JournalStore {
1040
1142
  * pass-local), which is what put the outposts over Node's heap ceiling.
1041
1143
  */
1042
1144
  replaceFromJournal(journal) {
1043
- const decoded = decodedJournalRows(journal);
1145
+ const proxied = journalRecordRows(journal.files);
1146
+ const decoded = proxied instanceof JournalRowStore || proxied instanceof JournalRowOverlay
1147
+ ? proxied
1148
+ : decodedJournalRows(journal);
1044
1149
  if (decoded) {
1045
- if (!this.usesSharedRows && this.files !== decoded)
1150
+ if (!this.usesSharedRows && this.files instanceof JournalRowStore && this.files !== decoded) {
1046
1151
  this.files.clear();
1152
+ }
1047
1153
  this.files = decoded;
1048
1154
  this.usesSharedRows = true;
1155
+ this.committedOverlay.clear();
1156
+ this.committedMoved.clear();
1049
1157
  this.version = journal.version;
1050
1158
  this.lastSync = journal.lastSync;
1051
1159
  this.pulls = structuredClone(journal.pulls ?? []);
1052
1160
  return;
1053
1161
  }
1054
- if (this.usesSharedRows)
1162
+ if (this.usesSharedRows || !(this.files instanceof JournalRowStore)) {
1055
1163
  this.files = new JournalRowStore({ trackChanges: false });
1164
+ }
1056
1165
  this.files.replaceFromRecord(journal.files);
1057
1166
  this.usesSharedRows = false;
1167
+ this.committedOverlay.clear();
1168
+ this.committedMoved.clear();
1058
1169
  this.version = journal.version;
1059
1170
  this.lastSync = journal.lastSync;
1060
1171
  this.pulls = structuredClone(journal.pulls ?? []);
@@ -1068,9 +1179,15 @@ export class JournalStore {
1068
1179
  throw new Error("journal: keyed session received an invalid WAL record");
1069
1180
  }
1070
1181
  for (const key of payload.deletes)
1071
- this.files.delete(key);
1072
- for (const [key, entry] of Object.entries(payload.upserts))
1073
- this.files.set(key, entry);
1182
+ this.committedOverlay.set(key, undefined);
1183
+ for (const [key, entry] of Object.entries(payload.upserts)) {
1184
+ if (this.committedOverlay.get(key) === undefined && this.committedOverlay.has(key)) {
1185
+ this.committedOverlay.delete(key);
1186
+ if (this.files.has(key))
1187
+ this.committedMoved.add(key);
1188
+ }
1189
+ this.committedOverlay.set(key, immutableCopy(entry));
1190
+ }
1074
1191
  this.version = payload.version;
1075
1192
  this.lastSync = payload.lastSync ?? this.lastSync;
1076
1193
  this.pulls = structuredClone(payload.pulls ?? this.pulls);
@@ -1081,7 +1198,9 @@ export class JournalStore {
1081
1198
  return;
1082
1199
  this.disposed = true;
1083
1200
  this.overlay.clear();
1084
- if (!this.usesSharedRows)
1201
+ this.committedOverlay.clear();
1202
+ this.committedMoved.clear();
1203
+ if (!this.usesSharedRows && this.files instanceof JournalRowStore)
1085
1204
  this.files.clear();
1086
1205
  this.store = undefined;
1087
1206
  this.areaMigration = undefined;
@@ -1108,6 +1227,69 @@ function immutableCopy(value) {
1108
1227
  freeze(copy);
1109
1228
  return copy;
1110
1229
  }
1230
+ // A scoped watcher pass must not recover a 700k-row snapshot just to inspect
1231
+ // one path. A runner can alternate the personal vault and its companies in the
1232
+ // same realtime batch, so one cache entry turns that alternation into a fresh
1233
+ // whole-snapshot recovery on every dispatch. Keep a deliberately small LRU:
1234
+ // enough for the runner's active journal set, bounded so an unusually broad
1235
+ // company fanout cannot retain every journal for the lifetime of the process.
1236
+ const MAX_RETAINED_SCOPED_JOURNALS = 4;
1237
+ const cachedScopedJournalStores = new Map();
1238
+ function scopedJournalStoreCacheKey(slug) {
1239
+ return `${getStateDir()}\u0000${slug}`;
1240
+ }
1241
+ function cachedScopedJournalStoreFor(slug) {
1242
+ const cacheKey = scopedJournalStoreCacheKey(slug);
1243
+ const cached = cachedScopedJournalStores.get(cacheKey);
1244
+ if (cached === undefined)
1245
+ return undefined;
1246
+ // Map insertion order is our LRU order. A hot scope must not be evicted by
1247
+ // a brief company fanout while the runner is still using it.
1248
+ cachedScopedJournalStores.delete(cacheKey);
1249
+ cachedScopedJournalStores.set(cacheKey, cached);
1250
+ return cached;
1251
+ }
1252
+ function retainScopedJournalStore(slug, store) {
1253
+ const cacheKey = scopedJournalStoreCacheKey(slug);
1254
+ const previous = cachedScopedJournalStores.get(cacheKey);
1255
+ if (previous !== undefined && previous !== store)
1256
+ previous.dispose();
1257
+ cachedScopedJournalStores.delete(cacheKey);
1258
+ cachedScopedJournalStores.set(cacheKey, store);
1259
+ while (cachedScopedJournalStores.size > MAX_RETAINED_SCOPED_JOURNALS) {
1260
+ const oldest = cachedScopedJournalStores.entries().next().value;
1261
+ if (oldest === undefined)
1262
+ break;
1263
+ const [oldestKey, oldestStore] = oldest;
1264
+ cachedScopedJournalStores.delete(oldestKey);
1265
+ oldestStore.dispose();
1266
+ }
1267
+ return store;
1268
+ }
1269
+ function scopedJournalStore(slug) {
1270
+ const cached = cachedScopedJournalStoreFor(slug);
1271
+ if (cached)
1272
+ return cached;
1273
+ return retainScopedJournalStore(slug, openJournalStoreSession(slug));
1274
+ }
1275
+ /** Drop one legacy keyed projection after an authoritative area cutover. */
1276
+ export function releaseScopedJournalStore(slug) {
1277
+ const cacheKey = scopedJournalStoreCacheKey(slug);
1278
+ const cached = cachedScopedJournalStores.get(cacheKey);
1279
+ if (cached === undefined)
1280
+ return;
1281
+ cachedScopedJournalStores.delete(cacheKey);
1282
+ cached.dispose();
1283
+ }
1284
+ /**
1285
+ * Release retained scoped projections when a watch runner stops. Durable state
1286
+ * remains in the StateStore; this only drops process-local decoded snapshots.
1287
+ */
1288
+ export function disposeScopedJournalStores() {
1289
+ for (const store of cachedScopedJournalStores.values())
1290
+ store.dispose();
1291
+ cachedScopedJournalStores.clear();
1292
+ }
1111
1293
  /** Open one keyed session, seeding a missing v3 store from legacy JSON once. */
1112
1294
  export function openJournalStoreSession(slug) {
1113
1295
  const areaMigration = openAreaMigrationIfAuthoritative(slug);
@@ -1620,6 +1802,47 @@ export function writeJournal(slug, journal) {
1620
1802
  ensureJournalLocator(slug);
1621
1803
  return;
1622
1804
  }
1805
+ const baseline = journalBaselines.get(journal);
1806
+ const cached = cachedScopedJournalStoreFor(slug);
1807
+ if (cached && (storeBacking || baseline)) {
1808
+ // The scoped reader captured only its named rows, so its baseline is the
1809
+ // authoritative delta boundary. The retained session refreshes and appends
1810
+ // under the normal StateStore lock; it never exposes partial durable rows
1811
+ // to another writer.
1812
+ const delta = storeBacking
1813
+ ? journalDeltaFromStoreDirtyLog(storeBacking, journal)
1814
+ : baseline
1815
+ ? journalDeltaFromBaseline(baseline, journal)
1816
+ : undefined;
1817
+ if (delta && hasJournalDelta(delta)) {
1818
+ // Carry ONLY the metadata fields this delta actually changed. The
1819
+ // retained session's `metadata()` is a snapshot that goes stale the
1820
+ // moment another writer advances `lastSync` or appends a pull record,
1821
+ // and filling the untouched field from it republished that stale value
1822
+ // over the concurrent writer's durable one — dropping pull history or
1823
+ // reverting `lastSync`. The store's reducer already preserves a field
1824
+ // the delta omits, so omission is the whole fix.
1825
+ const metadata = {};
1826
+ if (delta.lastSync !== undefined)
1827
+ metadata.lastSync = delta.lastSync;
1828
+ if (delta.pulls !== undefined)
1829
+ metadata.pulls = delta.pulls;
1830
+ const hasMetadata = delta.lastSync !== undefined || delta.pulls !== undefined;
1831
+ cached.applyDelta({
1832
+ upserts: delta.upserts,
1833
+ deletes: delta.deletes,
1834
+ ...(hasMetadata ? { metadata } : {}),
1835
+ });
1836
+ }
1837
+ if (storeBacking) {
1838
+ rebaseStoreBackedJournal(storeBacking, journal, cached.durableJournalForRebase());
1839
+ checkpointStoreBackedJournal(storeBacking, journal);
1840
+ }
1841
+ else
1842
+ rememberJournalBaseline(journal);
1843
+ ensureJournalLocator(slug);
1844
+ return;
1845
+ }
1623
1846
  const store = openJournalStore(slug, journal);
1624
1847
  if (store.wasCreated) {
1625
1848
  // The store was just seeded from this journal, so its state already matches
@@ -1643,7 +1866,6 @@ export function writeJournal(slug, journal) {
1643
1866
  // The WAL still compacts on its own schedule (record count or byte ceiling),
1644
1867
  // so a full snapshot is written periodically rather than never — recovery
1645
1868
  // stays bounded, it just no longer happens on every single update.
1646
- const baseline = journalBaselines.get(journal);
1647
1869
  const delta = storeBacking
1648
1870
  ? journalDeltaFromStoreDirtyLog(storeBacking, journal)
1649
1871
  : baseline
@@ -1652,8 +1874,10 @@ export function writeJournal(slug, journal) {
1652
1874
  store.append(JOURNAL_DELTA_RECORD_TYPE, delta);
1653
1875
  // A caller can checkpoint more than once during one pass. Advancing only its
1654
1876
  // own tracking state means the next checkpoint again publishes only new mutations.
1655
- if (storeBacking)
1877
+ if (storeBacking) {
1878
+ rebaseStoreBackedJournal(storeBacking, journal, store.getState().journal);
1656
1879
  checkpointStoreBackedJournal(storeBacking, journal);
1880
+ }
1657
1881
  else
1658
1882
  rememberJournalBaseline(journal);
1659
1883
  // The locator is immutable content, so it only needs writing if it is absent