@indigoai-us/hq-cloud 6.16.51 → 6.16.52-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/dist/backup-prune.d.ts +0 -20
  2. package/dist/backup-prune.d.ts.map +1 -1
  3. package/dist/backup-prune.js +3 -62
  4. package/dist/backup-prune.js.map +1 -1
  5. package/dist/backup-prune.test.js +0 -25
  6. package/dist/backup-prune.test.js.map +1 -1
  7. package/dist/bin/rescue-runner.js +1 -2
  8. package/dist/bin/rescue-runner.js.map +1 -1
  9. package/dist/bin/sync-runner-company.d.ts +6 -0
  10. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  11. package/dist/bin/sync-runner-company.js +30 -1
  12. package/dist/bin/sync-runner-company.js.map +1 -1
  13. package/dist/bin/sync-runner-company.test.js +55 -0
  14. package/dist/bin/sync-runner-company.test.js.map +1 -1
  15. package/dist/bin/sync-runner-planning.d.ts +12 -3
  16. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  17. package/dist/bin/sync-runner-planning.js +65 -3
  18. package/dist/bin/sync-runner-planning.js.map +1 -1
  19. package/dist/bin/sync-runner-planning.test.js +108 -1
  20. package/dist/bin/sync-runner-planning.test.js.map +1 -1
  21. package/dist/bin/sync-runner-watch-loop.d.ts +48 -2
  22. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  23. package/dist/bin/sync-runner-watch-loop.js +311 -116
  24. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  25. package/dist/bin/sync-runner.d.ts +33 -5
  26. package/dist/bin/sync-runner.d.ts.map +1 -1
  27. package/dist/bin/sync-runner.js +34 -4
  28. package/dist/bin/sync-runner.js.map +1 -1
  29. package/dist/bin/sync-runner.test.js +802 -79
  30. package/dist/bin/sync-runner.test.js.map +1 -1
  31. package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
  32. package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
  33. package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
  34. package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
  35. package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
  36. package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
  37. package/dist/cli/onepass-pull-planner.test.js +266 -0
  38. package/dist/cli/onepass-pull-planner.test.js.map +1 -0
  39. package/dist/cli/rescue-core.d.ts +0 -40
  40. package/dist/cli/rescue-core.d.ts.map +1 -1
  41. package/dist/cli/rescue-core.js +109 -1205
  42. package/dist/cli/rescue-core.js.map +1 -1
  43. package/dist/cli/rescue-rsync-preflight.test.js +4 -256
  44. package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
  45. package/dist/cli/rescue.d.ts +0 -4
  46. package/dist/cli/rescue.d.ts.map +1 -1
  47. package/dist/cli/rescue.js +1 -20
  48. package/dist/cli/rescue.js.map +1 -1
  49. package/dist/cli/rescue.test.js +1 -16
  50. package/dist/cli/rescue.test.js.map +1 -1
  51. package/dist/cli/share-presign-transport.test.js +0 -2
  52. package/dist/cli/share-presign-transport.test.js.map +1 -1
  53. package/dist/cli/share.d.ts +19 -0
  54. package/dist/cli/share.d.ts.map +1 -1
  55. package/dist/cli/share.js +57 -37
  56. package/dist/cli/share.js.map +1 -1
  57. package/dist/cli/share.test.js +87 -93
  58. package/dist/cli/share.test.js.map +1 -1
  59. package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
  60. package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
  61. package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
  62. package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
  63. package/dist/cli/sync-scope.test.js +8 -0
  64. package/dist/cli/sync-scope.test.js.map +1 -1
  65. package/dist/cli/sync.d.ts +12 -0
  66. package/dist/cli/sync.d.ts.map +1 -1
  67. package/dist/cli/sync.js +226 -118
  68. package/dist/cli/sync.js.map +1 -1
  69. package/dist/cli/sync.test.js +215 -2
  70. package/dist/cli/sync.test.js.map +1 -1
  71. package/dist/cli/watch-event-push-conflict.test.js +0 -2
  72. package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
  73. package/dist/cognito-auth.d.ts +0 -30
  74. package/dist/cognito-auth.d.ts.map +1 -1
  75. package/dist/cognito-auth.js +0 -175
  76. package/dist/cognito-auth.js.map +1 -1
  77. package/dist/index.d.ts +2 -2
  78. package/dist/index.d.ts.map +1 -1
  79. package/dist/index.js +1 -1
  80. package/dist/index.js.map +1 -1
  81. package/dist/journal-row-store.d.ts +9 -5
  82. package/dist/journal-row-store.d.ts.map +1 -1
  83. package/dist/journal-row-store.js +70 -10
  84. package/dist/journal-row-store.js.map +1 -1
  85. package/dist/journal-snapshot-v4.d.ts.map +1 -1
  86. package/dist/journal-snapshot-v4.js +11 -6
  87. package/dist/journal-snapshot-v4.js.map +1 -1
  88. package/dist/journal-store.test.js +259 -7
  89. package/dist/journal-store.test.js.map +1 -1
  90. package/dist/journal.d.ts +37 -12
  91. package/dist/journal.d.ts.map +1 -1
  92. package/dist/journal.js +291 -76
  93. package/dist/journal.js.map +1 -1
  94. package/dist/lib/exit-codes.d.ts +15 -0
  95. package/dist/lib/exit-codes.d.ts.map +1 -1
  96. package/dist/lib/exit-codes.js +15 -0
  97. package/dist/lib/exit-codes.js.map +1 -1
  98. package/dist/machine-auth.test.js +0 -295
  99. package/dist/machine-auth.test.js.map +1 -1
  100. package/dist/object-io.d.ts +2 -2
  101. package/dist/object-io.d.ts.map +1 -1
  102. package/dist/object-io.js +3 -18
  103. package/dist/object-io.js.map +1 -1
  104. package/dist/personal-vault.d.ts.map +1 -1
  105. package/dist/personal-vault.js +10 -3
  106. package/dist/personal-vault.js.map +1 -1
  107. package/dist/personal-vault.test.js +35 -0
  108. package/dist/personal-vault.test.js.map +1 -1
  109. package/dist/remote-pull.d.ts.map +1 -1
  110. package/dist/remote-pull.js +9 -2
  111. package/dist/remote-pull.js.map +1 -1
  112. package/dist/remote-pull.test.js +59 -2
  113. package/dist/remote-pull.test.js.map +1 -1
  114. package/dist/s3.d.ts +1 -18
  115. package/dist/s3.d.ts.map +1 -1
  116. package/dist/s3.js +24 -47
  117. package/dist/s3.js.map +1 -1
  118. package/dist/s3.list-cost.test.d.ts +2 -0
  119. package/dist/s3.list-cost.test.d.ts.map +1 -0
  120. package/dist/s3.list-cost.test.js +51 -0
  121. package/dist/s3.list-cost.test.js.map +1 -0
  122. package/dist/s3.test.js +1 -27
  123. package/dist/s3.test.js.map +1 -1
  124. package/dist/sync/area-ledger.js +4 -4
  125. package/dist/sync/area-ledger.js.map +1 -1
  126. package/dist/sync/parallel-sync-coordinator.test.js +29 -0
  127. package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
  128. package/dist/sync/realtime-telemetry.d.ts +1 -1
  129. package/dist/sync/realtime-telemetry.d.ts.map +1 -1
  130. package/dist/sync/realtime-telemetry.js +14 -6
  131. package/dist/sync/realtime-telemetry.js.map +1 -1
  132. package/dist/sync/realtime-telemetry.test.js +136 -16
  133. package/dist/sync/realtime-telemetry.test.js.map +1 -1
  134. package/dist/sync/remote-file-spool.d.ts +16 -0
  135. package/dist/sync/remote-file-spool.d.ts.map +1 -0
  136. package/dist/sync/remote-file-spool.js +91 -0
  137. package/dist/sync/remote-file-spool.js.map +1 -0
  138. package/dist/sync/remote-file-spool.test.d.ts +2 -0
  139. package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
  140. package/dist/sync/remote-file-spool.test.js +77 -0
  141. package/dist/sync/remote-file-spool.test.js.map +1 -0
  142. package/dist/sync-core.d.ts +4 -6
  143. package/dist/sync-core.d.ts.map +1 -1
  144. package/dist/sync-core.js +4 -10
  145. package/dist/sync-core.js.map +1 -1
  146. package/dist/sync-core.test.js +1 -25
  147. package/dist/sync-core.test.js.map +1 -1
  148. package/package.json +2 -1
  149. package/dist/cli/rescue-breaker.test.d.ts +0 -2
  150. package/dist/cli/rescue-breaker.test.d.ts.map +0 -1
  151. package/dist/cli/rescue-breaker.test.js +0 -334
  152. package/dist/cli/rescue-breaker.test.js.map +0 -1
  153. package/dist/cli/rescue-realpath.test.d.ts +0 -2
  154. package/dist/cli/rescue-realpath.test.d.ts.map +0 -1
  155. package/dist/cli/rescue-realpath.test.js +0 -109
  156. package/dist/cli/rescue-realpath.test.js.map +0 -1
  157. package/dist/cli/rescue-snapshot-copy.test.d.ts +0 -2
  158. package/dist/cli/rescue-snapshot-copy.test.d.ts.map +0 -1
  159. package/dist/cli/rescue-snapshot-copy.test.js +0 -435
  160. package/dist/cli/rescue-snapshot-copy.test.js.map +0 -1
  161. package/dist/cli/rescue-snapshot-recovery.test.d.ts +0 -2
  162. package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +0 -1
  163. package/dist/cli/rescue-snapshot-recovery.test.js +0 -260
  164. package/dist/cli/rescue-snapshot-recovery.test.js.map +0 -1
  165. package/dist/cli/rescue-source-switch-floor.test.d.ts +0 -2
  166. package/dist/cli/rescue-source-switch-floor.test.d.ts.map +0 -1
  167. package/dist/cli/rescue-source-switch-floor.test.js +0 -210
  168. package/dist/cli/rescue-source-switch-floor.test.js.map +0 -1
package/dist/journal.js CHANGED
@@ -17,7 +17,7 @@ import * as os from "os";
17
17
  import * as path from "path";
18
18
  import * as crypto from "crypto";
19
19
  import { JournalRowOverlay, JournalRowStore, journalEntryFingerprint, } from "./journal-row-store.js";
20
- import { createJournalRecordProxy } from "./journal-record-proxy.js";
20
+ import { createJournalRecordProxy, journalRecordRows } from "./journal-record-proxy.js";
21
21
  import { decodeJournalSnapshot, decodedJournalRows } from "./journal-snapshot-decoder.js";
22
22
  import { forgetSharedAreaJournalRows } from "./journal-snapshot-rows.js";
23
23
  import { encodeV4JournalSnapshotChunks } from "./journal-snapshot-v4.js";
@@ -114,6 +114,9 @@ export function journalRows(journal) {
114
114
  const backing = storeBackedJournals.get(journal);
115
115
  if (backing)
116
116
  return backing.store;
117
+ const proxied = journalRecordRows(journal.files);
118
+ if (proxied)
119
+ return proxied;
117
120
  const decoded = decodedJournalRows(journal);
118
121
  if (decoded)
119
122
  return decoded.rows;
@@ -191,7 +194,8 @@ function readStoreBackedJournal(journal, options = {}) {
191
194
  existing.pullsJson = JSON.stringify(journal.pulls ?? []);
192
195
  return journal;
193
196
  }
194
- const decoded = decodedJournalRows(journal);
197
+ const proxied = journalRecordRows(journal.files);
198
+ const decoded = proxied ?? decodedJournalRows(journal);
195
199
  if (decoded) {
196
200
  if (options.privateOverlay) {
197
201
  const store = new JournalRowOverlay(decoded);
@@ -207,7 +211,22 @@ function readStoreBackedJournal(journal, options = {}) {
207
211
  });
208
212
  return result;
209
213
  }
210
- decoded.enableChangeTracking();
214
+ if (decoded instanceof JournalRowStore)
215
+ decoded.enableChangeTracking();
216
+ else {
217
+ const store = new JournalRowOverlay(decoded);
218
+ const result = { ...journal, files: createJournalRecordProxy(store) };
219
+ if (Object.hasOwn(journal, "pulls"))
220
+ result.pulls = structuredClone(journal.pulls);
221
+ storeBackedJournals.set(result, {
222
+ store,
223
+ version: result.version,
224
+ lastSync: result.lastSync,
225
+ pullsJson: JSON.stringify(result.pulls ?? []),
226
+ keysNormalized: false,
227
+ });
228
+ return result;
229
+ }
211
230
  storeBackedJournals.set(journal, {
212
231
  store: decoded,
213
232
  version: journal.version,
@@ -279,11 +298,27 @@ function journalDeltaFromStoreDirtyLog(backing, next) {
279
298
  }
280
299
  function checkpointStoreBackedJournal(backing, journal) {
281
300
  backing.store.drainDirtyLog();
282
- backing.store.clearCommittedChanges?.();
283
301
  backing.version = journal.version;
284
302
  backing.lastSync = journal.lastSync;
285
303
  backing.pullsJson = JSON.stringify(journal.pulls ?? []);
286
304
  }
305
+ /**
306
+ * Move one full-pass facade onto the immutable durable revision it just
307
+ * published. Its prior private overlay contains the now-committed rows; merely
308
+ * clearing that overlay would make the caller's journal roll back to the old
309
+ * packed base. The StateStore reducer publishes a fresh overlay for every
310
+ * delta, so later scoped writes replace the aggregate instead of mutating this
311
+ * pass's newly established base.
312
+ */
313
+ function rebaseStoreBackedJournal(backing, journal, durable) {
314
+ const rows = journalRecordRows(durable.files) ?? decodedJournalRows(durable);
315
+ const store = new JournalRowOverlay(rows ?? new RecordJournalRows(durable.files));
316
+ journal.version = durable.version;
317
+ journal.lastSync = durable.lastSync;
318
+ journal.pulls = structuredClone(durable.pulls ?? []);
319
+ journal.files = createJournalRecordProxy(store);
320
+ backing.store = store;
321
+ }
287
322
  function storeBackedJournalNeedsMigration(backing, journal) {
288
323
  return journal.version !== JOURNAL_VERSION_CURRENT ||
289
324
  !Array.isArray(journal.pulls) ||
@@ -357,16 +392,15 @@ function entryFingerprint(entry) {
357
392
  */
358
393
  export function journalDeltaFromBaseline(baseline, next) {
359
394
  const upserts = {};
360
- const files = next.files;
361
- for (const key of Object.keys(files)) {
362
- const entry = files[key];
395
+ const nextRows = journalRows(next);
396
+ for (const [key, entry] of nextRows.entries()) {
363
397
  if (baseline.rows.get(key) === entryFingerprint(entry))
364
398
  continue;
365
399
  upserts[key] = entry;
366
400
  }
367
401
  const deletes = [];
368
402
  for (const key of baseline.rows.keys()) {
369
- if (!Object.hasOwn(files, key))
403
+ if (!nextRows.has(key))
370
404
  deletes.push(key);
371
405
  }
372
406
  const delta = {
@@ -510,7 +544,11 @@ export function readJournal(slug) {
510
544
  // mutable without deep-cloning the aggregate before every pass.
511
545
  return readStoreBackedJournal(areaMigration.readAreaJournalCached().journal, { privateOverlay: true });
512
546
  }
513
- return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration));
547
+ // The retained keyed projection and this full-pass facade can borrow the
548
+ // same immutable packed rows. Each owns only its own overlay, so an
549
+ // uncommitted full pass is never visible to a scoped reader (and a scoped
550
+ // delta cannot dirty the full pass).
551
+ return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration, true), { privateOverlay: true });
514
552
  }
515
553
  /**
516
554
  * Read the mutable outer shell required by a full pull without detaching every
@@ -521,9 +559,8 @@ export function readJournal(slug) {
521
559
  * own changes in a private overlay. `writeJournal()` drains that overlay's
522
560
  * dirty log, avoiding a full-ledger fingerprint at every download checkpoint.
523
561
  *
524
- * Legacy journals deliberately keep `readJournal()`'s packed-record boundary.
525
- * That path has different ownership and dirty-log semantics, and does not make
526
- * the area-authoritative structured-clone copy that this reader avoids.
562
+ * Legacy and area-authoritative journals both borrow immutable packed rows and
563
+ * put their caller-owned mutations in a private overlay.
527
564
  *
528
565
  * @internal Full-pull boundary; not a public journal record API.
529
566
  */
@@ -559,49 +596,55 @@ export function readJournalForPass(slug) {
559
596
  function readJournalAggregate(slug) {
560
597
  return readJournalAggregateWith(slug, openAreaMigrationIfAuthoritative(slug));
561
598
  }
562
- function readJournalAggregateWith(slug, areaMigration) {
599
+ function readJournalAggregateWith(slug, areaMigration, retainScopedProjection = false) {
563
600
  if (areaMigration)
564
601
  return areaMigration.readAreaJournal();
565
- if (StateStore.exists(getStateDir(), journalScopeId(slug))) {
566
- return openJournalStore(slug, emptyJournal()).getState().journal;
567
- }
568
- return openJournalStore(slug, readLegacyJournal(slug)).getState().journal;
602
+ const store = StateStore.exists(getStateDir(), journalScopeId(slug))
603
+ ? openJournalStore(slug, emptyJournal())
604
+ : openJournalStore(slug, readLegacyJournal(slug));
605
+ if (retainScopedProjection) {
606
+ // A full pass has already paid to recover this packed snapshot. Retain its
607
+ // projection for the next scoped pass instead of making that pass decode
608
+ // the same vault before it looks up even one named row. Read-only
609
+ // enumeration deliberately does not retain: watcher warm-up may inspect
610
+ // every locator and must not evict the hot personal projection.
611
+ retainScopedJournalStore(slug, new JournalStore(slug, store));
612
+ }
613
+ return store.getState().journal;
569
614
  }
570
615
  /**
571
616
  * Read a mutable, detached snapshot for named rows in a scoped pass.
572
617
  *
573
- * Area-authoritative journals use the ledger's keyed reader so the work is
574
- * proportional to the named keys rather than the full cached aggregate. The
575
- * legacy store has no keyed persistence surface, so it reads the named keys
576
- * straight out of the aggregate (O(K)) and, only when delete roots are named,
577
- * makes one linear pass over the rows with an ancestor-set lookup per row. It
578
- * must never be rows x roots: a watch-mode batch names one delete root per
579
- * changed path, and a warm-up catch-up hands the pass ~10k of them against a
580
- * 70k-row vault journal.
618
+ * Area-authoritative journals use the ledger's keyed reader. Other journals
619
+ * retain one refreshed packed projection after the preceding full pass, so a
620
+ * scoped read is proportional to named keys rather than a fresh snapshot
621
+ * recovery. Only named delete roots require a linear row scan, with an
622
+ * ancestor-set lookup per row; it must never be rows x roots.
581
623
  */
582
624
  export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
583
625
  const areaMigration = openAreaMigrationIfAuthoritative(slug);
584
626
  if (areaMigration) {
585
627
  return rememberJournalBaseline(areaMigration.readAreaJournalScoped(keys, deleteScopeRoots));
586
628
  }
587
- const full = readJournalAggregate(slug);
588
- const fullRows = journalRows(full);
629
+ const store = scopedJournalStore(slug);
630
+ store.refresh();
631
+ const metadata = store.metadata();
589
632
  const files = {};
590
633
  const roots = new Set(deleteScopeRoots);
591
634
  if (roots.has("")) {
592
- for (const [key, entry] of fullRows.entries()) {
635
+ for (const [key, entry] of store.scanPrefix("")) {
593
636
  files[key] = structuredClone(entry);
594
637
  }
595
638
  }
596
639
  else {
597
640
  for (const key of new Set(keys)) {
598
- const entry = fullRows.get(key);
641
+ const entry = store.get(key);
599
642
  if (entry === undefined)
600
643
  continue;
601
644
  files[key] = structuredClone(entry);
602
645
  }
603
646
  if (roots.size > 0) {
604
- for (const [key, entry] of fullRows.entries()) {
647
+ for (const [key, entry] of store.scanPrefix("")) {
605
648
  if (Object.hasOwn(files, key))
606
649
  continue;
607
650
  if (isUnderAnyRoot(key, roots))
@@ -610,10 +653,10 @@ export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
610
653
  }
611
654
  }
612
655
  return rememberJournalBaseline({
613
- version: full.version,
614
- lastSync: full.lastSync,
656
+ version: metadata.version,
657
+ lastSync: metadata.lastSync,
615
658
  files,
616
- pulls: structuredClone(full.pulls ?? []),
659
+ pulls: structuredClone(metadata.pulls),
617
660
  });
618
661
  }
619
662
  /**
@@ -694,9 +737,22 @@ function journalStoreOptions(slug, initialJournal) {
694
737
  }
695
738
  if (record.type === JOURNAL_DELTA_RECORD_TYPE) {
696
739
  const delta = record.payload;
697
- for (const key of delta.deletes)
698
- delete state.journal.files[key];
699
- Object.assign(state.journal.files, delta.upserts);
740
+ const existingRows = journalRecordRows(state.journal.files) ?? decodedJournalRows(state.journal);
741
+ if (existingRows) {
742
+ // StateStore's aggregate is also the immutable source for any
743
+ // full-pass facade opened before this append. Once a first delta is
744
+ // observed, put durable changes in a separate overlay rather than
745
+ // mutating that shared packed row store.
746
+ const rows = existingRows instanceof JournalRowOverlay
747
+ ? existingRows.withCommittedDelta(delta.upserts, delta.deletes)
748
+ : new JournalRowOverlay(existingRows).withCommittedDelta(delta.upserts, delta.deletes);
749
+ state.journal.files = createJournalRecordProxy(rows);
750
+ }
751
+ else {
752
+ for (const key of delta.deletes)
753
+ delete state.journal.files[key];
754
+ Object.assign(state.journal.files, delta.upserts);
755
+ }
700
756
  state.journal.version = delta.version;
701
757
  state.journal.lastSync = delta.lastSync ?? state.journal.lastSync;
702
758
  state.journal.pulls = delta.pulls ?? state.journal.pulls;
@@ -735,6 +791,10 @@ function openAreaMigrationIfAuthoritative(slug) {
735
791
  closeCachedAreaMigration(stateDir, slug);
736
792
  return undefined;
737
793
  }
794
+ // Area authority replaces the legacy v3 aggregate. Drop any retained legacy
795
+ // projection before a scoped reader can accidentally keep both generations
796
+ // resident (or use the obsolete writer after cutover).
797
+ releaseScopedJournalStore(slug);
738
798
  const resolverPath = path.join(areaLayoutDirectory(stateDir, slug), "area-resolver.json");
739
799
  let resolverJson;
740
800
  try {
@@ -799,19 +859,20 @@ export function openJournalStateStoreForMigration(slug) {
799
859
  async function openJournalStoreAsync(slug, initialJournal) {
800
860
  return StateStore.openAsync(journalStoreOptions(slug, initialJournal));
801
861
  }
802
- /**
803
- * Long-lived keyed projection over the v3 snapshot/WAL authority.
804
- *
805
- * The first open recovers once. Subsequent refreshes consume only authenticated
806
- * complete WAL frames, rebuilding the projection solely after a generation
807
- * publication or invalid-tail fallback.
808
- */
809
862
  export class JournalStore {
810
863
  slug;
811
864
  store;
812
865
  areaMigration;
813
866
  files = new JournalRowStore({ trackChanges: false });
814
867
  usesSharedRows = false;
868
+ /**
869
+ * Durable deltas this keyed session has observed after its base snapshot.
870
+ * The base can be shared with a full-pass facade, so publishing a scoped
871
+ * delta here must never mutate it in place.
872
+ */
873
+ committedOverlay = new Map();
874
+ /** Base keys deleted then reinserted by durable records move to the tail. */
875
+ committedMoved = new Set();
815
876
  /**
816
877
  * Pass-local rows that have not reached the shared projection yet. A missing
817
878
  * value represents a staged delete; every other value is a frozen copy of
@@ -841,27 +902,56 @@ export class JournalStore {
841
902
  this.assertOpen();
842
903
  if (this.overlay.has(key))
843
904
  return this.overlay.get(key);
905
+ if (this.committedOverlay.has(key))
906
+ return this.committedOverlay.get(key);
844
907
  return this.files.get(key);
845
908
  }
846
909
  scanPrefix(prefix) {
847
910
  this.assertOpen();
848
- const rows = new Map(this.files.entriesForPrefix(prefix));
849
911
  const normalizedPrefix = prefix.replace(/\/+$/, "");
850
- for (const [key, entry] of this.overlay) {
851
- if (!isJournalStorePrefixMatch(key, normalizedPrefix))
852
- continue;
853
- if (entry === undefined)
854
- rows.delete(key);
855
- else
856
- rows.set(key, entry);
857
- }
858
- return Array.from(rows.entries());
912
+ const files = this.files;
913
+ const committedOverlay = this.committedOverlay;
914
+ const committedMoved = this.committedMoved;
915
+ const overlay = this.overlay;
916
+ const iterate = function* () {
917
+ for (const [key, entry] of files.entriesForPrefix(normalizedPrefix)) {
918
+ if (committedMoved.has(key))
919
+ continue;
920
+ const replacement = overlay.has(key)
921
+ ? overlay.get(key)
922
+ : committedOverlay.has(key)
923
+ ? committedOverlay.get(key)
924
+ : entry;
925
+ if (replacement !== undefined)
926
+ yield [key, replacement];
927
+ }
928
+ // A Map replacement preserves the base row's position and appends newly
929
+ // introduced overlay rows. Preserve that observable order without holding
930
+ // a materialized JournalEntry for every base row at once.
931
+ const additions = new Map(committedOverlay);
932
+ for (const [key, entry] of overlay)
933
+ additions.set(key, entry);
934
+ for (const [key, entry] of additions) {
935
+ if (entry === undefined || (files.has(key) && !committedMoved.has(key)))
936
+ continue;
937
+ if (isJournalStorePrefixMatch(key, normalizedPrefix))
938
+ yield [key, entry];
939
+ }
940
+ };
941
+ const scan = {
942
+ [Symbol.iterator]: iterate,
943
+ filter: ((predicate, thisArg) => Array.from(iterate()).filter(predicate, thisArg)),
944
+ };
945
+ return scan;
859
946
  }
860
947
  rowCount() {
861
948
  if (this.disposed)
862
949
  return 0;
950
+ const replacements = new Map(this.committedOverlay);
951
+ for (const [key, entry] of this.overlay)
952
+ replacements.set(key, entry);
863
953
  let count = this.files.size;
864
- for (const [key, entry] of this.overlay) {
954
+ for (const [key, entry] of replacements) {
865
955
  const existsInSharedStore = this.files.has(key);
866
956
  if (entry === undefined) {
867
957
  if (existsInSharedStore)
@@ -877,7 +967,9 @@ export class JournalStore {
877
967
  set(key, entry) {
878
968
  this.assertOpen();
879
969
  const ownedEntry = immutableCopy(entry);
880
- const sharedEntry = this.files.get(key);
970
+ const sharedEntry = this.committedOverlay.has(key)
971
+ ? this.committedOverlay.get(key)
972
+ : this.files.get(key);
881
973
  if (sharedEntry !== undefined && sameEntry(sharedEntry, ownedEntry)) {
882
974
  this.overlay.delete(key);
883
975
  return;
@@ -890,12 +982,12 @@ export class JournalStore {
890
982
  if (this.overlay.has(key)) {
891
983
  if (this.overlay.get(key) === undefined)
892
984
  return false;
893
- if (!this.files.has(key)) {
985
+ if (!this.committedOverlay.has(key) && !this.files.has(key)) {
894
986
  this.overlay.delete(key);
895
987
  return true;
896
988
  }
897
989
  }
898
- else if (!this.files.has(key)) {
990
+ else if (this.get(key) === undefined) {
899
991
  return false;
900
992
  }
901
993
  this.overlay.set(key, undefined);
@@ -934,6 +1026,17 @@ export class JournalStore {
934
1026
  pulls: this.pulls,
935
1027
  });
936
1028
  }
1029
+ /**
1030
+ * The StateStore aggregate after a durable append. Its row view is immutable
1031
+ * across later deltas, which lets a full-pass facade rebase without exposing
1032
+ * its completed checkpoint to subsequent scoped writers.
1033
+ */
1034
+ durableJournalForRebase() {
1035
+ this.assertOpen();
1036
+ if (this.areaMigration)
1037
+ return this.areaMigration.readAreaJournalCached().journal;
1038
+ return this.store.getState().journal;
1039
+ }
937
1040
  refresh() {
938
1041
  this.assertOpen();
939
1042
  if (this.areaMigration) {
@@ -960,12 +1063,10 @@ export class JournalStore {
960
1063
  this.applyRefresh(result);
961
1064
  }
962
1065
  applyRefresh(result) {
963
- if (result.recovered) {
1066
+ if (result.recovered || result.records.length > 0) {
964
1067
  this.replaceFromJournal(this.store.getState().journal);
965
1068
  return;
966
1069
  }
967
- for (const record of result.records)
968
- this.applyRecord(record.type, record.payload);
969
1070
  }
970
1071
  applyDelta(delta) {
971
1072
  this.assertOpen();
@@ -1025,13 +1126,11 @@ export class JournalStore {
1025
1126
  this.applyCommitted(appended);
1026
1127
  ensureJournalLocator(this.slug);
1027
1128
  }
1028
- applyCommitted(appended) {
1029
- if (appended.refresh.recovered)
1030
- this.replaceFromJournal(this.store.getState().journal);
1031
- else
1032
- for (const record of appended.refresh.records)
1033
- this.applyRecord(record.type, record.payload);
1034
- this.applyRecord(appended.type, appended.payload);
1129
+ applyCommitted(_appended) {
1130
+ // StateStore has reduced the just-appended durable frame already. Repoint
1131
+ // this cache at that immutable aggregate rather than replaying the frame
1132
+ // into a second mutable overlay beside it.
1133
+ this.replaceFromJournal(this.store.getState().journal);
1035
1134
  }
1036
1135
  /**
1037
1136
  * Packing copies every source row into typed arrays, so this projection never
@@ -1040,21 +1139,30 @@ export class JournalStore {
1040
1139
  * pass-local), which is what put the outposts over Node's heap ceiling.
1041
1140
  */
1042
1141
  replaceFromJournal(journal) {
1043
- const decoded = decodedJournalRows(journal);
1142
+ const proxied = journalRecordRows(journal.files);
1143
+ const decoded = proxied instanceof JournalRowStore || proxied instanceof JournalRowOverlay
1144
+ ? proxied
1145
+ : decodedJournalRows(journal);
1044
1146
  if (decoded) {
1045
- if (!this.usesSharedRows && this.files !== decoded)
1147
+ if (!this.usesSharedRows && this.files instanceof JournalRowStore && this.files !== decoded) {
1046
1148
  this.files.clear();
1149
+ }
1047
1150
  this.files = decoded;
1048
1151
  this.usesSharedRows = true;
1152
+ this.committedOverlay.clear();
1153
+ this.committedMoved.clear();
1049
1154
  this.version = journal.version;
1050
1155
  this.lastSync = journal.lastSync;
1051
1156
  this.pulls = structuredClone(journal.pulls ?? []);
1052
1157
  return;
1053
1158
  }
1054
- if (this.usesSharedRows)
1159
+ if (this.usesSharedRows || !(this.files instanceof JournalRowStore)) {
1055
1160
  this.files = new JournalRowStore({ trackChanges: false });
1161
+ }
1056
1162
  this.files.replaceFromRecord(journal.files);
1057
1163
  this.usesSharedRows = false;
1164
+ this.committedOverlay.clear();
1165
+ this.committedMoved.clear();
1058
1166
  this.version = journal.version;
1059
1167
  this.lastSync = journal.lastSync;
1060
1168
  this.pulls = structuredClone(journal.pulls ?? []);
@@ -1068,9 +1176,15 @@ export class JournalStore {
1068
1176
  throw new Error("journal: keyed session received an invalid WAL record");
1069
1177
  }
1070
1178
  for (const key of payload.deletes)
1071
- this.files.delete(key);
1072
- for (const [key, entry] of Object.entries(payload.upserts))
1073
- this.files.set(key, entry);
1179
+ this.committedOverlay.set(key, undefined);
1180
+ for (const [key, entry] of Object.entries(payload.upserts)) {
1181
+ if (this.committedOverlay.get(key) === undefined && this.committedOverlay.has(key)) {
1182
+ this.committedOverlay.delete(key);
1183
+ if (this.files.has(key))
1184
+ this.committedMoved.add(key);
1185
+ }
1186
+ this.committedOverlay.set(key, immutableCopy(entry));
1187
+ }
1074
1188
  this.version = payload.version;
1075
1189
  this.lastSync = payload.lastSync ?? this.lastSync;
1076
1190
  this.pulls = structuredClone(payload.pulls ?? this.pulls);
@@ -1081,7 +1195,9 @@ export class JournalStore {
1081
1195
  return;
1082
1196
  this.disposed = true;
1083
1197
  this.overlay.clear();
1084
- if (!this.usesSharedRows)
1198
+ this.committedOverlay.clear();
1199
+ this.committedMoved.clear();
1200
+ if (!this.usesSharedRows && this.files instanceof JournalRowStore)
1085
1201
  this.files.clear();
1086
1202
  this.store = undefined;
1087
1203
  this.areaMigration = undefined;
@@ -1108,6 +1224,69 @@ function immutableCopy(value) {
1108
1224
  freeze(copy);
1109
1225
  return copy;
1110
1226
  }
1227
+ // A scoped watcher pass must not recover a 700k-row snapshot just to inspect
1228
+ // one path. A runner can alternate the personal vault and its companies in the
1229
+ // same realtime batch, so one cache entry turns that alternation into a fresh
1230
+ // whole-snapshot recovery on every dispatch. Keep a deliberately small LRU:
1231
+ // enough for the runner's active journal set, bounded so an unusually broad
1232
+ // company fanout cannot retain every journal for the lifetime of the process.
1233
+ const MAX_RETAINED_SCOPED_JOURNALS = 4;
1234
+ const cachedScopedJournalStores = new Map();
1235
+ function scopedJournalStoreCacheKey(slug) {
1236
+ return `${getStateDir()}\u0000${slug}`;
1237
+ }
1238
+ function cachedScopedJournalStoreFor(slug) {
1239
+ const cacheKey = scopedJournalStoreCacheKey(slug);
1240
+ const cached = cachedScopedJournalStores.get(cacheKey);
1241
+ if (cached === undefined)
1242
+ return undefined;
1243
+ // Map insertion order is our LRU order. A hot scope must not be evicted by
1244
+ // a brief company fanout while the runner is still using it.
1245
+ cachedScopedJournalStores.delete(cacheKey);
1246
+ cachedScopedJournalStores.set(cacheKey, cached);
1247
+ return cached;
1248
+ }
1249
+ function retainScopedJournalStore(slug, store) {
1250
+ const cacheKey = scopedJournalStoreCacheKey(slug);
1251
+ const previous = cachedScopedJournalStores.get(cacheKey);
1252
+ if (previous !== undefined && previous !== store)
1253
+ previous.dispose();
1254
+ cachedScopedJournalStores.delete(cacheKey);
1255
+ cachedScopedJournalStores.set(cacheKey, store);
1256
+ while (cachedScopedJournalStores.size > MAX_RETAINED_SCOPED_JOURNALS) {
1257
+ const oldest = cachedScopedJournalStores.entries().next().value;
1258
+ if (oldest === undefined)
1259
+ break;
1260
+ const [oldestKey, oldestStore] = oldest;
1261
+ cachedScopedJournalStores.delete(oldestKey);
1262
+ oldestStore.dispose();
1263
+ }
1264
+ return store;
1265
+ }
1266
+ function scopedJournalStore(slug) {
1267
+ const cached = cachedScopedJournalStoreFor(slug);
1268
+ if (cached)
1269
+ return cached;
1270
+ return retainScopedJournalStore(slug, openJournalStoreSession(slug));
1271
+ }
1272
+ /** Drop one legacy keyed projection after an authoritative area cutover. */
1273
+ export function releaseScopedJournalStore(slug) {
1274
+ const cacheKey = scopedJournalStoreCacheKey(slug);
1275
+ const cached = cachedScopedJournalStores.get(cacheKey);
1276
+ if (cached === undefined)
1277
+ return;
1278
+ cachedScopedJournalStores.delete(cacheKey);
1279
+ cached.dispose();
1280
+ }
1281
+ /**
1282
+ * Release retained scoped projections when a watch runner stops. Durable state
1283
+ * remains in the StateStore; this only drops process-local decoded snapshots.
1284
+ */
1285
+ export function disposeScopedJournalStores() {
1286
+ for (const store of cachedScopedJournalStores.values())
1287
+ store.dispose();
1288
+ cachedScopedJournalStores.clear();
1289
+ }
1111
1290
  /** Open one keyed session, seeding a missing v3 store from legacy JSON once. */
1112
1291
  export function openJournalStoreSession(slug) {
1113
1292
  const areaMigration = openAreaMigrationIfAuthoritative(slug);
@@ -1620,6 +1799,41 @@ export function writeJournal(slug, journal) {
1620
1799
  ensureJournalLocator(slug);
1621
1800
  return;
1622
1801
  }
1802
+ const baseline = journalBaselines.get(journal);
1803
+ const cached = cachedScopedJournalStoreFor(slug);
1804
+ if (cached && (storeBacking || baseline)) {
1805
+ // The scoped reader captured only its named rows, so its baseline is the
1806
+ // authoritative delta boundary. The retained session refreshes and appends
1807
+ // under the normal StateStore lock; it never exposes partial durable rows
1808
+ // to another writer.
1809
+ const delta = storeBacking
1810
+ ? journalDeltaFromStoreDirtyLog(storeBacking, journal)
1811
+ : baseline
1812
+ ? journalDeltaFromBaseline(baseline, journal)
1813
+ : undefined;
1814
+ if (delta && hasJournalDelta(delta)) {
1815
+ const currentMetadata = cached.metadata();
1816
+ const metadata = delta.lastSync === undefined && delta.pulls === undefined
1817
+ ? undefined
1818
+ : {
1819
+ lastSync: delta.lastSync ?? currentMetadata.lastSync,
1820
+ pulls: delta.pulls ?? currentMetadata.pulls,
1821
+ };
1822
+ cached.applyDelta({
1823
+ upserts: delta.upserts,
1824
+ deletes: delta.deletes,
1825
+ ...(metadata === undefined ? {} : { metadata }),
1826
+ });
1827
+ }
1828
+ if (storeBacking) {
1829
+ rebaseStoreBackedJournal(storeBacking, journal, cached.durableJournalForRebase());
1830
+ checkpointStoreBackedJournal(storeBacking, journal);
1831
+ }
1832
+ else
1833
+ rememberJournalBaseline(journal);
1834
+ ensureJournalLocator(slug);
1835
+ return;
1836
+ }
1623
1837
  const store = openJournalStore(slug, journal);
1624
1838
  if (store.wasCreated) {
1625
1839
  // The store was just seeded from this journal, so its state already matches
@@ -1643,7 +1857,6 @@ export function writeJournal(slug, journal) {
1643
1857
  // The WAL still compacts on its own schedule (record count or byte ceiling),
1644
1858
  // so a full snapshot is written periodically rather than never — recovery
1645
1859
  // stays bounded, it just no longer happens on every single update.
1646
- const baseline = journalBaselines.get(journal);
1647
1860
  const delta = storeBacking
1648
1861
  ? journalDeltaFromStoreDirtyLog(storeBacking, journal)
1649
1862
  : baseline
@@ -1652,8 +1865,10 @@ export function writeJournal(slug, journal) {
1652
1865
  store.append(JOURNAL_DELTA_RECORD_TYPE, delta);
1653
1866
  // A caller can checkpoint more than once during one pass. Advancing only its
1654
1867
  // own tracking state means the next checkpoint again publishes only new mutations.
1655
- if (storeBacking)
1868
+ if (storeBacking) {
1869
+ rebaseStoreBackedJournal(storeBacking, journal, store.getState().journal);
1656
1870
  checkpointStoreBackedJournal(storeBacking, journal);
1871
+ }
1657
1872
  else
1658
1873
  rememberJournalBaseline(journal);
1659
1874
  // The locator is immutable content, so it only needs writing if it is absent