@indigoai-us/hq-cloud 6.16.58-alpha.3 → 6.16.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/dist/bin/sync-runner-company.d.ts +0 -14
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-company.js +89 -57
  4. package/dist/bin/sync-runner-company.js.map +1 -1
  5. package/dist/bin/sync-runner-company.test.js +313 -56
  6. package/dist/bin/sync-runner-company.test.js.map +1 -1
  7. package/dist/bin/sync-runner-planning.d.ts +3 -12
  8. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  9. package/dist/bin/sync-runner-planning.js +3 -65
  10. package/dist/bin/sync-runner-planning.js.map +1 -1
  11. package/dist/bin/sync-runner-planning.test.js +1 -172
  12. package/dist/bin/sync-runner-planning.test.js.map +1 -1
  13. package/dist/bin/sync-runner-watch-loop.d.ts +2 -51
  14. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  15. package/dist/bin/sync-runner-watch-loop.js +165 -409
  16. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  17. package/dist/bin/sync-runner-watch-routes.d.ts +2 -19
  18. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
  19. package/dist/bin/sync-runner-watch-routes.js +5 -40
  20. package/dist/bin/sync-runner-watch-routes.js.map +1 -1
  21. package/dist/bin/sync-runner-watch-routes.test.js +0 -47
  22. package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
  23. package/dist/bin/sync-runner.d.ts +13 -59
  24. package/dist/bin/sync-runner.d.ts.map +1 -1
  25. package/dist/bin/sync-runner.js +4 -48
  26. package/dist/bin/sync-runner.js.map +1 -1
  27. package/dist/bin/sync-runner.test.js +229 -1499
  28. package/dist/bin/sync-runner.test.js.map +1 -1
  29. package/dist/cli/share.d.ts +0 -19
  30. package/dist/cli/share.d.ts.map +1 -1
  31. package/dist/cli/share.js +0 -55
  32. package/dist/cli/share.js.map +1 -1
  33. package/dist/cli/share.test.js +1 -87
  34. package/dist/cli/share.test.js.map +1 -1
  35. package/dist/cli/sync-scope.test.js +0 -8
  36. package/dist/cli/sync-scope.test.js.map +1 -1
  37. package/dist/cli/sync.d.ts +8 -12
  38. package/dist/cli/sync.d.ts.map +1 -1
  39. package/dist/cli/sync.js +152 -302
  40. package/dist/cli/sync.js.map +1 -1
  41. package/dist/cli/sync.test.js +126 -256
  42. package/dist/cli/sync.test.js.map +1 -1
  43. package/dist/cognito-auth.d.ts +34 -1
  44. package/dist/cognito-auth.d.ts.map +1 -1
  45. package/dist/cognito-auth.js +92 -21
  46. package/dist/cognito-auth.js.map +1 -1
  47. package/dist/context.d.ts +5 -0
  48. package/dist/context.d.ts.map +1 -1
  49. package/dist/context.js +92 -12
  50. package/dist/context.js.map +1 -1
  51. package/dist/context.test.js +77 -2
  52. package/dist/context.test.js.map +1 -1
  53. package/dist/ignore.d.ts +10 -0
  54. package/dist/ignore.d.ts.map +1 -1
  55. package/dist/ignore.js +47 -5
  56. package/dist/ignore.js.map +1 -1
  57. package/dist/ignore.test.js +30 -3
  58. package/dist/ignore.test.js.map +1 -1
  59. package/dist/journal-row-store.d.ts +5 -40
  60. package/dist/journal-row-store.d.ts.map +1 -1
  61. package/dist/journal-row-store.js +21 -240
  62. package/dist/journal-row-store.js.map +1 -1
  63. package/dist/journal-row-store.test.js +0 -54
  64. package/dist/journal-row-store.test.js.map +1 -1
  65. package/dist/journal-snapshot-v4.d.ts.map +1 -1
  66. package/dist/journal-snapshot-v4.js +6 -11
  67. package/dist/journal-snapshot-v4.js.map +1 -1
  68. package/dist/journal-store.test.js +7 -324
  69. package/dist/journal-store.test.js.map +1 -1
  70. package/dist/journal.d.ts +16 -51
  71. package/dist/journal.d.ts.map +1 -1
  72. package/dist/journal.js +79 -303
  73. package/dist/journal.js.map +1 -1
  74. package/dist/lib/exit-codes.d.ts +0 -15
  75. package/dist/lib/exit-codes.d.ts.map +1 -1
  76. package/dist/lib/exit-codes.js +0 -15
  77. package/dist/lib/exit-codes.js.map +1 -1
  78. package/dist/lib/net-errors.d.ts +12 -0
  79. package/dist/lib/net-errors.d.ts.map +1 -1
  80. package/dist/lib/net-errors.js +35 -0
  81. package/dist/lib/net-errors.js.map +1 -1
  82. package/dist/lib/net-errors.test.js +19 -1
  83. package/dist/lib/net-errors.test.js.map +1 -1
  84. package/dist/machine-auth.test.js +6 -4
  85. package/dist/machine-auth.test.js.map +1 -1
  86. package/dist/machine-token-cache-isolation.test.d.ts +24 -0
  87. package/dist/machine-token-cache-isolation.test.d.ts.map +1 -0
  88. package/dist/machine-token-cache-isolation.test.js +241 -0
  89. package/dist/machine-token-cache-isolation.test.js.map +1 -0
  90. package/dist/manifest/build-manifest-resident-parity.test.js +3 -3
  91. package/dist/manifest/build-manifest-resident-parity.test.js.map +1 -1
  92. package/dist/personal-vault.d.ts.map +1 -1
  93. package/dist/personal-vault.js +3 -10
  94. package/dist/personal-vault.js.map +1 -1
  95. package/dist/personal-vault.test.js +0 -35
  96. package/dist/personal-vault.test.js.map +1 -1
  97. package/dist/remote-pull.d.ts.map +1 -1
  98. package/dist/remote-pull.js +2 -9
  99. package/dist/remote-pull.js.map +1 -1
  100. package/dist/remote-pull.test.js +2 -59
  101. package/dist/remote-pull.test.js.map +1 -1
  102. package/dist/s3.d.ts +0 -1
  103. package/dist/s3.d.ts.map +1 -1
  104. package/dist/s3.js +2 -20
  105. package/dist/s3.js.map +1 -1
  106. package/dist/sync/area-ledger.d.ts +3 -27
  107. package/dist/sync/area-ledger.d.ts.map +1 -1
  108. package/dist/sync/area-ledger.js +39 -149
  109. package/dist/sync/area-ledger.js.map +1 -1
  110. package/dist/sync/area-ledger.test.js +5 -192
  111. package/dist/sync/area-ledger.test.js.map +1 -1
  112. package/dist/sync/parallel-sync-coordinator.test.js +0 -29
  113. package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
  114. package/dist/sync/realtime-telemetry.d.ts +1 -1
  115. package/dist/sync/realtime-telemetry.d.ts.map +1 -1
  116. package/dist/sync/realtime-telemetry.js +6 -14
  117. package/dist/sync/realtime-telemetry.js.map +1 -1
  118. package/dist/sync/realtime-telemetry.test.js +16 -136
  119. package/dist/sync/realtime-telemetry.test.js.map +1 -1
  120. package/dist/sync/state-store.d.ts +0 -6
  121. package/dist/sync/state-store.d.ts.map +1 -1
  122. package/dist/sync/state-store.js +2 -14
  123. package/dist/sync/state-store.js.map +1 -1
  124. package/dist/watcher.d.ts +0 -12
  125. package/dist/watcher.d.ts.map +1 -1
  126. package/dist/watcher.js +0 -131
  127. package/dist/watcher.js.map +1 -1
  128. package/dist/watcher.test.js +0 -63
  129. package/dist/watcher.test.js.map +1 -1
  130. package/package.json +1 -2
  131. package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
  132. package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
  133. package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
  134. package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
  135. package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
  136. package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
  137. package/dist/cli/onepass-pull-planner.test.js +0 -266
  138. package/dist/cli/onepass-pull-planner.test.js.map +0 -1
  139. package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
  140. package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
  141. package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
  142. package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
  143. package/dist/s3.list-cost.test.d.ts +0 -2
  144. package/dist/s3.list-cost.test.d.ts.map +0 -1
  145. package/dist/s3.list-cost.test.js +0 -51
  146. package/dist/s3.list-cost.test.js.map +0 -1
  147. package/dist/sync/remote-file-spool.d.ts +0 -16
  148. package/dist/sync/remote-file-spool.d.ts.map +0 -1
  149. package/dist/sync/remote-file-spool.js +0 -91
  150. package/dist/sync/remote-file-spool.js.map +0 -1
  151. package/dist/sync/remote-file-spool.test.d.ts +0 -2
  152. package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
  153. package/dist/sync/remote-file-spool.test.js +0 -77
  154. package/dist/sync/remote-file-spool.test.js.map +0 -1
package/dist/journal.js CHANGED
@@ -17,7 +17,7 @@ import * as os from "os";
17
17
  import * as path from "path";
18
18
  import * as crypto from "crypto";
19
19
  import { JournalRowOverlay, JournalRowStore, journalEntryFingerprint, } from "./journal-row-store.js";
20
- import { createJournalRecordProxy, journalRecordRows } from "./journal-record-proxy.js";
20
+ import { createJournalRecordProxy } from "./journal-record-proxy.js";
21
21
  import { decodeJournalSnapshot, decodedJournalRows } from "./journal-snapshot-decoder.js";
22
22
  import { forgetSharedAreaJournalRows } from "./journal-snapshot-rows.js";
23
23
  import { encodeV4JournalSnapshotChunks } from "./journal-snapshot-v4.js";
@@ -114,9 +114,6 @@ export function journalRows(journal) {
114
114
  const backing = storeBackedJournals.get(journal);
115
115
  if (backing)
116
116
  return backing.store;
117
- const proxied = journalRecordRows(journal.files);
118
- if (proxied)
119
- return proxied;
120
117
  const decoded = decodedJournalRows(journal);
121
118
  if (decoded)
122
119
  return decoded.rows;
@@ -149,12 +146,9 @@ export function closeCachedAreaJournalMigration(slug) {
149
146
  closeCachedAreaMigration(getStateDir(), slug);
150
147
  }
151
148
  /**
152
- * Release all process-local area bridges at runner shutdown.
153
- *
154
- * A bridge owns one packed aggregate per authoritative journal. It is retained
155
- * through ordinary watch passes so a later scoped callback cannot re-pack the
156
- * whole journal synchronously. Durable layout/phase changes use the narrower
157
- * close helper above; shutdown is the terminal memory boundary.
149
+ * Release all process-local area bridges after a completed runner pass.
150
+ * They hold lazy StateStore/aggregate caches but no file descriptors; durable
151
+ * state is authoritative, so a later pass can safely construct a fresh bridge.
158
152
  */
159
153
  export function releaseCachedAreaJournalMigrations() {
160
154
  for (const [cacheKey, cached] of cachedAreaMigrations) {
@@ -197,8 +191,7 @@ function readStoreBackedJournal(journal, options = {}) {
197
191
  existing.pullsJson = JSON.stringify(journal.pulls ?? []);
198
192
  return journal;
199
193
  }
200
- const proxied = journalRecordRows(journal.files);
201
- const decoded = proxied ?? decodedJournalRows(journal);
194
+ const decoded = decodedJournalRows(journal);
202
195
  if (decoded) {
203
196
  if (options.privateOverlay) {
204
197
  const store = new JournalRowOverlay(decoded);
@@ -214,22 +207,7 @@ function readStoreBackedJournal(journal, options = {}) {
214
207
  });
215
208
  return result;
216
209
  }
217
- if (decoded instanceof JournalRowStore)
218
- decoded.enableChangeTracking();
219
- else {
220
- const store = new JournalRowOverlay(decoded);
221
- const result = { ...journal, files: createJournalRecordProxy(store) };
222
- if (Object.hasOwn(journal, "pulls"))
223
- result.pulls = structuredClone(journal.pulls);
224
- storeBackedJournals.set(result, {
225
- store,
226
- version: result.version,
227
- lastSync: result.lastSync,
228
- pullsJson: JSON.stringify(result.pulls ?? []),
229
- keysNormalized: false,
230
- });
231
- return result;
232
- }
210
+ decoded.enableChangeTracking();
233
211
  storeBackedJournals.set(journal, {
234
212
  store: decoded,
235
213
  version: journal.version,
@@ -301,27 +279,11 @@ function journalDeltaFromStoreDirtyLog(backing, next) {
301
279
  }
302
280
  function checkpointStoreBackedJournal(backing, journal) {
303
281
  backing.store.drainDirtyLog();
282
+ backing.store.clearCommittedChanges?.();
304
283
  backing.version = journal.version;
305
284
  backing.lastSync = journal.lastSync;
306
285
  backing.pullsJson = JSON.stringify(journal.pulls ?? []);
307
286
  }
308
- /**
309
- * Move one full-pass facade onto the immutable durable revision it just
310
- * published. Its prior private overlay contains the now-committed rows; merely
311
- * clearing that overlay would make the caller's journal roll back to the old
312
- * packed base. The StateStore reducer publishes a fresh overlay for every
313
- * delta, so later scoped writes replace the aggregate instead of mutating this
314
- * pass's newly established base.
315
- */
316
- function rebaseStoreBackedJournal(backing, journal, durable) {
317
- const rows = journalRecordRows(durable.files) ?? decodedJournalRows(durable);
318
- const store = new JournalRowOverlay(rows ?? new RecordJournalRows(durable.files));
319
- journal.version = durable.version;
320
- journal.lastSync = durable.lastSync;
321
- journal.pulls = structuredClone(durable.pulls ?? []);
322
- journal.files = createJournalRecordProxy(store);
323
- backing.store = store;
324
- }
325
287
  function storeBackedJournalNeedsMigration(backing, journal) {
326
288
  return journal.version !== JOURNAL_VERSION_CURRENT ||
327
289
  !Array.isArray(journal.pulls) ||
@@ -395,15 +357,16 @@ function entryFingerprint(entry) {
395
357
  */
396
358
  export function journalDeltaFromBaseline(baseline, next) {
397
359
  const upserts = {};
398
- const nextRows = journalRows(next);
399
- for (const [key, entry] of nextRows.entries()) {
360
+ const files = next.files;
361
+ for (const key of Object.keys(files)) {
362
+ const entry = files[key];
400
363
  if (baseline.rows.get(key) === entryFingerprint(entry))
401
364
  continue;
402
365
  upserts[key] = entry;
403
366
  }
404
367
  const deletes = [];
405
368
  for (const key of baseline.rows.keys()) {
406
- if (!nextRows.has(key))
369
+ if (!Object.hasOwn(files, key))
407
370
  deletes.push(key);
408
371
  }
409
372
  const delta = {
@@ -547,11 +510,7 @@ export function readJournal(slug) {
547
510
  // mutable without deep-cloning the aggregate before every pass.
548
511
  return readStoreBackedJournal(areaMigration.readAreaJournalCached().journal, { privateOverlay: true });
549
512
  }
550
- // The retained keyed projection and this full-pass facade can borrow the
551
- // same immutable packed rows. Each owns only its own overlay, so an
552
- // uncommitted full pass is never visible to a scoped reader (and a scoped
553
- // delta cannot dirty the full pass).
554
- return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration, true), { privateOverlay: true });
513
+ return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration));
555
514
  }
556
515
  /**
557
516
  * Read the mutable outer shell required by a full pull without detaching every
@@ -562,8 +521,9 @@ export function readJournal(slug) {
562
521
  * own changes in a private overlay. `writeJournal()` drains that overlay's
563
522
  * dirty log, avoiding a full-ledger fingerprint at every download checkpoint.
564
523
  *
565
- * Legacy and area-authoritative journals both borrow immutable packed rows and
566
- * put their caller-owned mutations in a private overlay.
524
+ * Legacy journals deliberately keep `readJournal()`'s packed-record boundary.
525
+ * That path has different ownership and dirty-log semantics, and does not make
526
+ * the area-authoritative structured-clone copy that this reader avoids.
567
527
  *
568
528
  * @internal Full-pull boundary; not a public journal record API.
569
529
  */
@@ -599,55 +559,49 @@ export function readJournalForPass(slug) {
599
559
  function readJournalAggregate(slug) {
600
560
  return readJournalAggregateWith(slug, openAreaMigrationIfAuthoritative(slug));
601
561
  }
602
- function readJournalAggregateWith(slug, areaMigration, retainScopedProjection = false) {
562
+ function readJournalAggregateWith(slug, areaMigration) {
603
563
  if (areaMigration)
604
564
  return areaMigration.readAreaJournal();
605
- const store = StateStore.exists(getStateDir(), journalScopeId(slug))
606
- ? openJournalStore(slug, emptyJournal())
607
- : openJournalStore(slug, readLegacyJournal(slug));
608
- if (retainScopedProjection) {
609
- // A full pass has already paid to recover this packed snapshot. Retain its
610
- // projection for the next scoped pass instead of making that pass decode
611
- // the same vault before it looks up even one named row. Read-only
612
- // enumeration deliberately does not retain: watcher warm-up may inspect
613
- // every locator and must not evict the hot personal projection.
614
- retainScopedJournalStore(slug, new JournalStore(slug, store));
615
- }
616
- return store.getState().journal;
565
+ if (StateStore.exists(getStateDir(), journalScopeId(slug))) {
566
+ return openJournalStore(slug, emptyJournal()).getState().journal;
567
+ }
568
+ return openJournalStore(slug, readLegacyJournal(slug)).getState().journal;
617
569
  }
618
570
  /**
619
571
  * Read a mutable, detached snapshot for named rows in a scoped pass.
620
572
  *
621
- * Area-authoritative journals use the ledger's keyed reader. Other journals
622
- * retain one refreshed packed projection after the preceding full pass, so a
623
- * scoped read is proportional to named keys rather than a fresh snapshot
624
- * recovery. Only named delete roots require a linear row scan, with an
625
- * ancestor-set lookup per row; it must never be rows x roots.
573
+ * Area-authoritative journals use the ledger's keyed reader so the work is
574
+ * proportional to the named keys rather than the full cached aggregate. The
575
+ * legacy store has no keyed persistence surface, so it reads the named keys
576
+ * straight out of the aggregate (O(K)) and, only when delete roots are named,
577
+ * makes one linear pass over the rows with an ancestor-set lookup per row. It
578
+ * must never be rows x roots: a watch-mode batch names one delete root per
579
+ * changed path, and a warm-up catch-up hands the pass ~10k of them against a
580
+ * 70k-row vault journal.
626
581
  */
627
582
  export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
628
583
  const areaMigration = openAreaMigrationIfAuthoritative(slug);
629
584
  if (areaMigration) {
630
585
  return rememberJournalBaseline(areaMigration.readAreaJournalScoped(keys, deleteScopeRoots));
631
586
  }
632
- const store = scopedJournalStore(slug);
633
- store.refresh();
634
- const metadata = store.metadata();
587
+ const full = readJournalAggregate(slug);
588
+ const fullRows = journalRows(full);
635
589
  const files = {};
636
590
  const roots = new Set(deleteScopeRoots);
637
591
  if (roots.has("")) {
638
- for (const [key, entry] of store.scanPrefix("")) {
592
+ for (const [key, entry] of fullRows.entries()) {
639
593
  files[key] = structuredClone(entry);
640
594
  }
641
595
  }
642
596
  else {
643
597
  for (const key of new Set(keys)) {
644
- const entry = store.get(key);
598
+ const entry = fullRows.get(key);
645
599
  if (entry === undefined)
646
600
  continue;
647
601
  files[key] = structuredClone(entry);
648
602
  }
649
603
  if (roots.size > 0) {
650
- for (const [key, entry] of store.scanPrefix("")) {
604
+ for (const [key, entry] of fullRows.entries()) {
651
605
  if (Object.hasOwn(files, key))
652
606
  continue;
653
607
  if (isUnderAnyRoot(key, roots))
@@ -656,10 +610,10 @@ export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
656
610
  }
657
611
  }
658
612
  return rememberJournalBaseline({
659
- version: metadata.version,
660
- lastSync: metadata.lastSync,
613
+ version: full.version,
614
+ lastSync: full.lastSync,
661
615
  files,
662
- pulls: structuredClone(metadata.pulls),
616
+ pulls: structuredClone(full.pulls ?? []),
663
617
  });
664
618
  }
665
619
  /**
@@ -740,22 +694,9 @@ function journalStoreOptions(slug, initialJournal) {
740
694
  }
741
695
  if (record.type === JOURNAL_DELTA_RECORD_TYPE) {
742
696
  const delta = record.payload;
743
- const existingRows = journalRecordRows(state.journal.files) ?? decodedJournalRows(state.journal);
744
- if (existingRows) {
745
- // StateStore's aggregate is also the immutable source for any
746
- // full-pass facade opened before this append. Once a first delta is
747
- // observed, put durable changes in a separate overlay rather than
748
- // mutating that shared packed row store.
749
- const rows = existingRows instanceof JournalRowOverlay
750
- ? existingRows.withCommittedDelta(delta.upserts, delta.deletes)
751
- : new JournalRowOverlay(existingRows).withCommittedDelta(delta.upserts, delta.deletes);
752
- state.journal.files = createJournalRecordProxy(rows);
753
- }
754
- else {
755
- for (const key of delta.deletes)
756
- delete state.journal.files[key];
757
- Object.assign(state.journal.files, delta.upserts);
758
- }
697
+ for (const key of delta.deletes)
698
+ delete state.journal.files[key];
699
+ Object.assign(state.journal.files, delta.upserts);
759
700
  state.journal.version = delta.version;
760
701
  state.journal.lastSync = delta.lastSync ?? state.journal.lastSync;
761
702
  state.journal.pulls = delta.pulls ?? state.journal.pulls;
@@ -794,10 +735,6 @@ function openAreaMigrationIfAuthoritative(slug) {
794
735
  closeCachedAreaMigration(stateDir, slug);
795
736
  return undefined;
796
737
  }
797
- // Area authority replaces the legacy v3 aggregate. Drop any retained legacy
798
- // projection before a scoped reader can accidentally keep both generations
799
- // resident (or use the obsolete writer after cutover).
800
- releaseScopedJournalStore(slug);
801
738
  const resolverPath = path.join(areaLayoutDirectory(stateDir, slug), "area-resolver.json");
802
739
  let resolverJson;
803
740
  try {
@@ -862,20 +799,19 @@ export function openJournalStateStoreForMigration(slug) {
862
799
  async function openJournalStoreAsync(slug, initialJournal) {
863
800
  return StateStore.openAsync(journalStoreOptions(slug, initialJournal));
864
801
  }
802
+ /**
803
+ * Long-lived keyed projection over the v3 snapshot/WAL authority.
804
+ *
805
+ * The first open recovers once. Subsequent refreshes consume only authenticated
806
+ * complete WAL frames, rebuilding the projection solely after a generation
807
+ * publication or invalid-tail fallback.
808
+ */
865
809
  export class JournalStore {
866
810
  slug;
867
811
  store;
868
812
  areaMigration;
869
813
  files = new JournalRowStore({ trackChanges: false });
870
814
  usesSharedRows = false;
871
- /**
872
- * Durable deltas this keyed session has observed after its base snapshot.
873
- * The base can be shared with a full-pass facade, so publishing a scoped
874
- * delta here must never mutate it in place.
875
- */
876
- committedOverlay = new Map();
877
- /** Base keys deleted then reinserted by durable records move to the tail. */
878
- committedMoved = new Set();
879
815
  /**
880
816
  * Pass-local rows that have not reached the shared projection yet. A missing
881
817
  * value represents a staged delete; every other value is a frozen copy of
@@ -905,56 +841,27 @@ export class JournalStore {
905
841
  this.assertOpen();
906
842
  if (this.overlay.has(key))
907
843
  return this.overlay.get(key);
908
- if (this.committedOverlay.has(key))
909
- return this.committedOverlay.get(key);
910
844
  return this.files.get(key);
911
845
  }
912
846
  scanPrefix(prefix) {
913
847
  this.assertOpen();
848
+ const rows = new Map(this.files.entriesForPrefix(prefix));
914
849
  const normalizedPrefix = prefix.replace(/\/+$/, "");
915
- const files = this.files;
916
- const committedOverlay = this.committedOverlay;
917
- const committedMoved = this.committedMoved;
918
- const overlay = this.overlay;
919
- const iterate = function* () {
920
- for (const [key, entry] of files.entriesForPrefix(normalizedPrefix)) {
921
- if (committedMoved.has(key))
922
- continue;
923
- const replacement = overlay.has(key)
924
- ? overlay.get(key)
925
- : committedOverlay.has(key)
926
- ? committedOverlay.get(key)
927
- : entry;
928
- if (replacement !== undefined)
929
- yield [key, replacement];
930
- }
931
- // A Map replacement preserves the base row's position and appends newly
932
- // introduced overlay rows. Preserve that observable order without holding
933
- // a materialized JournalEntry for every base row at once.
934
- const additions = new Map(committedOverlay);
935
- for (const [key, entry] of overlay)
936
- additions.set(key, entry);
937
- for (const [key, entry] of additions) {
938
- if (entry === undefined || (files.has(key) && !committedMoved.has(key)))
939
- continue;
940
- if (isJournalStorePrefixMatch(key, normalizedPrefix))
941
- yield [key, entry];
942
- }
943
- };
944
- const scan = {
945
- [Symbol.iterator]: iterate,
946
- filter: ((predicate, thisArg) => Array.from(iterate()).filter(predicate, thisArg)),
947
- };
948
- return scan;
850
+ for (const [key, entry] of this.overlay) {
851
+ if (!isJournalStorePrefixMatch(key, normalizedPrefix))
852
+ continue;
853
+ if (entry === undefined)
854
+ rows.delete(key);
855
+ else
856
+ rows.set(key, entry);
857
+ }
858
+ return Array.from(rows.entries());
949
859
  }
950
860
  rowCount() {
951
861
  if (this.disposed)
952
862
  return 0;
953
- const replacements = new Map(this.committedOverlay);
954
- for (const [key, entry] of this.overlay)
955
- replacements.set(key, entry);
956
863
  let count = this.files.size;
957
- for (const [key, entry] of replacements) {
864
+ for (const [key, entry] of this.overlay) {
958
865
  const existsInSharedStore = this.files.has(key);
959
866
  if (entry === undefined) {
960
867
  if (existsInSharedStore)
@@ -970,9 +877,7 @@ export class JournalStore {
970
877
  set(key, entry) {
971
878
  this.assertOpen();
972
879
  const ownedEntry = immutableCopy(entry);
973
- const sharedEntry = this.committedOverlay.has(key)
974
- ? this.committedOverlay.get(key)
975
- : this.files.get(key);
880
+ const sharedEntry = this.files.get(key);
976
881
  if (sharedEntry !== undefined && sameEntry(sharedEntry, ownedEntry)) {
977
882
  this.overlay.delete(key);
978
883
  return;
@@ -985,12 +890,12 @@ export class JournalStore {
985
890
  if (this.overlay.has(key)) {
986
891
  if (this.overlay.get(key) === undefined)
987
892
  return false;
988
- if (!this.committedOverlay.has(key) && !this.files.has(key)) {
893
+ if (!this.files.has(key)) {
989
894
  this.overlay.delete(key);
990
895
  return true;
991
896
  }
992
897
  }
993
- else if (this.get(key) === undefined) {
898
+ else if (!this.files.has(key)) {
994
899
  return false;
995
900
  }
996
901
  this.overlay.set(key, undefined);
@@ -1029,17 +934,6 @@ export class JournalStore {
1029
934
  pulls: this.pulls,
1030
935
  });
1031
936
  }
1032
- /**
1033
- * The StateStore aggregate after a durable append. Its row view is immutable
1034
- * across later deltas, which lets a full-pass facade rebase without exposing
1035
- * its completed checkpoint to subsequent scoped writers.
1036
- */
1037
- durableJournalForRebase() {
1038
- this.assertOpen();
1039
- if (this.areaMigration)
1040
- return this.areaMigration.readAreaJournalCached().journal;
1041
- return this.store.getState().journal;
1042
- }
1043
937
  refresh() {
1044
938
  this.assertOpen();
1045
939
  if (this.areaMigration) {
@@ -1066,10 +960,12 @@ export class JournalStore {
1066
960
  this.applyRefresh(result);
1067
961
  }
1068
962
  applyRefresh(result) {
1069
- if (result.recovered || result.records.length > 0) {
963
+ if (result.recovered) {
1070
964
  this.replaceFromJournal(this.store.getState().journal);
1071
965
  return;
1072
966
  }
967
+ for (const record of result.records)
968
+ this.applyRecord(record.type, record.payload);
1073
969
  }
1074
970
  applyDelta(delta) {
1075
971
  this.assertOpen();
@@ -1129,11 +1025,13 @@ export class JournalStore {
1129
1025
  this.applyCommitted(appended);
1130
1026
  ensureJournalLocator(this.slug);
1131
1027
  }
1132
- applyCommitted(_appended) {
1133
- // StateStore has reduced the just-appended durable frame already. Repoint
1134
- // this cache at that immutable aggregate rather than replaying the frame
1135
- // into a second mutable overlay beside it.
1136
- this.replaceFromJournal(this.store.getState().journal);
1028
+ applyCommitted(appended) {
1029
+ if (appended.refresh.recovered)
1030
+ this.replaceFromJournal(this.store.getState().journal);
1031
+ else
1032
+ for (const record of appended.refresh.records)
1033
+ this.applyRecord(record.type, record.payload);
1034
+ this.applyRecord(appended.type, appended.payload);
1137
1035
  }
1138
1036
  /**
1139
1037
  * Packing copies every source row into typed arrays, so this projection never
@@ -1142,30 +1040,21 @@ export class JournalStore {
1142
1040
  * pass-local), which is what put the outposts over Node's heap ceiling.
1143
1041
  */
1144
1042
  replaceFromJournal(journal) {
1145
- const proxied = journalRecordRows(journal.files);
1146
- const decoded = proxied instanceof JournalRowStore || proxied instanceof JournalRowOverlay
1147
- ? proxied
1148
- : decodedJournalRows(journal);
1043
+ const decoded = decodedJournalRows(journal);
1149
1044
  if (decoded) {
1150
- if (!this.usesSharedRows && this.files instanceof JournalRowStore && this.files !== decoded) {
1045
+ if (!this.usesSharedRows && this.files !== decoded)
1151
1046
  this.files.clear();
1152
- }
1153
1047
  this.files = decoded;
1154
1048
  this.usesSharedRows = true;
1155
- this.committedOverlay.clear();
1156
- this.committedMoved.clear();
1157
1049
  this.version = journal.version;
1158
1050
  this.lastSync = journal.lastSync;
1159
1051
  this.pulls = structuredClone(journal.pulls ?? []);
1160
1052
  return;
1161
1053
  }
1162
- if (this.usesSharedRows || !(this.files instanceof JournalRowStore)) {
1054
+ if (this.usesSharedRows)
1163
1055
  this.files = new JournalRowStore({ trackChanges: false });
1164
- }
1165
1056
  this.files.replaceFromRecord(journal.files);
1166
1057
  this.usesSharedRows = false;
1167
- this.committedOverlay.clear();
1168
- this.committedMoved.clear();
1169
1058
  this.version = journal.version;
1170
1059
  this.lastSync = journal.lastSync;
1171
1060
  this.pulls = structuredClone(journal.pulls ?? []);
@@ -1179,15 +1068,9 @@ export class JournalStore {
1179
1068
  throw new Error("journal: keyed session received an invalid WAL record");
1180
1069
  }
1181
1070
  for (const key of payload.deletes)
1182
- this.committedOverlay.set(key, undefined);
1183
- for (const [key, entry] of Object.entries(payload.upserts)) {
1184
- if (this.committedOverlay.get(key) === undefined && this.committedOverlay.has(key)) {
1185
- this.committedOverlay.delete(key);
1186
- if (this.files.has(key))
1187
- this.committedMoved.add(key);
1188
- }
1189
- this.committedOverlay.set(key, immutableCopy(entry));
1190
- }
1071
+ this.files.delete(key);
1072
+ for (const [key, entry] of Object.entries(payload.upserts))
1073
+ this.files.set(key, entry);
1191
1074
  this.version = payload.version;
1192
1075
  this.lastSync = payload.lastSync ?? this.lastSync;
1193
1076
  this.pulls = structuredClone(payload.pulls ?? this.pulls);
@@ -1198,9 +1081,7 @@ export class JournalStore {
1198
1081
  return;
1199
1082
  this.disposed = true;
1200
1083
  this.overlay.clear();
1201
- this.committedOverlay.clear();
1202
- this.committedMoved.clear();
1203
- if (!this.usesSharedRows && this.files instanceof JournalRowStore)
1084
+ if (!this.usesSharedRows)
1204
1085
  this.files.clear();
1205
1086
  this.store = undefined;
1206
1087
  this.areaMigration = undefined;
@@ -1227,69 +1108,6 @@ function immutableCopy(value) {
1227
1108
  freeze(copy);
1228
1109
  return copy;
1229
1110
  }
1230
- // A scoped watcher pass must not recover a 700k-row snapshot just to inspect
1231
- // one path. A runner can alternate the personal vault and its companies in the
1232
- // same realtime batch, so one cache entry turns that alternation into a fresh
1233
- // whole-snapshot recovery on every dispatch. Keep a deliberately small LRU:
1234
- // enough for the runner's active journal set, bounded so an unusually broad
1235
- // company fanout cannot retain every journal for the lifetime of the process.
1236
- const MAX_RETAINED_SCOPED_JOURNALS = 4;
1237
- const cachedScopedJournalStores = new Map();
1238
- function scopedJournalStoreCacheKey(slug) {
1239
- return `${getStateDir()}\u0000${slug}`;
1240
- }
1241
- function cachedScopedJournalStoreFor(slug) {
1242
- const cacheKey = scopedJournalStoreCacheKey(slug);
1243
- const cached = cachedScopedJournalStores.get(cacheKey);
1244
- if (cached === undefined)
1245
- return undefined;
1246
- // Map insertion order is our LRU order. A hot scope must not be evicted by
1247
- // a brief company fanout while the runner is still using it.
1248
- cachedScopedJournalStores.delete(cacheKey);
1249
- cachedScopedJournalStores.set(cacheKey, cached);
1250
- return cached;
1251
- }
1252
- function retainScopedJournalStore(slug, store) {
1253
- const cacheKey = scopedJournalStoreCacheKey(slug);
1254
- const previous = cachedScopedJournalStores.get(cacheKey);
1255
- if (previous !== undefined && previous !== store)
1256
- previous.dispose();
1257
- cachedScopedJournalStores.delete(cacheKey);
1258
- cachedScopedJournalStores.set(cacheKey, store);
1259
- while (cachedScopedJournalStores.size > MAX_RETAINED_SCOPED_JOURNALS) {
1260
- const oldest = cachedScopedJournalStores.entries().next().value;
1261
- if (oldest === undefined)
1262
- break;
1263
- const [oldestKey, oldestStore] = oldest;
1264
- cachedScopedJournalStores.delete(oldestKey);
1265
- oldestStore.dispose();
1266
- }
1267
- return store;
1268
- }
1269
- function scopedJournalStore(slug) {
1270
- const cached = cachedScopedJournalStoreFor(slug);
1271
- if (cached)
1272
- return cached;
1273
- return retainScopedJournalStore(slug, openJournalStoreSession(slug));
1274
- }
1275
- /** Drop one legacy keyed projection after an authoritative area cutover. */
1276
- export function releaseScopedJournalStore(slug) {
1277
- const cacheKey = scopedJournalStoreCacheKey(slug);
1278
- const cached = cachedScopedJournalStores.get(cacheKey);
1279
- if (cached === undefined)
1280
- return;
1281
- cachedScopedJournalStores.delete(cacheKey);
1282
- cached.dispose();
1283
- }
1284
- /**
1285
- * Release retained scoped projections when a watch runner stops. Durable state
1286
- * remains in the StateStore; this only drops process-local decoded snapshots.
1287
- */
1288
- export function disposeScopedJournalStores() {
1289
- for (const store of cachedScopedJournalStores.values())
1290
- store.dispose();
1291
- cachedScopedJournalStores.clear();
1292
- }
1293
1111
  /** Open one keyed session, seeding a missing v3 store from legacy JSON once. */
1294
1112
  export function openJournalStoreSession(slug) {
1295
1113
  const areaMigration = openAreaMigrationIfAuthoritative(slug);
@@ -1802,47 +1620,6 @@ export function writeJournal(slug, journal) {
1802
1620
  ensureJournalLocator(slug);
1803
1621
  return;
1804
1622
  }
1805
- const baseline = journalBaselines.get(journal);
1806
- const cached = cachedScopedJournalStoreFor(slug);
1807
- if (cached && (storeBacking || baseline)) {
1808
- // The scoped reader captured only its named rows, so its baseline is the
1809
- // authoritative delta boundary. The retained session refreshes and appends
1810
- // under the normal StateStore lock; it never exposes partial durable rows
1811
- // to another writer.
1812
- const delta = storeBacking
1813
- ? journalDeltaFromStoreDirtyLog(storeBacking, journal)
1814
- : baseline
1815
- ? journalDeltaFromBaseline(baseline, journal)
1816
- : undefined;
1817
- if (delta && hasJournalDelta(delta)) {
1818
- // Carry ONLY the metadata fields this delta actually changed. The
1819
- // retained session's `metadata()` is a snapshot that goes stale the
1820
- // moment another writer advances `lastSync` or appends a pull record,
1821
- // and filling the untouched field from it republished that stale value
1822
- // over the concurrent writer's durable one — dropping pull history or
1823
- // reverting `lastSync`. The store's reducer already preserves a field
1824
- // the delta omits, so omission is the whole fix.
1825
- const metadata = {};
1826
- if (delta.lastSync !== undefined)
1827
- metadata.lastSync = delta.lastSync;
1828
- if (delta.pulls !== undefined)
1829
- metadata.pulls = delta.pulls;
1830
- const hasMetadata = delta.lastSync !== undefined || delta.pulls !== undefined;
1831
- cached.applyDelta({
1832
- upserts: delta.upserts,
1833
- deletes: delta.deletes,
1834
- ...(hasMetadata ? { metadata } : {}),
1835
- });
1836
- }
1837
- if (storeBacking) {
1838
- rebaseStoreBackedJournal(storeBacking, journal, cached.durableJournalForRebase());
1839
- checkpointStoreBackedJournal(storeBacking, journal);
1840
- }
1841
- else
1842
- rememberJournalBaseline(journal);
1843
- ensureJournalLocator(slug);
1844
- return;
1845
- }
1846
1623
  const store = openJournalStore(slug, journal);
1847
1624
  if (store.wasCreated) {
1848
1625
  // The store was just seeded from this journal, so its state already matches
@@ -1866,6 +1643,7 @@ export function writeJournal(slug, journal) {
1866
1643
  // The WAL still compacts on its own schedule (record count or byte ceiling),
1867
1644
  // so a full snapshot is written periodically rather than never — recovery
1868
1645
  // stays bounded, it just no longer happens on every single update.
1646
+ const baseline = journalBaselines.get(journal);
1869
1647
  const delta = storeBacking
1870
1648
  ? journalDeltaFromStoreDirtyLog(storeBacking, journal)
1871
1649
  : baseline
@@ -1874,10 +1652,8 @@ export function writeJournal(slug, journal) {
1874
1652
  store.append(JOURNAL_DELTA_RECORD_TYPE, delta);
1875
1653
  // A caller can checkpoint more than once during one pass. Advancing only its
1876
1654
  // own tracking state means the next checkpoint again publishes only new mutations.
1877
- if (storeBacking) {
1878
- rebaseStoreBackedJournal(storeBacking, journal, store.getState().journal);
1655
+ if (storeBacking)
1879
1656
  checkpointStoreBackedJournal(storeBacking, journal);
1880
- }
1881
1657
  else
1882
1658
  rememberJournalBaseline(journal);
1883
1659
  // The locator is immutable content, so it only needs writing if it is absent