@indigoai-us/hq-cloud 6.16.38 → 6.16.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/dist/bin/sync-runner-heap-telemetry.d.ts +145 -9
  2. package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-heap-telemetry.js +178 -11
  4. package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
  5. package/dist/bin/sync-runner-heap-telemetry.test.js +511 -2
  6. package/dist/bin/sync-runner-heap-telemetry.test.js.map +1 -1
  7. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  8. package/dist/bin/sync-runner-watch-loop.js +20 -10
  9. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  10. package/dist/bin/sync-runner-watch-loop.test.js +21 -0
  11. package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
  12. package/dist/bin/sync-runner.d.ts +16 -5
  13. package/dist/bin/sync-runner.d.ts.map +1 -1
  14. package/dist/bin/sync-runner.js.map +1 -1
  15. package/dist/cli/share.d.ts.map +1 -1
  16. package/dist/cli/share.js +28 -27
  17. package/dist/cli/share.js.map +1 -1
  18. package/dist/cli/share.test.js +49 -0
  19. package/dist/cli/share.test.js.map +1 -1
  20. package/dist/cli/sync.d.ts.map +1 -1
  21. package/dist/cli/sync.js +8 -8
  22. package/dist/cli/sync.js.map +1 -1
  23. package/dist/journal-area-ledger-read-boundary.test.js +46 -12
  24. package/dist/journal-area-ledger-read-boundary.test.js.map +1 -1
  25. package/dist/journal-record-proxy.d.ts +12 -2
  26. package/dist/journal-record-proxy.d.ts.map +1 -1
  27. package/dist/journal-record-proxy.js +8 -1
  28. package/dist/journal-record-proxy.js.map +1 -1
  29. package/dist/journal-row-store.d.ts +64 -0
  30. package/dist/journal-row-store.d.ts.map +1 -1
  31. package/dist/journal-row-store.js +129 -3
  32. package/dist/journal-row-store.js.map +1 -1
  33. package/dist/journal-row-store.test.js +3 -1
  34. package/dist/journal-row-store.test.js.map +1 -1
  35. package/dist/journal-snapshot-rows.d.ts +17 -1
  36. package/dist/journal-snapshot-rows.d.ts.map +1 -1
  37. package/dist/journal-snapshot-rows.js +18 -0
  38. package/dist/journal-snapshot-rows.js.map +1 -1
  39. package/dist/journal-snapshot-v4.d.ts +3 -2
  40. package/dist/journal-snapshot-v4.d.ts.map +1 -1
  41. package/dist/journal-snapshot-v4.js +9 -3
  42. package/dist/journal-snapshot-v4.js.map +1 -1
  43. package/dist/journal-snapshot-v4.test.js +1 -0
  44. package/dist/journal-snapshot-v4.test.js.map +1 -1
  45. package/dist/journal.d.ts +23 -5
  46. package/dist/journal.d.ts.map +1 -1
  47. package/dist/journal.js +309 -189
  48. package/dist/journal.js.map +1 -1
  49. package/dist/journal.test.js +58 -1
  50. package/dist/journal.test.js.map +1 -1
  51. package/dist/packed-journal-phase2.test.d.ts +2 -0
  52. package/dist/packed-journal-phase2.test.d.ts.map +1 -0
  53. package/dist/packed-journal-phase2.test.js +273 -0
  54. package/dist/packed-journal-phase2.test.js.map +1 -0
  55. package/dist/personal-vault.d.ts.map +1 -1
  56. package/dist/personal-vault.js +6 -4
  57. package/dist/personal-vault.js.map +1 -1
  58. package/dist/remote-pull.d.ts.map +1 -1
  59. package/dist/remote-pull.js +9 -8
  60. package/dist/remote-pull.js.map +1 -1
  61. package/dist/scope-shrink.js +2 -2
  62. package/dist/scope-shrink.js.map +1 -1
  63. package/dist/session-log-retention.d.ts.map +1 -1
  64. package/dist/session-log-retention.js +2 -2
  65. package/dist/session-log-retention.js.map +1 -1
  66. package/dist/sync/area-ledger-migration.d.ts +13 -6
  67. package/dist/sync/area-ledger-migration.d.ts.map +1 -1
  68. package/dist/sync/area-ledger-migration.js +45 -19
  69. package/dist/sync/area-ledger-migration.js.map +1 -1
  70. package/dist/sync/area-ledger-migration.test.js +23 -0
  71. package/dist/sync/area-ledger-migration.test.js.map +1 -1
  72. package/dist/sync/area-ledger.d.ts +31 -1
  73. package/dist/sync/area-ledger.d.ts.map +1 -1
  74. package/dist/sync/area-ledger.js +385 -88
  75. package/dist/sync/area-ledger.js.map +1 -1
  76. package/dist/sync/area-ledger.test.js +123 -10
  77. package/dist/sync/area-ledger.test.js.map +1 -1
  78. package/dist/types.d.ts +2 -0
  79. package/dist/types.d.ts.map +1 -1
  80. package/package.json +1 -1
@@ -7,6 +7,9 @@
7
7
  * not understand a layout continue to use their existing authority.
8
8
  */
9
9
  import * as crypto from "node:crypto";
10
+ import { JournalRowOverlay, JournalRowStore } from "../journal-row-store.js";
11
+ import { createJournalRecordProxy, journalRecordRows } from "../journal-record-proxy.js";
12
+ import { decodedJournalRows, rememberDecodedJournalRows, rememberSharedAreaJournalRows, sharedAreaJournalRows, } from "../journal-snapshot-rows.js";
10
13
  import { trimPullRecords } from "../journal.js";
11
14
  import { StateStore } from "./state-store.js";
12
15
  const DIRECT_DELTA = 1;
@@ -37,6 +40,54 @@ const META_FINALIZE = 2;
37
40
  export const UNROUTED_AREA_ID = "root:@unrouted";
38
41
  /** Nothing is held anywhere — for a ledger with no state on disk yet. */
39
42
  const NO_ROWS = { holds: () => false };
43
+ function areaJournalRowsIdentity(identities) {
44
+ return [...identities]
45
+ .filter(([areaId]) => areaId !== "@meta")
46
+ .sort(([left], [right]) => left.localeCompare(right))
47
+ .map(([areaId, identity]) => `${areaId}:${identity.generation}:${identity.lastRecordSequence}:${identity.walBytes}`)
48
+ .join("\u0000");
49
+ }
50
+ /** A state-store-facing slice over the one packed aggregate. */
51
+ class AreaJournalRows {
52
+ rows;
53
+ areaId;
54
+ defaultAreaForKey;
55
+ exceptionalOwners;
56
+ shadows;
57
+ write;
58
+ erase;
59
+ constructor(rows, areaId, defaultAreaForKey, exceptionalOwners, shadows, write, erase) {
60
+ this.rows = rows;
61
+ this.areaId = areaId;
62
+ this.defaultAreaForKey = defaultAreaForKey;
63
+ this.exceptionalOwners = exceptionalOwners;
64
+ this.shadows = shadows;
65
+ this.write = write;
66
+ this.erase = erase;
67
+ }
68
+ get(path) {
69
+ return this.shadows.get(path) ?? (this.owns(path) ? this.rows.get(path) : undefined);
70
+ }
71
+ has(path) {
72
+ return this.shadows.has(path) || (this.owns(path) && this.rows.has(path));
73
+ }
74
+ set(path, entry) {
75
+ this.write(path, entry);
76
+ }
77
+ delete(path) {
78
+ return this.erase(path);
79
+ }
80
+ *keys() {
81
+ yield* this.shadows.keys();
82
+ for (const key of this.rows.keys()) {
83
+ if (!this.shadows.has(key) && this.owns(key))
84
+ yield key;
85
+ }
86
+ }
87
+ owns(path) {
88
+ return (this.exceptionalOwners.get(path) ?? this.defaultAreaForKey(path)) === this.areaId;
89
+ }
90
+ }
40
91
  let testHooks = {};
41
92
  /** Test-only crash seam: data prepares are durable before a meta watermark. */
42
93
  export function setAreaLedgerTestHooksForTest(hooks) {
@@ -119,9 +170,22 @@ export class AreaLedger {
119
170
  }
120
171
  }
121
172
  const committed = this.committedTransactions(meta.getState(), states);
122
- // `combine` shares row objects with the area states; this one-off reader
123
- // hands out a detached copy, so a caller may mutate what it gets back.
124
- const aggregate = structuredClone(this.combine([...selected].map((areaId) => ({ areaId, journal: materializeArea(states.get(areaId), committed) }))));
173
+ // The overlay owns only caller changes; unchanged rows remain shared with
174
+ // the selected area projections instead of being deep-cloned as an
175
+ // aggregate before this scoped reader can inspect them.
176
+ const journals = [...selected].map((areaId) => ({
177
+ areaId,
178
+ // `materializeArea` returns the live StateStore journal when there are
179
+ // no pending records. Packing a scoped result must replace only this
180
+ // shell's facade, never the cached area's live journal facade.
181
+ journal: { ...materializeArea(states.get(areaId), committed) },
182
+ }));
183
+ const rows = new JournalRowStore(countJournalRows(journals), { trackChanges: false });
184
+ const owners = new Map();
185
+ const shadows = new Map();
186
+ this.packAreaJournals(journals, rows, owners, shadows);
187
+ const combined = this.combine(rows);
188
+ const aggregate = detachedJournalOverlay(combined.journal, combined.rows.rows);
125
189
  aggregate.lastSync = meta.getState().journal.lastSync;
126
190
  aggregate.pulls = structuredClone(meta.getState().journal.pulls);
127
191
  return aggregate;
@@ -255,7 +319,8 @@ export class AreaLedger {
255
319
  }
256
320
  /** Aggregate read. It is the parity view used by legacy SyncJournal callers. */
257
321
  read() {
258
- return structuredClone(this.readCached().journal);
322
+ const cached = this.readCached();
323
+ return detachedJournalOverlay(cached.journal, cached.rows);
259
324
  }
260
325
  /**
261
326
  * Internal aggregate view for long-lived keyed sessions. The returned journal
@@ -275,21 +340,22 @@ export class AreaLedger {
275
340
  this.primeAggregateCache();
276
341
  }
277
342
  this.aggregateCache.revision = ++this.aggregateRevision;
278
- return { journal: this.aggregateCache.aggregate, revision: this.aggregateCache.revision };
343
+ return this.cachedJournal(this.aggregateCache);
279
344
  }
280
345
  const cache = this.aggregateCache;
281
346
  const { metaChanged, areas } = this.refreshChangedStores(cache);
282
347
  if (!metaChanged && areas.size === 0)
283
- return { journal: cache.aggregate, revision: cache.revision };
348
+ return this.cachedJournal(cache);
284
349
  const meta = this.openMeta().getState();
285
350
  const states = this.currentAreaStates();
351
+ const identities = this.currentAreaIdentities();
286
352
  if (this.hasPendingTransactionEvidence(meta, states)) {
287
353
  // A different process published prepare/watermark evidence. Recovery is
288
354
  // exceptional and bounded to that evidence; the steady read path above
289
355
  // remains lock-free and write-free.
290
356
  this.recoverPendingTransactions();
291
357
  this.primeAggregateCache();
292
- return { journal: this.aggregateCache.aggregate, revision: this.aggregateCache.revision };
358
+ return this.cachedJournal(this.aggregateCache);
293
359
  }
294
360
  const committed = this.committedTransactions(meta, states);
295
361
  for (const transactionId of symmetricDifference(cache.committed, committed)) {
@@ -302,9 +368,9 @@ export class AreaLedger {
302
368
  this.updateCachedArea(cache, areaId, materializeArea(states.get(areaId), committed));
303
369
  }
304
370
  this.updateCacheMetadata(cache, meta, committed);
305
- this.refreshCacheIdentities(cache);
371
+ this.replaceCacheIdentities(cache, identities);
306
372
  cache.revision = ++this.aggregateRevision;
307
- return { journal: cache.aggregate, revision: cache.revision };
373
+ return this.cachedJournal(cache);
308
374
  }
309
375
  /**
310
376
  * Commit a partitioned journal delta. A metadata-only update still has a
@@ -312,21 +378,25 @@ export class AreaLedger {
312
378
  * protocol so multiple areas never expose a half transaction to `read()`.
313
379
  */
314
380
  append(delta) {
315
- this.invalidateAggregateCache();
381
+ // Bring a warm aggregate to the current durable generation before its
382
+ // caller's delta is appended. That preserves unrelated peer rows while
383
+ // letting the shared packed store stay resident after this write.
384
+ if (this.aggregateCache)
385
+ this.readCached();
316
386
  const partitioned = this.partition(delta);
317
387
  const metadata = cloneMetadata(delta.metadata);
318
388
  if (partitioned.length === 0) {
319
389
  if (metadata) {
320
390
  this.appendMeta({ transactionId: crypto.randomUUID(), areas: [], metadata });
321
391
  this.finalizePublishedTransactions();
322
- this.captureColdCacheIdentities([], true);
392
+ this.updateWarmAggregateAfterAppend(delta, []);
323
393
  }
324
394
  return;
325
395
  }
326
396
  if (partitioned.length === 1 && !metadata) {
327
397
  const [entry] = partitioned;
328
398
  this.openArea(entry.areaId).append(DIRECT_DELTA, entry.delta);
329
- this.captureColdCacheIdentities([entry.areaId], false);
399
+ this.updateWarmAggregateAfterAppend(delta, [entry.areaId]);
330
400
  return;
331
401
  }
332
402
  const transactionId = crypto.randomUUID();
@@ -350,7 +420,7 @@ export class AreaLedger {
350
420
  this.appendMeta({ transactionId, areas: areaIds, ...(metadata ? { metadata } : {}) });
351
421
  testHooks.afterWatermarkBeforeFinalize?.();
352
422
  this.finalizePublishedTransactions();
353
- this.captureColdCacheIdentities(areaIds, true);
423
+ this.updateWarmAggregateAfterAppend(delta, areaIds);
354
424
  }
355
425
  /**
356
426
  * Finish a transaction whose process died after durable area prepares but
@@ -470,10 +540,12 @@ export class AreaLedger {
470
540
  if (!source)
471
541
  return emptyJournal();
472
542
  const files = {};
473
- for (const [key, entry] of Object.entries(source.files)) {
543
+ const rows = rowsOf(source);
544
+ for (const key of rows.keys()) {
474
545
  // Nothing is on disk yet, so there is nothing to respect but the layout:
475
546
  // this is pure routing by construction.
476
- if (this.resolveArea(key, NO_ROWS) === areaId)
547
+ const entry = rows.get(key);
548
+ if (entry !== undefined && this.resolveArea(key, NO_ROWS) === areaId)
477
549
  files[key] = structuredClone(entry);
478
550
  }
479
551
  return { version: "2", lastSync: "", files, pulls: [] };
@@ -563,7 +635,7 @@ export class AreaLedger {
563
635
  identity: store.getGenerationIdentity(),
564
636
  metaIdentity: this.openMeta().getGenerationIdentity(),
565
637
  pending: Object.keys(state.prepared).some((transactionId) => !committed.has(transactionId)),
566
- keys: new Set(Object.keys(materializeArea(state, committed).files)),
638
+ keys: new Set(rowsOf(materializeArea(state, committed)).keys()),
567
639
  };
568
640
  }
569
641
  return this.overflowRows.keys.has(key);
@@ -590,7 +662,8 @@ export class AreaLedger {
590
662
  // when a real area has just taken it. This is a generation stat, not
591
663
  // a read, so an unchanged area still costs nothing.
592
664
  if (identity !== undefined && !this.openArea(areaId).hasChangedOnDisk(identity)) {
593
- return cache.areas.get(areaId)?.files[key] !== undefined;
665
+ const journal = cache.areas.get(areaId);
666
+ return journal === undefined ? false : rowsOf(journal).has(key);
594
667
  }
595
668
  return fromDisk.holds(areaId, key);
596
669
  },
@@ -696,21 +769,14 @@ export class AreaLedger {
696
769
  return "replace";
697
770
  throw new Error(`area-ledger: key ${JSON.stringify(key)} is present in multiple areas`);
698
771
  }
699
- combine(journals) {
700
- const aggregate = emptyJournal();
701
- const owner = new Map();
702
- for (const { areaId, journal } of journals) {
703
- for (const [key, entry] of Object.entries(journal.files)) {
704
- const incumbent = owner.get(key);
705
- if (incumbent !== undefined && this.resolveAreaCollision(key, incumbent, areaId) === "keep")
706
- continue;
707
- // The aggregate holds the SAME row object as the area projection and
708
- // the raw area state — never a copy. See `shareRow`.
709
- aggregate.files[key] = shareRow(entry);
710
- owner.set(key, areaId);
711
- }
712
- }
713
- return aggregate;
772
+ combine(rows) {
773
+ const aggregate = {
774
+ version: "2",
775
+ lastSync: "",
776
+ files: createJournalRecordProxy(rows),
777
+ pulls: [],
778
+ };
779
+ return { journal: aggregate, rows };
714
780
  }
715
781
  /** A watermark publishes only when every named area has prepared or folded it. */
716
782
  committedTransactions(meta, states) {
@@ -789,18 +855,61 @@ export class AreaLedger {
789
855
  const states = this.currentAreaStates();
790
856
  const meta = this.openMeta().getState();
791
857
  const committed = this.committedTransactions(meta, states);
858
+ const identities = this.currentAreaIdentities();
859
+ const shared = this.options.sharedJournalRowsCacheKey === undefined
860
+ ? undefined
861
+ : sharedAreaJournalRows(this.options.sharedJournalRowsCacheKey, areaJournalRowsIdentity(identities));
792
862
  const areas = new Map(this.areaIds.map((areaId) => [areaId, materializeArea(states.get(areaId), committed)]));
863
+ const rows = shared?.rows ?? new JournalRowStore(countJournalRows([...areas].map(([areaId, journal]) => ({ areaId, journal }))), { trackChanges: false });
864
+ const rowOwners = shared?.owners ?? new Map();
865
+ const shadowRows = shared?.shadows ?? new Map();
866
+ if (!shared)
867
+ this.packAreaJournals([...areas].map(([areaId, journal]) => ({ areaId, journal })), rows, rowOwners, shadowRows);
868
+ for (const [areaId, state] of states) {
869
+ this.attachAreaJournal(state, areaId, rows, rowOwners, shadowRows);
870
+ areas.set(areaId, materializeArea(state, committed));
871
+ }
872
+ const combined = this.combine(rows);
793
873
  const cache = {
794
- aggregate: this.combine([...areas].map(([areaId, journal]) => ({ areaId, journal }))),
874
+ aggregate: combined.journal,
875
+ rows: combined.rows,
876
+ rowOwners,
877
+ shadowRows,
795
878
  areas,
796
- identities: new Map(),
879
+ identities,
797
880
  committed,
798
881
  revision: this.aggregateRevision,
799
882
  };
800
883
  this.updateCacheMetadata(cache, meta, committed);
801
- this.refreshCacheIdentities(cache);
802
884
  this.aggregateCache = cache;
803
885
  }
886
+ /**
887
+ * Associate this aggregate view with the one packed store for its durable
888
+ * area generations. Sessions can borrow it without rebuilding every row;
889
+ * an identity change creates a new packed store before the new aggregate is
890
+ * exposed, so a later async opener cannot observe stale rows.
891
+ */
892
+ cachedJournal(cache) {
893
+ const cacheKey = this.options.sharedJournalRowsCacheKey;
894
+ if (cacheKey !== undefined) {
895
+ const identity = areaJournalRowsIdentity(cache.identities);
896
+ const shared = sharedAreaJournalRows(cacheKey, identity);
897
+ if (shared && shared.rows !== cache.rows) {
898
+ cache.rows = shared.rows;
899
+ cache.rowOwners = shared.owners;
900
+ cache.shadowRows = shared.shadows;
901
+ cache.aggregate.files = createJournalRecordProxy(shared.rows);
902
+ for (const [areaId, journal] of cache.areas) {
903
+ journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, cache.rows, cache.rowOwners, cache.shadowRows));
904
+ }
905
+ }
906
+ else if (!shared) {
907
+ rememberSharedAreaJournalRows(cacheKey, identity, cache.rows, cache.rowOwners, cache.shadowRows);
908
+ }
909
+ }
910
+ rememberDecodedJournalRows(cache.aggregate, cache.rows);
911
+ return { journal: cache.aggregate, rows: cache.rows.rows, revision: cache.revision };
912
+ }
804
913
  /** Generation pointer + WAL size observation takes neither append lock nor readdir. */
805
914
  refreshChangedStores(cache) {
806
915
  const meta = this.openMeta();
@@ -850,46 +959,35 @@ export class AreaLedger {
850
959
  }
851
960
  updateCachedArea(cache, areaId, journal) {
852
961
  const previous = cache.areas.get(areaId);
853
- // Keys this refresh removes that ANOTHER area also holds. Their aggregate
854
- // value may belong to that other area, so it has to be put back after the
855
- // removal loop below — otherwise refreshing one side of a tolerated
856
- // duplicate silently destroys the winning side's value, or drops the key.
857
- const shared = new Set();
858
- for (const key of Object.keys(previous.files)) {
859
- if (this.sharesKeyWithAnotherArea(cache, key, areaId))
860
- shared.add(key);
861
- delete cache.aggregate.files[key];
862
- }
863
- cache.areas.set(areaId, journal);
864
- for (const [key, entry] of Object.entries(journal.files)) {
865
- // Same rule as `combine`, on the incremental path. A key another area also
866
- // holds may have just been deleted from the aggregate by the loop above,
867
- // so `shared` — not the aggregate — is what says a collision is possible.
868
- // The incumbent's area is not carried alongside the aggregate, so recover
869
- // it from the per-area journals, only ever on an actual collision.
870
- // `shared` covers a key the removal loop above just deleted from the
871
- // aggregate; the aggregate itself covers a key this area did not
872
- // previously hold but another area does — the two-real-area corruption a
873
- // full rebuild rejects, which the incremental path must reject too.
874
- if (shared.has(key) || cache.aggregate.files[key] !== undefined) {
875
- const incumbent = this.findCachedOwner(cache, key, areaId);
876
- if (incumbent !== undefined && this.resolveAreaCollision(key, incumbent, areaId) === "keep") {
877
- cache.aggregate.files[key] = shareRow(cache.areas.get(incumbent).files[key]);
878
- continue;
879
- }
962
+ const overlay = journalRecordRows(journal.files);
963
+ if (overlay instanceof JournalRowOverlay) {
964
+ // Folding a prepared transaction puts a private overlay into the area
965
+ // StateStore. A later direct append mutates that same object in place,
966
+ // so identity alone cannot tell us whether the aggregate received its
967
+ // rows. Replay just the overlay's changed paths into the packed owner,
968
+ // then replace the overlay before it can become another base layer.
969
+ for (const change of overlay.dirtyLogSnapshot()) {
970
+ const entry = overlay.get(change.path);
971
+ if (entry === undefined)
972
+ this.deleteAreaRow(cache.rows, cache.rowOwners, cache.shadowRows, areaId, change.path);
973
+ else
974
+ this.writeAreaRow(cache.rows, cache.rowOwners, cache.shadowRows, areaId, change.path, entry);
880
975
  }
881
- cache.aggregate.files[key] = shareRow(entry);
976
+ journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, cache.rows, cache.rowOwners, cache.shadowRows));
977
+ cache.areas.set(areaId, journal);
978
+ return;
882
979
  }
883
- // Keys this refresh dropped that another area still holds: the value the
884
- // removal loop deleted may have been that other area's, so put it back.
885
- for (const key of shared) {
886
- if (cache.aggregate.files[key] !== undefined)
887
- continue;
888
- const owner = this.findCachedOwner(cache, key, areaId);
889
- if (owner === undefined)
890
- continue;
891
- cache.aggregate.files[key] = shareRow(cache.areas.get(owner).files[key]);
980
+ // WAL replay can apply a direct delta through this area's shared facade,
981
+ // so `previous` already observes the new rows. Repacking it after deleting
982
+ // from the same backing store would erase the source before it is copied.
983
+ if (previous === journal)
984
+ return;
985
+ for (const key of rowsOf(previous).keys()) {
986
+ this.deleteAreaRow(cache.rows, cache.rowOwners, cache.shadowRows, areaId, key);
892
987
  }
988
+ cache.shadowRows.delete(areaId);
989
+ this.packAreaJournals([{ areaId, journal }], cache.rows, cache.rowOwners, cache.shadowRows);
990
+ cache.areas.set(areaId, journal);
893
991
  }
894
992
  /**
895
993
  * Whether an area other than `areaId` also holds `key`.
@@ -902,7 +1000,8 @@ export class AreaLedger {
902
1000
  */
903
1001
  sharesKeyWithAnotherArea(cache, key, areaId) {
904
1002
  if (areaId !== UNROUTED_AREA_ID) {
905
- return cache.areas.get(UNROUTED_AREA_ID)?.files[key] !== undefined;
1003
+ const overflow = cache.areas.get(UNROUTED_AREA_ID);
1004
+ return overflow === undefined ? false : rowsOf(overflow).has(key);
906
1005
  }
907
1006
  return this.findCachedOwner(cache, key, areaId) !== undefined;
908
1007
  }
@@ -917,7 +1016,7 @@ export class AreaLedger {
917
1016
  for (const [candidateId, candidate] of cache.areas) {
918
1017
  if (candidateId === excludeAreaId)
919
1018
  continue;
920
- if (candidate.files[key] === undefined)
1019
+ if (!rowsOf(candidate).has(key))
921
1020
  continue;
922
1021
  if (candidateId === UNROUTED_AREA_ID)
923
1022
  overflow = candidateId;
@@ -931,10 +1030,112 @@ export class AreaLedger {
931
1030
  cache.aggregate.pulls = structuredClone(meta.journal.pulls);
932
1031
  cache.committed = new Set(committed);
933
1032
  }
934
- refreshCacheIdentities(cache) {
935
- cache.identities.set("@meta", this.openMeta().getGenerationIdentity());
1033
+ replaceCacheIdentities(cache, identities) {
1034
+ // These identities describe the same state we just materialized. Reading
1035
+ // fresh identities after reconciliation could acknowledge a concurrent
1036
+ // publication whose rows never reached the packed cache.
1037
+ cache.identities = new Map(identities);
1038
+ }
1039
+ currentAreaIdentities() {
1040
+ const identities = new Map();
1041
+ identities.set("@meta", this.openMeta().getGenerationIdentity());
936
1042
  for (const areaId of this.areaIds)
937
- cache.identities.set(areaId, this.openArea(areaId).getGenerationIdentity());
1043
+ identities.set(areaId, this.openArea(areaId).getGenerationIdentity());
1044
+ return identities;
1045
+ }
1046
+ defaultAreaForKey(key) {
1047
+ const resolved = this.options.resolveArea(key);
1048
+ return resolved !== undefined && this.areaIds.includes(resolved)
1049
+ ? resolved
1050
+ : UNROUTED_AREA_ID;
1051
+ }
1052
+ rowOwner(key, owners) {
1053
+ return owners.get(key) ?? this.defaultAreaForKey(key);
1054
+ }
1055
+ areaJournalRows(areaId, rows, owners, shadows) {
1056
+ const areaShadows = shadows.get(areaId) ?? new Map();
1057
+ shadows.set(areaId, areaShadows);
1058
+ return new AreaJournalRows(rows, areaId, (key) => this.defaultAreaForKey(key), owners, areaShadows, (key, entry) => this.writeAreaRow(rows, owners, shadows, areaId, key, entry), (key) => this.deleteAreaRow(rows, owners, shadows, areaId, key));
1059
+ }
1060
+ /** Replace an area record with its filtered facade over the shared packed rows. */
1061
+ attachAreaJournal(state, areaId, rows, owners, shadows) {
1062
+ state.journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, rows, owners, shadows));
1063
+ }
1064
+ /**
1065
+ * Move decoded area records into the one aggregate store, then discard each
1066
+ * record map in favour of an area-filtered facade over that store. A shared
1067
+ * aggregate bypasses this method entirely, so opening another bridge never
1068
+ * repacks every row merely to throw the new store away.
1069
+ */
1070
+ packAreaJournals(journals, rows, owners, shadows) {
1071
+ for (const { areaId, journal } of journals) {
1072
+ const files = journal.files;
1073
+ for (const key in files) {
1074
+ if (!Object.hasOwn(files, key))
1075
+ continue;
1076
+ const entry = files[key];
1077
+ this.writeAreaRow(rows, owners, shadows, areaId, key, entry);
1078
+ }
1079
+ journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, rows, owners, shadows));
1080
+ }
1081
+ }
1082
+ writeAreaRow(rows, owners, shadows, areaId, key, entry) {
1083
+ const areaShadows = shadows.get(areaId) ?? new Map();
1084
+ shadows.set(areaId, areaShadows);
1085
+ if (areaShadows.has(key)) {
1086
+ areaShadows.set(key, entry);
1087
+ return;
1088
+ }
1089
+ const incumbent = rows.has(key) ? this.rowOwner(key, owners) : undefined;
1090
+ if (incumbent !== undefined && incumbent !== areaId) {
1091
+ const resolution = this.resolveAreaCollision(key, incumbent, areaId);
1092
+ if (resolution === "keep") {
1093
+ areaShadows.set(key, entry);
1094
+ return;
1095
+ }
1096
+ const incumbentShadows = shadows.get(incumbent) ?? new Map();
1097
+ incumbentShadows.set(key, rows.get(key));
1098
+ shadows.set(incumbent, incumbentShadows);
1099
+ }
1100
+ rows.set(key, entry);
1101
+ const defaultOwner = this.defaultAreaForKey(key);
1102
+ if (defaultOwner === areaId)
1103
+ owners.delete(key);
1104
+ else
1105
+ owners.set(key, areaId);
1106
+ }
1107
+ deleteAreaRow(rows, owners, shadows, areaId, key) {
1108
+ const areaShadows = shadows.get(areaId);
1109
+ if (areaShadows?.delete(key))
1110
+ return true;
1111
+ if (this.rowOwner(key, owners) !== areaId)
1112
+ return false;
1113
+ const deleted = rows.delete(key);
1114
+ if (!deleted)
1115
+ return false;
1116
+ owners.delete(key);
1117
+ this.promoteShadowAfterOwnerDelete(rows, owners, shadows, key);
1118
+ return deleted;
1119
+ }
1120
+ /** A shadow remains durable when its aggregate owner is deleted, so make it visible. */
1121
+ promoteShadowAfterOwnerDelete(rows, owners, shadows, key) {
1122
+ let survivor;
1123
+ for (const [areaId, areaShadows] of shadows) {
1124
+ const entry = areaShadows.get(key);
1125
+ if (entry === undefined)
1126
+ continue;
1127
+ if (survivor === undefined) {
1128
+ survivor = { areaId, entry };
1129
+ continue;
1130
+ }
1131
+ if (this.resolveAreaCollision(key, survivor.areaId, areaId) === "replace") {
1132
+ survivor = { areaId, entry };
1133
+ }
1134
+ }
1135
+ if (survivor === undefined)
1136
+ return;
1137
+ shadows.get(survivor.areaId).delete(key);
1138
+ this.writeAreaRow(rows, owners, shadows, survivor.areaId, key, survivor.entry);
938
1139
  }
939
1140
  /** Refresh only identities that changed since this process's own cold-cache append. */
940
1141
  refreshChangedColdStores() {
@@ -960,6 +1161,34 @@ export class AreaLedger {
960
1161
  }
961
1162
  this.coldCacheIdentities = identities;
962
1163
  }
1164
+ /** Keep a warm packed aggregate current without rebuilding every unchanged row. */
1165
+ updateWarmAggregateAfterAppend(delta, changedAreaIds) {
1166
+ const cache = this.aggregateCache;
1167
+ if (!cache) {
1168
+ this.captureColdCacheIdentities(changedAreaIds, delta.metadata !== undefined);
1169
+ return;
1170
+ }
1171
+ const states = this.currentAreaStates();
1172
+ const identities = this.currentAreaIdentities();
1173
+ const meta = this.openMeta().getState();
1174
+ const committed = this.committedTransactions(meta, states);
1175
+ const areasToReconcile = new Set(changedAreaIds);
1176
+ for (const areaId of this.areaIds) {
1177
+ const previous = cache.identities.get(areaId);
1178
+ const current = identities.get(areaId);
1179
+ if (previous.generation !== current.generation ||
1180
+ previous.lastRecordSequence !== current.lastRecordSequence ||
1181
+ previous.walBytes !== current.walBytes) {
1182
+ areasToReconcile.add(areaId);
1183
+ }
1184
+ }
1185
+ for (const areaId of areasToReconcile) {
1186
+ this.updateCachedArea(cache, areaId, materializeArea(states.get(areaId), committed));
1187
+ }
1188
+ this.updateCacheMetadata(cache, meta, committed);
1189
+ this.replaceCacheIdentities(cache, identities);
1190
+ cache.revision = ++this.aggregateRevision;
1191
+ }
963
1192
  invalidateAggregateCache() {
964
1193
  if (this.aggregateCache)
965
1194
  this.coldCacheIdentities = new Map(this.aggregateCache.identities);
@@ -977,6 +1206,49 @@ function symmetricDifference(left, right) {
977
1206
  result.add(value);
978
1207
  return result;
979
1208
  }
1209
+ /** Count a source one key at a time so capacity planning never creates a key array. */
1210
+ function countJournalRows(journals) {
1211
+ let count = 0;
1212
+ for (const { journal } of journals) {
1213
+ const rows = decodedJournalRows(journal) ?? journalRecordRows(journal.files);
1214
+ if (rows?.size !== undefined) {
1215
+ count += rows.size;
1216
+ continue;
1217
+ }
1218
+ for (const key in journal.files) {
1219
+ if (Object.hasOwn(journal.files, key))
1220
+ count += 1;
1221
+ }
1222
+ }
1223
+ return count;
1224
+ }
1225
+ /** Keep the record proxy at the public edge while internal ledger code stays on lazy rows. */
1226
+ function rowsOf(journal) {
1227
+ const decoded = decodedJournalRows(journal);
1228
+ if (decoded)
1229
+ return decoded.rows;
1230
+ const proxied = journalRecordRows(journal.files);
1231
+ if (proxied)
1232
+ return proxied;
1233
+ const files = journal.files;
1234
+ return {
1235
+ get: (key) => files[key],
1236
+ has: (key) => Object.hasOwn(files, key),
1237
+ set: (key, entry) => { files[key] = entry; },
1238
+ delete: (key) => {
1239
+ if (!Object.hasOwn(files, key))
1240
+ return false;
1241
+ delete files[key];
1242
+ return true;
1243
+ },
1244
+ *keys() {
1245
+ for (const key in files) {
1246
+ if (Object.hasOwn(files, key))
1247
+ yield key;
1248
+ }
1249
+ },
1250
+ };
1251
+ }
980
1252
  function reduceAreaRecord(state, record) {
981
1253
  state.direct ??= [];
982
1254
  state.prepared ??= {};
@@ -1051,10 +1323,10 @@ function isFinalizeRecord(value) {
1051
1323
  }
1052
1324
  /**
1053
1325
  * A row is an immutable value. Every projection of a journal — the raw area
1054
- * state, the materialized per-area journal, the combined aggregate, and each
1055
- * `JournalStore` session opened on top — holds the SAME row object, and the
1056
- * object is frozen so an in-place mutation throws instead of silently
1057
- * rewriting every other projection at once.
1326
+ * state and the materialized per-area journal hold the same frozen row object.
1327
+ * The combined aggregate owns a packed representation; its record facade
1328
+ * materializes frozen rows on demand so an in-place mutation cannot silently
1329
+ * rewrite another projection.
1058
1330
  *
1059
1331
  * This is a memory bound, not a nicety. A personal vault on this fleet has
1060
1332
  * ~690k rows; deep-copying them per projection put five or six full copies in
@@ -1081,20 +1353,28 @@ function freezeDeep(value) {
1081
1353
  Object.freeze(value);
1082
1354
  }
1083
1355
  function materializeArea(state, committed) {
1084
- // Shallow: a new key map over the SAME row objects as the area state.
1356
+ const events = visibleEvents(state, committed);
1357
+ // A settled area is already a stable private projection. Returning it as-is
1358
+ // avoids the former whole-record clone that lived beside the packed aggregate.
1359
+ if (events.length === 0)
1360
+ return state.journal;
1361
+ // Pending direct/prepared records must remain isolated from their durable
1362
+ // base until a fold makes them visible. Keep only those changed keys in an
1363
+ // overlay instead of cloning every base row just to apply a small tail.
1364
+ const overlay = new JournalRowOverlay(rowsOf(state.journal));
1085
1365
  const journal = {
1086
1366
  ...state.journal,
1087
- files: { ...state.journal.files },
1367
+ files: createJournalRecordProxy(overlay),
1088
1368
  pulls: structuredClone(state.journal.pulls ?? []),
1089
1369
  };
1090
- for (const event of visibleEvents(state, committed))
1370
+ for (const event of events)
1091
1371
  applyDelta(journal, event.delta);
1092
1372
  return journal;
1093
1373
  }
1094
1374
  /** Apply only events relevant to one key; unlike materializeArea this never clones a whole journal. */
1095
1375
  function materializeRow(state, committed, key, events = visibleEvents(state, committed)) {
1096
1376
  testHooks.onMaterializeRow?.(key);
1097
- let row = state.journal.files[key];
1377
+ let row = rowsOf(state.journal).get(key);
1098
1378
  for (const event of events) {
1099
1379
  if (event.delta.deletes.includes(key))
1100
1380
  row = undefined;
@@ -1115,7 +1395,7 @@ function visibleEvents(state, committed) {
1115
1395
  }
1116
1396
  /** Key-only union used by scoped directory reads; values stay un-cloned until selected. */
1117
1397
  function visibleRowKeys(state, events) {
1118
- const keys = new Set(Object.keys(state.journal.files));
1398
+ const keys = new Set(rowsOf(state.journal).keys());
1119
1399
  for (const event of events) {
1120
1400
  for (const key of event.delta.deletes)
1121
1401
  keys.add(key);
@@ -1125,16 +1405,33 @@ function visibleRowKeys(state, events) {
1125
1405
  return keys;
1126
1406
  }
1127
1407
  function applyDelta(journal, delta) {
1408
+ const rows = rowsOf(journal);
1128
1409
  for (const key of delta.deletes)
1129
- delete journal.files[key];
1410
+ rows.delete(key);
1130
1411
  // Delta rows are already this ledger's own copies (see `partition` and the
1131
1412
  // WAL fold); adopt the objects, do not copy them again.
1132
1413
  for (const [key, entry] of Object.entries(delta.upserts))
1133
- journal.files[key] = shareRow(entry);
1414
+ rows.set(key, shareRow(entry));
1134
1415
  }
1135
1416
  function emptyJournal() {
1136
1417
  return { version: "2", lastSync: "", files: {}, pulls: [] };
1137
1418
  }
1419
+ /**
1420
+ * Public aggregate readers receive a mutable record facade without copying all
1421
+ * rows. Replacements and deletes stay private to this view; rows returned from
1422
+ * the immutable aggregate remain frozen, so in-place mutation cannot bypass a
1423
+ * later delta writer.
1424
+ */
1425
+ function detachedJournalOverlay(journal, source) {
1426
+ const rows = source ?? rowsOf(journal);
1427
+ const result = {
1428
+ ...journal,
1429
+ files: createJournalRecordProxy(new JournalRowOverlay(rows)),
1430
+ };
1431
+ if (Object.hasOwn(journal, "pulls"))
1432
+ result.pulls = structuredClone(journal.pulls);
1433
+ return result;
1434
+ }
1138
1435
  function capPullRecords(pulls) {
1139
1436
  const journal = { version: "2", lastSync: "", files: {}, pulls: structuredClone(pulls) };
1140
1437
  trimPullRecords(journal);