@indigoai-us/hq-cloud 6.16.41 → 6.16.42-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  2. package/dist/bin/sync-runner-watch-loop.js +9 -9
  3. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  4. package/dist/cli/share.d.ts.map +1 -1
  5. package/dist/cli/share.js +28 -27
  6. package/dist/cli/share.js.map +1 -1
  7. package/dist/cli/share.test.js +49 -0
  8. package/dist/cli/share.test.js.map +1 -1
  9. package/dist/cli/sync.d.ts.map +1 -1
  10. package/dist/cli/sync.js +8 -8
  11. package/dist/cli/sync.js.map +1 -1
  12. package/dist/journal-area-ledger-read-boundary.test.js +46 -12
  13. package/dist/journal-area-ledger-read-boundary.test.js.map +1 -1
  14. package/dist/journal-record-proxy.d.ts +12 -2
  15. package/dist/journal-record-proxy.d.ts.map +1 -1
  16. package/dist/journal-record-proxy.js +8 -1
  17. package/dist/journal-record-proxy.js.map +1 -1
  18. package/dist/journal-row-store.d.ts +70 -0
  19. package/dist/journal-row-store.d.ts.map +1 -1
  20. package/dist/journal-row-store.js +145 -3
  21. package/dist/journal-row-store.js.map +1 -1
  22. package/dist/journal-row-store.test.js +3 -1
  23. package/dist/journal-row-store.test.js.map +1 -1
  24. package/dist/journal-snapshot-rows.d.ts +17 -1
  25. package/dist/journal-snapshot-rows.d.ts.map +1 -1
  26. package/dist/journal-snapshot-rows.js +18 -0
  27. package/dist/journal-snapshot-rows.js.map +1 -1
  28. package/dist/journal-snapshot-v4.d.ts +3 -2
  29. package/dist/journal-snapshot-v4.d.ts.map +1 -1
  30. package/dist/journal-snapshot-v4.js +9 -3
  31. package/dist/journal-snapshot-v4.js.map +1 -1
  32. package/dist/journal-snapshot-v4.test.js +1 -0
  33. package/dist/journal-snapshot-v4.test.js.map +1 -1
  34. package/dist/journal.d.ts +23 -5
  35. package/dist/journal.d.ts.map +1 -1
  36. package/dist/journal.js +309 -189
  37. package/dist/journal.js.map +1 -1
  38. package/dist/journal.test.js +58 -1
  39. package/dist/journal.test.js.map +1 -1
  40. package/dist/packed-journal-phase2.test.d.ts +2 -0
  41. package/dist/packed-journal-phase2.test.d.ts.map +1 -0
  42. package/dist/packed-journal-phase2.test.js +273 -0
  43. package/dist/packed-journal-phase2.test.js.map +1 -0
  44. package/dist/personal-vault.d.ts.map +1 -1
  45. package/dist/personal-vault.js +6 -4
  46. package/dist/personal-vault.js.map +1 -1
  47. package/dist/remote-pull.d.ts.map +1 -1
  48. package/dist/remote-pull.js +9 -8
  49. package/dist/remote-pull.js.map +1 -1
  50. package/dist/scope-shrink.js +2 -2
  51. package/dist/scope-shrink.js.map +1 -1
  52. package/dist/session-log-retention.d.ts.map +1 -1
  53. package/dist/session-log-retention.js +2 -2
  54. package/dist/session-log-retention.js.map +1 -1
  55. package/dist/sync/area-ledger-migration.d.ts +13 -6
  56. package/dist/sync/area-ledger-migration.d.ts.map +1 -1
  57. package/dist/sync/area-ledger-migration.js +45 -19
  58. package/dist/sync/area-ledger-migration.js.map +1 -1
  59. package/dist/sync/area-ledger-migration.test.js +23 -0
  60. package/dist/sync/area-ledger-migration.test.js.map +1 -1
  61. package/dist/sync/area-ledger.d.ts +31 -1
  62. package/dist/sync/area-ledger.d.ts.map +1 -1
  63. package/dist/sync/area-ledger.js +438 -88
  64. package/dist/sync/area-ledger.js.map +1 -1
  65. package/dist/sync/area-ledger.test.js +155 -10
  66. package/dist/sync/area-ledger.test.js.map +1 -1
  67. package/dist/sync/area-resolver.d.ts.map +1 -1
  68. package/dist/sync/area-resolver.js +6 -6
  69. package/dist/sync/area-resolver.js.map +1 -1
  70. package/dist/types.d.ts +2 -0
  71. package/dist/types.d.ts.map +1 -1
  72. package/package.json +1 -1
@@ -7,6 +7,9 @@
7
7
  * not understand a layout continue to use their existing authority.
8
8
  */
9
9
  import * as crypto from "node:crypto";
10
+ import { JournalRowOverlay, JournalRowStore } from "../journal-row-store.js";
11
+ import { createJournalRecordProxy, journalRecordRows } from "../journal-record-proxy.js";
12
+ import { decodedJournalRows, rememberDecodedJournalRows, rememberSharedAreaJournalRows, sharedAreaJournalRows, } from "../journal-snapshot-rows.js";
10
13
  import { trimPullRecords } from "../journal.js";
11
14
  import { StateStore } from "./state-store.js";
12
15
  const DIRECT_DELTA = 1;
@@ -37,6 +40,100 @@ const META_FINALIZE = 2;
37
40
  export const UNROUTED_AREA_ID = "root:@unrouted";
38
41
  /** Nothing is held anywhere — for a ledger with no state on disk yet. */
39
42
  const NO_ROWS = { holds: () => false };
43
+ function areaJournalRowsIdentity(identities) {
44
+ return [...identities]
45
+ .filter(([areaId]) => areaId !== "@meta")
46
+ .sort(([left], [right]) => left.localeCompare(right))
47
+ .map(([areaId, identity]) => `${areaId}:${identity.generation}:${identity.lastRecordSequence}:${identity.walBytes}`)
48
+ .join("\u0000");
49
+ }
50
+ /**
51
+ * Index each aggregate row under its effective owner so per-area facades do
52
+ * not re-resolve every shared key on every enumeration.
53
+ */
54
+ class AreaJournalRowIndex {
55
+ rowIdsByArea = new Map();
56
+ /** A collision can move an old key into a later owner's bucket. */
57
+ reorderedAreas = new Set();
58
+ owns(areaId, rows, key) {
59
+ const rowId = rows.rowIdFor(key);
60
+ return rowId !== undefined && (this.rowIdsByArea.get(areaId)?.has(rowId) ?? false);
61
+ }
62
+ add(areaId, rowId, movedFromAnotherArea = false) {
63
+ const rowIds = this.rowIdsByArea.get(areaId) ?? new Set();
64
+ rowIds.add(rowId);
65
+ this.rowIdsByArea.set(areaId, rowIds);
66
+ if (movedFromAnotherArea)
67
+ this.reorderedAreas.add(areaId);
68
+ }
69
+ delete(areaId, rowId) {
70
+ this.rowIdsByArea.get(areaId)?.delete(rowId);
71
+ }
72
+ *keys(areaId, rows) {
73
+ let rowIds = this.rowIdsByArea.get(areaId);
74
+ if (!rowIds)
75
+ return;
76
+ if (this.reorderedAreas.delete(areaId)) {
77
+ // A replacement preserves the packed row's global position. Rebuild just
78
+ // this exceptional bucket so record own-key order remains unchanged.
79
+ const ordered = new Set();
80
+ for (const rowId of rows.rowIds())
81
+ if (rowIds.has(rowId))
82
+ ordered.add(rowId);
83
+ this.rowIdsByArea.set(areaId, ordered);
84
+ rowIds = ordered;
85
+ }
86
+ for (const rowId of rowIds)
87
+ yield rows.pathForRowId(rowId);
88
+ }
89
+ }
90
+ const areaJournalRowIndexes = new WeakMap();
91
+ function areaJournalRowIndex(rows) {
92
+ const existing = areaJournalRowIndexes.get(rows);
93
+ if (existing)
94
+ return existing;
95
+ const created = new AreaJournalRowIndex();
96
+ areaJournalRowIndexes.set(rows, created);
97
+ return created;
98
+ }
99
+ /** A state-store-facing slice over the one packed aggregate. */
100
+ class AreaJournalRows {
101
+ rows;
102
+ areaId;
103
+ rowIndex;
104
+ shadows;
105
+ write;
106
+ erase;
107
+ constructor(rows, areaId, rowIndex, shadows, write, erase) {
108
+ this.rows = rows;
109
+ this.areaId = areaId;
110
+ this.rowIndex = rowIndex;
111
+ this.shadows = shadows;
112
+ this.write = write;
113
+ this.erase = erase;
114
+ }
115
+ get(path) {
116
+ return this.shadows.get(path) ?? (this.owns(path) ? this.rows.get(path) : undefined);
117
+ }
118
+ has(path) {
119
+ return this.shadows.has(path) || (this.owns(path) && this.rows.has(path));
120
+ }
121
+ set(path, entry) {
122
+ this.write(path, entry);
123
+ }
124
+ delete(path) {
125
+ return this.erase(path);
126
+ }
127
+ *keys() {
128
+ yield* this.shadows.keys();
129
+ for (const key of this.rowIndex.keys(this.areaId, this.rows))
130
+ if (!this.shadows.has(key))
131
+ yield key;
132
+ }
133
+ owns(path) {
134
+ return this.rowIndex.owns(this.areaId, this.rows, path);
135
+ }
136
+ }
40
137
  let testHooks = {};
41
138
  /** Test-only crash seam: data prepares are durable before a meta watermark. */
42
139
  export function setAreaLedgerTestHooksForTest(hooks) {
@@ -119,9 +216,22 @@ export class AreaLedger {
119
216
  }
120
217
  }
121
218
  const committed = this.committedTransactions(meta.getState(), states);
122
- // `combine` shares row objects with the area states; this one-off reader
123
- // hands out a detached copy, so a caller may mutate what it gets back.
124
- const aggregate = structuredClone(this.combine([...selected].map((areaId) => ({ areaId, journal: materializeArea(states.get(areaId), committed) }))));
219
+ // The overlay owns only caller changes; unchanged rows remain shared with
220
+ // the selected area projections instead of being deep-cloned as an
221
+ // aggregate before this scoped reader can inspect them.
222
+ const journals = [...selected].map((areaId) => ({
223
+ areaId,
224
+ // `materializeArea` returns the live StateStore journal when there are
225
+ // no pending records. Packing a scoped result must replace only this
226
+ // shell's facade, never the cached area's live journal facade.
227
+ journal: { ...materializeArea(states.get(areaId), committed) },
228
+ }));
229
+ const rows = new JournalRowStore(countJournalRows(journals), { trackChanges: false });
230
+ const owners = new Map();
231
+ const shadows = new Map();
232
+ this.packAreaJournals(journals, rows, owners, shadows);
233
+ const combined = this.combine(rows);
234
+ const aggregate = detachedJournalOverlay(combined.journal, combined.rows.rows);
125
235
  aggregate.lastSync = meta.getState().journal.lastSync;
126
236
  aggregate.pulls = structuredClone(meta.getState().journal.pulls);
127
237
  return aggregate;
@@ -255,7 +365,8 @@ export class AreaLedger {
255
365
  }
256
366
  /** Aggregate read. It is the parity view used by legacy SyncJournal callers. */
257
367
  read() {
258
- return structuredClone(this.readCached().journal);
368
+ const cached = this.readCached();
369
+ return detachedJournalOverlay(cached.journal, cached.rows);
259
370
  }
260
371
  /**
261
372
  * Internal aggregate view for long-lived keyed sessions. The returned journal
@@ -275,21 +386,22 @@ export class AreaLedger {
275
386
  this.primeAggregateCache();
276
387
  }
277
388
  this.aggregateCache.revision = ++this.aggregateRevision;
278
- return { journal: this.aggregateCache.aggregate, revision: this.aggregateCache.revision };
389
+ return this.cachedJournal(this.aggregateCache);
279
390
  }
280
391
  const cache = this.aggregateCache;
281
392
  const { metaChanged, areas } = this.refreshChangedStores(cache);
282
393
  if (!metaChanged && areas.size === 0)
283
- return { journal: cache.aggregate, revision: cache.revision };
394
+ return this.cachedJournal(cache);
284
395
  const meta = this.openMeta().getState();
285
396
  const states = this.currentAreaStates();
397
+ const identities = this.currentAreaIdentities();
286
398
  if (this.hasPendingTransactionEvidence(meta, states)) {
287
399
  // A different process published prepare/watermark evidence. Recovery is
288
400
  // exceptional and bounded to that evidence; the steady read path above
289
401
  // remains lock-free and write-free.
290
402
  this.recoverPendingTransactions();
291
403
  this.primeAggregateCache();
292
- return { journal: this.aggregateCache.aggregate, revision: this.aggregateCache.revision };
404
+ return this.cachedJournal(this.aggregateCache);
293
405
  }
294
406
  const committed = this.committedTransactions(meta, states);
295
407
  for (const transactionId of symmetricDifference(cache.committed, committed)) {
@@ -302,9 +414,9 @@ export class AreaLedger {
302
414
  this.updateCachedArea(cache, areaId, materializeArea(states.get(areaId), committed));
303
415
  }
304
416
  this.updateCacheMetadata(cache, meta, committed);
305
- this.refreshCacheIdentities(cache);
417
+ this.replaceCacheIdentities(cache, identities);
306
418
  cache.revision = ++this.aggregateRevision;
307
- return { journal: cache.aggregate, revision: cache.revision };
419
+ return this.cachedJournal(cache);
308
420
  }
309
421
  /**
310
422
  * Commit a partitioned journal delta. A metadata-only update still has a
@@ -312,21 +424,25 @@ export class AreaLedger {
312
424
  * protocol so multiple areas never expose a half transaction to `read()`.
313
425
  */
314
426
  append(delta) {
315
- this.invalidateAggregateCache();
427
+ // Bring a warm aggregate to the current durable generation before its
428
+ // caller's delta is appended. That preserves unrelated peer rows while
429
+ // letting the shared packed store stay resident after this write.
430
+ if (this.aggregateCache)
431
+ this.readCached();
316
432
  const partitioned = this.partition(delta);
317
433
  const metadata = cloneMetadata(delta.metadata);
318
434
  if (partitioned.length === 0) {
319
435
  if (metadata) {
320
436
  this.appendMeta({ transactionId: crypto.randomUUID(), areas: [], metadata });
321
437
  this.finalizePublishedTransactions();
322
- this.captureColdCacheIdentities([], true);
438
+ this.updateWarmAggregateAfterAppend(delta, []);
323
439
  }
324
440
  return;
325
441
  }
326
442
  if (partitioned.length === 1 && !metadata) {
327
443
  const [entry] = partitioned;
328
444
  this.openArea(entry.areaId).append(DIRECT_DELTA, entry.delta);
329
- this.captureColdCacheIdentities([entry.areaId], false);
445
+ this.updateWarmAggregateAfterAppend(delta, [entry.areaId]);
330
446
  return;
331
447
  }
332
448
  const transactionId = crypto.randomUUID();
@@ -350,7 +466,7 @@ export class AreaLedger {
350
466
  this.appendMeta({ transactionId, areas: areaIds, ...(metadata ? { metadata } : {}) });
351
467
  testHooks.afterWatermarkBeforeFinalize?.();
352
468
  this.finalizePublishedTransactions();
353
- this.captureColdCacheIdentities(areaIds, true);
469
+ this.updateWarmAggregateAfterAppend(delta, areaIds);
354
470
  }
355
471
  /**
356
472
  * Finish a transaction whose process died after durable area prepares but
@@ -470,10 +586,12 @@ export class AreaLedger {
470
586
  if (!source)
471
587
  return emptyJournal();
472
588
  const files = {};
473
- for (const [key, entry] of Object.entries(source.files)) {
589
+ const rows = rowsOf(source);
590
+ for (const key of rows.keys()) {
474
591
  // Nothing is on disk yet, so there is nothing to respect but the layout:
475
592
  // this is pure routing by construction.
476
- if (this.resolveArea(key, NO_ROWS) === areaId)
593
+ const entry = rows.get(key);
594
+ if (entry !== undefined && this.resolveArea(key, NO_ROWS) === areaId)
477
595
  files[key] = structuredClone(entry);
478
596
  }
479
597
  return { version: "2", lastSync: "", files, pulls: [] };
@@ -563,7 +681,7 @@ export class AreaLedger {
563
681
  identity: store.getGenerationIdentity(),
564
682
  metaIdentity: this.openMeta().getGenerationIdentity(),
565
683
  pending: Object.keys(state.prepared).some((transactionId) => !committed.has(transactionId)),
566
- keys: new Set(Object.keys(materializeArea(state, committed).files)),
684
+ keys: new Set(rowsOf(materializeArea(state, committed)).keys()),
567
685
  };
568
686
  }
569
687
  return this.overflowRows.keys.has(key);
@@ -590,7 +708,8 @@ export class AreaLedger {
590
708
  // when a real area has just taken it. This is a generation stat, not
591
709
  // a read, so an unchanged area still costs nothing.
592
710
  if (identity !== undefined && !this.openArea(areaId).hasChangedOnDisk(identity)) {
593
- return cache.areas.get(areaId)?.files[key] !== undefined;
711
+ const journal = cache.areas.get(areaId);
712
+ return journal === undefined ? false : rowsOf(journal).has(key);
594
713
  }
595
714
  return fromDisk.holds(areaId, key);
596
715
  },
@@ -696,21 +815,14 @@ export class AreaLedger {
696
815
  return "replace";
697
816
  throw new Error(`area-ledger: key ${JSON.stringify(key)} is present in multiple areas`);
698
817
  }
699
- combine(journals) {
700
- const aggregate = emptyJournal();
701
- const owner = new Map();
702
- for (const { areaId, journal } of journals) {
703
- for (const [key, entry] of Object.entries(journal.files)) {
704
- const incumbent = owner.get(key);
705
- if (incumbent !== undefined && this.resolveAreaCollision(key, incumbent, areaId) === "keep")
706
- continue;
707
- // The aggregate holds the SAME row object as the area projection and
708
- // the raw area state — never a copy. See `shareRow`.
709
- aggregate.files[key] = shareRow(entry);
710
- owner.set(key, areaId);
711
- }
712
- }
713
- return aggregate;
818
+ combine(rows) {
819
+ const aggregate = {
820
+ version: "2",
821
+ lastSync: "",
822
+ files: createJournalRecordProxy(rows),
823
+ pulls: [],
824
+ };
825
+ return { journal: aggregate, rows };
714
826
  }
715
827
  /** A watermark publishes only when every named area has prepared or folded it. */
716
828
  committedTransactions(meta, states) {
@@ -789,18 +901,61 @@ export class AreaLedger {
789
901
  const states = this.currentAreaStates();
790
902
  const meta = this.openMeta().getState();
791
903
  const committed = this.committedTransactions(meta, states);
904
+ const identities = this.currentAreaIdentities();
905
+ const shared = this.options.sharedJournalRowsCacheKey === undefined
906
+ ? undefined
907
+ : sharedAreaJournalRows(this.options.sharedJournalRowsCacheKey, areaJournalRowsIdentity(identities));
792
908
  const areas = new Map(this.areaIds.map((areaId) => [areaId, materializeArea(states.get(areaId), committed)]));
909
+ const rows = shared?.rows ?? new JournalRowStore(countJournalRows([...areas].map(([areaId, journal]) => ({ areaId, journal }))), { trackChanges: false });
910
+ const rowOwners = shared?.owners ?? new Map();
911
+ const shadowRows = shared?.shadows ?? new Map();
912
+ if (!shared)
913
+ this.packAreaJournals([...areas].map(([areaId, journal]) => ({ areaId, journal })), rows, rowOwners, shadowRows);
914
+ for (const [areaId, state] of states) {
915
+ this.attachAreaJournal(state, areaId, rows, rowOwners, shadowRows);
916
+ areas.set(areaId, materializeArea(state, committed));
917
+ }
918
+ const combined = this.combine(rows);
793
919
  const cache = {
794
- aggregate: this.combine([...areas].map(([areaId, journal]) => ({ areaId, journal }))),
920
+ aggregate: combined.journal,
921
+ rows: combined.rows,
922
+ rowOwners,
923
+ shadowRows,
795
924
  areas,
796
- identities: new Map(),
925
+ identities,
797
926
  committed,
798
927
  revision: this.aggregateRevision,
799
928
  };
800
929
  this.updateCacheMetadata(cache, meta, committed);
801
- this.refreshCacheIdentities(cache);
802
930
  this.aggregateCache = cache;
803
931
  }
932
+ /**
933
+ * Associate this aggregate view with the one packed store for its durable
934
+ * area generations. Sessions can borrow it without rebuilding every row;
935
+ * an identity change creates a new packed store before the new aggregate is
936
+ * exposed, so a later async opener cannot observe stale rows.
937
+ */
938
+ cachedJournal(cache) {
939
+ const cacheKey = this.options.sharedJournalRowsCacheKey;
940
+ if (cacheKey !== undefined) {
941
+ const identity = areaJournalRowsIdentity(cache.identities);
942
+ const shared = sharedAreaJournalRows(cacheKey, identity);
943
+ if (shared && shared.rows !== cache.rows) {
944
+ cache.rows = shared.rows;
945
+ cache.rowOwners = shared.owners;
946
+ cache.shadowRows = shared.shadows;
947
+ cache.aggregate.files = createJournalRecordProxy(shared.rows);
948
+ for (const [areaId, journal] of cache.areas) {
949
+ journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, cache.rows, cache.rowOwners, cache.shadowRows));
950
+ }
951
+ }
952
+ else if (!shared) {
953
+ rememberSharedAreaJournalRows(cacheKey, identity, cache.rows, cache.rowOwners, cache.shadowRows);
954
+ }
955
+ }
956
+ rememberDecodedJournalRows(cache.aggregate, cache.rows);
957
+ return { journal: cache.aggregate, rows: cache.rows.rows, revision: cache.revision };
958
+ }
804
959
  /** Generation pointer + WAL size observation takes neither append lock nor readdir. */
805
960
  refreshChangedStores(cache) {
806
961
  const meta = this.openMeta();
@@ -850,46 +1005,35 @@ export class AreaLedger {
850
1005
  }
851
1006
  updateCachedArea(cache, areaId, journal) {
852
1007
  const previous = cache.areas.get(areaId);
853
- // Keys this refresh removes that ANOTHER area also holds. Their aggregate
854
- // value may belong to that other area, so it has to be put back after the
855
- // removal loop below — otherwise refreshing one side of a tolerated
856
- // duplicate silently destroys the winning side's value, or drops the key.
857
- const shared = new Set();
858
- for (const key of Object.keys(previous.files)) {
859
- if (this.sharesKeyWithAnotherArea(cache, key, areaId))
860
- shared.add(key);
861
- delete cache.aggregate.files[key];
862
- }
863
- cache.areas.set(areaId, journal);
864
- for (const [key, entry] of Object.entries(journal.files)) {
865
- // Same rule as `combine`, on the incremental path. A key another area also
866
- // holds may have just been deleted from the aggregate by the loop above,
867
- // so `shared` — not the aggregate — is what says a collision is possible.
868
- // The incumbent's area is not carried alongside the aggregate, so recover
869
- // it from the per-area journals, only ever on an actual collision.
870
- // `shared` covers a key the removal loop above just deleted from the
871
- // aggregate; the aggregate itself covers a key this area did not
872
- // previously hold but another area does — the two-real-area corruption a
873
- // full rebuild rejects, which the incremental path must reject too.
874
- if (shared.has(key) || cache.aggregate.files[key] !== undefined) {
875
- const incumbent = this.findCachedOwner(cache, key, areaId);
876
- if (incumbent !== undefined && this.resolveAreaCollision(key, incumbent, areaId) === "keep") {
877
- cache.aggregate.files[key] = shareRow(cache.areas.get(incumbent).files[key]);
878
- continue;
879
- }
1008
+ const overlay = journalRecordRows(journal.files);
1009
+ if (overlay instanceof JournalRowOverlay) {
1010
+ // Folding a prepared transaction puts a private overlay into the area
1011
+ // StateStore. A later direct append mutates that same object in place,
1012
+ // so identity alone cannot tell us whether the aggregate received its
1013
+ // rows. Replay just the overlay's changed paths into the packed owner,
1014
+ // then replace the overlay before it can become another base layer.
1015
+ for (const change of overlay.dirtyLogSnapshot()) {
1016
+ const entry = overlay.get(change.path);
1017
+ if (entry === undefined)
1018
+ this.deleteAreaRow(cache.rows, cache.rowOwners, cache.shadowRows, areaId, change.path);
1019
+ else
1020
+ this.writeAreaRow(cache.rows, cache.rowOwners, cache.shadowRows, areaId, change.path, entry);
880
1021
  }
881
- cache.aggregate.files[key] = shareRow(entry);
1022
+ journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, cache.rows, cache.rowOwners, cache.shadowRows));
1023
+ cache.areas.set(areaId, journal);
1024
+ return;
882
1025
  }
883
- // Keys this refresh dropped that another area still holds: the value the
884
- // removal loop deleted may have been that other area's, so put it back.
885
- for (const key of shared) {
886
- if (cache.aggregate.files[key] !== undefined)
887
- continue;
888
- const owner = this.findCachedOwner(cache, key, areaId);
889
- if (owner === undefined)
890
- continue;
891
- cache.aggregate.files[key] = shareRow(cache.areas.get(owner).files[key]);
1026
+ // WAL replay can apply a direct delta through this area's shared facade,
1027
+ // so `previous` already observes the new rows. Repacking it after deleting
1028
+ // from the same backing store would erase the source before it is copied.
1029
+ if (previous === journal)
1030
+ return;
1031
+ for (const key of rowsOf(previous).keys()) {
1032
+ this.deleteAreaRow(cache.rows, cache.rowOwners, cache.shadowRows, areaId, key);
892
1033
  }
1034
+ cache.shadowRows.delete(areaId);
1035
+ this.packAreaJournals([{ areaId, journal }], cache.rows, cache.rowOwners, cache.shadowRows);
1036
+ cache.areas.set(areaId, journal);
893
1037
  }
894
1038
  /**
895
1039
  * Whether an area other than `areaId` also holds `key`.
@@ -902,7 +1046,8 @@ export class AreaLedger {
902
1046
  */
903
1047
  sharesKeyWithAnotherArea(cache, key, areaId) {
904
1048
  if (areaId !== UNROUTED_AREA_ID) {
905
- return cache.areas.get(UNROUTED_AREA_ID)?.files[key] !== undefined;
1049
+ const overflow = cache.areas.get(UNROUTED_AREA_ID);
1050
+ return overflow === undefined ? false : rowsOf(overflow).has(key);
906
1051
  }
907
1052
  return this.findCachedOwner(cache, key, areaId) !== undefined;
908
1053
  }
@@ -917,7 +1062,7 @@ export class AreaLedger {
917
1062
  for (const [candidateId, candidate] of cache.areas) {
918
1063
  if (candidateId === excludeAreaId)
919
1064
  continue;
920
- if (candidate.files[key] === undefined)
1065
+ if (!rowsOf(candidate).has(key))
921
1066
  continue;
922
1067
  if (candidateId === UNROUTED_AREA_ID)
923
1068
  overflow = candidateId;
@@ -931,10 +1076,119 @@ export class AreaLedger {
931
1076
  cache.aggregate.pulls = structuredClone(meta.journal.pulls);
932
1077
  cache.committed = new Set(committed);
933
1078
  }
934
- refreshCacheIdentities(cache) {
935
- cache.identities.set("@meta", this.openMeta().getGenerationIdentity());
1079
+ replaceCacheIdentities(cache, identities) {
1080
+ // These identities describe the same state we just materialized. Reading
1081
+ // fresh identities after reconciliation could acknowledge a concurrent
1082
+ // publication whose rows never reached the packed cache.
1083
+ cache.identities = new Map(identities);
1084
+ }
1085
+ currentAreaIdentities() {
1086
+ const identities = new Map();
1087
+ identities.set("@meta", this.openMeta().getGenerationIdentity());
936
1088
  for (const areaId of this.areaIds)
937
- cache.identities.set(areaId, this.openArea(areaId).getGenerationIdentity());
1089
+ identities.set(areaId, this.openArea(areaId).getGenerationIdentity());
1090
+ return identities;
1091
+ }
1092
+ defaultAreaForKey(key) {
1093
+ const resolved = this.options.resolveArea(key);
1094
+ return resolved !== undefined && this.areaIds.includes(resolved)
1095
+ ? resolved
1096
+ : UNROUTED_AREA_ID;
1097
+ }
1098
+ rowOwner(key, owners) {
1099
+ return owners.get(key) ?? this.defaultAreaForKey(key);
1100
+ }
1101
+ areaJournalRows(areaId, rows, owners, shadows) {
1102
+ const areaShadows = shadows.get(areaId) ?? new Map();
1103
+ shadows.set(areaId, areaShadows);
1104
+ return new AreaJournalRows(rows, areaId, areaJournalRowIndex(rows), areaShadows, (key, entry) => this.writeAreaRow(rows, owners, shadows, areaId, key, entry), (key) => this.deleteAreaRow(rows, owners, shadows, areaId, key));
1105
+ }
1106
+ /** Replace an area record with its filtered facade over the shared packed rows. */
1107
+ attachAreaJournal(state, areaId, rows, owners, shadows) {
1108
+ state.journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, rows, owners, shadows));
1109
+ }
1110
+ /**
1111
+ * Move decoded area records into the one aggregate store, then discard each
1112
+ * record map in favour of an area-filtered facade over that store. A shared
1113
+ * aggregate bypasses this method entirely, so opening another bridge never
1114
+ * repacks every row merely to throw the new store away.
1115
+ */
1116
+ packAreaJournals(journals, rows, owners, shadows) {
1117
+ for (const { areaId, journal } of journals) {
1118
+ const files = journal.files;
1119
+ for (const key in files) {
1120
+ if (!Object.hasOwn(files, key))
1121
+ continue;
1122
+ const entry = files[key];
1123
+ this.writeAreaRow(rows, owners, shadows, areaId, key, entry);
1124
+ }
1125
+ journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, rows, owners, shadows));
1126
+ }
1127
+ }
1128
+ writeAreaRow(rows, owners, shadows, areaId, key, entry) {
1129
+ const areaShadows = shadows.get(areaId) ?? new Map();
1130
+ shadows.set(areaId, areaShadows);
1131
+ if (areaShadows.has(key)) {
1132
+ areaShadows.set(key, entry);
1133
+ return;
1134
+ }
1135
+ const incumbent = rows.has(key) ? this.rowOwner(key, owners) : undefined;
1136
+ if (incumbent !== undefined && incumbent !== areaId) {
1137
+ const resolution = this.resolveAreaCollision(key, incumbent, areaId);
1138
+ if (resolution === "keep") {
1139
+ areaShadows.set(key, entry);
1140
+ return;
1141
+ }
1142
+ const incumbentShadows = shadows.get(incumbent) ?? new Map();
1143
+ incumbentShadows.set(key, rows.get(key));
1144
+ shadows.set(incumbent, incumbentShadows);
1145
+ }
1146
+ rows.set(key, entry);
1147
+ const rowIndex = areaJournalRowIndex(rows);
1148
+ const rowId = rows.rowIdFor(key);
1149
+ if (incumbent !== undefined && incumbent !== areaId)
1150
+ rowIndex.delete(incumbent, rowId);
1151
+ rowIndex.add(areaId, rowId, incumbent !== undefined && incumbent !== areaId);
1152
+ const defaultOwner = this.defaultAreaForKey(key);
1153
+ if (defaultOwner === areaId)
1154
+ owners.delete(key);
1155
+ else
1156
+ owners.set(key, areaId);
1157
+ }
1158
+ deleteAreaRow(rows, owners, shadows, areaId, key) {
1159
+ const areaShadows = shadows.get(areaId);
1160
+ if (areaShadows?.delete(key))
1161
+ return true;
1162
+ if (this.rowOwner(key, owners) !== areaId)
1163
+ return false;
1164
+ const rowId = rows.rowIdFor(key);
1165
+ const deleted = rows.delete(key);
1166
+ if (!deleted)
1167
+ return false;
1168
+ areaJournalRowIndex(rows).delete(areaId, rowId);
1169
+ owners.delete(key);
1170
+ this.promoteShadowAfterOwnerDelete(rows, owners, shadows, key);
1171
+ return deleted;
1172
+ }
1173
+ /** A shadow remains durable when its aggregate owner is deleted, so make it visible. */
1174
+ promoteShadowAfterOwnerDelete(rows, owners, shadows, key) {
1175
+ let survivor;
1176
+ for (const [areaId, areaShadows] of shadows) {
1177
+ const entry = areaShadows.get(key);
1178
+ if (entry === undefined)
1179
+ continue;
1180
+ if (survivor === undefined) {
1181
+ survivor = { areaId, entry };
1182
+ continue;
1183
+ }
1184
+ if (this.resolveAreaCollision(key, survivor.areaId, areaId) === "replace") {
1185
+ survivor = { areaId, entry };
1186
+ }
1187
+ }
1188
+ if (survivor === undefined)
1189
+ return;
1190
+ shadows.get(survivor.areaId).delete(key);
1191
+ this.writeAreaRow(rows, owners, shadows, survivor.areaId, key, survivor.entry);
938
1192
  }
939
1193
  /** Refresh only identities that changed since this process's own cold-cache append. */
940
1194
  refreshChangedColdStores() {
@@ -960,6 +1214,34 @@ export class AreaLedger {
960
1214
  }
961
1215
  this.coldCacheIdentities = identities;
962
1216
  }
1217
+ /** Keep a warm packed aggregate current without rebuilding every unchanged row. */
1218
+ updateWarmAggregateAfterAppend(delta, changedAreaIds) {
1219
+ const cache = this.aggregateCache;
1220
+ if (!cache) {
1221
+ this.captureColdCacheIdentities(changedAreaIds, delta.metadata !== undefined);
1222
+ return;
1223
+ }
1224
+ const states = this.currentAreaStates();
1225
+ const identities = this.currentAreaIdentities();
1226
+ const meta = this.openMeta().getState();
1227
+ const committed = this.committedTransactions(meta, states);
1228
+ const areasToReconcile = new Set(changedAreaIds);
1229
+ for (const areaId of this.areaIds) {
1230
+ const previous = cache.identities.get(areaId);
1231
+ const current = identities.get(areaId);
1232
+ if (previous.generation !== current.generation ||
1233
+ previous.lastRecordSequence !== current.lastRecordSequence ||
1234
+ previous.walBytes !== current.walBytes) {
1235
+ areasToReconcile.add(areaId);
1236
+ }
1237
+ }
1238
+ for (const areaId of areasToReconcile) {
1239
+ this.updateCachedArea(cache, areaId, materializeArea(states.get(areaId), committed));
1240
+ }
1241
+ this.updateCacheMetadata(cache, meta, committed);
1242
+ this.replaceCacheIdentities(cache, identities);
1243
+ cache.revision = ++this.aggregateRevision;
1244
+ }
963
1245
  invalidateAggregateCache() {
964
1246
  if (this.aggregateCache)
965
1247
  this.coldCacheIdentities = new Map(this.aggregateCache.identities);
@@ -977,6 +1259,49 @@ function symmetricDifference(left, right) {
977
1259
  result.add(value);
978
1260
  return result;
979
1261
  }
1262
+ /** Count a source one key at a time so capacity planning never creates a key array. */
1263
+ function countJournalRows(journals) {
1264
+ let count = 0;
1265
+ for (const { journal } of journals) {
1266
+ const rows = decodedJournalRows(journal) ?? journalRecordRows(journal.files);
1267
+ if (rows?.size !== undefined) {
1268
+ count += rows.size;
1269
+ continue;
1270
+ }
1271
+ for (const key in journal.files) {
1272
+ if (Object.hasOwn(journal.files, key))
1273
+ count += 1;
1274
+ }
1275
+ }
1276
+ return count;
1277
+ }
1278
+ /** Keep the record proxy at the public edge while internal ledger code stays on lazy rows. */
1279
+ function rowsOf(journal) {
1280
+ const decoded = decodedJournalRows(journal);
1281
+ if (decoded)
1282
+ return decoded.rows;
1283
+ const proxied = journalRecordRows(journal.files);
1284
+ if (proxied)
1285
+ return proxied;
1286
+ const files = journal.files;
1287
+ return {
1288
+ get: (key) => files[key],
1289
+ has: (key) => Object.hasOwn(files, key),
1290
+ set: (key, entry) => { files[key] = entry; },
1291
+ delete: (key) => {
1292
+ if (!Object.hasOwn(files, key))
1293
+ return false;
1294
+ delete files[key];
1295
+ return true;
1296
+ },
1297
+ *keys() {
1298
+ for (const key in files) {
1299
+ if (Object.hasOwn(files, key))
1300
+ yield key;
1301
+ }
1302
+ },
1303
+ };
1304
+ }
980
1305
  function reduceAreaRecord(state, record) {
981
1306
  state.direct ??= [];
982
1307
  state.prepared ??= {};
@@ -1051,10 +1376,10 @@ function isFinalizeRecord(value) {
1051
1376
  }
1052
1377
  /**
1053
1378
  * A row is an immutable value. Every projection of a journal — the raw area
1054
- * state, the materialized per-area journal, the combined aggregate, and each
1055
- * `JournalStore` session opened on top — holds the SAME row object, and the
1056
- * object is frozen so an in-place mutation throws instead of silently
1057
- * rewriting every other projection at once.
1379
+ * state and the materialized per-area journal hold the same frozen row object.
1380
+ * The combined aggregate owns a packed representation; its record facade
1381
+ * materializes frozen rows on demand so an in-place mutation cannot silently
1382
+ * rewrite another projection.
1058
1383
  *
1059
1384
  * This is a memory bound, not a nicety. A personal vault on this fleet has
1060
1385
  * ~690k rows; deep-copying them per projection put five or six full copies in
@@ -1081,20 +1406,28 @@ function freezeDeep(value) {
1081
1406
  Object.freeze(value);
1082
1407
  }
1083
1408
  function materializeArea(state, committed) {
1084
- // Shallow: a new key map over the SAME row objects as the area state.
1409
+ const events = visibleEvents(state, committed);
1410
+ // A settled area is already a stable private projection. Returning it as-is
1411
+ // avoids the former whole-record clone that lived beside the packed aggregate.
1412
+ if (events.length === 0)
1413
+ return state.journal;
1414
+ // Pending direct/prepared records must remain isolated from their durable
1415
+ // base until a fold makes them visible. Keep only those changed keys in an
1416
+ // overlay instead of cloning every base row just to apply a small tail.
1417
+ const overlay = new JournalRowOverlay(rowsOf(state.journal));
1085
1418
  const journal = {
1086
1419
  ...state.journal,
1087
- files: { ...state.journal.files },
1420
+ files: createJournalRecordProxy(overlay),
1088
1421
  pulls: structuredClone(state.journal.pulls ?? []),
1089
1422
  };
1090
- for (const event of visibleEvents(state, committed))
1423
+ for (const event of events)
1091
1424
  applyDelta(journal, event.delta);
1092
1425
  return journal;
1093
1426
  }
1094
1427
  /** Apply only events relevant to one key; unlike materializeArea this never clones a whole journal. */
1095
1428
  function materializeRow(state, committed, key, events = visibleEvents(state, committed)) {
1096
1429
  testHooks.onMaterializeRow?.(key);
1097
- let row = state.journal.files[key];
1430
+ let row = rowsOf(state.journal).get(key);
1098
1431
  for (const event of events) {
1099
1432
  if (event.delta.deletes.includes(key))
1100
1433
  row = undefined;
@@ -1115,7 +1448,7 @@ function visibleEvents(state, committed) {
1115
1448
  }
1116
1449
  /** Key-only union used by scoped directory reads; values stay un-cloned until selected. */
1117
1450
  function visibleRowKeys(state, events) {
1118
- const keys = new Set(Object.keys(state.journal.files));
1451
+ const keys = new Set(rowsOf(state.journal).keys());
1119
1452
  for (const event of events) {
1120
1453
  for (const key of event.delta.deletes)
1121
1454
  keys.add(key);
@@ -1125,16 +1458,33 @@ function visibleRowKeys(state, events) {
1125
1458
  return keys;
1126
1459
  }
1127
1460
  function applyDelta(journal, delta) {
1461
+ const rows = rowsOf(journal);
1128
1462
  for (const key of delta.deletes)
1129
- delete journal.files[key];
1463
+ rows.delete(key);
1130
1464
  // Delta rows are already this ledger's own copies (see `partition` and the
1131
1465
  // WAL fold); adopt the objects, do not copy them again.
1132
1466
  for (const [key, entry] of Object.entries(delta.upserts))
1133
- journal.files[key] = shareRow(entry);
1467
+ rows.set(key, shareRow(entry));
1134
1468
  }
1135
1469
  function emptyJournal() {
1136
1470
  return { version: "2", lastSync: "", files: {}, pulls: [] };
1137
1471
  }
1472
+ /**
1473
+ * Public aggregate readers receive a mutable record facade without copying all
1474
+ * rows. Replacements and deletes stay private to this view; rows returned from
1475
+ * the immutable aggregate remain frozen, so in-place mutation cannot bypass a
1476
+ * later delta writer.
1477
+ */
1478
+ function detachedJournalOverlay(journal, source) {
1479
+ const rows = source ?? rowsOf(journal);
1480
+ const result = {
1481
+ ...journal,
1482
+ files: createJournalRecordProxy(new JournalRowOverlay(rows)),
1483
+ };
1484
+ if (Object.hasOwn(journal, "pulls"))
1485
+ result.pulls = structuredClone(journal.pulls);
1486
+ return result;
1487
+ }
1138
1488
  function capPullRecords(pulls) {
1139
1489
  const journal = { version: "2", lastSync: "", files: {}, pulls: structuredClone(pulls) };
1140
1490
  trimPullRecords(journal);