@indigoai-us/hq-cloud 6.16.39 → 6.16.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-heap-telemetry.d.ts +145 -9
- package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.js +178 -11
- package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.test.js +511 -2
- package/dist/bin/sync-runner-heap-telemetry.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +20 -10
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +21 -0
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +16 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +27 -28
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +0 -49
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +8 -8
- package/dist/cli/sync.js.map +1 -1
- package/dist/journal-area-ledger-read-boundary.test.js +12 -46
- package/dist/journal-area-ledger-read-boundary.test.js.map +1 -1
- package/dist/journal-record-proxy.d.ts +2 -12
- package/dist/journal-record-proxy.d.ts.map +1 -1
- package/dist/journal-record-proxy.js +1 -8
- package/dist/journal-record-proxy.js.map +1 -1
- package/dist/journal-row-store.d.ts +0 -64
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +3 -129
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +1 -3
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-rows.d.ts +1 -17
- package/dist/journal-snapshot-rows.d.ts.map +1 -1
- package/dist/journal-snapshot-rows.js +0 -18
- package/dist/journal-snapshot-rows.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts +2 -3
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +3 -9
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-snapshot-v4.test.js +0 -1
- package/dist/journal-snapshot-v4.test.js.map +1 -1
- package/dist/journal.d.ts +5 -23
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +189 -309
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +1 -58
- package/dist/journal.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +4 -6
- package/dist/personal-vault.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +8 -9
- package/dist/remote-pull.js.map +1 -1
- package/dist/scope-shrink.js +2 -2
- package/dist/scope-shrink.js.map +1 -1
- package/dist/session-log-retention.d.ts.map +1 -1
- package/dist/session-log-retention.js +2 -2
- package/dist/session-log-retention.js.map +1 -1
- package/dist/sync/area-ledger-migration.d.ts +6 -13
- package/dist/sync/area-ledger-migration.d.ts.map +1 -1
- package/dist/sync/area-ledger-migration.js +19 -45
- package/dist/sync/area-ledger-migration.js.map +1 -1
- package/dist/sync/area-ledger-migration.test.js +0 -23
- package/dist/sync/area-ledger-migration.test.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +1 -31
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +88 -385
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +10 -123
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/types.d.ts +0 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/packed-journal-phase2.test.d.ts +0 -2
- package/dist/packed-journal-phase2.test.d.ts.map +0 -1
- package/dist/packed-journal-phase2.test.js +0 -273
- package/dist/packed-journal-phase2.test.js.map +0 -1
package/dist/sync/area-ledger.js
CHANGED
|
@@ -7,9 +7,6 @@
|
|
|
7
7
|
* not understand a layout continue to use their existing authority.
|
|
8
8
|
*/
|
|
9
9
|
import * as crypto from "node:crypto";
|
|
10
|
-
import { JournalRowOverlay, JournalRowStore } from "../journal-row-store.js";
|
|
11
|
-
import { createJournalRecordProxy, journalRecordRows } from "../journal-record-proxy.js";
|
|
12
|
-
import { decodedJournalRows, rememberDecodedJournalRows, rememberSharedAreaJournalRows, sharedAreaJournalRows, } from "../journal-snapshot-rows.js";
|
|
13
10
|
import { trimPullRecords } from "../journal.js";
|
|
14
11
|
import { StateStore } from "./state-store.js";
|
|
15
12
|
const DIRECT_DELTA = 1;
|
|
@@ -40,54 +37,6 @@ const META_FINALIZE = 2;
|
|
|
40
37
|
export const UNROUTED_AREA_ID = "root:@unrouted";
|
|
41
38
|
/** Nothing is held anywhere — for a ledger with no state on disk yet. */
|
|
42
39
|
const NO_ROWS = { holds: () => false };
|
|
43
|
-
function areaJournalRowsIdentity(identities) {
|
|
44
|
-
return [...identities]
|
|
45
|
-
.filter(([areaId]) => areaId !== "@meta")
|
|
46
|
-
.sort(([left], [right]) => left.localeCompare(right))
|
|
47
|
-
.map(([areaId, identity]) => `${areaId}:${identity.generation}:${identity.lastRecordSequence}:${identity.walBytes}`)
|
|
48
|
-
.join("\u0000");
|
|
49
|
-
}
|
|
50
|
-
/** A state-store-facing slice over the one packed aggregate. */
|
|
51
|
-
class AreaJournalRows {
|
|
52
|
-
rows;
|
|
53
|
-
areaId;
|
|
54
|
-
defaultAreaForKey;
|
|
55
|
-
exceptionalOwners;
|
|
56
|
-
shadows;
|
|
57
|
-
write;
|
|
58
|
-
erase;
|
|
59
|
-
constructor(rows, areaId, defaultAreaForKey, exceptionalOwners, shadows, write, erase) {
|
|
60
|
-
this.rows = rows;
|
|
61
|
-
this.areaId = areaId;
|
|
62
|
-
this.defaultAreaForKey = defaultAreaForKey;
|
|
63
|
-
this.exceptionalOwners = exceptionalOwners;
|
|
64
|
-
this.shadows = shadows;
|
|
65
|
-
this.write = write;
|
|
66
|
-
this.erase = erase;
|
|
67
|
-
}
|
|
68
|
-
get(path) {
|
|
69
|
-
return this.shadows.get(path) ?? (this.owns(path) ? this.rows.get(path) : undefined);
|
|
70
|
-
}
|
|
71
|
-
has(path) {
|
|
72
|
-
return this.shadows.has(path) || (this.owns(path) && this.rows.has(path));
|
|
73
|
-
}
|
|
74
|
-
set(path, entry) {
|
|
75
|
-
this.write(path, entry);
|
|
76
|
-
}
|
|
77
|
-
delete(path) {
|
|
78
|
-
return this.erase(path);
|
|
79
|
-
}
|
|
80
|
-
*keys() {
|
|
81
|
-
yield* this.shadows.keys();
|
|
82
|
-
for (const key of this.rows.keys()) {
|
|
83
|
-
if (!this.shadows.has(key) && this.owns(key))
|
|
84
|
-
yield key;
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
owns(path) {
|
|
88
|
-
return (this.exceptionalOwners.get(path) ?? this.defaultAreaForKey(path)) === this.areaId;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
40
|
let testHooks = {};
|
|
92
41
|
/** Test-only crash seam: data prepares are durable before a meta watermark. */
|
|
93
42
|
export function setAreaLedgerTestHooksForTest(hooks) {
|
|
@@ -170,22 +119,9 @@ export class AreaLedger {
|
|
|
170
119
|
}
|
|
171
120
|
}
|
|
172
121
|
const committed = this.committedTransactions(meta.getState(), states);
|
|
173
|
-
//
|
|
174
|
-
//
|
|
175
|
-
|
|
176
|
-
const journals = [...selected].map((areaId) => ({
|
|
177
|
-
areaId,
|
|
178
|
-
// `materializeArea` returns the live StateStore journal when there are
|
|
179
|
-
// no pending records. Packing a scoped result must replace only this
|
|
180
|
-
// shell's facade, never the cached area's live journal facade.
|
|
181
|
-
journal: { ...materializeArea(states.get(areaId), committed) },
|
|
182
|
-
}));
|
|
183
|
-
const rows = new JournalRowStore(countJournalRows(journals), { trackChanges: false });
|
|
184
|
-
const owners = new Map();
|
|
185
|
-
const shadows = new Map();
|
|
186
|
-
this.packAreaJournals(journals, rows, owners, shadows);
|
|
187
|
-
const combined = this.combine(rows);
|
|
188
|
-
const aggregate = detachedJournalOverlay(combined.journal, combined.rows.rows);
|
|
122
|
+
// `combine` shares row objects with the area states; this one-off reader
|
|
123
|
+
// hands out a detached copy, so a caller may mutate what it gets back.
|
|
124
|
+
const aggregate = structuredClone(this.combine([...selected].map((areaId) => ({ areaId, journal: materializeArea(states.get(areaId), committed) }))));
|
|
189
125
|
aggregate.lastSync = meta.getState().journal.lastSync;
|
|
190
126
|
aggregate.pulls = structuredClone(meta.getState().journal.pulls);
|
|
191
127
|
return aggregate;
|
|
@@ -319,8 +255,7 @@ export class AreaLedger {
|
|
|
319
255
|
}
|
|
320
256
|
/** Aggregate read. It is the parity view used by legacy SyncJournal callers. */
|
|
321
257
|
read() {
|
|
322
|
-
|
|
323
|
-
return detachedJournalOverlay(cached.journal, cached.rows);
|
|
258
|
+
return structuredClone(this.readCached().journal);
|
|
324
259
|
}
|
|
325
260
|
/**
|
|
326
261
|
* Internal aggregate view for long-lived keyed sessions. The returned journal
|
|
@@ -340,22 +275,21 @@ export class AreaLedger {
|
|
|
340
275
|
this.primeAggregateCache();
|
|
341
276
|
}
|
|
342
277
|
this.aggregateCache.revision = ++this.aggregateRevision;
|
|
343
|
-
return this.
|
|
278
|
+
return { journal: this.aggregateCache.aggregate, revision: this.aggregateCache.revision };
|
|
344
279
|
}
|
|
345
280
|
const cache = this.aggregateCache;
|
|
346
281
|
const { metaChanged, areas } = this.refreshChangedStores(cache);
|
|
347
282
|
if (!metaChanged && areas.size === 0)
|
|
348
|
-
return
|
|
283
|
+
return { journal: cache.aggregate, revision: cache.revision };
|
|
349
284
|
const meta = this.openMeta().getState();
|
|
350
285
|
const states = this.currentAreaStates();
|
|
351
|
-
const identities = this.currentAreaIdentities();
|
|
352
286
|
if (this.hasPendingTransactionEvidence(meta, states)) {
|
|
353
287
|
// A different process published prepare/watermark evidence. Recovery is
|
|
354
288
|
// exceptional and bounded to that evidence; the steady read path above
|
|
355
289
|
// remains lock-free and write-free.
|
|
356
290
|
this.recoverPendingTransactions();
|
|
357
291
|
this.primeAggregateCache();
|
|
358
|
-
return this.
|
|
292
|
+
return { journal: this.aggregateCache.aggregate, revision: this.aggregateCache.revision };
|
|
359
293
|
}
|
|
360
294
|
const committed = this.committedTransactions(meta, states);
|
|
361
295
|
for (const transactionId of symmetricDifference(cache.committed, committed)) {
|
|
@@ -368,9 +302,9 @@ export class AreaLedger {
|
|
|
368
302
|
this.updateCachedArea(cache, areaId, materializeArea(states.get(areaId), committed));
|
|
369
303
|
}
|
|
370
304
|
this.updateCacheMetadata(cache, meta, committed);
|
|
371
|
-
this.
|
|
305
|
+
this.refreshCacheIdentities(cache);
|
|
372
306
|
cache.revision = ++this.aggregateRevision;
|
|
373
|
-
return
|
|
307
|
+
return { journal: cache.aggregate, revision: cache.revision };
|
|
374
308
|
}
|
|
375
309
|
/**
|
|
376
310
|
* Commit a partitioned journal delta. A metadata-only update still has a
|
|
@@ -378,25 +312,21 @@ export class AreaLedger {
|
|
|
378
312
|
* protocol so multiple areas never expose a half transaction to `read()`.
|
|
379
313
|
*/
|
|
380
314
|
append(delta) {
|
|
381
|
-
|
|
382
|
-
// caller's delta is appended. That preserves unrelated peer rows while
|
|
383
|
-
// letting the shared packed store stay resident after this write.
|
|
384
|
-
if (this.aggregateCache)
|
|
385
|
-
this.readCached();
|
|
315
|
+
this.invalidateAggregateCache();
|
|
386
316
|
const partitioned = this.partition(delta);
|
|
387
317
|
const metadata = cloneMetadata(delta.metadata);
|
|
388
318
|
if (partitioned.length === 0) {
|
|
389
319
|
if (metadata) {
|
|
390
320
|
this.appendMeta({ transactionId: crypto.randomUUID(), areas: [], metadata });
|
|
391
321
|
this.finalizePublishedTransactions();
|
|
392
|
-
this.
|
|
322
|
+
this.captureColdCacheIdentities([], true);
|
|
393
323
|
}
|
|
394
324
|
return;
|
|
395
325
|
}
|
|
396
326
|
if (partitioned.length === 1 && !metadata) {
|
|
397
327
|
const [entry] = partitioned;
|
|
398
328
|
this.openArea(entry.areaId).append(DIRECT_DELTA, entry.delta);
|
|
399
|
-
this.
|
|
329
|
+
this.captureColdCacheIdentities([entry.areaId], false);
|
|
400
330
|
return;
|
|
401
331
|
}
|
|
402
332
|
const transactionId = crypto.randomUUID();
|
|
@@ -420,7 +350,7 @@ export class AreaLedger {
|
|
|
420
350
|
this.appendMeta({ transactionId, areas: areaIds, ...(metadata ? { metadata } : {}) });
|
|
421
351
|
testHooks.afterWatermarkBeforeFinalize?.();
|
|
422
352
|
this.finalizePublishedTransactions();
|
|
423
|
-
this.
|
|
353
|
+
this.captureColdCacheIdentities(areaIds, true);
|
|
424
354
|
}
|
|
425
355
|
/**
|
|
426
356
|
* Finish a transaction whose process died after durable area prepares but
|
|
@@ -540,12 +470,10 @@ export class AreaLedger {
|
|
|
540
470
|
if (!source)
|
|
541
471
|
return emptyJournal();
|
|
542
472
|
const files = {};
|
|
543
|
-
const
|
|
544
|
-
for (const key of rows.keys()) {
|
|
473
|
+
for (const [key, entry] of Object.entries(source.files)) {
|
|
545
474
|
// Nothing is on disk yet, so there is nothing to respect but the layout:
|
|
546
475
|
// this is pure routing by construction.
|
|
547
|
-
|
|
548
|
-
if (entry !== undefined && this.resolveArea(key, NO_ROWS) === areaId)
|
|
476
|
+
if (this.resolveArea(key, NO_ROWS) === areaId)
|
|
549
477
|
files[key] = structuredClone(entry);
|
|
550
478
|
}
|
|
551
479
|
return { version: "2", lastSync: "", files, pulls: [] };
|
|
@@ -635,7 +563,7 @@ export class AreaLedger {
|
|
|
635
563
|
identity: store.getGenerationIdentity(),
|
|
636
564
|
metaIdentity: this.openMeta().getGenerationIdentity(),
|
|
637
565
|
pending: Object.keys(state.prepared).some((transactionId) => !committed.has(transactionId)),
|
|
638
|
-
keys: new Set(
|
|
566
|
+
keys: new Set(Object.keys(materializeArea(state, committed).files)),
|
|
639
567
|
};
|
|
640
568
|
}
|
|
641
569
|
return this.overflowRows.keys.has(key);
|
|
@@ -662,8 +590,7 @@ export class AreaLedger {
|
|
|
662
590
|
// when a real area has just taken it. This is a generation stat, not
|
|
663
591
|
// a read, so an unchanged area still costs nothing.
|
|
664
592
|
if (identity !== undefined && !this.openArea(areaId).hasChangedOnDisk(identity)) {
|
|
665
|
-
|
|
666
|
-
return journal === undefined ? false : rowsOf(journal).has(key);
|
|
593
|
+
return cache.areas.get(areaId)?.files[key] !== undefined;
|
|
667
594
|
}
|
|
668
595
|
return fromDisk.holds(areaId, key);
|
|
669
596
|
},
|
|
@@ -769,14 +696,21 @@ export class AreaLedger {
|
|
|
769
696
|
return "replace";
|
|
770
697
|
throw new Error(`area-ledger: key ${JSON.stringify(key)} is present in multiple areas`);
|
|
771
698
|
}
|
|
772
|
-
combine(
|
|
773
|
-
const aggregate =
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
699
|
+
combine(journals) {
|
|
700
|
+
const aggregate = emptyJournal();
|
|
701
|
+
const owner = new Map();
|
|
702
|
+
for (const { areaId, journal } of journals) {
|
|
703
|
+
for (const [key, entry] of Object.entries(journal.files)) {
|
|
704
|
+
const incumbent = owner.get(key);
|
|
705
|
+
if (incumbent !== undefined && this.resolveAreaCollision(key, incumbent, areaId) === "keep")
|
|
706
|
+
continue;
|
|
707
|
+
// The aggregate holds the SAME row object as the area projection and
|
|
708
|
+
// the raw area state — never a copy. See `shareRow`.
|
|
709
|
+
aggregate.files[key] = shareRow(entry);
|
|
710
|
+
owner.set(key, areaId);
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
return aggregate;
|
|
780
714
|
}
|
|
781
715
|
/** A watermark publishes only when every named area has prepared or folded it. */
|
|
782
716
|
committedTransactions(meta, states) {
|
|
@@ -855,61 +789,18 @@ export class AreaLedger {
|
|
|
855
789
|
const states = this.currentAreaStates();
|
|
856
790
|
const meta = this.openMeta().getState();
|
|
857
791
|
const committed = this.committedTransactions(meta, states);
|
|
858
|
-
const identities = this.currentAreaIdentities();
|
|
859
|
-
const shared = this.options.sharedJournalRowsCacheKey === undefined
|
|
860
|
-
? undefined
|
|
861
|
-
: sharedAreaJournalRows(this.options.sharedJournalRowsCacheKey, areaJournalRowsIdentity(identities));
|
|
862
792
|
const areas = new Map(this.areaIds.map((areaId) => [areaId, materializeArea(states.get(areaId), committed)]));
|
|
863
|
-
const rows = shared?.rows ?? new JournalRowStore(countJournalRows([...areas].map(([areaId, journal]) => ({ areaId, journal }))), { trackChanges: false });
|
|
864
|
-
const rowOwners = shared?.owners ?? new Map();
|
|
865
|
-
const shadowRows = shared?.shadows ?? new Map();
|
|
866
|
-
if (!shared)
|
|
867
|
-
this.packAreaJournals([...areas].map(([areaId, journal]) => ({ areaId, journal })), rows, rowOwners, shadowRows);
|
|
868
|
-
for (const [areaId, state] of states) {
|
|
869
|
-
this.attachAreaJournal(state, areaId, rows, rowOwners, shadowRows);
|
|
870
|
-
areas.set(areaId, materializeArea(state, committed));
|
|
871
|
-
}
|
|
872
|
-
const combined = this.combine(rows);
|
|
873
793
|
const cache = {
|
|
874
|
-
aggregate:
|
|
875
|
-
rows: combined.rows,
|
|
876
|
-
rowOwners,
|
|
877
|
-
shadowRows,
|
|
794
|
+
aggregate: this.combine([...areas].map(([areaId, journal]) => ({ areaId, journal }))),
|
|
878
795
|
areas,
|
|
879
|
-
identities,
|
|
796
|
+
identities: new Map(),
|
|
880
797
|
committed,
|
|
881
798
|
revision: this.aggregateRevision,
|
|
882
799
|
};
|
|
883
800
|
this.updateCacheMetadata(cache, meta, committed);
|
|
801
|
+
this.refreshCacheIdentities(cache);
|
|
884
802
|
this.aggregateCache = cache;
|
|
885
803
|
}
|
|
886
|
-
/**
|
|
887
|
-
* Associate this aggregate view with the one packed store for its durable
|
|
888
|
-
* area generations. Sessions can borrow it without rebuilding every row;
|
|
889
|
-
* an identity change creates a new packed store before the new aggregate is
|
|
890
|
-
* exposed, so a later async opener cannot observe stale rows.
|
|
891
|
-
*/
|
|
892
|
-
cachedJournal(cache) {
|
|
893
|
-
const cacheKey = this.options.sharedJournalRowsCacheKey;
|
|
894
|
-
if (cacheKey !== undefined) {
|
|
895
|
-
const identity = areaJournalRowsIdentity(cache.identities);
|
|
896
|
-
const shared = sharedAreaJournalRows(cacheKey, identity);
|
|
897
|
-
if (shared && shared.rows !== cache.rows) {
|
|
898
|
-
cache.rows = shared.rows;
|
|
899
|
-
cache.rowOwners = shared.owners;
|
|
900
|
-
cache.shadowRows = shared.shadows;
|
|
901
|
-
cache.aggregate.files = createJournalRecordProxy(shared.rows);
|
|
902
|
-
for (const [areaId, journal] of cache.areas) {
|
|
903
|
-
journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, cache.rows, cache.rowOwners, cache.shadowRows));
|
|
904
|
-
}
|
|
905
|
-
}
|
|
906
|
-
else if (!shared) {
|
|
907
|
-
rememberSharedAreaJournalRows(cacheKey, identity, cache.rows, cache.rowOwners, cache.shadowRows);
|
|
908
|
-
}
|
|
909
|
-
}
|
|
910
|
-
rememberDecodedJournalRows(cache.aggregate, cache.rows);
|
|
911
|
-
return { journal: cache.aggregate, rows: cache.rows.rows, revision: cache.revision };
|
|
912
|
-
}
|
|
913
804
|
/** Generation pointer + WAL size observation takes neither append lock nor readdir. */
|
|
914
805
|
refreshChangedStores(cache) {
|
|
915
806
|
const meta = this.openMeta();
|
|
@@ -959,35 +850,46 @@ export class AreaLedger {
|
|
|
959
850
|
}
|
|
960
851
|
updateCachedArea(cache, areaId, journal) {
|
|
961
852
|
const previous = cache.areas.get(areaId);
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
853
|
+
// Keys this refresh removes that ANOTHER area also holds. Their aggregate
|
|
854
|
+
// value may belong to that other area, so it has to be put back after the
|
|
855
|
+
// removal loop below — otherwise refreshing one side of a tolerated
|
|
856
|
+
// duplicate silently destroys the winning side's value, or drops the key.
|
|
857
|
+
const shared = new Set();
|
|
858
|
+
for (const key of Object.keys(previous.files)) {
|
|
859
|
+
if (this.sharesKeyWithAnotherArea(cache, key, areaId))
|
|
860
|
+
shared.add(key);
|
|
861
|
+
delete cache.aggregate.files[key];
|
|
862
|
+
}
|
|
863
|
+
cache.areas.set(areaId, journal);
|
|
864
|
+
for (const [key, entry] of Object.entries(journal.files)) {
|
|
865
|
+
// Same rule as `combine`, on the incremental path. A key another area also
|
|
866
|
+
// holds may have just been deleted from the aggregate by the loop above,
|
|
867
|
+
// so `shared` — not the aggregate — is what says a collision is possible.
|
|
868
|
+
// The incumbent's area is not carried alongside the aggregate, so recover
|
|
869
|
+
// it from the per-area journals, only ever on an actual collision.
|
|
870
|
+
// `shared` covers a key the removal loop above just deleted from the
|
|
871
|
+
// aggregate; the aggregate itself covers a key this area did not
|
|
872
|
+
// previously hold but another area does — the two-real-area corruption a
|
|
873
|
+
// full rebuild rejects, which the incremental path must reject too.
|
|
874
|
+
if (shared.has(key) || cache.aggregate.files[key] !== undefined) {
|
|
875
|
+
const incumbent = this.findCachedOwner(cache, key, areaId);
|
|
876
|
+
if (incumbent !== undefined && this.resolveAreaCollision(key, incumbent, areaId) === "keep") {
|
|
877
|
+
cache.aggregate.files[key] = shareRow(cache.areas.get(incumbent).files[key]);
|
|
878
|
+
continue;
|
|
879
|
+
}
|
|
975
880
|
}
|
|
976
|
-
|
|
977
|
-
cache.areas.set(areaId, journal);
|
|
978
|
-
return;
|
|
881
|
+
cache.aggregate.files[key] = shareRow(entry);
|
|
979
882
|
}
|
|
980
|
-
//
|
|
981
|
-
//
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
883
|
+
// Keys this refresh dropped that another area still holds: the value the
|
|
884
|
+
// removal loop deleted may have been that other area's, so put it back.
|
|
885
|
+
for (const key of shared) {
|
|
886
|
+
if (cache.aggregate.files[key] !== undefined)
|
|
887
|
+
continue;
|
|
888
|
+
const owner = this.findCachedOwner(cache, key, areaId);
|
|
889
|
+
if (owner === undefined)
|
|
890
|
+
continue;
|
|
891
|
+
cache.aggregate.files[key] = shareRow(cache.areas.get(owner).files[key]);
|
|
987
892
|
}
|
|
988
|
-
cache.shadowRows.delete(areaId);
|
|
989
|
-
this.packAreaJournals([{ areaId, journal }], cache.rows, cache.rowOwners, cache.shadowRows);
|
|
990
|
-
cache.areas.set(areaId, journal);
|
|
991
893
|
}
|
|
992
894
|
/**
|
|
993
895
|
* Whether an area other than `areaId` also holds `key`.
|
|
@@ -1000,8 +902,7 @@ export class AreaLedger {
|
|
|
1000
902
|
*/
|
|
1001
903
|
sharesKeyWithAnotherArea(cache, key, areaId) {
|
|
1002
904
|
if (areaId !== UNROUTED_AREA_ID) {
|
|
1003
|
-
|
|
1004
|
-
return overflow === undefined ? false : rowsOf(overflow).has(key);
|
|
905
|
+
return cache.areas.get(UNROUTED_AREA_ID)?.files[key] !== undefined;
|
|
1005
906
|
}
|
|
1006
907
|
return this.findCachedOwner(cache, key, areaId) !== undefined;
|
|
1007
908
|
}
|
|
@@ -1016,7 +917,7 @@ export class AreaLedger {
|
|
|
1016
917
|
for (const [candidateId, candidate] of cache.areas) {
|
|
1017
918
|
if (candidateId === excludeAreaId)
|
|
1018
919
|
continue;
|
|
1019
|
-
if (
|
|
920
|
+
if (candidate.files[key] === undefined)
|
|
1020
921
|
continue;
|
|
1021
922
|
if (candidateId === UNROUTED_AREA_ID)
|
|
1022
923
|
overflow = candidateId;
|
|
@@ -1030,112 +931,10 @@ export class AreaLedger {
|
|
|
1030
931
|
cache.aggregate.pulls = structuredClone(meta.journal.pulls);
|
|
1031
932
|
cache.committed = new Set(committed);
|
|
1032
933
|
}
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
// fresh identities after reconciliation could acknowledge a concurrent
|
|
1036
|
-
// publication whose rows never reached the packed cache.
|
|
1037
|
-
cache.identities = new Map(identities);
|
|
1038
|
-
}
|
|
1039
|
-
currentAreaIdentities() {
|
|
1040
|
-
const identities = new Map();
|
|
1041
|
-
identities.set("@meta", this.openMeta().getGenerationIdentity());
|
|
934
|
+
refreshCacheIdentities(cache) {
|
|
935
|
+
cache.identities.set("@meta", this.openMeta().getGenerationIdentity());
|
|
1042
936
|
for (const areaId of this.areaIds)
|
|
1043
|
-
identities.set(areaId, this.openArea(areaId).getGenerationIdentity());
|
|
1044
|
-
return identities;
|
|
1045
|
-
}
|
|
1046
|
-
defaultAreaForKey(key) {
|
|
1047
|
-
const resolved = this.options.resolveArea(key);
|
|
1048
|
-
return resolved !== undefined && this.areaIds.includes(resolved)
|
|
1049
|
-
? resolved
|
|
1050
|
-
: UNROUTED_AREA_ID;
|
|
1051
|
-
}
|
|
1052
|
-
rowOwner(key, owners) {
|
|
1053
|
-
return owners.get(key) ?? this.defaultAreaForKey(key);
|
|
1054
|
-
}
|
|
1055
|
-
areaJournalRows(areaId, rows, owners, shadows) {
|
|
1056
|
-
const areaShadows = shadows.get(areaId) ?? new Map();
|
|
1057
|
-
shadows.set(areaId, areaShadows);
|
|
1058
|
-
return new AreaJournalRows(rows, areaId, (key) => this.defaultAreaForKey(key), owners, areaShadows, (key, entry) => this.writeAreaRow(rows, owners, shadows, areaId, key, entry), (key) => this.deleteAreaRow(rows, owners, shadows, areaId, key));
|
|
1059
|
-
}
|
|
1060
|
-
/** Replace an area record with its filtered facade over the shared packed rows. */
|
|
1061
|
-
attachAreaJournal(state, areaId, rows, owners, shadows) {
|
|
1062
|
-
state.journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, rows, owners, shadows));
|
|
1063
|
-
}
|
|
1064
|
-
/**
|
|
1065
|
-
* Move decoded area records into the one aggregate store, then discard each
|
|
1066
|
-
* record map in favour of an area-filtered facade over that store. A shared
|
|
1067
|
-
* aggregate bypasses this method entirely, so opening another bridge never
|
|
1068
|
-
* repacks every row merely to throw the new store away.
|
|
1069
|
-
*/
|
|
1070
|
-
packAreaJournals(journals, rows, owners, shadows) {
|
|
1071
|
-
for (const { areaId, journal } of journals) {
|
|
1072
|
-
const files = journal.files;
|
|
1073
|
-
for (const key in files) {
|
|
1074
|
-
if (!Object.hasOwn(files, key))
|
|
1075
|
-
continue;
|
|
1076
|
-
const entry = files[key];
|
|
1077
|
-
this.writeAreaRow(rows, owners, shadows, areaId, key, entry);
|
|
1078
|
-
}
|
|
1079
|
-
journal.files = createJournalRecordProxy(this.areaJournalRows(areaId, rows, owners, shadows));
|
|
1080
|
-
}
|
|
1081
|
-
}
|
|
1082
|
-
writeAreaRow(rows, owners, shadows, areaId, key, entry) {
|
|
1083
|
-
const areaShadows = shadows.get(areaId) ?? new Map();
|
|
1084
|
-
shadows.set(areaId, areaShadows);
|
|
1085
|
-
if (areaShadows.has(key)) {
|
|
1086
|
-
areaShadows.set(key, entry);
|
|
1087
|
-
return;
|
|
1088
|
-
}
|
|
1089
|
-
const incumbent = rows.has(key) ? this.rowOwner(key, owners) : undefined;
|
|
1090
|
-
if (incumbent !== undefined && incumbent !== areaId) {
|
|
1091
|
-
const resolution = this.resolveAreaCollision(key, incumbent, areaId);
|
|
1092
|
-
if (resolution === "keep") {
|
|
1093
|
-
areaShadows.set(key, entry);
|
|
1094
|
-
return;
|
|
1095
|
-
}
|
|
1096
|
-
const incumbentShadows = shadows.get(incumbent) ?? new Map();
|
|
1097
|
-
incumbentShadows.set(key, rows.get(key));
|
|
1098
|
-
shadows.set(incumbent, incumbentShadows);
|
|
1099
|
-
}
|
|
1100
|
-
rows.set(key, entry);
|
|
1101
|
-
const defaultOwner = this.defaultAreaForKey(key);
|
|
1102
|
-
if (defaultOwner === areaId)
|
|
1103
|
-
owners.delete(key);
|
|
1104
|
-
else
|
|
1105
|
-
owners.set(key, areaId);
|
|
1106
|
-
}
|
|
1107
|
-
deleteAreaRow(rows, owners, shadows, areaId, key) {
|
|
1108
|
-
const areaShadows = shadows.get(areaId);
|
|
1109
|
-
if (areaShadows?.delete(key))
|
|
1110
|
-
return true;
|
|
1111
|
-
if (this.rowOwner(key, owners) !== areaId)
|
|
1112
|
-
return false;
|
|
1113
|
-
const deleted = rows.delete(key);
|
|
1114
|
-
if (!deleted)
|
|
1115
|
-
return false;
|
|
1116
|
-
owners.delete(key);
|
|
1117
|
-
this.promoteShadowAfterOwnerDelete(rows, owners, shadows, key);
|
|
1118
|
-
return deleted;
|
|
1119
|
-
}
|
|
1120
|
-
/** A shadow remains durable when its aggregate owner is deleted, so make it visible. */
|
|
1121
|
-
promoteShadowAfterOwnerDelete(rows, owners, shadows, key) {
|
|
1122
|
-
let survivor;
|
|
1123
|
-
for (const [areaId, areaShadows] of shadows) {
|
|
1124
|
-
const entry = areaShadows.get(key);
|
|
1125
|
-
if (entry === undefined)
|
|
1126
|
-
continue;
|
|
1127
|
-
if (survivor === undefined) {
|
|
1128
|
-
survivor = { areaId, entry };
|
|
1129
|
-
continue;
|
|
1130
|
-
}
|
|
1131
|
-
if (this.resolveAreaCollision(key, survivor.areaId, areaId) === "replace") {
|
|
1132
|
-
survivor = { areaId, entry };
|
|
1133
|
-
}
|
|
1134
|
-
}
|
|
1135
|
-
if (survivor === undefined)
|
|
1136
|
-
return;
|
|
1137
|
-
shadows.get(survivor.areaId).delete(key);
|
|
1138
|
-
this.writeAreaRow(rows, owners, shadows, survivor.areaId, key, survivor.entry);
|
|
937
|
+
cache.identities.set(areaId, this.openArea(areaId).getGenerationIdentity());
|
|
1139
938
|
}
|
|
1140
939
|
/** Refresh only identities that changed since this process's own cold-cache append. */
|
|
1141
940
|
refreshChangedColdStores() {
|
|
@@ -1161,34 +960,6 @@ export class AreaLedger {
|
|
|
1161
960
|
}
|
|
1162
961
|
this.coldCacheIdentities = identities;
|
|
1163
962
|
}
|
|
1164
|
-
/** Keep a warm packed aggregate current without rebuilding every unchanged row. */
|
|
1165
|
-
updateWarmAggregateAfterAppend(delta, changedAreaIds) {
|
|
1166
|
-
const cache = this.aggregateCache;
|
|
1167
|
-
if (!cache) {
|
|
1168
|
-
this.captureColdCacheIdentities(changedAreaIds, delta.metadata !== undefined);
|
|
1169
|
-
return;
|
|
1170
|
-
}
|
|
1171
|
-
const states = this.currentAreaStates();
|
|
1172
|
-
const identities = this.currentAreaIdentities();
|
|
1173
|
-
const meta = this.openMeta().getState();
|
|
1174
|
-
const committed = this.committedTransactions(meta, states);
|
|
1175
|
-
const areasToReconcile = new Set(changedAreaIds);
|
|
1176
|
-
for (const areaId of this.areaIds) {
|
|
1177
|
-
const previous = cache.identities.get(areaId);
|
|
1178
|
-
const current = identities.get(areaId);
|
|
1179
|
-
if (previous.generation !== current.generation ||
|
|
1180
|
-
previous.lastRecordSequence !== current.lastRecordSequence ||
|
|
1181
|
-
previous.walBytes !== current.walBytes) {
|
|
1182
|
-
areasToReconcile.add(areaId);
|
|
1183
|
-
}
|
|
1184
|
-
}
|
|
1185
|
-
for (const areaId of areasToReconcile) {
|
|
1186
|
-
this.updateCachedArea(cache, areaId, materializeArea(states.get(areaId), committed));
|
|
1187
|
-
}
|
|
1188
|
-
this.updateCacheMetadata(cache, meta, committed);
|
|
1189
|
-
this.replaceCacheIdentities(cache, identities);
|
|
1190
|
-
cache.revision = ++this.aggregateRevision;
|
|
1191
|
-
}
|
|
1192
963
|
invalidateAggregateCache() {
|
|
1193
964
|
if (this.aggregateCache)
|
|
1194
965
|
this.coldCacheIdentities = new Map(this.aggregateCache.identities);
|
|
@@ -1206,49 +977,6 @@ function symmetricDifference(left, right) {
|
|
|
1206
977
|
result.add(value);
|
|
1207
978
|
return result;
|
|
1208
979
|
}
|
|
1209
|
-
/** Count a source one key at a time so capacity planning never creates a key array. */
|
|
1210
|
-
function countJournalRows(journals) {
|
|
1211
|
-
let count = 0;
|
|
1212
|
-
for (const { journal } of journals) {
|
|
1213
|
-
const rows = decodedJournalRows(journal) ?? journalRecordRows(journal.files);
|
|
1214
|
-
if (rows?.size !== undefined) {
|
|
1215
|
-
count += rows.size;
|
|
1216
|
-
continue;
|
|
1217
|
-
}
|
|
1218
|
-
for (const key in journal.files) {
|
|
1219
|
-
if (Object.hasOwn(journal.files, key))
|
|
1220
|
-
count += 1;
|
|
1221
|
-
}
|
|
1222
|
-
}
|
|
1223
|
-
return count;
|
|
1224
|
-
}
|
|
1225
|
-
/** Keep the record proxy at the public edge while internal ledger code stays on lazy rows. */
|
|
1226
|
-
function rowsOf(journal) {
|
|
1227
|
-
const decoded = decodedJournalRows(journal);
|
|
1228
|
-
if (decoded)
|
|
1229
|
-
return decoded.rows;
|
|
1230
|
-
const proxied = journalRecordRows(journal.files);
|
|
1231
|
-
if (proxied)
|
|
1232
|
-
return proxied;
|
|
1233
|
-
const files = journal.files;
|
|
1234
|
-
return {
|
|
1235
|
-
get: (key) => files[key],
|
|
1236
|
-
has: (key) => Object.hasOwn(files, key),
|
|
1237
|
-
set: (key, entry) => { files[key] = entry; },
|
|
1238
|
-
delete: (key) => {
|
|
1239
|
-
if (!Object.hasOwn(files, key))
|
|
1240
|
-
return false;
|
|
1241
|
-
delete files[key];
|
|
1242
|
-
return true;
|
|
1243
|
-
},
|
|
1244
|
-
*keys() {
|
|
1245
|
-
for (const key in files) {
|
|
1246
|
-
if (Object.hasOwn(files, key))
|
|
1247
|
-
yield key;
|
|
1248
|
-
}
|
|
1249
|
-
},
|
|
1250
|
-
};
|
|
1251
|
-
}
|
|
1252
980
|
function reduceAreaRecord(state, record) {
|
|
1253
981
|
state.direct ??= [];
|
|
1254
982
|
state.prepared ??= {};
|
|
@@ -1323,10 +1051,10 @@ function isFinalizeRecord(value) {
|
|
|
1323
1051
|
}
|
|
1324
1052
|
/**
|
|
1325
1053
|
* A row is an immutable value. Every projection of a journal — the raw area
|
|
1326
|
-
* state
|
|
1327
|
-
*
|
|
1328
|
-
*
|
|
1329
|
-
*
|
|
1054
|
+
* state, the materialized per-area journal, the combined aggregate, and each
|
|
1055
|
+
* `JournalStore` session opened on top — holds the SAME row object, and the
|
|
1056
|
+
* object is frozen so an in-place mutation throws instead of silently
|
|
1057
|
+
* rewriting every other projection at once.
|
|
1330
1058
|
*
|
|
1331
1059
|
* This is a memory bound, not a nicety. A personal vault on this fleet has
|
|
1332
1060
|
* ~690k rows; deep-copying them per projection put five or six full copies in
|
|
@@ -1353,28 +1081,20 @@ function freezeDeep(value) {
|
|
|
1353
1081
|
Object.freeze(value);
|
|
1354
1082
|
}
|
|
1355
1083
|
function materializeArea(state, committed) {
|
|
1356
|
-
|
|
1357
|
-
// A settled area is already a stable private projection. Returning it as-is
|
|
1358
|
-
// avoids the former whole-record clone that lived beside the packed aggregate.
|
|
1359
|
-
if (events.length === 0)
|
|
1360
|
-
return state.journal;
|
|
1361
|
-
// Pending direct/prepared records must remain isolated from their durable
|
|
1362
|
-
// base until a fold makes them visible. Keep only those changed keys in an
|
|
1363
|
-
// overlay instead of cloning every base row just to apply a small tail.
|
|
1364
|
-
const overlay = new JournalRowOverlay(rowsOf(state.journal));
|
|
1084
|
+
// Shallow: a new key map over the SAME row objects as the area state.
|
|
1365
1085
|
const journal = {
|
|
1366
1086
|
...state.journal,
|
|
1367
|
-
files:
|
|
1087
|
+
files: { ...state.journal.files },
|
|
1368
1088
|
pulls: structuredClone(state.journal.pulls ?? []),
|
|
1369
1089
|
};
|
|
1370
|
-
for (const event of
|
|
1090
|
+
for (const event of visibleEvents(state, committed))
|
|
1371
1091
|
applyDelta(journal, event.delta);
|
|
1372
1092
|
return journal;
|
|
1373
1093
|
}
|
|
1374
1094
|
/** Apply only events relevant to one key; unlike materializeArea this never clones a whole journal. */
|
|
1375
1095
|
function materializeRow(state, committed, key, events = visibleEvents(state, committed)) {
|
|
1376
1096
|
testHooks.onMaterializeRow?.(key);
|
|
1377
|
-
let row =
|
|
1097
|
+
let row = state.journal.files[key];
|
|
1378
1098
|
for (const event of events) {
|
|
1379
1099
|
if (event.delta.deletes.includes(key))
|
|
1380
1100
|
row = undefined;
|
|
@@ -1395,7 +1115,7 @@ function visibleEvents(state, committed) {
|
|
|
1395
1115
|
}
|
|
1396
1116
|
/** Key-only union used by scoped directory reads; values stay un-cloned until selected. */
|
|
1397
1117
|
function visibleRowKeys(state, events) {
|
|
1398
|
-
const keys = new Set(
|
|
1118
|
+
const keys = new Set(Object.keys(state.journal.files));
|
|
1399
1119
|
for (const event of events) {
|
|
1400
1120
|
for (const key of event.delta.deletes)
|
|
1401
1121
|
keys.add(key);
|
|
@@ -1405,33 +1125,16 @@ function visibleRowKeys(state, events) {
|
|
|
1405
1125
|
return keys;
|
|
1406
1126
|
}
|
|
1407
1127
|
function applyDelta(journal, delta) {
|
|
1408
|
-
const rows = rowsOf(journal);
|
|
1409
1128
|
for (const key of delta.deletes)
|
|
1410
|
-
|
|
1129
|
+
delete journal.files[key];
|
|
1411
1130
|
// Delta rows are already this ledger's own copies (see `partition` and the
|
|
1412
1131
|
// WAL fold); adopt the objects, do not copy them again.
|
|
1413
1132
|
for (const [key, entry] of Object.entries(delta.upserts))
|
|
1414
|
-
|
|
1133
|
+
journal.files[key] = shareRow(entry);
|
|
1415
1134
|
}
|
|
1416
1135
|
function emptyJournal() {
|
|
1417
1136
|
return { version: "2", lastSync: "", files: {}, pulls: [] };
|
|
1418
1137
|
}
|
|
1419
|
-
/**
|
|
1420
|
-
* Public aggregate readers receive a mutable record facade without copying all
|
|
1421
|
-
* rows. Replacements and deletes stay private to this view; rows returned from
|
|
1422
|
-
* the immutable aggregate remain frozen, so in-place mutation cannot bypass a
|
|
1423
|
-
* later delta writer.
|
|
1424
|
-
*/
|
|
1425
|
-
function detachedJournalOverlay(journal, source) {
|
|
1426
|
-
const rows = source ?? rowsOf(journal);
|
|
1427
|
-
const result = {
|
|
1428
|
-
...journal,
|
|
1429
|
-
files: createJournalRecordProxy(new JournalRowOverlay(rows)),
|
|
1430
|
-
};
|
|
1431
|
-
if (Object.hasOwn(journal, "pulls"))
|
|
1432
|
-
result.pulls = structuredClone(journal.pulls);
|
|
1433
|
-
return result;
|
|
1434
|
-
}
|
|
1435
1138
|
function capPullRecords(pulls) {
|
|
1436
1139
|
const journal = { version: "2", lastSync: "", files: {}, pulls: structuredClone(pulls) };
|
|
1437
1140
|
trimPullRecords(journal);
|