@indigoai-us/hq-cloud 6.16.65 → 6.16.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +14 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +42 -8
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +62 -7
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +172 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +51 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +409 -165
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +19 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +40 -5
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +47 -0
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +59 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +48 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +1553 -192
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/push-scan-exclusions.test.js +4 -4
- package/dist/cli/push-scan-exclusions.test.js.map +1 -1
- package/dist/cli/share.d.ts +31 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +143 -8
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +350 -5
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +12 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +297 -125
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +255 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal-row-store.d.ts +40 -5
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +240 -21
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +54 -0
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +11 -6
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +324 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +51 -16
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +303 -79
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +20 -2
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/sync/area-ledger.d.ts +27 -3
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +149 -39
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +192 -5
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +29 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +6 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +14 -2
- package/dist/sync/state-store.js.map +1 -1
- package/dist/watcher.d.ts +12 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +131 -0
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +63 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +2 -1
package/dist/journal.js
CHANGED
|
@@ -17,7 +17,7 @@ import * as os from "os";
|
|
|
17
17
|
import * as path from "path";
|
|
18
18
|
import * as crypto from "crypto";
|
|
19
19
|
import { JournalRowOverlay, JournalRowStore, journalEntryFingerprint, } from "./journal-row-store.js";
|
|
20
|
-
import { createJournalRecordProxy } from "./journal-record-proxy.js";
|
|
20
|
+
import { createJournalRecordProxy, journalRecordRows } from "./journal-record-proxy.js";
|
|
21
21
|
import { decodeJournalSnapshot, decodedJournalRows } from "./journal-snapshot-decoder.js";
|
|
22
22
|
import { forgetSharedAreaJournalRows } from "./journal-snapshot-rows.js";
|
|
23
23
|
import { encodeV4JournalSnapshotChunks } from "./journal-snapshot-v4.js";
|
|
@@ -114,6 +114,9 @@ export function journalRows(journal) {
|
|
|
114
114
|
const backing = storeBackedJournals.get(journal);
|
|
115
115
|
if (backing)
|
|
116
116
|
return backing.store;
|
|
117
|
+
const proxied = journalRecordRows(journal.files);
|
|
118
|
+
if (proxied)
|
|
119
|
+
return proxied;
|
|
117
120
|
const decoded = decodedJournalRows(journal);
|
|
118
121
|
if (decoded)
|
|
119
122
|
return decoded.rows;
|
|
@@ -146,9 +149,12 @@ export function closeCachedAreaJournalMigration(slug) {
|
|
|
146
149
|
closeCachedAreaMigration(getStateDir(), slug);
|
|
147
150
|
}
|
|
148
151
|
/**
|
|
149
|
-
* Release all process-local area bridges
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
+
* Release all process-local area bridges at runner shutdown.
|
|
153
|
+
*
|
|
154
|
+
* A bridge owns one packed aggregate per authoritative journal. It is retained
|
|
155
|
+
* through ordinary watch passes so a later scoped callback cannot re-pack the
|
|
156
|
+
* whole journal synchronously. Durable layout/phase changes use the narrower
|
|
157
|
+
* close helper above; shutdown is the terminal memory boundary.
|
|
152
158
|
*/
|
|
153
159
|
export function releaseCachedAreaJournalMigrations() {
|
|
154
160
|
for (const [cacheKey, cached] of cachedAreaMigrations) {
|
|
@@ -191,7 +197,8 @@ function readStoreBackedJournal(journal, options = {}) {
|
|
|
191
197
|
existing.pullsJson = JSON.stringify(journal.pulls ?? []);
|
|
192
198
|
return journal;
|
|
193
199
|
}
|
|
194
|
-
const
|
|
200
|
+
const proxied = journalRecordRows(journal.files);
|
|
201
|
+
const decoded = proxied ?? decodedJournalRows(journal);
|
|
195
202
|
if (decoded) {
|
|
196
203
|
if (options.privateOverlay) {
|
|
197
204
|
const store = new JournalRowOverlay(decoded);
|
|
@@ -207,7 +214,22 @@ function readStoreBackedJournal(journal, options = {}) {
|
|
|
207
214
|
});
|
|
208
215
|
return result;
|
|
209
216
|
}
|
|
210
|
-
decoded
|
|
217
|
+
if (decoded instanceof JournalRowStore)
|
|
218
|
+
decoded.enableChangeTracking();
|
|
219
|
+
else {
|
|
220
|
+
const store = new JournalRowOverlay(decoded);
|
|
221
|
+
const result = { ...journal, files: createJournalRecordProxy(store) };
|
|
222
|
+
if (Object.hasOwn(journal, "pulls"))
|
|
223
|
+
result.pulls = structuredClone(journal.pulls);
|
|
224
|
+
storeBackedJournals.set(result, {
|
|
225
|
+
store,
|
|
226
|
+
version: result.version,
|
|
227
|
+
lastSync: result.lastSync,
|
|
228
|
+
pullsJson: JSON.stringify(result.pulls ?? []),
|
|
229
|
+
keysNormalized: false,
|
|
230
|
+
});
|
|
231
|
+
return result;
|
|
232
|
+
}
|
|
211
233
|
storeBackedJournals.set(journal, {
|
|
212
234
|
store: decoded,
|
|
213
235
|
version: journal.version,
|
|
@@ -279,11 +301,27 @@ function journalDeltaFromStoreDirtyLog(backing, next) {
|
|
|
279
301
|
}
|
|
280
302
|
function checkpointStoreBackedJournal(backing, journal) {
|
|
281
303
|
backing.store.drainDirtyLog();
|
|
282
|
-
backing.store.clearCommittedChanges?.();
|
|
283
304
|
backing.version = journal.version;
|
|
284
305
|
backing.lastSync = journal.lastSync;
|
|
285
306
|
backing.pullsJson = JSON.stringify(journal.pulls ?? []);
|
|
286
307
|
}
|
|
308
|
+
/**
|
|
309
|
+
* Move one full-pass facade onto the immutable durable revision it just
|
|
310
|
+
* published. Its prior private overlay contains the now-committed rows; merely
|
|
311
|
+
* clearing that overlay would make the caller's journal roll back to the old
|
|
312
|
+
* packed base. The StateStore reducer publishes a fresh overlay for every
|
|
313
|
+
* delta, so later scoped writes replace the aggregate instead of mutating this
|
|
314
|
+
* pass's newly established base.
|
|
315
|
+
*/
|
|
316
|
+
function rebaseStoreBackedJournal(backing, journal, durable) {
|
|
317
|
+
const rows = journalRecordRows(durable.files) ?? decodedJournalRows(durable);
|
|
318
|
+
const store = new JournalRowOverlay(rows ?? new RecordJournalRows(durable.files));
|
|
319
|
+
journal.version = durable.version;
|
|
320
|
+
journal.lastSync = durable.lastSync;
|
|
321
|
+
journal.pulls = structuredClone(durable.pulls ?? []);
|
|
322
|
+
journal.files = createJournalRecordProxy(store);
|
|
323
|
+
backing.store = store;
|
|
324
|
+
}
|
|
287
325
|
function storeBackedJournalNeedsMigration(backing, journal) {
|
|
288
326
|
return journal.version !== JOURNAL_VERSION_CURRENT ||
|
|
289
327
|
!Array.isArray(journal.pulls) ||
|
|
@@ -357,16 +395,15 @@ function entryFingerprint(entry) {
|
|
|
357
395
|
*/
|
|
358
396
|
export function journalDeltaFromBaseline(baseline, next) {
|
|
359
397
|
const upserts = {};
|
|
360
|
-
const
|
|
361
|
-
for (const key of
|
|
362
|
-
const entry = files[key];
|
|
398
|
+
const nextRows = journalRows(next);
|
|
399
|
+
for (const [key, entry] of nextRows.entries()) {
|
|
363
400
|
if (baseline.rows.get(key) === entryFingerprint(entry))
|
|
364
401
|
continue;
|
|
365
402
|
upserts[key] = entry;
|
|
366
403
|
}
|
|
367
404
|
const deletes = [];
|
|
368
405
|
for (const key of baseline.rows.keys()) {
|
|
369
|
-
if (!
|
|
406
|
+
if (!nextRows.has(key))
|
|
370
407
|
deletes.push(key);
|
|
371
408
|
}
|
|
372
409
|
const delta = {
|
|
@@ -510,7 +547,11 @@ export function readJournal(slug) {
|
|
|
510
547
|
// mutable without deep-cloning the aggregate before every pass.
|
|
511
548
|
return readStoreBackedJournal(areaMigration.readAreaJournalCached().journal, { privateOverlay: true });
|
|
512
549
|
}
|
|
513
|
-
|
|
550
|
+
// The retained keyed projection and this full-pass facade can borrow the
|
|
551
|
+
// same immutable packed rows. Each owns only its own overlay, so an
|
|
552
|
+
// uncommitted full pass is never visible to a scoped reader (and a scoped
|
|
553
|
+
// delta cannot dirty the full pass).
|
|
554
|
+
return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration, true), { privateOverlay: true });
|
|
514
555
|
}
|
|
515
556
|
/**
|
|
516
557
|
* Read the mutable outer shell required by a full pull without detaching every
|
|
@@ -521,9 +562,8 @@ export function readJournal(slug) {
|
|
|
521
562
|
* own changes in a private overlay. `writeJournal()` drains that overlay's
|
|
522
563
|
* dirty log, avoiding a full-ledger fingerprint at every download checkpoint.
|
|
523
564
|
*
|
|
524
|
-
* Legacy journals
|
|
525
|
-
*
|
|
526
|
-
* the area-authoritative structured-clone copy that this reader avoids.
|
|
565
|
+
* Legacy and area-authoritative journals both borrow immutable packed rows and
|
|
566
|
+
* put their caller-owned mutations in a private overlay.
|
|
527
567
|
*
|
|
528
568
|
* @internal Full-pull boundary; not a public journal record API.
|
|
529
569
|
*/
|
|
@@ -559,49 +599,55 @@ export function readJournalForPass(slug) {
|
|
|
559
599
|
function readJournalAggregate(slug) {
|
|
560
600
|
return readJournalAggregateWith(slug, openAreaMigrationIfAuthoritative(slug));
|
|
561
601
|
}
|
|
562
|
-
function readJournalAggregateWith(slug, areaMigration) {
|
|
602
|
+
function readJournalAggregateWith(slug, areaMigration, retainScopedProjection = false) {
|
|
563
603
|
if (areaMigration)
|
|
564
604
|
return areaMigration.readAreaJournal();
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
605
|
+
const store = StateStore.exists(getStateDir(), journalScopeId(slug))
|
|
606
|
+
? openJournalStore(slug, emptyJournal())
|
|
607
|
+
: openJournalStore(slug, readLegacyJournal(slug));
|
|
608
|
+
if (retainScopedProjection) {
|
|
609
|
+
// A full pass has already paid to recover this packed snapshot. Retain its
|
|
610
|
+
// projection for the next scoped pass instead of making that pass decode
|
|
611
|
+
// the same vault before it looks up even one named row. Read-only
|
|
612
|
+
// enumeration deliberately does not retain: watcher warm-up may inspect
|
|
613
|
+
// every locator and must not evict the hot personal projection.
|
|
614
|
+
retainScopedJournalStore(slug, new JournalStore(slug, store));
|
|
615
|
+
}
|
|
616
|
+
return store.getState().journal;
|
|
569
617
|
}
|
|
570
618
|
/**
|
|
571
619
|
* Read a mutable, detached snapshot for named rows in a scoped pass.
|
|
572
620
|
*
|
|
573
|
-
* Area-authoritative journals use the ledger's keyed reader
|
|
574
|
-
*
|
|
575
|
-
*
|
|
576
|
-
*
|
|
577
|
-
*
|
|
578
|
-
* must never be rows x roots: a watch-mode batch names one delete root per
|
|
579
|
-
* changed path, and a warm-up catch-up hands the pass ~10k of them against a
|
|
580
|
-
* 70k-row vault journal.
|
|
621
|
+
* Area-authoritative journals use the ledger's keyed reader. Other journals
|
|
622
|
+
* retain one refreshed packed projection after the preceding full pass, so a
|
|
623
|
+
* scoped read is proportional to named keys rather than a fresh snapshot
|
|
624
|
+
* recovery. Only named delete roots require a linear row scan, with an
|
|
625
|
+
* ancestor-set lookup per row; it must never be rows x roots.
|
|
581
626
|
*/
|
|
582
627
|
export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
|
|
583
628
|
const areaMigration = openAreaMigrationIfAuthoritative(slug);
|
|
584
629
|
if (areaMigration) {
|
|
585
630
|
return rememberJournalBaseline(areaMigration.readAreaJournalScoped(keys, deleteScopeRoots));
|
|
586
631
|
}
|
|
587
|
-
const
|
|
588
|
-
|
|
632
|
+
const store = scopedJournalStore(slug);
|
|
633
|
+
store.refresh();
|
|
634
|
+
const metadata = store.metadata();
|
|
589
635
|
const files = {};
|
|
590
636
|
const roots = new Set(deleteScopeRoots);
|
|
591
637
|
if (roots.has("")) {
|
|
592
|
-
for (const [key, entry] of
|
|
638
|
+
for (const [key, entry] of store.scanPrefix("")) {
|
|
593
639
|
files[key] = structuredClone(entry);
|
|
594
640
|
}
|
|
595
641
|
}
|
|
596
642
|
else {
|
|
597
643
|
for (const key of new Set(keys)) {
|
|
598
|
-
const entry =
|
|
644
|
+
const entry = store.get(key);
|
|
599
645
|
if (entry === undefined)
|
|
600
646
|
continue;
|
|
601
647
|
files[key] = structuredClone(entry);
|
|
602
648
|
}
|
|
603
649
|
if (roots.size > 0) {
|
|
604
|
-
for (const [key, entry] of
|
|
650
|
+
for (const [key, entry] of store.scanPrefix("")) {
|
|
605
651
|
if (Object.hasOwn(files, key))
|
|
606
652
|
continue;
|
|
607
653
|
if (isUnderAnyRoot(key, roots))
|
|
@@ -610,10 +656,10 @@ export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
|
|
|
610
656
|
}
|
|
611
657
|
}
|
|
612
658
|
return rememberJournalBaseline({
|
|
613
|
-
version:
|
|
614
|
-
lastSync:
|
|
659
|
+
version: metadata.version,
|
|
660
|
+
lastSync: metadata.lastSync,
|
|
615
661
|
files,
|
|
616
|
-
pulls: structuredClone(
|
|
662
|
+
pulls: structuredClone(metadata.pulls),
|
|
617
663
|
});
|
|
618
664
|
}
|
|
619
665
|
/**
|
|
@@ -694,9 +740,22 @@ function journalStoreOptions(slug, initialJournal) {
|
|
|
694
740
|
}
|
|
695
741
|
if (record.type === JOURNAL_DELTA_RECORD_TYPE) {
|
|
696
742
|
const delta = record.payload;
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
743
|
+
const existingRows = journalRecordRows(state.journal.files) ?? decodedJournalRows(state.journal);
|
|
744
|
+
if (existingRows) {
|
|
745
|
+
// StateStore's aggregate is also the immutable source for any
|
|
746
|
+
// full-pass facade opened before this append. Once a first delta is
|
|
747
|
+
// observed, put durable changes in a separate overlay rather than
|
|
748
|
+
// mutating that shared packed row store.
|
|
749
|
+
const rows = existingRows instanceof JournalRowOverlay
|
|
750
|
+
? existingRows.withCommittedDelta(delta.upserts, delta.deletes)
|
|
751
|
+
: new JournalRowOverlay(existingRows).withCommittedDelta(delta.upserts, delta.deletes);
|
|
752
|
+
state.journal.files = createJournalRecordProxy(rows);
|
|
753
|
+
}
|
|
754
|
+
else {
|
|
755
|
+
for (const key of delta.deletes)
|
|
756
|
+
delete state.journal.files[key];
|
|
757
|
+
Object.assign(state.journal.files, delta.upserts);
|
|
758
|
+
}
|
|
700
759
|
state.journal.version = delta.version;
|
|
701
760
|
state.journal.lastSync = delta.lastSync ?? state.journal.lastSync;
|
|
702
761
|
state.journal.pulls = delta.pulls ?? state.journal.pulls;
|
|
@@ -735,6 +794,10 @@ function openAreaMigrationIfAuthoritative(slug) {
|
|
|
735
794
|
closeCachedAreaMigration(stateDir, slug);
|
|
736
795
|
return undefined;
|
|
737
796
|
}
|
|
797
|
+
// Area authority replaces the legacy v3 aggregate. Drop any retained legacy
|
|
798
|
+
// projection before a scoped reader can accidentally keep both generations
|
|
799
|
+
// resident (or use the obsolete writer after cutover).
|
|
800
|
+
releaseScopedJournalStore(slug);
|
|
738
801
|
const resolverPath = path.join(areaLayoutDirectory(stateDir, slug), "area-resolver.json");
|
|
739
802
|
let resolverJson;
|
|
740
803
|
try {
|
|
@@ -799,19 +862,20 @@ export function openJournalStateStoreForMigration(slug) {
|
|
|
799
862
|
async function openJournalStoreAsync(slug, initialJournal) {
|
|
800
863
|
return StateStore.openAsync(journalStoreOptions(slug, initialJournal));
|
|
801
864
|
}
|
|
802
|
-
/**
|
|
803
|
-
* Long-lived keyed projection over the v3 snapshot/WAL authority.
|
|
804
|
-
*
|
|
805
|
-
* The first open recovers once. Subsequent refreshes consume only authenticated
|
|
806
|
-
* complete WAL frames, rebuilding the projection solely after a generation
|
|
807
|
-
* publication or invalid-tail fallback.
|
|
808
|
-
*/
|
|
809
865
|
export class JournalStore {
|
|
810
866
|
slug;
|
|
811
867
|
store;
|
|
812
868
|
areaMigration;
|
|
813
869
|
files = new JournalRowStore({ trackChanges: false });
|
|
814
870
|
usesSharedRows = false;
|
|
871
|
+
/**
|
|
872
|
+
* Durable deltas this keyed session has observed after its base snapshot.
|
|
873
|
+
* The base can be shared with a full-pass facade, so publishing a scoped
|
|
874
|
+
* delta here must never mutate it in place.
|
|
875
|
+
*/
|
|
876
|
+
committedOverlay = new Map();
|
|
877
|
+
/** Base keys deleted then reinserted by durable records move to the tail. */
|
|
878
|
+
committedMoved = new Set();
|
|
815
879
|
/**
|
|
816
880
|
* Pass-local rows that have not reached the shared projection yet. A missing
|
|
817
881
|
* value represents a staged delete; every other value is a frozen copy of
|
|
@@ -841,27 +905,56 @@ export class JournalStore {
|
|
|
841
905
|
this.assertOpen();
|
|
842
906
|
if (this.overlay.has(key))
|
|
843
907
|
return this.overlay.get(key);
|
|
908
|
+
if (this.committedOverlay.has(key))
|
|
909
|
+
return this.committedOverlay.get(key);
|
|
844
910
|
return this.files.get(key);
|
|
845
911
|
}
|
|
846
912
|
scanPrefix(prefix) {
|
|
847
913
|
this.assertOpen();
|
|
848
|
-
const rows = new Map(this.files.entriesForPrefix(prefix));
|
|
849
914
|
const normalizedPrefix = prefix.replace(/\/+$/, "");
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
915
|
+
const files = this.files;
|
|
916
|
+
const committedOverlay = this.committedOverlay;
|
|
917
|
+
const committedMoved = this.committedMoved;
|
|
918
|
+
const overlay = this.overlay;
|
|
919
|
+
const iterate = function* () {
|
|
920
|
+
for (const [key, entry] of files.entriesForPrefix(normalizedPrefix)) {
|
|
921
|
+
if (committedMoved.has(key))
|
|
922
|
+
continue;
|
|
923
|
+
const replacement = overlay.has(key)
|
|
924
|
+
? overlay.get(key)
|
|
925
|
+
: committedOverlay.has(key)
|
|
926
|
+
? committedOverlay.get(key)
|
|
927
|
+
: entry;
|
|
928
|
+
if (replacement !== undefined)
|
|
929
|
+
yield [key, replacement];
|
|
930
|
+
}
|
|
931
|
+
// A Map replacement preserves the base row's position and appends newly
|
|
932
|
+
// introduced overlay rows. Preserve that observable order without holding
|
|
933
|
+
// a materialized JournalEntry for every base row at once.
|
|
934
|
+
const additions = new Map(committedOverlay);
|
|
935
|
+
for (const [key, entry] of overlay)
|
|
936
|
+
additions.set(key, entry);
|
|
937
|
+
for (const [key, entry] of additions) {
|
|
938
|
+
if (entry === undefined || (files.has(key) && !committedMoved.has(key)))
|
|
939
|
+
continue;
|
|
940
|
+
if (isJournalStorePrefixMatch(key, normalizedPrefix))
|
|
941
|
+
yield [key, entry];
|
|
942
|
+
}
|
|
943
|
+
};
|
|
944
|
+
const scan = {
|
|
945
|
+
[Symbol.iterator]: iterate,
|
|
946
|
+
filter: ((predicate, thisArg) => Array.from(iterate()).filter(predicate, thisArg)),
|
|
947
|
+
};
|
|
948
|
+
return scan;
|
|
859
949
|
}
|
|
860
950
|
rowCount() {
|
|
861
951
|
if (this.disposed)
|
|
862
952
|
return 0;
|
|
953
|
+
const replacements = new Map(this.committedOverlay);
|
|
954
|
+
for (const [key, entry] of this.overlay)
|
|
955
|
+
replacements.set(key, entry);
|
|
863
956
|
let count = this.files.size;
|
|
864
|
-
for (const [key, entry] of
|
|
957
|
+
for (const [key, entry] of replacements) {
|
|
865
958
|
const existsInSharedStore = this.files.has(key);
|
|
866
959
|
if (entry === undefined) {
|
|
867
960
|
if (existsInSharedStore)
|
|
@@ -877,7 +970,9 @@ export class JournalStore {
|
|
|
877
970
|
set(key, entry) {
|
|
878
971
|
this.assertOpen();
|
|
879
972
|
const ownedEntry = immutableCopy(entry);
|
|
880
|
-
const sharedEntry = this.
|
|
973
|
+
const sharedEntry = this.committedOverlay.has(key)
|
|
974
|
+
? this.committedOverlay.get(key)
|
|
975
|
+
: this.files.get(key);
|
|
881
976
|
if (sharedEntry !== undefined && sameEntry(sharedEntry, ownedEntry)) {
|
|
882
977
|
this.overlay.delete(key);
|
|
883
978
|
return;
|
|
@@ -890,12 +985,12 @@ export class JournalStore {
|
|
|
890
985
|
if (this.overlay.has(key)) {
|
|
891
986
|
if (this.overlay.get(key) === undefined)
|
|
892
987
|
return false;
|
|
893
|
-
if (!this.files.has(key)) {
|
|
988
|
+
if (!this.committedOverlay.has(key) && !this.files.has(key)) {
|
|
894
989
|
this.overlay.delete(key);
|
|
895
990
|
return true;
|
|
896
991
|
}
|
|
897
992
|
}
|
|
898
|
-
else if (
|
|
993
|
+
else if (this.get(key) === undefined) {
|
|
899
994
|
return false;
|
|
900
995
|
}
|
|
901
996
|
this.overlay.set(key, undefined);
|
|
@@ -934,6 +1029,17 @@ export class JournalStore {
|
|
|
934
1029
|
pulls: this.pulls,
|
|
935
1030
|
});
|
|
936
1031
|
}
|
|
1032
|
+
/**
|
|
1033
|
+
* The StateStore aggregate after a durable append. Its row view is immutable
|
|
1034
|
+
* across later deltas, which lets a full-pass facade rebase without exposing
|
|
1035
|
+
* its completed checkpoint to subsequent scoped writers.
|
|
1036
|
+
*/
|
|
1037
|
+
durableJournalForRebase() {
|
|
1038
|
+
this.assertOpen();
|
|
1039
|
+
if (this.areaMigration)
|
|
1040
|
+
return this.areaMigration.readAreaJournalCached().journal;
|
|
1041
|
+
return this.store.getState().journal;
|
|
1042
|
+
}
|
|
937
1043
|
refresh() {
|
|
938
1044
|
this.assertOpen();
|
|
939
1045
|
if (this.areaMigration) {
|
|
@@ -960,12 +1066,10 @@ export class JournalStore {
|
|
|
960
1066
|
this.applyRefresh(result);
|
|
961
1067
|
}
|
|
962
1068
|
applyRefresh(result) {
|
|
963
|
-
if (result.recovered) {
|
|
1069
|
+
if (result.recovered || result.records.length > 0) {
|
|
964
1070
|
this.replaceFromJournal(this.store.getState().journal);
|
|
965
1071
|
return;
|
|
966
1072
|
}
|
|
967
|
-
for (const record of result.records)
|
|
968
|
-
this.applyRecord(record.type, record.payload);
|
|
969
1073
|
}
|
|
970
1074
|
applyDelta(delta) {
|
|
971
1075
|
this.assertOpen();
|
|
@@ -1025,13 +1129,11 @@ export class JournalStore {
|
|
|
1025
1129
|
this.applyCommitted(appended);
|
|
1026
1130
|
ensureJournalLocator(this.slug);
|
|
1027
1131
|
}
|
|
1028
|
-
applyCommitted(
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
this.applyRecord(record.type, record.payload);
|
|
1034
|
-
this.applyRecord(appended.type, appended.payload);
|
|
1132
|
+
applyCommitted(_appended) {
|
|
1133
|
+
// StateStore has reduced the just-appended durable frame already. Repoint
|
|
1134
|
+
// this cache at that immutable aggregate rather than replaying the frame
|
|
1135
|
+
// into a second mutable overlay beside it.
|
|
1136
|
+
this.replaceFromJournal(this.store.getState().journal);
|
|
1035
1137
|
}
|
|
1036
1138
|
/**
|
|
1037
1139
|
* Packing copies every source row into typed arrays, so this projection never
|
|
@@ -1040,21 +1142,30 @@ export class JournalStore {
|
|
|
1040
1142
|
* pass-local), which is what put the outposts over Node's heap ceiling.
|
|
1041
1143
|
*/
|
|
1042
1144
|
replaceFromJournal(journal) {
|
|
1043
|
-
const
|
|
1145
|
+
const proxied = journalRecordRows(journal.files);
|
|
1146
|
+
const decoded = proxied instanceof JournalRowStore || proxied instanceof JournalRowOverlay
|
|
1147
|
+
? proxied
|
|
1148
|
+
: decodedJournalRows(journal);
|
|
1044
1149
|
if (decoded) {
|
|
1045
|
-
if (!this.usesSharedRows && this.files !== decoded)
|
|
1150
|
+
if (!this.usesSharedRows && this.files instanceof JournalRowStore && this.files !== decoded) {
|
|
1046
1151
|
this.files.clear();
|
|
1152
|
+
}
|
|
1047
1153
|
this.files = decoded;
|
|
1048
1154
|
this.usesSharedRows = true;
|
|
1155
|
+
this.committedOverlay.clear();
|
|
1156
|
+
this.committedMoved.clear();
|
|
1049
1157
|
this.version = journal.version;
|
|
1050
1158
|
this.lastSync = journal.lastSync;
|
|
1051
1159
|
this.pulls = structuredClone(journal.pulls ?? []);
|
|
1052
1160
|
return;
|
|
1053
1161
|
}
|
|
1054
|
-
if (this.usesSharedRows)
|
|
1162
|
+
if (this.usesSharedRows || !(this.files instanceof JournalRowStore)) {
|
|
1055
1163
|
this.files = new JournalRowStore({ trackChanges: false });
|
|
1164
|
+
}
|
|
1056
1165
|
this.files.replaceFromRecord(journal.files);
|
|
1057
1166
|
this.usesSharedRows = false;
|
|
1167
|
+
this.committedOverlay.clear();
|
|
1168
|
+
this.committedMoved.clear();
|
|
1058
1169
|
this.version = journal.version;
|
|
1059
1170
|
this.lastSync = journal.lastSync;
|
|
1060
1171
|
this.pulls = structuredClone(journal.pulls ?? []);
|
|
@@ -1068,9 +1179,15 @@ export class JournalStore {
|
|
|
1068
1179
|
throw new Error("journal: keyed session received an invalid WAL record");
|
|
1069
1180
|
}
|
|
1070
1181
|
for (const key of payload.deletes)
|
|
1071
|
-
this.
|
|
1072
|
-
for (const [key, entry] of Object.entries(payload.upserts))
|
|
1073
|
-
this.
|
|
1182
|
+
this.committedOverlay.set(key, undefined);
|
|
1183
|
+
for (const [key, entry] of Object.entries(payload.upserts)) {
|
|
1184
|
+
if (this.committedOverlay.get(key) === undefined && this.committedOverlay.has(key)) {
|
|
1185
|
+
this.committedOverlay.delete(key);
|
|
1186
|
+
if (this.files.has(key))
|
|
1187
|
+
this.committedMoved.add(key);
|
|
1188
|
+
}
|
|
1189
|
+
this.committedOverlay.set(key, immutableCopy(entry));
|
|
1190
|
+
}
|
|
1074
1191
|
this.version = payload.version;
|
|
1075
1192
|
this.lastSync = payload.lastSync ?? this.lastSync;
|
|
1076
1193
|
this.pulls = structuredClone(payload.pulls ?? this.pulls);
|
|
@@ -1081,7 +1198,9 @@ export class JournalStore {
|
|
|
1081
1198
|
return;
|
|
1082
1199
|
this.disposed = true;
|
|
1083
1200
|
this.overlay.clear();
|
|
1084
|
-
|
|
1201
|
+
this.committedOverlay.clear();
|
|
1202
|
+
this.committedMoved.clear();
|
|
1203
|
+
if (!this.usesSharedRows && this.files instanceof JournalRowStore)
|
|
1085
1204
|
this.files.clear();
|
|
1086
1205
|
this.store = undefined;
|
|
1087
1206
|
this.areaMigration = undefined;
|
|
@@ -1108,6 +1227,69 @@ function immutableCopy(value) {
|
|
|
1108
1227
|
freeze(copy);
|
|
1109
1228
|
return copy;
|
|
1110
1229
|
}
|
|
1230
|
+
// A scoped watcher pass must not recover a 700k-row snapshot just to inspect
|
|
1231
|
+
// one path. A runner can alternate the personal vault and its companies in the
|
|
1232
|
+
// same realtime batch, so one cache entry turns that alternation into a fresh
|
|
1233
|
+
// whole-snapshot recovery on every dispatch. Keep a deliberately small LRU:
|
|
1234
|
+
// enough for the runner's active journal set, bounded so an unusually broad
|
|
1235
|
+
// company fanout cannot retain every journal for the lifetime of the process.
|
|
1236
|
+
const MAX_RETAINED_SCOPED_JOURNALS = 4;
|
|
1237
|
+
const cachedScopedJournalStores = new Map();
|
|
1238
|
+
function scopedJournalStoreCacheKey(slug) {
|
|
1239
|
+
return `${getStateDir()}\u0000${slug}`;
|
|
1240
|
+
}
|
|
1241
|
+
function cachedScopedJournalStoreFor(slug) {
|
|
1242
|
+
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1243
|
+
const cached = cachedScopedJournalStores.get(cacheKey);
|
|
1244
|
+
if (cached === undefined)
|
|
1245
|
+
return undefined;
|
|
1246
|
+
// Map insertion order is our LRU order. A hot scope must not be evicted by
|
|
1247
|
+
// a brief company fanout while the runner is still using it.
|
|
1248
|
+
cachedScopedJournalStores.delete(cacheKey);
|
|
1249
|
+
cachedScopedJournalStores.set(cacheKey, cached);
|
|
1250
|
+
return cached;
|
|
1251
|
+
}
|
|
1252
|
+
function retainScopedJournalStore(slug, store) {
|
|
1253
|
+
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1254
|
+
const previous = cachedScopedJournalStores.get(cacheKey);
|
|
1255
|
+
if (previous !== undefined && previous !== store)
|
|
1256
|
+
previous.dispose();
|
|
1257
|
+
cachedScopedJournalStores.delete(cacheKey);
|
|
1258
|
+
cachedScopedJournalStores.set(cacheKey, store);
|
|
1259
|
+
while (cachedScopedJournalStores.size > MAX_RETAINED_SCOPED_JOURNALS) {
|
|
1260
|
+
const oldest = cachedScopedJournalStores.entries().next().value;
|
|
1261
|
+
if (oldest === undefined)
|
|
1262
|
+
break;
|
|
1263
|
+
const [oldestKey, oldestStore] = oldest;
|
|
1264
|
+
cachedScopedJournalStores.delete(oldestKey);
|
|
1265
|
+
oldestStore.dispose();
|
|
1266
|
+
}
|
|
1267
|
+
return store;
|
|
1268
|
+
}
|
|
1269
|
+
function scopedJournalStore(slug) {
|
|
1270
|
+
const cached = cachedScopedJournalStoreFor(slug);
|
|
1271
|
+
if (cached)
|
|
1272
|
+
return cached;
|
|
1273
|
+
return retainScopedJournalStore(slug, openJournalStoreSession(slug));
|
|
1274
|
+
}
|
|
1275
|
+
/** Drop one legacy keyed projection after an authoritative area cutover. */
|
|
1276
|
+
export function releaseScopedJournalStore(slug) {
|
|
1277
|
+
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1278
|
+
const cached = cachedScopedJournalStores.get(cacheKey);
|
|
1279
|
+
if (cached === undefined)
|
|
1280
|
+
return;
|
|
1281
|
+
cachedScopedJournalStores.delete(cacheKey);
|
|
1282
|
+
cached.dispose();
|
|
1283
|
+
}
|
|
1284
|
+
/**
|
|
1285
|
+
* Release retained scoped projections when a watch runner stops. Durable state
|
|
1286
|
+
* remains in the StateStore; this only drops process-local decoded snapshots.
|
|
1287
|
+
*/
|
|
1288
|
+
export function disposeScopedJournalStores() {
|
|
1289
|
+
for (const store of cachedScopedJournalStores.values())
|
|
1290
|
+
store.dispose();
|
|
1291
|
+
cachedScopedJournalStores.clear();
|
|
1292
|
+
}
|
|
1111
1293
|
/** Open one keyed session, seeding a missing v3 store from legacy JSON once. */
|
|
1112
1294
|
export function openJournalStoreSession(slug) {
|
|
1113
1295
|
const areaMigration = openAreaMigrationIfAuthoritative(slug);
|
|
@@ -1620,6 +1802,47 @@ export function writeJournal(slug, journal) {
|
|
|
1620
1802
|
ensureJournalLocator(slug);
|
|
1621
1803
|
return;
|
|
1622
1804
|
}
|
|
1805
|
+
const baseline = journalBaselines.get(journal);
|
|
1806
|
+
const cached = cachedScopedJournalStoreFor(slug);
|
|
1807
|
+
if (cached && (storeBacking || baseline)) {
|
|
1808
|
+
// The scoped reader captured only its named rows, so its baseline is the
|
|
1809
|
+
// authoritative delta boundary. The retained session refreshes and appends
|
|
1810
|
+
// under the normal StateStore lock; it never exposes partial durable rows
|
|
1811
|
+
// to another writer.
|
|
1812
|
+
const delta = storeBacking
|
|
1813
|
+
? journalDeltaFromStoreDirtyLog(storeBacking, journal)
|
|
1814
|
+
: baseline
|
|
1815
|
+
? journalDeltaFromBaseline(baseline, journal)
|
|
1816
|
+
: undefined;
|
|
1817
|
+
if (delta && hasJournalDelta(delta)) {
|
|
1818
|
+
// Carry ONLY the metadata fields this delta actually changed. The
|
|
1819
|
+
// retained session's `metadata()` is a snapshot that goes stale the
|
|
1820
|
+
// moment another writer advances `lastSync` or appends a pull record,
|
|
1821
|
+
// and filling the untouched field from it republished that stale value
|
|
1822
|
+
// over the concurrent writer's durable one — dropping pull history or
|
|
1823
|
+
// reverting `lastSync`. The store's reducer already preserves a field
|
|
1824
|
+
// the delta omits, so omission is the whole fix.
|
|
1825
|
+
const metadata = {};
|
|
1826
|
+
if (delta.lastSync !== undefined)
|
|
1827
|
+
metadata.lastSync = delta.lastSync;
|
|
1828
|
+
if (delta.pulls !== undefined)
|
|
1829
|
+
metadata.pulls = delta.pulls;
|
|
1830
|
+
const hasMetadata = delta.lastSync !== undefined || delta.pulls !== undefined;
|
|
1831
|
+
cached.applyDelta({
|
|
1832
|
+
upserts: delta.upserts,
|
|
1833
|
+
deletes: delta.deletes,
|
|
1834
|
+
...(hasMetadata ? { metadata } : {}),
|
|
1835
|
+
});
|
|
1836
|
+
}
|
|
1837
|
+
if (storeBacking) {
|
|
1838
|
+
rebaseStoreBackedJournal(storeBacking, journal, cached.durableJournalForRebase());
|
|
1839
|
+
checkpointStoreBackedJournal(storeBacking, journal);
|
|
1840
|
+
}
|
|
1841
|
+
else
|
|
1842
|
+
rememberJournalBaseline(journal);
|
|
1843
|
+
ensureJournalLocator(slug);
|
|
1844
|
+
return;
|
|
1845
|
+
}
|
|
1623
1846
|
const store = openJournalStore(slug, journal);
|
|
1624
1847
|
if (store.wasCreated) {
|
|
1625
1848
|
// The store was just seeded from this journal, so its state already matches
|
|
@@ -1643,7 +1866,6 @@ export function writeJournal(slug, journal) {
|
|
|
1643
1866
|
// The WAL still compacts on its own schedule (record count or byte ceiling),
|
|
1644
1867
|
// so a full snapshot is written periodically rather than never — recovery
|
|
1645
1868
|
// stays bounded, it just no longer happens on every single update.
|
|
1646
|
-
const baseline = journalBaselines.get(journal);
|
|
1647
1869
|
const delta = storeBacking
|
|
1648
1870
|
? journalDeltaFromStoreDirtyLog(storeBacking, journal)
|
|
1649
1871
|
: baseline
|
|
@@ -1652,8 +1874,10 @@ export function writeJournal(slug, journal) {
|
|
|
1652
1874
|
store.append(JOURNAL_DELTA_RECORD_TYPE, delta);
|
|
1653
1875
|
// A caller can checkpoint more than once during one pass. Advancing only its
|
|
1654
1876
|
// own tracking state means the next checkpoint again publishes only new mutations.
|
|
1655
|
-
if (storeBacking)
|
|
1877
|
+
if (storeBacking) {
|
|
1878
|
+
rebaseStoreBackedJournal(storeBacking, journal, store.getState().journal);
|
|
1656
1879
|
checkpointStoreBackedJournal(storeBacking, journal);
|
|
1880
|
+
}
|
|
1657
1881
|
else
|
|
1658
1882
|
rememberJournalBaseline(journal);
|
|
1659
1883
|
// The locator is immutable content, so it only needs writing if it is absent
|