@indigoai-us/hq-cloud 6.16.51 → 6.16.52-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +0 -20
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +3 -62
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +0 -25
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +1 -2
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +6 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +30 -1
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +55 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +108 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +48 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +311 -116
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +33 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +34 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +802 -79
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +0 -40
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +109 -1205
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-rsync-preflight.test.js +4 -256
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue.d.ts +0 -4
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +1 -20
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +1 -16
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +0 -2
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +19 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +57 -37
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +87 -93
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +12 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +226 -118
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +215 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +0 -2
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +0 -30
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +0 -175
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +9 -5
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +70 -10
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +11 -6
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +259 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +37 -12
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +291 -76
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +0 -295
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +3 -18
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -18
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +24 -47
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/s3.test.js +1 -27
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/area-ledger.js +4 -4
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +29 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync-core.d.ts +4 -6
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +4 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +1 -25
- package/dist/sync-core.test.js.map +1 -1
- package/package.json +2 -1
- package/dist/cli/rescue-breaker.test.d.ts +0 -2
- package/dist/cli/rescue-breaker.test.d.ts.map +0 -1
- package/dist/cli/rescue-breaker.test.js +0 -334
- package/dist/cli/rescue-breaker.test.js.map +0 -1
- package/dist/cli/rescue-realpath.test.d.ts +0 -2
- package/dist/cli/rescue-realpath.test.d.ts.map +0 -1
- package/dist/cli/rescue-realpath.test.js +0 -109
- package/dist/cli/rescue-realpath.test.js.map +0 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +0 -2
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +0 -1
- package/dist/cli/rescue-snapshot-copy.test.js +0 -435
- package/dist/cli/rescue-snapshot-copy.test.js.map +0 -1
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +0 -2
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +0 -1
- package/dist/cli/rescue-snapshot-recovery.test.js +0 -260
- package/dist/cli/rescue-snapshot-recovery.test.js.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.d.ts +0 -2
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.js +0 -210
- package/dist/cli/rescue-source-switch-floor.test.js.map +0 -1
package/dist/journal.js
CHANGED
|
@@ -17,7 +17,7 @@ import * as os from "os";
|
|
|
17
17
|
import * as path from "path";
|
|
18
18
|
import * as crypto from "crypto";
|
|
19
19
|
import { JournalRowOverlay, JournalRowStore, journalEntryFingerprint, } from "./journal-row-store.js";
|
|
20
|
-
import { createJournalRecordProxy } from "./journal-record-proxy.js";
|
|
20
|
+
import { createJournalRecordProxy, journalRecordRows } from "./journal-record-proxy.js";
|
|
21
21
|
import { decodeJournalSnapshot, decodedJournalRows } from "./journal-snapshot-decoder.js";
|
|
22
22
|
import { forgetSharedAreaJournalRows } from "./journal-snapshot-rows.js";
|
|
23
23
|
import { encodeV4JournalSnapshotChunks } from "./journal-snapshot-v4.js";
|
|
@@ -114,6 +114,9 @@ export function journalRows(journal) {
|
|
|
114
114
|
const backing = storeBackedJournals.get(journal);
|
|
115
115
|
if (backing)
|
|
116
116
|
return backing.store;
|
|
117
|
+
const proxied = journalRecordRows(journal.files);
|
|
118
|
+
if (proxied)
|
|
119
|
+
return proxied;
|
|
117
120
|
const decoded = decodedJournalRows(journal);
|
|
118
121
|
if (decoded)
|
|
119
122
|
return decoded.rows;
|
|
@@ -191,7 +194,8 @@ function readStoreBackedJournal(journal, options = {}) {
|
|
|
191
194
|
existing.pullsJson = JSON.stringify(journal.pulls ?? []);
|
|
192
195
|
return journal;
|
|
193
196
|
}
|
|
194
|
-
const
|
|
197
|
+
const proxied = journalRecordRows(journal.files);
|
|
198
|
+
const decoded = proxied ?? decodedJournalRows(journal);
|
|
195
199
|
if (decoded) {
|
|
196
200
|
if (options.privateOverlay) {
|
|
197
201
|
const store = new JournalRowOverlay(decoded);
|
|
@@ -207,7 +211,22 @@ function readStoreBackedJournal(journal, options = {}) {
|
|
|
207
211
|
});
|
|
208
212
|
return result;
|
|
209
213
|
}
|
|
210
|
-
decoded
|
|
214
|
+
if (decoded instanceof JournalRowStore)
|
|
215
|
+
decoded.enableChangeTracking();
|
|
216
|
+
else {
|
|
217
|
+
const store = new JournalRowOverlay(decoded);
|
|
218
|
+
const result = { ...journal, files: createJournalRecordProxy(store) };
|
|
219
|
+
if (Object.hasOwn(journal, "pulls"))
|
|
220
|
+
result.pulls = structuredClone(journal.pulls);
|
|
221
|
+
storeBackedJournals.set(result, {
|
|
222
|
+
store,
|
|
223
|
+
version: result.version,
|
|
224
|
+
lastSync: result.lastSync,
|
|
225
|
+
pullsJson: JSON.stringify(result.pulls ?? []),
|
|
226
|
+
keysNormalized: false,
|
|
227
|
+
});
|
|
228
|
+
return result;
|
|
229
|
+
}
|
|
211
230
|
storeBackedJournals.set(journal, {
|
|
212
231
|
store: decoded,
|
|
213
232
|
version: journal.version,
|
|
@@ -279,11 +298,27 @@ function journalDeltaFromStoreDirtyLog(backing, next) {
|
|
|
279
298
|
}
|
|
280
299
|
function checkpointStoreBackedJournal(backing, journal) {
|
|
281
300
|
backing.store.drainDirtyLog();
|
|
282
|
-
backing.store.clearCommittedChanges?.();
|
|
283
301
|
backing.version = journal.version;
|
|
284
302
|
backing.lastSync = journal.lastSync;
|
|
285
303
|
backing.pullsJson = JSON.stringify(journal.pulls ?? []);
|
|
286
304
|
}
|
|
305
|
+
/**
|
|
306
|
+
* Move one full-pass facade onto the immutable durable revision it just
|
|
307
|
+
* published. Its prior private overlay contains the now-committed rows; merely
|
|
308
|
+
* clearing that overlay would make the caller's journal roll back to the old
|
|
309
|
+
* packed base. The StateStore reducer publishes a fresh overlay for every
|
|
310
|
+
* delta, so later scoped writes replace the aggregate instead of mutating this
|
|
311
|
+
* pass's newly established base.
|
|
312
|
+
*/
|
|
313
|
+
function rebaseStoreBackedJournal(backing, journal, durable) {
|
|
314
|
+
const rows = journalRecordRows(durable.files) ?? decodedJournalRows(durable);
|
|
315
|
+
const store = new JournalRowOverlay(rows ?? new RecordJournalRows(durable.files));
|
|
316
|
+
journal.version = durable.version;
|
|
317
|
+
journal.lastSync = durable.lastSync;
|
|
318
|
+
journal.pulls = structuredClone(durable.pulls ?? []);
|
|
319
|
+
journal.files = createJournalRecordProxy(store);
|
|
320
|
+
backing.store = store;
|
|
321
|
+
}
|
|
287
322
|
function storeBackedJournalNeedsMigration(backing, journal) {
|
|
288
323
|
return journal.version !== JOURNAL_VERSION_CURRENT ||
|
|
289
324
|
!Array.isArray(journal.pulls) ||
|
|
@@ -357,16 +392,15 @@ function entryFingerprint(entry) {
|
|
|
357
392
|
*/
|
|
358
393
|
export function journalDeltaFromBaseline(baseline, next) {
|
|
359
394
|
const upserts = {};
|
|
360
|
-
const
|
|
361
|
-
for (const key of
|
|
362
|
-
const entry = files[key];
|
|
395
|
+
const nextRows = journalRows(next);
|
|
396
|
+
for (const [key, entry] of nextRows.entries()) {
|
|
363
397
|
if (baseline.rows.get(key) === entryFingerprint(entry))
|
|
364
398
|
continue;
|
|
365
399
|
upserts[key] = entry;
|
|
366
400
|
}
|
|
367
401
|
const deletes = [];
|
|
368
402
|
for (const key of baseline.rows.keys()) {
|
|
369
|
-
if (!
|
|
403
|
+
if (!nextRows.has(key))
|
|
370
404
|
deletes.push(key);
|
|
371
405
|
}
|
|
372
406
|
const delta = {
|
|
@@ -510,7 +544,11 @@ export function readJournal(slug) {
|
|
|
510
544
|
// mutable without deep-cloning the aggregate before every pass.
|
|
511
545
|
return readStoreBackedJournal(areaMigration.readAreaJournalCached().journal, { privateOverlay: true });
|
|
512
546
|
}
|
|
513
|
-
|
|
547
|
+
// The retained keyed projection and this full-pass facade can borrow the
|
|
548
|
+
// same immutable packed rows. Each owns only its own overlay, so an
|
|
549
|
+
// uncommitted full pass is never visible to a scoped reader (and a scoped
|
|
550
|
+
// delta cannot dirty the full pass).
|
|
551
|
+
return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration, true), { privateOverlay: true });
|
|
514
552
|
}
|
|
515
553
|
/**
|
|
516
554
|
* Read the mutable outer shell required by a full pull without detaching every
|
|
@@ -521,9 +559,8 @@ export function readJournal(slug) {
|
|
|
521
559
|
* own changes in a private overlay. `writeJournal()` drains that overlay's
|
|
522
560
|
* dirty log, avoiding a full-ledger fingerprint at every download checkpoint.
|
|
523
561
|
*
|
|
524
|
-
* Legacy journals
|
|
525
|
-
*
|
|
526
|
-
* the area-authoritative structured-clone copy that this reader avoids.
|
|
562
|
+
* Legacy and area-authoritative journals both borrow immutable packed rows and
|
|
563
|
+
* put their caller-owned mutations in a private overlay.
|
|
527
564
|
*
|
|
528
565
|
* @internal Full-pull boundary; not a public journal record API.
|
|
529
566
|
*/
|
|
@@ -559,49 +596,55 @@ export function readJournalForPass(slug) {
|
|
|
559
596
|
function readJournalAggregate(slug) {
|
|
560
597
|
return readJournalAggregateWith(slug, openAreaMigrationIfAuthoritative(slug));
|
|
561
598
|
}
|
|
562
|
-
function readJournalAggregateWith(slug, areaMigration) {
|
|
599
|
+
function readJournalAggregateWith(slug, areaMigration, retainScopedProjection = false) {
|
|
563
600
|
if (areaMigration)
|
|
564
601
|
return areaMigration.readAreaJournal();
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
602
|
+
const store = StateStore.exists(getStateDir(), journalScopeId(slug))
|
|
603
|
+
? openJournalStore(slug, emptyJournal())
|
|
604
|
+
: openJournalStore(slug, readLegacyJournal(slug));
|
|
605
|
+
if (retainScopedProjection) {
|
|
606
|
+
// A full pass has already paid to recover this packed snapshot. Retain its
|
|
607
|
+
// projection for the next scoped pass instead of making that pass decode
|
|
608
|
+
// the same vault before it looks up even one named row. Read-only
|
|
609
|
+
// enumeration deliberately does not retain: watcher warm-up may inspect
|
|
610
|
+
// every locator and must not evict the hot personal projection.
|
|
611
|
+
retainScopedJournalStore(slug, new JournalStore(slug, store));
|
|
612
|
+
}
|
|
613
|
+
return store.getState().journal;
|
|
569
614
|
}
|
|
570
615
|
/**
|
|
571
616
|
* Read a mutable, detached snapshot for named rows in a scoped pass.
|
|
572
617
|
*
|
|
573
|
-
* Area-authoritative journals use the ledger's keyed reader
|
|
574
|
-
*
|
|
575
|
-
*
|
|
576
|
-
*
|
|
577
|
-
*
|
|
578
|
-
* must never be rows x roots: a watch-mode batch names one delete root per
|
|
579
|
-
* changed path, and a warm-up catch-up hands the pass ~10k of them against a
|
|
580
|
-
* 70k-row vault journal.
|
|
618
|
+
* Area-authoritative journals use the ledger's keyed reader. Other journals
|
|
619
|
+
* retain one refreshed packed projection after the preceding full pass, so a
|
|
620
|
+
* scoped read is proportional to named keys rather than a fresh snapshot
|
|
621
|
+
* recovery. Only named delete roots require a linear row scan, with an
|
|
622
|
+
* ancestor-set lookup per row; it must never be rows x roots.
|
|
581
623
|
*/
|
|
582
624
|
export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
|
|
583
625
|
const areaMigration = openAreaMigrationIfAuthoritative(slug);
|
|
584
626
|
if (areaMigration) {
|
|
585
627
|
return rememberJournalBaseline(areaMigration.readAreaJournalScoped(keys, deleteScopeRoots));
|
|
586
628
|
}
|
|
587
|
-
const
|
|
588
|
-
|
|
629
|
+
const store = scopedJournalStore(slug);
|
|
630
|
+
store.refresh();
|
|
631
|
+
const metadata = store.metadata();
|
|
589
632
|
const files = {};
|
|
590
633
|
const roots = new Set(deleteScopeRoots);
|
|
591
634
|
if (roots.has("")) {
|
|
592
|
-
for (const [key, entry] of
|
|
635
|
+
for (const [key, entry] of store.scanPrefix("")) {
|
|
593
636
|
files[key] = structuredClone(entry);
|
|
594
637
|
}
|
|
595
638
|
}
|
|
596
639
|
else {
|
|
597
640
|
for (const key of new Set(keys)) {
|
|
598
|
-
const entry =
|
|
641
|
+
const entry = store.get(key);
|
|
599
642
|
if (entry === undefined)
|
|
600
643
|
continue;
|
|
601
644
|
files[key] = structuredClone(entry);
|
|
602
645
|
}
|
|
603
646
|
if (roots.size > 0) {
|
|
604
|
-
for (const [key, entry] of
|
|
647
|
+
for (const [key, entry] of store.scanPrefix("")) {
|
|
605
648
|
if (Object.hasOwn(files, key))
|
|
606
649
|
continue;
|
|
607
650
|
if (isUnderAnyRoot(key, roots))
|
|
@@ -610,10 +653,10 @@ export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
|
|
|
610
653
|
}
|
|
611
654
|
}
|
|
612
655
|
return rememberJournalBaseline({
|
|
613
|
-
version:
|
|
614
|
-
lastSync:
|
|
656
|
+
version: metadata.version,
|
|
657
|
+
lastSync: metadata.lastSync,
|
|
615
658
|
files,
|
|
616
|
-
pulls: structuredClone(
|
|
659
|
+
pulls: structuredClone(metadata.pulls),
|
|
617
660
|
});
|
|
618
661
|
}
|
|
619
662
|
/**
|
|
@@ -694,9 +737,22 @@ function journalStoreOptions(slug, initialJournal) {
|
|
|
694
737
|
}
|
|
695
738
|
if (record.type === JOURNAL_DELTA_RECORD_TYPE) {
|
|
696
739
|
const delta = record.payload;
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
740
|
+
const existingRows = journalRecordRows(state.journal.files) ?? decodedJournalRows(state.journal);
|
|
741
|
+
if (existingRows) {
|
|
742
|
+
// StateStore's aggregate is also the immutable source for any
|
|
743
|
+
// full-pass facade opened before this append. Once a first delta is
|
|
744
|
+
// observed, put durable changes in a separate overlay rather than
|
|
745
|
+
// mutating that shared packed row store.
|
|
746
|
+
const rows = existingRows instanceof JournalRowOverlay
|
|
747
|
+
? existingRows.withCommittedDelta(delta.upserts, delta.deletes)
|
|
748
|
+
: new JournalRowOverlay(existingRows).withCommittedDelta(delta.upserts, delta.deletes);
|
|
749
|
+
state.journal.files = createJournalRecordProxy(rows);
|
|
750
|
+
}
|
|
751
|
+
else {
|
|
752
|
+
for (const key of delta.deletes)
|
|
753
|
+
delete state.journal.files[key];
|
|
754
|
+
Object.assign(state.journal.files, delta.upserts);
|
|
755
|
+
}
|
|
700
756
|
state.journal.version = delta.version;
|
|
701
757
|
state.journal.lastSync = delta.lastSync ?? state.journal.lastSync;
|
|
702
758
|
state.journal.pulls = delta.pulls ?? state.journal.pulls;
|
|
@@ -735,6 +791,10 @@ function openAreaMigrationIfAuthoritative(slug) {
|
|
|
735
791
|
closeCachedAreaMigration(stateDir, slug);
|
|
736
792
|
return undefined;
|
|
737
793
|
}
|
|
794
|
+
// Area authority replaces the legacy v3 aggregate. Drop any retained legacy
|
|
795
|
+
// projection before a scoped reader can accidentally keep both generations
|
|
796
|
+
// resident (or use the obsolete writer after cutover).
|
|
797
|
+
releaseScopedJournalStore(slug);
|
|
738
798
|
const resolverPath = path.join(areaLayoutDirectory(stateDir, slug), "area-resolver.json");
|
|
739
799
|
let resolverJson;
|
|
740
800
|
try {
|
|
@@ -799,19 +859,20 @@ export function openJournalStateStoreForMigration(slug) {
|
|
|
799
859
|
async function openJournalStoreAsync(slug, initialJournal) {
|
|
800
860
|
return StateStore.openAsync(journalStoreOptions(slug, initialJournal));
|
|
801
861
|
}
|
|
802
|
-
/**
|
|
803
|
-
* Long-lived keyed projection over the v3 snapshot/WAL authority.
|
|
804
|
-
*
|
|
805
|
-
* The first open recovers once. Subsequent refreshes consume only authenticated
|
|
806
|
-
* complete WAL frames, rebuilding the projection solely after a generation
|
|
807
|
-
* publication or invalid-tail fallback.
|
|
808
|
-
*/
|
|
809
862
|
export class JournalStore {
|
|
810
863
|
slug;
|
|
811
864
|
store;
|
|
812
865
|
areaMigration;
|
|
813
866
|
files = new JournalRowStore({ trackChanges: false });
|
|
814
867
|
usesSharedRows = false;
|
|
868
|
+
/**
|
|
869
|
+
* Durable deltas this keyed session has observed after its base snapshot.
|
|
870
|
+
* The base can be shared with a full-pass facade, so publishing a scoped
|
|
871
|
+
* delta here must never mutate it in place.
|
|
872
|
+
*/
|
|
873
|
+
committedOverlay = new Map();
|
|
874
|
+
/** Base keys deleted then reinserted by durable records move to the tail. */
|
|
875
|
+
committedMoved = new Set();
|
|
815
876
|
/**
|
|
816
877
|
* Pass-local rows that have not reached the shared projection yet. A missing
|
|
817
878
|
* value represents a staged delete; every other value is a frozen copy of
|
|
@@ -841,27 +902,56 @@ export class JournalStore {
|
|
|
841
902
|
this.assertOpen();
|
|
842
903
|
if (this.overlay.has(key))
|
|
843
904
|
return this.overlay.get(key);
|
|
905
|
+
if (this.committedOverlay.has(key))
|
|
906
|
+
return this.committedOverlay.get(key);
|
|
844
907
|
return this.files.get(key);
|
|
845
908
|
}
|
|
846
909
|
scanPrefix(prefix) {
|
|
847
910
|
this.assertOpen();
|
|
848
|
-
const rows = new Map(this.files.entriesForPrefix(prefix));
|
|
849
911
|
const normalizedPrefix = prefix.replace(/\/+$/, "");
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
912
|
+
const files = this.files;
|
|
913
|
+
const committedOverlay = this.committedOverlay;
|
|
914
|
+
const committedMoved = this.committedMoved;
|
|
915
|
+
const overlay = this.overlay;
|
|
916
|
+
const iterate = function* () {
|
|
917
|
+
for (const [key, entry] of files.entriesForPrefix(normalizedPrefix)) {
|
|
918
|
+
if (committedMoved.has(key))
|
|
919
|
+
continue;
|
|
920
|
+
const replacement = overlay.has(key)
|
|
921
|
+
? overlay.get(key)
|
|
922
|
+
: committedOverlay.has(key)
|
|
923
|
+
? committedOverlay.get(key)
|
|
924
|
+
: entry;
|
|
925
|
+
if (replacement !== undefined)
|
|
926
|
+
yield [key, replacement];
|
|
927
|
+
}
|
|
928
|
+
// A Map replacement preserves the base row's position and appends newly
|
|
929
|
+
// introduced overlay rows. Preserve that observable order without holding
|
|
930
|
+
// a materialized JournalEntry for every base row at once.
|
|
931
|
+
const additions = new Map(committedOverlay);
|
|
932
|
+
for (const [key, entry] of overlay)
|
|
933
|
+
additions.set(key, entry);
|
|
934
|
+
for (const [key, entry] of additions) {
|
|
935
|
+
if (entry === undefined || (files.has(key) && !committedMoved.has(key)))
|
|
936
|
+
continue;
|
|
937
|
+
if (isJournalStorePrefixMatch(key, normalizedPrefix))
|
|
938
|
+
yield [key, entry];
|
|
939
|
+
}
|
|
940
|
+
};
|
|
941
|
+
const scan = {
|
|
942
|
+
[Symbol.iterator]: iterate,
|
|
943
|
+
filter: ((predicate, thisArg) => Array.from(iterate()).filter(predicate, thisArg)),
|
|
944
|
+
};
|
|
945
|
+
return scan;
|
|
859
946
|
}
|
|
860
947
|
rowCount() {
|
|
861
948
|
if (this.disposed)
|
|
862
949
|
return 0;
|
|
950
|
+
const replacements = new Map(this.committedOverlay);
|
|
951
|
+
for (const [key, entry] of this.overlay)
|
|
952
|
+
replacements.set(key, entry);
|
|
863
953
|
let count = this.files.size;
|
|
864
|
-
for (const [key, entry] of
|
|
954
|
+
for (const [key, entry] of replacements) {
|
|
865
955
|
const existsInSharedStore = this.files.has(key);
|
|
866
956
|
if (entry === undefined) {
|
|
867
957
|
if (existsInSharedStore)
|
|
@@ -877,7 +967,9 @@ export class JournalStore {
|
|
|
877
967
|
set(key, entry) {
|
|
878
968
|
this.assertOpen();
|
|
879
969
|
const ownedEntry = immutableCopy(entry);
|
|
880
|
-
const sharedEntry = this.
|
|
970
|
+
const sharedEntry = this.committedOverlay.has(key)
|
|
971
|
+
? this.committedOverlay.get(key)
|
|
972
|
+
: this.files.get(key);
|
|
881
973
|
if (sharedEntry !== undefined && sameEntry(sharedEntry, ownedEntry)) {
|
|
882
974
|
this.overlay.delete(key);
|
|
883
975
|
return;
|
|
@@ -890,12 +982,12 @@ export class JournalStore {
|
|
|
890
982
|
if (this.overlay.has(key)) {
|
|
891
983
|
if (this.overlay.get(key) === undefined)
|
|
892
984
|
return false;
|
|
893
|
-
if (!this.files.has(key)) {
|
|
985
|
+
if (!this.committedOverlay.has(key) && !this.files.has(key)) {
|
|
894
986
|
this.overlay.delete(key);
|
|
895
987
|
return true;
|
|
896
988
|
}
|
|
897
989
|
}
|
|
898
|
-
else if (
|
|
990
|
+
else if (this.get(key) === undefined) {
|
|
899
991
|
return false;
|
|
900
992
|
}
|
|
901
993
|
this.overlay.set(key, undefined);
|
|
@@ -934,6 +1026,17 @@ export class JournalStore {
|
|
|
934
1026
|
pulls: this.pulls,
|
|
935
1027
|
});
|
|
936
1028
|
}
|
|
1029
|
+
/**
|
|
1030
|
+
* The StateStore aggregate after a durable append. Its row view is immutable
|
|
1031
|
+
* across later deltas, which lets a full-pass facade rebase without exposing
|
|
1032
|
+
* its completed checkpoint to subsequent scoped writers.
|
|
1033
|
+
*/
|
|
1034
|
+
durableJournalForRebase() {
|
|
1035
|
+
this.assertOpen();
|
|
1036
|
+
if (this.areaMigration)
|
|
1037
|
+
return this.areaMigration.readAreaJournalCached().journal;
|
|
1038
|
+
return this.store.getState().journal;
|
|
1039
|
+
}
|
|
937
1040
|
refresh() {
|
|
938
1041
|
this.assertOpen();
|
|
939
1042
|
if (this.areaMigration) {
|
|
@@ -960,12 +1063,10 @@ export class JournalStore {
|
|
|
960
1063
|
this.applyRefresh(result);
|
|
961
1064
|
}
|
|
962
1065
|
applyRefresh(result) {
|
|
963
|
-
if (result.recovered) {
|
|
1066
|
+
if (result.recovered || result.records.length > 0) {
|
|
964
1067
|
this.replaceFromJournal(this.store.getState().journal);
|
|
965
1068
|
return;
|
|
966
1069
|
}
|
|
967
|
-
for (const record of result.records)
|
|
968
|
-
this.applyRecord(record.type, record.payload);
|
|
969
1070
|
}
|
|
970
1071
|
applyDelta(delta) {
|
|
971
1072
|
this.assertOpen();
|
|
@@ -1025,13 +1126,11 @@ export class JournalStore {
|
|
|
1025
1126
|
this.applyCommitted(appended);
|
|
1026
1127
|
ensureJournalLocator(this.slug);
|
|
1027
1128
|
}
|
|
1028
|
-
applyCommitted(
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
this.applyRecord(record.type, record.payload);
|
|
1034
|
-
this.applyRecord(appended.type, appended.payload);
|
|
1129
|
+
applyCommitted(_appended) {
|
|
1130
|
+
// StateStore has reduced the just-appended durable frame already. Repoint
|
|
1131
|
+
// this cache at that immutable aggregate rather than replaying the frame
|
|
1132
|
+
// into a second mutable overlay beside it.
|
|
1133
|
+
this.replaceFromJournal(this.store.getState().journal);
|
|
1035
1134
|
}
|
|
1036
1135
|
/**
|
|
1037
1136
|
* Packing copies every source row into typed arrays, so this projection never
|
|
@@ -1040,21 +1139,30 @@ export class JournalStore {
|
|
|
1040
1139
|
* pass-local), which is what put the outposts over Node's heap ceiling.
|
|
1041
1140
|
*/
|
|
1042
1141
|
replaceFromJournal(journal) {
|
|
1043
|
-
const
|
|
1142
|
+
const proxied = journalRecordRows(journal.files);
|
|
1143
|
+
const decoded = proxied instanceof JournalRowStore || proxied instanceof JournalRowOverlay
|
|
1144
|
+
? proxied
|
|
1145
|
+
: decodedJournalRows(journal);
|
|
1044
1146
|
if (decoded) {
|
|
1045
|
-
if (!this.usesSharedRows && this.files !== decoded)
|
|
1147
|
+
if (!this.usesSharedRows && this.files instanceof JournalRowStore && this.files !== decoded) {
|
|
1046
1148
|
this.files.clear();
|
|
1149
|
+
}
|
|
1047
1150
|
this.files = decoded;
|
|
1048
1151
|
this.usesSharedRows = true;
|
|
1152
|
+
this.committedOverlay.clear();
|
|
1153
|
+
this.committedMoved.clear();
|
|
1049
1154
|
this.version = journal.version;
|
|
1050
1155
|
this.lastSync = journal.lastSync;
|
|
1051
1156
|
this.pulls = structuredClone(journal.pulls ?? []);
|
|
1052
1157
|
return;
|
|
1053
1158
|
}
|
|
1054
|
-
if (this.usesSharedRows)
|
|
1159
|
+
if (this.usesSharedRows || !(this.files instanceof JournalRowStore)) {
|
|
1055
1160
|
this.files = new JournalRowStore({ trackChanges: false });
|
|
1161
|
+
}
|
|
1056
1162
|
this.files.replaceFromRecord(journal.files);
|
|
1057
1163
|
this.usesSharedRows = false;
|
|
1164
|
+
this.committedOverlay.clear();
|
|
1165
|
+
this.committedMoved.clear();
|
|
1058
1166
|
this.version = journal.version;
|
|
1059
1167
|
this.lastSync = journal.lastSync;
|
|
1060
1168
|
this.pulls = structuredClone(journal.pulls ?? []);
|
|
@@ -1068,9 +1176,15 @@ export class JournalStore {
|
|
|
1068
1176
|
throw new Error("journal: keyed session received an invalid WAL record");
|
|
1069
1177
|
}
|
|
1070
1178
|
for (const key of payload.deletes)
|
|
1071
|
-
this.
|
|
1072
|
-
for (const [key, entry] of Object.entries(payload.upserts))
|
|
1073
|
-
this.
|
|
1179
|
+
this.committedOverlay.set(key, undefined);
|
|
1180
|
+
for (const [key, entry] of Object.entries(payload.upserts)) {
|
|
1181
|
+
if (this.committedOverlay.get(key) === undefined && this.committedOverlay.has(key)) {
|
|
1182
|
+
this.committedOverlay.delete(key);
|
|
1183
|
+
if (this.files.has(key))
|
|
1184
|
+
this.committedMoved.add(key);
|
|
1185
|
+
}
|
|
1186
|
+
this.committedOverlay.set(key, immutableCopy(entry));
|
|
1187
|
+
}
|
|
1074
1188
|
this.version = payload.version;
|
|
1075
1189
|
this.lastSync = payload.lastSync ?? this.lastSync;
|
|
1076
1190
|
this.pulls = structuredClone(payload.pulls ?? this.pulls);
|
|
@@ -1081,7 +1195,9 @@ export class JournalStore {
|
|
|
1081
1195
|
return;
|
|
1082
1196
|
this.disposed = true;
|
|
1083
1197
|
this.overlay.clear();
|
|
1084
|
-
|
|
1198
|
+
this.committedOverlay.clear();
|
|
1199
|
+
this.committedMoved.clear();
|
|
1200
|
+
if (!this.usesSharedRows && this.files instanceof JournalRowStore)
|
|
1085
1201
|
this.files.clear();
|
|
1086
1202
|
this.store = undefined;
|
|
1087
1203
|
this.areaMigration = undefined;
|
|
@@ -1108,6 +1224,69 @@ function immutableCopy(value) {
|
|
|
1108
1224
|
freeze(copy);
|
|
1109
1225
|
return copy;
|
|
1110
1226
|
}
|
|
1227
|
+
// A scoped watcher pass must not recover a 700k-row snapshot just to inspect
|
|
1228
|
+
// one path. A runner can alternate the personal vault and its companies in the
|
|
1229
|
+
// same realtime batch, so one cache entry turns that alternation into a fresh
|
|
1230
|
+
// whole-snapshot recovery on every dispatch. Keep a deliberately small LRU:
|
|
1231
|
+
// enough for the runner's active journal set, bounded so an unusually broad
|
|
1232
|
+
// company fanout cannot retain every journal for the lifetime of the process.
|
|
1233
|
+
const MAX_RETAINED_SCOPED_JOURNALS = 4;
|
|
1234
|
+
const cachedScopedJournalStores = new Map();
|
|
1235
|
+
function scopedJournalStoreCacheKey(slug) {
|
|
1236
|
+
return `${getStateDir()}\u0000${slug}`;
|
|
1237
|
+
}
|
|
1238
|
+
function cachedScopedJournalStoreFor(slug) {
|
|
1239
|
+
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1240
|
+
const cached = cachedScopedJournalStores.get(cacheKey);
|
|
1241
|
+
if (cached === undefined)
|
|
1242
|
+
return undefined;
|
|
1243
|
+
// Map insertion order is our LRU order. A hot scope must not be evicted by
|
|
1244
|
+
// a brief company fanout while the runner is still using it.
|
|
1245
|
+
cachedScopedJournalStores.delete(cacheKey);
|
|
1246
|
+
cachedScopedJournalStores.set(cacheKey, cached);
|
|
1247
|
+
return cached;
|
|
1248
|
+
}
|
|
1249
|
+
function retainScopedJournalStore(slug, store) {
|
|
1250
|
+
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1251
|
+
const previous = cachedScopedJournalStores.get(cacheKey);
|
|
1252
|
+
if (previous !== undefined && previous !== store)
|
|
1253
|
+
previous.dispose();
|
|
1254
|
+
cachedScopedJournalStores.delete(cacheKey);
|
|
1255
|
+
cachedScopedJournalStores.set(cacheKey, store);
|
|
1256
|
+
while (cachedScopedJournalStores.size > MAX_RETAINED_SCOPED_JOURNALS) {
|
|
1257
|
+
const oldest = cachedScopedJournalStores.entries().next().value;
|
|
1258
|
+
if (oldest === undefined)
|
|
1259
|
+
break;
|
|
1260
|
+
const [oldestKey, oldestStore] = oldest;
|
|
1261
|
+
cachedScopedJournalStores.delete(oldestKey);
|
|
1262
|
+
oldestStore.dispose();
|
|
1263
|
+
}
|
|
1264
|
+
return store;
|
|
1265
|
+
}
|
|
1266
|
+
function scopedJournalStore(slug) {
|
|
1267
|
+
const cached = cachedScopedJournalStoreFor(slug);
|
|
1268
|
+
if (cached)
|
|
1269
|
+
return cached;
|
|
1270
|
+
return retainScopedJournalStore(slug, openJournalStoreSession(slug));
|
|
1271
|
+
}
|
|
1272
|
+
/** Drop one legacy keyed projection after an authoritative area cutover. */
|
|
1273
|
+
export function releaseScopedJournalStore(slug) {
|
|
1274
|
+
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1275
|
+
const cached = cachedScopedJournalStores.get(cacheKey);
|
|
1276
|
+
if (cached === undefined)
|
|
1277
|
+
return;
|
|
1278
|
+
cachedScopedJournalStores.delete(cacheKey);
|
|
1279
|
+
cached.dispose();
|
|
1280
|
+
}
|
|
1281
|
+
/**
|
|
1282
|
+
* Release retained scoped projections when a watch runner stops. Durable state
|
|
1283
|
+
* remains in the StateStore; this only drops process-local decoded snapshots.
|
|
1284
|
+
*/
|
|
1285
|
+
export function disposeScopedJournalStores() {
|
|
1286
|
+
for (const store of cachedScopedJournalStores.values())
|
|
1287
|
+
store.dispose();
|
|
1288
|
+
cachedScopedJournalStores.clear();
|
|
1289
|
+
}
|
|
1111
1290
|
/** Open one keyed session, seeding a missing v3 store from legacy JSON once. */
|
|
1112
1291
|
export function openJournalStoreSession(slug) {
|
|
1113
1292
|
const areaMigration = openAreaMigrationIfAuthoritative(slug);
|
|
@@ -1620,6 +1799,41 @@ export function writeJournal(slug, journal) {
|
|
|
1620
1799
|
ensureJournalLocator(slug);
|
|
1621
1800
|
return;
|
|
1622
1801
|
}
|
|
1802
|
+
const baseline = journalBaselines.get(journal);
|
|
1803
|
+
const cached = cachedScopedJournalStoreFor(slug);
|
|
1804
|
+
if (cached && (storeBacking || baseline)) {
|
|
1805
|
+
// The scoped reader captured only its named rows, so its baseline is the
|
|
1806
|
+
// authoritative delta boundary. The retained session refreshes and appends
|
|
1807
|
+
// under the normal StateStore lock; it never exposes partial durable rows
|
|
1808
|
+
// to another writer.
|
|
1809
|
+
const delta = storeBacking
|
|
1810
|
+
? journalDeltaFromStoreDirtyLog(storeBacking, journal)
|
|
1811
|
+
: baseline
|
|
1812
|
+
? journalDeltaFromBaseline(baseline, journal)
|
|
1813
|
+
: undefined;
|
|
1814
|
+
if (delta && hasJournalDelta(delta)) {
|
|
1815
|
+
const currentMetadata = cached.metadata();
|
|
1816
|
+
const metadata = delta.lastSync === undefined && delta.pulls === undefined
|
|
1817
|
+
? undefined
|
|
1818
|
+
: {
|
|
1819
|
+
lastSync: delta.lastSync ?? currentMetadata.lastSync,
|
|
1820
|
+
pulls: delta.pulls ?? currentMetadata.pulls,
|
|
1821
|
+
};
|
|
1822
|
+
cached.applyDelta({
|
|
1823
|
+
upserts: delta.upserts,
|
|
1824
|
+
deletes: delta.deletes,
|
|
1825
|
+
...(metadata === undefined ? {} : { metadata }),
|
|
1826
|
+
});
|
|
1827
|
+
}
|
|
1828
|
+
if (storeBacking) {
|
|
1829
|
+
rebaseStoreBackedJournal(storeBacking, journal, cached.durableJournalForRebase());
|
|
1830
|
+
checkpointStoreBackedJournal(storeBacking, journal);
|
|
1831
|
+
}
|
|
1832
|
+
else
|
|
1833
|
+
rememberJournalBaseline(journal);
|
|
1834
|
+
ensureJournalLocator(slug);
|
|
1835
|
+
return;
|
|
1836
|
+
}
|
|
1623
1837
|
const store = openJournalStore(slug, journal);
|
|
1624
1838
|
if (store.wasCreated) {
|
|
1625
1839
|
// The store was just seeded from this journal, so its state already matches
|
|
@@ -1643,7 +1857,6 @@ export function writeJournal(slug, journal) {
|
|
|
1643
1857
|
// The WAL still compacts on its own schedule (record count or byte ceiling),
|
|
1644
1858
|
// so a full snapshot is written periodically rather than never — recovery
|
|
1645
1859
|
// stays bounded, it just no longer happens on every single update.
|
|
1646
|
-
const baseline = journalBaselines.get(journal);
|
|
1647
1860
|
const delta = storeBacking
|
|
1648
1861
|
? journalDeltaFromStoreDirtyLog(storeBacking, journal)
|
|
1649
1862
|
: baseline
|
|
@@ -1652,8 +1865,10 @@ export function writeJournal(slug, journal) {
|
|
|
1652
1865
|
store.append(JOURNAL_DELTA_RECORD_TYPE, delta);
|
|
1653
1866
|
// A caller can checkpoint more than once during one pass. Advancing only its
|
|
1654
1867
|
// own tracking state means the next checkpoint again publishes only new mutations.
|
|
1655
|
-
if (storeBacking)
|
|
1868
|
+
if (storeBacking) {
|
|
1869
|
+
rebaseStoreBackedJournal(storeBacking, journal, store.getState().journal);
|
|
1656
1870
|
checkpointStoreBackedJournal(storeBacking, journal);
|
|
1871
|
+
}
|
|
1657
1872
|
else
|
|
1658
1873
|
rememberJournalBaseline(journal);
|
|
1659
1874
|
// The locator is immutable content, so it only needs writing if it is absent
|