@indigoai-us/hq-cloud 6.16.52-alpha.0 → 6.16.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +23 -1
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +92 -8
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +36 -0
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +2 -1
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +0 -6
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +1 -30
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +0 -55
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +3 -12
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +3 -65
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +1 -108
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -48
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +116 -311
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -33
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +4 -34
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +79 -802
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-breaker.test.d.ts +2 -0
- package/dist/cli/rescue-breaker.test.d.ts.map +1 -0
- package/dist/cli/rescue-breaker.test.js +334 -0
- package/dist/cli/rescue-breaker.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +40 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +1454 -162
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-realpath.test.d.ts +2 -0
- package/dist/cli/rescue-realpath.test.d.ts.map +1 -0
- package/dist/cli/rescue-realpath.test.js +109 -0
- package/dist/cli/rescue-realpath.test.js.map +1 -0
- package/dist/cli/rescue-rsync-preflight.test.js +256 -4
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-copy.test.js +627 -0
- package/dist/cli/rescue-snapshot-copy.test.js.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.js +260 -0
- package/dist/cli/rescue-snapshot-recovery.test.js.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts +2 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.js +210 -0
- package/dist/cli/rescue-source-switch-floor.test.js.map +1 -0
- package/dist/cli/rescue.d.ts +4 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +20 -1
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +16 -1
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +2 -0
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +0 -19
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +37 -57
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +93 -87
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +0 -8
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +0 -12
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +118 -226
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +2 -215
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +2 -0
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +30 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +175 -0
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +5 -9
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +10 -70
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -11
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +7 -259
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +12 -37
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +76 -291
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +0 -15
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +0 -15
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +295 -0
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +18 -3
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +0 -35
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +2 -9
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +2 -59
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +18 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +47 -24
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +27 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/area-ledger.js +4 -4
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +0 -29
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +6 -14
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +16 -136
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync-core.d.ts +6 -4
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +10 -4
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +25 -1
- package/dist/sync-core.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
- package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner.test.js +0 -266
- package/dist/cli/onepass-pull-planner.test.js.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
- package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
- package/dist/s3.list-cost.test.d.ts +0 -2
- package/dist/s3.list-cost.test.d.ts.map +0 -1
- package/dist/s3.list-cost.test.js +0 -51
- package/dist/s3.list-cost.test.js.map +0 -1
- package/dist/sync/remote-file-spool.d.ts +0 -16
- package/dist/sync/remote-file-spool.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.js +0 -91
- package/dist/sync/remote-file-spool.js.map +0 -1
- package/dist/sync/remote-file-spool.test.d.ts +0 -2
- package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.test.js +0 -77
- package/dist/sync/remote-file-spool.test.js.map +0 -1
package/dist/journal.js
CHANGED
|
@@ -17,7 +17,7 @@ import * as os from "os";
|
|
|
17
17
|
import * as path from "path";
|
|
18
18
|
import * as crypto from "crypto";
|
|
19
19
|
import { JournalRowOverlay, JournalRowStore, journalEntryFingerprint, } from "./journal-row-store.js";
|
|
20
|
-
import { createJournalRecordProxy
|
|
20
|
+
import { createJournalRecordProxy } from "./journal-record-proxy.js";
|
|
21
21
|
import { decodeJournalSnapshot, decodedJournalRows } from "./journal-snapshot-decoder.js";
|
|
22
22
|
import { forgetSharedAreaJournalRows } from "./journal-snapshot-rows.js";
|
|
23
23
|
import { encodeV4JournalSnapshotChunks } from "./journal-snapshot-v4.js";
|
|
@@ -114,9 +114,6 @@ export function journalRows(journal) {
|
|
|
114
114
|
const backing = storeBackedJournals.get(journal);
|
|
115
115
|
if (backing)
|
|
116
116
|
return backing.store;
|
|
117
|
-
const proxied = journalRecordRows(journal.files);
|
|
118
|
-
if (proxied)
|
|
119
|
-
return proxied;
|
|
120
117
|
const decoded = decodedJournalRows(journal);
|
|
121
118
|
if (decoded)
|
|
122
119
|
return decoded.rows;
|
|
@@ -194,8 +191,7 @@ function readStoreBackedJournal(journal, options = {}) {
|
|
|
194
191
|
existing.pullsJson = JSON.stringify(journal.pulls ?? []);
|
|
195
192
|
return journal;
|
|
196
193
|
}
|
|
197
|
-
const
|
|
198
|
-
const decoded = proxied ?? decodedJournalRows(journal);
|
|
194
|
+
const decoded = decodedJournalRows(journal);
|
|
199
195
|
if (decoded) {
|
|
200
196
|
if (options.privateOverlay) {
|
|
201
197
|
const store = new JournalRowOverlay(decoded);
|
|
@@ -211,22 +207,7 @@ function readStoreBackedJournal(journal, options = {}) {
|
|
|
211
207
|
});
|
|
212
208
|
return result;
|
|
213
209
|
}
|
|
214
|
-
|
|
215
|
-
decoded.enableChangeTracking();
|
|
216
|
-
else {
|
|
217
|
-
const store = new JournalRowOverlay(decoded);
|
|
218
|
-
const result = { ...journal, files: createJournalRecordProxy(store) };
|
|
219
|
-
if (Object.hasOwn(journal, "pulls"))
|
|
220
|
-
result.pulls = structuredClone(journal.pulls);
|
|
221
|
-
storeBackedJournals.set(result, {
|
|
222
|
-
store,
|
|
223
|
-
version: result.version,
|
|
224
|
-
lastSync: result.lastSync,
|
|
225
|
-
pullsJson: JSON.stringify(result.pulls ?? []),
|
|
226
|
-
keysNormalized: false,
|
|
227
|
-
});
|
|
228
|
-
return result;
|
|
229
|
-
}
|
|
210
|
+
decoded.enableChangeTracking();
|
|
230
211
|
storeBackedJournals.set(journal, {
|
|
231
212
|
store: decoded,
|
|
232
213
|
version: journal.version,
|
|
@@ -298,27 +279,11 @@ function journalDeltaFromStoreDirtyLog(backing, next) {
|
|
|
298
279
|
}
|
|
299
280
|
function checkpointStoreBackedJournal(backing, journal) {
|
|
300
281
|
backing.store.drainDirtyLog();
|
|
282
|
+
backing.store.clearCommittedChanges?.();
|
|
301
283
|
backing.version = journal.version;
|
|
302
284
|
backing.lastSync = journal.lastSync;
|
|
303
285
|
backing.pullsJson = JSON.stringify(journal.pulls ?? []);
|
|
304
286
|
}
|
|
305
|
-
/**
|
|
306
|
-
* Move one full-pass facade onto the immutable durable revision it just
|
|
307
|
-
* published. Its prior private overlay contains the now-committed rows; merely
|
|
308
|
-
* clearing that overlay would make the caller's journal roll back to the old
|
|
309
|
-
* packed base. The StateStore reducer publishes a fresh overlay for every
|
|
310
|
-
* delta, so later scoped writes replace the aggregate instead of mutating this
|
|
311
|
-
* pass's newly established base.
|
|
312
|
-
*/
|
|
313
|
-
function rebaseStoreBackedJournal(backing, journal, durable) {
|
|
314
|
-
const rows = journalRecordRows(durable.files) ?? decodedJournalRows(durable);
|
|
315
|
-
const store = new JournalRowOverlay(rows ?? new RecordJournalRows(durable.files));
|
|
316
|
-
journal.version = durable.version;
|
|
317
|
-
journal.lastSync = durable.lastSync;
|
|
318
|
-
journal.pulls = structuredClone(durable.pulls ?? []);
|
|
319
|
-
journal.files = createJournalRecordProxy(store);
|
|
320
|
-
backing.store = store;
|
|
321
|
-
}
|
|
322
287
|
function storeBackedJournalNeedsMigration(backing, journal) {
|
|
323
288
|
return journal.version !== JOURNAL_VERSION_CURRENT ||
|
|
324
289
|
!Array.isArray(journal.pulls) ||
|
|
@@ -392,15 +357,16 @@ function entryFingerprint(entry) {
|
|
|
392
357
|
*/
|
|
393
358
|
export function journalDeltaFromBaseline(baseline, next) {
|
|
394
359
|
const upserts = {};
|
|
395
|
-
const
|
|
396
|
-
for (const
|
|
360
|
+
const files = next.files;
|
|
361
|
+
for (const key of Object.keys(files)) {
|
|
362
|
+
const entry = files[key];
|
|
397
363
|
if (baseline.rows.get(key) === entryFingerprint(entry))
|
|
398
364
|
continue;
|
|
399
365
|
upserts[key] = entry;
|
|
400
366
|
}
|
|
401
367
|
const deletes = [];
|
|
402
368
|
for (const key of baseline.rows.keys()) {
|
|
403
|
-
if (!
|
|
369
|
+
if (!Object.hasOwn(files, key))
|
|
404
370
|
deletes.push(key);
|
|
405
371
|
}
|
|
406
372
|
const delta = {
|
|
@@ -544,11 +510,7 @@ export function readJournal(slug) {
|
|
|
544
510
|
// mutable without deep-cloning the aggregate before every pass.
|
|
545
511
|
return readStoreBackedJournal(areaMigration.readAreaJournalCached().journal, { privateOverlay: true });
|
|
546
512
|
}
|
|
547
|
-
|
|
548
|
-
// same immutable packed rows. Each owns only its own overlay, so an
|
|
549
|
-
// uncommitted full pass is never visible to a scoped reader (and a scoped
|
|
550
|
-
// delta cannot dirty the full pass).
|
|
551
|
-
return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration, true), { privateOverlay: true });
|
|
513
|
+
return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration));
|
|
552
514
|
}
|
|
553
515
|
/**
|
|
554
516
|
* Read the mutable outer shell required by a full pull without detaching every
|
|
@@ -559,8 +521,9 @@ export function readJournal(slug) {
|
|
|
559
521
|
* own changes in a private overlay. `writeJournal()` drains that overlay's
|
|
560
522
|
* dirty log, avoiding a full-ledger fingerprint at every download checkpoint.
|
|
561
523
|
*
|
|
562
|
-
* Legacy
|
|
563
|
-
*
|
|
524
|
+
* Legacy journals deliberately keep `readJournal()`'s packed-record boundary.
|
|
525
|
+
* That path has different ownership and dirty-log semantics, and does not make
|
|
526
|
+
* the area-authoritative structured-clone copy that this reader avoids.
|
|
564
527
|
*
|
|
565
528
|
* @internal Full-pull boundary; not a public journal record API.
|
|
566
529
|
*/
|
|
@@ -596,55 +559,49 @@ export function readJournalForPass(slug) {
|
|
|
596
559
|
function readJournalAggregate(slug) {
|
|
597
560
|
return readJournalAggregateWith(slug, openAreaMigrationIfAuthoritative(slug));
|
|
598
561
|
}
|
|
599
|
-
function readJournalAggregateWith(slug, areaMigration
|
|
562
|
+
function readJournalAggregateWith(slug, areaMigration) {
|
|
600
563
|
if (areaMigration)
|
|
601
564
|
return areaMigration.readAreaJournal();
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
// A full pass has already paid to recover this packed snapshot. Retain its
|
|
607
|
-
// projection for the next scoped pass instead of making that pass decode
|
|
608
|
-
// the same vault before it looks up even one named row. Read-only
|
|
609
|
-
// enumeration deliberately does not retain: watcher warm-up may inspect
|
|
610
|
-
// every locator and must not evict the hot personal projection.
|
|
611
|
-
retainScopedJournalStore(slug, new JournalStore(slug, store));
|
|
612
|
-
}
|
|
613
|
-
return store.getState().journal;
|
|
565
|
+
if (StateStore.exists(getStateDir(), journalScopeId(slug))) {
|
|
566
|
+
return openJournalStore(slug, emptyJournal()).getState().journal;
|
|
567
|
+
}
|
|
568
|
+
return openJournalStore(slug, readLegacyJournal(slug)).getState().journal;
|
|
614
569
|
}
|
|
615
570
|
/**
|
|
616
571
|
* Read a mutable, detached snapshot for named rows in a scoped pass.
|
|
617
572
|
*
|
|
618
|
-
* Area-authoritative journals use the ledger's keyed reader
|
|
619
|
-
*
|
|
620
|
-
*
|
|
621
|
-
*
|
|
622
|
-
*
|
|
573
|
+
* Area-authoritative journals use the ledger's keyed reader so the work is
|
|
574
|
+
* proportional to the named keys rather than the full cached aggregate. The
|
|
575
|
+
* legacy store has no keyed persistence surface, so it reads the named keys
|
|
576
|
+
* straight out of the aggregate (O(K)) and, only when delete roots are named,
|
|
577
|
+
* makes one linear pass over the rows with an ancestor-set lookup per row. It
|
|
578
|
+
* must never be rows x roots: a watch-mode batch names one delete root per
|
|
579
|
+
* changed path, and a warm-up catch-up hands the pass ~10k of them against a
|
|
580
|
+
* 70k-row vault journal.
|
|
623
581
|
*/
|
|
624
582
|
export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
|
|
625
583
|
const areaMigration = openAreaMigrationIfAuthoritative(slug);
|
|
626
584
|
if (areaMigration) {
|
|
627
585
|
return rememberJournalBaseline(areaMigration.readAreaJournalScoped(keys, deleteScopeRoots));
|
|
628
586
|
}
|
|
629
|
-
const
|
|
630
|
-
|
|
631
|
-
const metadata = store.metadata();
|
|
587
|
+
const full = readJournalAggregate(slug);
|
|
588
|
+
const fullRows = journalRows(full);
|
|
632
589
|
const files = {};
|
|
633
590
|
const roots = new Set(deleteScopeRoots);
|
|
634
591
|
if (roots.has("")) {
|
|
635
|
-
for (const [key, entry] of
|
|
592
|
+
for (const [key, entry] of fullRows.entries()) {
|
|
636
593
|
files[key] = structuredClone(entry);
|
|
637
594
|
}
|
|
638
595
|
}
|
|
639
596
|
else {
|
|
640
597
|
for (const key of new Set(keys)) {
|
|
641
|
-
const entry =
|
|
598
|
+
const entry = fullRows.get(key);
|
|
642
599
|
if (entry === undefined)
|
|
643
600
|
continue;
|
|
644
601
|
files[key] = structuredClone(entry);
|
|
645
602
|
}
|
|
646
603
|
if (roots.size > 0) {
|
|
647
|
-
for (const [key, entry] of
|
|
604
|
+
for (const [key, entry] of fullRows.entries()) {
|
|
648
605
|
if (Object.hasOwn(files, key))
|
|
649
606
|
continue;
|
|
650
607
|
if (isUnderAnyRoot(key, roots))
|
|
@@ -653,10 +610,10 @@ export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
|
|
|
653
610
|
}
|
|
654
611
|
}
|
|
655
612
|
return rememberJournalBaseline({
|
|
656
|
-
version:
|
|
657
|
-
lastSync:
|
|
613
|
+
version: full.version,
|
|
614
|
+
lastSync: full.lastSync,
|
|
658
615
|
files,
|
|
659
|
-
pulls: structuredClone(
|
|
616
|
+
pulls: structuredClone(full.pulls ?? []),
|
|
660
617
|
});
|
|
661
618
|
}
|
|
662
619
|
/**
|
|
@@ -737,22 +694,9 @@ function journalStoreOptions(slug, initialJournal) {
|
|
|
737
694
|
}
|
|
738
695
|
if (record.type === JOURNAL_DELTA_RECORD_TYPE) {
|
|
739
696
|
const delta = record.payload;
|
|
740
|
-
const
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
// full-pass facade opened before this append. Once a first delta is
|
|
744
|
-
// observed, put durable changes in a separate overlay rather than
|
|
745
|
-
// mutating that shared packed row store.
|
|
746
|
-
const rows = existingRows instanceof JournalRowOverlay
|
|
747
|
-
? existingRows.withCommittedDelta(delta.upserts, delta.deletes)
|
|
748
|
-
: new JournalRowOverlay(existingRows).withCommittedDelta(delta.upserts, delta.deletes);
|
|
749
|
-
state.journal.files = createJournalRecordProxy(rows);
|
|
750
|
-
}
|
|
751
|
-
else {
|
|
752
|
-
for (const key of delta.deletes)
|
|
753
|
-
delete state.journal.files[key];
|
|
754
|
-
Object.assign(state.journal.files, delta.upserts);
|
|
755
|
-
}
|
|
697
|
+
for (const key of delta.deletes)
|
|
698
|
+
delete state.journal.files[key];
|
|
699
|
+
Object.assign(state.journal.files, delta.upserts);
|
|
756
700
|
state.journal.version = delta.version;
|
|
757
701
|
state.journal.lastSync = delta.lastSync ?? state.journal.lastSync;
|
|
758
702
|
state.journal.pulls = delta.pulls ?? state.journal.pulls;
|
|
@@ -791,10 +735,6 @@ function openAreaMigrationIfAuthoritative(slug) {
|
|
|
791
735
|
closeCachedAreaMigration(stateDir, slug);
|
|
792
736
|
return undefined;
|
|
793
737
|
}
|
|
794
|
-
// Area authority replaces the legacy v3 aggregate. Drop any retained legacy
|
|
795
|
-
// projection before a scoped reader can accidentally keep both generations
|
|
796
|
-
// resident (or use the obsolete writer after cutover).
|
|
797
|
-
releaseScopedJournalStore(slug);
|
|
798
738
|
const resolverPath = path.join(areaLayoutDirectory(stateDir, slug), "area-resolver.json");
|
|
799
739
|
let resolverJson;
|
|
800
740
|
try {
|
|
@@ -859,20 +799,19 @@ export function openJournalStateStoreForMigration(slug) {
|
|
|
859
799
|
async function openJournalStoreAsync(slug, initialJournal) {
|
|
860
800
|
return StateStore.openAsync(journalStoreOptions(slug, initialJournal));
|
|
861
801
|
}
|
|
802
|
+
/**
|
|
803
|
+
* Long-lived keyed projection over the v3 snapshot/WAL authority.
|
|
804
|
+
*
|
|
805
|
+
* The first open recovers once. Subsequent refreshes consume only authenticated
|
|
806
|
+
* complete WAL frames, rebuilding the projection solely after a generation
|
|
807
|
+
* publication or invalid-tail fallback.
|
|
808
|
+
*/
|
|
862
809
|
export class JournalStore {
|
|
863
810
|
slug;
|
|
864
811
|
store;
|
|
865
812
|
areaMigration;
|
|
866
813
|
files = new JournalRowStore({ trackChanges: false });
|
|
867
814
|
usesSharedRows = false;
|
|
868
|
-
/**
|
|
869
|
-
* Durable deltas this keyed session has observed after its base snapshot.
|
|
870
|
-
* The base can be shared with a full-pass facade, so publishing a scoped
|
|
871
|
-
* delta here must never mutate it in place.
|
|
872
|
-
*/
|
|
873
|
-
committedOverlay = new Map();
|
|
874
|
-
/** Base keys deleted then reinserted by durable records move to the tail. */
|
|
875
|
-
committedMoved = new Set();
|
|
876
815
|
/**
|
|
877
816
|
* Pass-local rows that have not reached the shared projection yet. A missing
|
|
878
817
|
* value represents a staged delete; every other value is a frozen copy of
|
|
@@ -902,56 +841,27 @@ export class JournalStore {
|
|
|
902
841
|
this.assertOpen();
|
|
903
842
|
if (this.overlay.has(key))
|
|
904
843
|
return this.overlay.get(key);
|
|
905
|
-
if (this.committedOverlay.has(key))
|
|
906
|
-
return this.committedOverlay.get(key);
|
|
907
844
|
return this.files.get(key);
|
|
908
845
|
}
|
|
909
846
|
scanPrefix(prefix) {
|
|
910
847
|
this.assertOpen();
|
|
848
|
+
const rows = new Map(this.files.entriesForPrefix(prefix));
|
|
911
849
|
const normalizedPrefix = prefix.replace(/\/+$/, "");
|
|
912
|
-
const
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
? overlay.get(key)
|
|
922
|
-
: committedOverlay.has(key)
|
|
923
|
-
? committedOverlay.get(key)
|
|
924
|
-
: entry;
|
|
925
|
-
if (replacement !== undefined)
|
|
926
|
-
yield [key, replacement];
|
|
927
|
-
}
|
|
928
|
-
// A Map replacement preserves the base row's position and appends newly
|
|
929
|
-
// introduced overlay rows. Preserve that observable order without holding
|
|
930
|
-
// a materialized JournalEntry for every base row at once.
|
|
931
|
-
const additions = new Map(committedOverlay);
|
|
932
|
-
for (const [key, entry] of overlay)
|
|
933
|
-
additions.set(key, entry);
|
|
934
|
-
for (const [key, entry] of additions) {
|
|
935
|
-
if (entry === undefined || (files.has(key) && !committedMoved.has(key)))
|
|
936
|
-
continue;
|
|
937
|
-
if (isJournalStorePrefixMatch(key, normalizedPrefix))
|
|
938
|
-
yield [key, entry];
|
|
939
|
-
}
|
|
940
|
-
};
|
|
941
|
-
const scan = {
|
|
942
|
-
[Symbol.iterator]: iterate,
|
|
943
|
-
filter: ((predicate, thisArg) => Array.from(iterate()).filter(predicate, thisArg)),
|
|
944
|
-
};
|
|
945
|
-
return scan;
|
|
850
|
+
for (const [key, entry] of this.overlay) {
|
|
851
|
+
if (!isJournalStorePrefixMatch(key, normalizedPrefix))
|
|
852
|
+
continue;
|
|
853
|
+
if (entry === undefined)
|
|
854
|
+
rows.delete(key);
|
|
855
|
+
else
|
|
856
|
+
rows.set(key, entry);
|
|
857
|
+
}
|
|
858
|
+
return Array.from(rows.entries());
|
|
946
859
|
}
|
|
947
860
|
rowCount() {
|
|
948
861
|
if (this.disposed)
|
|
949
862
|
return 0;
|
|
950
|
-
const replacements = new Map(this.committedOverlay);
|
|
951
|
-
for (const [key, entry] of this.overlay)
|
|
952
|
-
replacements.set(key, entry);
|
|
953
863
|
let count = this.files.size;
|
|
954
|
-
for (const [key, entry] of
|
|
864
|
+
for (const [key, entry] of this.overlay) {
|
|
955
865
|
const existsInSharedStore = this.files.has(key);
|
|
956
866
|
if (entry === undefined) {
|
|
957
867
|
if (existsInSharedStore)
|
|
@@ -967,9 +877,7 @@ export class JournalStore {
|
|
|
967
877
|
set(key, entry) {
|
|
968
878
|
this.assertOpen();
|
|
969
879
|
const ownedEntry = immutableCopy(entry);
|
|
970
|
-
const sharedEntry = this.
|
|
971
|
-
? this.committedOverlay.get(key)
|
|
972
|
-
: this.files.get(key);
|
|
880
|
+
const sharedEntry = this.files.get(key);
|
|
973
881
|
if (sharedEntry !== undefined && sameEntry(sharedEntry, ownedEntry)) {
|
|
974
882
|
this.overlay.delete(key);
|
|
975
883
|
return;
|
|
@@ -982,12 +890,12 @@ export class JournalStore {
|
|
|
982
890
|
if (this.overlay.has(key)) {
|
|
983
891
|
if (this.overlay.get(key) === undefined)
|
|
984
892
|
return false;
|
|
985
|
-
if (!this.
|
|
893
|
+
if (!this.files.has(key)) {
|
|
986
894
|
this.overlay.delete(key);
|
|
987
895
|
return true;
|
|
988
896
|
}
|
|
989
897
|
}
|
|
990
|
-
else if (this.
|
|
898
|
+
else if (!this.files.has(key)) {
|
|
991
899
|
return false;
|
|
992
900
|
}
|
|
993
901
|
this.overlay.set(key, undefined);
|
|
@@ -1026,17 +934,6 @@ export class JournalStore {
|
|
|
1026
934
|
pulls: this.pulls,
|
|
1027
935
|
});
|
|
1028
936
|
}
|
|
1029
|
-
/**
|
|
1030
|
-
* The StateStore aggregate after a durable append. Its row view is immutable
|
|
1031
|
-
* across later deltas, which lets a full-pass facade rebase without exposing
|
|
1032
|
-
* its completed checkpoint to subsequent scoped writers.
|
|
1033
|
-
*/
|
|
1034
|
-
durableJournalForRebase() {
|
|
1035
|
-
this.assertOpen();
|
|
1036
|
-
if (this.areaMigration)
|
|
1037
|
-
return this.areaMigration.readAreaJournalCached().journal;
|
|
1038
|
-
return this.store.getState().journal;
|
|
1039
|
-
}
|
|
1040
937
|
refresh() {
|
|
1041
938
|
this.assertOpen();
|
|
1042
939
|
if (this.areaMigration) {
|
|
@@ -1063,10 +960,12 @@ export class JournalStore {
|
|
|
1063
960
|
this.applyRefresh(result);
|
|
1064
961
|
}
|
|
1065
962
|
applyRefresh(result) {
|
|
1066
|
-
if (result.recovered
|
|
963
|
+
if (result.recovered) {
|
|
1067
964
|
this.replaceFromJournal(this.store.getState().journal);
|
|
1068
965
|
return;
|
|
1069
966
|
}
|
|
967
|
+
for (const record of result.records)
|
|
968
|
+
this.applyRecord(record.type, record.payload);
|
|
1070
969
|
}
|
|
1071
970
|
applyDelta(delta) {
|
|
1072
971
|
this.assertOpen();
|
|
@@ -1126,11 +1025,13 @@ export class JournalStore {
|
|
|
1126
1025
|
this.applyCommitted(appended);
|
|
1127
1026
|
ensureJournalLocator(this.slug);
|
|
1128
1027
|
}
|
|
1129
|
-
applyCommitted(
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1028
|
+
applyCommitted(appended) {
|
|
1029
|
+
if (appended.refresh.recovered)
|
|
1030
|
+
this.replaceFromJournal(this.store.getState().journal);
|
|
1031
|
+
else
|
|
1032
|
+
for (const record of appended.refresh.records)
|
|
1033
|
+
this.applyRecord(record.type, record.payload);
|
|
1034
|
+
this.applyRecord(appended.type, appended.payload);
|
|
1134
1035
|
}
|
|
1135
1036
|
/**
|
|
1136
1037
|
* Packing copies every source row into typed arrays, so this projection never
|
|
@@ -1139,30 +1040,21 @@ export class JournalStore {
|
|
|
1139
1040
|
* pass-local), which is what put the outposts over Node's heap ceiling.
|
|
1140
1041
|
*/
|
|
1141
1042
|
replaceFromJournal(journal) {
|
|
1142
|
-
const
|
|
1143
|
-
const decoded = proxied instanceof JournalRowStore || proxied instanceof JournalRowOverlay
|
|
1144
|
-
? proxied
|
|
1145
|
-
: decodedJournalRows(journal);
|
|
1043
|
+
const decoded = decodedJournalRows(journal);
|
|
1146
1044
|
if (decoded) {
|
|
1147
|
-
if (!this.usesSharedRows && this.files
|
|
1045
|
+
if (!this.usesSharedRows && this.files !== decoded)
|
|
1148
1046
|
this.files.clear();
|
|
1149
|
-
}
|
|
1150
1047
|
this.files = decoded;
|
|
1151
1048
|
this.usesSharedRows = true;
|
|
1152
|
-
this.committedOverlay.clear();
|
|
1153
|
-
this.committedMoved.clear();
|
|
1154
1049
|
this.version = journal.version;
|
|
1155
1050
|
this.lastSync = journal.lastSync;
|
|
1156
1051
|
this.pulls = structuredClone(journal.pulls ?? []);
|
|
1157
1052
|
return;
|
|
1158
1053
|
}
|
|
1159
|
-
if (this.usesSharedRows
|
|
1054
|
+
if (this.usesSharedRows)
|
|
1160
1055
|
this.files = new JournalRowStore({ trackChanges: false });
|
|
1161
|
-
}
|
|
1162
1056
|
this.files.replaceFromRecord(journal.files);
|
|
1163
1057
|
this.usesSharedRows = false;
|
|
1164
|
-
this.committedOverlay.clear();
|
|
1165
|
-
this.committedMoved.clear();
|
|
1166
1058
|
this.version = journal.version;
|
|
1167
1059
|
this.lastSync = journal.lastSync;
|
|
1168
1060
|
this.pulls = structuredClone(journal.pulls ?? []);
|
|
@@ -1176,15 +1068,9 @@ export class JournalStore {
|
|
|
1176
1068
|
throw new Error("journal: keyed session received an invalid WAL record");
|
|
1177
1069
|
}
|
|
1178
1070
|
for (const key of payload.deletes)
|
|
1179
|
-
this.
|
|
1180
|
-
for (const [key, entry] of Object.entries(payload.upserts))
|
|
1181
|
-
|
|
1182
|
-
this.committedOverlay.delete(key);
|
|
1183
|
-
if (this.files.has(key))
|
|
1184
|
-
this.committedMoved.add(key);
|
|
1185
|
-
}
|
|
1186
|
-
this.committedOverlay.set(key, immutableCopy(entry));
|
|
1187
|
-
}
|
|
1071
|
+
this.files.delete(key);
|
|
1072
|
+
for (const [key, entry] of Object.entries(payload.upserts))
|
|
1073
|
+
this.files.set(key, entry);
|
|
1188
1074
|
this.version = payload.version;
|
|
1189
1075
|
this.lastSync = payload.lastSync ?? this.lastSync;
|
|
1190
1076
|
this.pulls = structuredClone(payload.pulls ?? this.pulls);
|
|
@@ -1195,9 +1081,7 @@ export class JournalStore {
|
|
|
1195
1081
|
return;
|
|
1196
1082
|
this.disposed = true;
|
|
1197
1083
|
this.overlay.clear();
|
|
1198
|
-
this.
|
|
1199
|
-
this.committedMoved.clear();
|
|
1200
|
-
if (!this.usesSharedRows && this.files instanceof JournalRowStore)
|
|
1084
|
+
if (!this.usesSharedRows)
|
|
1201
1085
|
this.files.clear();
|
|
1202
1086
|
this.store = undefined;
|
|
1203
1087
|
this.areaMigration = undefined;
|
|
@@ -1224,69 +1108,6 @@ function immutableCopy(value) {
|
|
|
1224
1108
|
freeze(copy);
|
|
1225
1109
|
return copy;
|
|
1226
1110
|
}
|
|
1227
|
-
// A scoped watcher pass must not recover a 700k-row snapshot just to inspect
|
|
1228
|
-
// one path. A runner can alternate the personal vault and its companies in the
|
|
1229
|
-
// same realtime batch, so one cache entry turns that alternation into a fresh
|
|
1230
|
-
// whole-snapshot recovery on every dispatch. Keep a deliberately small LRU:
|
|
1231
|
-
// enough for the runner's active journal set, bounded so an unusually broad
|
|
1232
|
-
// company fanout cannot retain every journal for the lifetime of the process.
|
|
1233
|
-
const MAX_RETAINED_SCOPED_JOURNALS = 4;
|
|
1234
|
-
const cachedScopedJournalStores = new Map();
|
|
1235
|
-
function scopedJournalStoreCacheKey(slug) {
|
|
1236
|
-
return `${getStateDir()}\u0000${slug}`;
|
|
1237
|
-
}
|
|
1238
|
-
function cachedScopedJournalStoreFor(slug) {
|
|
1239
|
-
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1240
|
-
const cached = cachedScopedJournalStores.get(cacheKey);
|
|
1241
|
-
if (cached === undefined)
|
|
1242
|
-
return undefined;
|
|
1243
|
-
// Map insertion order is our LRU order. A hot scope must not be evicted by
|
|
1244
|
-
// a brief company fanout while the runner is still using it.
|
|
1245
|
-
cachedScopedJournalStores.delete(cacheKey);
|
|
1246
|
-
cachedScopedJournalStores.set(cacheKey, cached);
|
|
1247
|
-
return cached;
|
|
1248
|
-
}
|
|
1249
|
-
function retainScopedJournalStore(slug, store) {
|
|
1250
|
-
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1251
|
-
const previous = cachedScopedJournalStores.get(cacheKey);
|
|
1252
|
-
if (previous !== undefined && previous !== store)
|
|
1253
|
-
previous.dispose();
|
|
1254
|
-
cachedScopedJournalStores.delete(cacheKey);
|
|
1255
|
-
cachedScopedJournalStores.set(cacheKey, store);
|
|
1256
|
-
while (cachedScopedJournalStores.size > MAX_RETAINED_SCOPED_JOURNALS) {
|
|
1257
|
-
const oldest = cachedScopedJournalStores.entries().next().value;
|
|
1258
|
-
if (oldest === undefined)
|
|
1259
|
-
break;
|
|
1260
|
-
const [oldestKey, oldestStore] = oldest;
|
|
1261
|
-
cachedScopedJournalStores.delete(oldestKey);
|
|
1262
|
-
oldestStore.dispose();
|
|
1263
|
-
}
|
|
1264
|
-
return store;
|
|
1265
|
-
}
|
|
1266
|
-
function scopedJournalStore(slug) {
|
|
1267
|
-
const cached = cachedScopedJournalStoreFor(slug);
|
|
1268
|
-
if (cached)
|
|
1269
|
-
return cached;
|
|
1270
|
-
return retainScopedJournalStore(slug, openJournalStoreSession(slug));
|
|
1271
|
-
}
|
|
1272
|
-
/** Drop one legacy keyed projection after an authoritative area cutover. */
|
|
1273
|
-
export function releaseScopedJournalStore(slug) {
|
|
1274
|
-
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1275
|
-
const cached = cachedScopedJournalStores.get(cacheKey);
|
|
1276
|
-
if (cached === undefined)
|
|
1277
|
-
return;
|
|
1278
|
-
cachedScopedJournalStores.delete(cacheKey);
|
|
1279
|
-
cached.dispose();
|
|
1280
|
-
}
|
|
1281
|
-
/**
|
|
1282
|
-
* Release retained scoped projections when a watch runner stops. Durable state
|
|
1283
|
-
* remains in the StateStore; this only drops process-local decoded snapshots.
|
|
1284
|
-
*/
|
|
1285
|
-
export function disposeScopedJournalStores() {
|
|
1286
|
-
for (const store of cachedScopedJournalStores.values())
|
|
1287
|
-
store.dispose();
|
|
1288
|
-
cachedScopedJournalStores.clear();
|
|
1289
|
-
}
|
|
1290
1111
|
/** Open one keyed session, seeding a missing v3 store from legacy JSON once. */
|
|
1291
1112
|
export function openJournalStoreSession(slug) {
|
|
1292
1113
|
const areaMigration = openAreaMigrationIfAuthoritative(slug);
|
|
@@ -1799,41 +1620,6 @@ export function writeJournal(slug, journal) {
|
|
|
1799
1620
|
ensureJournalLocator(slug);
|
|
1800
1621
|
return;
|
|
1801
1622
|
}
|
|
1802
|
-
const baseline = journalBaselines.get(journal);
|
|
1803
|
-
const cached = cachedScopedJournalStoreFor(slug);
|
|
1804
|
-
if (cached && (storeBacking || baseline)) {
|
|
1805
|
-
// The scoped reader captured only its named rows, so its baseline is the
|
|
1806
|
-
// authoritative delta boundary. The retained session refreshes and appends
|
|
1807
|
-
// under the normal StateStore lock; it never exposes partial durable rows
|
|
1808
|
-
// to another writer.
|
|
1809
|
-
const delta = storeBacking
|
|
1810
|
-
? journalDeltaFromStoreDirtyLog(storeBacking, journal)
|
|
1811
|
-
: baseline
|
|
1812
|
-
? journalDeltaFromBaseline(baseline, journal)
|
|
1813
|
-
: undefined;
|
|
1814
|
-
if (delta && hasJournalDelta(delta)) {
|
|
1815
|
-
const currentMetadata = cached.metadata();
|
|
1816
|
-
const metadata = delta.lastSync === undefined && delta.pulls === undefined
|
|
1817
|
-
? undefined
|
|
1818
|
-
: {
|
|
1819
|
-
lastSync: delta.lastSync ?? currentMetadata.lastSync,
|
|
1820
|
-
pulls: delta.pulls ?? currentMetadata.pulls,
|
|
1821
|
-
};
|
|
1822
|
-
cached.applyDelta({
|
|
1823
|
-
upserts: delta.upserts,
|
|
1824
|
-
deletes: delta.deletes,
|
|
1825
|
-
...(metadata === undefined ? {} : { metadata }),
|
|
1826
|
-
});
|
|
1827
|
-
}
|
|
1828
|
-
if (storeBacking) {
|
|
1829
|
-
rebaseStoreBackedJournal(storeBacking, journal, cached.durableJournalForRebase());
|
|
1830
|
-
checkpointStoreBackedJournal(storeBacking, journal);
|
|
1831
|
-
}
|
|
1832
|
-
else
|
|
1833
|
-
rememberJournalBaseline(journal);
|
|
1834
|
-
ensureJournalLocator(slug);
|
|
1835
|
-
return;
|
|
1836
|
-
}
|
|
1837
1623
|
const store = openJournalStore(slug, journal);
|
|
1838
1624
|
if (store.wasCreated) {
|
|
1839
1625
|
// The store was just seeded from this journal, so its state already matches
|
|
@@ -1857,6 +1643,7 @@ export function writeJournal(slug, journal) {
|
|
|
1857
1643
|
// The WAL still compacts on its own schedule (record count or byte ceiling),
|
|
1858
1644
|
// so a full snapshot is written periodically rather than never — recovery
|
|
1859
1645
|
// stays bounded, it just no longer happens on every single update.
|
|
1646
|
+
const baseline = journalBaselines.get(journal);
|
|
1860
1647
|
const delta = storeBacking
|
|
1861
1648
|
? journalDeltaFromStoreDirtyLog(storeBacking, journal)
|
|
1862
1649
|
: baseline
|
|
@@ -1865,10 +1652,8 @@ export function writeJournal(slug, journal) {
|
|
|
1865
1652
|
store.append(JOURNAL_DELTA_RECORD_TYPE, delta);
|
|
1866
1653
|
// A caller can checkpoint more than once during one pass. Advancing only its
|
|
1867
1654
|
// own tracking state means the next checkpoint again publishes only new mutations.
|
|
1868
|
-
if (storeBacking)
|
|
1869
|
-
rebaseStoreBackedJournal(storeBacking, journal, store.getState().journal);
|
|
1655
|
+
if (storeBacking)
|
|
1870
1656
|
checkpointStoreBackedJournal(storeBacking, journal);
|
|
1871
|
-
}
|
|
1872
1657
|
else
|
|
1873
1658
|
rememberJournalBaseline(journal);
|
|
1874
1659
|
// The locator is immutable content, so it only needs writing if it is absent
|