@indigoai-us/hq-cloud 6.16.58-alpha.3 → 6.16.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +0 -14
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +89 -57
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +313 -56
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +3 -12
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +3 -65
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +1 -172
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -51
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +165 -409
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +2 -19
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +5 -40
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +0 -47
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +13 -59
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +4 -48
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +229 -1499
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +0 -19
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +0 -55
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +1 -87
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +0 -8
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +8 -12
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +152 -302
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +126 -256
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +34 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +92 -21
- package/dist/cognito-auth.js.map +1 -1
- package/dist/context.d.ts +5 -0
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +92 -12
- package/dist/context.js.map +1 -1
- package/dist/context.test.js +77 -2
- package/dist/context.test.js.map +1 -1
- package/dist/ignore.d.ts +10 -0
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +47 -5
- package/dist/ignore.js.map +1 -1
- package/dist/ignore.test.js +30 -3
- package/dist/ignore.test.js.map +1 -1
- package/dist/journal-row-store.d.ts +5 -40
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +21 -240
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +0 -54
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -11
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +7 -324
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +16 -51
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +79 -303
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +0 -15
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +0 -15
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/lib/net-errors.d.ts +12 -0
- package/dist/lib/net-errors.d.ts.map +1 -1
- package/dist/lib/net-errors.js +35 -0
- package/dist/lib/net-errors.js.map +1 -1
- package/dist/lib/net-errors.test.js +19 -1
- package/dist/lib/net-errors.test.js.map +1 -1
- package/dist/machine-auth.test.js +6 -4
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/machine-token-cache-isolation.test.d.ts +24 -0
- package/dist/machine-token-cache-isolation.test.d.ts.map +1 -0
- package/dist/machine-token-cache-isolation.test.js +241 -0
- package/dist/machine-token-cache-isolation.test.js.map +1 -0
- package/dist/manifest/build-manifest-resident-parity.test.js +3 -3
- package/dist/manifest/build-manifest-resident-parity.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +0 -35
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +2 -9
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +2 -59
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +0 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +2 -20
- package/dist/s3.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +3 -27
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +39 -149
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +5 -192
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +0 -29
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +6 -14
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +16 -136
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/state-store.d.ts +0 -6
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +2 -14
- package/dist/sync/state-store.js.map +1 -1
- package/dist/watcher.d.ts +0 -12
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +0 -131
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +0 -63
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
- package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner.test.js +0 -266
- package/dist/cli/onepass-pull-planner.test.js.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
- package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
- package/dist/s3.list-cost.test.d.ts +0 -2
- package/dist/s3.list-cost.test.d.ts.map +0 -1
- package/dist/s3.list-cost.test.js +0 -51
- package/dist/s3.list-cost.test.js.map +0 -1
- package/dist/sync/remote-file-spool.d.ts +0 -16
- package/dist/sync/remote-file-spool.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.js +0 -91
- package/dist/sync/remote-file-spool.js.map +0 -1
- package/dist/sync/remote-file-spool.test.d.ts +0 -2
- package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.test.js +0 -77
- package/dist/sync/remote-file-spool.test.js.map +0 -1
package/dist/journal.js
CHANGED
|
@@ -17,7 +17,7 @@ import * as os from "os";
|
|
|
17
17
|
import * as path from "path";
|
|
18
18
|
import * as crypto from "crypto";
|
|
19
19
|
import { JournalRowOverlay, JournalRowStore, journalEntryFingerprint, } from "./journal-row-store.js";
|
|
20
|
-
import { createJournalRecordProxy
|
|
20
|
+
import { createJournalRecordProxy } from "./journal-record-proxy.js";
|
|
21
21
|
import { decodeJournalSnapshot, decodedJournalRows } from "./journal-snapshot-decoder.js";
|
|
22
22
|
import { forgetSharedAreaJournalRows } from "./journal-snapshot-rows.js";
|
|
23
23
|
import { encodeV4JournalSnapshotChunks } from "./journal-snapshot-v4.js";
|
|
@@ -114,9 +114,6 @@ export function journalRows(journal) {
|
|
|
114
114
|
const backing = storeBackedJournals.get(journal);
|
|
115
115
|
if (backing)
|
|
116
116
|
return backing.store;
|
|
117
|
-
const proxied = journalRecordRows(journal.files);
|
|
118
|
-
if (proxied)
|
|
119
|
-
return proxied;
|
|
120
117
|
const decoded = decodedJournalRows(journal);
|
|
121
118
|
if (decoded)
|
|
122
119
|
return decoded.rows;
|
|
@@ -149,12 +146,9 @@ export function closeCachedAreaJournalMigration(slug) {
|
|
|
149
146
|
closeCachedAreaMigration(getStateDir(), slug);
|
|
150
147
|
}
|
|
151
148
|
/**
|
|
152
|
-
* Release all process-local area bridges
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
* through ordinary watch passes so a later scoped callback cannot re-pack the
|
|
156
|
-
* whole journal synchronously. Durable layout/phase changes use the narrower
|
|
157
|
-
* close helper above; shutdown is the terminal memory boundary.
|
|
149
|
+
* Release all process-local area bridges after a completed runner pass.
|
|
150
|
+
* They hold lazy StateStore/aggregate caches but no file descriptors; durable
|
|
151
|
+
* state is authoritative, so a later pass can safely construct a fresh bridge.
|
|
158
152
|
*/
|
|
159
153
|
export function releaseCachedAreaJournalMigrations() {
|
|
160
154
|
for (const [cacheKey, cached] of cachedAreaMigrations) {
|
|
@@ -197,8 +191,7 @@ function readStoreBackedJournal(journal, options = {}) {
|
|
|
197
191
|
existing.pullsJson = JSON.stringify(journal.pulls ?? []);
|
|
198
192
|
return journal;
|
|
199
193
|
}
|
|
200
|
-
const
|
|
201
|
-
const decoded = proxied ?? decodedJournalRows(journal);
|
|
194
|
+
const decoded = decodedJournalRows(journal);
|
|
202
195
|
if (decoded) {
|
|
203
196
|
if (options.privateOverlay) {
|
|
204
197
|
const store = new JournalRowOverlay(decoded);
|
|
@@ -214,22 +207,7 @@ function readStoreBackedJournal(journal, options = {}) {
|
|
|
214
207
|
});
|
|
215
208
|
return result;
|
|
216
209
|
}
|
|
217
|
-
|
|
218
|
-
decoded.enableChangeTracking();
|
|
219
|
-
else {
|
|
220
|
-
const store = new JournalRowOverlay(decoded);
|
|
221
|
-
const result = { ...journal, files: createJournalRecordProxy(store) };
|
|
222
|
-
if (Object.hasOwn(journal, "pulls"))
|
|
223
|
-
result.pulls = structuredClone(journal.pulls);
|
|
224
|
-
storeBackedJournals.set(result, {
|
|
225
|
-
store,
|
|
226
|
-
version: result.version,
|
|
227
|
-
lastSync: result.lastSync,
|
|
228
|
-
pullsJson: JSON.stringify(result.pulls ?? []),
|
|
229
|
-
keysNormalized: false,
|
|
230
|
-
});
|
|
231
|
-
return result;
|
|
232
|
-
}
|
|
210
|
+
decoded.enableChangeTracking();
|
|
233
211
|
storeBackedJournals.set(journal, {
|
|
234
212
|
store: decoded,
|
|
235
213
|
version: journal.version,
|
|
@@ -301,27 +279,11 @@ function journalDeltaFromStoreDirtyLog(backing, next) {
|
|
|
301
279
|
}
|
|
302
280
|
function checkpointStoreBackedJournal(backing, journal) {
|
|
303
281
|
backing.store.drainDirtyLog();
|
|
282
|
+
backing.store.clearCommittedChanges?.();
|
|
304
283
|
backing.version = journal.version;
|
|
305
284
|
backing.lastSync = journal.lastSync;
|
|
306
285
|
backing.pullsJson = JSON.stringify(journal.pulls ?? []);
|
|
307
286
|
}
|
|
308
|
-
/**
|
|
309
|
-
* Move one full-pass facade onto the immutable durable revision it just
|
|
310
|
-
* published. Its prior private overlay contains the now-committed rows; merely
|
|
311
|
-
* clearing that overlay would make the caller's journal roll back to the old
|
|
312
|
-
* packed base. The StateStore reducer publishes a fresh overlay for every
|
|
313
|
-
* delta, so later scoped writes replace the aggregate instead of mutating this
|
|
314
|
-
* pass's newly established base.
|
|
315
|
-
*/
|
|
316
|
-
function rebaseStoreBackedJournal(backing, journal, durable) {
|
|
317
|
-
const rows = journalRecordRows(durable.files) ?? decodedJournalRows(durable);
|
|
318
|
-
const store = new JournalRowOverlay(rows ?? new RecordJournalRows(durable.files));
|
|
319
|
-
journal.version = durable.version;
|
|
320
|
-
journal.lastSync = durable.lastSync;
|
|
321
|
-
journal.pulls = structuredClone(durable.pulls ?? []);
|
|
322
|
-
journal.files = createJournalRecordProxy(store);
|
|
323
|
-
backing.store = store;
|
|
324
|
-
}
|
|
325
287
|
function storeBackedJournalNeedsMigration(backing, journal) {
|
|
326
288
|
return journal.version !== JOURNAL_VERSION_CURRENT ||
|
|
327
289
|
!Array.isArray(journal.pulls) ||
|
|
@@ -395,15 +357,16 @@ function entryFingerprint(entry) {
|
|
|
395
357
|
*/
|
|
396
358
|
export function journalDeltaFromBaseline(baseline, next) {
|
|
397
359
|
const upserts = {};
|
|
398
|
-
const
|
|
399
|
-
for (const
|
|
360
|
+
const files = next.files;
|
|
361
|
+
for (const key of Object.keys(files)) {
|
|
362
|
+
const entry = files[key];
|
|
400
363
|
if (baseline.rows.get(key) === entryFingerprint(entry))
|
|
401
364
|
continue;
|
|
402
365
|
upserts[key] = entry;
|
|
403
366
|
}
|
|
404
367
|
const deletes = [];
|
|
405
368
|
for (const key of baseline.rows.keys()) {
|
|
406
|
-
if (!
|
|
369
|
+
if (!Object.hasOwn(files, key))
|
|
407
370
|
deletes.push(key);
|
|
408
371
|
}
|
|
409
372
|
const delta = {
|
|
@@ -547,11 +510,7 @@ export function readJournal(slug) {
|
|
|
547
510
|
// mutable without deep-cloning the aggregate before every pass.
|
|
548
511
|
return readStoreBackedJournal(areaMigration.readAreaJournalCached().journal, { privateOverlay: true });
|
|
549
512
|
}
|
|
550
|
-
|
|
551
|
-
// same immutable packed rows. Each owns only its own overlay, so an
|
|
552
|
-
// uncommitted full pass is never visible to a scoped reader (and a scoped
|
|
553
|
-
// delta cannot dirty the full pass).
|
|
554
|
-
return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration, true), { privateOverlay: true });
|
|
513
|
+
return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration));
|
|
555
514
|
}
|
|
556
515
|
/**
|
|
557
516
|
* Read the mutable outer shell required by a full pull without detaching every
|
|
@@ -562,8 +521,9 @@ export function readJournal(slug) {
|
|
|
562
521
|
* own changes in a private overlay. `writeJournal()` drains that overlay's
|
|
563
522
|
* dirty log, avoiding a full-ledger fingerprint at every download checkpoint.
|
|
564
523
|
*
|
|
565
|
-
* Legacy
|
|
566
|
-
*
|
|
524
|
+
* Legacy journals deliberately keep `readJournal()`'s packed-record boundary.
|
|
525
|
+
* That path has different ownership and dirty-log semantics, and does not make
|
|
526
|
+
* the area-authoritative structured-clone copy that this reader avoids.
|
|
567
527
|
*
|
|
568
528
|
* @internal Full-pull boundary; not a public journal record API.
|
|
569
529
|
*/
|
|
@@ -599,55 +559,49 @@ export function readJournalForPass(slug) {
|
|
|
599
559
|
function readJournalAggregate(slug) {
|
|
600
560
|
return readJournalAggregateWith(slug, openAreaMigrationIfAuthoritative(slug));
|
|
601
561
|
}
|
|
602
|
-
function readJournalAggregateWith(slug, areaMigration
|
|
562
|
+
function readJournalAggregateWith(slug, areaMigration) {
|
|
603
563
|
if (areaMigration)
|
|
604
564
|
return areaMigration.readAreaJournal();
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
// A full pass has already paid to recover this packed snapshot. Retain its
|
|
610
|
-
// projection for the next scoped pass instead of making that pass decode
|
|
611
|
-
// the same vault before it looks up even one named row. Read-only
|
|
612
|
-
// enumeration deliberately does not retain: watcher warm-up may inspect
|
|
613
|
-
// every locator and must not evict the hot personal projection.
|
|
614
|
-
retainScopedJournalStore(slug, new JournalStore(slug, store));
|
|
615
|
-
}
|
|
616
|
-
return store.getState().journal;
|
|
565
|
+
if (StateStore.exists(getStateDir(), journalScopeId(slug))) {
|
|
566
|
+
return openJournalStore(slug, emptyJournal()).getState().journal;
|
|
567
|
+
}
|
|
568
|
+
return openJournalStore(slug, readLegacyJournal(slug)).getState().journal;
|
|
617
569
|
}
|
|
618
570
|
/**
|
|
619
571
|
* Read a mutable, detached snapshot for named rows in a scoped pass.
|
|
620
572
|
*
|
|
621
|
-
* Area-authoritative journals use the ledger's keyed reader
|
|
622
|
-
*
|
|
623
|
-
*
|
|
624
|
-
*
|
|
625
|
-
*
|
|
573
|
+
* Area-authoritative journals use the ledger's keyed reader so the work is
|
|
574
|
+
* proportional to the named keys rather than the full cached aggregate. The
|
|
575
|
+
* legacy store has no keyed persistence surface, so it reads the named keys
|
|
576
|
+
* straight out of the aggregate (O(K)) and, only when delete roots are named,
|
|
577
|
+
* makes one linear pass over the rows with an ancestor-set lookup per row. It
|
|
578
|
+
* must never be rows x roots: a watch-mode batch names one delete root per
|
|
579
|
+
* changed path, and a warm-up catch-up hands the pass ~10k of them against a
|
|
580
|
+
* 70k-row vault journal.
|
|
626
581
|
*/
|
|
627
582
|
export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
|
|
628
583
|
const areaMigration = openAreaMigrationIfAuthoritative(slug);
|
|
629
584
|
if (areaMigration) {
|
|
630
585
|
return rememberJournalBaseline(areaMigration.readAreaJournalScoped(keys, deleteScopeRoots));
|
|
631
586
|
}
|
|
632
|
-
const
|
|
633
|
-
|
|
634
|
-
const metadata = store.metadata();
|
|
587
|
+
const full = readJournalAggregate(slug);
|
|
588
|
+
const fullRows = journalRows(full);
|
|
635
589
|
const files = {};
|
|
636
590
|
const roots = new Set(deleteScopeRoots);
|
|
637
591
|
if (roots.has("")) {
|
|
638
|
-
for (const [key, entry] of
|
|
592
|
+
for (const [key, entry] of fullRows.entries()) {
|
|
639
593
|
files[key] = structuredClone(entry);
|
|
640
594
|
}
|
|
641
595
|
}
|
|
642
596
|
else {
|
|
643
597
|
for (const key of new Set(keys)) {
|
|
644
|
-
const entry =
|
|
598
|
+
const entry = fullRows.get(key);
|
|
645
599
|
if (entry === undefined)
|
|
646
600
|
continue;
|
|
647
601
|
files[key] = structuredClone(entry);
|
|
648
602
|
}
|
|
649
603
|
if (roots.size > 0) {
|
|
650
|
-
for (const [key, entry] of
|
|
604
|
+
for (const [key, entry] of fullRows.entries()) {
|
|
651
605
|
if (Object.hasOwn(files, key))
|
|
652
606
|
continue;
|
|
653
607
|
if (isUnderAnyRoot(key, roots))
|
|
@@ -656,10 +610,10 @@ export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
|
|
|
656
610
|
}
|
|
657
611
|
}
|
|
658
612
|
return rememberJournalBaseline({
|
|
659
|
-
version:
|
|
660
|
-
lastSync:
|
|
613
|
+
version: full.version,
|
|
614
|
+
lastSync: full.lastSync,
|
|
661
615
|
files,
|
|
662
|
-
pulls: structuredClone(
|
|
616
|
+
pulls: structuredClone(full.pulls ?? []),
|
|
663
617
|
});
|
|
664
618
|
}
|
|
665
619
|
/**
|
|
@@ -740,22 +694,9 @@ function journalStoreOptions(slug, initialJournal) {
|
|
|
740
694
|
}
|
|
741
695
|
if (record.type === JOURNAL_DELTA_RECORD_TYPE) {
|
|
742
696
|
const delta = record.payload;
|
|
743
|
-
const
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
// full-pass facade opened before this append. Once a first delta is
|
|
747
|
-
// observed, put durable changes in a separate overlay rather than
|
|
748
|
-
// mutating that shared packed row store.
|
|
749
|
-
const rows = existingRows instanceof JournalRowOverlay
|
|
750
|
-
? existingRows.withCommittedDelta(delta.upserts, delta.deletes)
|
|
751
|
-
: new JournalRowOverlay(existingRows).withCommittedDelta(delta.upserts, delta.deletes);
|
|
752
|
-
state.journal.files = createJournalRecordProxy(rows);
|
|
753
|
-
}
|
|
754
|
-
else {
|
|
755
|
-
for (const key of delta.deletes)
|
|
756
|
-
delete state.journal.files[key];
|
|
757
|
-
Object.assign(state.journal.files, delta.upserts);
|
|
758
|
-
}
|
|
697
|
+
for (const key of delta.deletes)
|
|
698
|
+
delete state.journal.files[key];
|
|
699
|
+
Object.assign(state.journal.files, delta.upserts);
|
|
759
700
|
state.journal.version = delta.version;
|
|
760
701
|
state.journal.lastSync = delta.lastSync ?? state.journal.lastSync;
|
|
761
702
|
state.journal.pulls = delta.pulls ?? state.journal.pulls;
|
|
@@ -794,10 +735,6 @@ function openAreaMigrationIfAuthoritative(slug) {
|
|
|
794
735
|
closeCachedAreaMigration(stateDir, slug);
|
|
795
736
|
return undefined;
|
|
796
737
|
}
|
|
797
|
-
// Area authority replaces the legacy v3 aggregate. Drop any retained legacy
|
|
798
|
-
// projection before a scoped reader can accidentally keep both generations
|
|
799
|
-
// resident (or use the obsolete writer after cutover).
|
|
800
|
-
releaseScopedJournalStore(slug);
|
|
801
738
|
const resolverPath = path.join(areaLayoutDirectory(stateDir, slug), "area-resolver.json");
|
|
802
739
|
let resolverJson;
|
|
803
740
|
try {
|
|
@@ -862,20 +799,19 @@ export function openJournalStateStoreForMigration(slug) {
|
|
|
862
799
|
async function openJournalStoreAsync(slug, initialJournal) {
|
|
863
800
|
return StateStore.openAsync(journalStoreOptions(slug, initialJournal));
|
|
864
801
|
}
|
|
802
|
+
/**
|
|
803
|
+
* Long-lived keyed projection over the v3 snapshot/WAL authority.
|
|
804
|
+
*
|
|
805
|
+
* The first open recovers once. Subsequent refreshes consume only authenticated
|
|
806
|
+
* complete WAL frames, rebuilding the projection solely after a generation
|
|
807
|
+
* publication or invalid-tail fallback.
|
|
808
|
+
*/
|
|
865
809
|
export class JournalStore {
|
|
866
810
|
slug;
|
|
867
811
|
store;
|
|
868
812
|
areaMigration;
|
|
869
813
|
files = new JournalRowStore({ trackChanges: false });
|
|
870
814
|
usesSharedRows = false;
|
|
871
|
-
/**
|
|
872
|
-
* Durable deltas this keyed session has observed after its base snapshot.
|
|
873
|
-
* The base can be shared with a full-pass facade, so publishing a scoped
|
|
874
|
-
* delta here must never mutate it in place.
|
|
875
|
-
*/
|
|
876
|
-
committedOverlay = new Map();
|
|
877
|
-
/** Base keys deleted then reinserted by durable records move to the tail. */
|
|
878
|
-
committedMoved = new Set();
|
|
879
815
|
/**
|
|
880
816
|
* Pass-local rows that have not reached the shared projection yet. A missing
|
|
881
817
|
* value represents a staged delete; every other value is a frozen copy of
|
|
@@ -905,56 +841,27 @@ export class JournalStore {
|
|
|
905
841
|
this.assertOpen();
|
|
906
842
|
if (this.overlay.has(key))
|
|
907
843
|
return this.overlay.get(key);
|
|
908
|
-
if (this.committedOverlay.has(key))
|
|
909
|
-
return this.committedOverlay.get(key);
|
|
910
844
|
return this.files.get(key);
|
|
911
845
|
}
|
|
912
846
|
scanPrefix(prefix) {
|
|
913
847
|
this.assertOpen();
|
|
848
|
+
const rows = new Map(this.files.entriesForPrefix(prefix));
|
|
914
849
|
const normalizedPrefix = prefix.replace(/\/+$/, "");
|
|
915
|
-
const
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
? overlay.get(key)
|
|
925
|
-
: committedOverlay.has(key)
|
|
926
|
-
? committedOverlay.get(key)
|
|
927
|
-
: entry;
|
|
928
|
-
if (replacement !== undefined)
|
|
929
|
-
yield [key, replacement];
|
|
930
|
-
}
|
|
931
|
-
// A Map replacement preserves the base row's position and appends newly
|
|
932
|
-
// introduced overlay rows. Preserve that observable order without holding
|
|
933
|
-
// a materialized JournalEntry for every base row at once.
|
|
934
|
-
const additions = new Map(committedOverlay);
|
|
935
|
-
for (const [key, entry] of overlay)
|
|
936
|
-
additions.set(key, entry);
|
|
937
|
-
for (const [key, entry] of additions) {
|
|
938
|
-
if (entry === undefined || (files.has(key) && !committedMoved.has(key)))
|
|
939
|
-
continue;
|
|
940
|
-
if (isJournalStorePrefixMatch(key, normalizedPrefix))
|
|
941
|
-
yield [key, entry];
|
|
942
|
-
}
|
|
943
|
-
};
|
|
944
|
-
const scan = {
|
|
945
|
-
[Symbol.iterator]: iterate,
|
|
946
|
-
filter: ((predicate, thisArg) => Array.from(iterate()).filter(predicate, thisArg)),
|
|
947
|
-
};
|
|
948
|
-
return scan;
|
|
850
|
+
for (const [key, entry] of this.overlay) {
|
|
851
|
+
if (!isJournalStorePrefixMatch(key, normalizedPrefix))
|
|
852
|
+
continue;
|
|
853
|
+
if (entry === undefined)
|
|
854
|
+
rows.delete(key);
|
|
855
|
+
else
|
|
856
|
+
rows.set(key, entry);
|
|
857
|
+
}
|
|
858
|
+
return Array.from(rows.entries());
|
|
949
859
|
}
|
|
950
860
|
rowCount() {
|
|
951
861
|
if (this.disposed)
|
|
952
862
|
return 0;
|
|
953
|
-
const replacements = new Map(this.committedOverlay);
|
|
954
|
-
for (const [key, entry] of this.overlay)
|
|
955
|
-
replacements.set(key, entry);
|
|
956
863
|
let count = this.files.size;
|
|
957
|
-
for (const [key, entry] of
|
|
864
|
+
for (const [key, entry] of this.overlay) {
|
|
958
865
|
const existsInSharedStore = this.files.has(key);
|
|
959
866
|
if (entry === undefined) {
|
|
960
867
|
if (existsInSharedStore)
|
|
@@ -970,9 +877,7 @@ export class JournalStore {
|
|
|
970
877
|
set(key, entry) {
|
|
971
878
|
this.assertOpen();
|
|
972
879
|
const ownedEntry = immutableCopy(entry);
|
|
973
|
-
const sharedEntry = this.
|
|
974
|
-
? this.committedOverlay.get(key)
|
|
975
|
-
: this.files.get(key);
|
|
880
|
+
const sharedEntry = this.files.get(key);
|
|
976
881
|
if (sharedEntry !== undefined && sameEntry(sharedEntry, ownedEntry)) {
|
|
977
882
|
this.overlay.delete(key);
|
|
978
883
|
return;
|
|
@@ -985,12 +890,12 @@ export class JournalStore {
|
|
|
985
890
|
if (this.overlay.has(key)) {
|
|
986
891
|
if (this.overlay.get(key) === undefined)
|
|
987
892
|
return false;
|
|
988
|
-
if (!this.
|
|
893
|
+
if (!this.files.has(key)) {
|
|
989
894
|
this.overlay.delete(key);
|
|
990
895
|
return true;
|
|
991
896
|
}
|
|
992
897
|
}
|
|
993
|
-
else if (this.
|
|
898
|
+
else if (!this.files.has(key)) {
|
|
994
899
|
return false;
|
|
995
900
|
}
|
|
996
901
|
this.overlay.set(key, undefined);
|
|
@@ -1029,17 +934,6 @@ export class JournalStore {
|
|
|
1029
934
|
pulls: this.pulls,
|
|
1030
935
|
});
|
|
1031
936
|
}
|
|
1032
|
-
/**
|
|
1033
|
-
* The StateStore aggregate after a durable append. Its row view is immutable
|
|
1034
|
-
* across later deltas, which lets a full-pass facade rebase without exposing
|
|
1035
|
-
* its completed checkpoint to subsequent scoped writers.
|
|
1036
|
-
*/
|
|
1037
|
-
durableJournalForRebase() {
|
|
1038
|
-
this.assertOpen();
|
|
1039
|
-
if (this.areaMigration)
|
|
1040
|
-
return this.areaMigration.readAreaJournalCached().journal;
|
|
1041
|
-
return this.store.getState().journal;
|
|
1042
|
-
}
|
|
1043
937
|
refresh() {
|
|
1044
938
|
this.assertOpen();
|
|
1045
939
|
if (this.areaMigration) {
|
|
@@ -1066,10 +960,12 @@ export class JournalStore {
|
|
|
1066
960
|
this.applyRefresh(result);
|
|
1067
961
|
}
|
|
1068
962
|
applyRefresh(result) {
|
|
1069
|
-
if (result.recovered
|
|
963
|
+
if (result.recovered) {
|
|
1070
964
|
this.replaceFromJournal(this.store.getState().journal);
|
|
1071
965
|
return;
|
|
1072
966
|
}
|
|
967
|
+
for (const record of result.records)
|
|
968
|
+
this.applyRecord(record.type, record.payload);
|
|
1073
969
|
}
|
|
1074
970
|
applyDelta(delta) {
|
|
1075
971
|
this.assertOpen();
|
|
@@ -1129,11 +1025,13 @@ export class JournalStore {
|
|
|
1129
1025
|
this.applyCommitted(appended);
|
|
1130
1026
|
ensureJournalLocator(this.slug);
|
|
1131
1027
|
}
|
|
1132
|
-
applyCommitted(
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1028
|
+
applyCommitted(appended) {
|
|
1029
|
+
if (appended.refresh.recovered)
|
|
1030
|
+
this.replaceFromJournal(this.store.getState().journal);
|
|
1031
|
+
else
|
|
1032
|
+
for (const record of appended.refresh.records)
|
|
1033
|
+
this.applyRecord(record.type, record.payload);
|
|
1034
|
+
this.applyRecord(appended.type, appended.payload);
|
|
1137
1035
|
}
|
|
1138
1036
|
/**
|
|
1139
1037
|
* Packing copies every source row into typed arrays, so this projection never
|
|
@@ -1142,30 +1040,21 @@ export class JournalStore {
|
|
|
1142
1040
|
* pass-local), which is what put the outposts over Node's heap ceiling.
|
|
1143
1041
|
*/
|
|
1144
1042
|
replaceFromJournal(journal) {
|
|
1145
|
-
const
|
|
1146
|
-
const decoded = proxied instanceof JournalRowStore || proxied instanceof JournalRowOverlay
|
|
1147
|
-
? proxied
|
|
1148
|
-
: decodedJournalRows(journal);
|
|
1043
|
+
const decoded = decodedJournalRows(journal);
|
|
1149
1044
|
if (decoded) {
|
|
1150
|
-
if (!this.usesSharedRows && this.files
|
|
1045
|
+
if (!this.usesSharedRows && this.files !== decoded)
|
|
1151
1046
|
this.files.clear();
|
|
1152
|
-
}
|
|
1153
1047
|
this.files = decoded;
|
|
1154
1048
|
this.usesSharedRows = true;
|
|
1155
|
-
this.committedOverlay.clear();
|
|
1156
|
-
this.committedMoved.clear();
|
|
1157
1049
|
this.version = journal.version;
|
|
1158
1050
|
this.lastSync = journal.lastSync;
|
|
1159
1051
|
this.pulls = structuredClone(journal.pulls ?? []);
|
|
1160
1052
|
return;
|
|
1161
1053
|
}
|
|
1162
|
-
if (this.usesSharedRows
|
|
1054
|
+
if (this.usesSharedRows)
|
|
1163
1055
|
this.files = new JournalRowStore({ trackChanges: false });
|
|
1164
|
-
}
|
|
1165
1056
|
this.files.replaceFromRecord(journal.files);
|
|
1166
1057
|
this.usesSharedRows = false;
|
|
1167
|
-
this.committedOverlay.clear();
|
|
1168
|
-
this.committedMoved.clear();
|
|
1169
1058
|
this.version = journal.version;
|
|
1170
1059
|
this.lastSync = journal.lastSync;
|
|
1171
1060
|
this.pulls = structuredClone(journal.pulls ?? []);
|
|
@@ -1179,15 +1068,9 @@ export class JournalStore {
|
|
|
1179
1068
|
throw new Error("journal: keyed session received an invalid WAL record");
|
|
1180
1069
|
}
|
|
1181
1070
|
for (const key of payload.deletes)
|
|
1182
|
-
this.
|
|
1183
|
-
for (const [key, entry] of Object.entries(payload.upserts))
|
|
1184
|
-
|
|
1185
|
-
this.committedOverlay.delete(key);
|
|
1186
|
-
if (this.files.has(key))
|
|
1187
|
-
this.committedMoved.add(key);
|
|
1188
|
-
}
|
|
1189
|
-
this.committedOverlay.set(key, immutableCopy(entry));
|
|
1190
|
-
}
|
|
1071
|
+
this.files.delete(key);
|
|
1072
|
+
for (const [key, entry] of Object.entries(payload.upserts))
|
|
1073
|
+
this.files.set(key, entry);
|
|
1191
1074
|
this.version = payload.version;
|
|
1192
1075
|
this.lastSync = payload.lastSync ?? this.lastSync;
|
|
1193
1076
|
this.pulls = structuredClone(payload.pulls ?? this.pulls);
|
|
@@ -1198,9 +1081,7 @@ export class JournalStore {
|
|
|
1198
1081
|
return;
|
|
1199
1082
|
this.disposed = true;
|
|
1200
1083
|
this.overlay.clear();
|
|
1201
|
-
this.
|
|
1202
|
-
this.committedMoved.clear();
|
|
1203
|
-
if (!this.usesSharedRows && this.files instanceof JournalRowStore)
|
|
1084
|
+
if (!this.usesSharedRows)
|
|
1204
1085
|
this.files.clear();
|
|
1205
1086
|
this.store = undefined;
|
|
1206
1087
|
this.areaMigration = undefined;
|
|
@@ -1227,69 +1108,6 @@ function immutableCopy(value) {
|
|
|
1227
1108
|
freeze(copy);
|
|
1228
1109
|
return copy;
|
|
1229
1110
|
}
|
|
1230
|
-
// A scoped watcher pass must not recover a 700k-row snapshot just to inspect
|
|
1231
|
-
// one path. A runner can alternate the personal vault and its companies in the
|
|
1232
|
-
// same realtime batch, so one cache entry turns that alternation into a fresh
|
|
1233
|
-
// whole-snapshot recovery on every dispatch. Keep a deliberately small LRU:
|
|
1234
|
-
// enough for the runner's active journal set, bounded so an unusually broad
|
|
1235
|
-
// company fanout cannot retain every journal for the lifetime of the process.
|
|
1236
|
-
const MAX_RETAINED_SCOPED_JOURNALS = 4;
|
|
1237
|
-
const cachedScopedJournalStores = new Map();
|
|
1238
|
-
function scopedJournalStoreCacheKey(slug) {
|
|
1239
|
-
return `${getStateDir()}\u0000${slug}`;
|
|
1240
|
-
}
|
|
1241
|
-
function cachedScopedJournalStoreFor(slug) {
|
|
1242
|
-
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1243
|
-
const cached = cachedScopedJournalStores.get(cacheKey);
|
|
1244
|
-
if (cached === undefined)
|
|
1245
|
-
return undefined;
|
|
1246
|
-
// Map insertion order is our LRU order. A hot scope must not be evicted by
|
|
1247
|
-
// a brief company fanout while the runner is still using it.
|
|
1248
|
-
cachedScopedJournalStores.delete(cacheKey);
|
|
1249
|
-
cachedScopedJournalStores.set(cacheKey, cached);
|
|
1250
|
-
return cached;
|
|
1251
|
-
}
|
|
1252
|
-
function retainScopedJournalStore(slug, store) {
|
|
1253
|
-
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1254
|
-
const previous = cachedScopedJournalStores.get(cacheKey);
|
|
1255
|
-
if (previous !== undefined && previous !== store)
|
|
1256
|
-
previous.dispose();
|
|
1257
|
-
cachedScopedJournalStores.delete(cacheKey);
|
|
1258
|
-
cachedScopedJournalStores.set(cacheKey, store);
|
|
1259
|
-
while (cachedScopedJournalStores.size > MAX_RETAINED_SCOPED_JOURNALS) {
|
|
1260
|
-
const oldest = cachedScopedJournalStores.entries().next().value;
|
|
1261
|
-
if (oldest === undefined)
|
|
1262
|
-
break;
|
|
1263
|
-
const [oldestKey, oldestStore] = oldest;
|
|
1264
|
-
cachedScopedJournalStores.delete(oldestKey);
|
|
1265
|
-
oldestStore.dispose();
|
|
1266
|
-
}
|
|
1267
|
-
return store;
|
|
1268
|
-
}
|
|
1269
|
-
function scopedJournalStore(slug) {
|
|
1270
|
-
const cached = cachedScopedJournalStoreFor(slug);
|
|
1271
|
-
if (cached)
|
|
1272
|
-
return cached;
|
|
1273
|
-
return retainScopedJournalStore(slug, openJournalStoreSession(slug));
|
|
1274
|
-
}
|
|
1275
|
-
/** Drop one legacy keyed projection after an authoritative area cutover. */
|
|
1276
|
-
export function releaseScopedJournalStore(slug) {
|
|
1277
|
-
const cacheKey = scopedJournalStoreCacheKey(slug);
|
|
1278
|
-
const cached = cachedScopedJournalStores.get(cacheKey);
|
|
1279
|
-
if (cached === undefined)
|
|
1280
|
-
return;
|
|
1281
|
-
cachedScopedJournalStores.delete(cacheKey);
|
|
1282
|
-
cached.dispose();
|
|
1283
|
-
}
|
|
1284
|
-
/**
|
|
1285
|
-
* Release retained scoped projections when a watch runner stops. Durable state
|
|
1286
|
-
* remains in the StateStore; this only drops process-local decoded snapshots.
|
|
1287
|
-
*/
|
|
1288
|
-
export function disposeScopedJournalStores() {
|
|
1289
|
-
for (const store of cachedScopedJournalStores.values())
|
|
1290
|
-
store.dispose();
|
|
1291
|
-
cachedScopedJournalStores.clear();
|
|
1292
|
-
}
|
|
1293
1111
|
/** Open one keyed session, seeding a missing v3 store from legacy JSON once. */
|
|
1294
1112
|
export function openJournalStoreSession(slug) {
|
|
1295
1113
|
const areaMigration = openAreaMigrationIfAuthoritative(slug);
|
|
@@ -1802,47 +1620,6 @@ export function writeJournal(slug, journal) {
|
|
|
1802
1620
|
ensureJournalLocator(slug);
|
|
1803
1621
|
return;
|
|
1804
1622
|
}
|
|
1805
|
-
const baseline = journalBaselines.get(journal);
|
|
1806
|
-
const cached = cachedScopedJournalStoreFor(slug);
|
|
1807
|
-
if (cached && (storeBacking || baseline)) {
|
|
1808
|
-
// The scoped reader captured only its named rows, so its baseline is the
|
|
1809
|
-
// authoritative delta boundary. The retained session refreshes and appends
|
|
1810
|
-
// under the normal StateStore lock; it never exposes partial durable rows
|
|
1811
|
-
// to another writer.
|
|
1812
|
-
const delta = storeBacking
|
|
1813
|
-
? journalDeltaFromStoreDirtyLog(storeBacking, journal)
|
|
1814
|
-
: baseline
|
|
1815
|
-
? journalDeltaFromBaseline(baseline, journal)
|
|
1816
|
-
: undefined;
|
|
1817
|
-
if (delta && hasJournalDelta(delta)) {
|
|
1818
|
-
// Carry ONLY the metadata fields this delta actually changed. The
|
|
1819
|
-
// retained session's `metadata()` is a snapshot that goes stale the
|
|
1820
|
-
// moment another writer advances `lastSync` or appends a pull record,
|
|
1821
|
-
// and filling the untouched field from it republished that stale value
|
|
1822
|
-
// over the concurrent writer's durable one — dropping pull history or
|
|
1823
|
-
// reverting `lastSync`. The store's reducer already preserves a field
|
|
1824
|
-
// the delta omits, so omission is the whole fix.
|
|
1825
|
-
const metadata = {};
|
|
1826
|
-
if (delta.lastSync !== undefined)
|
|
1827
|
-
metadata.lastSync = delta.lastSync;
|
|
1828
|
-
if (delta.pulls !== undefined)
|
|
1829
|
-
metadata.pulls = delta.pulls;
|
|
1830
|
-
const hasMetadata = delta.lastSync !== undefined || delta.pulls !== undefined;
|
|
1831
|
-
cached.applyDelta({
|
|
1832
|
-
upserts: delta.upserts,
|
|
1833
|
-
deletes: delta.deletes,
|
|
1834
|
-
...(hasMetadata ? { metadata } : {}),
|
|
1835
|
-
});
|
|
1836
|
-
}
|
|
1837
|
-
if (storeBacking) {
|
|
1838
|
-
rebaseStoreBackedJournal(storeBacking, journal, cached.durableJournalForRebase());
|
|
1839
|
-
checkpointStoreBackedJournal(storeBacking, journal);
|
|
1840
|
-
}
|
|
1841
|
-
else
|
|
1842
|
-
rememberJournalBaseline(journal);
|
|
1843
|
-
ensureJournalLocator(slug);
|
|
1844
|
-
return;
|
|
1845
|
-
}
|
|
1846
1623
|
const store = openJournalStore(slug, journal);
|
|
1847
1624
|
if (store.wasCreated) {
|
|
1848
1625
|
// The store was just seeded from this journal, so its state already matches
|
|
@@ -1866,6 +1643,7 @@ export function writeJournal(slug, journal) {
|
|
|
1866
1643
|
// The WAL still compacts on its own schedule (record count or byte ceiling),
|
|
1867
1644
|
// so a full snapshot is written periodically rather than never — recovery
|
|
1868
1645
|
// stays bounded, it just no longer happens on every single update.
|
|
1646
|
+
const baseline = journalBaselines.get(journal);
|
|
1869
1647
|
const delta = storeBacking
|
|
1870
1648
|
? journalDeltaFromStoreDirtyLog(storeBacking, journal)
|
|
1871
1649
|
: baseline
|
|
@@ -1874,10 +1652,8 @@ export function writeJournal(slug, journal) {
|
|
|
1874
1652
|
store.append(JOURNAL_DELTA_RECORD_TYPE, delta);
|
|
1875
1653
|
// A caller can checkpoint more than once during one pass. Advancing only its
|
|
1876
1654
|
// own tracking state means the next checkpoint again publishes only new mutations.
|
|
1877
|
-
if (storeBacking)
|
|
1878
|
-
rebaseStoreBackedJournal(storeBacking, journal, store.getState().journal);
|
|
1655
|
+
if (storeBacking)
|
|
1879
1656
|
checkpointStoreBackedJournal(storeBacking, journal);
|
|
1880
|
-
}
|
|
1881
1657
|
else
|
|
1882
1658
|
rememberJournalBaseline(journal);
|
|
1883
1659
|
// The locator is immutable content, so it only needs writing if it is absent
|