@indigoai-us/hq-cloud 6.16.38 → 6.16.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/dist/bin/sync-runner-heap-telemetry.d.ts +145 -9
  2. package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-heap-telemetry.js +178 -11
  4. package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
  5. package/dist/bin/sync-runner-heap-telemetry.test.js +511 -2
  6. package/dist/bin/sync-runner-heap-telemetry.test.js.map +1 -1
  7. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  8. package/dist/bin/sync-runner-watch-loop.js +20 -10
  9. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  10. package/dist/bin/sync-runner-watch-loop.test.js +21 -0
  11. package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
  12. package/dist/bin/sync-runner.d.ts +16 -5
  13. package/dist/bin/sync-runner.d.ts.map +1 -1
  14. package/dist/bin/sync-runner.js.map +1 -1
  15. package/dist/cli/share.d.ts.map +1 -1
  16. package/dist/cli/share.js +28 -27
  17. package/dist/cli/share.js.map +1 -1
  18. package/dist/cli/share.test.js +49 -0
  19. package/dist/cli/share.test.js.map +1 -1
  20. package/dist/cli/sync.d.ts.map +1 -1
  21. package/dist/cli/sync.js +8 -8
  22. package/dist/cli/sync.js.map +1 -1
  23. package/dist/journal-area-ledger-read-boundary.test.js +46 -12
  24. package/dist/journal-area-ledger-read-boundary.test.js.map +1 -1
  25. package/dist/journal-record-proxy.d.ts +12 -2
  26. package/dist/journal-record-proxy.d.ts.map +1 -1
  27. package/dist/journal-record-proxy.js +8 -1
  28. package/dist/journal-record-proxy.js.map +1 -1
  29. package/dist/journal-row-store.d.ts +64 -0
  30. package/dist/journal-row-store.d.ts.map +1 -1
  31. package/dist/journal-row-store.js +129 -3
  32. package/dist/journal-row-store.js.map +1 -1
  33. package/dist/journal-row-store.test.js +3 -1
  34. package/dist/journal-row-store.test.js.map +1 -1
  35. package/dist/journal-snapshot-rows.d.ts +17 -1
  36. package/dist/journal-snapshot-rows.d.ts.map +1 -1
  37. package/dist/journal-snapshot-rows.js +18 -0
  38. package/dist/journal-snapshot-rows.js.map +1 -1
  39. package/dist/journal-snapshot-v4.d.ts +3 -2
  40. package/dist/journal-snapshot-v4.d.ts.map +1 -1
  41. package/dist/journal-snapshot-v4.js +9 -3
  42. package/dist/journal-snapshot-v4.js.map +1 -1
  43. package/dist/journal-snapshot-v4.test.js +1 -0
  44. package/dist/journal-snapshot-v4.test.js.map +1 -1
  45. package/dist/journal.d.ts +23 -5
  46. package/dist/journal.d.ts.map +1 -1
  47. package/dist/journal.js +309 -189
  48. package/dist/journal.js.map +1 -1
  49. package/dist/journal.test.js +58 -1
  50. package/dist/journal.test.js.map +1 -1
  51. package/dist/packed-journal-phase2.test.d.ts +2 -0
  52. package/dist/packed-journal-phase2.test.d.ts.map +1 -0
  53. package/dist/packed-journal-phase2.test.js +273 -0
  54. package/dist/packed-journal-phase2.test.js.map +1 -0
  55. package/dist/personal-vault.d.ts.map +1 -1
  56. package/dist/personal-vault.js +6 -4
  57. package/dist/personal-vault.js.map +1 -1
  58. package/dist/remote-pull.d.ts.map +1 -1
  59. package/dist/remote-pull.js +9 -8
  60. package/dist/remote-pull.js.map +1 -1
  61. package/dist/scope-shrink.js +2 -2
  62. package/dist/scope-shrink.js.map +1 -1
  63. package/dist/session-log-retention.d.ts.map +1 -1
  64. package/dist/session-log-retention.js +2 -2
  65. package/dist/session-log-retention.js.map +1 -1
  66. package/dist/sync/area-ledger-migration.d.ts +13 -6
  67. package/dist/sync/area-ledger-migration.d.ts.map +1 -1
  68. package/dist/sync/area-ledger-migration.js +45 -19
  69. package/dist/sync/area-ledger-migration.js.map +1 -1
  70. package/dist/sync/area-ledger-migration.test.js +23 -0
  71. package/dist/sync/area-ledger-migration.test.js.map +1 -1
  72. package/dist/sync/area-ledger.d.ts +31 -1
  73. package/dist/sync/area-ledger.d.ts.map +1 -1
  74. package/dist/sync/area-ledger.js +385 -88
  75. package/dist/sync/area-ledger.js.map +1 -1
  76. package/dist/sync/area-ledger.test.js +123 -10
  77. package/dist/sync/area-ledger.test.js.map +1 -1
  78. package/dist/types.d.ts +2 -0
  79. package/dist/types.d.ts.map +1 -1
  80. package/package.json +1 -1
package/dist/journal.js CHANGED
@@ -16,9 +16,10 @@ import * as fs from "fs";
16
16
  import * as os from "os";
17
17
  import * as path from "path";
18
18
  import * as crypto from "crypto";
19
- import { JournalRowStore, journalEntryFingerprint } from "./journal-row-store.js";
19
+ import { JournalRowOverlay, JournalRowStore, journalEntryFingerprint, } from "./journal-row-store.js";
20
20
  import { createJournalRecordProxy } from "./journal-record-proxy.js";
21
21
  import { decodeJournalSnapshot, decodedJournalRows } from "./journal-snapshot-decoder.js";
22
+ import { forgetSharedAreaJournalRows } from "./journal-snapshot-rows.js";
22
23
  import { encodeV4JournalSnapshotChunks } from "./journal-snapshot-v4.js";
23
24
  import { toPosixKey } from "./posix-key.js";
24
25
  import { StateStore, StateStoreCorruptionError, SNAPSHOT_V4_MAGIC, } from "./sync/state-store.js";
@@ -51,11 +52,73 @@ const journalBaselines = new WeakMap();
51
52
  // the public journal object while letting writeJournal select its O(changes)
52
53
  // dirty-log path only for journals returned by readJournal.
53
54
  const storeBackedJournals = new WeakMap();
54
- // Full pulls need a plain, shallow files map so they can share frozen ledger
55
- // rows. This separate brand gives that map the same O(changes) checkpoint
56
- // behavior as the packed record proxy without projecting every row into a
57
- // second JournalRowStore.
58
- const mutationTrackedJournals = new WeakMap();
55
+ /** Full-pass journals must not retain the post-cutover v3 rollback mirror between checkpoints. */
56
+ const fullPassAreaJournals = new WeakSet();
57
+ /**
58
+ * Record-shaped journals are a public compatibility boundary. Internal callers
59
+ * use this adapter so packed rows stay lazy through reads, scans and writes.
60
+ * Plain records only reach it from legacy/public inputs, where the adapter
61
+ * yields one row at a time rather than materialising a whole entry array.
62
+ */
63
+ class RecordJournalRows {
64
+ files;
65
+ constructor(files) {
66
+ this.files = files;
67
+ }
68
+ get size() {
69
+ let size = 0;
70
+ for (const key in this.files) {
71
+ if (Object.hasOwn(this.files, key))
72
+ size += 1;
73
+ }
74
+ return size;
75
+ }
76
+ get(path) {
77
+ return this.files[path];
78
+ }
79
+ has(path) {
80
+ return Object.hasOwn(this.files, path);
81
+ }
82
+ set(path, entry) {
83
+ this.files[path] = entry;
84
+ }
85
+ delete(path) {
86
+ if (!Object.hasOwn(this.files, path))
87
+ return false;
88
+ delete this.files[path];
89
+ return true;
90
+ }
91
+ *keys() {
92
+ for (const key in this.files) {
93
+ if (Object.hasOwn(this.files, key))
94
+ yield key;
95
+ }
96
+ }
97
+ *entries() {
98
+ for (const key of this.keys()) {
99
+ const entry = this.files[key];
100
+ if (entry !== undefined)
101
+ yield [key, entry];
102
+ }
103
+ }
104
+ *values() {
105
+ for (const [, entry] of this.entries())
106
+ yield entry;
107
+ }
108
+ toRecord() {
109
+ return this.files;
110
+ }
111
+ }
112
+ /** Return the lazy internal row view for any journal without exposing its record proxy. */
113
+ export function journalRows(journal) {
114
+ const backing = storeBackedJournals.get(journal);
115
+ if (backing)
116
+ return backing.store;
117
+ const decoded = decodedJournalRows(journal);
118
+ if (decoded)
119
+ return decoded.rows;
120
+ return new RecordJournalRows(journal.files);
121
+ }
59
122
  // Area-authoritative reads still refresh every participating area so a cached
60
123
  // bridge observes committed writers in this and other processes. The expensive
61
124
  // part we can share is the bridge construction: it owns the lazy AreaLedger
@@ -69,10 +132,11 @@ function areaMigrationCacheKey(stateDir, slug) {
69
132
  function closeCachedAreaMigration(stateDir, slug) {
70
133
  const cacheKey = areaMigrationCacheKey(stateDir, slug);
71
134
  const cached = cachedAreaMigrations.get(cacheKey);
72
- if (!cached)
73
- return;
74
- cached.migration.dispose();
75
- cachedAreaMigrations.delete(cacheKey);
135
+ if (cached) {
136
+ cached.migration.dispose();
137
+ cachedAreaMigrations.delete(cacheKey);
138
+ }
139
+ forgetSharedAreaJournalRows(cacheKey);
76
140
  }
77
141
  /**
78
142
  * Release this process's cached area bridge for a journal. The durable stores
@@ -90,6 +154,7 @@ export function releaseCachedAreaJournalMigrations() {
90
154
  for (const [cacheKey, cached] of cachedAreaMigrations) {
91
155
  cached.migration.dispose();
92
156
  cachedAreaMigrations.delete(cacheKey);
157
+ forgetSharedAreaJournalRows(cacheKey);
93
158
  }
94
159
  }
95
160
  /** Path-free count exported solely for the runner heap census. */
@@ -102,9 +167,8 @@ export function journalCacheCensusSizes() {
102
167
  */
103
168
  export function buildJournalBaseline(journal) {
104
169
  const rows = new Map();
105
- const files = journal.files;
106
- for (const key of Object.keys(files))
107
- rows.set(key, entryFingerprint(files[key]));
170
+ for (const [key, entry] of journalRows(journal).entries())
171
+ rows.set(key, entryFingerprint(entry));
108
172
  return {
109
173
  version: journal.version,
110
174
  lastSync: journal.lastSync,
@@ -116,7 +180,7 @@ function rememberJournalBaseline(journal) {
116
180
  journalBaselines.set(journal, buildJournalBaseline(journal));
117
181
  return journal;
118
182
  }
119
- function readStoreBackedJournal(journal) {
183
+ function readStoreBackedJournal(journal, options = {}) {
120
184
  const existing = storeBackedJournals.get(journal);
121
185
  if (existing) {
122
186
  // Snapshot decode starts the store's change log before WAL replay. The
@@ -129,17 +193,32 @@ function readStoreBackedJournal(journal) {
129
193
  }
130
194
  const decoded = decodedJournalRows(journal);
131
195
  if (decoded) {
196
+ if (options.privateOverlay) {
197
+ const store = new JournalRowOverlay(decoded);
198
+ const result = { ...journal, files: createJournalRecordProxy(store) };
199
+ if (Object.hasOwn(journal, "pulls"))
200
+ result.pulls = structuredClone(journal.pulls);
201
+ storeBackedJournals.set(result, {
202
+ store,
203
+ version: result.version,
204
+ lastSync: result.lastSync,
205
+ pullsJson: JSON.stringify(result.pulls ?? []),
206
+ keysNormalized: false,
207
+ });
208
+ return result;
209
+ }
132
210
  decoded.enableChangeTracking();
133
211
  storeBackedJournals.set(journal, {
134
212
  store: decoded,
135
213
  version: journal.version,
136
214
  lastSync: journal.lastSync,
137
215
  pullsJson: JSON.stringify(journal.pulls ?? []),
216
+ keysNormalized: false,
138
217
  });
139
218
  return journal;
140
219
  }
141
220
  const store = new JournalRowStore();
142
- for (const [key, entry] of Object.entries(journal.files))
221
+ for (const [key, entry] of journalRows(journal).entries())
143
222
  store.set(key, entry);
144
223
  // Loading establishes the read baseline; only caller mutations belong in the
145
224
  // write delta. The store remains change-tracked after this checkpoint.
@@ -154,77 +233,10 @@ function readStoreBackedJournal(journal) {
154
233
  version: result.version,
155
234
  lastSync: result.lastSync,
156
235
  pullsJson: JSON.stringify(result.pulls ?? []),
236
+ keysNormalized: false,
157
237
  });
158
238
  return result;
159
239
  }
160
- function recordMutation(backing, path, hadPrevious, previous) {
161
- if (backing.dirtyPaths.has(path))
162
- return;
163
- backing.dirtyPaths.add(path);
164
- backing.dirtyLog.push({
165
- path,
166
- hadPrevious,
167
- previousFingerprint: previous === undefined ? undefined : journalEntryFingerprint(previous),
168
- });
169
- }
170
- function mutationTrackedFiles(backing) {
171
- return new Proxy(backing.files, {
172
- set(target, key, value) {
173
- if (typeof key !== "string")
174
- return Reflect.set(target, key, value);
175
- const hadPrevious = Object.hasOwn(target, key);
176
- const previous = target[key];
177
- const written = Reflect.set(target, key, value);
178
- if (written) {
179
- recordMutation(backing, key, hadPrevious, previous);
180
- if (toPosixKey(key) !== key)
181
- backing.needsNormalization = true;
182
- }
183
- return written;
184
- },
185
- deleteProperty(target, key) {
186
- if (typeof key !== "string")
187
- return Reflect.deleteProperty(target, key);
188
- const hadPrevious = Object.hasOwn(target, key);
189
- const previous = target[key];
190
- const deleted = Reflect.deleteProperty(target, key);
191
- if (deleted && hadPrevious)
192
- recordMutation(backing, key, true, previous);
193
- return deleted;
194
- },
195
- defineProperty(target, key, descriptor) {
196
- if (typeof key !== "string")
197
- return Reflect.defineProperty(target, key, descriptor);
198
- const hadPrevious = Object.hasOwn(target, key);
199
- const previous = target[key];
200
- const defined = Reflect.defineProperty(target, key, descriptor);
201
- if (defined) {
202
- recordMutation(backing, key, hadPrevious, previous);
203
- if (toPosixKey(key) !== key)
204
- backing.needsNormalization = true;
205
- }
206
- return defined;
207
- },
208
- });
209
- }
210
- function readMutationTrackedJournal(journal) {
211
- const backing = {
212
- files: journal.files,
213
- dirtyLog: [],
214
- dirtyPaths: new Set(),
215
- needsNormalization: false,
216
- version: journal.version,
217
- lastSync: journal.lastSync,
218
- pullsJson: JSON.stringify(journal.pulls ?? []),
219
- };
220
- journal.files = mutationTrackedFiles(backing);
221
- mutationTrackedJournals.set(journal, backing);
222
- // Area authority may originate from an old on-disk journal. Normalize its
223
- // shallow pass map once at read time; any repaired keys are in this dirty log
224
- // and will be durably emitted by the first checkpoint, not silently hidden.
225
- migrateToV2(journal);
226
- return journal;
227
- }
228
240
  function journalDeltaFromDirtyLog(dirtyLog, getCurrent, lastSync, pullsJson, next) {
229
241
  const previousByPath = new Map();
230
242
  for (const change of dirtyLog) {
@@ -265,29 +277,20 @@ function journalDeltaFromDirtyLog(dirtyLog, getCurrent, lastSync, pullsJson, nex
265
277
  function journalDeltaFromStoreDirtyLog(backing, next) {
266
278
  return journalDeltaFromDirtyLog(backing.store.dirtyLogSnapshot(), (key) => backing.store.get(key), backing.lastSync, backing.pullsJson, next);
267
279
  }
268
- function journalDeltaFromMutationLog(backing, next) {
269
- return journalDeltaFromDirtyLog(backing.dirtyLog, (key) => backing.files[key], backing.lastSync, backing.pullsJson, next);
270
- }
271
280
  function checkpointStoreBackedJournal(backing, journal) {
272
281
  backing.store.drainDirtyLog();
282
+ backing.store.clearCommittedChanges?.();
273
283
  backing.version = journal.version;
274
284
  backing.lastSync = journal.lastSync;
275
285
  backing.pullsJson = JSON.stringify(journal.pulls ?? []);
276
286
  }
277
- function checkpointMutationTrackedJournal(backing, journal) {
278
- // The pass's append-only mutation log is drained here, after durable write,
279
- // so a later download checkpoint only publishes rows it changed since this
280
- // checkpoint rather than retaining a pass-long list.
281
- backing.dirtyLog = [];
282
- backing.dirtyPaths.clear();
283
- backing.version = journal.version;
284
- backing.lastSync = journal.lastSync;
285
- backing.pullsJson = JSON.stringify(journal.pulls ?? []);
287
+ function storeBackedJournalNeedsMigration(backing, journal) {
288
+ return journal.version !== JOURNAL_VERSION_CURRENT ||
289
+ !Array.isArray(journal.pulls) ||
290
+ typeof journal.lastSync !== "string";
286
291
  }
287
292
  function journalDeltaPath(journal) {
288
- return storeBackedJournals.has(journal) || mutationTrackedJournals.has(journal)
289
- ? "dirty-log"
290
- : "fingerprint";
293
+ return storeBackedJournals.has(journal) ? "dirty-log" : "fingerprint";
291
294
  }
292
295
  /** Exposed only to pin write-path selection at the public boundary. */
293
296
  export function journalDeltaPathForTest(journal) {
@@ -386,9 +389,6 @@ export function journalDeltaForTest(journal) {
386
389
  const backing = storeBackedJournals.get(journal);
387
390
  if (backing)
388
391
  return journalDeltaFromStoreDirtyLog(backing, journal);
389
- const mutationTracking = mutationTrackedJournals.get(journal);
390
- if (mutationTracking)
391
- return journalDeltaFromMutationLog(mutationTracking, journal);
392
392
  const baseline = journalBaselines.get(journal);
393
393
  if (!baseline)
394
394
  throw new Error("journalDeltaForTest: journal has no remembered baseline");
@@ -505,10 +505,10 @@ export function migratePersonalVaultJournal() {
505
505
  export function readJournal(slug) {
506
506
  const areaMigration = openAreaMigrationIfAuthoritative(slug);
507
507
  if (areaMigration) {
508
- // `readJournal` is the legacy mutable snapshot boundary. Direct area-ledger
509
- // reads retain their frozen rows; only a caller that requested this mutable
510
- // aggregate pays to detach it for a later writeJournal() diff.
511
- return readStoreBackedJournal(structuredClone(areaMigration.readAreaJournalCached().journal));
508
+ // Area rows stay in their shared packed store. This record-compatible
509
+ // overlay owns only the caller's writes, so the public boundary remains
510
+ // mutable without deep-cloning the aggregate before every pass.
511
+ return readStoreBackedJournal(areaMigration.readAreaJournalCached().journal, { privateOverlay: true });
512
512
  }
513
513
  return readStoreBackedJournal(readJournalAggregateWith(slug, areaMigration));
514
514
  }
@@ -516,11 +516,10 @@ export function readJournal(slug) {
516
516
  * Read the mutable outer shell required by a full pull without detaching every
517
517
  * authoritative-ledger row from its immutable source.
518
518
  *
519
- * A full pull only replaces or removes `files` keys. It never edits fields on
520
- * an existing row, so it can safely share the ledger's deeply frozen entries
521
- * while owning the map that it mutates. That map records only pass mutations,
522
- * so `writeJournal()` emits an incremental delta without fingerprinting the
523
- * whole ledger at each download checkpoint.
519
+ * A full pull only replaces or removes rows. It never edits an existing row,
520
+ * so it can safely borrow the ledger's frozen packed rows and place only its
521
+ * own changes in a private overlay. `writeJournal()` drains that overlay's
522
+ * dirty log, avoiding a full-ledger fingerprint at every download checkpoint.
524
523
  *
525
524
  * Legacy journals deliberately keep `readJournal()`'s packed-record boundary.
526
525
  * That path has different ownership and dirty-log semantics, and does not make
@@ -532,13 +531,11 @@ export function readJournalForPass(slug) {
532
531
  const areaMigration = openAreaMigrationIfAuthoritative(slug);
533
532
  if (!areaMigration)
534
533
  return readJournal(slug);
535
- const cached = areaMigration.readAreaJournalCached().journal;
536
- return readMutationTrackedJournal({
537
- version: cached.version,
538
- lastSync: cached.lastSync,
539
- files: { ...cached.files },
540
- pulls: structuredClone(cached.pulls ?? []),
541
- });
534
+ // The overlay owns only keys this pass changes; it borrows all untouched
535
+ // packed rows from the cached aggregate and keeps the public record facade.
536
+ const journal = readStoreBackedJournal(areaMigration.readAreaJournalCached().journal, { privateOverlay: true });
537
+ fullPassAreaJournals.add(journal);
538
+ return journal;
542
539
  }
543
540
  /**
544
541
  * The live aggregate for `slug` without recording a delta baseline.
@@ -588,21 +585,23 @@ export function readJournalScoped(slug, keys, deleteScopeRoots = []) {
588
585
  return rememberJournalBaseline(areaMigration.readAreaJournalScoped(keys, deleteScopeRoots));
589
586
  }
590
587
  const full = readJournalAggregate(slug);
588
+ const fullRows = journalRows(full);
591
589
  const files = {};
592
590
  const roots = new Set(deleteScopeRoots);
593
591
  if (roots.has("")) {
594
- for (const [key, entry] of Object.entries(full.files)) {
592
+ for (const [key, entry] of fullRows.entries()) {
595
593
  files[key] = structuredClone(entry);
596
594
  }
597
595
  }
598
596
  else {
599
597
  for (const key of new Set(keys)) {
600
- if (!Object.hasOwn(full.files, key))
598
+ const entry = fullRows.get(key);
599
+ if (entry === undefined)
601
600
  continue;
602
- files[key] = structuredClone(full.files[key]);
601
+ files[key] = structuredClone(entry);
603
602
  }
604
603
  if (roots.size > 0) {
605
- for (const [key, entry] of Object.entries(full.files)) {
604
+ for (const [key, entry] of fullRows.entries()) {
606
605
  if (Object.hasOwn(files, key))
607
606
  continue;
608
607
  if (isUnderAnyRoot(key, roots))
@@ -812,6 +811,14 @@ export class JournalStore {
812
811
  store;
813
812
  areaMigration;
814
813
  files = new JournalRowStore({ trackChanges: false });
814
+ usesSharedRows = false;
815
+ /**
816
+ * Pass-local rows that have not reached the shared projection yet. A missing
817
+ * value represents a staged delete; every other value is a frozen copy of
818
+ * the caller's entry, so mutating that entry after `set()` cannot alter what
819
+ * this session commits.
820
+ */
821
+ overlay = new Map();
815
822
  version = JOURNAL_VERSION_CURRENT;
816
823
  lastSync = "";
817
824
  pulls = [];
@@ -832,14 +839,87 @@ export class JournalStore {
832
839
  }
833
840
  get(key) {
834
841
  this.assertOpen();
842
+ if (this.overlay.has(key))
843
+ return this.overlay.get(key);
835
844
  return this.files.get(key);
836
845
  }
837
846
  scanPrefix(prefix) {
838
847
  this.assertOpen();
839
- return Array.from(this.files.entriesForPrefix(prefix));
848
+ const rows = new Map(this.files.entriesForPrefix(prefix));
849
+ const normalizedPrefix = prefix.replace(/\/+$/, "");
850
+ for (const [key, entry] of this.overlay) {
851
+ if (!isJournalStorePrefixMatch(key, normalizedPrefix))
852
+ continue;
853
+ if (entry === undefined)
854
+ rows.delete(key);
855
+ else
856
+ rows.set(key, entry);
857
+ }
858
+ return Array.from(rows.entries());
840
859
  }
841
860
  rowCount() {
842
- return this.files.size;
861
+ if (this.disposed)
862
+ return 0;
863
+ let count = this.files.size;
864
+ for (const [key, entry] of this.overlay) {
865
+ const existsInSharedStore = this.files.has(key);
866
+ if (entry === undefined) {
867
+ if (existsInSharedStore)
868
+ count -= 1;
869
+ }
870
+ else if (!existsInSharedStore) {
871
+ count += 1;
872
+ }
873
+ }
874
+ return count;
875
+ }
876
+ /** Stage an upsert in this session without exposing it to other sessions. */
877
+ set(key, entry) {
878
+ this.assertOpen();
879
+ const ownedEntry = immutableCopy(entry);
880
+ const sharedEntry = this.files.get(key);
881
+ if (sharedEntry !== undefined && sameEntry(sharedEntry, ownedEntry)) {
882
+ this.overlay.delete(key);
883
+ return;
884
+ }
885
+ this.overlay.set(key, ownedEntry);
886
+ }
887
+ /** Stage a delete in this session. Returns false when no session-visible row exists. */
888
+ delete(key) {
889
+ this.assertOpen();
890
+ if (this.overlay.has(key)) {
891
+ if (this.overlay.get(key) === undefined)
892
+ return false;
893
+ if (!this.files.has(key)) {
894
+ this.overlay.delete(key);
895
+ return true;
896
+ }
897
+ }
898
+ else if (!this.files.has(key)) {
899
+ return false;
900
+ }
901
+ this.overlay.set(key, undefined);
902
+ return true;
903
+ }
904
+ /** Persist this session's staged rows, then release its private overlay. */
905
+ commit() {
906
+ this.assertOpen();
907
+ if (this.overlay.size === 0)
908
+ return;
909
+ const upserts = {};
910
+ const deletes = [];
911
+ for (const [key, entry] of this.overlay) {
912
+ if (entry === undefined)
913
+ deletes.push(key);
914
+ else
915
+ upserts[key] = entry;
916
+ }
917
+ this.applyDelta({ upserts, deletes });
918
+ this.overlay.clear();
919
+ }
920
+ /** Test-only visibility into the bounded, caller-owned session state. */
921
+ overlaySizeForTest() {
922
+ return this.overlay.size;
843
923
  }
844
924
  pullHistory() {
845
925
  this.assertOpen();
@@ -962,15 +1042,19 @@ export class JournalStore {
962
1042
  replaceFromJournal(journal) {
963
1043
  const decoded = decodedJournalRows(journal);
964
1044
  if (decoded) {
1045
+ if (!this.usesSharedRows && this.files !== decoded)
1046
+ this.files.clear();
965
1047
  this.files = decoded;
1048
+ this.usesSharedRows = true;
966
1049
  this.version = journal.version;
967
1050
  this.lastSync = journal.lastSync;
968
1051
  this.pulls = structuredClone(journal.pulls ?? []);
969
1052
  return;
970
1053
  }
971
- this.files.clear();
972
- for (const [key, entry] of Object.entries(journal.files))
973
- this.files.set(key, entry);
1054
+ if (this.usesSharedRows)
1055
+ this.files = new JournalRowStore({ trackChanges: false });
1056
+ this.files.replaceFromRecord(journal.files);
1057
+ this.usesSharedRows = false;
974
1058
  this.version = journal.version;
975
1059
  this.lastSync = journal.lastSync;
976
1060
  this.pulls = structuredClone(journal.pulls ?? []);
@@ -996,7 +1080,9 @@ export class JournalStore {
996
1080
  if (this.disposed)
997
1081
  return;
998
1082
  this.disposed = true;
999
- this.files.clear();
1083
+ this.overlay.clear();
1084
+ if (!this.usesSharedRows)
1085
+ this.files.clear();
1000
1086
  this.store = undefined;
1001
1087
  this.areaMigration = undefined;
1002
1088
  this.pulls = [];
@@ -1006,6 +1092,9 @@ export class JournalStore {
1006
1092
  throw new Error(`journal: session for ${this.slug} is disposed`);
1007
1093
  }
1008
1094
  }
1095
+ function isJournalStorePrefixMatch(key, normalizedPrefix) {
1096
+ return normalizedPrefix === "" || key === normalizedPrefix || key.startsWith(`${normalizedPrefix}/`);
1097
+ }
1009
1098
  /** Return a detached, deeply frozen JSON-shaped metadata value. */
1010
1099
  function immutableCopy(value) {
1011
1100
  const copy = structuredClone(value);
@@ -1411,22 +1500,70 @@ export function listJournals() {
1411
1500
  export function normalizeJournalKeys(journal) {
1412
1501
  if (!journal.files)
1413
1502
  return 0;
1503
+ const rows = journalRows(journal);
1504
+ const rewrites = [];
1414
1505
  let rewritten = 0;
1415
- // Snapshot keys up front — we mutate journal.files during the walk.
1416
- for (const key of Object.keys(journal.files)) {
1506
+ // Preserve only malformed keys while scanning. This avoids an all-key array
1507
+ // on a normal journal, where the common result is no rewrite at all.
1508
+ for (const key of rows.keys()) {
1417
1509
  const posix = toPosixKey(key);
1418
1510
  if (posix === key)
1419
1511
  continue; // already canonical (no backslash)
1420
- const entry = journal.files[key];
1421
- delete journal.files[key];
1512
+ const entry = rows.get(key);
1513
+ if (entry !== undefined)
1514
+ rewrites.push({ key, posix, entry });
1515
+ }
1516
+ for (const { key, posix, entry } of rewrites) {
1517
+ rows.delete(key);
1422
1518
  rewritten++;
1423
1519
  // POSIX twin already present → it wins; drop the malformed duplicate.
1424
- if (!(posix in journal.files)) {
1425
- journal.files[posix] = entry;
1426
- }
1520
+ if (!rows.has(posix))
1521
+ rows.set(posix, entry);
1427
1522
  }
1428
1523
  return rewritten;
1429
1524
  }
1525
+ /**
1526
+ * Normalize a store-backed journal without repeatedly walking its immutable
1527
+ * packed base. The first checkpoint verifies every inherited key because old
1528
+ * state may already contain a Windows-shaped path. Once that pass succeeds,
1529
+ * only keys named by the dirty log can introduce a new malformed spelling.
1530
+ */
1531
+ function normalizeStoreBackedJournalKeys(backing, journal) {
1532
+ if (!backing.keysNormalized) {
1533
+ const rewritten = normalizeJournalKeys(journal);
1534
+ backing.keysNormalized = true;
1535
+ return rewritten;
1536
+ }
1537
+ let rewritten = 0;
1538
+ const rows = backing.store;
1539
+ const dirtyPaths = new Set(backing.store.dirtyLogSnapshot().map((change) => change.path));
1540
+ for (const key of dirtyPaths) {
1541
+ const posix = toPosixKey(key);
1542
+ if (posix === key)
1543
+ continue;
1544
+ const entry = rows.get(key);
1545
+ // A delete of a malformed key already leaves no non-canonical row. A
1546
+ // canonical twin wins when both were changed in one checkpoint.
1547
+ if (entry === undefined)
1548
+ continue;
1549
+ rows.delete(key);
1550
+ rewritten++;
1551
+ if (!rows.has(posix))
1552
+ rows.set(posix, entry);
1553
+ }
1554
+ return rewritten;
1555
+ }
1556
+ function migrateJournalShapeToV2(journal) {
1557
+ if (journal.version === "2" && Array.isArray(journal.pulls) && typeof journal.lastSync === "string") {
1558
+ return journal;
1559
+ }
1560
+ journal.version = JOURNAL_VERSION_CURRENT;
1561
+ if (typeof journal.lastSync !== "string")
1562
+ journal.lastSync = "";
1563
+ if (!Array.isArray(journal.pulls))
1564
+ journal.pulls = [];
1565
+ return journal;
1566
+ }
1430
1567
  /**
1431
1568
  * Coerce any-version journal into a v2 shape. Idempotent for v2 inputs.
1432
1569
  * Mutates the input and returns it for chainable use. Call this immediately
@@ -1442,15 +1579,7 @@ export function migrateToV2(journal) {
1442
1579
  // already carry version "2" (the landmine predates the schema bump), so this
1443
1580
  // must precede the v2 early-return or already-v2 journals stay poisoned.
1444
1581
  normalizeJournalKeys(journal);
1445
- if (journal.version === "2" && Array.isArray(journal.pulls) && typeof journal.lastSync === "string") {
1446
- return journal;
1447
- }
1448
- journal.version = JOURNAL_VERSION_CURRENT;
1449
- if (typeof journal.lastSync !== "string")
1450
- journal.lastSync = "";
1451
- if (!Array.isArray(journal.pulls))
1452
- journal.pulls = [];
1453
- return journal;
1582
+ return migrateJournalShapeToV2(journal);
1454
1583
  }
1455
1584
  /**
1456
1585
  * Write a journal to disk, migrating to the current schema version in-place.
@@ -1459,36 +1588,31 @@ export function migrateToV2(journal) {
1459
1588
  */
1460
1589
  export function writeJournal(slug, journal) {
1461
1590
  const storeBacking = storeBackedJournals.get(journal);
1462
- const mutationTracking = mutationTrackedJournals.get(journal);
1463
- // A full-pull journal was normalized when its shallow tracked map was
1464
- // opened. Re-running normalizeJournalKeys here would enumerate every row at
1465
- // each 10k download checkpoint. Preserve writeJournal's general migration
1466
- // contract when metadata changes shape or a pass introduces a non-POSIX key.
1467
- const mutationTrackingNeedsMigration = mutationTracking !== undefined && (mutationTracking.needsNormalization ||
1468
- journal.version !== JOURNAL_VERSION_CURRENT ||
1469
- !Array.isArray(journal.pulls) ||
1470
- typeof journal.lastSync !== "string");
1471
- if (!mutationTracking || mutationTrackingNeedsMigration) {
1591
+ if (storeBacking) {
1592
+ // The first checkpoint repairs inherited poison. Afterwards the dirty log
1593
+ // is the complete set of newly introduced keys, so stay O(changes).
1594
+ normalizeStoreBackedJournalKeys(storeBacking, journal);
1595
+ if (storeBackedJournalNeedsMigration(storeBacking, journal))
1596
+ migrateJournalShapeToV2(journal);
1597
+ }
1598
+ else {
1472
1599
  migrateToV2(journal);
1473
- if (mutationTracking)
1474
- mutationTracking.needsNormalization = false;
1475
1600
  }
1476
1601
  const areaMigration = openAreaMigrationIfAuthoritative(slug);
1477
1602
  if (areaMigration) {
1478
1603
  const baseline = journalBaselines.get(journal);
1479
1604
  const delta = storeBacking
1480
1605
  ? journalDeltaFromStoreDirtyLog(storeBacking, journal)
1481
- : mutationTracking
1482
- ? journalDeltaFromMutationLog(mutationTracking, journal)
1483
- : baseline
1484
- ? journalDeltaFromBaseline(baseline, journal)
1485
- : journalDelta(areaMigration.readAreaJournal(), journal);
1486
- if (hasJournalDelta(delta))
1487
- areaMigration.reverseShadow(toAreaLedgerDelta(delta));
1606
+ : baseline
1607
+ ? journalDeltaFromBaseline(baseline, journal)
1608
+ : journalDelta(areaMigration.readAreaJournal(), journal);
1609
+ if (hasJournalDelta(delta)) {
1610
+ areaMigration.reverseShadow(toAreaLedgerDelta(delta), {
1611
+ releaseLegacyStoreImmediately: fullPassAreaJournals.has(journal),
1612
+ });
1613
+ }
1488
1614
  if (storeBacking)
1489
1615
  checkpointStoreBackedJournal(storeBacking, journal);
1490
- else if (mutationTracking)
1491
- checkpointMutationTrackedJournal(mutationTracking, journal);
1492
1616
  else
1493
1617
  rememberJournalBaseline(journal);
1494
1618
  ensureJournalLocator(slug);
@@ -1500,8 +1624,6 @@ export function writeJournal(slug, journal) {
1500
1624
  // and there is no delta to record. Only the locator is missing.
1501
1625
  if (storeBacking)
1502
1626
  checkpointStoreBackedJournal(storeBacking, journal);
1503
- else if (mutationTracking)
1504
- checkpointMutationTrackedJournal(mutationTracking, journal);
1505
1627
  writeJournalLocator(slug);
1506
1628
  return;
1507
1629
  }
@@ -1522,18 +1644,14 @@ export function writeJournal(slug, journal) {
1522
1644
  const baseline = journalBaselines.get(journal);
1523
1645
  const delta = storeBacking
1524
1646
  ? journalDeltaFromStoreDirtyLog(storeBacking, journal)
1525
- : mutationTracking
1526
- ? journalDeltaFromMutationLog(mutationTracking, journal)
1527
- : baseline
1528
- ? journalDeltaFromBaseline(baseline, journal)
1529
- : journalDelta(store.getState().journal, journal);
1647
+ : baseline
1648
+ ? journalDeltaFromBaseline(baseline, journal)
1649
+ : journalDelta(store.getState().journal, journal);
1530
1650
  store.append(JOURNAL_DELTA_RECORD_TYPE, delta);
1531
1651
  // A caller can checkpoint more than once during one pass. Advancing only its
1532
1652
  // own tracking state means the next checkpoint again publishes only new mutations.
1533
1653
  if (storeBacking)
1534
1654
  checkpointStoreBackedJournal(storeBacking, journal);
1535
- else if (mutationTracking)
1536
- checkpointMutationTrackedJournal(mutationTracking, journal);
1537
1655
  else
1538
1656
  rememberJournalBaseline(journal);
1539
1657
  // The locator is immutable content, so it only needs writing if it is absent
@@ -1549,8 +1667,10 @@ export function writeJournal(slug, journal) {
1549
1667
  */
1550
1668
  function journalDelta(previous, next) {
1551
1669
  const upserts = {};
1552
- for (const [key, entry] of Object.entries(next.files)) {
1553
- const before = previous.files[key];
1670
+ const previousRows = journalRows(previous);
1671
+ const nextRows = journalRows(next);
1672
+ for (const [key, entry] of nextRows.entries()) {
1673
+ const before = previousRows.get(key);
1554
1674
  // Value comparison, not identity: `previous` was just deserialized from the
1555
1675
  // snapshot on disk, so nothing in it shares a reference with the caller's
1556
1676
  // journal and an identity check would mark every file changed.
@@ -1559,8 +1679,8 @@ function journalDelta(previous, next) {
1559
1679
  upserts[key] = entry;
1560
1680
  }
1561
1681
  const deletes = [];
1562
- for (const key of Object.keys(previous.files)) {
1563
- if (!(key in next.files))
1682
+ for (const key of previousRows.keys()) {
1683
+ if (!nextRows.has(key))
1564
1684
  deletes.push(key);
1565
1685
  }
1566
1686
  const delta = {
@@ -1909,7 +2029,7 @@ export function updateEntry(journal, relativePath, hash, size, direction, verify
1909
2029
  if (skillMetadataPending === true) {
1910
2030
  entry.skillMetadataPending = true;
1911
2031
  }
1912
- journal.files[relativePath] = entry;
2032
+ journalRows(journal).set(relativePath, entry);
1913
2033
  journal.lastSync = new Date().toISOString();
1914
2034
  }
1915
2035
  /**
@@ -1923,10 +2043,10 @@ export function normalizeEtag(etag) {
1923
2043
  return etag.replace(/^"|"$/g, "");
1924
2044
  }
1925
2045
  export function getEntry(journal, relativePath) {
1926
- return journal.files[relativePath];
2046
+ return journalRows(journal).get(relativePath);
1927
2047
  }
1928
2048
  export function removeEntry(journal, relativePath) {
1929
- delete journal.files[relativePath];
2049
+ journalRows(journal).delete(relativePath);
1930
2050
  }
1931
2051
  // ─── Journal v2 (US-005): pulls, tombstones, GC ─────────────────────────────
1932
2052
  /** Crockford base32 alphabet (ULID-compatible). */
@@ -2027,10 +2147,10 @@ export function appendPullRecord(journal, record) {
2027
2147
  * They are GC'd after `TOMBSTONE_TTL_MS` via `gcTombstones`.
2028
2148
  */
2029
2149
  export function tombstoneEntry(journal, relativePath, reason, now = new Date().toISOString()) {
2030
- const entry = journal.files[relativePath];
2150
+ const entry = journalRows(journal).get(relativePath);
2031
2151
  if (!entry)
2032
2152
  return;
2033
- journal.files[relativePath] = { ...entry, removedAt: now, removedReason: reason };
2153
+ journalRows(journal).set(relativePath, { ...entry, removedAt: now, removedReason: reason });
2034
2154
  }
2035
2155
  /**
2036
2156
  * Mark a journal entry for a user-authorized local delete. Callers must
@@ -2038,14 +2158,14 @@ export function tombstoneEntry(journal, relativePath, reason, now = new Date().t
2038
2158
  * alone is never deletion intent.
2039
2159
  */
2040
2160
  export function markLocalDeleteIntent(journal, relativePath, localHash, localKind, expectedRemoteEtag) {
2041
- const entry = journal.files[relativePath];
2161
+ const entry = journalRows(journal).get(relativePath);
2042
2162
  if (!entry?.remoteEtag ||
2043
2163
  entry.remoteEtag !== normalizeEtag(expectedRemoteEtag) ||
2044
2164
  entry.hash !== localHash ||
2045
2165
  entry.kind !== localKind) {
2046
2166
  return false;
2047
2167
  }
2048
- journal.files[relativePath] = {
2168
+ journalRows(journal).set(relativePath, {
2049
2169
  ...entry,
2050
2170
  localDeleteIntent: {
2051
2171
  version: 1,
@@ -2053,16 +2173,16 @@ export function markLocalDeleteIntent(journal, relativePath, localHash, localKin
2053
2173
  localHash,
2054
2174
  localKind,
2055
2175
  },
2056
- };
2176
+ });
2057
2177
  return true;
2058
2178
  }
2059
2179
  /** Invalidate a prior delete authorization after a watcher-observed recreate. */
2060
2180
  export function clearLocalDeleteIntent(journal, relativePath) {
2061
- const entry = journal.files[relativePath];
2181
+ const entry = journalRows(journal).get(relativePath);
2062
2182
  if (!entry?.localDeleteIntent)
2063
2183
  return false;
2064
2184
  const { localDeleteIntent: _localDeleteIntent, ...cleared } = entry;
2065
- journal.files[relativePath] = cleared;
2185
+ journalRows(journal).set(relativePath, cleared);
2066
2186
  return true;
2067
2187
  }
2068
2188
  /** True if the entry is a tombstone (set by `tombstoneEntry`). */
@@ -2077,14 +2197,14 @@ export function isTombstone(entry) {
2077
2197
  */
2078
2198
  export function gcTombstones(journal, now = Date.now()) {
2079
2199
  let removed = 0;
2080
- for (const [path, entry] of Object.entries(journal.files)) {
2200
+ for (const [path, entry] of journalRows(journal).entries()) {
2081
2201
  if (!entry.removedAt)
2082
2202
  continue;
2083
2203
  const removedTime = Date.parse(entry.removedAt);
2084
2204
  if (Number.isNaN(removedTime))
2085
2205
  continue;
2086
2206
  if (now - removedTime > TOMBSTONE_TTL_MS) {
2087
- delete journal.files[path];
2207
+ journalRows(journal).delete(path);
2088
2208
  removed++;
2089
2209
  }
2090
2210
  }