@indigoai-us/hq-cloud 6.16.34 → 6.16.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  2. package/dist/bin/sync-runner-watch-loop.js +20 -8
  3. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  4. package/dist/bin/sync-runner-watch-loop.test.js +1 -2
  5. package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
  6. package/dist/bin/sync-runner.test.js +21 -24
  7. package/dist/bin/sync-runner.test.js.map +1 -1
  8. package/dist/cli/share.d.ts.map +1 -1
  9. package/dist/cli/share.js +18 -7
  10. package/dist/cli/share.js.map +1 -1
  11. package/dist/cli/share.test.js +30 -3
  12. package/dist/cli/share.test.js.map +1 -1
  13. package/dist/cli/sync.d.ts.map +1 -1
  14. package/dist/cli/sync.js +3 -2
  15. package/dist/cli/sync.js.map +1 -1
  16. package/dist/cli/sync.test.js +6 -6
  17. package/dist/cli/sync.test.js.map +1 -1
  18. package/dist/company-vault-transport.d.ts +0 -2
  19. package/dist/company-vault-transport.d.ts.map +1 -1
  20. package/dist/company-vault-transport.js +0 -2
  21. package/dist/company-vault-transport.js.map +1 -1
  22. package/dist/index.d.ts +2 -2
  23. package/dist/index.d.ts.map +1 -1
  24. package/dist/index.js +1 -1
  25. package/dist/index.js.map +1 -1
  26. package/dist/journal-baseline.test.js +18 -15
  27. package/dist/journal-baseline.test.js.map +1 -1
  28. package/dist/journal-delta-writes.test.js +2 -2
  29. package/dist/journal-delta-writes.test.js.map +1 -1
  30. package/dist/journal-public-boundary.test.d.ts +2 -0
  31. package/dist/journal-public-boundary.test.d.ts.map +1 -0
  32. package/dist/journal-public-boundary.test.js +216 -0
  33. package/dist/journal-public-boundary.test.js.map +1 -0
  34. package/dist/journal-record-proxy.d.ts +11 -0
  35. package/dist/journal-record-proxy.d.ts.map +1 -0
  36. package/dist/journal-record-proxy.js +71 -0
  37. package/dist/journal-record-proxy.js.map +1 -0
  38. package/dist/journal-row-store.d.ts +76 -3
  39. package/dist/journal-row-store.d.ts.map +1 -1
  40. package/dist/journal-row-store.js +450 -43
  41. package/dist/journal-row-store.js.map +1 -1
  42. package/dist/journal-row-store.test.js +139 -6
  43. package/dist/journal-row-store.test.js.map +1 -1
  44. package/dist/journal-snapshot-decoder.d.ts +18 -0
  45. package/dist/journal-snapshot-decoder.d.ts.map +1 -0
  46. package/dist/journal-snapshot-decoder.js +373 -0
  47. package/dist/journal-snapshot-decoder.js.map +1 -0
  48. package/dist/journal-snapshot-memory-helpers.d.ts +30 -0
  49. package/dist/journal-snapshot-memory-helpers.d.ts.map +1 -0
  50. package/dist/journal-snapshot-memory-helpers.js +203 -0
  51. package/dist/journal-snapshot-memory-helpers.js.map +1 -0
  52. package/dist/journal-snapshot-memory.test.d.ts +2 -0
  53. package/dist/journal-snapshot-memory.test.d.ts.map +1 -0
  54. package/dist/journal-snapshot-memory.test.js +32 -0
  55. package/dist/journal-snapshot-memory.test.js.map +1 -0
  56. package/dist/journal-snapshot-v4-memory.test.d.ts +2 -0
  57. package/dist/journal-snapshot-v4-memory.test.d.ts.map +1 -0
  58. package/dist/journal-snapshot-v4-memory.test.js +24 -0
  59. package/dist/journal-snapshot-v4-memory.test.js.map +1 -0
  60. package/dist/journal-snapshot-v4.d.ts +52 -0
  61. package/dist/journal-snapshot-v4.d.ts.map +1 -0
  62. package/dist/journal-snapshot-v4.js +352 -0
  63. package/dist/journal-snapshot-v4.js.map +1 -0
  64. package/dist/journal-snapshot-v4.test.d.ts +2 -0
  65. package/dist/journal-snapshot-v4.test.d.ts.map +1 -0
  66. package/dist/journal-snapshot-v4.test.js +280 -0
  67. package/dist/journal-snapshot-v4.test.js.map +1 -0
  68. package/dist/journal-store.test.js +335 -17
  69. package/dist/journal-store.test.js.map +1 -1
  70. package/dist/journal.d.ts +10 -8
  71. package/dist/journal.d.ts.map +1 -1
  72. package/dist/journal.js +165 -50
  73. package/dist/journal.js.map +1 -1
  74. package/dist/journal.test.js +1 -1
  75. package/dist/journal.test.js.map +1 -1
  76. package/dist/manifest/build-manifest.bench.test.js +165 -58
  77. package/dist/manifest/build-manifest.bench.test.js.map +1 -1
  78. package/dist/remote-pull.js +6 -4
  79. package/dist/remote-pull.js.map +1 -1
  80. package/dist/remote-pull.test.js +72 -0
  81. package/dist/remote-pull.test.js.map +1 -1
  82. package/dist/session-log-retention.test.js +3 -3
  83. package/dist/session-log-retention.test.js.map +1 -1
  84. package/dist/skill-telemetry.d.ts +1 -2
  85. package/dist/skill-telemetry.d.ts.map +1 -1
  86. package/dist/skill-telemetry.js +1 -10
  87. package/dist/skill-telemetry.js.map +1 -1
  88. package/dist/sync/child-process-sync-worker-env.test.d.ts +2 -0
  89. package/dist/sync/child-process-sync-worker-env.test.d.ts.map +1 -0
  90. package/dist/sync/child-process-sync-worker-env.test.js +38 -0
  91. package/dist/sync/child-process-sync-worker-env.test.js.map +1 -0
  92. package/dist/sync/feature-flags.d.ts +0 -16
  93. package/dist/sync/feature-flags.d.ts.map +1 -1
  94. package/dist/sync/feature-flags.js +0 -11
  95. package/dist/sync/feature-flags.js.map +1 -1
  96. package/dist/sync/index.d.ts +2 -2
  97. package/dist/sync/index.d.ts.map +1 -1
  98. package/dist/sync/index.js +1 -1
  99. package/dist/sync/index.js.map +1 -1
  100. package/dist/sync/logger.test.js +1 -3
  101. package/dist/sync/logger.test.js.map +1 -1
  102. package/dist/sync/metrics.test.js +0 -3
  103. package/dist/sync/metrics.test.js.map +1 -1
  104. package/dist/sync/push-receiver.d.ts +0 -7
  105. package/dist/sync/push-receiver.d.ts.map +1 -1
  106. package/dist/sync/push-receiver.js.map +1 -1
  107. package/dist/sync/state-store.d.ts +23 -0
  108. package/dist/sync/state-store.d.ts.map +1 -1
  109. package/dist/sync/state-store.js +114 -39
  110. package/dist/sync/state-store.js.map +1 -1
  111. package/dist/sync/state-store.test.js +82 -0
  112. package/dist/sync/state-store.test.js.map +1 -1
  113. package/dist/watcher.test.js +7 -8
  114. package/dist/watcher.test.js.map +1 -1
  115. package/package.json +1 -1
  116. package/dist/daemon-worker.d.ts +0 -11
  117. package/dist/daemon-worker.d.ts.map +0 -1
  118. package/dist/daemon-worker.js +0 -22
  119. package/dist/daemon-worker.js.map +0 -1
@@ -40,6 +40,23 @@ const SYNCED_AT_FALLBACK_PRESENT = 1 << 15;
40
40
  const textDecoder = new TextDecoder();
41
41
  const textEncoder = new TextEncoder();
42
42
  const lowerHex = /^[0-9a-f]+$/;
43
+ const CANONICAL_ENTRY_KEY_ORDER = [
44
+ "createdBySub",
45
+ "ctimeMs",
46
+ "direction",
47
+ "hash",
48
+ "kind",
49
+ "localDeleteIntent",
50
+ "localDiverges",
51
+ "mtimeMs",
52
+ "outOfScopeProtected",
53
+ "remoteEtag",
54
+ "removedAt",
55
+ "removedReason",
56
+ "size",
57
+ "skillMetadataPending",
58
+ "syncedAt",
59
+ ];
43
60
  function splitPath(path) {
44
61
  const slash = path.lastIndexOf("/");
45
62
  return slash === -1
@@ -93,6 +110,40 @@ function freezeEntry(entry) {
93
110
  Object.freeze(entry.localDeleteIntent);
94
111
  return Object.freeze(entry);
95
112
  }
113
+ /** Compact before-image used by the dirty log to avoid retaining row objects. */
114
+ export function journalEntryFingerprint(entry) {
115
+ const record = entry;
116
+ const keys = Object.keys(record).sort();
117
+ let text = "";
118
+ for (const key of keys) {
119
+ const value = record[key];
120
+ text += key;
121
+ text += "\u0001";
122
+ if (value === undefined)
123
+ text += "u";
124
+ else if (value === null)
125
+ text += "n";
126
+ else if (typeof value === "object")
127
+ text += "o" + JSON.stringify(value);
128
+ else if (typeof value === "string")
129
+ text += "s" + JSON.stringify(value);
130
+ else
131
+ text += typeof value + String(value);
132
+ text += "\u0000";
133
+ }
134
+ let h1 = 0xdeadbeef;
135
+ let h2 = 0x41c6ce57;
136
+ for (let index = 0; index < text.length; index++) {
137
+ const character = text.charCodeAt(index);
138
+ h1 = Math.imul(h1 ^ character, 2654435761);
139
+ h2 = Math.imul(h2 ^ character, 1597334677);
140
+ }
141
+ h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
142
+ h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
143
+ h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
144
+ h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
145
+ return 4294967296 * (2097151 & h2) + (h1 >>> 0);
146
+ }
96
147
  /**
97
148
  * The record-like view used by internal callers. It has no object-indexing
98
149
  * semantics: paths and rows are materialised only at the operation boundary.
@@ -145,6 +196,9 @@ export class JournalRowStore {
145
196
  createdBySubs;
146
197
  dirIds;
147
198
  leaves;
199
+ propertyOrderIds;
200
+ iterationNext;
201
+ iterationPrevious;
148
202
  leafBlob;
149
203
  leafLength = 0;
150
204
  buckets;
@@ -161,11 +215,26 @@ export class JournalRowStore {
161
215
  removedAts = new Map();
162
216
  removedReasons = new Map();
163
217
  localDeleteIntents = new Map();
218
+ propertyOrders = [[...CANONICAL_ENTRY_KEY_ORDER]];
219
+ propertyOrderIdsByLayout = new Map([
220
+ [CANONICAL_ENTRY_KEY_ORDER.join("\u0000"), 0],
221
+ ]);
164
222
  dirtyLog = [];
223
+ dirtyPaths = new Set();
224
+ trackChanges;
165
225
  nextRowId = 0;
166
226
  liveRows = 0;
227
+ firstIterationRowId = -1;
228
+ lastIterationRowId = -1;
167
229
  rows = new JournalRows(this);
168
- constructor(initialCapacity = INITIAL_ROW_CAPACITY) {
230
+ constructor(initialCapacityOrOptions = INITIAL_ROW_CAPACITY, options = {}) {
231
+ const initialCapacity = typeof initialCapacityOrOptions === "number"
232
+ ? initialCapacityOrOptions
233
+ : INITIAL_ROW_CAPACITY;
234
+ const selectedOptions = typeof initialCapacityOrOptions === "number"
235
+ ? options
236
+ : initialCapacityOrOptions;
237
+ this.trackChanges = selectedOptions.trackChanges ?? true;
169
238
  const capacity = Math.max(INITIAL_ROW_CAPACITY, initialCapacity);
170
239
  this.hashes = new Uint8Array(HASH_BYTES * capacity);
171
240
  this.remoteEtags = new Uint8Array(ETAG_BYTES * capacity);
@@ -177,12 +246,21 @@ export class JournalRowStore {
177
246
  this.createdBySubs = new Uint32Array(capacity);
178
247
  this.dirIds = new Uint32Array(capacity);
179
248
  this.leaves = new Uint32Array(capacity);
249
+ this.propertyOrderIds = new Uint16Array(capacity);
250
+ this.iterationNext = new Int32Array(capacity).fill(-1);
251
+ this.iterationPrevious = new Int32Array(capacity).fill(-1);
180
252
  this.leafBlob = new Uint8Array(Math.max(INITIAL_LEAF_CAPACITY, capacity * 32));
181
253
  this.buckets = new Uint32Array(Math.max(INITIAL_BUCKET_CAPACITY, bucketCapacityFor(capacity)));
182
254
  }
183
255
  get size() {
184
256
  return this.liveRows;
185
257
  }
258
+ /** Start retaining caller mutations after a snapshot has established its baseline. */
259
+ enableChangeTracking() {
260
+ this.trackChanges = true;
261
+ this.dirtyLog = [];
262
+ this.dirtyPaths.clear();
263
+ }
186
264
  /** Exposed for diagnostics and path-identity tests; id zero is the root directory. */
187
265
  get directoryCount() {
188
266
  return this.directories.length;
@@ -194,11 +272,18 @@ export class JournalRowStore {
194
272
  has(path) {
195
273
  return this.findRowId(path) !== undefined;
196
274
  }
275
+ /** Remove all rows and pending changes so the store can be reused cleanly. */
276
+ clear() {
277
+ this.reset(INITIAL_ROW_CAPACITY);
278
+ this.dirtyLog = [];
279
+ this.dirtyPaths.clear();
280
+ }
197
281
  set(path, entry) {
198
282
  const { directory, leaf } = splitPath(path);
199
283
  const existingRowId = this.findRowId(path);
200
284
  if (existingRowId !== undefined) {
201
- this.recordDirty(existingRowId, path, this.materialize(existingRowId));
285
+ if (this.trackChanges)
286
+ this.recordDirty(existingRowId, path, true, journalEntryFingerprint(this.materialize(existingRowId)));
202
287
  this.writeEntry(existingRowId, entry);
203
288
  this.storeLeafFallback(existingRowId, leaf, this.leafBytes(existingRowId));
204
289
  return;
@@ -207,21 +292,24 @@ export class JournalRowStore {
207
292
  const leafBytes = textEncoder.encode(leaf);
208
293
  this.ensureBucketCapacity();
209
294
  const rowId = this.allocateRow();
210
- this.recordDirty(rowId, path, undefined);
295
+ if (this.trackChanges)
296
+ this.recordDirty(rowId, path, false, undefined);
211
297
  this.dirIds[rowId] = directoryId;
212
298
  this.leaves[rowId] = this.appendLeaf(leafBytes);
213
299
  this.writeEntry(rowId, entry);
214
300
  this.storeLeafFallback(rowId, leaf, leafBytes);
215
301
  this.insertIndex(rowId);
302
+ this.appendIterationRow(rowId);
216
303
  this.liveRows++;
217
304
  }
218
305
  delete(path) {
219
306
  const rowId = this.findRowId(path);
220
307
  if (rowId === undefined)
221
308
  return false;
222
- const previous = this.materialize(rowId);
223
- this.recordDirty(rowId, path, previous);
309
+ if (this.trackChanges)
310
+ this.recordDirty(rowId, path, true, journalEntryFingerprint(this.materialize(rowId)));
224
311
  this.deleteIndex(rowId);
312
+ this.removeIterationRow(rowId);
225
313
  this.clearRow(rowId);
226
314
  this.freeRows.push(rowId);
227
315
  this.freeRowSet.add(rowId);
@@ -229,15 +317,31 @@ export class JournalRowStore {
229
317
  return true;
230
318
  }
231
319
  *entries() {
232
- for (let rowId = 0; rowId < this.nextRowId; rowId++) {
233
- if (this.freeRowSet.has(rowId))
234
- continue;
320
+ for (let rowId = this.firstIterationRowId; rowId !== -1; rowId = this.iterationNext[rowId]) {
235
321
  yield [this.pathForRow(rowId), this.materialize(rowId)];
236
322
  }
237
323
  }
238
324
  *keys() {
239
- for (const [path] of this.entries())
240
- yield path;
325
+ for (let rowId = this.firstIterationRowId; rowId !== -1; rowId = this.iterationNext[rowId]) {
326
+ yield this.pathForRow(rowId);
327
+ }
328
+ }
329
+ *entriesForPrefix(prefix) {
330
+ const normalizedPrefix = prefix.replace(/\/+$/, "");
331
+ if (normalizedPrefix === "") {
332
+ yield* this.entries();
333
+ return;
334
+ }
335
+ const exactRowId = this.findRowId(normalizedPrefix);
336
+ const descendantPrefix = `${normalizedPrefix}/`;
337
+ for (let rowId = this.firstIterationRowId; rowId !== -1; rowId = this.iterationNext[rowId]) {
338
+ const directory = this.directories[this.dirIds[rowId]];
339
+ if (rowId === exactRowId ||
340
+ directory === normalizedPrefix ||
341
+ directory.startsWith(descendantPrefix)) {
342
+ yield [this.pathForRow(rowId), this.materialize(rowId)];
343
+ }
344
+ }
241
345
  }
242
346
  *values() {
243
347
  for (const [, entry] of this.entries())
@@ -255,12 +359,26 @@ export class JournalRowStore {
255
359
  }
256
360
  return record;
257
361
  }
258
- /** Returns and clears changes since the prior checkpoint. */
362
+ /**
363
+ * Returns and clears changes since the prior checkpoint.
364
+ *
365
+ * A non-tracking store has no change stream; asking it to drain is a caller
366
+ * error rather than an empty checkpoint.
367
+ */
259
368
  drainDirtyLog() {
369
+ if (!this.trackChanges)
370
+ throw new Error("Change tracking is disabled for this journal row store");
260
371
  const drained = this.dirtyLog;
261
372
  this.dirtyLog = [];
373
+ this.dirtyPaths.clear();
262
374
  return drained;
263
375
  }
376
+ /** Read pending changes without acknowledging them; writeJournal snapshots before append. */
377
+ dirtyLogSnapshot() {
378
+ if (!this.trackChanges)
379
+ throw new Error("Change tracking is disabled for this journal row store");
380
+ return this.dirtyLog;
381
+ }
264
382
  /** Alias for callers that treat the dirty log as a change stream. */
265
383
  drainChanges() {
266
384
  return this.drainDirtyLog();
@@ -270,13 +388,109 @@ export class JournalRowStore {
270
388
  * row ids in already-emitted records remain meaningful.
271
389
  */
272
390
  compact() {
273
- if (this.dirtyLog.length !== 0)
391
+ if (this.trackChanges && this.dirtyLog.length !== 0)
274
392
  throw new Error("Drain the journal dirty log before compaction");
275
393
  const entries = [...this.entries()];
276
394
  this.reset(Math.max(INITIAL_ROW_CAPACITY, entries.length));
277
395
  for (const [path, entry] of entries)
278
396
  this.insertWithoutDirtyLog(path, entry);
279
397
  }
398
+ /**
399
+ * Export a hole-free immutable view for the HQSNAP4 encoder. Snapshot
400
+ * publication is already a compaction boundary, but this keeps the encoder
401
+ * correct when a future caller invokes it before that boundary.
402
+ */
403
+ toSnapshot() {
404
+ if (this.nextRowId !== this.liveRows || this.freeRows.length !== 0) {
405
+ const compacted = new JournalRowStore(this.liveRows, { trackChanges: false });
406
+ for (const [path, entry] of this.entries())
407
+ compacted.insertWithoutDirtyLog(path, entry);
408
+ return compacted.toSnapshot();
409
+ }
410
+ return {
411
+ rowCount: this.liveRows,
412
+ hashes: this.hashes.subarray(0, this.liveRows * HASH_BYTES),
413
+ remoteEtags: this.remoteEtags.subarray(0, this.liveRows * ETAG_BYTES),
414
+ sizes: this.sizes.subarray(0, this.liveRows),
415
+ mtimes: this.mtimes.subarray(0, this.liveRows),
416
+ ctimes: this.ctimes.subarray(0, this.liveRows),
417
+ syncedAts: this.syncedAts.subarray(0, this.liveRows),
418
+ flags: this.flags.subarray(0, this.liveRows),
419
+ createdBySubs: this.createdBySubs.subarray(0, this.liveRows),
420
+ dirIds: this.dirIds.subarray(0, this.liveRows),
421
+ leaves: this.leaves.subarray(0, this.liveRows),
422
+ propertyOrderIds: this.propertyOrderIds.subarray(0, this.liveRows),
423
+ iterationNext: this.iterationNext.subarray(0, this.liveRows),
424
+ iterationPrevious: this.iterationPrevious.subarray(0, this.liveRows),
425
+ leafBlob: this.leafBlob.subarray(0, this.leafLength),
426
+ buckets: this.buckets,
427
+ directories: [...this.directories],
428
+ authors: [...this.authors],
429
+ propertyOrders: this.propertyOrders.map((order) => [...order]),
430
+ hashFallbacks: sortedStringEntries(this.hashFallbacks),
431
+ etagFallbacks: sortedStringEntries(this.etagFallbacks),
432
+ syncedAtFallbacks: sortedStringEntries(this.syncedAtFallbacks),
433
+ leafFallbacks: sortedStringEntries(this.leafFallbacks),
434
+ removedAts: sortedStringEntries(this.removedAts),
435
+ removedReasons: sortedStringEntries(this.removedReasons),
436
+ localDeleteIntents: [...this.localDeleteIntents.entries()]
437
+ .sort(([left], [right]) => left - right)
438
+ .map(([rowId, intent]) => [rowId, cloneIntent(intent)]),
439
+ };
440
+ }
441
+ /** Install the already-authenticated HQSNAP4 column views without row parsing. */
442
+ static fromSnapshot(snapshot) {
443
+ validateSnapshot(snapshot);
444
+ const iteration = validateIterationChain(snapshot.iterationNext, snapshot.iterationPrevious);
445
+ const store = new JournalRowStore(snapshot.rowCount, { trackChanges: false });
446
+ store.hashes = snapshot.hashes;
447
+ store.remoteEtags = snapshot.remoteEtags;
448
+ store.sizes = snapshot.sizes;
449
+ store.mtimes = snapshot.mtimes;
450
+ store.ctimes = snapshot.ctimes;
451
+ store.syncedAts = snapshot.syncedAts;
452
+ store.flags = snapshot.flags;
453
+ store.createdBySubs = snapshot.createdBySubs;
454
+ store.dirIds = snapshot.dirIds;
455
+ store.leaves = snapshot.leaves;
456
+ store.propertyOrderIds = snapshot.propertyOrderIds;
457
+ store.iterationNext = snapshot.iterationNext;
458
+ store.iterationPrevious = snapshot.iterationPrevious;
459
+ store.leafBlob = new Uint8Array(Math.max(INITIAL_LEAF_CAPACITY, snapshot.leafBlob.length));
460
+ store.leafBlob.set(snapshot.leafBlob);
461
+ store.leafLength = snapshot.leafBlob.length;
462
+ store.buckets = snapshot.buckets;
463
+ replaceStrings(store.directories, snapshot.directories, "directory");
464
+ rebuildStringIds(store.directoryIds, store.directories, 0, "directory");
465
+ replaceStrings(store.authors, snapshot.authors, "author");
466
+ rebuildStringIds(store.authorIds, store.authors, 1, "author");
467
+ store.propertyOrders.splice(0, store.propertyOrders.length, ...snapshot.propertyOrders.map((order) => [...order]));
468
+ store.propertyOrderIdsByLayout.clear();
469
+ for (const [id, order] of store.propertyOrders.entries()) {
470
+ const layout = order.join("\u0000");
471
+ if (store.propertyOrderIdsByLayout.has(layout))
472
+ throw new Error("HQSNAP4 has duplicate property order");
473
+ store.propertyOrderIdsByLayout.set(layout, id);
474
+ }
475
+ restoreStringEntries(store.hashFallbacks, snapshot.hashFallbacks, snapshot.rowCount, "hash fallback");
476
+ restoreStringEntries(store.etagFallbacks, snapshot.etagFallbacks, snapshot.rowCount, "etag fallback");
477
+ restoreStringEntries(store.syncedAtFallbacks, snapshot.syncedAtFallbacks, snapshot.rowCount, "synced-at fallback");
478
+ restoreStringEntries(store.leafFallbacks, snapshot.leafFallbacks, snapshot.rowCount, "leaf fallback");
479
+ restoreStringEntries(store.removedAts, snapshot.removedAts, snapshot.rowCount, "removed-at");
480
+ restoreStringEntries(store.removedReasons, snapshot.removedReasons, snapshot.rowCount, "removed-reason");
481
+ for (const [rowId, intent] of snapshot.localDeleteIntents) {
482
+ assertSnapshotRowId(rowId, snapshot.rowCount, "local delete intent");
483
+ if (store.localDeleteIntents.has(rowId) || !isLocalDeleteIntent(intent)) {
484
+ throw new Error("HQSNAP4 has an invalid local delete intent");
485
+ }
486
+ store.localDeleteIntents.set(rowId, cloneIntent(intent));
487
+ }
488
+ store.nextRowId = snapshot.rowCount;
489
+ store.liveRows = snapshot.rowCount;
490
+ store.firstIterationRowId = iteration.first;
491
+ store.lastIterationRowId = iteration.last;
492
+ return store;
493
+ }
280
494
  reset(rowCapacity) {
281
495
  this.hashes = new Uint8Array(HASH_BYTES * rowCapacity);
282
496
  this.remoteEtags = new Uint8Array(ETAG_BYTES * rowCapacity);
@@ -288,6 +502,9 @@ export class JournalRowStore {
288
502
  this.createdBySubs = new Uint32Array(rowCapacity);
289
503
  this.dirIds = new Uint32Array(rowCapacity);
290
504
  this.leaves = new Uint32Array(rowCapacity);
505
+ this.propertyOrderIds = new Uint16Array(rowCapacity);
506
+ this.iterationNext = new Int32Array(rowCapacity).fill(-1);
507
+ this.iterationPrevious = new Int32Array(rowCapacity).fill(-1);
291
508
  this.leafBlob = new Uint8Array(Math.max(INITIAL_LEAF_CAPACITY, rowCapacity * 8));
292
509
  this.leafLength = 0;
293
510
  this.buckets = new Uint32Array(bucketCapacityFor(rowCapacity));
@@ -307,6 +524,8 @@ export class JournalRowStore {
307
524
  this.localDeleteIntents.clear();
308
525
  this.nextRowId = 0;
309
526
  this.liveRows = 0;
527
+ this.firstIterationRowId = -1;
528
+ this.lastIterationRowId = -1;
310
529
  }
311
530
  insertWithoutDirtyLog(path, entry) {
312
531
  const { directory, leaf } = splitPath(path);
@@ -318,10 +537,14 @@ export class JournalRowStore {
318
537
  this.writeEntry(rowId, entry);
319
538
  this.storeLeafFallback(rowId, leaf, leafBytes);
320
539
  this.insertIndex(rowId);
540
+ this.appendIterationRow(rowId);
321
541
  this.liveRows++;
322
542
  }
323
- recordDirty(rowId, path, previous) {
324
- this.dirtyLog.push(Object.freeze({ rowId, path, previous }));
543
+ recordDirty(rowId, path, hadPrevious, previousFingerprint) {
544
+ if (this.dirtyPaths.has(path))
545
+ return;
546
+ this.dirtyPaths.add(path);
547
+ this.dirtyLog.push(Object.freeze({ rowId, path, hadPrevious, previousFingerprint }));
325
548
  }
326
549
  allocateRow() {
327
550
  const reused = this.freeRows.pop();
@@ -336,6 +559,7 @@ export class JournalRowStore {
336
559
  ensureRowCapacity(required) {
337
560
  if (required <= this.flags.length)
338
561
  return;
562
+ const previousCapacity = this.flags.length;
339
563
  const capacity = Math.max(required, this.flags.length * 2);
340
564
  this.hashes = growUint8(this.hashes, capacity * HASH_BYTES);
341
565
  this.remoteEtags = growUint8(this.remoteEtags, capacity * ETAG_BYTES);
@@ -347,13 +571,24 @@ export class JournalRowStore {
347
571
  this.createdBySubs = growUint32(this.createdBySubs, capacity);
348
572
  this.dirIds = growUint32(this.dirIds, capacity);
349
573
  this.leaves = growUint32(this.leaves, capacity);
574
+ this.propertyOrderIds = growUint16(this.propertyOrderIds, capacity);
575
+ this.iterationNext = growInt32(this.iterationNext, capacity);
576
+ this.iterationPrevious = growInt32(this.iterationPrevious, capacity);
577
+ this.iterationNext.fill(-1, previousCapacity);
578
+ this.iterationPrevious.fill(-1, previousCapacity);
350
579
  }
351
580
  writeEntry(rowId, entry) {
352
581
  const directoryId = this.dirIds[rowId];
353
582
  const leafOffset = this.leaves[rowId];
583
+ const nextIterationRowId = this.iterationNext[rowId];
584
+ const previousIterationRowId = this.iterationPrevious[rowId];
585
+ const propertyOrderId = this.propertyOrderIdFor(entry);
354
586
  this.clearRow(rowId);
355
587
  this.dirIds[rowId] = directoryId;
356
588
  this.leaves[rowId] = leafOffset;
589
+ this.iterationNext[rowId] = nextIterationRowId;
590
+ this.iterationPrevious[rowId] = previousIterationRowId;
591
+ this.propertyOrderIds[rowId] = propertyOrderId;
357
592
  let flags = entry.direction === "down" ? DIRECTION_DOWN : 0;
358
593
  if (isCanonicalHex(entry.hash, HASH_BYTES * 2)) {
359
594
  decodeHex(this.hashes, rowId * HASH_BYTES, entry.hash);
@@ -420,6 +655,9 @@ export class JournalRowStore {
420
655
  this.createdBySubs[rowId] = 0;
421
656
  this.dirIds[rowId] = 0;
422
657
  this.leaves[rowId] = 0;
658
+ this.propertyOrderIds[rowId] = 0;
659
+ this.iterationNext[rowId] = -1;
660
+ this.iterationPrevious[rowId] = -1;
423
661
  this.hashFallbacks.delete(rowId);
424
662
  this.etagFallbacks.delete(rowId);
425
663
  this.syncedAtFallbacks.delete(rowId);
@@ -428,43 +666,86 @@ export class JournalRowStore {
428
666
  this.removedReasons.delete(rowId);
429
667
  this.localDeleteIntents.delete(rowId);
430
668
  }
669
+ propertyOrderIdFor(entry) {
670
+ const keys = Object.keys(entry);
671
+ const layout = keys.join("\u0000");
672
+ const known = this.propertyOrderIdsByLayout.get(layout);
673
+ if (known !== undefined)
674
+ return known;
675
+ const id = this.propertyOrders.length;
676
+ if (id > 0xffff)
677
+ throw new Error("Too many journal entry property layouts");
678
+ this.propertyOrders.push(keys);
679
+ this.propertyOrderIdsByLayout.set(layout, id);
680
+ return id;
681
+ }
682
+ appendIterationRow(rowId) {
683
+ const previous = this.lastIterationRowId;
684
+ this.iterationPrevious[rowId] = previous;
685
+ this.iterationNext[rowId] = -1;
686
+ if (previous === -1)
687
+ this.firstIterationRowId = rowId;
688
+ else
689
+ this.iterationNext[previous] = rowId;
690
+ this.lastIterationRowId = rowId;
691
+ }
692
+ removeIterationRow(rowId) {
693
+ const previous = this.iterationPrevious[rowId];
694
+ const next = this.iterationNext[rowId];
695
+ if (previous === -1)
696
+ this.firstIterationRowId = next;
697
+ else
698
+ this.iterationNext[previous] = next;
699
+ if (next === -1)
700
+ this.lastIterationRowId = previous;
701
+ else
702
+ this.iterationPrevious[next] = previous;
703
+ this.iterationNext[rowId] = -1;
704
+ this.iterationPrevious[rowId] = -1;
705
+ }
431
706
  materialize(rowId) {
432
707
  const flags = this.flags[rowId];
433
- const entry = {
434
- hash: flags & HASH_FALLBACK_PRESENT
435
- ? this.hashFallbacks.get(rowId)
436
- : encodeHex(this.hashes, rowId * HASH_BYTES, HASH_BYTES),
437
- size: this.sizes[rowId],
438
- syncedAt: flags & SYNCED_AT_FALLBACK_PRESENT
439
- ? this.syncedAtFallbacks.get(rowId)
440
- : new Date(this.syncedAts[rowId]).toISOString(),
441
- direction: flags & DIRECTION_DOWN ? "down" : "up",
442
- };
443
- if (flags & KIND_PRESENT)
444
- entry.kind = flags & KIND_SYMLINK ? "symlink" : "file";
708
+ const values = {};
445
709
  if (flags & CREATED_BY_SUB_PRESENT)
446
- entry.createdBySub = this.authors[this.createdBySubs[rowId]];
710
+ values.createdBySub = this.authors[this.createdBySubs[rowId]];
711
+ if (flags & CTIME_PRESENT)
712
+ values.ctimeMs = this.ctimes[rowId];
713
+ values.direction = flags & DIRECTION_DOWN ? "down" : "up";
714
+ values.hash = flags & HASH_FALLBACK_PRESENT
715
+ ? this.hashFallbacks.get(rowId)
716
+ : encodeHex(this.hashes, rowId * HASH_BYTES, HASH_BYTES);
717
+ if (flags & KIND_PRESENT)
718
+ values.kind = flags & KIND_SYMLINK ? "symlink" : "file";
719
+ const localDeleteIntent = this.localDeleteIntents.get(rowId);
720
+ if (localDeleteIntent !== undefined)
721
+ values.localDeleteIntent = cloneIntent(localDeleteIntent);
722
+ readOptionalBoolean(values, "localDiverges", flags, LOCAL_DIVERGES_PRESENT, LOCAL_DIVERGES_VALUE);
723
+ if (flags & MTIME_PRESENT)
724
+ values.mtimeMs = this.mtimes[rowId];
725
+ readOptionalBoolean(values, "outOfScopeProtected", flags, OUT_OF_SCOPE_PROTECTED_PRESENT, OUT_OF_SCOPE_PROTECTED_VALUE);
447
726
  if (flags & REMOTE_ETAG_PRESENT) {
448
- entry.remoteEtag = flags & ETAG_FALLBACK_PRESENT
727
+ values.remoteEtag = flags & ETAG_FALLBACK_PRESENT
449
728
  ? this.etagFallbacks.get(rowId)
450
729
  : encodeHex(this.remoteEtags, rowId * ETAG_BYTES, ETAG_BYTES);
451
730
  }
452
- if (flags & MTIME_PRESENT)
453
- entry.mtimeMs = this.mtimes[rowId];
454
- if (flags & CTIME_PRESENT)
455
- entry.ctimeMs = this.ctimes[rowId];
456
731
  const removedAt = this.removedAts.get(rowId);
457
732
  if (removedAt !== undefined)
458
- entry.removedAt = removedAt;
733
+ values.removedAt = removedAt;
459
734
  const removedReason = this.removedReasons.get(rowId);
460
735
  if (removedReason !== undefined)
461
- entry.removedReason = removedReason;
462
- const localDeleteIntent = this.localDeleteIntents.get(rowId);
463
- if (localDeleteIntent !== undefined)
464
- entry.localDeleteIntent = cloneIntent(localDeleteIntent);
465
- readOptionalBoolean(entry, "outOfScopeProtected", flags, OUT_OF_SCOPE_PROTECTED_PRESENT, OUT_OF_SCOPE_PROTECTED_VALUE);
466
- readOptionalBoolean(entry, "skillMetadataPending", flags, SKILL_METADATA_PENDING_PRESENT, SKILL_METADATA_PENDING_VALUE);
467
- readOptionalBoolean(entry, "localDiverges", flags, LOCAL_DIVERGES_PRESENT, LOCAL_DIVERGES_VALUE);
736
+ values.removedReason = removedReason;
737
+ values.size = this.sizes[rowId];
738
+ readOptionalBoolean(values, "skillMetadataPending", flags, SKILL_METADATA_PENDING_PRESENT, SKILL_METADATA_PENDING_VALUE);
739
+ values.syncedAt = flags & SYNCED_AT_FALLBACK_PRESENT
740
+ ? this.syncedAtFallbacks.get(rowId)
741
+ : new Date(this.syncedAts[rowId]).toISOString();
742
+ const entry = {};
743
+ const source = values;
744
+ const target = entry;
745
+ for (const key of this.propertyOrders[this.propertyOrderIds[rowId]]) {
746
+ if (Object.hasOwn(source, key))
747
+ target[key] = source[key];
748
+ }
468
749
  return freezeEntry(entry);
469
750
  }
470
751
  internDirectory(directory) {
@@ -489,9 +770,8 @@ export class JournalRowStore {
489
770
  const length = leaf.length;
490
771
  const needed = this.leafLength + 4 + length;
491
772
  if (needed > this.leafBlob.length) {
492
- let capacity = this.leafBlob.length;
493
- while (capacity < needed)
494
- capacity *= 2;
773
+ const minimumCapacity = Math.max(INITIAL_LEAF_CAPACITY, this.leafBlob.length, needed);
774
+ const capacity = 2 ** Math.ceil(Math.log2(minimumCapacity));
495
775
  this.leafBlob = growUint8(this.leafBlob, capacity);
496
776
  }
497
777
  const offset = this.leafLength;
@@ -595,6 +875,128 @@ function bucketCapacityFor(rows) {
595
875
  capacity *= 2;
596
876
  return capacity;
597
877
  }
878
+ function sortedStringEntries(source) {
879
+ return [...source.entries()].sort(([left], [right]) => left - right);
880
+ }
881
+ function replaceStrings(target, source, label) {
882
+ if (source.length === 0 || source.some((value) => typeof value !== "string")) {
883
+ throw new Error(`HQSNAP4 has an invalid ${label} table`);
884
+ }
885
+ target.splice(0, target.length, ...source);
886
+ }
887
+ function rebuildStringIds(target, values, start, label) {
888
+ target.clear();
889
+ for (let index = start; index < values.length; index += 1) {
890
+ const value = values[index];
891
+ if (target.has(value))
892
+ throw new Error(`HQSNAP4 has a duplicate ${label}`);
893
+ target.set(value, index);
894
+ }
895
+ }
896
+ function restoreStringEntries(target, source, rowCount, label) {
897
+ target.clear();
898
+ for (const [rowId, value] of source) {
899
+ assertSnapshotRowId(rowId, rowCount, label);
900
+ if (typeof value !== "string" || target.has(rowId))
901
+ throw new Error(`HQSNAP4 has an invalid ${label}`);
902
+ target.set(rowId, value);
903
+ }
904
+ }
905
+ function assertSnapshotRowId(rowId, rowCount, label) {
906
+ if (!Number.isInteger(rowId) || rowId < 0 || rowId >= rowCount) {
907
+ throw new Error(`HQSNAP4 has an invalid ${label} row id`);
908
+ }
909
+ }
910
+ function validateIterationChain(iterationNext, iterationPrevious) {
911
+ if (iterationNext.length === 0)
912
+ return { first: -1, last: -1 };
913
+ let first = -1;
914
+ let last = -1;
915
+ for (let rowId = 0; rowId < iterationNext.length; rowId += 1) {
916
+ if (iterationPrevious[rowId] === -1) {
917
+ if (first !== -1)
918
+ throw new Error("HQSNAP4 has an invalid iteration chain");
919
+ first = rowId;
920
+ }
921
+ if (iterationNext[rowId] === -1) {
922
+ if (last !== -1)
923
+ throw new Error("HQSNAP4 has an invalid iteration chain");
924
+ last = rowId;
925
+ }
926
+ }
927
+ if (first === -1 || last === -1)
928
+ throw new Error("HQSNAP4 has an invalid iteration chain");
929
+ const visited = new Uint8Array(iterationNext.length);
930
+ let previous = -1;
931
+ let rowId = first;
932
+ let visitedCount = 0;
933
+ while (rowId !== -1) {
934
+ if (rowId < 0 || rowId >= iterationNext.length || visited[rowId] !== 0
935
+ || iterationPrevious[rowId] !== previous) {
936
+ throw new Error("HQSNAP4 has an invalid iteration chain");
937
+ }
938
+ visited[rowId] = 1;
939
+ visitedCount += 1;
940
+ const next = iterationNext[rowId];
941
+ if (next !== -1 && (next < 0 || next >= iterationNext.length || iterationPrevious[next] !== rowId)) {
942
+ throw new Error("HQSNAP4 has an invalid iteration chain");
943
+ }
944
+ previous = rowId;
945
+ rowId = next;
946
+ }
947
+ if (visitedCount !== iterationNext.length || previous !== last) {
948
+ throw new Error("HQSNAP4 has an invalid iteration chain");
949
+ }
950
+ return { first, last };
951
+ }
952
+ function isLocalDeleteIntent(value) {
953
+ return value.version === 1
954
+ && typeof value.remoteEtag === "string"
955
+ && typeof value.localHash === "string"
956
+ && (value.localKind === "file" || value.localKind === "symlink");
957
+ }
958
+ function validateSnapshot(snapshot) {
959
+ const { rowCount } = snapshot;
960
+ if (!Number.isSafeInteger(rowCount) || rowCount < 0)
961
+ throw new Error("HQSNAP4 has an invalid row count");
962
+ const rowArrays = [
963
+ ["hashes", snapshot.hashes, rowCount * HASH_BYTES],
964
+ ["remote etags", snapshot.remoteEtags, rowCount * ETAG_BYTES],
965
+ ["sizes", snapshot.sizes, rowCount],
966
+ ["mtimes", snapshot.mtimes, rowCount],
967
+ ["ctimes", snapshot.ctimes, rowCount],
968
+ ["synced ats", snapshot.syncedAts, rowCount],
969
+ ["flags", snapshot.flags, rowCount],
970
+ ["created-by subs", snapshot.createdBySubs, rowCount],
971
+ ["directory ids", snapshot.dirIds, rowCount],
972
+ ["leaves", snapshot.leaves, rowCount],
973
+ ["property order ids", snapshot.propertyOrderIds, rowCount],
974
+ ["iteration next", snapshot.iterationNext, rowCount],
975
+ ["iteration previous", snapshot.iterationPrevious, rowCount],
976
+ ];
977
+ for (const [label, values, length] of rowArrays) {
978
+ if (values.length !== length)
979
+ throw new Error(`HQSNAP4 has an invalid ${label} column length`);
980
+ }
981
+ if (snapshot.buckets.length === 0 || (snapshot.buckets.length & (snapshot.buckets.length - 1)) !== 0) {
982
+ throw new Error("HQSNAP4 has an invalid bucket table");
983
+ }
984
+ if (snapshot.directories[0] !== "" || snapshot.authors[0] !== "" || snapshot.propertyOrders.length === 0) {
985
+ throw new Error("HQSNAP4 has invalid string tables");
986
+ }
987
+ for (const directoryId of snapshot.dirIds) {
988
+ if (directoryId >= snapshot.directories.length)
989
+ throw new Error("HQSNAP4 has an invalid directory id");
990
+ }
991
+ for (const authorId of snapshot.createdBySubs) {
992
+ if (authorId >= snapshot.authors.length)
993
+ throw new Error("HQSNAP4 has an invalid author id");
994
+ }
995
+ for (const propertyOrderId of snapshot.propertyOrderIds) {
996
+ if (propertyOrderId >= snapshot.propertyOrders.length)
997
+ throw new Error("HQSNAP4 has an invalid property order id");
998
+ }
999
+ }
598
1000
  function growUint8(source, length) {
599
1001
  const next = new Uint8Array(length);
600
1002
  next.set(source);
@@ -610,6 +1012,11 @@ function growUint32(source, length) {
610
1012
  next.set(source);
611
1013
  return next;
612
1014
  }
1015
+ function growInt32(source, length) {
1016
+ const next = new Int32Array(length);
1017
+ next.set(source);
1018
+ return next;
1019
+ }
613
1020
  function growFloat64(source, length) {
614
1021
  const next = new Float64Array(length);
615
1022
  next.set(source);