@optimystic/db-core 1.25.0 → 1.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/src/cluster/structs.d.ts +10 -0
  2. package/dist/src/cluster/structs.d.ts.map +1 -1
  3. package/dist/src/cohort-topic/addressing.d.ts +26 -2
  4. package/dist/src/cohort-topic/addressing.d.ts.map +1 -1
  5. package/dist/src/cohort-topic/addressing.js +35 -4
  6. package/dist/src/cohort-topic/addressing.js.map +1 -1
  7. package/dist/src/cohort-topic/member-engine.d.ts +17 -10
  8. package/dist/src/cohort-topic/member-engine.d.ts.map +1 -1
  9. package/dist/src/cohort-topic/member-engine.js +20 -18
  10. package/dist/src/cohort-topic/member-engine.js.map +1 -1
  11. package/dist/src/cohort-topic/registration/renewal.d.ts +21 -14
  12. package/dist/src/cohort-topic/registration/renewal.d.ts.map +1 -1
  13. package/dist/src/cohort-topic/registration/renewal.js +38 -28
  14. package/dist/src/cohort-topic/registration/renewal.js.map +1 -1
  15. package/dist/src/collection/collection.d.ts +123 -34
  16. package/dist/src/collection/collection.d.ts.map +1 -1
  17. package/dist/src/collection/collection.js +231 -74
  18. package/dist/src/collection/collection.js.map +1 -1
  19. package/dist/src/collection/segments.d.ts +71 -14
  20. package/dist/src/collection/segments.d.ts.map +1 -1
  21. package/dist/src/collection/segments.js +94 -14
  22. package/dist/src/collection/segments.js.map +1 -1
  23. package/dist/src/network/i-key-network.d.ts +28 -0
  24. package/dist/src/network/i-key-network.d.ts.map +1 -1
  25. package/dist/src/network/i-repo.d.ts +16 -0
  26. package/dist/src/network/i-repo.d.ts.map +1 -1
  27. package/dist/src/offline/queue.d.ts +13 -0
  28. package/dist/src/offline/queue.d.ts.map +1 -1
  29. package/dist/src/offline/queue.js +22 -3
  30. package/dist/src/offline/queue.js.map +1 -1
  31. package/dist/src/reactivity/topic-anchor.d.ts.map +1 -1
  32. package/dist/src/reactivity/topic-anchor.js +20 -2
  33. package/dist/src/reactivity/topic-anchor.js.map +1 -1
  34. package/dist/src/transaction/coordinator.d.ts +1 -1
  35. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  36. package/dist/src/transaction/coordinator.js +15 -16
  37. package/dist/src/transaction/coordinator.js.map +1 -1
  38. package/dist/src/transactor/network-transactor.d.ts +16 -3
  39. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  40. package/dist/src/transactor/network-transactor.js +22 -5
  41. package/dist/src/transactor/network-transactor.js.map +1 -1
  42. package/dist/src/transform/atomic.d.ts +8 -2
  43. package/dist/src/transform/atomic.d.ts.map +1 -1
  44. package/dist/src/transform/atomic.js +10 -4
  45. package/dist/src/transform/atomic.js.map +1 -1
  46. package/dist/src/transform/helpers.d.ts +29 -3
  47. package/dist/src/transform/helpers.d.ts.map +1 -1
  48. package/dist/src/transform/helpers.js +62 -3
  49. package/dist/src/transform/helpers.js.map +1 -1
  50. package/dist/src/transform/tracker.d.ts +57 -5
  51. package/dist/src/transform/tracker.d.ts.map +1 -1
  52. package/dist/src/transform/tracker.js +111 -11
  53. package/dist/src/transform/tracker.js.map +1 -1
  54. package/dist/src/utility/expiring-lru-map.d.ts +16 -0
  55. package/dist/src/utility/expiring-lru-map.d.ts.map +1 -0
  56. package/dist/src/utility/expiring-lru-map.js +39 -0
  57. package/dist/src/utility/expiring-lru-map.js.map +1 -0
  58. package/package.json +1 -1
  59. package/src/cluster/structs.ts +10 -0
  60. package/src/cohort-topic/addressing.ts +42 -4
  61. package/src/cohort-topic/member-engine.ts +35 -28
  62. package/src/cohort-topic/registration/renewal.ts +62 -42
  63. package/src/collection/collection.ts +3462 -3272
  64. package/src/collection/segments.ts +140 -22
  65. package/src/network/i-key-network.ts +30 -0
  66. package/src/network/i-repo.ts +16 -0
  67. package/src/offline/queue.ts +23 -3
  68. package/src/reactivity/topic-anchor.ts +23 -3
  69. package/src/transaction/coordinator.ts +15 -16
  70. package/src/transactor/network-transactor.ts +24 -6
  71. package/src/transform/atomic.ts +12 -5
  72. package/src/transform/helpers.ts +61 -3
  73. package/src/transform/tracker.ts +110 -12
  74. package/src/utility/expiring-lru-map.ts +42 -0
@@ -56,9 +56,8 @@ export function emptyTransforms(): Transforms {
56
56
  *
57
57
  * @pitfall Both `inserts` and `updates` MUST be deep cloned. A shallow copy like
58
58
  * `{ ...transform.updates }` or `{ ...transform.inserts }` shares references, causing
59
- * mutations in one consumer to affect others. Specifically, `Tracker.update` mutates
60
- * inserted block objects in place via `applyOperation`, so a shallow-cloned snapshot
61
- * tracker will leak ops back into the original `Transforms.inserts`.
59
+ * mutations in one consumer to affect others. {@link shallowCopyTransforms} is the one
60
+ * exception, and only for values that `Tracker` alone changes.
62
61
  * @see docs/internals.md "Shallow Copy of Transforms" pitfall
63
62
  */
64
63
  export function copyTransforms(transform: Transforms): Transforms {
@@ -71,6 +70,21 @@ export function copyTransforms(transform: Transforms): Transforms {
71
70
  return { inserts, updates, deletes: transform.deletes ? [...transform.deletes] : undefined };
72
71
  }
73
72
 
73
+ /**
74
+ * New outer maps and a new `deletes` over the SAME inserted blocks and update lists, for a staged-state
75
+ * snapshot and whatever is built from it. Safe only because `Tracker` is the sole thing that changes
76
+ * those values and copies one before it first changes it, so anything that changes staged values
77
+ * outside `Tracker` needs {@link copyTransforms}.
78
+ * @see docs/internals.md "Shallow Copy of Transforms" pitfall
79
+ */
80
+ export function shallowCopyTransforms(transform: Transforms): Transforms {
81
+ return {
82
+ inserts: { ...transform.inserts },
83
+ updates: transform.updates ? { ...transform.updates } : undefined,
84
+ deletes: transform.deletes ? [...transform.deletes] : undefined,
85
+ };
86
+ }
87
+
74
88
  export function mergeTransforms(a: Transforms, b: Transforms): Transforms {
75
89
  // updates: for a block id present in both sides, concatenate a's ops then b's ops
76
90
  // (order-preserving) rather than letting b's array replace a's. inserts stay last-wins.
@@ -85,6 +99,50 @@ export function mergeTransforms(a: Transforms, b: Transforms): Transforms {
85
99
  };
86
100
  }
87
101
 
102
+ /**
103
+ * Fold `delta` into `target` under `Tracker`'s semantics — what `target` would hold had a tracker
104
+ * holding it absorbed `delta` (`Tracker.absorb`): a delete removes the id's insert and updates and
105
+ * lists it deleted; an insert replaces the id's block and takes it off the deletes; an update of a
106
+ * block `target` inserts is applied into that block, and otherwise appended to the id's update list.
107
+ * Changes `target`'s maps in place but never a value in them: a value the delta changes is replaced
108
+ * by a changed copy, so a snapshot sharing `target`'s values ({@link shallowCopyTransforms}) stays as
109
+ * captured. `delta`'s values are taken by reference and never changed, so a delta folded here can be
110
+ * folded again elsewhere.
111
+ *
112
+ * `SegmentList.seal` folds the open segment's per-action deltas into one with it.
113
+ * {@link mergeTransforms} is the other combination, for transforms that are not one tracker's: it
114
+ * keeps an insert and a later update of it apart and never takes anything off the deletes.
115
+ *
116
+ * NOTE: an update of a block `target` inserts copies that block once per delta that updates it — at
117
+ * the seal, one copy per later action of a transaction that writes a node an earlier action of it
118
+ * created. If a long transaction ever shows it, fold a second update into the copy the first made.
119
+ */
120
+ export function foldTransforms(target: Transforms, delta: Transforms): void {
121
+ const inserts = target.inserts ??= {};
122
+ const updates = target.updates ??= {};
123
+ const deletes = target.deletes ??= [];
124
+ for (const id of delta.deletes ?? []) {
125
+ delete inserts[id];
126
+ delete updates[id];
127
+ if (!deletes.includes(id)) deletes.push(id);
128
+ }
129
+ for (const [id, block] of Object.entries(delta.inserts ?? {})) {
130
+ inserts[id] = block;
131
+ const index = deletes.indexOf(id);
132
+ if (index >= 0) deletes.splice(index, 1);
133
+ }
134
+ for (const [id, ops] of Object.entries(delta.updates ?? {})) {
135
+ if (Object.hasOwn(inserts, id)) {
136
+ const block = structuredClone(inserts[id]!);
137
+ applyOperations(block, ops);
138
+ inserts[id] = block;
139
+ } else {
140
+ const held = updates[id];
141
+ updates[id] = held === undefined ? ops : [...held, ...ops];
142
+ }
143
+ }
144
+ }
145
+
88
146
  export function isTransformsEmpty(transform: Transforms): boolean {
89
147
  return Object.keys(transform.inserts ?? {}).length === 0
90
148
  && Object.keys(transform.updates ?? {}).length === 0
@@ -1,8 +1,8 @@
1
1
  import type { IBlock, BlockId, BlockStore as IBlockStore, BlockHeader, BlockOperation, BlockType, BlockSource as IBlockSource, ReadPurpose } from "../index.js";
2
- import { applyOperation, applyOperations, applyTransform, emptyTransforms, blockIdsForTransforms, transformForBlockId } from "./helpers.js";
2
+ import { applyOperation, applyOperations, applyTransform, emptyTransforms, blockIdsForTransforms, shallowCopyTransforms, transformForBlockId } from "./helpers.js";
3
+ import type { Transforms } from "./struct.js";
3
4
  import { BasePins } from "./base-pins.js";
4
5
  import type { PinnedBase } from "./base-pins.js";
5
- import { ensured } from "../utility/ensured.js";
6
6
 
7
7
  /** The two base probes a source may offer, duck-typed because Tracker layers over test doubles as
8
8
  * well as CacheSource. `peek` must return a clone (CacheSource's does). */
@@ -37,6 +37,14 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
37
37
  * the base block content the memo was built from. */
38
38
  private materialized = new Map<BlockId, { block: T; gen: number }>();
39
39
 
40
+ /** The ids whose staged value (the inserted block, or the update list) only this tracker
41
+ * references, so it may change that value in place. Any other value is shared with a snapshot
42
+ * ({@link shareTransforms}) or was handed in (the constructor, {@link reset}), and
43
+ * {@link stageUpdate} copies it before its first change. One set covers both shapes because
44
+ * {@link stageUpdate} reaches an id's update list only while the id has no staged insert, and
45
+ * only {@link delete}, which removes both, takes an insert away. */
46
+ private readonly owned = new Set<BlockId>();
47
+
40
48
  constructor(
41
49
  private readonly source: IBlockSource<T>,
42
50
  /** The collected set of transformations to be applied. Treat as immutable */
@@ -308,15 +316,12 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
308
316
  insert(block: T) {
309
317
  const inserts = this.transforms.inserts ??= {};
310
318
  inserts[block.header.id] = structuredClone(block);
319
+ this.owned.add(block.header.id);
311
320
  // Served from `inserts` now, not source+updates — the materialized memo no longer applies.
312
321
  this.materialized.delete(block.header.id);
313
322
  // An insert makes the materialized result base-independent, so any pinned base is moot.
314
323
  this.pins.delete(block.header.id);
315
- const deletes = this.transforms.deletes;
316
- const deleteIndex = deletes?.indexOf(block.header.id) ?? -1;
317
- if (deleteIndex >= 0) {
318
- deletes!.splice(deleteIndex, 1);
319
- }
324
+ this.unlistDelete(block.header.id);
320
325
  }
321
326
 
322
327
  update(blockId: BlockId, op: BlockOperation) {
@@ -332,8 +337,11 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
332
337
  * the block at a newer revision), and probing would put the newer revision on operations built
333
338
  * for the older one — the wrong base the storage-side guard cannot catch. An adopted pin the
334
339
  * source no longer describes at its revision is judged moved on its next use, like any pin. An
335
- * update the child did NOT pin (a blind one) is pinned here exactly as a direct update would be. */
336
- absorb(child: Tracker<T>): void {
340
+ * update the child did NOT pin (a blind one) is pinned here exactly as a direct update would be.
341
+ *
342
+ * @returns the child's transforms, as they were. Everything kept here was copied from them, so
343
+ * the caller may keep them: `Collection.stageInto` records them as what the actions staged. */
344
+ absorb(child: Tracker<T>): Transforms {
337
345
  this.pins.adopt(child.pins);
338
346
  const adopted = new Set(child.pins.ids());
339
347
  const transform = child.reset();
@@ -345,6 +353,56 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
345
353
  else this.recheckPin(blockId);
346
354
  }
347
355
  }
356
+ return transform;
357
+ }
358
+
359
+ /** Stage the ids in `touched` again from `deltas` alone — the transforms of the units still
360
+ * staged, in staging order — leaving every other id as it is. For taking a committed head
361
+ * segment out of a collection's live tracker (`Collection.subtractCommittedHead`): the source
362
+ * now holds what the head staged over those ids, so each is forgotten here ({@link forget}) and
363
+ * then re-folded from the remaining deltas exactly as {@link absorb} folds a child — deletes,
364
+ * inserts, then updates, one delta after another — over the source as it is now. An id left
365
+ * update-only pins its base from the source now ({@link pinBase}). That is not a re-pin of
366
+ * operations built for another base: the remaining operations were computed against content
367
+ * that included the head's effect, and that content is what the source holds since the head
368
+ * committed. The deltas are left as they are; what is kept here is copied, as {@link absorb}
369
+ * copies. Synchronous, with nothing between the forgetting and the re-folding.
370
+ *
371
+ * NOTE: every touched id is looked up in every delta, so a head commit over a queue of n
372
+ * segments costs O(touched × n) lookups here — the small quadratic term left in a drain. If it
373
+ * ever shows up, keep an id-to-segments index beside the deltas and visit only the deltas that
374
+ * touch the id. */
375
+ refold(touched: Iterable<BlockId>, deltas: readonly Transforms[]): void {
376
+ const ids = [...new Set(touched)];
377
+ for (const id of ids) this.forget(id);
378
+ for (const delta of deltas) {
379
+ for (const id of ids) {
380
+ if (delta.deletes?.includes(id)) this.delete(id);
381
+ if (delta.inserts && Object.hasOwn(delta.inserts, id)) this.insert(delta.inserts[id] as T);
382
+ for (const op of delta.updates?.[id] ?? []) {
383
+ if (this.stageUpdate(id, op)) this.pinBase(id);
384
+ else this.recheckPin(id);
385
+ }
386
+ }
387
+ }
388
+ }
389
+
390
+ /** Drop everything this tracker holds for `id` — its staged insert, update list or delete, the
391
+ * memo, the ownership and the pin — as if nothing had been staged for it. */
392
+ private forget(id: BlockId): void {
393
+ if (this.transforms.inserts) delete this.transforms.inserts[id];
394
+ if (this.transforms.updates) delete this.transforms.updates[id];
395
+ this.unlistDelete(id);
396
+ this.owned.delete(id);
397
+ this.materialized.delete(id);
398
+ this.pins.delete(id);
399
+ }
400
+
401
+ /** Take `id` off the staged deletes, if it is there. */
402
+ private unlistDelete(id: BlockId): void {
403
+ const deletes = this.transforms.deletes;
404
+ const index = deletes?.indexOf(id) ?? -1;
405
+ if (index >= 0) deletes!.splice(index, 1);
348
406
  }
349
407
 
350
408
  /** Stage `op` for `blockId` — folded into a staged insert if there is one, else appended to the
@@ -353,11 +411,10 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
353
411
  private stageUpdate(blockId: BlockId, op: BlockOperation): boolean {
354
412
  const inserted = this.transforms.inserts?.[blockId];
355
413
  if (inserted) {
356
- applyOperation(inserted, op);
414
+ applyOperation(this.ownedInsert(blockId, inserted), op);
357
415
  return false;
358
416
  }
359
- const updates = this.transforms.updates ??= {};
360
- const ops = ensured(updates, blockId, () => []);
417
+ const ops = this.ownedUpdates(blockId);
361
418
  const first = ops.length === 0;
362
419
  ops.push(structuredClone(op));
363
420
  // The memo already equals (base source content + prior ops); applying just the new op
@@ -371,6 +428,28 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
371
428
  return first;
372
429
  }
373
430
 
431
+ /** `id`'s staged insert, safe to change in place: `applyOperation` mutates it, so a block this
432
+ * tracker does not own is replaced by a clone first. */
433
+ private ownedInsert(id: BlockId, inserted: IBlock): IBlock {
434
+ if (this.owned.has(id)) return inserted;
435
+ const copy = structuredClone(inserted);
436
+ this.transforms.inserts![id] = copy;
437
+ this.owned.add(id);
438
+ return copy;
439
+ }
440
+
441
+ /** `id`'s staged update list, safe to append to. A list this tracker does not own is replaced by
442
+ * a plain array copy, which is enough because a staged operation is never changed once pushed. */
443
+ private ownedUpdates(id: BlockId): BlockOperation[] {
444
+ const updates = this.transforms.updates ??= {};
445
+ const held = updates[id];
446
+ if (held && this.owned.has(id)) return held;
447
+ const list: BlockOperation[] = held ? [...held] : [];
448
+ updates[id] = list;
449
+ this.owned.add(id);
450
+ return list;
451
+ }
452
+
374
453
  /** Fix the base of `id`'s staged operations at the moment the FIRST of them is staged in this
375
454
  * tracker: the caller just read the block, so the base source describes what the operation was
376
455
  * computed against. Whatever the store held for the id before is replaced (or dropped, when the
@@ -395,6 +474,7 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
395
474
  delete(blockId: BlockId) {
396
475
  if (this.transforms.inserts) delete this.transforms.inserts[blockId];
397
476
  if (this.transforms.updates) delete this.transforms.updates[blockId];
477
+ this.owned.delete(blockId);
398
478
  this.materialized.delete(blockId);
399
479
  // A delete materializes to nothing (delete-last-wins), so the pinned base is moot.
400
480
  this.pins.delete(blockId);
@@ -405,6 +485,7 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
405
485
  reset(newTransform = emptyTransforms()) {
406
486
  const oldTransform = this.transforms;
407
487
  this.transforms = newTransform;
488
+ this.owned.clear();
408
489
  this.materialized.clear();
409
490
  // The single reclamation point for pins: a plain reset clears them (empty updates), a
410
491
  // rollback-style reset(transforms) keeps exactly the pins — moved marks included — for ids
@@ -413,6 +494,23 @@ export class Tracker<T extends IBlock> implements IBlockStore<T> {
413
494
  return oldTransform;
414
495
  }
415
496
 
497
+ /** The staged transforms for a snapshot to keep: new outer maps over the same values, plus a copy
498
+ * of `deletes`. The maps must be new because {@link insert}, {@link update} and {@link delete}
499
+ * write to them; the values can be shared because this tracker gives up owning them here, so it
500
+ * copies each one before changing it again.
501
+ *
502
+ * NOTE: a capture costs one map entry per staged block, across every queued transaction, and the
503
+ * Quereus bridge takes one per statement per written collection. Staged blocks are tree nodes
504
+ * shared by many rows, so their count grows far more slowly than the queue. The first change to a
505
+ * shared update list after a capture also copies that list, whose length grows with the writes
506
+ * staged to the block. If either ever shows in a profile, keep what the sealed segments staged in
507
+ * a layer the captures share; a capture cannot simply leave it out, because `Tree.readView` reads
508
+ * those blocks through the snapshot. */
509
+ shareTransforms(): Transforms {
510
+ this.owned.clear();
511
+ return shallowCopyTransforms(this.transforms);
512
+ }
513
+
416
514
  transformedBlockIds(): BlockId[] {
417
515
  return blockIdsForTransforms(this.transforms);
418
516
  }
@@ -0,0 +1,42 @@
1
+ import { LruMap } from './lru-map.js';
2
+
3
+ interface Stamped<V> {
4
+ readonly value: V;
5
+ usedAt: number;
6
+ }
7
+
8
+ /**
9
+ * An {@link LruMap} whose entries also expire `idleMs` after their last use. The size cap bounds memory
10
+ * while the cache is busy; the idle expiry releases entries a long-lived process stopped asking for even
11
+ * when the cap is never reached. Expiry is judged on lookup, so the map needs no timer and no teardown.
12
+ */
13
+ export class ExpiringLruMap<K, V> {
14
+ private readonly entries: LruMap<K, Stamped<V>>;
15
+
16
+ constructor(maxSize: number, private readonly idleMs: number, private readonly now: () => number = Date.now) {
17
+ if (!(idleMs > 0)) throw new Error('ExpiringLruMap idleMs must be > 0');
18
+ this.entries = new LruMap(maxSize);
19
+ }
20
+
21
+ /** The live value for `key`, refreshing its recency and its idle clock; `undefined` once expired. */
22
+ get(key: K): V | undefined {
23
+ const entry = this.entries.get(key);
24
+ if (entry === undefined) return undefined;
25
+ const at = this.now();
26
+ if (at - entry.usedAt > this.idleMs) {
27
+ this.entries.delete(key);
28
+ return undefined;
29
+ }
30
+ entry.usedAt = at;
31
+ return entry.value;
32
+ }
33
+
34
+ set(key: K, value: V): this {
35
+ this.entries.set(key, { value, usedAt: this.now() });
36
+ return this;
37
+ }
38
+
39
+ get size(): number {
40
+ return this.entries.size;
41
+ }
42
+ }