@volter/world-core 3.0.155 → 3.0.156

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dist/generated/pack-facts.json +143 -12
  2. package/dist/inject.cjs +150 -86
  3. package/dist/src/actions.d.ts +15 -0
  4. package/dist/src/actions.js +82 -21
  5. package/dist/src/ancestry.d.ts +5 -4
  6. package/dist/src/ancestry.js +53 -28
  7. package/dist/src/changeset.js +1 -1
  8. package/dist/src/clickhouse/compression.d.ts +28 -0
  9. package/dist/src/clickhouse/compression.js +281 -0
  10. package/dist/src/clickhouse/index.d.ts +1 -0
  11. package/dist/src/clickhouse/index.js +1 -0
  12. package/dist/src/derived-core.d.ts +12 -1
  13. package/dist/src/derived-core.js +145 -11
  14. package/dist/src/derived.js +3 -0
  15. package/dist/src/fork.js +1 -1
  16. package/dist/src/grpc-wire.js +2 -1
  17. package/dist/src/h2.js +63 -10
  18. package/dist/src/head.d.ts +3 -0
  19. package/dist/src/head.js +9 -0
  20. package/dist/src/history.d.ts +1 -1
  21. package/dist/src/history.js +7 -7
  22. package/dist/src/index.d.ts +4 -1
  23. package/dist/src/index.js +5 -1
  24. package/dist/src/log.d.ts +23 -3
  25. package/dist/src/log.js +160 -37
  26. package/dist/src/managed-original-inputs.d.ts +101 -0
  27. package/dist/src/managed-original-inputs.js +125 -0
  28. package/dist/src/mongodb/client.d.ts +8 -0
  29. package/dist/src/mongodb/client.js +27 -0
  30. package/dist/src/mongodb/index.d.ts +2 -0
  31. package/dist/src/mongodb/index.js +5 -0
  32. package/dist/src/mongodb/stream.d.ts +74 -0
  33. package/dist/src/mongodb/stream.js +206 -0
  34. package/dist/src/observe.js +1 -1
  35. package/dist/src/pack-fetch.js +20 -8
  36. package/dist/src/packRegistry.d.ts +10 -14
  37. package/dist/src/packRegistry.js +2 -13
  38. package/dist/src/protocol.d.ts +13 -0
  39. package/dist/src/protocol.js +13 -0
  40. package/dist/src/redis/engine.d.ts +10 -4
  41. package/dist/src/redis/engine.js +103 -32
  42. package/dist/src/redis/member-storage.js +20 -8
  43. package/dist/src/redis/service-times.d.ts +25 -0
  44. package/dist/src/redis/service-times.js +87 -0
  45. package/dist/src/redis/stream.d.ts +2 -1
  46. package/dist/src/redis/stream.js +109 -20
  47. package/dist/src/runtime-state.d.ts +11 -0
  48. package/dist/src/runtime-state.js +13 -0
  49. package/dist/src/runtime.d.ts +4 -2
  50. package/dist/src/runtime.js +5 -2
  51. package/dist/src/serve-http.d.ts +2 -0
  52. package/dist/src/serve-http.js +21 -4
  53. package/dist/src/serve.d.ts +10 -0
  54. package/dist/src/serve.js +43 -3
  55. package/dist/src/storage.d.ts +2 -1
  56. package/dist/src/storage.js +6 -3
  57. package/dist/src/world-boot.d.ts +1 -0
  58. package/dist/src/world-boot.js +6 -0
  59. package/dist/src/world-clock.js +6 -2
  60. package/dist/src/world-store.d.ts +108 -7
  61. package/dist/src/world-store.js +883 -103
  62. package/dist/src/xml.d.ts +30 -0
  63. package/dist/src/xml.js +74 -0
  64. package/generated/pack-facts.json +143 -12
  65. package/inject.cjs +150 -86
  66. package/package.json +20 -1
  67. package/src/actions.ts +78 -21
  68. package/src/ancestry.ts +46 -24
  69. package/src/changeset.ts +1 -1
  70. package/src/clickhouse/compression.ts +172 -0
  71. package/src/clickhouse/index.ts +2 -0
  72. package/src/derived-core.ts +142 -11
  73. package/src/derived.ts +3 -0
  74. package/src/fork.ts +1 -1
  75. package/src/grpc-wire.ts +2 -1
  76. package/src/h2.ts +46 -5
  77. package/src/head.ts +9 -0
  78. package/src/history.ts +7 -7
  79. package/src/index.ts +6 -1
  80. package/src/log.ts +147 -32
  81. package/src/managed-original-inputs.ts +163 -0
  82. package/src/mongodb/client.ts +27 -0
  83. package/src/mongodb/index.ts +5 -0
  84. package/src/mongodb/stream.ts +213 -0
  85. package/src/observe.ts +1 -1
  86. package/src/pack-fetch.ts +20 -8
  87. package/src/packRegistry.ts +3 -15
  88. package/src/protocol.ts +13 -0
  89. package/src/redis/engine.ts +100 -57
  90. package/src/redis/member-storage.ts +17 -9
  91. package/src/redis/service-times.ts +81 -0
  92. package/src/redis/stream.ts +66 -14
  93. package/src/runtime-state.ts +14 -0
  94. package/src/runtime.ts +6 -1
  95. package/src/serve-http.ts +23 -4
  96. package/src/serve.ts +45 -3
  97. package/src/storage.ts +5 -3
  98. package/src/world-boot.ts +6 -0
  99. package/src/world-clock.ts +6 -2
  100. package/src/world-store.ts +704 -91
  101. package/src/xml.ts +71 -0
package/src/log.ts CHANGED
@@ -30,7 +30,7 @@ import { resolveReferences } from './references.ts';
30
30
  import { hashFieldValue } from './hash.ts';
31
31
  import type { SubjectFields } from './hash.ts';
32
32
  import { worldPaths } from './storage.ts';
33
- import { getActiveWorldStore } from './world-store.ts';
33
+ import { getActiveWorldStore, TREE_FILES } from './world-store.ts';
34
34
  import type { WorldServiceEvent } from './types.ts';
35
35
  import type { TwinResource } from './serve.ts';
36
36
 
@@ -97,8 +97,8 @@ const TREE_PROJECTION_VERSION = 2;
97
97
  export const DELTA_SUFFIX = '.delta';
98
98
 
99
99
  export function parentLogPath(service: string, root?: string): string { return worldPaths(service, root).events; }
100
- export function branchLogPath(service: string, root?: string): string { return join(dirname(worldPaths(service, root).events), 'actions.jsonl'); }
101
- export function branchMetaPath(service: string, root?: string): string { return join(dirname(worldPaths(service, root).events), 'branch.json'); }
100
+ export function branchLogPath(service: string, root?: string): string { return join(dirname(worldPaths(service, root).events), TREE_FILES.branch); }
101
+ export function branchMetaPath(service: string, root?: string): string { return join(dirname(worldPaths(service, root).events), TREE_FILES.pointer); }
102
102
  /** The fetched cache of a URL parent's whole log — what `fetch` extends and a read folds up to the position. */
103
103
  export function originLogPath(service: string, root?: string): string { return join(dirname(worldPaths(service, root).events), 'origin.jsonl'); }
104
104
  /** A parent is at a URL when it is not a path on this machine. */
@@ -173,8 +173,9 @@ function readRows<T>(path: string): T[] {
173
173
  return rows.slice();
174
174
  }
175
175
 
176
- export function readBranchMeta(service: string, root?: string): BranchMeta | null {
177
- const raw = getActiveWorldStore().read(branchMetaPath(service, root));
176
+ export function readBranchMeta(service: string, root?: string, read?: string | null): BranchMeta | null {
177
+ // `read`: branch.json's text where the caller has just read it (a tree's stamp reads it once for both uses).
178
+ const raw = read !== undefined ? read : getActiveWorldStore().read(branchMetaPath(service, root));
178
179
  if (raw === null) return null;
179
180
  const meta = parseExactJson(raw) as BranchMeta & { parent?: { root?: string; parentCount?: number; branchCount?: number } };
180
181
  // a pointer written before positions were one number: the parent's whole log is parent then branch
@@ -228,8 +229,8 @@ export function writeBranchMeta(service: string, meta: BranchMeta, root?: string
228
229
  getActiveWorldStore().mkdir(dirname(path));
229
230
  getActiveWorldStore().writeAtomic(path, `${JSON.stringify(meta, null, 2)}\n`);
230
231
  if (meta.parent?.directory && !isUrlParent(meta.parent.at)) commitParentPin(meta.parent.directory, path);
231
- });
232
- });
232
+ }, { operation: 'pointer publication' });
233
+ }, { operation: 'pointer publication' });
233
234
  }
234
235
 
235
236
  /** A v1 observed row as an entry: a delta event folds its `changed.*.after`; anything else folds
@@ -304,7 +305,7 @@ export function appendOriginEntry(entry: Entry, root?: string): { appended: bool
304
305
  if (readRows<{ id?: string }>(path).some((r) => r.id === entry.id)) return { appended: false };
305
306
  store.append(path, `${JSON.stringify(entry)}\n`);
306
307
  return { appended: true };
307
- }));
308
+ }, { operation: 'origin append' }), { operation: 'origin append' });
308
309
  }
309
310
 
310
311
  /** Where the parent stands now: the length of its whole log (live at a path, the cache for a URL). */
@@ -338,7 +339,7 @@ export function aliasesFrom(parent: Entry[]): Map<string, string> {
338
339
  }
339
340
 
340
341
  const META = new Set(['id', 'type', 'updatedAt']);
341
- type Subject = { type: string; id: string; updatedAt: string; fields: SubjectFields };
342
+ export type Subject = { type: string; id: string; updatedAt: string; fields: SubjectFields };
342
343
  export type Tree = Map<string, Subject>;
343
344
 
344
345
  /** Last write per field. `deleted: true` is the tombstone a delete carries; it holds until the subject
@@ -351,6 +352,24 @@ function overlay(tree: Tree, type: string, id: string, fields: SubjectFields, at
351
352
  tree.set(key, { ...existing, updatedAt: at, fields: merged });
352
353
  }
353
354
 
355
+ /** One subject read by the same guarded key as a write's answer; restore the vendor's own metadata fields. */
356
+ export function heldSubject(service: string, type: string, id: string, root?: string): Subject | undefined {
357
+ const row = readSubject(service, type, id, root);
358
+ return row ? { type: row.type, id: row.id, updatedAt: row.updatedAt, fields: ownFields(row) } : undefined;
359
+ }
360
+
361
+ /**
362
+ * One local write folded onto its subject as the tree folds its entry (`overlay`): the subject as it was (undefined
363
+ * for one new to the tree) is copied, never changed. A command's writes staged in `ctx.batch` are read back by
364
+ * folding them this way, so a read inside the batch sees what the tree will hold once they land (derived-core.ts).
365
+ */
366
+ export function foldLocalWrite(previous: Subject | undefined, type: string, id: string, fields: SubjectFields, at: string): Subject {
367
+ const key = `${type}:${id}`;
368
+ const tree: Tree = previous ? new Map([[key, { ...previous, fields: { ...previous.fields } }]]) : new Map();
369
+ overlay(tree, type, id, fields, at);
370
+ return tree.get(key)!;
371
+ }
372
+
354
373
  /** Fold entries onto a tree, in order: `set` overlays its fields and applies its projection; a
355
374
  * reverted entry folds nothing; a `confirm` row (v1) folds nothing. `skip` names entries the
356
375
  * parent already holds. */
@@ -447,6 +466,22 @@ export function foldHistory(entries: Entry[], layout: HistoryLayout, service?: s
447
466
  foldAliases(tree, aliasesFrom(parent), service);
448
467
  return tree;
449
468
  }
469
+ /**
470
+ * One subject of the current tree as a resource, by its type and id: the resource `projectResources(...).find` gives
471
+ * for the same type and id, read by the subject's key without cloning or projecting the rest of the tree. Every subject is
472
+ * held under `${type}:${id}` (overlay is the only setter, and an alias re-keys through it), so the key is the
473
+ * subject. A write's answer read back the whole tree this way (serve.ts, applyTwinWrite's `resource`), once per
474
+ * written subject: a derived pack writing each document, index and collection of a database's startup paid a
475
+ * projection of every subject the World holds for each (S2 R17).
476
+ */
477
+ export function readSubject(service: string, type: string, id: string, root?: string): TwinResource | undefined {
478
+ const key = `${type}:${id}`;
479
+ const held = currentTree(service, root).tree.get(key);
480
+ // a type or id holding ':' can share a key with another (type, id): only the subject itself is it
481
+ if (!held || held.type !== type || held.id !== id) return undefined;
482
+ return treeResources(new Map([[key, { ...held, fields: { ...held.fields } }]]))[0];
483
+ }
484
+
450
485
  export function readTree(service: string, root?: string, opts: { until?: string; at?: number; view?: string } = {}): TwinResource[] {
451
486
  return treeResources(readTreeMap(service, root, opts));
452
487
  }
@@ -460,7 +495,7 @@ export function readParentTreeMap(service: string, root?: string): Tree {
460
495
  return foldHistory([...(inherited ? historyEntries(inherited) : []), ...own], {
461
496
  ...(inherited ? { base: inherited.layout } : {}), parent: own.length, branch: 0,
462
497
  }, service);
463
- });
498
+ }, { operation: 'tree fold' });
464
499
  }
465
500
 
466
501
  // A READ'S TREE, MEMOIZED per store and per process or isolate. What a read folds is named by cheap
@@ -483,11 +518,44 @@ const TREE_STEPS_KEPT = 64;
483
518
  function statKey(path: string): string { const s = getActiveWorldStore().stat(path); return s ? `${s.size}:${s.mtimeMs}` : '-'; }
484
519
  /** What a service's PARENT side at `root` is folded from (its parent log, branch.json and inherited history), as one
485
520
  * string: while it stands, its aliases and landed ids are the same, whatever the branch's own log does. */
486
- export function parentStamp(service: string, root?: string): string { return parentKeyOf(service, root); }
487
- function parentKeyOf(service: string, root?: string): string { return `${TREE_PROJECTION_VERSION}|${statKey(parentLogPath(service, root))}|${getActiveWorldStore().read(branchMetaPath(service, root)) ?? ''}|${inheritedHistoryKey(service, root)}`; }
521
+ export function parentStamp(service: string, root?: string): string { return treeFacts(service, root).parentKey; }
522
+ function parentKeyOf(service: string, root?: string): string { return parentFacts(service, root).key; }
523
+ /** The parent side's key, and whether it names a file outside the tree's own directory (an inherited view). */
524
+ function parentFacts(service: string, root?: string): { key: string; inherits: boolean } {
525
+ // branch.json is small and every stamp needs it: kept while its own facts stand (world-store.ts readKept), so an
526
+ // unchanged pointer costs one check. Its text is still in the key, so a changed pointer still moves the stamp.
527
+ const store = getActiveWorldStore(), pointer = branchMetaPath(service, root);
528
+ const meta = store.readKept ? store.readKept(pointer) : store.read(pointer);
529
+ const inherited = inheritedHistoryKey(service, root, meta);
530
+ return { key: `${TREE_PROJECTION_VERSION}|${statKey(parentLogPath(service, root))}|${meta ?? ''}|${inherited}`, inherits: inherited !== '' };
531
+ }
532
+ // ONE QUESTION PER STAMP, WHERE THE STORE KEEPS A GENERATION (world-store.ts FsWorldStore.generation, which states
533
+ // what makes it sound and where that stops). A stamp was three questions of the file system, on every operation of
534
+ // every twin: the parent log, the pointer and the branch log, the first two of files a World with no parent does not
535
+ // have. The facts are kept under the generation read BEFORE them, and served while the generation still reads the
536
+ // same. Never kept: where the directory has no generation yet (nothing has been written there by a store that keeps
537
+ // one), and where the parent side names files of other directories (an inherited view: its ancestors' writes move no
538
+ // generation here). Both are asked in full, as before.
539
+ type TreeFacts = { parentKey: string; branchKey: string };
540
+ const keptFacts = new WeakMap<object, Map<string, TreeFacts & { generation: string }>>();
541
+ function treeFacts(service: string, root?: string): TreeFacts {
542
+ const store = getActiveWorldStore();
543
+ const branchLog = branchLogPath(service, root);
544
+ if (!store.generation) return { parentKey: parentKeyOf(service, root), branchKey: statKey(branchLog) };
545
+ const directory = dirname(branchLog);
546
+ const generation = store.generation(directory);
547
+ let kept = keptFacts.get(store);
548
+ if (!kept) { kept = new Map(); keptFacts.set(store, kept); }
549
+ const held = kept.get(directory);
550
+ if (generation !== null && held?.generation === generation) return held;
551
+ const parent = parentFacts(service, root);
552
+ const facts = { parentKey: parent.key, branchKey: statKey(branchLog) };
553
+ if (generation !== null && !parent.inherits) kept.set(directory, { ...facts, generation }); else kept.delete(directory);
554
+ return facts;
555
+ }
488
556
  /** What a service's tree (and every subject's history) at `root` is folded from, as one string: while it
489
557
  * stands, nothing was written. A pack that projects the tree into its own state memoizes on it. */
490
- export function treeStamp(service: string, root?: string): string { return `${parentKeyOf(service, root)}#${statKey(branchLogPath(service, root))}`; }
558
+ export function treeStamp(service: string, root?: string): string { const facts = treeFacts(service, root); return `${facts.parentKey}#${facts.branchKey}`; }
491
559
  const cloneTree = (tree: Tree): Tree => new Map([...tree].map(([k, s]) => [k, { ...s, fields: { ...s.fields } }]));
492
560
  const memoStamp = (m: TreeMemo): string => `${m.parentKey}#${m.branchKey}`;
493
561
 
@@ -529,7 +597,7 @@ function currentTree(service: string, root?: string): { tree: Tree; memo?: TreeM
529
597
  // A hit needs no lock: a memo is only ever stored from a fold made under the history lock, and while the
530
598
  // facts still match that fold, nothing was written since. A writer mid-batch has moved them.
531
599
  const unlocked = memos.get(slot);
532
- if (unlocked && unlocked.parentKey === parentKeyOf(service, root) && unlocked.branchKey === statKey(branchLogPath(service, root))) return { tree: unlocked.tree, memo: unlocked };
600
+ if (unlocked) { const facts = treeFacts(service, root); if (unlocked.parentKey === facts.parentKey && unlocked.branchKey === facts.branchKey) return { tree: unlocked.tree, memo: unlocked }; }
533
601
  return withHistoryLock(worldPaths(service, root).dir, () => {
534
602
  const parentKey = parentKeyOf(service, root); const branchKey = statKey(branchLogPath(service, root));
535
603
  const held = memos.get(slot);
@@ -539,19 +607,7 @@ function currentTree(service: string, root?: string): { tree: Tree; memo?: TreeM
539
607
  if (held && held.parentKey === parentKey && branch.length >= held.branchCount
540
608
  && (held.branchCount === 0 ? held.lastBranchId === null : branch[held.branchCount - 1]?.id === held.lastBranchId)
541
609
  && !branch.slice(held.branchCount).some((e) => e.op === 'revert')) {
542
- const fresh = branch.slice(held.branchCount);
543
- const keys = touchedKeys(fresh, held.aliases);
544
- const from = memoStamp(held);
545
- let appended: string[];
546
- try {
547
- appended = insertionsDuring(held.tree, () => {
548
- foldEntries(held.tree, fresh, held.landed);
549
- foldAliases(held.tree, held.aliases, service, keys);
550
- });
551
- } catch (error) { memos.delete(slot); throw error; } // a half-folded memo is never read again
552
- Object.assign(held, { branchKey, branchCount: branch.length, lastBranchId: last(branch) });
553
- held.steps.push({ from, to: memoStamp(held), keys, appended });
554
- if (held.steps.length > TREE_STEPS_KEPT) held.steps.splice(0, held.steps.length - TREE_STEPS_KEPT);
610
+ extendMemo(memos, slot, held, branch.slice(held.branchCount), branchKey, service);
555
611
  return { tree: held.tree, memo: held };
556
612
  }
557
613
  const tree = readTreeMapFull(service, root);
@@ -563,7 +619,66 @@ function currentTree(service: string, root?: string): { tree: Tree; memo?: TreeM
563
619
  return { tree, memo };
564
620
  }
565
621
  return { tree };
566
- });
622
+ }, { operation: 'tree fold' });
623
+ }
624
+
625
+ /** The memo extended in place by `fresh`, the branch log's entries after its own, which leave the log at `branchKey`,
626
+ * with the step that records the subjects they touched. */
627
+ function extendMemo(memos: Map<string, TreeMemo>, slot: string, held: TreeMemo, fresh: Entry[], branchKey: string, service: string): void {
628
+ const keys = touchedKeys(fresh, held.aliases);
629
+ const from = memoStamp(held);
630
+ let appended: string[];
631
+ try {
632
+ appended = insertionsDuring(held.tree, () => {
633
+ foldEntries(held.tree, fresh, held.landed);
634
+ foldAliases(held.tree, held.aliases, service, keys);
635
+ });
636
+ } catch (error) { memos.delete(slot); throw error; } // a half-folded memo is never read again
637
+ Object.assign(held, { branchKey, branchCount: held.branchCount + fresh.length, lastBranchId: fresh.length ? fresh[fresh.length - 1]!.id : held.lastBranchId });
638
+ held.steps.push({ from, to: memoStamp(held), keys, appended });
639
+ if (held.steps.length > TREE_STEPS_KEPT) held.steps.splice(0, held.steps.length - TREE_STEPS_KEPT);
640
+ }
641
+
642
+ // AN OWN APPEND EXTENDS THE READS' MEMO (S2 R17c). A write appended its entries and the next read found the facts
643
+ // moved, so it took the history lock, read the facts again and the log's new bytes, and extended the memo with the
644
+ // entries this process had just written: in a tab, where each of those is a call into the kernel, about twenty calls
645
+ // per command that writes, taken again by the command after it. The writer, still holding the history lock, extends
646
+ // the memo with its own entries and keeps the tree's facts under the generation its own write left, so the next read
647
+ // asks one question. It does so only where the store can say its write was the one write since the generation it
648
+ // read just before (`generationAfterOwnWrite`), the facts were kept under that generation and the memo stood at them,
649
+ // the parent side's facts read after the append are still the kept ones, and the log grew by exactly its bytes;
650
+ // otherwise it extends nothing, and the next read checks as before.
651
+ /** Append `data` (whole JSONL lines of a service's branch log, `append` writing them) and, when the reads' memo held
652
+ * the tree as it stood, extend it by them. Called holding the directory's history lock. Answers the log's facts
653
+ * after the append (`size:mtimeMs`) when it read them, for a caller keyed on the same. */
654
+ export function appendToBranchLog(service: string, root: string | undefined, data: string, append: () => void): string | undefined {
655
+ const store = getActiveWorldStore();
656
+ const memos = treeMemos.get(store);
657
+ const slot = `${service}\u0000${root ?? ''}`;
658
+ const held = memos?.get(slot);
659
+ const branchLog = branchLogPath(service, root);
660
+ const directory = dirname(branchLog);
661
+ const kept = keptFacts.get(store)?.get(directory);
662
+ const standing = held !== undefined && kept !== undefined && kept.parentKey === held.parentKey && kept.branchKey === held.branchKey;
663
+ const before = standing && store.generation && store.generationAfterOwnWrite ? store.generation(directory) : null;
664
+ append();
665
+ if (!memos || !held || !kept || before === null || kept.generation !== before) return undefined;
666
+ const after = store.generationAfterOwnWrite!(directory, before);
667
+ if (after === null) return undefined;
668
+ // read after the generation, as treeFacts reads its facts. The parent side is read again too: a writer the generation
669
+ // does not see (an older version's, or one that died before its second byte) may have moved it under the kept
670
+ // generation, and carried forward its old key would outlive every append this process makes (S2 R17c review).
671
+ const branchKey = statKey(branchLog);
672
+ if (parentKeyOf(service, root) !== kept.parentKey) return branchKey;
673
+ const grown = (key: string): number => (key === '-' ? 0 : Number(key.slice(0, key.indexOf(':'))));
674
+ const fresh: Entry[] = [];
675
+ // line numbers in a refusal count from this append's first line
676
+ parseLines(`${branchLog} (appended)`, data.split('\n'), 0, fresh);
677
+ if (grown(branchKey) !== grown(held.branchKey) + Buffer.byteLength(data) || fresh.some((e) => e.op === 'revert')) return branchKey;
678
+ // the write has landed: a fold that fails here drops the memo (extendMemo) and the next read folds in full, never failing the write
679
+ try { extendMemo(memos, slot, held, fresh, branchKey, service); } catch { return branchKey; }
680
+ keptFacts.get(store)!.set(directory, { parentKey: kept.parentKey, branchKey, generation: after });
681
+ return branchKey;
567
682
  }
568
683
 
569
684
  /** What changed in a service's tree since `since` (a `treeStamp` an earlier read was made at): the subjects
@@ -649,7 +764,7 @@ function readTreeMapFull(service: string, root?: string, opts: { until?: string;
649
764
  getActiveWorldStore().writeAtomic(checkpointPath(service, root), JSON.stringify({ parentDigest, branchDigest, branchCount: branch.length, subjects: [...tree.values()], ...(now === facts ? { facts } : {}) }));
650
765
  }
651
766
  return tree;
652
- });
767
+ }, { operation: 'tree fold' });
653
768
  }
654
769
  export function cutCheckpoint(service: string, root?: string): void {
655
770
  return withHistoryLock(worldPaths(service, root).dir, () => {
@@ -657,7 +772,7 @@ export function cutCheckpoint(service: string, root?: string): void {
657
772
  const subjects = [...readTreeMap(service, root).values()];
658
773
  const inherited = inheritedHistory(service, root); const own = ownParentEntries(service, root);
659
774
  getActiveWorldStore().writeAtomic(checkpointPath(service, root), JSON.stringify({ parentDigest: hashFieldValue({ projectionVersion: TREE_PROJECTION_VERSION, inherited, own }), branchDigest: hashFieldValue(branchEntries(service, root)), branchCount: branchEntries(service, root).length, subjects }));
660
- });
775
+ }, { operation: 'checkpoint' });
661
776
  }
662
777
 
663
778
  /** The branch's entries the parent does not hold: `set` rows, not reverted, not landed. */
@@ -744,7 +859,7 @@ export function appendParentEntry(entry: Entry, root?: string): { appended: bool
744
859
  index.ids.add(entry.id);
745
860
  index.key = statKey(path);
746
861
  return { appended: true };
747
- }));
862
+ }, { operation: 'parent append' }), { operation: 'parent append' });
748
863
  }
749
864
 
750
865
  /** The whole log of a twin as a served world hands it out: the parent view, then the branch's own
@@ -815,7 +930,7 @@ export function rebaseBranch(service: string, root?: string, opts: { origin?: bo
815
930
  dropCheckpoint(service, root);
816
931
  }
817
932
  return { moved, from, to, conflicts };
818
- });
933
+ }, { operation: 'branch-pointer adjustment' });
819
934
  }
820
935
 
821
936
  /**
@@ -0,0 +1,163 @@
1
+ /** Source-bound files at a managed replacement. No vendor interpretation or host paths. */
2
+ import * as nodeCrypto from 'node:crypto';
3
+ const digest = (algorithm: 'sha1' | 'sha256', value: string | Uint8Array): string => nodeCrypto.createHash(algorithm).update(value).digest('hex');
4
+ export interface OriginalInputReference {
5
+ version: 1;
6
+ /** Relative to the needs file, or to the World configuration after init copies it. */
7
+ root: string;
8
+ manifest: "manifest.json";
9
+ sha256: string;
10
+ inventoryDigest: string;
11
+ }
12
+
13
+ export interface OriginalInputCapability {
14
+ version: 1;
15
+ manifestArg: "--original-inputs";
16
+ rootArg: "--original-root";
17
+ digestArg: "--original-inputs-sha256";
18
+ /** Image paths whose complete overlay contents the consumer requires. */
19
+ paths: string[];
20
+ }
21
+
22
+ export interface OriginalInputEntry {
23
+ target: string;
24
+ type: "file" | "directory" | "link";
25
+ mode: number;
26
+ bytes?: number;
27
+ sha256?: string;
28
+ object?: string;
29
+ link?: string;
30
+ source: { kind: "image" | "project"; path: string; layer?: string; blob?: string };
31
+ }
32
+
33
+ export interface ManagedOriginalInputs {
34
+ kind: "volter.managed-original-inputs";
35
+ version: 1;
36
+ service: string;
37
+ family: string;
38
+ /** Canonical selected service declaration; used to refuse a different stack on reuse. */
39
+ declaration: string;
40
+ source: { image: string; manifest: string; config: string; layers: string[]; commit: string; compose: string; composeSha256: string; composeBytes: number; composeObject: string };
41
+ /** Authenticated complete image metadata, before the declared mounts overlay it. */
42
+ imageInventory: Array<{ target: string; type: string; mode: number; bytes: number; layer: string; link?: string }>;
43
+ imageInventoryDigest: string;
44
+ projectInventory: Array<{ path: string; mode: number; blob: string }>;
45
+ projectInventoryDigest: string;
46
+ roots: string[];
47
+ /** Complete contents beneath roots, plus every original project bind mount. */
48
+ inventory: OriginalInputEntry[];
49
+ inventoryDigest: string;
50
+ start: {
51
+ imageConfig: Record<string, unknown>;
52
+ service: Record<string, unknown>;
53
+ imageEntrypoint: string[] | null;
54
+ imageCommand: string[] | null;
55
+ command?: string[];
56
+ entrypoint?: string[];
57
+ environment: Record<string, string>;
58
+ envFiles: Array<{ path: string; bytes: number; sha256: string; object: string }>;
59
+ unset: string[];
60
+ selfHosts: string[];
61
+ mounts: Array<{ kind: string; source?: string; target: string; readOnly: boolean }>;
62
+ };
63
+ }
64
+
65
+ export const originalTarget = (path: string): boolean => path === "/" || (path.startsWith("/") && !path.slice(1).split("/").some(part => !part || part === "." || part === ".." || /[\0\r\n\\]/u.test(part)));
66
+ export const originalRelative = (path: string): boolean => !!path && !path.split("/").some(part => !part || part === "." || part === ".." || /[\0\r\n\\:]/u.test(part));
67
+
68
+ const hex = (value: unknown): value is string => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value);
69
+ const oci = (value: unknown): value is string => typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value);
70
+ const bytesOf = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) >= 0;
71
+ const strings = (value: unknown): value is string[] => Array.isArray(value) && value.every(entry => typeof entry === 'string' && !/[\0\r\n]/.test(entry));
72
+ const bound = 128 * 1024 * 1024;
73
+
74
+ export function referenceOf(value: unknown): OriginalInputReference {
75
+ if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Original inputs need a versioned reference');
76
+ const ref = value as OriginalInputReference;
77
+ if (ref.version !== 1 || ref.manifest !== 'manifest.json' || typeof ref.root !== 'string' || !originalRelative(ref.root) || !hex(ref.sha256) || !hex(ref.inventoryDigest)
78
+ || Object.keys(ref).some(key => !['version', 'root', 'manifest', 'sha256', 'inventoryDigest'].includes(key))) throw new Error('Invalid original-input reference');
79
+ return { ...ref };
80
+ }
81
+
82
+ /** Hash and validate neutral carrier data. The caller supplies I/O; no pack/vendor behavior or import effects. */
83
+ export function readSnapshot(bytes: Uint8Array, expected: string, readObject: (path: string, bytes: number) => Uint8Array): { manifest: ManagedOriginalInputs; objects: ReadonlyMap<string, Uint8Array> } {
84
+ if (!(bytes instanceof Uint8Array) || bytes.length > 64 * 1024 * 1024 || !hex(expected) || digest('sha256', bytes) !== expected) throw new Error('Original-input manifest identity differs');
85
+ const manifest = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)) as ManagedOriginalInputs;
86
+ const source = manifest.source, start = manifest.start;
87
+ if (manifest.kind !== 'volter.managed-original-inputs' || manifest.version !== 1 || typeof manifest.service !== 'string' || !manifest.service || typeof manifest.family !== 'string' || !/^[a-z][a-z0-9-]*$/.test(manifest.family)
88
+ || typeof manifest.declaration !== 'string' || !source || !/^[a-f0-9]{40}$/.test(source.commit) || !/^[A-Za-z0-9./:_-]+@sha256:[a-f0-9]{64}$/.test(source.image)
89
+ || !oci(source.manifest) || !oci(source.config) || !Array.isArray(source.layers) || !source.layers.every(oci) || !originalRelative(source.compose)
90
+ || !Array.isArray(manifest.imageInventory) || !Array.isArray(manifest.projectInventory) || !Array.isArray(manifest.inventory) || !strings(manifest.roots) || !manifest.roots.length || !manifest.roots.every(originalTarget) || new Set(manifest.roots).size !== manifest.roots.length
91
+ || !hex(manifest.imageInventoryDigest) || digest('sha256', JSON.stringify(manifest.imageInventory)) !== manifest.imageInventoryDigest
92
+ || !hex(manifest.inventoryDigest) || digest('sha256', JSON.stringify(manifest.inventory)) !== manifest.inventoryDigest
93
+ || !hex(manifest.projectInventoryDigest) || digest('sha256', JSON.stringify(manifest.projectInventory)) !== manifest.projectInventoryDigest
94
+ || !start || !start.imageConfig || typeof start.imageConfig !== 'object' || Array.isArray(start.imageConfig) || !start.service || typeof start.service !== 'object' || Array.isArray(start.service)
95
+ || !strings(start.selfHosts) || !start.selfHosts.length || !strings(start.unset) || !Array.isArray(start.envFiles) || !Array.isArray(start.mounts)
96
+ || !start.environment || typeof start.environment !== 'object' || Array.isArray(start.environment) || Object.entries(start.environment).some(([key, value]) => !/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || typeof value !== 'string')
97
+ || [start.imageEntrypoint, start.imageCommand].some(value => value !== null && !strings(value)) || [start.command, start.entrypoint].some(value => value !== undefined && !strings(value))) throw new Error('Invalid original-input manifest');
98
+ if (JSON.stringify(JSON.parse(manifest.declaration)) !== JSON.stringify(start.service) || start.service.name !== manifest.service
99
+ || JSON.stringify(start.command) !== JSON.stringify(start.service.command) || JSON.stringify(start.entrypoint) !== JSON.stringify(start.service.entrypoint)
100
+ || JSON.stringify(start.imageEntrypoint) !== JSON.stringify(start.imageConfig.Entrypoint ?? null) || JSON.stringify(start.imageCommand) !== JSON.stringify(start.imageConfig.Cmd ?? null)
101
+ || !Array.isArray(start.service.volumes)
102
+ || JSON.stringify(start.mounts.map(mount => [mount.kind, mount.source ?? null, mount.target, mount.readOnly])) !== JSON.stringify((start.service.volumes as ManagedOriginalInputs['start']['mounts']).map(mount => [mount.kind, mount.source ?? null, mount.target, mount.readOnly]))) throw new Error('Original-input start declaration differs');
103
+ const project = new Map<string, ManagedOriginalInputs['projectInventory'][number]>();
104
+ for (const entry of manifest.projectInventory) {
105
+ if (!entry || !originalRelative(entry.path) || project.has(entry.path) || ![0o100644, 0o100755, 0o120000, 0o160000].includes(entry.mode) || !/^[a-f0-9]{40}$/.test(entry.blob)) throw new Error('Invalid original project inventory');
106
+ project.set(entry.path, entry);
107
+ }
108
+ const image = new Map<string, ManagedOriginalInputs['imageInventory'][number]>();
109
+ for (const entry of manifest.imageInventory) {
110
+ if (!entry || !originalTarget(entry.target) || image.has(entry.target) || !['file', 'directory', 'link', 'other'].includes(entry.type) || !Number.isInteger(entry.mode) || entry.mode < 0 || entry.mode > 0o7777 || !bytesOf(entry.bytes) || !source.layers.includes(entry.layer)
111
+ || (entry.type === 'link' && (typeof entry.link !== 'string' || /[\0\r\n\\]/.test(entry.link)))) throw new Error('Invalid original image inventory');
112
+ image.set(entry.target, entry);
113
+ }
114
+ const targets = new Set<string>(), selected = new Map<string, OriginalInputEntry>(), expectedObjects = new Map<string, number>();
115
+ const hold = (entry: { sha256?: string; bytes?: number; object?: string }): void => {
116
+ if (!hex(entry.sha256) || !bytesOf(entry.bytes) || entry.object !== `objects/${entry.sha256}` || (expectedObjects.has(entry.sha256) && expectedObjects.get(entry.sha256) !== entry.bytes)) throw new Error('Invalid original-input payload identity');
117
+ expectedObjects.set(entry.sha256, entry.bytes);
118
+ };
119
+ for (const entry of manifest.inventory) {
120
+ if (!entry || !originalTarget(entry.target) || targets.has(entry.target) || !['file', 'directory', 'link'].includes(entry.type) || !Number.isInteger(entry.mode) || entry.mode < 0 || entry.mode > 0o7777 || !entry.source || !['image', 'project'].includes(entry.source.kind)) throw new Error('Invalid original-input inventory');
121
+ targets.add(entry.target);
122
+ selected.set(entry.target, entry);
123
+ if (entry.source.kind === 'image') {
124
+ const original = image.get(entry.source.path);
125
+ if (!original || original.layer !== entry.source.layer || original.type !== entry.type || original.mode !== entry.mode || (entry.type === 'file' && original.bytes !== entry.bytes) || (entry.type === 'link' && original.link !== entry.link)) throw new Error('Original-input image provenance differs');
126
+ } else {
127
+ const original = project.get(entry.source.path);
128
+ if (!originalRelative(entry.source.path) || (entry.type === 'file' && (!original || original.blob !== entry.source.blob || (original.mode & 0o777) !== entry.mode))) throw new Error('Original-input project provenance differs');
129
+ }
130
+ if (entry.type === 'file') hold(entry);
131
+ else if (entry.object !== undefined || entry.bytes !== undefined || entry.sha256 !== undefined || (entry.type === 'link' && (typeof entry.link !== 'string' || /[\0\r\n\\]/.test(entry.link)))) throw new Error('Invalid original-input non-file');
132
+ }
133
+ for (const mount of start.mounts) if (!mount || !['bind', 'named', 'anonymous'].includes(mount.kind) || !originalTarget(mount.target) || typeof mount.readOnly !== 'boolean' || (mount.source !== undefined && typeof mount.source !== 'string')) throw new Error('Invalid original-input mount declaration');
134
+ const roots = new Set(manifest.roots), mounted = new Set(start.mounts.filter(mount => mount.kind === 'bind').map(mount => mount.target));
135
+ const under = (path: string, roots: ReadonlySet<string>): boolean => {
136
+ for (let at = path;; at = at.slice(0, at.lastIndexOf('/')) || '/') {
137
+ if (roots.has(at)) return true;
138
+ if (at === '/') return false;
139
+ }
140
+ };
141
+ for (const entry of image.values()) if (under(entry.target, roots) && !under(entry.target, mounted)) {
142
+ const captured = selected.get(entry.target);
143
+ if (!captured || captured.source.kind !== 'image' || captured.source.path !== entry.target) throw new Error('Original image input was omitted from its declared root');
144
+ }
145
+ hold({ sha256: source.composeSha256, bytes: source.composeBytes, object: source.composeObject });
146
+ for (const file of start.envFiles) {
147
+ if (!file || !originalRelative(file.path)) throw new Error('Invalid original-input env_file');
148
+ hold(file);
149
+ }
150
+ if ([...expectedObjects.values()].reduce((sum, bytes) => sum + bytes, 0) > bound) throw new Error('Original-input snapshot exceeds its byte bound');
151
+ const objects = new Map<string, Uint8Array>();
152
+ for (const [sha256, size] of expectedObjects) {
153
+ const data = readObject(`objects/${sha256}`, size);
154
+ if (!(data instanceof Uint8Array) || data.length !== size || digest('sha256', data) !== sha256) throw new Error('Original-input payload bytes differ');
155
+ objects.set(sha256, data.slice());
156
+ }
157
+ for (const entry of manifest.inventory) if (entry.type === 'file' && entry.source.kind === 'project') {
158
+ const data = objects.get(entry.sha256!)!, header = new TextEncoder().encode(`blob ${data.length}\0`), blob = new Uint8Array(header.length + data.length);
159
+ blob.set(header); blob.set(data, header.length);
160
+ if (digest('sha1', blob) !== entry.source.blob) throw new Error('Original-input project blob identity differs');
161
+ }
162
+ return { manifest, objects };
163
+ }
@@ -0,0 +1,27 @@
1
+ // A MongoDB client over any byte exchange: the other side of ./stream.ts, so a journey speaks the wire as an
2
+ // application's driver does. It sends one command document as an OP_MSG and returns the document the server
3
+ // answered; it reads what the protocol answers and decides nothing.
4
+ // source: archive:https://registry.npmjs.org/mongodb/-/mongodb-6.20.0.tgz#sha256=a12bd599cb4f39cdf222b32380ea3efd1f60c6d88615e0700632a303a9006b02!/package/src/cmap/commands.ts "moreToCome is an OP_MSG only concept"
5
+ import { MongoOp, decodeMsg, encodeMsg, type MongoCodec, type MongoDocument } from './stream.ts';
6
+
7
+ /** Sends bytes and returns every byte the server answered to them. */
8
+ export type MongoExchange = (bytes: Uint8Array) => Promise<Uint8Array>;
9
+
10
+ /** A client on one connection: `command` sends a document to `database` and resolves with the reply document,
11
+ * whatever its `ok`. A reply that is not one whole OP_MSG answering this request is an error. */
12
+ export function mongoClient(exchange: MongoExchange, codec: MongoCodec): { command(database: string, command: MongoDocument): Promise<MongoDocument> } {
13
+ let nextRequest = 0;
14
+ return {
15
+ async command(database, command) {
16
+ const requestId = ++nextRequest;
17
+ const answered = await exchange(encodeMsg(codec, { ...command, $db: database }, requestId, 0));
18
+ if (answered.length < 16) throw new Error('the server answered no message');
19
+ const view = new DataView(answered.buffer, answered.byteOffset, answered.byteLength);
20
+ const length = view.getInt32(0, true), responseTo = view.getInt32(8, true), opcode = view.getInt32(12, true);
21
+ if (length !== answered.length) throw new Error(`the server answered ${answered.length} bytes, and its message says ${length}`);
22
+ if (opcode !== MongoOp.MSG) throw new Error(`the server answered opcode ${opcode}, not OP_MSG`);
23
+ if (responseTo !== requestId) throw new Error(`the server answered request ${responseTo}, not ${requestId}`);
24
+ return decodeMsg(codec, answered).document;
25
+ },
26
+ };
27
+ }
@@ -0,0 +1,5 @@
1
+ // THE KERNEL'S MONGODB WIRE (`@volter/world-core/mongodb`): MongoDB's message framing as a TwinStream, and its client.
2
+ // A pack that serves MongoDB passes what a command means (its engine) and the document codec; the kernel names no vendor.
3
+ // docs/contributing/architecture.md, "Other wires", the MongoDB wire.
4
+ export * from './stream.ts';
5
+ export * from './client.ts';