@volter/world-core 3.0.155 → 3.0.156

package/dist/src/actions.js

@@ -12,16 +12,16 @@ import { parseExactJson } from "./exact-json.js";
 //
 // Projection = observed mirror, then apply each `set` transaction in order,
 // skipping any transaction that was reverted or confirmed.
-import { withHistoryLock } from "./ancestry.js";
+import { canonicalStatePath, withHistoryLock } from "./ancestry.js";
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { randomUUID } from 'node:crypto';
 import { dirname, join } from 'node:path';
 import { canonicalJson, subjectKey } from "./hash.js";
 import { appendDurable, eventsLockPath, ownerStoreRoots, projectionLockPath, twinLog, withFileLock, withoutIdentityClaim, worldPaths } from "./storage.js";
 import { landAsPlaceholder, placeholderPullActive } from "./placeholder-remote.js";
-import { aliasesFrom, dropCheckpoint, landedCopy, landedIds, parentEntries, readTree, treeStamp } from "./log.js";
+import { aliasesFrom, appendToBranchLog, dropCheckpoint, landedCopy, landedIds, parentEntries, readTree, treeStamp } from "./log.js";
 import { inVendorMove, refuseReadOnlyWrite } from "./request-scope.js";
-import { getActiveWorldStore } from "./world-store.js";
+import { getActiveWorldStore, storageLockScope } from "./world-store.js";
 import { currentTraceparent, traceparentForDelivery } from "./trace-context.js";
 export class TwinActionPreconditionError extends Error {
     actionId;
@@ -36,8 +36,11 @@ export class TwinActionPreconditionError extends Error {
 function actionsPath(service, root) {
     return join(dirname(worldPaths(service, root).events), 'actions.jsonl');
 }
-const actionsLock = (service, root) => `${actionsPath(service, root)}.lock`;
-const projectionLock = (service, root) => projectionLockPath(worldPaths(service, root));
+// Canonical, as the history lock's path is (ancestry.ts withHistoryLock), so a take inside it is seen as nested in the same
+// directory whatever the path it was asked by (/var or /private/var). The file is the same one, so a take by another
+// process through the other spelling still excludes this one.
+const actionsLock = (service, root) => join(canonicalStatePath(worldPaths(service, root).dir), 'actions.jsonl.lock');
+const projectionLock = (service, root) => { const paths = worldPaths(service, root); return projectionLockPath({ ...paths, dir: canonicalStatePath(paths.dir) }); };
 /** Every appended action carries a correlationId — generate one when the caller
  *  hasn't supplied it, so downstream joins (push ledger, logs) always have an id
  *  to key on (D3). Preserves a caller-supplied id (e.g. propagated from an HTTP
@@ -94,24 +97,38 @@ function actionIds(service, root) {
 }
 /** A service's projection lock, under its history lock: one directory's locks are taken history, projection,
  *  actions, events, in that order (architecture, the history lock). */
-function withProjection(service, root, fn) {
-    return withHistoryLock(worldPaths(service, root).dir, () => withFileLock(projectionLock(service, root), fn));
+function withProjection(service, root, fn, options = {}) {
+    const scope = storageLockScope(options);
+    return withHistoryLock(worldPaths(service, root).dir, () => withFileLock(projectionLock(service, root), fn, scope), scope);
 }
 function appendActionRaw(action, root) {
-    const path = actionsPath(action.service, root);
-    getActiveWorldStore().mkdir(dirname(path));
+    appendActionsRaw([action], root);
+}
+/** One service's actions appended as ONE store append, in order: the same lines `appendActionRaw` writes one at a time. */
+function appendActionsRaw(actions, root) {
+    if (!actions.length)
+        return;
+    const service = actions[0].service;
+    const path = actionsPath(service, root);
+    // No mkdir: every append is made holding the projection lock (withProjection), whose file is in this same directory
+    // and was made there by the take (S2 R17 review).
     const memos = idMemoSlot(path);
     const held = memos.get(path);
     const current = held !== undefined && held.key === logKey(path);
-    withHistoryLock(worldPaths(action.service, root).dir, () => appendDurable(path, `${JSON.stringify(action)}\n`));
+    const data = actions.map((action) => `${JSON.stringify(action)}\n`).join('');
+    // the log is the branch log a read's tree is folded from: the append extends that memo too (log.ts appendToBranchLog)
+    const after = withHistoryLock(worldPaths(service, root).dir, () => appendToBranchLog(service, root, data, () => appendDurable(path, data)), { operation: 'action append' });
     if (current) {
-        held.ids.add(action.id);
-        held.key = logKey(path);
+        for (const action of actions)
+            held.ids.add(action.id);
+        held.key = after ?? logKey(path);
     }
     else
         memos.delete(path);
-    twinLog('action.append', { service: action.service, id: action.id, op: action.op, correlationId: action.correlationId });
-    appendObserver?.(action, root);
+    for (const action of actions) {
+        twinLog('action.append', { service: action.service, id: action.id, op: action.op, correlationId: action.correlationId });
+        appendObserver?.(action, root);
+    }
 }
 /** ONE observer of appended entries (state-system.ts: the runtime's `deploy: auto` hook). Called
  *  under the append locks, so it must only schedule work, never do it. */
@@ -173,7 +190,7 @@ export function appendAction(action, root) {
             return placeholder;
         appendActionRaw(stamped, root);
         return stamped;
-    }));
+    }, { operation: 'action append' }), { operation: 'action append' });
 }
 /** Append `action` only if no exact action with the same id already exists — the whole
  *  replay/precondition/append decision runs under the service projection + actions locks, so it's
@@ -196,7 +213,7 @@ export function appendActionIfAbsent(action, root) {
             return { action: placeholder, appended: true, placeholder: true };
         appendActionRaw(stamped, root);
         return { action: stamped, appended: true };
-    }));
+    }, { operation: 'action append' }), { operation: 'action append' });
 }
 /**
  * Append `build(n)` as an OCCURRENCE: the nth time this exact write has been made.
@@ -224,7 +241,51 @@ export function appendActionOccurrence(base, root) {
             return { action: placeholder, appended: true, placeholder: true };
         appendActionRaw(stamped, root);
         return { action: stamped, appended: true };
-    }));
+    }, { operation: 'action append' }), { operation: 'action append' });
+}
+/**
+ * Append several writes of ONE service as occurrences, in order, under one projection + actions lock and as one store
+ * append: each lands with the id, ordinal and correlation `appendActionOccurrence` would give it called once per write
+ * in the same order (a write's ordinal counts the batch's earlier writes as taken). A Redis run's flush made one
+ * append per changed member and key, each taking the history, projection and actions locks and appending alone, one
+ * after another under the twin's single command queue (S2 R19: Twenty's queue scripts spent up to 2.1 s of a 2 s
+ * window in flush while 38 commands waited). A write carrying preconditions is not taken here: each is checked against
+ * the state the earlier ones made, which only the one-at-a-time path sees. While a placeholder pull is open a write
+ * lands as observed events, as `appendActionOccurrence`'s does.
+ */
+export function appendActionOccurrences(bases, root) {
+    if (!bases.length)
+        return [];
+    const service = bases[0].service;
+    if (bases.some((base) => base.service !== service))
+        throw new Error(`appendActionOccurrences: one service per batch, got ${[...new Set(bases.map((base) => base.service))].join(', ')}`);
+    if (bases.some((base) => base.preconditions?.length))
+        throw new Error('appendActionOccurrences: a write with preconditions is appended alone');
+    refuseReadOnlyWrite(service);
+    return withProjection(service, root, () => withFileLock(actionsLock(service, root), () => {
+        const logged = actionIds(service, root);
+        const batch = new Set();
+        const taken = { has: (id) => batch.has(id) || logged.has(id) };
+        const landed = [];
+        let unwritten = [];
+        for (const base of bases) {
+            const stamped = occurrenceOf(base, taken, root);
+            const placeholder = landIfPlaceholderPull(stamped, root);
+            if (placeholder) {
+                // no action row: its id is not taken, as it is not in the log the one-at-a-time path reads
+                // the lines before it are written first, so the log and the pull keep the writes' order
+                appendActionsRaw(unwritten, root);
+                unwritten = [];
+                landed.push({ action: placeholder, appended: true, placeholder: true });
+                continue;
+            }
+            batch.add(stamped.id);
+            unwritten.push(stamped);
+            landed.push({ action: stamped, appended: true });
+        }
+        appendActionsRaw(unwritten, root);
+        return landed;
+    }, { operation: 'action append' }), { operation: 'action append' });
 }
 /** The occurrence rule, shared by both appenders that use it — the ordinal AND the merge base.
  *  Pure over (base, the log as it stands): the caller holds the lock. */
@@ -288,7 +349,7 @@ read = () => projectResources(service, root)) {
             return { value: decision.value, action: placeholder, appended: true, placeholder: true };
         appendActionRaw(stamped, root);
         return { value: decision.value, action: stamped, appended: true };
-    }));
+    }, { operation: 'action decision' }), { operation: 'action decision' });
 }
 export function appendTransactionCommit(commit, root) {
     return appendAction(commit, root);
@@ -439,9 +500,9 @@ export function confirmAction(opts) {
             for (const copy of copies)
                 if (!held.has(copy.id))
                     appendDurable(paths.events, `${JSON.stringify(copy)}\n`);
-        });
+        }, { operation: 'action confirmation' });
         dropCheckpoint(opts.service, opts.root);
-    });
+    }, { operation: 'action confirmation' });
     const observedEventIds = copies.map((c) => c.id);
     return { observedEventId: observedEventIds.at(-1), observedEventIds };
 }
@@ -477,7 +538,7 @@ export function revertAction(opts) {
         });
         appendActionRaw(revert, opts.root);
         return { status: 'reverted', revertId: revert.id };
-    }));
+    }, { operation: 'action revert' }), { operation: 'action revert' });
 }
 /** The LOCAL OVERLAY: pending actions (set, not reverted, not yet confirmed) — the divergence
  *  from the mirror, what a twin's projection folds over the observed state. A QUARANTINED row

package/dist/src/ancestry.d.ts

@@ -1,18 +1,19 @@
+import { type WorldLockOptions } from './world-store.js';
 export declare const canonicalStatePath: (path: string) => string;
 /** Synchronous only; use exclusive acquisition without age-stealing a live owner.
  * The coordinator lives outside service, instance and hosted World deletion trees. It is held only for ancestry
  * bookkeeping (generations, pins, branch pointers, removal receipts) and the not-being-removed check: every World
  * sharing the VOLTER_HOME waits on it, so no append, fold or caller's critical section runs inside it, and no state lock
  * is taken inside it (`withStateLock` takes its lock first). */
-export declare function withAncestryLock<T>(fn: () => T): T;
+export declare function withAncestryLock<T>(fn: () => T, options?: WorldLockOptions): T;
 /** Run `fn` holding the cross-process lock at `lockPath`, a lock of the directory it sits in: taken first, then the
  *  coordinator only to check that the directory is not being removed. Re-entrant within a process. */
-export declare function withStateLock<T>(lockPath: string, fn: () => T): T;
+export declare function withStateLock<T>(lockPath: string, fn: () => T, options?: WorldLockOptions): T;
 /** A state directory's history lock: its appends, captures, observation batches and folds. A branch's is taken before
  *  its ancestors'. */
-export declare function withHistoryLock<T>(directory: string, fn: () => T): T;
+export declare function withHistoryLock<T>(directory: string, fn: () => T, options?: WorldLockOptions): T;
 /** Several sibling directories' history locks at once (a World's services, a changeset's targets), in one order. */
-export declare function withHistoryLocks<T>(directories: string[], fn: () => T): T;
+export declare function withHistoryLocks<T>(directories: string[], fn: () => T, options?: WorldLockOptions): T;
 /** Create an identity only for an existing parent. A recreated path gets a new identity. */
 export declare function stateGeneration(dir: string): string;
 /** Must be called in the same critical section as child pointer publication. A crash before

package/dist/src/ancestry.js

@@ -4,7 +4,7 @@ import { createHash, randomUUID } from 'node:crypto';
 import { volterHome } from "./volter-home.js";
 import { basename, dirname, join, relative, resolve, sep } from 'node:path';
 import { stateDirName } from "./storage.js";
-import { getActiveWorldStore } from "./world-store.js";
+import { getActiveWorldStore, storageLockScope, withStorageLockOperation } from "./world-store.js";
 let entered;
 export const canonicalStatePath = (path) => getActiveWorldStore().canonicalPath?.(path) ?? resolve(path);
 const metaPath = (dir) => join(dir, 'ancestry.json');
@@ -14,7 +14,8 @@ const inside = (base, path) => { const rel = relative(canonicalStatePath(base),
  * bookkeeping (generations, pins, branch pointers, removal receipts) and the not-being-removed check: every World
  * sharing the VOLTER_HOME waits on it, so no append, fold or caller's critical section runs inside it, and no state lock
  * is taken inside it (`withStateLock` takes its lock first). */
-export function withAncestryLock(fn) {
+export function withAncestryLock(fn, options = {}) {
+    const scope = storageLockScope(options);
     const store = getActiveWorldStore();
     const active = entered ??= new WeakSet();
     if (active.has(store))
@@ -23,13 +24,13 @@ export function withAncestryLock(fn) {
     return store.withLock(path, () => {
         active.add(store);
         try {
-            recoverRemovals();
+            recoverRemovals(path);
             return fn();
         }
         finally {
             active.delete(store);
         }
-    }, { retainLiveOwner: true });
+    }, { ...scope, retainLiveOwner: true });
 }
 /** The state locks this process holds, per store: taken again inside one, a lock is already ours. */
 let heldLocks;
@@ -39,15 +40,27 @@ const heldBy = (store) => { const map = heldLocks ??= new WeakMap(); let set = m
 } return set; };
 /** Run `fn` holding the cross-process lock at `lockPath`, a lock of the directory it sits in: taken first, then the
  *  coordinator only to check that the directory is not being removed. Re-entrant within a process. */
-export function withStateLock(lockPath, fn) {
+export function withStateLock(lockPath, fn, options = {}) {
+    const scope = storageLockScope(options);
     const store = getActiveWorldStore();
     const held = heldBy(store);
     if (held.has(lockPath))
         return fn();
     if (entered?.has(store))
         throw new Error(`State lock ${lockPath} taken inside the ancestry coordinator: a state lock is taken before it, never inside it`);
+    // A lock taken inside a held lock of the SAME directory asks the coordinator nothing: the outer take checked that
+    // the directory is not being removed, and a removal recorded since waits on that outer lock before it deletes
+    // anything, so this operation finishes first, as a holder that checked before the receipt does. Asked again, each
+    // write took the machine-wide coordinator three times for one directory.
+    const directory = dirname(lockPath);
+    const nested = [...held].some((path) => dirname(path) === directory);
     return store.withLock(lockPath, () => {
-        withAncestryLock(() => assertNotBeingRemoved(dirname(lockPath)));
+        // The removal check reads the receipts without the coordinator, and takes it only where one is there (S2 R17;
+        // architecture, the history lock): this holder already has the directory's lock, and a removal writes its receipt
+        // under the coordinator before it waits on every lock inside the tree, so a receipt not yet visible here belongs to
+        // a removal that waits for this holder, which finishes first. A visible receipt is checked, recovered, as before.
+        if (!nested && removalRecorded())
+            withAncestryLock(() => assertNotBeingRemoved(directory), { operation: 'removal preflight' });
         held.add(lockPath);
         try {
             return fn();
@@ -55,12 +68,12 @@ export function withStateLock(lockPath, fn) {
         finally {
             held.delete(lockPath);
         }
-    });
+    }, scope);
 }
 /** A state directory's history lock: its appends, captures, observation batches and folds. A branch's is taken before
  *  its ancestors'. */
-export function withHistoryLock(directory, fn) {
-    return withStateLock(join(canonicalStatePath(directory), 'history.lock'), fn);
+export function withHistoryLock(directory, fn, options = {}) {
+    return withStateLock(join(canonicalStatePath(directory), 'history.lock'), fn, storageLockScope(options));
 }
 /** A lock file a holder has recorded itself in (`{pid, hostname}`, written before the holder runs anything): a file
  *  that only ends in `.lock` (a yarn.lock) is not one. */
@@ -74,9 +87,10 @@ function isHeldLock(text) {
     }
 }
 /** Several sibling directories' history locks at once (a World's services, a changeset's targets), in one order. */
-export function withHistoryLocks(directories, fn) {
+export function withHistoryLocks(directories, fn, options = {}) {
+    const scope = storageLockScope(options);
     const sorted = [...new Set(directories.map((d) => canonicalStatePath(d)))].sort();
-    const take = (i) => (i === sorted.length ? fn() : withHistoryLock(sorted[i], () => take(i + 1)));
+    const take = (i) => (i === sorted.length ? fn() : withHistoryLock(sorted[i], () => take(i + 1), scope));
     return take(0);
 }
 /** The locks inside a tree a removal waits on before deleting it. */
@@ -125,7 +139,7 @@ export function stateGeneration(dir) {
             store.writeAtomic(metaPath(dir), JSON.stringify(meta));
         }
         return meta.generation;
-    });
+    }, { operation: 'state generation' });
 }
 /** Must be called in the same critical section as child pointer publication. A crash before
  * publication leaves a pending pin; it can be ignored only once its writer is provably dead. */
@@ -254,23 +268,34 @@ function releaseOutgoing(removal) {
 }
 /** A crash after deleting files cannot strand committed child pins. Receipts are replayed
  * before any subsequent ancestry publication, so path reuse cannot hide completed removal. */
-function recoverRemovals() {
+function recoverRemovals(lockPath) {
     const store = getActiveWorldStore();
+    const receipts = store.list(removalDir()).filter(n => n.endsWith('.json'));
+    if (!receipts.length) {
+        beingRemoved = [];
+        return;
+    }
     const pending = [];
-    for (const name of store.list(removalDir()).filter(n => n.endsWith('.json'))) {
-        const file = join(removalDir(), name);
-        const removal = JSON.parse(store.read(file));
-        if (typeof removal.path !== 'string' || !Array.isArray(removal.outgoing))
-            throw new Error('Invalid branch removal receipt; retained');
-        if (!store.exists(removal.path)) {
-            releaseOutgoing(removal);
-            store.remove(file);
+    withStorageLockOperation(lockPath, 'ancestry removal recovery', () => {
+        for (const name of receipts) {
+            const file = join(removalDir(), name);
+            const removal = JSON.parse(store.read(file));
+            if (typeof removal.path !== 'string' || !Array.isArray(removal.outgoing))
+                throw new Error('Invalid branch removal receipt; retained');
+            if (!store.exists(removal.path)) {
+                releaseOutgoing(removal);
+                store.remove(file);
+            }
+            else
+                pending.push(removal.path);
         }
-        else
-            pending.push(removal.path);
-    }
+    });
     beingRemoved = pending;
 }
+/** Whether any removal receipt is recorded now: read without the coordinator (withStateLock). */
+function removalRecorded() {
+    return getActiveWorldStore().list(removalDir()).some((name) => name.endsWith('.json'));
+}
 /** State whose removal is recorded and not yet complete: its deletion runs outside the coordinator, so nothing may pin
  * it as a parent or write into it meanwhile. Read under the coordinator, where recoverRemovals has just refreshed it. */
 let beingRemoved = [];
@@ -298,21 +323,21 @@ export function withStateRemoval(path, remove, check) {
         store.writeAtomic(receipt, JSON.stringify(removal));
         beingRemoved = [...beingRemoved, removal.path];
         return { removal, receipt };
-    });
+    }, { operation: 'removal preflight' });
     // Each lock is waited on alone, taken and let go: a holder that checked before the receipt finishes, and a taker
     // after it refuses once it holds the lock (withStateLock), so the removal never holds one lock while waiting on
     // another and no lock order can deadlock it. A wait that fails leaves nothing deleted: the receipt is released.
     const held = heldBy(store);
     try {
         for (const lock of locksUnder(path).filter((l) => !held.has(l)))
-            store.withLock(lock, () => undefined);
+            store.withLock(lock, () => undefined, { operation: 'removal preflight' });
     }
     catch (error) {
         withAncestryLock(() => {
             if (store.exists(receipt))
                 store.remove(receipt);
             beingRemoved = beingRemoved.filter((p) => p !== removal.path);
-        });
+        }, { operation: 'ancestry removal recovery' });
         throw error;
     }
     const result = remove();
@@ -323,6 +348,6 @@ export function withStateRemoval(path, remove, check) {
         releaseOutgoing(removal);
         if (store.exists(receipt))
             store.remove(receipt);
-    });
+    }, { operation: 'ancestry removal recovery' });
     return result;
 }

package/dist/src/changeset.js

@@ -727,7 +727,7 @@ export function narrationDrift(changeset) {
     return expected === changeset.narration ? null : { recorded: changeset.narration, expected };
 }
 export function rebaseChangeset(changeset, opts) {
-    return withHistoryLocks(opts.targets.map((t) => worldPaths(t.stateService, t.controlRoot).dir), () => rebaseChangesetLocked(changeset, opts));
+    return withHistoryLocks(opts.targets.map((t) => worldPaths(t.stateService, t.controlRoot).dir), () => rebaseChangesetLocked(changeset, opts), { operation: 'branch-pointer adjustment' });
 }
 function rebaseChangesetLocked(changeset, opts) {
     const at = opts.at ?? new Date().toISOString();

package/dist/src/clickhouse/compression.d.ts

@@ -0,0 +1,28 @@
+type Pair = [bigint, bigint];
+/** The CityHash 1.0.2 checksum used by ClickHouse native compression frames. */
+export declare function cityHash128(bytes: Uint8Array): Pair;
+export declare const MAX_COMPRESSED_BLOCK_SIZE = 1073741824;
+export declare const MAX_DECOMPRESSED_BLOCK_SIZE = 1073741824;
+/** LZ4 block sequences. Validate all lengths before allocating the output buffer. */
+export declare function decodeLz4Block(input: Uint8Array, size: number): Uint8Array;
+/** A valid literal-only LZ4 block. */
+export declare function encodeLz4Literals(input: Uint8Array): Uint8Array;
+export declare class ClickHouseCompressionError extends Error {
+    readonly code: number;
+    constructor(code: number, message: string);
+}
+/** ClickHouse compressed frame, including its sixteen-byte checksum.
+ * ICompressionCodec.cpp:62–69,102–116 (SHA2562cfbdbd7…) writes the9-byte header.
+ * CompressedReadBufferBase.cpp:193–194,255–256 (SHA2566f5af3a8…) reads low/high
+ * little-endian checksum words and verifies the complete header+payload before decompressing.
+ */
+export declare function encodeFrame(input: Uint8Array, method?: 0x82 | 0x02): Uint8Array;
+/** Decode one frame; undefined means the input has not yet supplied the whole frame.
+ * Sizes are validated from the header before waiting for its body, and the checksum
+ * precedes decompression. bytesRead allows the caller to retain subsequent frames.
+ */
+export declare function decodeFrame(input: Uint8Array): {
+    data: Uint8Array;
+    bytesRead: number;
+} | undefined;
+export {};