@loomwork/event-runtime 0.0.0-stage → 0.1.1

package/sequence.d.ts

@@ -0,0 +1 @@
+export * from "./dist/sequenceRange";

package/sequence.js

@@ -0,0 +1,5 @@
+// Browser-safe subpath: only the pure sequence-range helpers, so importing them
+// never pulls the SQLite store (better-sqlite3 / node:fs) that the package barrel
+// re-exports. (These packages compile to CommonJS, which bundlers cannot
+// tree-shake — hence a dedicated subpath rather than relying on `sideEffects`.)
+module.exports = require("./dist/sequenceRange");

package/src/BranchRegistry.ts

@@ -0,0 +1,161 @@
+/**
+ * Git-style branching for event-sourced executions — branch only, no merge.
+ *
+ * A {@link BranchRef} is a movable pointer (like a git branch) naming the head
+ * execution of a line of work. A {@link LineageRecord} is an immutable edge in
+ * the fork DAG (child → parent at a sequence). Together they let speculative
+ * agent executions fork from any point, run independently, and be selected with
+ * `promote()` ("pick-a-winner"). There is deliberately no merge/rebase: two
+ * divergent ExecutionStates have no principled join, so we don't fake one.
+ */
+
+/**
+ * SPI version for the {@link BranchRegistry} contract (ADR 0033). Pinned by
+ * `runBranchRegistryConformance` in `@loomwork/testing`.
+ *
+ * **2.0.0 (L21, ADR 0108 addendum, MAJOR):** `listRefs` orders names in Unicode **code
+ * point** order — the same order `EventStore.listExecutionsPage` pins (ADR 0108, SPI
+ * 2.0.0): UTF-8 byte order, SQL `COLLATE "C"`, SQLite BINARY, and `compareExecutionIds`
+ * (`@loomwork/event-runtime`) as the reference comparator. Not a database's default collation,
+ * and not `String.prototype.localeCompare` — both can disagree with code point order, and
+ * `localeCompare` can return 0 for two DISTINCT, canonically-equivalent names (an NFC "é"
+ * and its NFD decomposition). `after` may be **any string**, not only a name a previous
+ * `listRefs` call returned, and the page holds the names strictly after it. Before this, the
+ * three backends ordered `listRefs` three different ways (Postgres: database collation;
+ * SQLite: BINARY, already conforming; in-memory: `localeCompare`), so a cursor produced by
+ * one order and consumed by another skipped or repeated refs at a page boundary — the same
+ * failure mode ADR 0108 fixed for execution ids. MAJOR because a store conforming to 1.x
+ * (any order, an opaque cursor) can fail the new conformance case (the ADR 0091 rule); no
+ * method was added or removed.
+ */
+export const BRANCH_REGISTRY_SPI_VERSION = "2.0.0";
+
+/** A named, movable pointer to an execution head. */
+export interface BranchRef {
+  /** Branch name, e.g. "main" or "spec/scale-replicas". Unique. */
+  name: string;
+  /** The execution this ref currently points at. */
+  executionId: string;
+  /** Latest known sequence on that execution (advanced via `moveTip`). */
+  tip: number;
+  /** If this branch was created by a fork, the execution it forked from. */
+  parentExecutionId?: string;
+  /** If this branch was created by a fork, the sequence it forked at. */
+  forkSequence?: number;
+  createdAt: number;
+  updatedAt: number;
+}
+
+/** An immutable edge in the fork DAG: `executionId` forked from `parentExecutionId` at `forkSequence`. */
+export interface LineageRecord {
+  executionId: string;
+  parentExecutionId: string;
+  forkSequence: number;
+  createdAt: number;
+}
+
+/** Paging for {@link BranchRegistry.listRefs}. Keyset, because refs are unique and name-ordered. */
+export interface ListRefsOptions {
+  /** Max rows to return. Must be a positive integer — an unbounded page is not a default. */
+  limit?: number;
+  /**
+   * Return refs whose name sorts strictly AFTER this one, in Unicode CODE POINT order
+   * (SPI 2.0.0, ADR 0108 addendum / L21 — {@link BRANCH_REGISTRY_SPI_VERSION}). May be ANY
+   * string, not only a name a previous `listRefs` call returned — pass the last name of the
+   * previous page.
+   */
+  after?: string;
+}
+
+/** Options for {@link BranchRegistry.promote}. */
+export interface PromoteOptions {
+  /**
+   * Compare-and-set. The executionId `toName` is expected to name right now, or `null` for
+   * "must not exist yet". The repoint happens only if the expectation holds; otherwise it
+   * throws {@link BranchPromotionConflictError} and **nothing moves**.
+   *
+   * Without it, `promote` is last-write-wins: two nodes promoting different winners onto
+   * `main` both succeed and the loser silently wins. That is the 1.0.0 behaviour, preserved
+   * byte-identically when `expect` is omitted.
+   */
+  expect?: string | null;
+}
+
+/**
+ * A compare-and-set {@link BranchRegistry.promote} lost: `toName` does not name what the
+ * caller expected, so the promotion was refused. The registry is untouched — re-read the ref
+ * and decide, rather than retrying blind.
+ */
+export class BranchPromotionConflictError extends Error {
+  constructor(
+    readonly toName: string,
+    readonly expected: string | null,
+    readonly actual: string | null
+  ) {
+    super(
+      `promote onto "${toName}" refused: expected it to name ${expected === null ? "nothing" : `"${expected}"`}, but it names ${actual === null ? "nothing" : `"${actual}"`}`
+    );
+    this.name = "BranchPromotionConflictError";
+    Object.setPrototypeOf(this, BranchPromotionConflictError.prototype);
+  }
+}
+
+/**
+ * Refuse a nonsense page size before it reaches a backend. SQLite treats `LIMIT -1` as "no
+ * limit", so an unvalidated negative silently returns the whole table — the opposite of what
+ * a caller asking for a page wants.
+ */
+export function assertListRefsLimit(limit: number | undefined): void {
+  if (limit === undefined) return;
+  if (!Number.isInteger(limit) || limit <= 0) {
+    throw new RangeError(`listRefs: limit must be a positive integer, got ${limit}`);
+  }
+}
+
+export interface BranchRegistry {
+  /** Create or move a ref (upsert by name). */
+  putRef(ref: BranchRef): Promise<void>;
+  getRef(name: string): Promise<BranchRef | null>;
+  /**
+   * Refs in name order — Unicode CODE POINT order (SPI 2.0.0, see
+   * {@link BRANCH_REGISTRY_SPI_VERSION}), never a database collation or
+   * `String.prototype.localeCompare`. Unpaged by default; pass `limit`/`after` to walk a
+   * registry that has grown — forking is cheap, so the ref table is unbounded by design.
+   */
+  listRefs(options?: ListRefsOptions): Promise<BranchRef[]>;
+  /** Advance a ref's tip as its execution grows. No-op if the ref is absent. */
+  moveTip(name: string, tip: number): Promise<void>;
+  /**
+   * Pick-a-winner: repoint `toName` onto the execution `fromName` currently
+   * names, adopting its (executionId, tip). This is the entire "merge"
+   * replacement — a pointer move, no history rewrite, no state reconciliation.
+   * The losing execution remains immutable and independently resumable.
+   * Throws if `fromName` does not exist. Pass `options.expect` to make it a compare-and-set
+   * (see {@link PromoteOptions}) — without it, a concurrent promotion is a silent overwrite.
+   */
+  promote(fromName: string, toName: string, options?: PromoteOptions): Promise<void>;
+  deleteRef(name: string): Promise<void>;
+
+  /** Record a fork edge in the DAG. */
+  recordLineage(rec: Omit<LineageRecord, "createdAt">): Promise<void>;
+  getLineage(executionId: string): Promise<LineageRecord | null>;
+  /**
+   * OPTIONAL: forget one fork edge — the child's edge to its parent.
+   *
+   * Lineage is the alternatives-considered audit trail and must outlive a branch's own
+   * life. It must NOT outlive the executions it describes: once retention has reclaimed a
+   * branch, its edge points at data that no longer exists, and keeping a permanent index of
+   * deleted executions contradicts the retention policy that deleted them. It also costs —
+   * `LogLifecycle` walks `children()` on every sweep, so dead edges make retention slower
+   * forever, in proportion to every fork ever taken.
+   *
+   * Called by `LogLifecycle` at the moment an execution is deleted, which is the only point
+   * where the id is known and the data is provably gone. A store that omits it simply keeps
+   * its edges (today's behaviour, byte-identically).
+   */
+  removeLineage?(executionId: string): Promise<void>;
+  /** Direct children forked from `executionId`. */
+  children(executionId: string): Promise<LineageRecord[]>;
+
+  close(): Promise<void>;
+}

package/src/CoalescingEventStore.ts

@@ -0,0 +1,214 @@
+import type { EventEnvelope } from "@loomwork/core";
+import { type DecoratorPlan, installOptional } from "./installOptional";
+import { type EventStore, type OptionalEventStoreMethod, appendBatchOf } from "./types";
+
+export interface CoalescingEventStoreOptions {
+  /**
+   * Flush once this many envelopes are queued. The measured knee is ~16; past ~64 both
+   * throughput and tail latency regress. Default 16.
+   */
+  maxBatch?: number;
+  /**
+   * Upper bound in milliseconds on how long the FIRST envelope of a batch waits for
+   * company. Default 0 — flush on the next microtask, which under concurrency already
+   * collects everything enqueued in the same turn without adding measurable latency.
+   * Raise it only if you are deliberately trading latency for batch size.
+   */
+  maxLingerMs?: number;
+}
+
+interface Pending {
+  envelope: EventEnvelope;
+  resolve: () => void;
+  reject: (err: unknown) => void;
+}
+
+/**
+ * Merges concurrent appends **across executions** into one `appendBatch` (ADR 0074 D2).
+ *
+ * Why a decorator and not the journal: `ExecutionJournal` is per-execution, so a
+ * journal-side batcher can only ever hold one execution's envelopes — a 1-step flow emits
+ * four, and with a realistic slow effect the serial tail drains between `STEP_STARTED` and
+ * `STEP_COMPLETED`, so it collects one. Measured; it is ADR 0065's own gate-2 case. A store
+ * decorator sees every live execution at once and can actually fill a batch.
+ *
+ * **Ordering.** Each `ExecutionJournal` still serializes its own execution (D3 unchanged) —
+ * it awaits each `append` before issuing the next, so this store never holds two envelopes
+ * of one execution concurrently through the normal path. Should it anyway (a second writer,
+ * a caller not using the journal), the queue is FIFO and `appendBatch` is specified to
+ * apply in array order, so arrival order is preserved regardless.
+ *
+ * **Durability.** An `append` promise resolves only once its batch has committed. Nothing
+ * resolves early, so every `drain()` barrier keeps exactly the meaning it has today. This
+ * is the property that distinguishes coalescing from the durability classes ADR 0065
+ * rejected on invariants.
+ *
+ * **Failure.** A batch rejection rejects every caller in that batch, so a partial write can
+ * never look like success. Combined with `appendBatch`'s all-or-nothing contract, a failed
+ * flush leaves the log untouched and every writer informed — **but only when the inner store
+ * implements `appendBatch`**. Over a store without it, `appendBatchOf` falls back to a
+ * sequential `append` loop, which is not atomic: envelopes ahead of the failing one are
+ * already durable while their writers are told the batch failed. Every decorator between
+ * this one and the backend must therefore carry `appendBatch` down (ADR 0074 Amendment —
+ * `runEventStoreDecoratorConformance` checks it).
+ *
+ * **Blast radius.** One stale-fence zombie in a batch rejects the whole batch, so every
+ * healthy writer coalesced with it is told "failed" too (nothing of theirs lands; they abort
+ * and recovery re-drives them from the log). Coalescing widens a fencing rejection from one
+ * execution to up to `maxBatch` of them — a price worth paying while fencing is rare.
+ *
+ * Inert unless wired: compose it explicitly in `storage`, outermost-but-one (sharding
+ * outside it, since routing must pick a shard before a batch is formed for that shard).
+ */
+export class CoalescingEventStore implements EventStore {
+  readonly head?: NonNullable<EventStore["head"]>;
+  readonly readRange?: NonNullable<EventStore["readRange"]>;
+  readonly acquireFence?: NonNullable<EventStore["acquireFence"]>;
+  readonly acquireFenceIf?: NonNullable<EventStore["acquireFenceIf"]>;
+  readonly appendIfUnchanged?: NonNullable<EventStore["appendIfUnchanged"]>;
+  readonly storeNow?: NonNullable<EventStore["storeNow"]>;
+  readonly sealingCapability?: NonNullable<EventStore["sealingCapability"]>;
+  readonly readAtRest?: NonNullable<EventStore["readAtRest"]>;
+  readonly copyPrefix?: NonNullable<EventStore["copyPrefix"]>;
+  readonly deleteExecution?: NonNullable<EventStore["deleteExecution"]>;
+  readonly deleteThrough?: NonNullable<EventStore["deleteThrough"]>;
+  readonly truncatePrefix?: NonNullable<EventStore["truncatePrefix"]>;
+  readonly truncatePrefixKeeping?: NonNullable<EventStore["truncatePrefixKeeping"]>;
+  readonly listExecutions?: NonNullable<EventStore["listExecutions"]>;
+  readonly listExecutionsPage?: NonNullable<EventStore["listExecutionsPage"]>;
+
+  private readonly maxBatch: number;
+  private readonly maxLingerMs: number;
+  private queue: Pending[] = [];
+  private flushScheduled = false;
+  private timer: ReturnType<typeof setTimeout> | undefined;
+  /** Serializes flushes so two batches can never be in flight against the inner store. */
+  private tail: Promise<void> = Promise.resolve();
+  private closed = false;
+
+  constructor(
+    private readonly inner: EventStore,
+    options: CoalescingEventStoreOptions = {}
+  ) {
+    this.maxBatch = Math.max(1, options.maxBatch ?? 16);
+    this.maxLingerMs = Math.max(0, options.maxLingerMs ?? 0);
+    // Pass through every optional capability the inner store has, so wrapping never
+    // silently downgrades a backend (a missing `readRange` would push callers onto the
+    // slow fallback for no reason). `appendBatch` is this class's own method — batching is
+    // what it is for — so the plan covers every other optional method.
+    const plan = {
+      head: "forward",
+      readRange: "forward",
+      acquireFence: "forward",
+      acquireFenceIf: "forward",
+      appendIfUnchanged: "forward",
+      storeNow: "forward",
+      sealingCapability: "forward",
+      readAtRest: "forward",
+      copyPrefix: "forward",
+      deleteExecution: "forward",
+      deleteThrough: "forward",
+      truncatePrefix: "forward",
+      truncatePrefixKeeping: "forward",
+      listExecutions: "forward",
+      listExecutionsPage: "forward",
+    } satisfies DecoratorPlan<EventStore, Exclude<OptionalEventStoreMethod, "appendBatch">>;
+    installOptional<EventStore, Exclude<OptionalEventStoreMethod, "appendBatch">>(
+      this,
+      inner,
+      plan
+    );
+  }
+
+  append(envelope: EventEnvelope): Promise<void> {
+    if (this.closed) return Promise.reject(new Error("CoalescingEventStore is closed"));
+    return new Promise<void>((resolve, reject) => {
+      this.queue.push({ envelope, resolve, reject });
+      if (this.queue.length >= this.maxBatch) this.flushNow();
+      else this.scheduleFlush();
+    });
+  }
+
+  /**
+   * A caller that already has a batch skips the queue entirely — it is by definition
+   * already coalesced, and threading it through the queue would only risk splitting it.
+   */
+  appendBatch(envelopes: readonly EventEnvelope[]): Promise<void> {
+    if (envelopes.length === 0) return Promise.resolve();
+    if (this.closed) return Promise.reject(new Error("CoalescingEventStore is closed"));
+    return this.enqueueFlush(() => appendBatchOf(this.inner, envelopes));
+  }
+
+  private scheduleFlush(): void {
+    if (this.flushScheduled) return;
+    this.flushScheduled = true;
+    if (this.maxLingerMs === 0) {
+      // Next microtask: collects everything enqueued in this turn — which under concurrency
+      // is the whole in-flight set — without deferring to a timer.
+      queueMicrotask(() => this.flushNow());
+    } else {
+      this.timer = setTimeout(() => this.flushNow(), this.maxLingerMs);
+    }
+  }
+
+  private flushNow(): void {
+    if (this.timer !== undefined) {
+      clearTimeout(this.timer);
+      this.timer = undefined;
+    }
+    this.flushScheduled = false;
+    if (this.queue.length === 0) return;
+    const batch = this.queue;
+    this.queue = [];
+    void this.enqueueFlush(async () => {
+      await appendBatchOf(
+        this.inner,
+        batch.map((p) => p.envelope)
+      );
+    }).then(
+      () => {
+        for (const p of batch) p.resolve();
+      },
+      (err: unknown) => {
+        // Every writer in a failed batch learns it failed — a partial write must never be
+        // reported as success.
+        for (const p of batch) p.reject(err);
+      }
+    );
+  }
+
+  /** Chain onto the serial tail so batches commit one at a time, in enqueue order. */
+  private enqueueFlush(run: () => Promise<void>): Promise<void> {
+    const result = this.tail.then(run);
+    // The tail itself must never reject, or every later flush would inherit the failure.
+    this.tail = result.then(
+      () => undefined,
+      () => undefined
+    );
+    return result;
+  }
+
+  read(executionId: string): Promise<EventEnvelope[]> {
+    return this.inner.read(executionId);
+  }
+
+  readFrom(executionId: string, fromSequence: number): Promise<EventEnvelope[]> {
+    return this.inner.readFrom(executionId, fromSequence);
+  }
+
+  subscribe(executionId: string, handler: (event: EventEnvelope) => void): Promise<() => void> {
+    return this.inner.subscribe(executionId, handler);
+  }
+
+  /** Flush anything queued and wait for in-flight batches. Safe to call repeatedly. */
+  async drain(): Promise<void> {
+    this.flushNow();
+    await this.tail;
+  }
+
+  async close(): Promise<void> {
+    await this.drain();
+    this.closed = true;
+    await this.inner.close();
+  }
+}

package/src/InMemoryArchiveStore.ts

@@ -0,0 +1,69 @@
+import { pageIds } from "./archive";
+import {
+  type ArchiveExecutionPage,
+  type ArchiveSegment,
+  type ArchiveSegmentHeader,
+  type ArchiveStore,
+  assertReplaceable,
+  classifyAppend,
+  headerOf,
+} from "./archiveSpi";
+import { compareExecutionIds } from "./executionIdOrder";
+
+/** In-memory {@link ArchiveStore}: for tests, the memory storage kind and in-memory event stores. */
+export class InMemoryArchiveStore implements ArchiveStore {
+  private readonly byExecution = new Map<string, ArchiveSegment[]>();
+
+  async append(executionId: string, segment: ArchiveSegment): Promise<void> {
+    const stored = this.byExecution.get(executionId) ?? [];
+    if (classifyAppend(executionId, stored, segment) === "noop") return;
+    // A copy through JSON, as a durable backend would hold it: the caller's objects never alias.
+    const copy = JSON.parse(JSON.stringify(segment)) as ArchiveSegment;
+    this.byExecution.set(executionId, [...stored, copy]);
+  }
+
+  async headers(executionId: string): Promise<ArchiveSegmentHeader[]> {
+    return (this.byExecution.get(executionId) ?? []).map(headerOf);
+  }
+
+  async listExecutions(
+    opts: { cursor?: string; limit?: number } = {}
+  ): Promise<ArchiveExecutionPage> {
+    const ids = [...this.byExecution.entries()]
+      .filter(([, segs]) => segs.length > 0)
+      .map(([id]) => id)
+      .sort(compareExecutionIds);
+    return pageIds(ids, opts);
+  }
+
+  async read(
+    executionId: string,
+    range: { readonly from?: number; readonly through?: number } = {}
+  ): Promise<ArchiveSegment[]> {
+    return (this.byExecution.get(executionId) ?? [])
+      .filter(
+        (s) =>
+          (range.from === undefined || s.through >= range.from) &&
+          (range.through === undefined || s.from <= range.through)
+      )
+      .map((s) => JSON.parse(JSON.stringify(s)) as ArchiveSegment);
+  }
+
+  async replaceSegments(
+    executionId: string,
+    expected: readonly ArchiveSegmentHeader[],
+    next: readonly ArchiveSegment[]
+  ): Promise<void> {
+    const stored = this.byExecution.get(executionId) ?? [];
+    assertReplaceable(executionId, stored.map(headerOf), expected, next);
+    const copy = JSON.parse(JSON.stringify(next)) as ArchiveSegment[];
+    if (copy.length === 0) this.byExecution.delete(executionId);
+    else this.byExecution.set(executionId, copy);
+  }
+
+  async deleteExecution(executionId: string): Promise<void> {
+    this.byExecution.delete(executionId);
+  }
+
+  async close(): Promise<void> {}
+}