@humwick/server 0.0.0-stage → 0.130.0

package/node_modules/@humwick/kernel/workflow-activation.ts

@@ -0,0 +1,25 @@
+import { z } from "zod";
+import { workflowDefinitionSchema } from "./workflow-definition";
+import { workflowObject } from "./workflow-values";
+
+export const workflowActivationSchema = z.object({
+  id: z.string(), runId: z.string(), path: z.string(), entry: z.number().int(), attempt: z.number().int(),
+  kind: z.enum(["role", "script", "decision", "workflow", "for-each", "parallel"]),
+  state: z.enum(["pending", "ready", "running", "waiting", "paused", "succeeded", "failed", "skipped", "cancelled"]),
+  inputs: workflowObject, outputs: workflowObject.optional(),
+  error: workflowObject.optional(), childRunId: z.string().optional(), taskId: z.number().int().optional(),
+  waitKey: z.string(), deadline: z.number().int().optional(), generation: z.number().int(),
+  outcome: z.string().optional(), createdAt: z.number().int(), updatedAt: z.number().int(),
+  parentId: z.string().optional(), index: z.number().int().optional(), item: z.unknown().optional(),
+  key: z.string().optional(), needs: z.array(z.string()).optional(), successors: z.array(z.string()).optional(), remaining: z.number().int().optional(),
+  lane: workflowObject.optional(),
+  active: z.boolean().optional(), scheduling: z.boolean().optional(),
+  container: z.object({ total: z.number().int(), next: z.number().int(), live: z.number().int(), settled: z.number().int(), failed: z.number().int(), stopped: z.boolean(), decision: z.boolean().optional(), winner: z.string().optional() }).optional(),
+});
+export const workflowSnapshotSchema = z.record(z.string(), z.object({
+  definition: workflowDefinitionSchema, source: z.enum(["bundled", "project", "project-overrides-bundled"]),
+  roles: z.record(z.string(), workflowObject),
+}));
+export const workflowBudgetSchema = z.object({ children: z.number().int(), activations: z.number().int(), agents: z.number().int(), scripts: z.number().int().optional() });
+export const workflowReservationSchema = z.object({ id: z.string(), runId: z.string(), rootRunId: z.string(), state: z.enum(["live", "released"]), kind: z.enum(["agents", "scripts"]).optional() });
+export type WorkflowActivation = z.output<typeof workflowActivationSchema>;

package/node_modules/@humwick/kernel/workflow-backstop.ts

@@ -0,0 +1,40 @@
+import { z } from "zod";
+import type { ModuleCtx } from "./contract";
+import { waitTimeoutMs } from "./workflow-definition";
+import type { Stores, WorkflowRunRecord } from "./workflow.pilot";
+
+/** #3285 — leave at least two hours beyond the ordinary deadline, and double longer waits. */
+export function workflowOverdueAt(deadline: number, timeoutMs: number): number {
+  return deadline + Math.max(timeoutMs, 2 * 60 * 60_000);
+}
+
+export function workflowBackstop(run: Pick<WorkflowRunRecord, "definition" | "step">, since: number, deadline = since + waitTimeoutMs(run.definition, run.step)) {
+  if (run.definition.steps[run.step]?.decision) return { overdueAt: undefined, overdueSince: since, overdueNotifiedAt: undefined };
+  return { overdueAt: workflowOverdueAt(deadline, waitTimeoutMs(run.definition, run.step)), overdueSince: since, overdueNotifiedAt: undefined };
+}
+
+export function workflowStepChange(run: WorkflowRunRecord, step: string, now: number) {
+  return { step, ...workflowBackstop({ ...run, step }, now), awaiting: null, updatedAt: now };
+}
+
+export const workflowBackstopInput = z.object({ runId: z.string().min(1), now: z.number().int() });
+
+/** An independent maintenance pass writes the same durable escalation intent the supervisor sends. */
+export async function checkWorkflowBackstop(ctx: ModuleCtx, input: z.output<typeof workflowBackstopInput>, stores: Stores): Promise<{ created: boolean; effectId?: string }> {
+  const run = await stores.run.get(input.runId);
+  if (!run || run.status !== "running" || run.cancel) return { created: false };
+  if (run.definition.steps[run.step]?.decision) return { created: false };
+  const wait = run.awaiting;
+  const since = run.overdueSince ?? wait?.since ?? run.createdAt;
+  const overdueAt = run.overdueAt ?? workflowBackstop(run, since, wait?.deadline).overdueAt;
+  if (overdueAt === undefined) return { created: false };
+  // Old runs gain the same bound without resetting their age at deployment or restart.
+  if (run.overdueAt === undefined) await stores.run.update(run.id, { overdueAt, overdueSince: since });
+  const id = `${run.id}:overdue:${run.waitSeq}:${run.resumes?.length ?? 0}:${overdueAt}`;
+  if (run.overdueNotifiedAt !== undefined) return { created: false, effectId: id };
+  if (input.now < overdueAt) return { created: false };
+  const payload = { reason: "run-overdue", alert: true, step: run.step, round: run.round, since, overdueAt, waitingFor: { kind: wait?.kind ?? "step transition", taskId: wait?.taskId ?? run.taskId } };
+  await stores.effect.create({ id, runId: run.id, kind: "escalate", ...(wait ? { waitKey: wait.key } : {}), payload: { ...payload, ...(wait ? { wait } : {}) }, status: "pending", attempts: 0, createdAt: input.now });
+  await stores.run.update(run.id, { overdueNotifiedAt: input.now, escalation: { id, at: input.now, ...(wait ? { waitKey: wait.key } : {}) }, updatedAt: input.now });
+  return { created: true, effectId: id };
+}

package/node_modules/@humwick/kernel/workflow-cancel-tree.ts

@@ -0,0 +1,20 @@
+import { z } from "zod";
+import type { ModuleCtx } from "./contract";
+import { ADMIN_GRANT, stores } from "./workflow-state";
+
+const input = z.object({ id: z.string(), agentId: z.string(), reason: z.string() });
+
+/** Direct delegated children share the workflow driver's durable cleanup outbox. */
+export const workflowCancelTreeOps = {
+  cancelAgentTree: {
+    kind: "mutation", idempotent: true, grant: ADMIN_GRANT,
+    summary: "Persist cleanup of a stopped delegated agent subtree (system-only).",
+    expose: { http: false, mcp: false, cli: false }, input, output: z.object({ created: z.boolean() }),
+    handler: async (ctx: ModuleCtx, args: z.output<typeof input>) => {
+      const effects = stores(ctx).effect;
+      if (await effects.get(args.id)) return { created: false };
+      await effects.create({ id: args.id, runId: args.agentId, kind: "cancel-agent-tree", payload: args, status: "pending", attempts: 0, createdAt: ctx.clock.now() });
+      return { created: true };
+    },
+  },
+} as const;

package/node_modules/@humwick/kernel/workflow-chaos-worker.ts

@@ -0,0 +1,213 @@
+import { relayEnv } from "@humwick/sdk/env";
+/**
+ * M4 kill-9 chaos harness — the spawned WORKER entrypoint. This is NOT a test file (it takes env and
+ * drives one workflow against a shared on-disk SQLite/WAL database, possibly SIGKILLing itself at a
+ * labeled boundary). `kernel/workflow-chaos.integration.test.ts` spawns it as real OS processes.
+ *
+ * The harness proves exactly-once step execution survives a REAL `kill -9` across process boundaries,
+ * which the single-thread `bun:sqlite` unit tests structurally cannot: only real processes against
+ * one shared WAL file expose cross-process contention (SQLITE_BUSY) and a genuinely-aborted commit
+ * transaction. A worker either drives a workflow to a crash at an armed boundary (self-`SIGKILL`) or,
+ * as a fresh process, RECOVERS orphaned instances via the runtime's recover()/wakeup() entry points.
+ *
+ * The idempotency-keyed side-effect ledger (harness-owned tables, distinct from the kernel journal):
+ *   - `chaos_applied(key PRIMARY KEY)`  — an external effect keyed by `${inputId}:${stepName}`,
+ *     written in the step body via INSERT OR IGNORE. A crash+resume re-runs the body, which repeats
+ *     the SAME key → absorbed. This proves the key was applied AT LEAST once (`applied.has(key)`);
+ *     it CANNOT prove "at most once" — `key` is a PRIMARY KEY, so a duplicate row is structurally
+ *     impossible regardless of runtime behavior. That claim instead rests on the raw `chaos_attempts`
+ *     counter plus the completed-journal-row set (see the integration test).
+ *   - `chaos_attempts(key)`            — an append-only raw counter of every body execution, so the
+ *     at-least-once nature is directly observable (a resumed step's body ran ≥ once).
+ * The K4 `mark` entity write inside each step is the IN-TRANSACTION effect: exactly-once by
+ * construction (it commits atomically with the journal memo; a committed step is never re-run).
+ *
+ * The kernel runtime stays deterministic. Only THIS harness scaffolding uses real processes / real
+ * wall-clock / real signals — exactly what the working agreement sanctions for the acceptance gate.
+ */
+import { Database } from "bun:sqlite";
+import { z } from "zod";
+import {
+  type EntityStore,
+  type Grant,
+  type ModuleCtx,
+  type StepResult,
+  type WorkflowStepCommitFaults,
+  createSqliteModuleHost,
+  createRealClock,
+  defineModule,
+} from "./index";
+
+const DB_PATH = required("CHAOS_DB");
+const CMD = required("CHAOS_CMD"); // "start" | "recover"
+const WORKFLOW = process.env.CHAOS_WORKFLOW ?? "chain";
+const KEY = process.env.CHAOS_KEY ?? "k";
+const BOUNDARY = process.env.CHAOS_BOUNDARY ?? "none";
+const TARGET_STEP = process.env.CHAOS_TARGET_STEP ?? (WORKFLOW === "flaky" ? "attempt" : "s2");
+const LEASE_TTL_MS = Number(process.env.CHAOS_LEASE_TTL_MS ?? "200");
+const RECOVER_DEADLINE_MS = Number(process.env.CHAOS_RECOVER_DEADLINE_MS ?? "8000");
+const BUSY_TIMEOUT_MS = Number(process.env.CHAOS_BUSY_TIMEOUT_MS ?? "5000");
+
+function required(name: string): string {
+  const value = relayEnv(process.env, name);
+  if (!value) throw new Error(`chaos worker requires env ${name}`);
+  return value;
+}
+
+function selfKill(): never {
+  // A real, uncatchable kill-9 at the exact labeled boundary. SIGKILL cannot be trapped, so no
+  // cleanup runs — precisely the crash the durability argument must survive.
+  process.kill(process.pid, "SIGKILL");
+  throw new Error("unreachable after SIGKILL");
+}
+
+/** Map a labeled COMMIT boundary to the fault hook that self-kills there. */
+function commitFaultFor(boundary: string): WorkflowStepCommitFaults | undefined {
+  switch (boundary) {
+    case "before-commit": return { beforeCommit: selfKill };
+    case "after-fence": return { afterFenceBeforeEntities: selfKill };
+    case "between-entities-events": return { betweenEntitiesAndEvents: selfKill };
+    case "before-journal": return { insideTxBeforeJournal: selfKill };
+    case "after-journal": return { afterJournalBeforeCommit: selfKill };
+    case "after-commit": return { afterCommit: selfKill };
+    // A crash while journaling the FAILED attempt / parking the retry backoff row.
+    case "backoff": return { insideTxBeforeJournal: selfKill };
+    default: return undefined; // "mid-body" is a body-level kill; "none" arms nothing.
+  }
+}
+
+const markSchema = z.object({ id: z.string(), step: z.string(), n: z.number().int() });
+function markStore(ctx: ModuleCtx): EntityStore<{ schema: typeof markSchema }> {
+  return (ctx.entities as unknown as { readonly mark: EntityStore<{ schema: typeof markSchema }> }).mark;
+}
+
+/** Open the shared on-disk connection in WAL with a real busy_timeout, and install the harness-owned
+ *  idempotency ledger tables (idempotent — every worker may run this). */
+function openDb(): Database {
+  const db = new Database(DB_PATH);
+  // busy_timeout FIRST: switching the file to WAL and creating the ledger tables both need the write
+  // lock, and many workers open the shared db at once — without the timeout set first, that startup
+  // race would fail with a raw `database is locked` before the runtime is even built.
+  db.run(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`);
+  db.run("PRAGMA journal_mode = WAL");
+  db.run("CREATE TABLE IF NOT EXISTS chaos_applied (key TEXT PRIMARY KEY)");
+  db.run("CREATE TABLE IF NOT EXISTS chaos_attempts (key TEXT NOT NULL)");
+  return db;
+}
+
+/** Record the external, idempotency-keyed side effect for a step: an idempotent apply + a raw
+ *  attempt, both autocommit writes OUTSIDE the step's K4/K5 commit tx — modelling a keyed external
+ *  effect that a resume re-issues but the key absorbs. Keyed by the per-instance-unique input id. */
+function recordExternalEffect(db: Database, inputId: string, stepName: string): void {
+  const key = `${inputId}:${stepName}`;
+  db.query("INSERT INTO chaos_attempts (key) VALUES (?)").run(key);
+  db.query("INSERT OR IGNORE INTO chaos_applied (key) VALUES (?)").run(key);
+}
+
+const idInput = z.object({ id: z.string() });
+const chainState = z.object({ id: z.string(), n: z.number().int() });
+type ChainState = z.output<typeof chainState>;
+
+function buildModule(db: Database) {
+  // Each step: (1) record the keyed EXTERNAL effect, (2) optionally self-kill mid-body (effect
+  // emitted, process dies before commit), (3) do the IN-TX K4 write + K5 emit, (4) advance. A `chain`
+  // is a straight s1→s2→s3(done); `flaky` is one step that throws until its ledger says it may pass.
+  async function chainStep(name: string, next: string, ctx: ModuleCtx, s: ChainState): Promise<StepResult<ChainState>> {
+    recordExternalEffect(db, s.id, name);
+    if (BOUNDARY === "mid-body" && name === TARGET_STEP) selfKill();
+    await markStore(ctx).create({ id: `${s.id}:${name}`, step: name, n: s.n + 1 });
+    await ctx.events.emit("chaos.stepped", { id: s.id, step: name });
+    return next === "done"
+      ? { state: { ...s, n: s.n + 1 }, next: "done", output: { n: s.n + 1 } }
+      : { state: { ...s, n: s.n + 1 }, next };
+  }
+
+  return defineModule({
+    id: "chaos",
+    title: "M4 kill-9 chaos pilot",
+    description: "Real multi-process crash-safety fixture over a shared WAL db.",
+    entities: { mark: { schema: markSchema, indexes: ["step"] } },
+    events: { "chaos.stepped": z.object({ id: z.string(), step: z.string() }) },
+    workflows: {
+      chain: {
+        input: idInput, state: chainState,
+        init: (i): ChainState => ({ id: i.id, n: 0 }), start: "s1",
+        steps: {
+          s1: { run: (ctx, s: ChainState) => chainStep("s1", "s2", ctx, s) },
+          s2: { run: (ctx, s: ChainState) => chainStep("s2", "s3", ctx, s) },
+          s3: { run: (ctx, s: ChainState) => chainStep("s3", "done", ctx, s) },
+        },
+      },
+      flaky: {
+        input: idInput, state: chainState,
+        init: (i): ChainState => ({ id: i.id, n: 0 }), start: "attempt",
+        steps: {
+          attempt: {
+            run: async (ctx, s: ChainState): Promise<StepResult<ChainState>> => {
+              recordExternalEffect(db, s.id, "attempt");
+              // Fail on the first two body runs (observed via the raw attempt counter), then succeed —
+              // so a `backoff` crash lands while the retry backoff row is being journaled.
+              const attempts = (db.query("SELECT COUNT(*) AS c FROM chaos_attempts WHERE key = ?")
+                .get(`${s.id}:attempt`) as { c: number }).c;
+              if (attempts <= 2) throw new Error(`transient (attempt ${attempts})`);
+              await markStore(ctx).create({ id: `${s.id}:attempt`, step: "attempt", n: s.n + 1 });
+              return { state: { ...s, n: s.n + 1 }, next: "done", output: { n: s.n + 1 } };
+            },
+            retry: { maxAttempts: 5, backoff: { kind: "fixed", delayMs: 40 } },
+          },
+        },
+      },
+    },
+  });
+}
+
+type Driver = ReturnType<ReturnType<typeof createSqliteModuleHost>["workflowDriver"]>;
+
+function buildDriver(db: Database): Driver {
+  const module = buildModule(db);
+  const host = createSqliteModuleHost({
+    db,
+    clock: createRealClock(),
+    leaseTtlMs: LEASE_TTL_MS,
+    busyTimeoutMs: BUSY_TIMEOUT_MS,
+    workflowCommitFaults: (_instanceId, stepName) =>
+      (stepName === TARGET_STEP ? commitFaultFor(BOUNDARY) : undefined),
+  });
+  host.register(module);
+  return host.workflowDriver(module, { actor: "user:chaos", grants: [] as Grant[] });
+}
+
+/** Fresh-process recovery: reclaim orphaned instances and fire due timers/backoffs until every
+ *  instance is terminal or the deadline elapses. Polls on a short real-clock interval; a crashed
+ *  worker's lease is reclaimable only after its TTL lapses, so the loop simply waits it out. */
+async function recoverLoop(driver: Driver, db: Database): Promise<void> {
+  const deadline = Date.now() + RECOVER_DEADLINE_MS;
+  const openCount = () => (db.query(
+    "SELECT COUNT(*) AS c FROM workflow_instances WHERE status NOT IN ('done','failed','canceled')",
+  ).get() as { c: number }).c;
+  while (Date.now() < deadline) {
+    await driver.recover(); // reclaim expired-lease pending/running instances
+    await driver.wakeup();   // fire due sleepers (retry backoffs) + satisfied waiters
+    if (openCount() === 0) return;
+    await Bun.sleep(15);
+  }
+}
+
+async function main(): Promise<void> {
+  const db = openDb();
+  const driver = buildDriver(db);
+
+  if (CMD === "start") {
+    // start() seeds the pending instance then drives it; an armed crash boundary self-kills mid-drive
+    // (this process never returns), otherwise the workflow runs to its terminal/parked state.
+    await driver.start(WORKFLOW as never, { id: KEY } as never, { idempotencyKey: KEY });
+    process.exit(0);
+  }
+  if (CMD === "recover") {
+    await recoverLoop(driver, db);
+    process.exit(0);
+  }
+  throw new Error(`unknown CHAOS_CMD '${CMD}'`);
+}
+
+await main();

package/node_modules/@humwick/kernel/workflow-composition.ts

@@ -0,0 +1,231 @@
+import { isDeepStrictEqual } from "node:util";
+import { z } from "zod";
+import { OpError, type ModuleCtx } from "./contract";
+import { workflowActivationSchema } from "./workflow-activation";
+import { workflowBackstop } from "./workflow-backstop";
+import { actionKind } from "./workflow-lane";
+import { validateWorkflowValue, workflowValue } from "./workflow-values";
+import { ADMIN_GRANT, runSchema, stores, requireRun, linkTask, writeEffect, type WorkflowRunRecord } from "./workflow-state";
+
+export function valueContext(run: WorkflowRunRecord): Record<string, unknown> {
+  return { inputs: run.inputs ?? {}, steps: run.results ?? {}, ...run.itemContext, run: { id: run.id, rootRunId: run.rootRunId ?? run.id, taskId: run.taskId, round: run.round, headSha: run.head ?? "", landedSha: run.landed?.sha ?? "" } };
+}
+export async function rootRun(ctx: ModuleCtx, run: WorkflowRunRecord): Promise<WorkflowRunRecord> {
+  return run.rootRunId && run.rootRunId !== run.id ? requireRun(ctx, run.rootRunId) : run;
+}
+export async function allocateRunId(ctx: ModuleCtx, taskId: number): Promise<string> {
+  const key = `run:${taskId}`;
+  let allocation = await stores(ctx).allocation.get(key);
+  if (!allocation) {
+    let max = 0;
+    let cursor: string | undefined;
+    do {
+      const page = await stores(ctx).run.list({ where: { taskId }, limit: 200, ...(cursor ? { cursor } : {}) });
+      for (const run of page.items) max = Math.max(max, Number(run.id.split("-").at(-1)) || 0);
+      cursor = page.cursor;
+    } while (cursor);
+    allocation = await stores(ctx).allocation.create({ id: key, value: max });
+  }
+  await stores(ctx).allocation.update(key, { value: allocation.value + 1 });
+  return `wf-T${taskId}-${allocation.value + 1}`;
+}
+
+export async function enterActivation(ctx: ModuleCtx, run: WorkflowRunRecord, key: string, deadline: number | undefined, skipped = false): Promise<Record<string, unknown>> {
+  if (!run.rootRunId) return {};
+  const root = await rootRun(ctx, run);
+  if (root.cancel || root.status !== "running") throw new OpError("conflict", 409, "ROOT_STOPPED: root has stopped admissions");
+  const step = run.definition.steps[run.step];
+  if (run.itemContext) {
+    const activation = await stores(ctx).activation.get(run.id);
+    const inputs = activation!.inputs;
+    if (new TextEncoder().encode(JSON.stringify(inputs)).length > 65536) throw new OpError("invalid_input", 400, "INPUT_INVALID: inputs exceed 64 KiB");
+    await stores(ctx).activation.update(run.id, { inputs, waitKey: key, deadline, state: "waiting", updatedAt: ctx.clock.now() });
+    return { activationId: activation!.id, inputs };
+  }
+  const id = `${run.id}:${run.waitSeq + 1}:1`;
+  const inputs = skipped ? {} : workflowValue(step?.inputs ?? {}, valueContext(run)) as Record<string, unknown>;
+  if (new TextEncoder().encode(JSON.stringify(inputs)).length > 65536) throw new OpError("invalid_input", 400, "INPUT_INVALID: inputs exceed 64 KiB");
+  const budget = root.budget ?? { children: 0, activations: 0, agents: 0 };
+  if (budget.activations >= (root.limits?.activations ?? 4096)) throw new OpError("conflict", 409, "ACTIVATION_LIMIT: root budget exhausted");
+  const now = ctx.clock.now();
+  await stores(ctx).activation.create({ id, runId: run.id, path: run.step, entry: run.waitSeq + 1, attempt: run.scriptPoll?.attempt ?? 1,
+    kind: actionKind(step!), state: "waiting", inputs, waitKey: key, deadline,
+    generation: root.generation ?? 0, createdAt: now, updatedAt: now });
+  await stores(ctx).run.update(root.id, { budget: { ...budget, activations: budget.activations + 1 } });
+  await stores(ctx).run.update(run.id, { activationId: id });
+  if (!skipped && step?.task === "input" && !step.script && !step.workflow) await linkTask(ctx, run.taskId, run.id, step.gate, id);
+  if (step?.workflow) {
+    const link = await stores(ctx).link.get(String(run.taskId));
+    if (link?.runId === run.id) await stores(ctx).link.delete(link.id);
+  }
+  return { activationId: id, inputs };
+}
+
+export async function settleActivation(ctx: ModuleCtx, run: WorkflowRunRecord, outcome: string, outputs: Record<string, unknown>, error?: Record<string, unknown>): Promise<Partial<WorkflowRunRecord>> {
+  if (!run.activationId) return {};
+  const successful = outcome === "complete" || outcome === "pass";
+  const state = outcome === "skipped" ? "skipped" : error ? "failed" : successful ? "succeeded" : outcome === "blocked" ? "paused" : "failed";
+  await stores(ctx).activation.update(run.activationId, { state, outcome, outputs: outcome === "skipped" ? undefined : outputs, ...(error ? { error } : {}), updatedAt: ctx.clock.now() });
+  return { results: { ...run.results, [run.step]: { status: state, outcome, ...(outcome === "skipped" ? {} : { outputs }), ...(error ? { error } : {}) } } };
+}
+
+export async function releaseReservation(ctx: ModuleCtx, effectId: string): Promise<boolean> {
+  const reservation = await stores(ctx).reservation.get(effectId);
+  if (!reservation || reservation.state === "released") return false;
+  const root = await requireRun(ctx, reservation.rootRunId);
+  await stores(ctx).reservation.update(effectId, { state: "released" });
+  const kind = reservation.kind ?? "agents";
+  await stores(ctx).run.update(root.id, { budget: { ...root.budget!, [kind]: Math.max(0, (root.budget?.[kind] ?? 1) - 1) } });
+  return true;
+}
+
+export async function parkDecision(ctx: ModuleCtx, child: WorkflowRunRecord, parent: WorkflowRunRecord, reason: string, sourceEffectId?: string): Promise<{ created: boolean; rootRunId: string; decisionEffectId?: string }> {
+  const root = await rootRun(ctx, parent);
+  if (root.cancel || root.status !== "running") return { created: false, rootRunId: root.id, ...(root.escalation ? { decisionEffectId: root.escalation.id } : {}) };
+  const source = sourceEffectId ? await stores(ctx).effect.get(sourceEffectId) : undefined;
+  const now = ctx.clock.now();
+  let ancestor: WorkflowRunRecord | undefined = parent;
+  while (ancestor) {
+    if (ancestor.activationId) await stores(ctx).activation.update(ancestor.activationId, { state: "paused", error: { code: "CHILD_PAUSED", childRunId: child.id, message: reason }, updatedAt: now });
+    await stores(ctx).run.update(ancestor.id, { status: "escalated", reason, decisionChildRunId: child.id, updatedAt: now });
+    ancestor = ancestor.parentRunId ? await requireRun(ctx, ancestor.parentRunId) : undefined;
+  }
+  const seq = root.waitSeq + 1;
+  const id = await writeEffect(ctx, root, seq, "escalate", { reason, childRunId: child.id,
+    childDecision: { ...source?.payload, runId: child.id, step: source?.payload.step ?? child.step, ...(source?.payload.wait ? { wait: source.payload.wait } : child.awaiting ? { wait: child.awaiting } : {}), ...(sourceEffectId ? { sourceEffectId } : {}) } });
+  await stores(ctx).run.update(root.id, { waitSeq: seq, escalation: { id, at: now } });
+  return { created: true, rootRunId: root.id, decisionEffectId: id };
+}
+
+export const compositionOps = {
+  parkChild: {
+    kind: "mutation", idempotent: true, grant: ADMIN_GRANT, summary: "Park the owning tree on one child's durable decision.", expose: { http: false, mcp: false, cli: false },
+    input: z.object({ runId: z.string(), childRunId: z.string(), reason: z.string().optional() }), output: z.object({ created: z.boolean() }),
+    handler: async (ctx: ModuleCtx, input: { runId: string; childRunId: string; reason?: string }) => {
+      const child = await requireRun(ctx, input.childRunId);
+      const parent = await requireRun(ctx, input.runId);
+      if (child.parentRunId !== parent.id || (child.status !== "escalated" && !input.reason) || parent.cancel) return { created: false };
+      const reason = input.reason ?? `Child ${child.id} parked: ${child.reason ?? "decision required"}`;
+      const decision = await parkDecision(ctx, child, parent, reason, input.reason ? undefined : child.escalation?.id);
+      return { created: decision.created };
+    },
+  },
+  routeDecision: {
+    kind: "mutation", idempotent: true, grant: ADMIN_GRANT, summary: "Route a descendant decision fact through the root before acknowledging it.", expose: { http: false, mcp: false, cli: false },
+    input: z.object({ effectId: z.string() }), output: z.object({ created: z.boolean(), rootRunId: z.string(), decisionEffectId: z.string().optional(), obsolete: z.boolean().optional() }),
+    handler: async (ctx: ModuleCtx, input: { effectId: string }) => {
+      const effect = await stores(ctx).effect.get(input.effectId);
+      if (!effect || effect.kind !== "escalate") throw new OpError("not_found", 404, "decision effect not found");
+      const child = await requireRun(ctx, effect.runId);
+      if (!child.parentRunId) throw new OpError("invalid_input", 400, "decision already belongs to root");
+      if (effect.payload.alert === true && (child.status !== "running" || (effect.waitKey && effect.waitKey !== child.awaiting?.key))) return { created: false, rootRunId: child.rootRunId!, obsolete: true };
+      return parkDecision(ctx, child, await requireRun(ctx, child.parentRunId), `Child ${child.id}: ${String(effect.payload.reason)}`, effect.id);
+    },
+  },
+  cleanup: {
+    kind: "mutation", idempotent: true, grant: ADMIN_GRANT, summary: "Record cancellation cleanup without changing the terminal outcome.", expose: { http: false, mcp: false, cli: false },
+    input: z.object({ runId: z.string(), complete: z.boolean(), error: z.string().optional() }), output: z.object({ recorded: z.boolean() }),
+    handler: async (ctx: ModuleCtx, input: { runId: string; complete: boolean; error?: string }) => {
+      const run = await requireRun(ctx, input.runId);
+      if (!run.cleanup) return { recorded: false };
+      const late = ctx.clock.now() >= run.cleanup.deadline;
+      await stores(ctx).run.update(run.id, { cleanup: { ...run.cleanup, status: input.complete ? "complete" : late ? "incomplete" : "pending", ...(input.error ? { error: input.error } : {}) } });
+      if (late && !input.complete) {
+        const root = await rootRun(ctx, run);
+        const id = `${root.id}:cleanup:${root.generation ?? 0}`;
+        if (!await stores(ctx).effect.get(id)) await stores(ctx).effect.create({ id, runId: root.id, kind: "escalate", status: "pending", attempts: 0, payload: { reason: `CLEANUP_INCOMPLETE: ${input.error ?? run.id}`, cleanupRunId: run.id }, createdAt: ctx.clock.now() });
+      }
+      return { recorded: true };
+    },
+  },
+  reserveAgent: {
+    kind: "mutation", idempotent: true, grant: ADMIN_GRANT, summary: "Reserve one live root-tree agent before dispatch.", expose: { http: false, mcp: false, cli: false },
+    input: z.object({ runId: z.string(), effectId: z.string() }), output: z.object({ reserved: z.boolean() }),
+    handler: async (ctx: ModuleCtx, input: { runId: string; effectId: string }) => {
+      const run = await requireRun(ctx, input.runId);
+      if (!run.rootRunId) return { reserved: false };
+      const existing = await stores(ctx).reservation.get(input.effectId);
+      if (existing) return { reserved: existing.state === "live" };
+      const root = await rootRun(ctx, run);
+      if (root.cancel || root.status !== "running" || run.status !== "running") throw new OpError("conflict", 409, "ROOT_STOPPED: dispatch is stale");
+      const budget = root.budget!;
+      if (budget.agents >= Math.min(run.limits?.agents ?? 8, root.limits?.agents ?? 8)) throw new OpError("conflict", 409, "AGENT_BUDGET_BLOCKED: retained agents exhaust root capacity; change retention or limits");
+      await stores(ctx).reservation.create({ id: input.effectId, runId: run.id, rootRunId: root.id, state: "live" });
+      await stores(ctx).run.update(root.id, { budget: { ...budget, agents: budget.agents + 1 } });
+      return { reserved: true };
+    },
+  },
+  reserveScript: {
+    kind: "mutation", idempotent: true, grant: ADMIN_GRANT, summary: "Reserve a root-tree script command before executor admission.", expose: { http: false, mcp: false, cli: false },
+    input: z.object({ runId: z.string(), effectId: z.string() }), output: z.object({ reserved: z.boolean() }),
+    handler: async (ctx: ModuleCtx, input: { runId: string; effectId: string }) => {
+      const run = await requireRun(ctx, input.runId);
+      if (!run.rootRunId) return { reserved: true };
+      const existing = await stores(ctx).reservation.get(input.effectId);
+      if (existing) return { reserved: existing.state === "live" };
+      const root = await rootRun(ctx, run);
+      if (root.cancel || run.cancel || root.status !== "running" || run.status !== "running") return { reserved: false };
+      if ((root.budget?.scripts ?? 0) >= Math.min(run.limits?.scripts ?? 8, root.limits?.scripts ?? 8)) return { reserved: false };
+      await stores(ctx).reservation.create({ id: input.effectId, runId: run.id, rootRunId: root.id, kind: "scripts", state: "live" });
+      await stores(ctx).run.update(root.id, { budget: { ...root.budget!, scripts: (root.budget?.scripts ?? 0) + 1 } });
+      return { reserved: true };
+    },
+  },
+  releaseReservation: {
+    kind: "mutation", idempotent: true, grant: ADMIN_GRANT, summary: "Release a confirmed ended or refused invocation's reservation.", expose: { http: false, mcp: false, cli: false },
+    input: z.object({ effectId: z.string() }), output: z.object({ released: z.boolean() }),
+    handler: async (ctx: ModuleCtx, input: { effectId: string }) => ({ released: await releaseReservation(ctx, input.effectId) }),
+  },
+  activations: {
+    kind: "query", grant: ADMIN_GRANT, summary: "Page a run's sequential activation lane.", expose: { http: false, mcp: false, cli: false },
+    input: z.object({ runId: z.string(), cursor: z.string().optional() }),
+    output: z.object({ items: z.array(workflowActivationSchema), cursor: z.string().optional() }),
+    handler: async (ctx: ModuleCtx, input: { runId: string; cursor?: string }) => stores(ctx).activation.list({ where: { runId: input.runId }, limit: 100, ...(input.cursor ? { cursor: input.cursor } : {}) }),
+  },
+  children: {
+    kind: "query", grant: ADMIN_GRANT, summary: "Page linked child runs.", expose: { http: false, mcp: false, cli: false },
+    input: z.object({ runId: z.string(), cursor: z.string().optional() }),
+    output: z.object({ items: z.array(runSchema), cursor: z.string().optional() }),
+    handler: async (ctx: ModuleCtx, input: { runId: string; cursor?: string }) => stores(ctx).run.list({ where: { parentRunId: input.runId }, limit: 100, ...(input.cursor ? { cursor: input.cursor } : {}) }),
+  },
+  createChild: {
+    kind: "mutation", idempotent: true, grant: ADMIN_GRANT, summary: "Atomically allocate/link one pinned child and its first driver.", expose: { http: false, mcp: false, cli: false },
+    input: z.object({ runId: z.string(), activationId: z.string(), afterEventId: z.number().int(), taskId: z.number().int().positive(), inputs: z.record(z.string(), z.unknown()) }),
+    output: z.object({ run: runSchema, created: z.boolean() }),
+    handler: async (ctx: ModuleCtx, input: { runId: string; activationId: string; afterEventId: number; taskId: number; inputs: Record<string, unknown> }) => {
+      const parent = await requireRun(ctx, input.runId);
+      const activation = await stores(ctx).activation.get(input.activationId);
+      if (!activation || (activation.runId !== parent.id && activation.id !== parent.id)) throw new OpError("not_found", 404, "activation not found");
+      if (activation.childRunId) {
+        const child = await requireRun(ctx, activation.childRunId);
+        if (!isDeepStrictEqual(child.inputs, input.inputs) || child.taskId !== input.taskId) throw new OpError("conflict", 409, "CHILD_INPUT_CONFLICT: child already allocated with other inputs");
+        return { run: child, created: false };
+      }
+      const root = await rootRun(ctx, parent);
+      if (root.cancel || root.status !== "running" || parent.status !== "running" || parent.activationId !== activation.id) throw new OpError("conflict", 409, "ROOT_STOPPED: child admission is stale");
+      const name = parent.definition.steps[parent.step]!.workflow!;
+      const snapshot = root.snapshot?.[name];
+      if (!snapshot) throw new OpError("invalid_input", 400, "SOURCE_UNAVAILABLE: child missing from pinned closure");
+      validateWorkflowValue(snapshot.definition.inputs, input.inputs, "INPUT_INVALID");
+      const depth = (parent.depth ?? 0) + 1;
+      const budget = root.budget!;
+      const owner = parent.itemContext ? await requireRun(ctx, activation.runId) : parent;
+      if (depth > (root.limits?.depth ?? 8) || (owner.childCount ?? 0) >= (owner.limits?.children ?? root.limits?.children ?? 256) || budget.children >= (root.limits?.treeChildren ?? 1024)) throw new OpError("conflict", 409, "CHILD_LIMIT: root child budget exhausted");
+      const id = await allocateRunId(ctx, input.taskId);
+      const now = ctx.clock.now();
+      const run = await stores(ctx).run.create({ id, workflow: name, ref: root.ref, sha: root.sha, source: snapshot.source,
+        taskId: input.taskId, coordinator: root.coordinator, callerAgentId: root.callerAgentId, maxLiveAgents: root.maxLiveAgents, cwd: root.cwd, orchestratorId: root.orchestratorId,
+        definition: snapshot.definition, inputs: input.inputs, routingDefaults: root.routingDefaults, parentRunId: parent.id, parentActivationId: activation.id,
+        rootRunId: root.id, depth, generation: root.generation ?? 0, limits: Object.fromEntries(Object.entries({ ...root.limits, ...snapshot.definition.limits }).map(([key, value]) => [key, Math.min(value, root.limits?.[key] ?? value)])),
+        status: "running", step: snapshot.definition.start, round: 1, entered: [], counters: {}, waitSeq: 0, awaiting: null,
+        cursor: input.afterEventId, feedback: null, rounds: [], ...workflowBackstop({ definition: snapshot.definition, step: snapshot.definition.start }, now), createdAt: now, updatedAt: now });
+      const first = snapshot.definition.steps[snapshot.definition.start]!;
+      if (first.task === "input" && !first.script && !first.workflow) await linkTask(ctx, input.taskId, id, first.gate);
+      await stores(ctx).activation.update(activation.id, { childRunId: id, taskId: input.taskId, updatedAt: now });
+      await stores(ctx).run.update(root.id, { budget: { ...budget, children: budget.children + 1 } });
+      await stores(ctx).run.update(owner.id, { childCount: (owner.childCount ?? 0) + 1 });
+      await ctx.workflows.start("run", { runId: id }, { idempotencyKey: id, enlist: "op-commit" });
+      return { run, created: true };
+    },
+  },
+} as const;

package/node_modules/@humwick/kernel/workflow-container.ts

@@ -0,0 +1,72 @@
+import { OpError, type ModuleCtx } from "./contract";
+import type { WorkflowActivation } from "./workflow-activation";
+import { workflowItemAction } from "./workflow-definition";
+import { actionKind } from "./workflow-lane";
+import { rootRun, valueContext } from "./workflow-composition";
+import { workflowValue, workflowCondition } from "./workflow-values";
+import { stores, type WorkflowRunRecord } from "./workflow-state";
+
+export function itemResult(slot: WorkflowActivation): Record<string, unknown> {
+  return { index: slot.index, key: slot.key ?? String(slot.index), status: slot.state, outcome: slot.outcome ?? slot.state,
+    ...(slot.needs ? { needs: slot.needs } : {}), ...(slot.outputs ? { outputs: slot.outputs } : {}), ...(slot.error ? { error: slot.error } : {}), ...(slot.childRunId ? { childRunId: slot.childRunId } : {}) };
+}
+
+export async function expandItems(ctx: ModuleCtx, run: WorkflowRunRecord): Promise<void> {
+  const container = (await stores(ctx).activation.get(run.activationId!))!;
+  if (container.container) return;
+  const loop = run.definition.steps[run.step]!.forEach!;
+  const items = typeof loop.items === "string" ? workflowValue(loop.items, valueContext(run)) : loop.items;
+  if (!Array.isArray(items)) throw new OpError("invalid_input", 400, "ITEMS_INVALID: source must be an array");
+  const root = await rootRun(ctx, run);
+  if (items.length > 1000 || new TextEncoder().encode(JSON.stringify(items)).length > 1048576 || root.budget!.activations + items.length > (root.limits?.activations ?? 4096)) throw new OpError("invalid_input", 400, "EXPANSION_LIMIT: entire item expansion refused");
+  const slots = items.map((item, index) => {
+    const context = { ...valueContext(run), item, index };
+    const key = loop.branches ? String(item) : loop.key === undefined ? String(index) : workflowValue(loop.key, context);
+    const needs = workflowValue(loop.needs ?? [], context);
+    if (typeof key !== "string" || !key.length || key.length > 256 || !Array.isArray(needs) || needs.some(key => typeof key !== "string") || new Set(needs).size !== needs.length) throw new OpError("invalid_input", 400, "DAG_INVALID: keys must be unique strings and needs must be distinct keys");
+    const action = workflowItemAction(loop, { key, item });
+    const inputs = action.condition !== undefined && !workflowCondition(action.condition, context) ? {} : workflowValue(action.inputs ?? {}, context) as Record<string, unknown>;
+    if (new TextEncoder().encode(JSON.stringify(inputs)).length > 65536) throw new OpError("invalid_input", 400, "INPUT_INVALID: item inputs exceed 64 KiB");
+    return { id: `${container.id}:item:${String(index).padStart(4, "0")}`, key, needs: needs as string[], successors: [] as string[], item, index, action, inputs };
+  });
+  const keyed = new Map(slots.map(slot => [slot.key, slot]));
+  if (keyed.size !== slots.length) throw new OpError("invalid_input", 400, "DAG_INVALID: duplicate key");
+  for (const slot of slots) for (const key of slot.needs) {
+    const predecessor = keyed.get(key);
+    if (!predecessor) throw new OpError("invalid_input", 400, `DAG_INVALID: missing dependency ${key}`);
+    predecessor.successors.push(slot.id);
+  }
+  const byId = new Map(slots.map(slot => [slot.id, slot]));
+  const remaining = new Map(slots.map(slot => [slot.id, slot.needs.length]));
+  const ready = slots.filter(slot => !slot.needs.length).map(slot => slot.id);
+  for (let index = 0; index < ready.length; index++) for (const id of byId.get(ready[index]!)!.successors) {
+    remaining.set(id, remaining.get(id)! - 1);
+    if (!remaining.get(id)) ready.push(id);
+  }
+  if (ready.length !== slots.length) throw new OpError("invalid_input", 400, "DAG_INVALID: dependency cycle");
+  const children = slots.filter(slot => slot.action.workflow).length;
+  if (children && (children + (run.childCount ?? 0) > (run.limits?.children ?? 256) || children + root.budget!.children > (root.limits?.treeChildren ?? 1024) || (run.depth ?? 0) + 1 > (root.limits?.depth ?? 8))) throw new OpError("invalid_input", 400, "EXPANSION_LIMIT: child allocation budget would be exceeded");
+  const now = ctx.clock.now();
+  for (const { action, ...slot } of slots) await stores(ctx).activation.create({ ...slot, runId: container.runId, parentId: container.id,
+    path: loop.branches ? `${container.path}.${slot.key}` : `${container.path}[${slot.index}]`, entry: container.entry, attempt: 1,
+    active: false, remaining: slot.needs.length, kind: actionKind(action), state: slot.needs.length ? "pending" : "ready", waitKey: "", deadline: container.deadline, generation: root.generation ?? 0, createdAt: now, updatedAt: now });
+  await stores(ctx).run.update(root.id, { budget: { ...root.budget!, activations: root.budget!.activations + items.length } });
+  await stores(ctx).activation.update(container.id, { scheduling: true, container: { total: items.length, next: 0, live: 0, settled: 0, failed: 0, stopped: false } });
+}
+
+export async function settleDependents(ctx: ModuleCtx, slot: WorkflowActivation, failed: boolean): Promise<number> {
+  let skipped = 0;
+  const pending = [...(slot.successors ?? [])];
+  for (let index = 0; index < pending.length; index++) {
+    const next = (await stores(ctx).activation.get(pending[index]!))!;
+    if (next.state !== "pending") continue;
+    if (failed) {
+      await stores(ctx).activation.update(next.id, { state: "skipped", outcome: "skipped", error: { code: "DEPENDENCY_FAILED", predecessor: slot.key ?? String(slot.index) } });
+      skipped++; pending.push(...(next.successors ?? []));
+    } else {
+      const remaining = next.remaining! - 1;
+      await stores(ctx).activation.update(next.id, { remaining, ...(remaining === 0 ? { state: "ready" } : {}) });
+    }
+  }
+  return skipped;
+}