@strav/durable 0.4.27

package/src/engine/compensate_handler.ts

@@ -0,0 +1,70 @@
+import { transaction } from '@strav/database'
+import { registry } from '../registry.ts'
+import type { JournalWrite } from '../types.ts'
+import { loadJournal, writeJournal } from '../models/journal.ts'
+import { buildContext } from './context.ts'
+import { enqueueCompensate } from './enqueue.ts'
+import { failRun } from './finalize.ts'
+import { loadRun, lockRun, type Tx } from './run_store.ts'
+import { runCompensator } from './compensation_driver.ts'
+
+/** Payload of a `durable:compensate` job. */
+export interface CompensatePayload {
+  runId: number
+  compensateIndex: number
+}
+
+/**
+ * The `durable:compensate` queue handler — rolls back one step of a failed
+ * run. The chain walks `compensation_cursor` downward; each step's compensation
+ * is journaled (`<step>#compensate`), so rollback resumes crash-safely. When
+ * the cursor reaches below zero the run is marked `failed`.
+ */
+export async function compensateHandler(payload: CompensatePayload): Promise<void> {
+  const { runId, compensateIndex } = payload
+
+  const run = await loadRun(runId)
+  if (!run) return
+  if (run.status !== 'compensating') return
+  if (run.compensationCursor !== compensateIndex) return
+
+  // Phase A — run the compensator outside the transaction (it may be slow).
+  let writes: JournalWrite[] = []
+  if (compensateIndex >= 0) {
+    const step = registry.get(run.workflowName).steps[compensateIndex]
+    if (step) {
+      const journal = await loadJournal(runId)
+      const alreadyDone =
+        journal.get(`${step.name}#compensate`)?.status === 'completed'
+      if (!alreadyDone) {
+        writes = await runCompensator(step, buildContext(run, 1, step.name), journal)
+      }
+    }
+  }
+
+  // Phase B — record the compensation and advance the cursor atomically.
+  await transaction(async (trx: Tx) => {
+    const locked = await lockRun(trx, runId)
+    if (
+      !locked ||
+      locked.status !== 'compensating' ||
+      locked.compensationCursor !== compensateIndex
+    ) {
+      return
+    }
+
+    if (writes.length > 0) await writeJournal(trx, runId, writes)
+
+    const next = compensateIndex - 1
+    if (next < 0) {
+      await failRun(trx, locked, locked.error ?? 'workflow failed')
+    } else {
+      await trx`
+        UPDATE "_strav_workflow_runs"
+        SET "compensation_cursor" = ${next}, "updated_at" = NOW()
+        WHERE "id" = ${runId}
+      `
+      await enqueueCompensate(trx, runId, next)
+    }
+  })
+}

package/src/engine/compensation_driver.ts

@@ -0,0 +1,61 @@
+import type {
+  DurableContext,
+  DurableStep,
+  JournalRecord,
+  JournalWrite,
+} from '../types.ts'
+
+function message(err: unknown): string {
+  return err instanceof Error ? err.message : String(err)
+}
+
+/**
+ * Run the compensator(s) for one step during saga rollback.
+ *
+ * Mirrors `@strav/workflow`'s compensation surface: only `step` (sequential)
+ * and `parallel` entries carry compensators, and only journaled-completed
+ * units are compensated. The result is a single `<step>#compensate` journal
+ * write — so the compensation chain is itself crash-safe (a redelivered
+ * compensate job sees the marker and skips).
+ *
+ * A throwing compensator is best-effort: its error is recorded on the journal
+ * row and the rollback still advances (it does not get stuck).
+ */
+export async function runCompensator(
+  step: DurableStep,
+  ctx: DurableContext,
+  journal: Map<string, JournalRecord>
+): Promise<JournalWrite[]> {
+  const errors: string[] = []
+
+  if (step.type === 'step') {
+    if (journal.get(step.name)?.status === 'completed' && step.compensate) {
+      try {
+        await step.compensate(ctx)
+      } catch (err) {
+        errors.push(message(err))
+      }
+    }
+  } else if (step.type === 'parallel') {
+    for (const entry of step.entries) {
+      const done = journal.get(`${step.name}#${entry.name}`)?.status === 'completed'
+      if (done && entry.compensate) {
+        try {
+          await entry.compensate(ctx)
+        } catch (err) {
+          errors.push(`${entry.name}: ${message(err)}`)
+        }
+      }
+    }
+  }
+  // route / loop / sleep / signal / child have no compensators.
+
+  return [
+    {
+      stepId: `${step.name}#compensate`,
+      status: errors.length > 0 ? 'failed' : 'completed',
+      error: errors.length > 0 ? errors.join('; ') : undefined,
+      attempt: 1,
+    },
+  ]
+}

package/src/engine/context.ts

@@ -0,0 +1,36 @@
+import type { DurableContext, RunRow } from '../types.ts'
+
+const INTERNAL_PREFIX = '__strav_'
+
+/**
+ * Build the `DurableContext` handed to step handlers from a run row.
+ *
+ * `results` is the run's accumulated `state` with engine-internal keys
+ * (e.g. `__strav_resume__`) filtered out, so it reads exactly like a
+ * `@strav/workflow` context. `resumeData()` exposes the raw resume payload.
+ */
+export function buildContext(
+  run: RunRow,
+  attempt: number,
+  stepName: string
+): DurableContext {
+  const rawState = run.state ?? {}
+  const results: Record<string, unknown> = {}
+  for (const [key, value] of Object.entries(rawState)) {
+    if (!key.startsWith(INTERNAL_PREFIX)) results[key] = value
+  }
+
+  return {
+    input: run.input ?? {},
+    results,
+    runId: run.id,
+    attempt,
+    stepName,
+    signal<T = unknown>(name: string): T | undefined {
+      return results[name] as T | undefined
+    },
+    resumeData<T = unknown>(): T | undefined {
+      return rawState['__strav_resume__'] as T | undefined
+    },
+  }
+}

package/src/engine/enqueue.ts

@@ -0,0 +1,62 @@
+import { getConfig } from '../config.ts'
+
+/**
+ * Enqueue durable jobs by INSERTing directly into `_strav_jobs` on the caller's
+ * transaction handle.
+ *
+ * This is a deliberate, documented coupling to `@strav/queue`'s table. It is
+ * NOT done via `Queue.push` because `Queue.push` runs its INSERT on a separate
+ * connection — outside the engine's transaction — which would break the atomic
+ * `{ journal write + run-row update + next-job enqueue }` commit that the whole
+ * crash-safety story depends on. The clean long-term fix is an upstream
+ * `Queue.pushTx(trx, ...)`; until then, the INSERT shape is replicated here.
+ */
+
+/** A Bun SQL transaction handle (tagged-template callable). */
+type Tx = (strings: TemplateStringsArray, ...values: unknown[]) => Promise<unknown>
+
+async function enqueueJob(
+  trx: Tx,
+  job: string,
+  payload: unknown,
+  delay: number
+): Promise<void> {
+  const cfg = getConfig()
+  const availableAt = delay > 0 ? new Date(Date.now() + delay) : new Date()
+  await trx`
+    INSERT INTO "_strav_jobs"
+      ("queue", "job", "payload", "max_attempts", "timeout", "available_at")
+    VALUES (
+      ${cfg.queue},
+      ${job},
+      ${JSON.stringify(payload)},
+      ${cfg.maxAttempts},
+      ${cfg.jobTimeout},
+      ${availableAt}
+    )
+  `
+}
+
+/** Enqueue a `durable:advance` continuation for a run at a given step index. */
+export async function enqueueAdvance(
+  trx: Tx,
+  runId: number,
+  stepIndex: number,
+  opts?: { attempt?: number; delay?: number }
+): Promise<void> {
+  await enqueueJob(
+    trx,
+    'durable:advance',
+    { runId, stepIndex, attempt: opts?.attempt ?? 1 },
+    opts?.delay ?? 0
+  )
+}
+
+/** Enqueue a `durable:compensate` job for a run at a given compensation cursor. */
+export async function enqueueCompensate(
+  trx: Tx,
+  runId: number,
+  compensateIndex: number
+): Promise<void> {
+  await enqueueJob(trx, 'durable:compensate', { runId, compensateIndex }, 0)
+}

package/src/engine/finalize.ts

@@ -0,0 +1,111 @@
+import { transaction } from '@strav/database'
+import type { RunRow } from '../types.ts'
+import { writeJournal } from '../models/journal.ts'
+import { enqueueAdvance, enqueueCompensate } from './enqueue.ts'
+import { lockRun, type Tx } from './run_store.ts'
+
+/** Merge a result patch into a run's accumulated state. */
+export function applyPatch(
+  state: Record<string, unknown>,
+  patch: Record<string, unknown>
+): Record<string, unknown> {
+  return { ...state, ...patch }
+}
+
+/**
+ * Transition a run into compensation, rolling back from the step before the
+ * one that failed. If nothing was completed before it, fail the run directly.
+ */
+export async function beginCompensation(
+  trx: Tx,
+  run: RunRow,
+  failedStepIndex: number,
+  failure: string
+): Promise<void> {
+  const cursor = failedStepIndex - 1
+  if (cursor < 0) {
+    await failRun(trx, run, failure)
+    return
+  }
+  await trx`
+    UPDATE "_strav_workflow_runs"
+    SET "status" = 'compensating', "compensation_cursor" = ${cursor},
+        "error" = ${failure}, "updated_at" = NOW()
+    WHERE "id" = ${run.id}
+  `
+  await enqueueCompensate(trx, run.id, cursor)
+}
+
+/** Mark a run failed and, if it is a child, propagate failure to its parent. */
+export async function failRun(trx: Tx, run: RunRow, failure: string): Promise<void> {
+  await trx`
+    UPDATE "_strav_workflow_runs"
+    SET "status" = 'failed', "error" = ${failure}, "updated_at" = NOW()
+    WHERE "id" = ${run.id}
+  `
+  if (run.parentRunId != null) {
+    await finalizeChildIntoParent(trx, run, 'failed')
+  }
+}
+
+/**
+ * Complete a run. Opens its own transaction (called as the terminal advance).
+ * If the run is a child, fan in: write its result into the parent journal and
+ * resume the parent — all in the same transaction, so the parent's
+ * continuation job is visible only once the child result is.
+ */
+export async function completeRun(runId: number): Promise<void> {
+  await transaction(async (trx: Tx) => {
+    const run = await lockRun(trx, runId)
+    if (!run || run.status !== 'running') return
+    await trx`
+      UPDATE "_strav_workflow_runs"
+      SET "status" = 'completed', "result" = ${JSON.stringify(run.state)},
+          "updated_at" = NOW()
+      WHERE "id" = ${runId}
+    `
+    if (run.parentRunId != null) {
+      await finalizeChildIntoParent(trx, run, 'completed')
+    }
+  })
+}
+
+/**
+ * Fan a finished child run into its parent. Locks the parent (child→parent
+ * lock order is consistent everywhere, so no deadlock).
+ */
+export async function finalizeChildIntoParent(
+  trx: Tx,
+  childRun: RunRow,
+  outcome: 'completed' | 'failed'
+): Promise<void> {
+  const parentId = childRun.parentRunId
+  const childStepId = childRun.parentStepId
+  if (parentId == null || childStepId == null) return
+
+  const parent = await lockRun(trx, parentId)
+  if (!parent || parent.status !== 'suspended') return
+
+  const parentStepIndex = parent.currentStep
+
+  if (outcome === 'completed') {
+    await writeJournal(trx, parentId, [
+      { stepId: childStepId, status: 'completed', result: childRun.state, attempt: 1 },
+    ])
+    const newState = applyPatch(parent.state, { [childStepId]: childRun.state })
+    const next = parentStepIndex + 1
+    await trx`
+      UPDATE "_strav_workflow_runs"
+      SET "status" = 'running', "state" = ${JSON.stringify(newState)},
+          "current_step" = ${next}, "updated_at" = NOW()
+      WHERE "id" = ${parentId}
+    `
+    await enqueueAdvance(trx, parentId, next)
+  } else {
+    const failure = `child workflow "${childRun.workflowName}" failed`
+    await writeJournal(trx, parentId, [
+      { stepId: childStepId, status: 'failed', error: failure, attempt: 1 },
+    ])
+    await beginCompensation(trx, parent, parentStepIndex, failure)
+  }
+}

package/src/engine/index.ts

@@ -0,0 +1,20 @@
+export { advanceHandler } from './advance_handler.ts'
+export type { AdvancePayload } from './advance_handler.ts'
+export { compensateHandler } from './compensate_handler.ts'
+export type { CompensatePayload } from './compensate_handler.ts'
+export { runDurableStep } from './step_driver.ts'
+export type { StepOutcome } from './step_driver.ts'
+export { runCompensator } from './compensation_driver.ts'
+export { buildContext } from './context.ts'
+export { isSuspendedRun } from './suspended_run.ts'
+export type { SuspendedRunLike } from './suspended_run.ts'
+export { enqueueAdvance, enqueueCompensate } from './enqueue.ts'
+export { loadRun, lockRun, hydrateRun } from './run_store.ts'
+export type { Tx } from './run_store.ts'
+export {
+  applyPatch,
+  beginCompensation,
+  completeRun,
+  failRun,
+  finalizeChildIntoParent,
+} from './finalize.ts'

package/src/engine/run_store.ts

@@ -0,0 +1,42 @@
+import { sql } from '@strav/database'
+import type { RunRow } from '../types.ts'
+import { parseJson } from '../util.ts'
+
+/** A Bun SQL transaction handle (tagged-template callable). */
+export type Tx = (strings: TemplateStringsArray, ...values: unknown[]) => Promise<unknown>
+
+/** Hydrate a raw `_strav_workflow_runs` row into a typed `RunRow`. */
+export function hydrateRun(row: Record<string, unknown>): RunRow {
+  return {
+    id: Number(row.id),
+    workflowName: row.workflow_name as string,
+    input: parseJson<Record<string, unknown>>(row.input) ?? {},
+    status: row.status as RunRow['status'],
+    state: parseJson<Record<string, unknown>>(row.state) ?? {},
+    currentStep: Number(row.current_step),
+    compensationCursor:
+      row.compensation_cursor == null ? null : Number(row.compensation_cursor),
+    parentRunId: row.parent_run_id == null ? null : Number(row.parent_run_id),
+    parentStepId: (row.parent_step_id as string | null) ?? null,
+    awaitingSignal: (row.awaiting_signal as string | null) ?? null,
+    wakeAt: (row.wake_at as Date | null) ?? null,
+    error: (row.error as string | null) ?? null,
+    result: parseJson<Record<string, unknown>>(row.result) ?? null,
+  }
+}
+
+/** Load a run by id (unlocked read). */
+export async function loadRun(runId: number): Promise<RunRow | null> {
+  const rows = (await sql`
+    SELECT * FROM "_strav_workflow_runs" WHERE "id" = ${runId}
+  `) as Record<string, unknown>[]
+  return rows.length > 0 ? hydrateRun(rows[0]!) : null
+}
+
+/** Load a run `FOR UPDATE` inside a transaction (row-locked). */
+export async function lockRun(trx: Tx, runId: number): Promise<RunRow | null> {
+  const rows = (await trx`
+    SELECT * FROM "_strav_workflow_runs" WHERE "id" = ${runId} FOR UPDATE
+  `) as Record<string, unknown>[]
+  return rows.length > 0 ? hydrateRun(rows[0]!) : null
+}

package/src/engine/step_driver.ts

@@ -0,0 +1,291 @@
+import type {
+  DurableContext,
+  DurableStep,
+  JournalRecord,
+  JournalWrite,
+} from '../types.ts'
+import { backoffDelay } from '../util.ts'
+import { isSuspendedRun } from './suspended_run.ts'
+
+/**
+ * The result of executing one top-level step. The engine applies it
+ * atomically in Phase B (a row-locked transaction).
+ */
+export type StepOutcome =
+  /** Step (and all sub-units) completed — journal results and move forward. */
+  | { kind: 'advance'; journal: JournalWrite[]; resultPatch: Record<string, unknown> }
+  /** A durable timer — journal, move forward, enqueue a delayed continuation. */
+  | { kind: 'sleep'; journal: JournalWrite[]; resultPatch: Record<string, unknown>; wakeAt: Date }
+  /** Suspend awaiting an external signal (human-in-the-loop). */
+  | { kind: 'suspend-signal'; signal: string }
+  /** Suspend on a brain agent `SuspendedRun` — resume re-enters this step. */
+  | { kind: 'suspend-agent'; stepName: string; snapshot: unknown }
+  /** Spawn a child workflow and wait for it. */
+  | { kind: 'await-child'; childName: string; childInput: Record<string, unknown>; childStepId: string }
+  /** Step failed but retries remain — re-enqueue the same step with backoff. */
+  | { kind: 'retry'; journal: JournalWrite[]; attempt: number; backoffMs: number; failure: string }
+  /** Step failed terminally — begin saga compensation. */
+  | { kind: 'compensate'; journal: JournalWrite[]; failure: string }
+
+type RetryableStep = {
+  name: string
+  maxRetries: number
+  retryBackoff: 'exponential' | 'linear'
+}
+
+function message(err: unknown): string {
+  return err instanceof Error ? err.message : String(err)
+}
+
+/** Decide between retry and compensation when a step's handler throws. */
+function failureOutcome(
+  step: RetryableStep,
+  attempt: number,
+  err: unknown,
+  partialJournal: JournalWrite[]
+): StepOutcome {
+  const failure = message(err)
+  if (attempt < step.maxRetries) {
+    return {
+      kind: 'retry',
+      journal: partialJournal,
+      attempt: attempt + 1,
+      backoffMs: backoffDelay(attempt, step.retryBackoff),
+      failure,
+    }
+  }
+  return {
+    kind: 'compensate',
+    journal: [
+      ...partialJournal,
+      { stepId: step.name, status: 'failed', error: failure, attempt },
+    ],
+    failure,
+  }
+}
+
+/**
+ * Execute one top-level step against the journal. Completed sub-units
+ * (parallel entries, loop iterations, the route decision) are read back from
+ * the journal rather than re-run, so redelivery fast-forwards to the first
+ * incomplete unit.
+ */
+export async function runDurableStep(
+  step: DurableStep,
+  ctx: DurableContext,
+  journal: Map<string, JournalRecord>
+): Promise<StepOutcome> {
+  switch (step.type) {
+    case 'step':
+      return runSequential(step, ctx)
+    case 'parallel':
+      return runParallel(step, ctx, journal)
+    case 'route':
+      return runRoute(step, ctx, journal)
+    case 'loop':
+      return runLoop(step, ctx, journal)
+    case 'sleep':
+      return runSleep(step)
+    case 'signal':
+      return { kind: 'suspend-signal', signal: step.signal }
+    case 'child':
+      return runChild(step, ctx)
+  }
+}
+
+// ── step ────────────────────────────────────────────────────────────────────
+
+async function runSequential(
+  step: Extract<DurableStep, { type: 'step' }>,
+  ctx: DurableContext
+): Promise<StepOutcome> {
+  try {
+    const result = await step.handler(ctx)
+    if (isSuspendedRun(result)) {
+      return { kind: 'suspend-agent', stepName: step.name, snapshot: result }
+    }
+    return {
+      kind: 'advance',
+      journal: [{ stepId: step.name, status: 'completed', result, attempt: ctx.attempt }],
+      resultPatch: { [step.name]: result },
+    }
+  } catch (err) {
+    return failureOutcome(step, ctx.attempt, err, [])
+  }
+}
+
+// ── parallel ────────────────────────────────────────────────────────────────
+
+async function runParallel(
+  step: Extract<DurableStep, { type: 'parallel' }>,
+  ctx: DurableContext,
+  journal: Map<string, JournalRecord>
+): Promise<StepOutcome> {
+  const settled = await Promise.all(
+    step.entries.map(async entry => {
+      const jid = `${step.name}#${entry.name}`
+      const existing = journal.get(jid)
+      if (existing?.status === 'completed') {
+        return { entry, ok: true as const, result: existing.result, fresh: false }
+      }
+      try {
+        const result = await entry.handler(ctx)
+        return { entry, ok: true as const, result, fresh: true }
+      } catch (err) {
+        return { entry, ok: false as const, err, fresh: true }
+      }
+    })
+  )
+
+  const writes: JournalWrite[] = []
+  const resultPatch: Record<string, unknown> = {}
+  let firstError: unknown
+
+  for (const s of settled) {
+    if (s.ok) {
+      resultPatch[s.entry.name] = s.result
+      if (s.fresh) {
+        writes.push({
+          stepId: `${step.name}#${s.entry.name}`,
+          status: 'completed',
+          result: s.result,
+          attempt: ctx.attempt,
+        })
+      }
+    } else if (firstError === undefined) {
+      firstError = s.err
+    }
+  }
+
+  if (firstError === undefined) {
+    writes.push({
+      stepId: step.name,
+      status: 'completed',
+      result: resultPatch,
+      attempt: ctx.attempt,
+    })
+    return { kind: 'advance', journal: writes, resultPatch }
+  }
+  return failureOutcome(step, ctx.attempt, firstError, writes)
+}
+
+// ── route ───────────────────────────────────────────────────────────────────
+
+async function runRoute(
+  step: Extract<DurableStep, { type: 'route' }>,
+  ctx: DurableContext,
+  journal: Map<string, JournalRecord>
+): Promise<StepOutcome> {
+  const routeJid = `${step.name}#route`
+  const writes: JournalWrite[] = []
+  let routeKey: string
+
+  const existingRoute = journal.get(routeJid)
+  try {
+    if (existingRoute?.status === 'completed') {
+      routeKey = existingRoute.result as string
+    } else {
+      routeKey = await step.resolver(ctx)
+      writes.push({ stepId: routeJid, status: 'completed', result: routeKey, attempt: ctx.attempt })
+    }
+  } catch (err) {
+    return failureOutcome(step, ctx.attempt, err, [])
+  }
+
+  const branch = step.branches[routeKey]
+  try {
+    let result: unknown = null
+    const existingBranch = journal.get(step.name)
+    if (existingBranch?.status === 'completed') {
+      result = existingBranch.result
+    } else if (branch) {
+      result = await branch(ctx)
+    }
+    writes.push({ stepId: step.name, status: 'completed', result, attempt: ctx.attempt })
+    return {
+      kind: 'advance',
+      journal: writes,
+      resultPatch: branch ? { [step.name]: result } : {},
+    }
+  } catch (err) {
+    // Persist the route decision so a retry takes the same branch.
+    return failureOutcome(step, ctx.attempt, err, writes)
+  }
+}
+
+// ── loop ────────────────────────────────────────────────────────────────────
+
+async function runLoop(
+  step: Extract<DurableStep, { type: 'loop' }>,
+  ctx: DurableContext,
+  journal: Map<string, JournalRecord>
+): Promise<StepOutcome> {
+  let currentInput: unknown = step.mapInput ? step.mapInput(ctx) : ctx.input
+  let lastResult: unknown
+  let ran = false
+  const writes: JournalWrite[] = []
+
+  for (let i = 0; i < step.maxIterations; i++) {
+    ran = true
+    const jid = `${step.name}#iter${i}`
+    const existing = journal.get(jid)
+
+    if (existing?.status === 'completed') {
+      lastResult = existing.result
+    } else {
+      try {
+        lastResult = await step.handler(currentInput, ctx)
+      } catch (err) {
+        return failureOutcome(step, ctx.attempt, err, writes)
+      }
+      writes.push({ stepId: jid, status: 'completed', result: lastResult, attempt: ctx.attempt })
+    }
+
+    if (step.until?.(lastResult, i + 1)) break
+    if (step.feedback) currentInput = step.feedback(lastResult)
+  }
+
+  writes.push({ stepId: step.name, status: 'completed', result: lastResult ?? null, attempt: ctx.attempt })
+  return {
+    kind: 'advance',
+    journal: writes,
+    resultPatch: ran ? { [step.name]: lastResult } : {},
+  }
+}
+
+// ── sleep ───────────────────────────────────────────────────────────────────
+
+function runSleep(step: Extract<DurableStep, { type: 'sleep' }>): StepOutcome {
+  const wakeAt =
+    step.duration instanceof Date
+      ? step.duration
+      : new Date(Date.now() + step.duration)
+  return {
+    kind: 'sleep',
+    journal: [
+      {
+        stepId: step.name,
+        status: 'completed',
+        result: { wakeAt: wakeAt.toISOString() },
+        attempt: 1,
+      },
+    ],
+    resultPatch: {},
+    wakeAt,
+  }
+}
+
+// ── child ───────────────────────────────────────────────────────────────────
+
+function runChild(
+  step: Extract<DurableStep, { type: 'child' }>,
+  ctx: DurableContext
+): StepOutcome {
+  const childInput = step.mapInput ? step.mapInput(ctx) : ctx.input
+  return {
+    kind: 'await-child',
+    childName: step.childName,
+    childInput,
+    childStepId: step.name,
+  }
+}