@vercel/factory 0.0.15 → 0.0.16

package/docs/recipes/persistence-recovery.md

@@ -0,0 +1,268 @@
+# Persist and recover workflow work
+
+Use the Postgres storage adapter when Task state must survive process restarts. This recipe uses
+PGlite's filesystem-backed PostgreSQL runtime, so it exercises the public Postgres adapter and its
+migrations without credentials or a database service.
+
+The example deliberately runs in three fresh Node.js processes. The initialize phase creates and
+starts typed workflow work, the recover phase reopens the database and completes that work, and the
+verify phase reopens it again to prove that the result and idempotency evidence were persisted.
+
+## Install and run
+
+In an ESM project running Node.js 24 or later, install the runtime packages:
+
+```sh
+pnpm add @vercel/factory @electric-sql/pglite drizzle-orm eve zod
+```
+
+Save the runnable block below as `persistence-recovery.mts`, then run each phase as a separate
+process against the same workspace directory:
+
+```sh
+node persistence-recovery.mts initialize ./factory-recovery-data
+node persistence-recovery.mts recover ./factory-recovery-data
+node persistence-recovery.mts verify ./factory-recovery-data
+```
+
+Each command prints one JSON object and exits unsuccessfully if an expected persistence or replay
+property is missing. Use an empty workspace directory for a new run.
+
+<!-- runnable-example:start -->
+
+```ts
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { PGlite } from "@electric-sql/pglite";
+import { drizzle } from "drizzle-orm/pglite";
+import { z } from "zod";
+import { createStores } from "@vercel/factory/storage";
+import { applyFactoryMigrations, createPostgresDriver } from "@vercel/factory/storage/postgres";
+import { taskIdSchema } from "@vercel/factory/tasks";
+import { defineWorkflow, defineWorkflows, taskWork } from "@vercel/factory/workflows";
+
+const phase = process.argv[2];
+const workspaceArgument = process.argv[3];
+if (
+  (phase !== "initialize" && phase !== "recover" && phase !== "verify") ||
+  workspaceArgument === undefined
+) {
+  throw new Error(
+    "Usage: persistence-recovery.mts <initialize|recover|verify> <workspace-directory>",
+  );
+}
+
+const workspace = path.resolve(workspaceArgument);
+const databasePath = path.join(workspace, "postgres");
+const taskIdPath = path.join(workspace, "root-task-id");
+const greetingWorkflow = defineWorkflow({
+  id: "persistent-greeting",
+  version: 1,
+  input: z.strictObject({ name: z.string().min(1) }),
+  output: z.strictObject({ greeting: z.string().min(1) }),
+});
+const workflows = defineWorkflows([greetingWorkflow]);
+
+async function openFactory() {
+  const client = new PGlite(databasePath);
+  const db = drizzle(client);
+  const appliedMigrations = await applyFactoryMigrations(db);
+  const stores = createStores({
+    driver: createPostgresDriver(db),
+    workflows,
+  });
+  return { appliedMigrations, client, stores };
+}
+
+async function readRootTaskId() {
+  return taskIdSchema.parse((await readFile(taskIdPath, "utf8")).trim());
+}
+
+async function initialize() {
+  await mkdir(workspace, { recursive: true });
+  const { appliedMigrations, client, stores } = await openFactory();
+  try {
+    const repository = await stores.repositories.put({
+      schemaVersion: 1,
+      id: "repo_persistrecipe",
+      slug: "example/project",
+      defaultBranch: "main",
+      enabled: true,
+      dependsOn: [],
+      connectedAt: new Date().toISOString(),
+    });
+    const task = await stores.tasks.create({
+      repositoryIds: [repository.id],
+      kind: "example",
+      origin: { operator: "local:recipe" },
+      replyTo: { channel: "local", address: "persistence-recovery" },
+      work: {
+        ...taskWork({
+          title: "Create a persistent greeting",
+          input: { name: "Factory" },
+          workflow: greetingWorkflow,
+        }),
+        completionCriteria: ["Return a greeting for the persisted name"],
+      },
+      dedupeKey: "persistence-recovery-root",
+    });
+    const running = await stores.tasks.transition(task.id, "running");
+    await writeFile(taskIdPath, `${running.id}\n`, { encoding: "utf8", flag: "wx" });
+    if (running.rootTaskId !== running.id || running.attempt !== 1) {
+      throw new Error("Expected a first-attempt root Task");
+    }
+    console.log(
+      JSON.stringify({
+        phase: "initialize",
+        taskId: running.id,
+        attempt: running.attempt,
+        workflow: running.work.workflow,
+        state: running.state,
+        appliedMigrations,
+      }),
+    );
+  } finally {
+    await client.close();
+  }
+}
+
+async function recoverAndComplete() {
+  const rootTaskId = await readRootTaskId();
+  const { appliedMigrations, client, stores } = await openFactory();
+  try {
+    const recovered = await stores.work.recover(rootTaskId);
+    const task = recovered.find((candidate) => candidate.id === rootTaskId);
+    if (task === undefined || recovered.length !== 1 || task.state !== "running") {
+      throw new Error("Expected to recover one running root Task");
+    }
+    const input = greetingWorkflow.input.parse(task.work.input);
+    const completed = await stores.work.completeWorkflow({
+      task: { taskId: task.id, attempt: task.attempt },
+      workflow: greetingWorkflow.binding,
+      output: { greeting: `Hello, ${input.name}!` },
+    });
+    const output = greetingWorkflow.output.parse(completed.workResult?.output);
+    if (completed.state !== "succeeded" || output.greeting !== "Hello, Factory!") {
+      throw new Error("Expected recovered work to complete with its validated output");
+    }
+    console.log(
+      JSON.stringify({
+        phase: "recover",
+        taskId: completed.id,
+        attempt: completed.attempt,
+        workflow: completed.work.workflow,
+        recoveredState: task.state,
+        state: completed.state,
+        output,
+        appliedMigrations,
+      }),
+    );
+  } finally {
+    await client.close();
+  }
+}
+
+async function verify() {
+  const rootTaskId = await readRootTaskId();
+  const { appliedMigrations, client, stores } = await openFactory();
+  try {
+    const recovered = await stores.work.recover(rootTaskId);
+    const task = recovered.find((candidate) => candidate.id === rootTaskId);
+    if (task === undefined || task.state !== "succeeded") {
+      throw new Error("Expected the completed Task to survive another restart");
+    }
+    const output = greetingWorkflow.output.parse(task.workResult?.output);
+    const succeededReceiptsBefore = (await stores.receipts.list()).filter(
+      (receipt) => receipt.taskId === task.id && receipt.state === "succeeded",
+    ).length;
+    const replayed = await stores.work.completeWorkflow({
+      task: { taskId: task.id, attempt: task.attempt },
+      workflow: greetingWorkflow.binding,
+      output,
+    });
+    const succeededReceiptsAfterReplay = (await stores.receipts.list()).filter(
+      (receipt) => receipt.taskId === task.id && receipt.state === "succeeded",
+    ).length;
+
+    let conflictingOutputRejected = false;
+    try {
+      await stores.work.completeWorkflow({
+        task: { taskId: task.id, attempt: task.attempt },
+        workflow: greetingWorkflow.binding,
+        output: { greeting: "Goodbye, Factory!" },
+      });
+    } catch {
+      conflictingOutputRejected = true;
+    }
+    const succeededReceiptCount = (await stores.receipts.list()).filter(
+      (receipt) => receipt.taskId === task.id && receipt.state === "succeeded",
+    ).length;
+    if (
+      replayed.state !== "succeeded" ||
+      replayed.workResult?.completedAt !== task.workResult?.completedAt ||
+      succeededReceiptsBefore !== 1 ||
+      succeededReceiptsAfterReplay !== 1 ||
+      succeededReceiptCount !== 1 ||
+      !conflictingOutputRejected
+    ) {
+      throw new Error("Expected idempotent replay and rejection of conflicting output");
+    }
+    console.log(
+      JSON.stringify({
+        phase: "verify",
+        taskId: replayed.id,
+        attempt: replayed.attempt,
+        workflow: replayed.work.workflow,
+        state: replayed.state,
+        replayed: true,
+        conflictingOutputRejected,
+        succeededReceiptCount,
+        appliedMigrations,
+      }),
+    );
+  } finally {
+    await client.close();
+  }
+}
+
+if (phase === "initialize") await initialize();
+if (phase === "recover") await recoverAndComplete();
+if (phase === "verify") await verify();
+```
+
+<!-- runnable-example:end -->
+
+Expected states are `running`, then `succeeded`, then `succeeded`. The verify phase also reports
+`replayed: true`, `conflictingOutputRejected: true`, and `succeededReceiptCount: 1`.
+
+## What recovery means
+
+This example demonstrates three separate responsibilities:
+
+- Reopening storage constructs a new PGlite connection, Postgres driver, store engine, and workflow
+  registry over the same database directory. No JavaScript process state is reused.
+- `stores.work.recover(rootTaskId)` reads the canonical graph by its root ID, validates every Task
+  against its exact registered workflow, and repairs a missing canonical Task snapshot from its
+  recorded graph admission when necessary. It does not resume an agent, restart provider work, or
+  replay arbitrary external effects.
+- Execution recovery belongs to workflow code. For provider-owned execution, inspect the persisted
+  execution reference, reattach only when the provider confirms it is active, and apply the
+  workflow's retry policy otherwise. This recipe uses deterministic local, workflow-owned work, so
+  it can complete directly from the persisted input and attempt. See
+  [Retry a failed Eve attempt](./retry-recovery.md) and
+  [Cancel an active Task](./cancellation.md) for those lifecycle paths.
+
+Register every historical workflow ID and version that stored Tasks can reference whenever a
+process creates its stores. A missing exact definition causes reads and recovery to fail rather
+than silently selecting a newer contract.
+
+`applyFactoryMigrations` updates the Factory SQL tables and migration ledger. It does not rewrite
+old Task JSON when a persisted application contract changes. Such a release needs an explicit data
+migration or reset decision before deployment.
+
+PGlite owns the `postgres` subdirectory, while `root-task-id` is the durable pointer used by each
+fresh process. Remove the workspace after the example when its data is no longer needed:
+
+```sh
+rm -rf ./factory-recovery-data
+```

package/docs/recipes/retry-recovery.md

@@ -0,0 +1,241 @@
+# Retry a failed Eve attempt
+
+`createFactoryHooks` turns an Eve `turn.failed` or `session.failed` event into durable recovery
+policy. It requeues the exact route-bound Task while attempts remain, ignores late events from a
+replaced attempt, and fails the Task when the configured attempt limit is exhausted. A scheduled
+`sweepQueued` pass starts each replacement.
+
+## Run the recipe
+
+Save this as `retry-recovery.mts` in an ESM project with `@vercel/factory`, `eve`, and `zod`
+installed. Injected session responses make the example credential-free.
+
+<!-- runnable-example:start -->
+
+```ts
+import type { ToolContext } from "eve/tools";
+import {
+  createFactoryHooks,
+  createTaskTools,
+  factoryTaskAttemptAttribute,
+  factoryTaskAttribute,
+  startEveSession,
+  taskHeaders,
+} from "@vercel/factory";
+import {
+  ExecutionLaunchPendingError,
+  sweepQueued,
+  type TaskLauncher,
+} from "@vercel/factory/execution";
+import { createInMemoryDriver, createStores } from "@vercel/factory/storage";
+import type { Task } from "@vercel/factory/tasks";
+import { parseAgentRouteBinding, routeKey, taskWork } from "@vercel/factory/workflows";
+
+const stores = createStores({ driver: createInMemoryDriver() });
+const repository = await stores.repositories.put({
+  schemaVersion: 1,
+  id: "repo_recipe04",
+  slug: "example/project",
+  defaultBranch: "main",
+  enabled: true,
+  dependsOn: [],
+  connectedAt: new Date().toISOString(),
+});
+const route = parseAgentRouteBinding({ id: "worker", version: 1 });
+const sessions: string[] = [];
+const launchers = {
+  [routeKey(route)]: (claimed: Task) =>
+    startEveSession({
+      agentUrl: "https://factory.example/eve/agents/worker",
+      message: `Work Task ${claimed.id}`,
+      operationId: `factory-task:${claimed.id}:${claimed.attempt}`,
+      headers: taskHeaders({ taskId: claimed.id, attempt: claimed.attempt }),
+      fetch: async () => {
+        const sessionId = `ses_${claimed.id}_${claimed.attempt}`;
+        sessions.push(sessionId);
+        return Response.json({ sessionId });
+      },
+    }),
+};
+
+const createTask = (title: string) =>
+  stores.tasks.create({
+    repositoryIds: [repository.id],
+    kind: "example",
+    origin: { operator: "local:recipe" },
+    replyTo: { channel: "local", address: "retry-recovery" },
+    work: { ...taskWork({ title }), route },
+  });
+
+function sessionContext(task: Task) {
+  if (task.execution?.provider !== "eve") throw new Error("Expected an Eve execution");
+  return {
+    session: {
+      id: task.execution.sessionId,
+      auth: {
+        initiator: {
+          attributes: {
+            [factoryTaskAttribute]: task.id,
+            [factoryTaskAttemptAttribute]: String(task.attempt),
+          },
+        },
+      },
+    },
+  };
+}
+
+const events = createFactoryHooks(stores, {
+  maxAttempts: 2,
+  presentQuestion: async () => {},
+}).factory.events;
+const turnFailed = events?.["turn.failed"];
+const sessionFailed = events?.["session.failed"];
+if (turnFailed === undefined || sessionFailed === undefined) {
+  throw new Error("Expected Eve failure hook handlers");
+}
+const turnFailure = {
+  type: "turn.failed",
+  data: {
+    code: "MODEL_CALL_FAILED",
+    message: "Provider unavailable",
+    sequence: 1,
+    turnId: "turn_1",
+  },
+} as Parameters<typeof turnFailed>[0];
+const sessionFailure = {
+  type: "session.failed",
+  data: {
+    code: "SESSION_FAILED",
+    message: "Session stopped",
+    sessionId: "session_1",
+  },
+} as Parameters<typeof sessionFailed>[0];
+
+const recoveredTask = await createTask("Recover after one failure");
+await sweepQueued(stores, { launchers });
+const firstAttempt = await stores.tasks.get(recoveredTask.id);
+if (firstAttempt?.state !== "running") throw new Error("First attempt did not launch");
+
+await turnFailed(
+  turnFailure,
+  sessionContext(firstAttempt) as unknown as Parameters<typeof turnFailed>[1],
+);
+const queuedReplacement = await stores.tasks.get(recoveredTask.id);
+if (queuedReplacement?.state !== "queued" || queuedReplacement.attempt !== 2) {
+  throw new Error("The failed first attempt was not requeued");
+}
+
+await sweepQueued(stores, { launchers });
+const replacement = await stores.tasks.get(recoveredTask.id);
+if (replacement?.state !== "running" || replacement.attempt !== 2) {
+  throw new Error("The replacement attempt did not launch");
+}
+await turnFailed(
+  turnFailure,
+  sessionContext(firstAttempt) as unknown as Parameters<typeof turnFailed>[1],
+);
+if ((await stores.tasks.get(recoveredTask.id))?.state !== "running") {
+  throw new Error("A stale failure changed the replacement attempt");
+}
+
+const finish = await createTaskTools(stores).finish_task.execute(
+  { taskId: replacement.id, summary: "Recovered on the replacement attempt" },
+  sessionContext(replacement) as unknown as ToolContext,
+);
+if (!finish.finished) throw new Error(finish.reason);
+
+const exhaustedTask = await createTask("Exhaust the retry policy");
+await sweepQueued(stores, { launchers });
+const exhaustedFirst = await stores.tasks.get(exhaustedTask.id);
+if (exhaustedFirst?.state !== "running") throw new Error("Exhaustion attempt did not launch");
+await turnFailed(
+  turnFailure,
+  sessionContext(exhaustedFirst) as unknown as Parameters<typeof turnFailed>[1],
+);
+await sweepQueued(stores, { launchers });
+const exhaustedSecond = await stores.tasks.get(exhaustedTask.id);
+if (exhaustedSecond?.state !== "running" || exhaustedSecond.attempt !== 2) {
+  throw new Error("Final allowed attempt did not launch");
+}
+await sessionFailed(
+  sessionFailure,
+  sessionContext(exhaustedSecond) as unknown as Parameters<typeof sessionFailed>[1],
+);
+const failed = await stores.tasks.get(exhaustedTask.id);
+const finalSweep = await sweepQueued(stores, { launchers });
+if (
+  failed?.state !== "failed" ||
+  failed.attempt !== 2 ||
+  finalSweep.dispatched.includes(exhaustedTask.id) ||
+  sessions.length !== 4
+) {
+  throw new Error("The attempt limit did not produce terminal failure");
+}
+
+const ambiguousTask = await createTask("Reconnect an ambiguously accepted launch");
+const operationIds: string[] = [];
+let launchCalls = 0;
+const reconnectingLauncher: TaskLauncher = async (claimed) => {
+  const operationId = `factory-task:${claimed.id}:${claimed.attempt}`;
+  operationIds.push(operationId);
+  launchCalls += 1;
+  if (launchCalls === 1) {
+    throw new ExecutionLaunchPendingError("Eve accepted the request but its response was lost");
+  }
+  return startEveSession({
+    agentUrl: "https://factory.example/eve/agents/worker",
+    message: `Work Task ${claimed.id}`,
+    operationId,
+    headers: taskHeaders({ taskId: claimed.id, attempt: claimed.attempt }),
+    fetch: async () => Response.json({ sessionId: "ses_ambiguous_reconnected" }),
+  });
+};
+const ambiguousSweep = await sweepQueued(stores, {
+  launchers: { [routeKey(route)]: reconnectingLauncher },
+  tasks: [ambiguousTask],
+});
+const pendingLaunch = await stores.tasks.get(ambiguousTask.id);
+if (
+  ambiguousSweep.failed[0]?.reasonCode !== "launch_failed" ||
+  pendingLaunch?.state !== "running" ||
+  pendingLaunch.execution !== undefined
+) {
+  throw new Error("Ambiguous acceptance did not preserve the running attempt for recovery");
+}
+const recoveredExecution = await reconnectingLauncher(pendingLaunch, repository);
+await stores.tasks.recordExecution(pendingLaunch.id, {
+  execution: recoveredExecution,
+  expectAttempt: pendingLaunch.attempt,
+});
+const reconnected = await stores.tasks.get(pendingLaunch.id);
+if (
+  reconnected?.execution?.provider !== "eve" ||
+  reconnected.execution.sessionId !== "ses_ambiguous_reconnected" ||
+  operationIds.length !== 2 ||
+  operationIds[0] !== operationIds[1]
+) {
+  throw new Error("The recovery launch did not reconnect with the stable operation ID");
+}
+
+console.log(
+  JSON.stringify({
+    recovered: (await stores.tasks.get(recoveredTask.id))?.state,
+    recoveredAttempt: replacement.attempt,
+    exhausted: failed.state,
+    exhaustedAttempt: failed.attempt,
+    reconnectedSession: reconnected.execution.sessionId,
+  }),
+);
+```
+
+<!-- runnable-example:end -->
+
+The first failure queues attempt 2 and clears the old execution. The late attempt-1 event is
+ignored, and only the replacement session can call `finish_task`. The other Task demonstrates the
+terminal path at `maxAttempts`; another sweep has nothing to relaunch.
+
+Run `sweepQueued` from a durable schedule or queue wakeup, including after failures, approvals,
+answers, and dependency completion. An ambiguous launch deliberately remains `running` without an
+execution reference, so an ordinary queued sweep will not pick it up. A recovery worker must find
+that state, invoke the exact stored route again with the same Task-and-attempt `operationId`, and
+record the reconnected execution with `expectAttempt`, as the final part of the example does.