@osolmaz/pi-workflows 0.16.1 → 0.16.3

package/src/{host/worker-store.ts → server/workflow-runner-store.ts}

@@ -4,8 +4,6 @@ import type { WorkflowSettingsScopeRecord } from "../workflows/settings.js";
 import {
   createDefinitionSnapshot,
   type InitializeWorkflowRunOptions,
-  type LoadedWorkflowRun,
-  type ReadWorkflowRunOptions,
   type WorkflowExecutionStore,
 } from "../workflows/store.js";
 import type {
@@ -22,28 +20,31 @@ import type {
   WorkflowUpdateInput,
   WorkflowUpdateRecord,
 } from "../workflows/types.js";
-import type { WorkerMessageKind, WorkerStoreOperation } from "./worker-protocol.js";
-import { workerKindForOperation } from "./worker-protocol.js";
+import type {
+  WorkflowRunnerMessageKind,
+  WorkflowRunnerStoreOperation,
+} from "./workflow-runner-protocol.js";
+import { runnerKindForOperation } from "./workflow-runner-protocol.js";
 
-export interface WorkerStoreTransport {
+export interface WorkflowRunnerStoreTransport {
   request(options: {
     messageId: string;
-    operation: WorkerStoreOperation;
-    kind: WorkerMessageKind;
+    operation: WorkflowRunnerStoreOperation;
+    kind: WorkflowRunnerMessageKind;
     expectedRevision: number;
     attemptId?: string;
     payload: JsonValue;
   }): Promise<{ result?: JsonValue; revision?: number }>;
 }
 
-/** A worker-side store that can only propose state changes to its host. */
-export class HostBackedWorkflowStore implements WorkflowExecutionStore {
-  readonly databasePath = "host://pi-workflows-state";
+/** A runner-side store that can only propose state changes to its server. */
+export class ServerBackedWorkflowStore implements WorkflowExecutionStore {
+  readonly databasePath = "server://pi-workflows-state";
   private revision: number;
 
   constructor(
     private readonly runId: string,
-    private readonly transport: WorkerStoreTransport,
+    private readonly transport: WorkflowRunnerStoreTransport,
     initialRevision = 0,
   ) {
     this.revision = initialRevision;
@@ -62,15 +63,12 @@ export class HostBackedWorkflowStore implements WorkflowExecutionStore {
     });
   }
 
-  async prepareRunResume(runId: string): Promise<LoadedWorkflowRun> {
-    return await this.call<LoadedWorkflowRun>("store.prepareRunResume", { runId });
+  async prepareRunResume(runId: string): Promise<WorkflowRunState> {
+    return await this.call<WorkflowRunState>("store.prepareRunResume", { runId });
   }
 
-  async readRun(
-    runId: string,
-    options: ReadWorkflowRunOptions = {},
-  ): Promise<LoadedWorkflowRun | null> {
-    return await this.call<LoadedWorkflowRun | null>("store.readRun", { runId, options });
+  async readRunState(runId: string): Promise<WorkflowRunState | null> {
+    return await this.call<WorkflowRunState | null>("store.readRunState", { runId });
   }
 
   async writeSnapshot(
@@ -211,14 +209,14 @@ export class HostBackedWorkflowStore implements WorkflowExecutionStore {
   }
 
   private async call<T>(
-    operation: WorkerStoreOperation,
+    operation: WorkflowRunnerStoreOperation,
     payload: Record<string, unknown>,
     attemptId?: string,
   ): Promise<T> {
     const response = await this.transport.request({
       messageId: randomUUID(),
       operation,
-      kind: workerKindForOperation(operation, payload),
+      kind: runnerKindForOperation(operation, payload),
       expectedRevision: this.revision,
       ...(attemptId === undefined ? {} : { attemptId }),
       payload: payload as JsonValue,

package/src/server/workflow-runner-supervisor.ts

@@ -0,0 +1,79 @@
+import fs from "node:fs";
+import { createRequire } from "node:module";
+import { fileURLToPath } from "node:url";
+import { canonicalJson } from "../state/json.js";
+import { ChildRunnerSupervisor, type SupervisedRunnerResult } from "./child-runner-supervisor.js";
+import { ServerProcessRegistry, type ProcessIdentity } from "./processes.js";
+import type { WorkflowRunnerLaunchEnvelope, RunnerOutcome } from "./state.js";
+import {
+  MAX_WORKFLOW_RUNNER_PROTOCOL_MESSAGE_BYTES,
+  encodeRunnerLine,
+  parseRunnerMessage,
+  type WorkflowRunnerMessage,
+  type WorkflowRunnerResponse,
+} from "./workflow-runner-protocol.js";
+
+export type WorkflowRunnerResult = SupervisedRunnerResult;
+
+export class WorkflowRunnerSupervisor {
+  private readonly supervisor: ChildRunnerSupervisor<WorkflowRunnerMessage, WorkflowRunnerResponse>;
+
+  constructor(
+    readonly envelope: WorkflowRunnerLaunchEnvelope,
+    private readonly options: {
+      registry: ServerProcessRegistry;
+      onMessage: (message: WorkflowRunnerMessage) => Promise<WorkflowRunnerResponse>;
+      env?: Record<string, string>;
+      startupTimeoutMs?: number;
+      runnerEntryPath?: string;
+      onSpawn?: (identity: ProcessIdentity) => void;
+      onDiagnostic?: (message: string) => void;
+    },
+  ) {
+    this.supervisor = new ChildRunnerSupervisor({
+      label: "Workflow runner",
+      registry: options.registry,
+      onMessage: options.onMessage,
+      parseMessage: parseRunnerMessage,
+      encodeResponse: encodeRunnerLine,
+      isReady: (message) => message.kind === "runner.ready",
+      isTerminal: (message) => message.kind === "runner.exiting",
+      maxMessageBytes: MAX_WORKFLOW_RUNNER_PROTOCOL_MESSAGE_BYTES,
+      oversizedMessage: "Runner protocol message exceeds 1 MiB",
+      ...(options.startupTimeoutMs === undefined
+        ? {}
+        : { startupTimeoutMs: options.startupTimeoutMs }),
+      ...(options.onSpawn === undefined ? {} : { onSpawn: options.onSpawn }),
+      ...(options.onDiagnostic === undefined ? {} : { onDiagnostic: options.onDiagnostic }),
+    });
+  }
+
+  async start(): Promise<void> {
+    const builtEntry = fileURLToPath(new URL("./workflow-runner-entry.js", import.meta.url));
+    const sourceEntry = fileURLToPath(new URL("./workflow-runner-entry.ts", import.meta.url));
+    const runnerEntry = this.options.runnerEntryPath ?? builtEntry;
+    const args =
+      this.options.runnerEntryPath === undefined && !fs.existsSync(builtEntry)
+        ? ["--import", createRequire(import.meta.url).resolve("tsx"), sourceEntry]
+        : [runnerEntry];
+    await this.supervisor.start({
+      args,
+      cwd: this.envelope.projectPath,
+      env: {
+        ...process.env,
+        ...this.options.env,
+        PI_WORKFLOWS_WORKFLOW_RUNNER_LAUNCH: Buffer.from(canonicalJson(this.envelope)).toString(
+          "base64url",
+        ),
+      },
+    });
+  }
+
+  async wait(): Promise<WorkflowRunnerResult> {
+    return await this.supervisor.wait();
+  }
+
+  async stop(outcome: RunnerOutcome = "cancelled"): Promise<void> {
+    await this.supervisor.stop(outcome);
+  }
+}

package/src/state/prune.ts

@@ -48,6 +48,7 @@ export async function pruneState(
   state: StateDatabase,
   databasePath: string,
   options: StatePruneOptions,
+  activeBlobHashes: () => readonly Buffer[] = () => [],
 ): Promise<StatePruneReport> {
   validateStatePruneOptions(options);
   const cutoff = parseCutoff(options.before);
@@ -114,8 +115,9 @@ export async function pruneState(
            )`,
         )
         .run();
-      const blobStats = unreferencedBlobStats(state.connection);
-      deleteUnreferencedBlobs(state.connection);
+      const retainedBlobHashes = activeBlobHashes();
+      const blobStats = unreferencedBlobStats(state.connection, retainedBlobHashes);
+      deleteUnreferencedBlobs(state.connection, retainedBlobHashes);
       deletedRows = totalChanges(state.connection) - beforeChanges;
       deletedBlobs = blobStats.count;
       deletedBlobBytes = blobStats.bytes;
@@ -356,21 +358,31 @@ function selectionSignature(database: Database.Database, runIds: string[]): stri
   return JSON.stringify([resourceRows, queueRows]);
 }
 
-function unreferencedBlobStats(database: Database.Database): { count: number; bytes: number } {
-  const predicate = blobReferencePredicate(database);
+function unreferencedBlobStats(
+  database: Database.Database,
+  retainedBlobHashes: readonly Buffer[],
+): { count: number; bytes: number } {
+  const predicate = blobReferencePredicate(database, retainedBlobHashes.length);
   const row = database
     .prepare(
       `SELECT count(*) AS count, COALESCE(sum(byte_length), 0) AS bytes FROM blobs WHERE ${predicate}`,
     )
-    .get();
+    .get(...retainedBlobHashes);
   return isBlobStatsRow(row) ? row : { count: 0, bytes: 0 };
 }
 
-function deleteUnreferencedBlobs(database: Database.Database): void {
-  database.prepare(`DELETE FROM blobs WHERE ${blobReferencePredicate(database)}`).run();
+function deleteUnreferencedBlobs(
+  database: Database.Database,
+  retainedBlobHashes: readonly Buffer[],
+): void {
+  database
+    .prepare(
+      `DELETE FROM blobs WHERE ${blobReferencePredicate(database, retainedBlobHashes.length)}`,
+    )
+    .run(...retainedBlobHashes);
 }
 
-function blobReferencePredicate(database: Database.Database): string {
+function blobReferencePredicate(database: Database.Database, retainedBlobCount: number): string {
   const tables = database
     .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")
     .all()
@@ -387,7 +399,13 @@ function blobReferencePredicate(database: Database.Database): string {
       );
     }
   }
-  return references.length === 0 ? "1 = 1" : `blob_hash NOT IN (${references.join(" UNION ")})`;
+  const unreferenced =
+    references.length === 0 ? "1 = 1" : `blob_hash NOT IN (${references.join(" UNION ")})`;
+  if (retainedBlobCount === 0) return unreferenced;
+  const retainedPlaceholders = placeholders(
+    Array.from({ length: retainedBlobCount }, () => undefined),
+  );
+  return `${unreferenced} AND blob_hash NOT IN (${retainedPlaceholders})`;
 }
 
 function descendants(root: string, children: Map<string, string[]>): string[] {

package/src/state/schema.ts

@@ -302,8 +302,8 @@ CREATE TABLE host_commands (
     'workflowMessage.reportBranch', 'workflowTurn.report', 'sessionView.clearTerminal',
     'run.restart', 'run.changeSettings', 'followUp.queue', 'followUp.remove', 'session.record',
     'channel.reload', 'channel.recover',
-    'controller.list', 'controller.get', 'controller.apply', 'controller.reconcile', 'controller.delete',
-    'host.status', 'host.stop', 'state.backup', 'state.prune'
+    'resourceManager.list', 'resourceManager.get', 'resourceManager.apply', 'resourceManager.reconcile', 'resourceManager.delete',
+    'server.status', 'server.stop', 'state.backup', 'state.prune'
   )),
   idempotency_key TEXT NOT NULL,
   request_fingerprint BLOB NOT NULL CHECK (length(request_fingerprint) = 32),

package/src/state/workflow-messages.ts

@@ -92,7 +92,7 @@ type WorkflowTurnRow = {
   endedAt: number | null;
 };
 
-/** Durable workflow content that the host requires Pi to add to one origin session. */
+/** Durable workflow content that the server requires Pi to add to one origin session. */
 export class WorkflowMessageStore {
   constructor(readonly state: StateDatabase) {}
 
@@ -279,6 +279,12 @@ export class WorkflowMessageStore {
         throw new Error("Workflow turn does not match its message");
       }
       if (message.status !== "sent") throw new Error("Workflow turn requires a sent message");
+      const open = this.openTurnForMessage(options.workflowMessageId);
+      if (open !== undefined) {
+        throw new Error(
+          `Workflow message ${options.workflowMessageId} already has open turn ${open.workflowTurnId}`,
+        );
+      }
       this.state.connection
         .prepare(
           `INSERT INTO workflow_turns(
@@ -337,6 +343,51 @@ export class WorkflowMessageStore {
     });
   }
 
+  settleOpenTurnsForRun(
+    runId: string,
+    stopReason: WorkflowTurnStopReason = "lost",
+    now: number = Date.now(),
+  ): WorkflowTurn[] {
+    return this.state.transaction(() => {
+      const open = this.state.connection
+        .prepare(
+          `SELECT workflow_turn_id AS workflowTurnId, workflow_message_id AS workflowMessageId,
+                  run_id AS runId, target_session_id AS targetSessionId, state,
+                  stop_reason AS stopReason, response_session_entry_id AS responseSessionEntryId,
+                  started_at AS startedAt, ended_at AS endedAt
+           FROM workflow_turns WHERE run_id = ? AND state = 'started' ORDER BY started_at`,
+        )
+        .all(runId)
+        .filter(isWorkflowTurnRow)
+        .map(mapTurn);
+      this.state.connection
+        .prepare(
+          `UPDATE workflow_turns SET state = 'ended', stop_reason = ?, ended_at = ?
+           WHERE run_id = ? AND state = 'started'`,
+        )
+        .run(stopReason, now, runId);
+      return open.map((turn) => this.requireTurn(turn.workflowTurnId));
+    });
+  }
+
+  cancelPendingForRun(
+    runId: string,
+    now: number = Date.now(),
+    kinds: readonly WorkflowMessageKind[] = ["step", "decision", "followUp"],
+  ): number {
+    if (kinds.length === 0) return 0;
+    return this.state.transaction(
+      () =>
+        this.state.connection
+          .prepare(
+            `UPDATE workflow_messages SET status = 'cancelled', updated_at = ?
+             WHERE run_id = ? AND status = 'pending'
+               AND kind IN (${kinds.map(() => "?").join(", ")})`,
+          )
+          .run(now, runId, ...kinds).changes,
+    );
+  }
+
   getTurn(workflowTurnId: string): WorkflowTurn | undefined {
     const row = this.state.connection
       .prepare(

package/src/viewer/backup.ts

@@ -10,7 +10,7 @@ export function verifyInactiveBackup(
   const resolved = path.resolve(databasePath);
   const active = path.resolve(activeDatabasePath);
   if (sameFile(resolved, active)) {
-    throw new Error("Active workflow state must be verified through the workflow host");
+    throw new Error("Active workflow state must be verified through the workflow server");
   }
   const state = new StateDatabase({ filePath: resolved, mode: "read-only" });
   try {

package/src/viewer/cli.ts

@@ -12,35 +12,35 @@ import { renderClientView, runViewer } from "./tui.js";
 
 const CLI_CLIENT_ID = "pi-workflows-cli";
 
-const USAGE = `pi-workflows — workflow runs and controller resources
+const USAGE = `pi-workflows — workflow runs and managed resources
 
 Usage:
   pi-workflows view [runId] [--once]
   pi-workflows runs
-  pi-workflows controllers
-  pi-workflows controller <controller> <key>
+  pi-workflows resource-managers
+  pi-workflows resource-manager <resource-manager> <key>
   pi-workflows state status
   pi-workflows state verify [inactive-backup]
   pi-workflows state backup <destination>
   pi-workflows state prune --before <timestamp> --dry-run
   pi-workflows state prune --before <timestamp> --backup <absolute-path> --apply
-  pi-workflows host start
-  pi-workflows host status
-  pi-workflows host stop
-  pi-workflows host run [-- <extra pi args>]
+  pi-workflows server start
+  pi-workflows server status
+  pi-workflows server stop
+  pi-workflows server run [-- <extra pi args>]
   pi-workflows herdr sync [--json]
   pi-workflows herdr setup [--json]
 
-Active state is available only through the package-owned workflow host.
+Active state is available only through the package-owned workflow server.
 `;
 
 export type CliArgs = {
   command: string;
   runId?: string;
-  controllerName?: string;
+  resourceManagerName?: string;
   resourceKey?: string;
   herdrAction?: string;
-  hostAction?: "start" | "status" | "stop" | "run";
+  serverAction?: "start" | "status" | "stop" | "run";
   stateAction?: "status" | "verify" | "backup" | "prune";
   backupDestination?: string;
   verifyBackup?: string;
@@ -55,9 +55,16 @@ export function parseCliArgs(argv: string[]): CliArgs {
   const args = [...argv];
   const command = args[0] && !args[0].startsWith("-") ? (args.shift() as string) : "view";
   if (
-    !["view", "runs", "controllers", "controller", "state", "host", "herdr", "help"].includes(
-      command,
-    )
+    ![
+      "view",
+      "runs",
+      "resource-managers",
+      "resource-manager",
+      "state",
+      "server",
+      "herdr",
+      "help",
+    ].includes(command)
   ) {
     throw new Error(`Unknown command: ${command}`);
   }
@@ -87,23 +94,24 @@ export function parseCliArgs(argv: string[]): CliArgs {
   }
 
   if (command !== "herdr" && json) throw new Error("--json is available only for herdr sync");
-  if (command === "host") {
-    if (project !== undefined) throw new Error("The global host does not accept --project");
+  if (command === "server") {
+    if (project !== undefined) throw new Error("The global server does not accept --project");
     const action = positionals[0];
     if (action !== "start" && action !== "status" && action !== "stop" && action !== "run") {
-      throw new Error("host requires start, status, stop, or run");
+      throw new Error("server requires start, status, stop, or run");
     }
-    if (positionals.length !== 1) throw new Error("host accepts one lifecycle action");
+    if (positionals.length !== 1) throw new Error("server accepts one lifecycle action");
     if (piArgs.length > 0 && action !== "run") {
-      throw new Error("Extra Pi arguments are available only for host run");
+      throw new Error("Extra Pi arguments are available only for server run");
     }
-    return { command, hostAction: action, once, json, piArgs };
+    return { command, serverAction: action, once, json, piArgs };
   }
-  if (command === "controller") {
-    if (positionals.length !== 2) throw new Error("controller requires <controller> and <key>");
+  if (command === "resource-manager") {
+    if (positionals.length !== 2)
+      throw new Error("resource-manager requires <resource-manager> and <key>");
     return {
       command,
-      controllerName: positionals[0] as string,
+      resourceManagerName: positionals[0] as string,
       resourceKey: positionals[1] as string,
       once,
       json,
@@ -188,8 +196,8 @@ export async function main(argv = process.argv.slice(2)): Promise<number> {
   }
 
   try {
-    if (args.command === "host") {
-      return await runHost(args.hostAction as "start" | "status" | "stop" | "run", args.piArgs);
+    if (args.command === "server") {
+      return await runServer(args.serverAction as "start" | "status" | "stop" | "run", args.piArgs);
     }
     if (args.command === "herdr") {
       const result = syncHerdrPlugin(packageRoot());
@@ -223,13 +231,17 @@ export async function main(argv = process.argv.slice(2)): Promise<number> {
         printJsonLines(runs);
         return 0;
       }
-      if (args.command === "controllers" || args.command === "controller") {
+      if (args.command === "resource-managers" || args.command === "resource-manager") {
         const response = await client.request({
-          operation: args.command === "controllers" ? "controller.list" : "controller.get",
+          operation:
+            args.command === "resource-managers" ? "resourceManager.list" : "resourceManager.get",
           payload: {
             projectPath: process.cwd(),
-            ...(args.command === "controller"
-              ? { controller: args.controllerName as string, key: args.resourceKey as string }
+            ...(args.command === "resource-manager"
+              ? {
+                  resourceManager: args.resourceManagerName as string,
+                  key: args.resourceKey as string,
+                }
               : {}),
           },
         });
@@ -298,19 +310,19 @@ async function runStateCommand(client: WorkflowClient, args: CliArgs): Promise<v
   process.stdout.write(`${JSON.stringify(response.receipt ?? {})}\n`);
 }
 
-async function runHost(
+async function runServer(
   action: "start" | "status" | "stop" | "run",
   piArgs: string[] | undefined,
 ): Promise<number> {
   if (action === "run") {
-    const { WorkflowHost } = await import("../host/runner.js");
-    const host = new WorkflowHost({
+    const { WorkflowServer } = await import("../server/server.js");
+    const server = new WorkflowServer({
       piArgs: piArgs ?? [],
-      onLog: (message) => process.stdout.write(`[host] ${message}\n`),
+      onLog: (message) => process.stdout.write(`[server] ${message}\n`),
     });
-    await host.start();
+    await server.start();
     await new Promise<void>((resolve) => {
-      const shutdown = () => void host.stop().then(resolve);
+      const shutdown = () => void server.stop().then(resolve);
       process.once("SIGTERM", shutdown);
       process.once("SIGINT", shutdown);
     });
@@ -321,7 +333,9 @@ async function runHost(
     const response =
       action === "start"
         ? await client.ensureRunning()
-        : await client.request({ operation: action === "status" ? "host.status" : "host.stop" });
+        : await client.request({
+            operation: action === "status" ? "server.status" : "server.stop",
+          });
     process.stdout.write(`${JSON.stringify(response.receipt ?? {})}\n`);
     return response.outcome === "accepted" || response.outcome === "adopted" ? 0 : 1;
   } finally {
@@ -364,7 +378,7 @@ function printJsonLines(value: JsonValue): void {
 
 function requireAccepted(response: { outcome: string; error?: string }): void {
   if (response.outcome !== "accepted" && response.outcome !== "adopted") {
-    throw new Error(response.error ?? `Workflow host returned ${response.outcome}`);
+    throw new Error(response.error ?? `Workflow server returned ${response.outcome}`);
   }
 }
 

package/src/viewer/tui.ts

@@ -13,7 +13,7 @@ export type ViewerOptions = {
   runId?: string | undefined;
 };
 
-/** Interactive client view. All durable reads stay in the workflow host. */
+/** Interactive client view. All durable reads stay in the workflow server. */
 export async function runViewer(options: ViewerOptions): Promise<void> {
   let value: JsonValue = options.runId === undefined ? [] : null;
   let selected = 0;

package/src/workflows/command-batch.ts

@@ -117,8 +117,8 @@ export async function runCommandBatch(
     }
   };
 
-  const workerCount = Math.min(request.maxConcurrency, total);
-  await Promise.all(Array.from({ length: workerCount }, worker));
+  const runnerCount = Math.min(request.maxConcurrency, total);
+  await Promise.all(Array.from({ length: runnerCount }, worker));
 
   for (let index = 0; index < total; index += 1) {
     if (results[index] !== undefined) continue;

package/src/workflows/composition.ts

@@ -589,15 +589,10 @@ function wrappedEffect(
   project: (context: WorkflowNodeContext) => WorkflowNodeContext,
 ): WorkflowManagedEffect | undefined {
   if (node.effect === undefined) return undefined;
-  const idempotencyKey = node.effect.idempotencyKey;
   const request = node.effect.request;
   return {
     type: node.effect.type,
     recovery: node.effect.recovery,
-    idempotencyKey:
-      typeof idempotencyKey === "function"
-        ? (context: WorkflowNodeContext) => idempotencyKey(project(context))
-        : idempotencyKey,
     request:
       typeof request === "function"
         ? (context: WorkflowNodeContext) => request(project(context))

package/src/workflows/definition.ts

@@ -191,14 +191,6 @@ function workflowEffect(
   return {
     type: normalized,
     recovery,
-    idempotencyKey: ({ state }: WorkflowNodeContext) => {
-      const nodeId = state.currentNode;
-      if (nodeId === undefined) {
-        throw new Error("Managed effect resolution requires an active workflow node");
-      }
-      const nodeInvocation = state.steps.filter((step) => step.nodeId === nodeId).length + 1;
-      return `${state.runId}:${normalized}:${nodeId}:${nodeInvocation}`;
-    },
     request: ({ input, outputs }: WorkflowNodeContext) => ({ input, outputs }),
   };
 }