@osolmaz/pi-workflows 0.3.0 → 0.5.0

package/src/extension/index.ts

@@ -1,20 +1,19 @@
 import { randomUUID } from "node:crypto";
+import { isDeepStrictEqual } from "node:util";
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
-import {
-  projectControllerStorePath,
-  type RunEventRecord,
-  SqliteControllerStore,
-} from "../controllers/index.js";
+import { builtinWorkflowCatalog } from "../builtins/catalog.js";
+import { projectControllerStorePath, SqliteControllerStore } from "../controllers/index.js";
 import type { JsonObject } from "../controllers/types.js";
 import type { WorkflowSchedulerResult } from "../controllers/workflows.js";
 import { WorkflowEngine } from "../workflows/engine.js";
-import { ClaimLostError, errorMessage, isClaimLostError } from "../workflows/errors.js";
 import {
-  discoverWorkflows,
-  hashWorkflowSource,
-  loadWorkflowFile,
-  resolveWorkflowRef,
-} from "../workflows/loader.js";
+  ClaimLostError,
+  errorMessage,
+  isClaimLostError,
+  TimeoutError,
+} from "../workflows/errors.js";
+import { discoverWorkflows, resolveWorkflowRef } from "../workflows/loader.js";
+import { migrateLegacyWorkflowSources } from "../workflows/migrate-sources.js";
 import {
   createRunId,
   listRunBundles,
@@ -24,6 +23,7 @@ import {
   createDefinitionSnapshot,
 } from "../workflows/store.js";
 import type {
+  AgentStepContract,
   WorkflowDefinition,
   WorkflowDefinitionSnapshot,
   WorkflowRunResult,
@@ -42,6 +42,7 @@ import { WorkflowToolParameters, type WorkflowToolInput } from "./workflow-tool.
 const RUN_CLAIM_LEASE_MS = 30_000;
 const RUN_CLAIM_RENEW_MS = 10_000;
 const RUN_SYNC_POLL_MS = 3_000;
+const NOTIFICATION_DELIVERY_LEASE_MS = 30_000;
 const WIDGET_KEY = "pi-workflows";
 const PRESENTATION_MESSAGE_TYPE = "pi-workflows-presentation";
 const FINAL_WIDGET_TTL_MS = 60_000;
@@ -207,16 +208,14 @@ export default function piWorkflows(pi: ExtensionAPI) {
   // One runner identity per session; it names this session in run claims.
   const runnerId = randomUUID();
   let runQueueStore: SqliteControllerStore | null = null;
+  const migrationBlockedRuns = new Set<string>();
   const ensureRunQueueStore = (cwd: string): SqliteControllerStore => {
     runQueueStore ??= new SqliteControllerStore(projectControllerStorePath(cwd));
     return runQueueStore;
   };
 
-  // Session sync: a per-session watermark over the run event feed keeps this
-  // session's context current with runs other runners drove.
-  // The sync watermark is project-scoped: a reopened session catches up
-  // where the last one stopped, and two open sessions share one pointer.
-  const SYNC_WATERMARK_KEY = "project";
+  // Session-addressed delivery: each session polls only its durable outbox.
+  // Run events remain an audit feed and never enter a conversation.
   let syncArmed = false;
   let runSyncTimer: ReturnType<typeof setInterval> | null = null;
 
@@ -233,109 +232,54 @@ export default function piWorkflows(pi: ExtensionAPI) {
     }
   };
 
-  const describeRunEvent = (event: RunEventRecord): string => {
-    const label = `${event.workflowRef} run ${event.runId}`;
-    switch (event.type) {
-      case "waiting": {
-        const waitingOn =
-          typeof event.payload.waitingOn === "string" ? event.payload.waitingOn : "a checkpoint";
-        return `${label} waits at checkpoint ${waitingOn} — answer with /workflow answer`;
+  const deliveredNotificationIds = (ctx: ExtensionContext): Set<string> => {
+    const ids = new Set<string>();
+    for (const entry of ctx.sessionManager.getBranch()) {
+      if (entry.type !== "custom_message" || entry.customType !== "pi-workflows-notification") {
+        continue;
       }
-      case "parked":
-        return `${label} was parked and will resume when a runner is available`;
-      case "failed": {
-        const detail = typeof event.payload.error === "string" ? `: ${event.payload.error}` : "";
-        return `${label} failed${detail}`;
+      const details = entry.details;
+      if (details !== null && typeof details === "object" && !Array.isArray(details)) {
+        const notificationId = (details as { notificationId?: unknown }).notificationId;
+        if (typeof notificationId === "string") ids.add(notificationId);
       }
-      default:
-        return `${label} ${event.type}`;
     }
+    return ids;
   };
 
-  const runSyncPass = async (ctx: ExtensionContext): Promise<void> => {
-    if (runQueueStore === null || !syncArmed) {
-      return;
-    }
+  const runSyncPass = (ctx: ExtensionContext): void => {
+    if (runQueueStore === null || !syncArmed) return;
     try {
-      const watermark = runQueueStore.getSessionWatermark(SYNC_WATERMARK_KEY);
-      if (watermark === 0) {
-        // First sync ever for this project: never replay the feed (stale
-        // "waits at checkpoint" lines included). Fast-forward, then catch
-        // up from current state instead — what is parked, resuming, or
-        // waiting for an answer right now.
-        const latest = runQueueStore.latestRunEventSeq();
-        if (latest > 0) {
-          runQueueStore.setSessionWatermark(SYNC_WATERMARK_KEY, latest);
+      const sessionId = ctx.sessionManager.getSessionId();
+      const alreadyDelivered = deliveredNotificationIds(ctx);
+      const claimToken = randomUUID();
+      for (const notification of runQueueStore.claimPendingWorkflowNotifications({
+        targetSessionId: sessionId,
+        claimToken,
+        leaseMs: NOTIFICATION_DELIVERY_LEASE_MS,
+      })) {
+        if (!alreadyDelivered.has(notification.notificationId)) {
+          pi.sendMessage({
+            customType: "pi-workflows-notification",
+            content: notification.content,
+            display: true,
+            details: {
+              notificationId: notification.notificationId,
+              runId: notification.runId,
+              kind: notification.kind,
+            },
+          });
+          alreadyDelivered.add(notification.notificationId);
         }
-        await sendStateSnapshot(ctx);
-        return;
-      }
-      const events = runQueueStore.listRunEventsAfter(watermark, { limit: 20 });
-      if (events.length === 0) {
-        return;
-      }
-      // Persist the watermark first. A crash after this point skips the
-      // message, but snapshots recompute from the store, so no information
-      // stays lost; a duplicated state line is the worst outcome.
-      runQueueStore.setSessionWatermark(SYNC_WATERMARK_KEY, events[events.length - 1]?.seq ?? 0);
-      const noteworthy = events.filter(
-        (event) =>
-          event.runnerId !== runnerId &&
-          ["completed", "failed", "timed_out", "cancelled", "waiting", "parked"].includes(
-            event.type,
-          ),
-      );
-      if (noteworthy.length === 0) {
-        return;
+        runQueueStore.markWorkflowNotificationDelivered({
+          notificationId: notification.notificationId,
+          targetSessionId: sessionId,
+          claimToken,
+        });
       }
-      const content = `Workflow run update:\n${noteworthy.map(describeRunEvent).join("\n")}`;
-      pi.sendMessage(
-        { customType: "pi-workflows-run-sync", content, display: false },
-        { deliverAs: "steer", triggerTurn: false },
-      );
-      notify(ctx, noteworthy.map(describeRunEvent).join("; "));
     } catch {
-      // Sync is observational.
-    }
-  };
-
-  // The first-use catch-up: a snapshot of runs that need attention now.
-  const sendStateSnapshot = async (ctx: ExtensionContext) => {
-    if (runQueueStore === null) {
-      return;
+      // Delivery retries on the next poll. It never affects workflow execution.
     }
-    const lines: string[] = [];
-    const rows = runQueueStore.listWorkflowRuns();
-    for (const row of rows) {
-      if (row.status === "parked") {
-        lines.push(`${row.workflowRef} run ${row.runId} is parked and will resume`);
-      }
-    }
-    const known = new Set(rows.map((row) => row.runId));
-    const continued = new Set(
-      rows.map((row) => row.parentRunId).filter((parent): parent is string => parent !== null),
-    );
-    const bundles = await listRunBundles(new WorkflowRunStore().outputRoot);
-    for (const bundle of bundles) {
-      if (
-        bundle.state.status === "waiting" &&
-        known.has(bundle.state.runId) &&
-        !continued.has(bundle.state.runId)
-      ) {
-        lines.push(
-          `${bundle.state.workflowName} run ${bundle.state.runId} waits at checkpoint ${bundle.state.waitingOn ?? "?"} — answer with /workflow answer`,
-        );
-      }
-    }
-    if (lines.length === 0) {
-      return;
-    }
-    const content = `Workflow runs needing attention:\n${lines.join("\n")}`;
-    pi.sendMessage(
-      { customType: "pi-workflows-run-sync", content, display: false },
-      { deliverAs: "steer", triggerTurn: false },
-    );
-    notify(ctx, lines.join("; "));
   };
 
   const startRunSync = (ctx: ExtensionContext) => {
@@ -348,6 +292,8 @@ export default function piWorkflows(pi: ExtensionAPI) {
     runSyncTimer.unref?.();
   };
   let activeRun: ActiveRun | null = null;
+  let systemTurnAbort: AgentStepContract | null = null;
+  let lastExpiredAttempt: { contract: AgentStepContract; reason: string } | null = null;
   let pendingToolLaunch: {
     ctx: ExtensionContext;
     ref: string;
@@ -506,7 +452,7 @@ export default function piWorkflows(pi: ExtensionAPI) {
     if (
       sessionClosed ||
       run.generation !== runGeneration ||
-      state.status === "cancelled" ||
+      (state.status !== "completed" && state.status !== "waiting") ||
       run.presentationPrompt === undefined
     ) {
       return;
@@ -639,7 +585,9 @@ export default function piWorkflows(pi: ExtensionAPI) {
     const summary =
       state.status === "waiting" && state.waitingOn
         ? `Workflow ${state.workflowName} parked at checkpoint ${state.waitingOn} — answer with /workflow answer <json> (run ${state.runId})`
-        : `Workflow ${state.workflowName} ${state.status} (run ${state.runId})`;
+        : `Workflow ${state.workflowName} ${state.status} (run ${state.runId})${
+            state.error !== undefined ? `: ${state.error.slice(0, MAX_STATUS_ERROR_CHARS)}` : ""
+          }`;
     notify(ctx, summary, state.status === "completed" ? "info" : "warning");
     try {
       const childResult =
@@ -689,13 +637,13 @@ export default function piWorkflows(pi: ExtensionAPI) {
     }
     supersedePresentation();
     const generation = runGeneration;
-    const resolved = await resolveWorkflowRef(ref, { cwd: ctx.cwd });
-    const workflow = await loadWorkflowFile(resolved.path);
+    const resolved = await resolveWorkflowRef(ref, { cwd: ctx.cwd }, builtinWorkflowCatalog);
+    const workflow = resolved.definition;
     if (options.signal?.aborted) {
       throw options.signal.reason ?? new Error("Workflow startup aborted");
     }
     const snapshot = createDefinitionSnapshot(workflow);
-    const workflowHash = await hashWorkflowSource(resolved.path);
+    const workflowSource = resolved.source;
     const runId = options.runId ?? createRunId(workflow.name);
 
     // Continuations validate the parent before touching the queue: a
@@ -706,7 +654,10 @@ export default function piWorkflows(pi: ExtensionAPI) {
       if (parent === null || parent.state.status !== "waiting") {
         throw new Error(`Workflow run ${options.parentRunId} is not waiting at a checkpoint`);
       }
-      if (parent.state.workflowHash !== undefined && parent.state.workflowHash !== workflowHash) {
+      if (
+        parent.state.workflowSource !== undefined &&
+        !isDeepStrictEqual(parent.state.workflowSource, workflowSource)
+      ) {
         throw new Error(
           `Workflow source changed since run ${options.parentRunId} started; revert the edit to answer its checkpoint`,
         );
@@ -725,12 +676,16 @@ export default function piWorkflows(pi: ExtensionAPI) {
         const token = randomUUID();
         queueStore.enqueueWorkflowRun({
           runId,
-          workflowRef: ref,
-          workflowPath: resolved.path,
+          workflowName: workflow.name,
+          workflowSourceRef:
+            workflowSource.kind === "builtin"
+              ? `builtin:${workflowSource.id}`
+              : workflowSource.path,
           input,
           runnerId,
           claimToken: token,
           leaseMs: RUN_CLAIM_LEASE_MS,
+          originSessionId: ctx.sessionManager.getSessionId(),
           ...(options.parentRunId !== undefined ? { parentRunId: options.parentRunId } : {}),
         });
         claimToken = token;
@@ -757,6 +712,16 @@ export default function piWorkflows(pi: ExtensionAPI) {
       sendPrompt: ({ prompt, streaming }) => {
         pi.sendUserMessage(prompt, streaming ? { deliverAs: "steer" } : undefined);
       },
+      onAbort: (contract, reason) => {
+        lastExpiredAttempt = {
+          contract,
+          reason: reason instanceof TimeoutError ? "timed out" : `ended: ${errorMessage(reason)}`,
+        };
+        if (!executor.held && !ctx.isIdle()) {
+          systemTurnAbort = contract;
+          ctx.abort();
+        }
+      },
       conversation: {
         beginAttempt: (contract) => run.recorder?.beginAttempt(contract),
         mark: () => run.recorder?.mark() ?? 0,
@@ -773,6 +738,24 @@ export default function piWorkflows(pi: ExtensionAPI) {
     const engine = new WorkflowEngine({
       executor,
       store,
+      notificationSink: {
+        notify: (request) => {
+          fence?.();
+          if (queueStore === null) throw new Error("Workflow notifications require a queued run");
+          const record = queueStore.getWorkflowRun(request.runId);
+          if (record?.originSessionId === null || record?.originSessionId === undefined) {
+            throw new Error(`Workflow run ${request.runId} has no origin session`);
+          }
+          const notification = queueStore.enqueueWorkflowNotification({
+            ...request,
+            targetSessionId: record.originSessionId,
+          });
+          return {
+            notificationId: notification.notificationId,
+            targetSessionId: notification.targetSessionId,
+          };
+        },
+      },
       // Awaited by the engine after run_started is persisted, so the session
       // binding and its trace event always precede node and terminal events.
       onRunStarted: async (runDir, state) => {
@@ -840,12 +823,11 @@ export default function piWorkflows(pi: ExtensionAPI) {
 
     run.completion = (
       options.resume === true
-        ? engine.resumeRun(workflow, runId, { workflowHash })
+        ? engine.resumeRun(workflow, runId, { workflowSource })
         : options.parentRunId === undefined
-          ? engine.run(workflow, input, { workflowPath: resolved.path, workflowHash, runId })
+          ? engine.run(workflow, input, { workflowSource, runId })
           : engine.continueRun(workflow, options.parentRunId, input, {
-              workflowPath: resolved.path,
-              workflowHash,
+              workflowSource,
               runId,
             })
     )
@@ -955,13 +937,22 @@ export default function piWorkflows(pi: ExtensionAPI) {
       runnerId,
       claimToken,
       leaseMs: RUN_CLAIM_LEASE_MS,
+      excludeRunIds: [...migrationBlockedRuns],
+      sessionId: ctx.sessionManager.getSessionId(),
     });
     if (claimed === undefined) {
       return;
     }
     let started: string | undefined;
     try {
-      started = await startRun(ctx, claimed.workflowPath, claimed.input, {
+      const bundle = await readRunBundle(new WorkflowRunStore().runDirFor(claimed.runId));
+      const sourceRef =
+        bundle?.state.workflowSource === undefined
+          ? claimed.workflowSourceRef
+          : bundle.state.workflowSource.kind === "builtin"
+            ? `builtin:${bundle.state.workflowSource.id}`
+            : bundle.state.workflowSource.path;
+      started = await startRun(ctx, sourceRef, claimed.input, {
         resume: true,
         runId: claimed.runId,
         claimToken,
@@ -971,7 +962,7 @@ export default function piWorkflows(pi: ExtensionAPI) {
       throw error;
     }
     if (started !== undefined) {
-      notify(ctx, `Resumed workflow run ${claimed.runId} (${claimed.workflowRef}).`);
+      notify(ctx, `Resumed workflow run ${claimed.runId} (${claimed.workflowName}).`);
     } else {
       queueStore.parkWorkflowRun({ runId: claimed.runId, claimToken });
     }
@@ -1051,7 +1042,7 @@ export default function piWorkflows(pi: ExtensionAPI) {
     ctx: ExtensionContext,
     offset = 0,
   ): Promise<WorkflowControlResult> => {
-    const discovered = await discoverWorkflows({ cwd: ctx.cwd });
+    const discovered = await discoverWorkflows({ cwd: ctx.cwd }, builtinWorkflowCatalog);
     if (discovered.length === 0) {
       return {
         message:
@@ -1255,11 +1246,16 @@ export default function piWorkflows(pi: ExtensionAPI) {
   const resolveWaitingWorkflow = async (
     ctx: ExtensionContext,
     requestedRunId?: string,
-  ): Promise<{ parentRunId: string; workflowPath: string }> => {
+  ): Promise<{ parentRunId: string; workflowRef: string }> => {
     let parentRunId = requestedRunId ?? lastWaitingRunId;
     if (parentRunId === null) {
       const rows = ensureRunQueueStore(ctx.cwd).listWorkflowRuns();
-      const known = new Set(rows.map((row) => row.runId));
+      const sessionId = ctx.sessionManager.getSessionId();
+      const known = new Set(
+        rows
+          .filter((row) => row.originSessionId === null || row.originSessionId === sessionId)
+          .map((row) => row.runId),
+      );
       const continued = new Set(
         rows.map((row) => row.parentRunId).filter((parent): parent is string => parent !== null),
       );
@@ -1275,18 +1271,32 @@ export default function piWorkflows(pi: ExtensionAPI) {
     if (parentRunId === null) {
       throw new Error("No workflow is waiting for an answer.");
     }
+    const queueRecord = ensureRunQueueStore(ctx.cwd).getWorkflowRun(parentRunId);
+    if (
+      queueRecord?.originSessionId !== null &&
+      queueRecord?.originSessionId !== undefined &&
+      queueRecord.originSessionId !== ctx.sessionManager.getSessionId()
+    ) {
+      throw new Error(`Workflow run ${parentRunId} belongs to another Pi session.`);
+    }
     const parent = await readRunBundle(new WorkflowRunStore().runDirFor(parentRunId));
     if (
       parent === null ||
       parent.state.status !== "waiting" ||
-      parent.state.workflowPath === undefined
+      parent.state.workflowSource === undefined
     ) {
       if (parentRunId === lastWaitingRunId) {
         lastWaitingRunId = null;
       }
       throw new Error(`Workflow run ${parentRunId} is no longer waiting.`);
     }
-    return { parentRunId, workflowPath: parent.state.workflowPath };
+    return {
+      parentRunId,
+      workflowRef:
+        parent.state.workflowSource.kind === "builtin"
+          ? `builtin:${parent.state.workflowSource.id}`
+          : parent.state.workflowSource.path,
+    };
   };
 
   const answerWorkflowControl = async (
@@ -1295,7 +1305,7 @@ export default function piWorkflows(pi: ExtensionAPI) {
     requestedRunId?: string,
   ): Promise<WorkflowControlResult> => {
     const waiting = await resolveWaitingWorkflow(ctx, requestedRunId);
-    const continued = await startRun(ctx, waiting.workflowPath, input, {
+    const continued = await startRun(ctx, waiting.workflowRef, input, {
       parentRunId: waiting.parentRunId,
     });
     if (continued === undefined) {
@@ -1356,8 +1366,8 @@ export default function piWorkflows(pi: ExtensionAPI) {
     const reservation = { ctx, ref, input, options };
     pendingToolLaunch = reservation;
     try {
-      const resolved = await resolveWorkflowRef(ref, { cwd: ctx.cwd });
-      const workflow = await loadWorkflowFile(resolved.path);
+      const resolved = await resolveWorkflowRef(ref, { cwd: ctx.cwd }, builtinWorkflowCatalog);
+      const workflow = resolved.definition;
       if (pendingToolLaunch !== reservation) {
         throw new Error("The queued workflow launch was cancelled before validation finished.");
       }
@@ -1382,7 +1392,7 @@ export default function piWorkflows(pi: ExtensionAPI) {
     description:
       "Run or manage a workflow: /workflow <name-or-path> [task | --input-json {…}]; also: status, pause, resume, cancel, answer",
     getArgumentCompletions: async (prefix: string) => {
-      const discovered = await discoverWorkflows({ cwd: process.cwd() });
+      const discovered = await discoverWorkflows({ cwd: process.cwd() }, builtinWorkflowCatalog);
       const items = [
         ...discovered.map((workflow) => ({ value: workflow.name, label: workflow.name })),
         { value: "status", label: "status" },
@@ -1556,13 +1566,21 @@ export default function piWorkflows(pi: ExtensionAPI) {
           break;
         case "answer": {
           const waiting = await resolveWaitingWorkflow(ctx, params.runId);
-          control = await queueToolLaunch(ctx, waiting.workflowPath, params.input, {
+          control = await queueToolLaunch(ctx, waiting.workflowRef, params.input, {
             parentRunId: waiting.parentRunId,
           });
           break;
         }
         case "submit": {
           if (!activeRun) {
+            if (
+              lastExpiredAttempt?.contract.attemptId === params.attempt &&
+              lastExpiredAttempt.contract.nodeId === params.step
+            ) {
+              throw new Error(
+                `Workflow step ${JSON.stringify(params.step)} attempt ${JSON.stringify(params.attempt)} ${lastExpiredAttempt.reason}; its output is no longer accepted.`,
+              );
+            }
             throw new Error("No workflow step is waiting for output.");
           }
           // Flush the conversation into the bundle before accepting, so the
@@ -1603,6 +1621,24 @@ export default function piWorkflows(pi: ExtensionAPI) {
   pi.on("session_start", async (_event, ctx) => {
     sessionClosed = false;
     controllerContext = ctx;
+    try {
+      const queue = ensureRunQueueStore(ctx.cwd);
+      const migration = await migrateLegacyWorkflowSources({
+        catalog: builtinWorkflowCatalog,
+        queue,
+      });
+      migrationBlockedRuns.clear();
+      for (const blocked of migration.blocked) migrationBlockedRuns.add(blocked.runId);
+      if (migration.blocked.length > 0) {
+        notify(
+          ctx,
+          `Could not migrate ${migration.blocked.length} legacy workflow source(s).`,
+          "warning",
+        );
+      }
+    } catch (error) {
+      notify(ctx, `Could not migrate legacy workflow sources: ${errorMessage(error)}`, "warning");
+    }
     try {
       syncArmed = true;
       startRunSync(ctx);
@@ -1632,13 +1668,6 @@ export default function piWorkflows(pi: ExtensionAPI) {
   });
 
   pi.on("agent_end", (event, ctx) => {
-    const run = activeRun;
-    if (!run) {
-      return;
-    }
-    // An aborted turn means the user hit escape to take the conversation
-    // back. Nudging or dispatching the next step would immediately steal it
-    // again, so hold the run until an explicit /workflow resume.
     const aborted = event.messages.some(
       (message) =>
         typeof message === "object" &&
@@ -1646,6 +1675,17 @@ export default function piWorkflows(pi: ExtensionAPI) {
         "stopReason" in message &&
         (message as { stopReason?: string }).stopReason === "aborted",
     );
+    if (aborted && systemTurnAbort !== null) {
+      systemTurnAbort = null;
+      return;
+    }
+    const run = activeRun;
+    if (!run) {
+      return;
+    }
+    // An aborted turn means the user hit escape to take the conversation
+    // back. Nudging or dispatching the next step would immediately steal it
+    // again, so hold the run until an explicit /workflow resume.
     if (!aborted || runHeld()) {
       return;
     }
@@ -1719,6 +1759,7 @@ export default function piWorkflows(pi: ExtensionAPI) {
 
   pi.on("session_shutdown", async () => {
     sessionClosed = true;
+    systemTurnAbort = null;
     supersedePresentation();
     const run = activeRun;
     if (run !== null && run.claimToken !== undefined) {