@osolmaz/pi-workflows 0.6.1 → 0.8.0

package/skills/pi-workflows/SKILL.md

@@ -0,0 +1,71 @@
+---
+name: pi-workflows
+description: Use when creating, reviewing, debugging, starting, inspecting, or controlling Pi Workflows; authoring .workflow.ts files; using the workflow tool; handling workflow step contracts, checkpoints, updates, or progress; or deciding how a task should compose workflow primitives.
+compatibility: Requires the Pi Workflows extension.
+---
+
+# Pi Workflows
+
+Use Pi Workflows for durable multi-step work that needs explicit routing, retries, checkpoints, scheduled waits, or progress. Keep simple one-turn work outside a workflow.
+
+The `workflow` tool schema is the authority for call shapes. A workflow step message is the authority for its current step id, attempt id, and expected output. Do not guess these values from an earlier attempt.
+
+## Operate workflows
+
+Use the smallest applicable action:
+
+- `list` discovers available workflows. Use its offset for later pages.
+- `start` starts a discovered workflow name or workflow file path with structured input.
+- `status` reads the active run, or the named run when `runId` is supplied.
+- `pause`, `resume`, and `cancel` control the current active run.
+- `answer` supplies input to a waiting checkpoint.
+- `update` publishes a non-completing durable update for the active step attempt.
+- `submit` completes the active agent step with its required output.
+
+Use `start` only once for one requested run. Do not build a manual polling loop around a workflow that already schedules its own work. Use the `monitor` skill for monitoring requests.
+
+## Complete agent steps
+
+When a workflow step message arrives:
+
+1. Do the requested work with the available tools.
+2. Produce output that matches the exact expected shape.
+3. Call `workflow` with `action: "submit"` exactly once, using the step and attempt ids from that message.
+4. If validation rejects the output, correct it and submit again with the same current ids.
+5. After acceptance, end the turn. The workflow sends the next step or presentation message when needed.
+
+A node id can run more than once in a loop. Each run has a new attempt id. Never reuse an attempt id from conversation history.
+
+## Publish updates and progress
+
+Use `update` only while the named step attempt is active. An update does not complete the step and does not control routing.
+
+For progress, publish `pi-workflows.progress.v1` data with stable track keys. Report observed counts and source-provided estimates. Do not invent totals, rates, confidence, or completion times. Use separate keys for concurrent processes and send explicit terminal states before a track disappears.
+
+The workflow definition should publish progress from function and shell actions when the runtime already has exact counts. Do not add an agent step only to format data that code can publish directly.
+
+Read [../../docs/WORKFLOW_UPDATES.md](../../docs/WORKFLOW_UPDATES.md) before adding update producers or progress estimation.
+
+## Author workflows
+
+A workflow is a `.workflow.ts`, `.workflow.js`, `.workflow.mts`, or `.workflow.mjs` module whose default export comes from `defineWorkflow(...)`.
+
+Follow these rules:
+
+- Compose the existing node and edge primitives before adding a new primitive.
+- Keep `compute` pure. Put external effects in agent, function-action, or shell-action nodes.
+- Use structured node outputs for routing.
+- Use a checkpoint when progress requires human input.
+- Set explicit step and command timeouts.
+- Bound ordinary loops with `maxSteps` or another clear finish rule.
+- Use a controller instead of a workflow for indefinite resource reconciliation.
+- Keep presentation separate from execution. Use `presentationPrompt` only when a final assistant response is needed.
+- Preserve the single active workflow rule in one Pi session.
+
+Read [../../docs/workflows.md](../../docs/workflows.md) before creating or changing a workflow. Read [../../docs/DESIGN_PHILOSOPHY.md](../../docs/DESIGN_PHILOSOPHY.md) before adding public primitives. Use the examples under [../../examples/workflows](../../examples/workflows) as starting points.
+
+## Verify changes
+
+For workflow definitions, test success, failure, routing, retries or loops, checkpoints, timeouts, cancellation, and resume behavior that applies.
+
+For extension or engine changes, run the repository checks and the real-Pi end-to-end suite. Verify discovery through the installed package path rather than only loading the source extension file.

package/src/job-progress/index.ts

@@ -0,0 +1,24 @@
+export {
+  createJobProgressReporter,
+  estimateJobProgress,
+  type JobProgressReporter,
+} from "./reporter.js";
+export {
+  isTerminalJobProgressState,
+  MAX_JOB_PROGRESS_BYTES,
+  MAX_JOB_PROGRESS_TRACKS,
+  validateJobProgressSnapshot,
+} from "./validation.js";
+export {
+  JOB_PROGRESS_SCHEMA,
+  type JobProgressCost,
+  type JobProgressEstimate,
+  type JobProgressIdentity,
+  type JobProgressPublish,
+  type JobProgressPublishResult,
+  type JobProgressReporterOptions,
+  type JobProgressSnapshot,
+  type JobProgressState,
+  type JobProgressTrack,
+  type JobProgressUpdate,
+} from "./types.js";

package/src/job-progress/reporter.ts

@@ -0,0 +1,304 @@
+import {
+  estimateProgress,
+  type ProgressSample,
+  type ProgressTrackState,
+} from "../workflows/index.js";
+import {
+  JOB_PROGRESS_SCHEMA,
+  type JobProgressEstimate,
+  type JobProgressPublishResult,
+  type JobProgressReporterOptions,
+  type JobProgressSnapshot,
+  type JobProgressTrack,
+  type JobProgressUpdate,
+} from "./types.js";
+import { isTerminalJobProgressState, validateJobProgressSnapshot } from "./validation.js";
+
+const DEFAULT_MINIMUM_INTERVAL_MS = 30_000;
+const DEFAULT_PUBLISH_TIMEOUT_MS = 15_000;
+
+export type JobProgressReporter = {
+  report(update: JobProgressUpdate): Promise<JobProgressPublishResult>;
+  flush(): Promise<JobProgressPublishResult>;
+  snapshot(): JobProgressSnapshot;
+};
+
+export function createJobProgressReporter(
+  options: JobProgressReporterOptions,
+): JobProgressReporter {
+  const now = options.now ?? (() => new Date());
+  const minimumIntervalMs = nonNegativeInteger(
+    options.minimumIntervalMs ?? DEFAULT_MINIMUM_INTERVAL_MS,
+    "minimumIntervalMs",
+  );
+  const publishTimeoutMs = positiveInteger(
+    options.publishTimeoutMs ?? DEFAULT_PUBLISH_TIMEOUT_MS,
+    "publishTimeoutMs",
+  );
+  let current =
+    options.previousSnapshot === undefined
+      ? validateJobProgressSnapshot({
+          schema: JOB_PROGRESS_SCHEMA,
+          application: options.application,
+          component: options.component,
+          jobId: options.jobId,
+          sourceRevision: options.sourceRevision,
+          contractHash: options.contractHash,
+          sequence: 0,
+          state: options.initialState ?? "running",
+          phase: options.initialPhase ?? "starting",
+          startedAt: options.startedAt,
+          updatedAt: options.startedAt,
+          ...(options.deadlineAt === undefined ? {} : { deadlineAt: options.deadlineAt }),
+          tracks: options.initialTracks,
+        })
+      : validatePreviousSnapshot(options, options.previousSnapshot);
+  const resumed = options.previousSnapshot !== undefined;
+  let lastQueuedAtMs = resumed ? Date.parse(current.updatedAt) : Number.NEGATIVE_INFINITY;
+  let lastQueuedSequence = -1;
+  let lastPublishedSequence = resumed ? current.sequence : -1;
+  let queuedOperation: Promise<void> | undefined;
+  let tail: Promise<void> = Promise.resolve();
+
+  const enqueue = (snapshot: JobProgressSnapshot): Promise<void> => {
+    if (lastQueuedSequence === snapshot.sequence && queuedOperation !== undefined) {
+      return queuedOperation;
+    }
+    lastQueuedAtMs = Date.parse(snapshot.updatedAt);
+    lastQueuedSequence = snapshot.sequence;
+    const started = tail.then(() =>
+      startPublication(options.publish, cloneSnapshot(snapshot), publishTimeoutMs),
+    );
+    const operation = started.then(async (publication) => {
+      await publication.result;
+      lastPublishedSequence = Math.max(lastPublishedSequence, snapshot.sequence);
+    });
+    queuedOperation = operation;
+    tail = started.then((publication) => publication.settled).catch(() => undefined);
+    void operation
+      .finally(() => {
+        if (lastQueuedSequence === snapshot.sequence) {
+          lastQueuedSequence = -1;
+          queuedOperation = undefined;
+        }
+      })
+      .catch(() => undefined);
+    return operation;
+  };
+
+  return {
+    async report(update) {
+      if (isTerminalJobProgressState(current.state)) {
+        throw new Error("job progress cannot change after a terminal state");
+      }
+      if (current.sequence >= Number.MAX_SAFE_INTEGER) {
+        throw new Error("job progress sequence is exhausted");
+      }
+      const timestamp = now().toISOString();
+      const state = update.state ?? current.state;
+      const phase = update.phase ?? current.phase;
+      const finishedAt = isTerminalJobProgressState(state)
+        ? (update.finishedAt ?? timestamp)
+        : update.finishedAt;
+      const tracks = isTerminalJobProgressState(state)
+        ? terminalizeTracks(update.tracks ?? current.tracks, state)
+        : (update.tracks ?? current.tracks);
+      const next = validateJobProgressSnapshot({
+        ...current,
+        sequence: current.sequence + 1,
+        state,
+        phase,
+        updatedAt: timestamp,
+        tracks,
+        ...(update.cost === undefined ? {} : { cost: update.cost }),
+        ...(finishedAt === undefined ? {} : { finishedAt }),
+      });
+      assertMonotonicTracks(current, next);
+      const publishNow =
+        update.force === true ||
+        phase !== current.phase ||
+        isTerminalJobProgressState(state) ||
+        Date.parse(timestamp) - lastQueuedAtMs >= minimumIntervalMs;
+      current = next;
+      if (!publishNow) return { snapshot: cloneSnapshot(next), published: false };
+      await enqueue(next);
+      return { snapshot: cloneSnapshot(next), published: true };
+    },
+
+    async flush() {
+      const snapshot = current;
+      if (lastPublishedSequence >= snapshot.sequence) {
+        await tail;
+        return { snapshot: cloneSnapshot(snapshot), published: false };
+      }
+      await enqueue(snapshot);
+      return { snapshot: cloneSnapshot(snapshot), published: true };
+    },
+
+    snapshot() {
+      return cloneSnapshot(current);
+    },
+  };
+}
+
+export function estimateJobProgress(
+  snapshots: JobProgressSnapshot[],
+  now = new Date(),
+): JobProgressEstimate {
+  if (snapshots.length === 0)
+    throw new Error("job progress estimation requires at least one snapshot");
+  const validated = snapshots
+    .map(validateJobProgressSnapshot)
+    .sort((left, right) => left.sequence - right.sequence);
+  const latest = validated.at(-1) as JobProgressSnapshot;
+  for (const snapshot of validated) assertSameIdentity(latest, snapshot);
+  for (let index = 1; index < validated.length; index += 1) {
+    const previous = validated[index - 1] as JobProgressSnapshot;
+    const current = validated[index] as JobProgressSnapshot;
+    if (previous.sequence === current.sequence) {
+      throw new Error(`job progress sequence ${current.sequence} is duplicated`);
+    }
+    if (Date.parse(previous.updatedAt) > Date.parse(current.updatedAt)) {
+      throw new Error("job progress updatedAt must not decrease as sequence increases");
+    }
+  }
+  const keys = latest.tracks.map((track) => track.key);
+  const tracks: ProgressTrackState[] = keys.map((key) => {
+    const samples: ProgressSample[] = validated.flatMap((snapshot) => {
+      const track = snapshot.tracks.find((candidate) => candidate.key === key);
+      return track === undefined ? [] : [{ at: snapshot.updatedAt, data: track.data }];
+    });
+    return { key, samples, estimate: estimateProgress(key, samples, now) };
+  });
+  return {
+    snapshot: cloneSnapshot(latest),
+    tracks,
+    estimates: tracks.map((track) => track.estimate),
+  };
+}
+
+function terminalizeTracks(
+  tracks: JobProgressTrack[],
+  state: "completed" | "failed" | "cancelled",
+): JobProgressTrack[] {
+  return tracks.map((track) => ({
+    key: track.key,
+    data: {
+      ...track.data,
+      status:
+        track.data.status === "completed" ||
+        track.data.status === "failed" ||
+        track.data.status === "cancelled"
+          ? track.data.status
+          : state,
+    },
+  }));
+}
+
+function assertMonotonicTracks(previous: JobProgressSnapshot, next: JobProgressSnapshot): void {
+  if (previous.phase !== next.phase) return;
+  for (const track of next.tracks) {
+    const prior = previous.tracks.find((candidate) => candidate.key === track.key);
+    if (prior === undefined || resetsTrack(prior, track)) continue;
+    if (
+      prior.data.completed !== undefined &&
+      track.data.completed !== undefined &&
+      track.data.completed < prior.data.completed
+    ) {
+      throw new Error(`job progress track ${track.key} completed value must not decrease`);
+    }
+  }
+}
+
+function resetsTrack(previous: JobProgressTrack, next: JobProgressTrack): boolean {
+  return (
+    previous.data.phase !== next.data.phase ||
+    previous.data.unit !== next.data.unit ||
+    previous.data.total !== next.data.total
+  );
+}
+
+function assertSameIdentity(expected: JobProgressSnapshot, actual: JobProgressSnapshot): void {
+  for (const field of [
+    "application",
+    "component",
+    "jobId",
+    "sourceRevision",
+    "contractHash",
+    "startedAt",
+  ] as const) {
+    if (expected[field] !== actual[field]) {
+      throw new Error(`job progress sample ${field} does not match`);
+    }
+  }
+}
+
+function startPublication(
+  publish: JobProgressReporterOptions["publish"],
+  snapshot: JobProgressSnapshot,
+  timeoutMs: number,
+): { result: Promise<void>; settled: Promise<void> } {
+  const controller = new AbortController();
+  let timeout: NodeJS.Timeout | undefined;
+  const write = publish(snapshot, controller.signal);
+  const deadline = new Promise<never>((_resolve, reject) => {
+    timeout = setTimeout(() => {
+      controller.abort();
+      reject(new Error(`job progress publication timed out after ${timeoutMs} ms`));
+    }, timeoutMs);
+  });
+  const result = Promise.race([write, deadline]).finally(() => {
+    if (timeout !== undefined) clearTimeout(timeout);
+  });
+  return {
+    result,
+    settled: write.then(
+      () => undefined,
+      () => undefined,
+    ),
+  };
+}
+
+function validatePreviousSnapshot(
+  identity: JobProgressReporterOptions,
+  previous: JobProgressSnapshot,
+): JobProgressSnapshot {
+  const validated = validateJobProgressSnapshot(previous);
+  for (const field of [
+    "application",
+    "component",
+    "jobId",
+    "sourceRevision",
+    "contractHash",
+    "startedAt",
+  ] as const) {
+    if (identity[field] !== validated[field]) {
+      throw new Error(`job progress previous snapshot ${field} does not match`);
+    }
+  }
+  if (isTerminalJobProgressState(validated.state)) {
+    throw new Error("job progress cannot resume from a terminal snapshot");
+  }
+  if (identity.deadlineAt !== validated.deadlineAt) {
+    throw new Error("job progress previous snapshot deadlineAt does not match");
+  }
+  return validated;
+}
+
+function cloneSnapshot(snapshot: JobProgressSnapshot): JobProgressSnapshot {
+  return structuredClone(snapshot);
+}
+
+function nonNegativeInteger(value: number, field: string): number {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new Error(`${field} must be a non-negative safe integer`);
+  }
+  return value;
+}
+
+function positiveInteger(value: number, field: string): number {
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(`${field} must be a positive safe integer`);
+  }
+  return value;
+}

package/src/job-progress/types.ts

@@ -0,0 +1,92 @@
+import type {
+  ProgressEstimate,
+  ProgressTrackState,
+  WorkflowProgressData,
+} from "../workflows/index.js";
+
+export const JOB_PROGRESS_SCHEMA = "pi-workflows.job-progress.v1" as const;
+
+export type JobProgressState =
+  | "queued"
+  | "running"
+  | "waiting"
+  | "blocked"
+  | "completed"
+  | "failed"
+  | "cancelled"
+  | "unknown";
+
+export type JobProgressTrack = {
+  key: string;
+  data: WorkflowProgressData;
+};
+
+export type JobProgressCost = {
+  settledUsd: number;
+  reservedUsd: number;
+};
+
+export type JobProgressSnapshot = {
+  schema: typeof JOB_PROGRESS_SCHEMA;
+  application: string;
+  component: string;
+  jobId: string;
+  sourceRevision: string;
+  contractHash: string;
+  sequence: number;
+  state: JobProgressState;
+  phase: string;
+  startedAt: string;
+  updatedAt: string;
+  deadlineAt?: string;
+  finishedAt?: string;
+  tracks: JobProgressTrack[];
+  cost?: JobProgressCost;
+};
+
+export type JobProgressIdentity = Pick<
+  JobProgressSnapshot,
+  "application" | "component" | "jobId" | "sourceRevision" | "contractHash" | "startedAt"
+> & {
+  deadlineAt?: string;
+};
+
+export type JobProgressUpdate = {
+  state?: JobProgressState;
+  phase?: string;
+  tracks?: JobProgressTrack[];
+  cost?: JobProgressCost;
+  finishedAt?: string;
+  force?: boolean;
+};
+
+export type JobProgressPublish = (
+  snapshot: JobProgressSnapshot,
+  signal: AbortSignal,
+) => Promise<void>;
+
+type JobProgressReporterBaseOptions = JobProgressIdentity & {
+  initialState?: Exclude<JobProgressState, "completed" | "failed" | "cancelled">;
+  initialPhase?: string;
+  publish: JobProgressPublish;
+  minimumIntervalMs?: number;
+  publishTimeoutMs?: number;
+  now?: () => Date;
+};
+
+export type JobProgressReporterOptions = JobProgressReporterBaseOptions &
+  (
+    | { initialTracks: JobProgressTrack[]; previousSnapshot?: never }
+    | { initialTracks?: JobProgressTrack[]; previousSnapshot: JobProgressSnapshot }
+  );
+
+export type JobProgressPublishResult = {
+  snapshot: JobProgressSnapshot;
+  published: boolean;
+};
+
+export type JobProgressEstimate = {
+  snapshot: JobProgressSnapshot;
+  tracks: ProgressTrackState[];
+  estimates: ProgressEstimate[];
+};