@osolmaz/pi-workflows 0.2.0 → 0.3.0

package/src/controllers/store.ts

@@ -0,0 +1,160 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type {
+  ChildWorkflowRecord,
+  ControllerEvent,
+  ControllerQueueClaim,
+  ControllerResource,
+  ControllerResourceRef,
+  ControllerResourceStatus,
+  EffectRecord,
+  JsonObject,
+} from "./types.js";
+
+export const CONTROLLER_STORE_SCHEMA = "pi-workflows.controller-store.v1" as const;
+
+export type QueueRequeueOptions = {
+  availableAt: string;
+  error?: string;
+};
+
+export type QueueItem = {
+  controller: string;
+  key: string;
+  availableAt: string;
+  consecutiveErrors: number;
+  claimExpiresAt?: string;
+};
+
+export type EffectReservation = {
+  record: EffectRecord;
+  created: boolean;
+};
+
+export type WorkflowReservation = {
+  record: ChildWorkflowRecord;
+  created: boolean;
+};
+
+export type WorkflowRecordUpdate = {
+  runId?: string | null;
+  state: ChildWorkflowRecord["state"];
+  attempt?: number;
+  error?: string | null;
+};
+
+export interface ControllerStore {
+  close(): void;
+
+  putResource<TSpec, TStatus>(options: {
+    controller: string;
+    key: string;
+    spec: TSpec;
+    initialStatus: TStatus;
+    now?: string;
+  }): ControllerResource<TSpec, TStatus>;
+  getResource<TSpec = unknown, TStatus = unknown>(
+    ref: ControllerResourceRef,
+  ): ControllerResource<TSpec, TStatus> | undefined;
+  getResourceByUid(uid: string): ControllerResource | undefined;
+  listResources<TSpec = unknown, TStatus = unknown>(options?: {
+    controller?: string;
+  }): ControllerResource<TSpec, TStatus>[];
+  updateStatus<TStatus>(options: {
+    ref: ControllerResourceRef;
+    expectedResourceVersion: number;
+    status: ControllerResourceStatus<TStatus>;
+    finalizers?: string[];
+    now?: string;
+  }): ControllerResource<unknown, TStatus>;
+  requestDeletion(ref: ControllerResourceRef, now?: string): ControllerResource;
+  updateFinalizers(options: {
+    ref: ControllerResourceRef;
+    expectedResourceVersion: number;
+    finalizers: string[];
+    now?: string;
+  }): ControllerResource;
+  deleteResource(ref: ControllerResourceRef, expectedResourceVersion: number): boolean;
+
+  enqueue(ref: ControllerResourceRef, availableAt?: string): void;
+  claimNext(options: {
+    controllers: string[];
+    leaseMs: number;
+    now?: string;
+  }): ControllerQueueClaim | undefined;
+  renewClaim(claim: ControllerQueueClaim, leaseMs: number, now?: string): boolean;
+  settleClaim(claim: ControllerQueueClaim, now?: string): boolean;
+  requeueClaim(claim: ControllerQueueClaim, options: QueueRequeueOptions, now?: string): boolean;
+  listQueue(): QueueItem[];
+
+  reserveEffect(options: {
+    key: string;
+    resourceUid: string;
+    generation: number;
+    kind: string;
+    requestFingerprint: string;
+    now?: string;
+  }): EffectReservation;
+  getEffect(resourceUid: string, key: string): EffectRecord | undefined;
+  updateEffect(options: {
+    resourceUid: string;
+    key: string;
+    state: EffectRecord["state"];
+    externalRef?: string;
+    error?: string;
+    now?: string;
+  }): EffectRecord;
+  listEffects(resourceUid: string): EffectRecord[];
+
+  reserveWorkflow(options: {
+    resourceUid: string;
+    requestKey: string;
+    workflow: string;
+    inputFingerprint: string;
+  }): WorkflowReservation;
+  getWorkflow(resourceUid: string, requestKey: string): ChildWorkflowRecord | undefined;
+  getWorkflowByRequestId(requestId: string): ChildWorkflowRecord | undefined;
+  updateWorkflow(requestId: string, update: WorkflowRecordUpdate): ChildWorkflowRecord;
+  listWorkflows(resourceUid: string): ChildWorkflowRecord[];
+
+  recordEvent(options: {
+    controller: string;
+    key: string;
+    type: string;
+    payload?: JsonObject;
+    now?: string;
+  }): ControllerEvent;
+  listEvents(options?: { controller?: string; key?: string; limit?: number }): ControllerEvent[];
+}
+
+export function controllerStoreBaseDir(homeDir: string = os.homedir()): string {
+  const override = process.env.PI_WORKFLOWS_CONTROLLER_DIR;
+  if (override !== undefined && override.length > 0) {
+    return override;
+  }
+  return path.join(homeDir, ".pi", "agent", "workflows", "controllers");
+}
+
+export function controllerStorePath(homeDir: string = os.homedir()): string {
+  return path.join(controllerStoreBaseDir(homeDir), "controller.sqlite");
+}
+
+export function controllerProjectScope(cwd: string): string {
+  let canonicalCwd: string;
+  try {
+    canonicalCwd = fs.realpathSync.native(cwd);
+  } catch {
+    canonicalCwd = path.resolve(cwd);
+  }
+  return createHash("sha256").update(canonicalCwd).digest("hex").slice(0, 24);
+}
+
+export function projectControllerStoreBaseDir(cwd: string, homeDir: string = os.homedir()): string {
+  return path.join(controllerStoreBaseDir(homeDir), "projects", controllerProjectScope(cwd));
+}
+
+export function projectControllerStorePath(cwd: string, homeDir: string = os.homedir()): string {
+  return path.join(projectControllerStoreBaseDir(cwd, homeDir), "controller.sqlite");
+}

package/src/controllers/types.ts

@@ -0,0 +1,183 @@
+export type MaybePromise<T> = T | Promise<T>;
+export type JsonPrimitive = string | number | boolean | null;
+export type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue };
+export type JsonObject = { [key: string]: JsonValue };
+
+export type ControllerConditionStatus = true | false | "unknown";
+
+export type ControllerCondition = {
+  type: string;
+  status: ControllerConditionStatus;
+  reason: string;
+  message?: string;
+  observedGeneration: number;
+  lastTransitionTime: string;
+};
+
+export type ChildWorkflowState =
+  | "pending"
+  | "running"
+  | "waiting"
+  | "succeeded"
+  | "failed"
+  | "interrupted";
+
+export type ChildWorkflowReference = {
+  requestId: string;
+  runId?: string;
+  state: ChildWorkflowState;
+  attempt: number;
+};
+
+export type ControllerResourceStatus<TStatus> = {
+  observedGeneration: number;
+  conditions: ControllerCondition[];
+  controllerStatus: TStatus;
+  workflowRun?: ChildWorkflowReference;
+};
+
+export type ControllerResource<TSpec = unknown, TStatus = unknown> = {
+  metadata: {
+    uid: string;
+    controller: string;
+    key: string;
+    resourceVersion: number;
+    generation: number;
+    deletionTimestamp?: string;
+    finalizers: string[];
+  };
+  spec: TSpec;
+  status: ControllerResourceStatus<TStatus>;
+};
+
+export type ControllerResourceRef = {
+  controller: string;
+  key: string;
+};
+
+export type ControllerConditionInput = {
+  type: string;
+  status: ControllerConditionStatus;
+  reason: string;
+  message?: string;
+};
+
+export type ControllerStatusPatch<TStatus> = {
+  controllerStatus?: TStatus;
+  conditions?: ControllerConditionInput[];
+  workflowRun?: ChildWorkflowReference | null;
+  finalizers?: string[];
+};
+
+export type ReconcileResult<TStatus> =
+  | {
+      kind: "settled";
+      status?: ControllerStatusPatch<TStatus>;
+    }
+  | {
+      kind: "requeue";
+      afterMs?: number;
+      status?: ControllerStatusPatch<TStatus>;
+    };
+
+export type EffectState = "pending" | "applied" | "rejected" | "indeterminate";
+
+export type EffectRecord = {
+  key: string;
+  resourceUid: string;
+  generation: number;
+  kind: string;
+  state: EffectState;
+  requestFingerprint: string;
+  startedAt: string;
+  completedAt?: string;
+  externalRef?: string;
+  error?: string;
+};
+
+export type EffectObservation =
+  | { state: "applied"; externalRef?: string }
+  | { state: "not_applied" }
+  | { state: "indeterminate" };
+
+export type EffectApplication =
+  | { state: "applied"; externalRef?: string }
+  | { state: "rejected"; error: string }
+  | { state: "indeterminate"; error?: string };
+
+export type EffectDefinition<TRequest> = {
+  key: string;
+  kind: string;
+  request: TRequest;
+  observe: (signal: AbortSignal) => MaybePromise<EffectObservation>;
+  apply: (signal: AbortSignal) => MaybePromise<EffectApplication>;
+};
+
+export interface ControllerEffects {
+  ensure<TRequest>(definition: EffectDefinition<TRequest>): Promise<EffectRecord>;
+}
+
+export type ChildWorkflowRequest = {
+  requestKey: string;
+  workflow: string;
+  input: unknown;
+};
+
+export type ChildWorkflowRecord = ChildWorkflowReference & {
+  resourceUid: string;
+  requestKey: string;
+  inputFingerprint: string;
+  workflow: string;
+  error?: string;
+};
+
+export interface ControllerWorkflows {
+  ensure(request: ChildWorkflowRequest): Promise<ChildWorkflowRecord>;
+}
+
+export type ReconcileContext<TStatus> = {
+  signal: AbortSignal;
+  effects: ControllerEffects;
+  workflows: ControllerWorkflows;
+  settled(status?: ControllerStatusPatch<TStatus>): ReconcileResult<TStatus>;
+  requeue(status?: ControllerStatusPatch<TStatus>): ReconcileResult<TStatus>;
+  requeueAfter(afterMs: number, status?: ControllerStatusPatch<TStatus>): ReconcileResult<TStatus>;
+};
+
+export type ControllerDefinition<TSpec = unknown, TStatus = unknown> = {
+  name: string;
+  initialStatus: (spec: TSpec) => TStatus;
+  reconcile: (
+    context: ReconcileContext<TStatus>,
+    resource: ControllerResource<TSpec, TStatus>,
+  ) => MaybePromise<ReconcileResult<TStatus>>;
+  timeoutMs?: number;
+};
+
+export type AnyControllerDefinition = {
+  name: string;
+  initialStatus(spec: unknown): unknown;
+  reconcile(
+    context: ReconcileContext<unknown>,
+    resource: ControllerResource,
+  ): MaybePromise<ReconcileResult<unknown>>;
+  timeoutMs?: number;
+};
+
+export type ControllerQueueClaim = {
+  controller: string;
+  key: string;
+  token: string;
+  queueVersion: number;
+  consecutiveErrors: number;
+  expiresAt: string;
+};
+
+export type ControllerEvent = {
+  seq: number;
+  recordedAt: string;
+  controller: string;
+  key: string;
+  type: string;
+  payload: JsonObject;
+};

package/src/controllers/workflow-engine-scheduler.ts

@@ -0,0 +1,145 @@
+import { WorkflowEngine } from "../workflows/engine.js";
+import { readLastTraceEvent, readRunBundle, type WorkflowRunStore } from "../workflows/store.js";
+import type {
+  WorkflowDefinition,
+  WorkflowRunResult,
+  WorkflowRunStatus,
+} from "../workflows/types.js";
+import type {
+  ControllerWorkflowScheduler,
+  WorkflowSchedulerRequest,
+  WorkflowSchedulerResult,
+} from "./workflows.js";
+
+export type ResolvedChildWorkflow = {
+  workflow: WorkflowDefinition;
+  workflowPath?: string;
+};
+
+export type WorkflowEngineSchedulerOptions = {
+  store: WorkflowRunStore;
+  resolveWorkflow: (name: string) => Promise<ResolvedChildWorkflow>;
+  createEngine: (request: WorkflowSchedulerRequest) => WorkflowEngine;
+  /**
+   * Release resources the engine held after its run settles, for example a
+   * headless child process. Called once per finished run.
+   */
+  disposeEngine?: (engine: WorkflowEngine) => Promise<void>;
+};
+
+export class WorkflowEngineScheduler implements ControllerWorkflowScheduler {
+  private readonly active = new Map<string, { runId: string; promise: Promise<void> }>();
+
+  constructor(private readonly options: WorkflowEngineSchedulerOptions) {}
+
+  async ensure(
+    request: WorkflowSchedulerRequest,
+    signal: AbortSignal,
+    onComplete: (result: WorkflowSchedulerResult) => void,
+  ): Promise<WorkflowSchedulerResult> {
+    if (signal.aborted) {
+      throw signal.reason ?? new Error("Workflow scheduling aborted");
+    }
+    const activeKey = requestKey(request);
+    const active = this.active.get(activeKey);
+    if (active !== undefined) {
+      return { state: "running", runId: active.runId };
+    }
+
+    const bundle = await readRunBundle(this.options.store.runDirFor(request.runId));
+    if (bundle !== null) {
+      const recovered =
+        bundle.state.status === "running"
+          ? await this.options.store.markRunInterrupted(request.runId)
+          : bundle;
+      if (recovered !== null) {
+        const lastTraceEvent = await readLastTraceEvent(
+          recovered.runDir,
+          recovered.manifest.paths.trace,
+        );
+        return resultFromStatus(
+          recovered.state.status,
+          request.runId,
+          recovered.state.error,
+          lastTraceEvent?.type === "run_interrupted",
+        );
+      }
+      return { state: "pending" };
+    }
+
+    await this.options.store.quarantineIncompleteRun(request.runId);
+    const resolved = await this.options.resolveWorkflow(request.workflow);
+    if (signal.aborted) {
+      throw signal.reason ?? new Error("Workflow scheduling aborted");
+    }
+    const runId = request.runId;
+    const engine = this.options.createEngine(request);
+    const promise = engine
+      .run(resolved.workflow, request.input, {
+        runId,
+        ...(resolved.workflowPath !== undefined ? { workflowPath: resolved.workflowPath } : {}),
+      })
+      .then((result) => {
+        callCompletion(onComplete, resultFromRun(result));
+      })
+      .catch((error: unknown) => {
+        callCompletion(onComplete, {
+          state: "failed",
+          runId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      })
+      .finally(() => {
+        this.active.delete(activeKey);
+        void this.options.disposeEngine?.(engine).catch(() => undefined);
+      });
+    this.active.set(activeKey, { runId, promise });
+    return { state: "running", runId };
+  }
+
+  async waitForIdle(): Promise<void> {
+    await Promise.all([...this.active.values()].map((entry) => entry.promise));
+  }
+}
+
+function callCompletion(
+  callback: (result: WorkflowSchedulerResult) => void,
+  result: WorkflowSchedulerResult,
+): void {
+  try {
+    callback(result);
+  } catch {
+    // A later reconciliation recovers completion from the immutable run bundle.
+  }
+}
+
+function requestKey(request: WorkflowSchedulerRequest): string {
+  return `${request.requestId}:${request.attempt}`;
+}
+
+function resultFromRun(result: WorkflowRunResult): WorkflowSchedulerResult {
+  return resultFromStatus(result.state.status, result.state.runId, result.state.error);
+}
+
+function resultFromStatus(
+  status: WorkflowRunStatus,
+  runId: string,
+  error?: string,
+  interrupted = false,
+): WorkflowSchedulerResult {
+  if (interrupted) {
+    return { state: "interrupted", runId, ...(error !== undefined ? { error } : {}) };
+  }
+  switch (status) {
+    case "running":
+      return { state: "running", runId };
+    case "waiting":
+      return { state: "waiting", runId };
+    case "completed":
+      return { state: "succeeded", runId };
+    case "failed":
+    case "timed_out":
+    case "cancelled":
+      return { state: "failed", runId, ...(error !== undefined ? { error } : {}) };
+  }
+}

package/src/controllers/workflows.ts

@@ -0,0 +1,152 @@
+import { createRunId } from "../workflows/store.js";
+import { jsonFingerprint } from "./json.js";
+import type { ControllerStore } from "./store.js";
+import type {
+  ChildWorkflowRecord,
+  ChildWorkflowRequest,
+  ChildWorkflowState,
+  ControllerResource,
+  ControllerWorkflows,
+} from "./types.js";
+
+export type WorkflowSchedulerRequest = {
+  requestId: string;
+  attempt: number;
+  workflow: string;
+  input: unknown;
+  runId: string;
+};
+
+export type WorkflowSchedulerResult = {
+  state: ChildWorkflowState;
+  runId?: string;
+  error?: string;
+};
+
+export interface ControllerWorkflowScheduler {
+  ensure(
+    request: WorkflowSchedulerRequest,
+    signal: AbortSignal,
+    onComplete: (result: WorkflowSchedulerResult) => void,
+  ): Promise<WorkflowSchedulerResult>;
+}
+
+export class ControllerWorkflowCoordinator {
+  constructor(
+    private readonly store: ControllerStore,
+    private readonly scheduler?: ControllerWorkflowScheduler,
+  ) {}
+
+  forResource(resource: ControllerResource, signal: AbortSignal): ControllerWorkflows {
+    return {
+      ensure: async (request) => await this.ensure(resource, request, signal),
+    };
+  }
+
+  complete(requestId: string, result: WorkflowSchedulerResult): ChildWorkflowRecord {
+    const previous = this.store.getWorkflowByRequestId(requestId);
+    if (previous === undefined) {
+      throw new Error(`Workflow request not found: ${requestId}`);
+    }
+    const record = this.store.updateWorkflow(requestId, {
+      state: result.state,
+      ...(result.runId !== undefined ? { runId: result.runId } : {}),
+      ...(result.error !== undefined ? { error: result.error } : { error: null }),
+    });
+    this.enqueueParent(record.resourceUid);
+    this.recordEvent(record, "workflow_state_changed");
+    return record;
+  }
+
+  private async ensure(
+    resource: ControllerResource,
+    request: ChildWorkflowRequest,
+    signal: AbortSignal,
+  ): Promise<ChildWorkflowRecord> {
+    const inputFingerprint = jsonFingerprint(request.input);
+    const reservation = this.store.reserveWorkflow({
+      resourceUid: resource.metadata.uid,
+      requestKey: request.requestKey,
+      workflow: request.workflow,
+      inputFingerprint,
+    });
+    let record = reservation.record;
+    if (record.state === "succeeded" || record.state === "failed" || record.state === "waiting") {
+      return record;
+    }
+    if (this.scheduler === undefined) {
+      return record;
+    }
+    if (record.state === "interrupted" || record.attempt === 0) {
+      record = this.store.updateWorkflow(record.requestId, {
+        state: "pending",
+        attempt: record.attempt + 1,
+        runId: createRunId(record.workflow),
+        error: null,
+      });
+    } else if (record.runId === undefined) {
+      record = this.store.updateWorkflow(record.requestId, {
+        state: "pending",
+        runId: createRunId(record.workflow),
+        error: null,
+      });
+    }
+    if (record.runId === undefined) {
+      throw new Error(`Workflow request has no reserved run ID: ${record.requestId}`);
+    }
+    const result = await this.scheduler.ensure(
+      {
+        requestId: record.requestId,
+        attempt: record.attempt,
+        workflow: record.workflow,
+        input: request.input,
+        runId: record.runId,
+      },
+      signal,
+      (completed) => {
+        this.complete(record.requestId, completed);
+      },
+    );
+    record = this.store.updateWorkflow(record.requestId, {
+      state: result.state,
+      ...(result.runId !== undefined ? { runId: result.runId } : {}),
+      ...(result.error !== undefined ? { error: result.error } : { error: null }),
+    });
+    if (reservation.created || result.state !== "running") {
+      this.recordEvent(
+        record,
+        reservation.created ? "workflow_requested" : "workflow_state_changed",
+      );
+    }
+    return record;
+  }
+
+  private enqueueParent(resourceUid: string): void {
+    const resource = this.store.getResourceByUid(resourceUid);
+    if (resource !== undefined) {
+      this.store.enqueue({
+        controller: resource.metadata.controller,
+        key: resource.metadata.key,
+      });
+    }
+  }
+
+  private recordEvent(record: ChildWorkflowRecord, type: string): void {
+    const resource = this.store.getResourceByUid(record.resourceUid);
+    if (resource === undefined) {
+      return;
+    }
+    this.store.recordEvent({
+      controller: resource.metadata.controller,
+      key: resource.metadata.key,
+      type,
+      payload: {
+        requestId: record.requestId,
+        requestKey: record.requestKey,
+        state: record.state,
+        attempt: record.attempt,
+        ...(record.runId !== undefined ? { runId: record.runId } : {}),
+      },
+    });
+  }
+}