@voyantjs/workflows-orchestrator-node 0.28.3 → 0.30.0

package/src/node-standalone-driver.ts

@@ -0,0 +1,491 @@
+// Mode 2 driver — pure Node, Postgres-backed.
+//
+// Returns a `DriverFactory` (per architecture doc §6.3) that the framework
+// invokes after `createApp()` has assembled its `ModuleContainer`. Composes:
+//
+//   * `createPostgresRunRecordStore` — primary state, against
+//      `voyant_snapshot_runs.run_record` JSONB.
+//   * `createPostgresManifestStore`   — manifest history, against
+//      `voyant_workflow_manifests`.
+//   * In-process step handler glued to `handleStepRequest` from
+//      `@voyantjs/workflows/handler` — the workflow body executes in the
+//      same Node process as the driver.
+//
+// The Postgres time wheel (`createPersistentWakeupManager`) is started via
+// the returned driver's `start()` lifecycle helper or — when used from
+// `createApp()` — by the framework's bootstrap.
+//
+// See architecture doc §7 for the full Mode 2 design.
+
+import type {
+  ListRunsOptions,
+  Run,
+  RunDetail,
+  RunSummary,
+  TriggerOptions,
+} from "@voyantjs/workflows"
+import type {
+  DriverFactory,
+  DriverFactoryDeps,
+  IngestEventArgs,
+  IngestEventResponse,
+  IngestMatch,
+  WorkflowAdmin,
+  WorkflowDriver,
+} from "@voyantjs/workflows/driver"
+import { deriveStableEventId } from "@voyantjs/workflows/events"
+import { handleStepRequest, type WorkflowStepRequest } from "@voyantjs/workflows/handler"
+import type { WorkflowManifest } from "@voyantjs/workflows/protocol"
+import {
+  cancel as orchestratorCancel,
+  trigger as orchestratorTrigger,
+  type RunRecord,
+  routeEvent,
+  type StepHandler,
+} from "@voyantjs/workflows-orchestrator"
+import type { drizzle } from "drizzle-orm/node-postgres"
+
+import {
+  createPersistentWakeupManager,
+  type PersistentWakeupManager,
+} from "./persistent-wakeup-manager.js"
+import { createPostgresManifestStore } from "./postgres-manifest-store.js"
+import { createPostgresRunRecordStore } from "./postgres-run-record-store.js"
+import { createPostgresWakeupStore } from "./postgres-wakeup-store.js"
+import { syncWakeupFromRecord, type WakeupStore } from "./wakeup-store.js"
+
+type Db = ReturnType<typeof drizzle>
+
+// ---- Public factory options ----
+
+export interface NodeStandaloneDriverOptions {
+  /** Long-lived Postgres connection (drizzle-orm `node-postgres` adapter). */
+  db: Db
+  /** Default environment for `trigger()` calls that don't specify one. */
+  defaultEnvironment?: "production" | "preview" | "development"
+  /** Tenant metadata stamped onto every triggered run. */
+  tenantMeta?: RunRecord["tenantMeta"]
+  /** Injectable clock; defaults to `Date.now`. */
+  now?: () => number
+  /**
+   * Step handler override. Defaults to in-process `handleStepRequest`
+   * with the framework-supplied `services` container plumbed through
+   * (so step bodies can resolve via `ctx.services.resolve(...)`).
+   */
+  handler?: StepHandler
+  /**
+   * Latest N manifest versions to retain per environment after each
+   * registerManifest. Defaults to 3 (per architecture doc §14.2). Set to
+   * a high number to disable pruning effectively.
+   */
+  manifestVersionsToKeep?: number
+  /**
+   * Time-wheel poll interval, ms. The wakeup manager (architecture doc
+   * §7.2) polls `voyant_wakeups` for due alarms and resumes parked runs
+   * via the orchestrator. Defaults to 1_000 ms. Lower values reduce
+   * sleep-resume latency at the cost of DB load.
+   */
+  wakeupPollIntervalMs?: number
+  /**
+   * Wakeup lease TTL, ms. A poll instance leases a due wakeup for this
+   * long; if the process dies mid-process, another instance picks the
+   * wakeup back up after the lease expires. Defaults to 4× the poll
+   * interval (or 5_000 ms, whichever is greater).
+   */
+  wakeupLeaseMs?: number
+  /**
+   * Lease owner identifier. Used to disambiguate poller instances
+   * across processes. Defaults to a random per-driver token.
+   */
+  wakeupLeaseOwner?: string
+  /**
+   * When `true`, the wakeup poller does NOT auto-start on construction.
+   * Callers must invoke the returned driver's lifecycle hooks
+   * themselves — useful for tests that want to control the poll
+   * cadence. Defaults to `false` (poller starts immediately).
+   */
+  disableTimeWheel?: boolean
+}
+
+const DEFAULT_TENANT_META: RunRecord["tenantMeta"] = {
+  tenantId: "default",
+  projectId: "default",
+  organizationId: "default",
+}
+
+const DEFAULT_MANIFEST_KEEP = 3
+
+/**
+ * Build the Mode 2 driver factory. The factory closes over its options
+ * and returns a fresh `WorkflowDriver` when `createApp()` (or a test)
+ * calls it with `DriverFactoryDeps`.
+ *
+ * Usage:
+ *
+ *     createApp({
+ *       workflows: {
+ *         driver: createNodeStandaloneDriver({ db, defaultEnvironment: "production" }),
+ *       },
+ *     })
+ *
+ * Or in compliance tests:
+ *
+ *     const driver = createNodeStandaloneDriver({ db: testDb })(testFactoryDeps())
+ */
+export function createNodeStandaloneDriver(opts: NodeStandaloneDriverOptions): DriverFactory {
+  return (deps: DriverFactoryDeps): WorkflowDriver => {
+    const runStore = createPostgresRunRecordStore({ db: opts.db })
+    const manifestStore = createPostgresManifestStore({ db: opts.db })
+    const wakeupStore: WakeupStore = createPostgresWakeupStore({ db: opts.db })
+    const now = opts.now ?? deps.now ?? (() => Date.now())
+    const tenantMeta = opts.tenantMeta ?? DEFAULT_TENANT_META
+    const defaultEnv = opts.defaultEnvironment ?? "development"
+    const keep = opts.manifestVersionsToKeep ?? DEFAULT_MANIFEST_KEEP
+    const leaseOwner = opts.wakeupLeaseOwner ?? `node-standalone-${randomToken()}`
+
+    // Wire the framework-supplied service container through to step bodies.
+    // The handler closes over `deps.services` so every step invocation
+    // surfaces it as `ctx.services` inside the workflow body.
+    const handler: StepHandler =
+      opts.handler ??
+      (async (req: WorkflowStepRequest, stepOpts) =>
+        handleStepRequest(req, { services: deps.services }, stepOpts))
+
+    // Persistent wakeup manager — polls `voyant_wakeups` for runs
+    // parked on DATETIME waitpoints and resumes them via the orchestrator's
+    // `resumeDueAlarms`. This is what makes `ctx.sleep(...)` actually
+    // wake up in Mode 2 (architecture doc §7.2).
+    const wakeupManager: PersistentWakeupManager<RunRecord> = createPersistentWakeupManager({
+      wakeupStore,
+      handler,
+      leaseOwner,
+      leaseMs: opts.wakeupLeaseMs,
+      intervalMs: opts.wakeupPollIntervalMs,
+      now,
+      logger: (level, message, data) => deps.logger(level, message, data),
+      // For Mode 2 the "stored" representation IS the RunRecord — the
+      // postgres-run-record-store carries the full state on `run_record`
+      // JSONB. So toRecord/fromRecord are identity.
+      async getRun(runId) {
+        return runStore.get(runId)
+      },
+      async saveRun(record) {
+        await runStore.save(record)
+        return record
+      },
+      toRecord: (record) => record,
+      fromRecord: (record) => record,
+      async listRuns() {
+        // Bootstrap-time list of currently-parked runs to seed the wakeup
+        // store. Mode 2 uses status="waiting" filter on the run-record store.
+        return runStore.list({ status: "waiting" })
+      },
+    })
+
+    if (!opts.disableTimeWheel) {
+      // Auto-start the poller. Callers can opt out via `disableTimeWheel`
+      // for tests that want to control the cadence manually (poll explicitly
+      // via `manager.poll()`).
+      wakeupManager.start()
+    }
+
+    let shuttingDown = false
+
+    // ---- WorkflowDriver implementation ----
+
+    async function registerManifest(args: {
+      environment: WorkflowDriver["registerManifest"] extends (a: infer A) => unknown
+        ? A extends { environment: infer E }
+          ? E
+          : never
+        : never
+      manifest: WorkflowManifest
+    }): Promise<{ versionId: string }> {
+      assertNotShutdown(shuttingDown)
+      const result = await manifestStore.registerManifest({
+        environment: args.environment as string,
+        versionId: args.manifest.versionId,
+        manifest: args.manifest as unknown as Record<string, unknown>,
+      })
+      // Best-effort prune; failures here shouldn't fail boot.
+      try {
+        await manifestStore.pruneToVersions(args.environment as string, keep)
+      } catch (err) {
+        deps.logger("warn", "manifest prune failed (non-fatal)", {
+          environment: args.environment,
+          error: err instanceof Error ? err.message : String(err),
+        })
+      }
+      return result
+    }
+
+    async function getManifest(args: { environment: string }): Promise<WorkflowManifest | null> {
+      const envelope = await manifestStore.getCurrent(args.environment)
+      if (!envelope) return null
+      return envelope.manifest as unknown as WorkflowManifest
+    }
+
+    async function trigger<TIn, TOut>(
+      workflow: { id: string } | string,
+      input: TIn,
+      triggerOpts?: TriggerOptions,
+    ): Promise<Run<TOut>> {
+      assertNotShutdown(shuttingDown)
+      const workflowId = typeof workflow === "string" ? workflow : workflow.id
+      const env = triggerOpts?.environment ?? defaultEnv
+
+      const record = await orchestratorTrigger(
+        {
+          workflowId,
+          workflowVersion: triggerOpts?.lockToVersion ?? "v1",
+          input: input as unknown,
+          tenantMeta,
+          environment: env,
+          tags: triggerOpts?.tags,
+          idempotencyKey: triggerOpts?.idempotencyKey,
+        },
+        { store: runStore, handler, now },
+      )
+      // Sync wakeup row so the time-wheel can resume DATETIME-parked runs.
+      // No-op if the run completed inline (status !== "waiting").
+      await syncWakeupFromRecord(wakeupStore, record)
+      return runRecordToRun<TOut>(record)
+    }
+
+    async function ingestEvent(args: IngestEventArgs): Promise<IngestEventResponse> {
+      assertNotShutdown(shuttingDown)
+      const stored = await manifestStore.getCurrent(args.environment)
+      if (!stored) {
+        return {
+          ok: false,
+          reason: "manifest_not_registered",
+          message: `No manifest is registered for environment "${args.environment}".`,
+        }
+      }
+      const eventId = await ensureEventId(args.envelope)
+      const manifest = stored.manifest as unknown as WorkflowManifest
+      const routed = routeEvent({
+        manifest,
+        envelope: args.envelope,
+        eventId,
+        idempotencyOverride: args.idempotencyKey,
+      })
+
+      const matches: IngestMatch[] = []
+      let anyTriggered = false
+      let anyFailed = false
+      for (const entry of routed) {
+        if (entry.status === "skipped") {
+          matches.push({
+            filterId: entry.filterId,
+            status: "skipped",
+            reason: entry.reason,
+            details: entry.details,
+          })
+          continue
+        }
+        try {
+          const record = await orchestratorTrigger(
+            {
+              workflowId: entry.targetWorkflowId,
+              workflowVersion: "v1",
+              input: entry.input,
+              tenantMeta,
+              environment: args.environment,
+              idempotencyKey: entry.idempotencyKey,
+              triggeredBy: {
+                kind: "event",
+                eventId,
+                eventType: args.envelope.name,
+                filterId: entry.filterId,
+              },
+            },
+            { store: runStore, handler, now },
+          )
+          await syncWakeupFromRecord(wakeupStore, record)
+          matches.push({
+            filterId: entry.filterId,
+            targetWorkflowId: entry.targetWorkflowId,
+            runId: record.id,
+            idempotencyKey: entry.idempotencyKey,
+            status: "queued",
+          })
+          anyTriggered = true
+        } catch (err) {
+          matches.push({
+            filterId: entry.filterId,
+            targetWorkflowId: entry.targetWorkflowId,
+            status: "error",
+            reason: err instanceof Error ? err.message : String(err),
+          })
+          anyFailed = true
+        }
+      }
+
+      if (matches.length > 0 && !anyTriggered && anyFailed) {
+        return {
+          ok: false,
+          reason: "trigger_failed_for_all_matches",
+          message: "every matched filter failed to trigger",
+        }
+      }
+      return { ok: true, eventId, matches }
+    }
+
+    async function shutdown(): Promise<void> {
+      shuttingDown = true
+      // Stop the time-wheel poller so the process can exit cleanly.
+      // Idempotent — calling stop() on an already-stopped manager is a
+      // no-op.
+      wakeupManager.stop()
+    }
+
+    // ---- WorkflowAdmin (full; Mode 2 has Postgres-native query support) ----
+
+    const admin: WorkflowAdmin = {
+      async listRuns(listOpts?: ListRunsOptions) {
+        const filterStatus = normalizeStatusFilter(listOpts?.status)
+        const filterEnv = listOpts?.environment
+        const filterWorkflow = listOpts?.workflowId
+        const filterTag = listOpts?.tag
+        const filterSince = toEpoch(listOpts?.since)
+        const filterUntil = toEpoch(listOpts?.until)
+        const limit = listOpts?.limit ?? 100
+
+        // Take a generous fetch window; in-memory filter for fields the
+        // store doesn't index (env, tag, since/until). For real load we'd
+        // push these down into the query — out of scope for PR1 step 6.
+        const records = await runStore.list({
+          workflowId: filterWorkflow,
+          status: filterStatus?.[0] as never,
+          limit: limit * 2,
+        })
+
+        const results: RunSummary[] = []
+        for (const rec of records) {
+          if (filterStatus && !filterStatus.includes(rec.status as never)) continue
+          if (filterEnv && rec.environment !== filterEnv) continue
+          if (filterTag && !rec.tags.includes(filterTag)) continue
+          if (filterSince !== undefined && rec.startedAt < filterSince) continue
+          if (filterUntil !== undefined && rec.startedAt > filterUntil) continue
+          results.push(runRecordToSummary(rec))
+        }
+        const page = results.slice(0, limit)
+        const nextCursor = results.length > limit ? String(limit) : undefined
+        return { runs: page, nextCursor }
+      },
+
+      async getRun(runId: string): Promise<RunDetail | null> {
+        const rec = await runStore.get(runId)
+        return rec ? runRecordToDetail(rec) : null
+      },
+
+      async cancelRun(runId: string, cancelOpts?: { reason?: string; compensate?: boolean }) {
+        // The orchestrator core's cancel() does NOT run compensations by
+        // default (architecture doc §21.21). The `compensate` flag is
+        // accepted but no-ops in v1.
+        void cancelOpts?.compensate
+        await orchestratorCancel(
+          { runId, reason: cancelOpts?.reason },
+          { store: runStore, handler, now },
+        )
+      },
+
+      streamRun(runId: string): AsyncIterable<never> {
+        // Live journal-event streaming is a follow-up — needs LISTEN/NOTIFY
+        // wired against the run store or a polling source. PR1 ships
+        // listRuns + getRun; streamRun returns an immediately-exhausted
+        // iterable so dashboards probing it get a clean empty stream
+        // instead of an undefined.
+        void runId
+        return {
+          [Symbol.asyncIterator]() {
+            return {
+              next: async () => ({ value: undefined as never, done: true as const }),
+            }
+          },
+        }
+      },
+    }
+
+    return {
+      registerManifest,
+      trigger,
+      ingestEvent,
+      getManifest,
+      shutdown,
+      admin,
+    }
+  }
+}
+
+// ---- Helpers ----
+
+function assertNotShutdown(shuttingDown: boolean): void {
+  if (shuttingDown) {
+    throw new Error("NodeStandaloneDriver: shutdown() has been called; new operations are refused.")
+  }
+}
+
+function randomToken(): string {
+  return Math.floor(Math.random() * 1_000_000_000)
+    .toString(36)
+    .padStart(6, "0")
+}
+
+async function ensureEventId(envelope: {
+  name: string
+  data: unknown
+  metadata?: { eventId?: string }
+  emittedAt: string
+}): Promise<string> {
+  if (envelope.metadata?.eventId) return envelope.metadata.eventId
+  // Content-derived fallback per architecture doc §15.2 — closes the
+  // dedup hole reviewer P2.2 flagged.
+  return deriveStableEventId(envelope)
+}
+
+function runRecordToRun<TOut>(rec: RunRecord): Run<TOut> {
+  return {
+    id: rec.id,
+    workflowId: rec.workflowId,
+    status: rec.status as Run["status"],
+    startedAt: rec.startedAt,
+  }
+}
+
+function runRecordToSummary(rec: RunRecord): RunSummary {
+  return {
+    id: rec.id,
+    workflowId: rec.workflowId,
+    status: rec.status as RunSummary["status"],
+    startedAt: rec.startedAt,
+    completedAt: rec.completedAt,
+    tags: [...rec.tags],
+    environment: rec.environment,
+  }
+}
+
+function runRecordToDetail(rec: RunRecord): RunDetail {
+  return {
+    ...runRecordToSummary(rec),
+    version: rec.workflowVersion,
+    input: rec.input,
+    output: rec.output,
+    error: rec.error,
+    durationMs:
+      rec.completedAt !== undefined ? Math.max(0, rec.completedAt - rec.startedAt) : undefined,
+  }
+}
+
+function normalizeStatusFilter(
+  s: ListRunsOptions["status"] | undefined,
+): readonly string[] | undefined {
+  if (s === undefined) return undefined
+  return Array.isArray(s) ? s : [s]
+}
+
+function toEpoch(v: number | Date | undefined): number | undefined {
+  if (v === undefined) return undefined
+  return typeof v === "number" ? v : v.getTime()
+}

package/src/postgres-manifest-store.ts

@@ -0,0 +1,144 @@
+// Postgres-backed manifest store. Holds the serialized WorkflowManifest
+// pushed at `createApp()` boot via `driver.registerManifest(...)` and read
+// by `driver.getManifest(...)` for boot-time mismatch detection and the
+// dashboard's filter inspector.
+//
+// One row is "current" per environment, enforced by the partial unique
+// index `voyant_workflow_manifests_current_idx` (migration 0004). History
+// is retained — `pruneToVersions(n)` keeps the latest N per environment.
+//
+// See architecture doc §14 for the manifest lifecycle.
+
+import { and, desc, eq, ne, sql } from "drizzle-orm"
+import type { drizzle } from "drizzle-orm/node-postgres"
+
+import { workflowManifestsTable } from "./postgres-schema.js"
+
+type ManifestDb = ReturnType<typeof drizzle>
+
+/**
+ * Structural view of `WorkflowManifest` (from `@voyantjs/workflows/protocol`).
+ * Declared locally to avoid pulling the workflows package's protocol export
+ * into this store — every consumer satisfies the shape via TypeScript
+ * structural compat, same pattern Voyant uses elsewhere.
+ */
+export interface ManifestEnvelope {
+  environment: string
+  versionId: string
+  manifest: Record<string, unknown>
+}
+
+export interface ManifestStore {
+  /**
+   * Idempotent. Same `(environment, versionId)` returns without re-write.
+   * New `versionId` for an existing environment marks the new row
+   * `is_current = true` and the previous current `is_current = false`.
+   */
+  registerManifest(envelope: ManifestEnvelope): Promise<{ versionId: string }>
+
+  /** Returns the current manifest for the environment, or null. */
+  getCurrent(environment: string): Promise<ManifestEnvelope | null>
+
+  /** Retain the latest `keep` versions per environment; delete older. */
+  pruneToVersions(environment: string, keep: number): Promise<{ deleted: number }>
+}
+
+export interface PostgresManifestStoreOptions {
+  db: ManifestDb
+}
+
+export function createPostgresManifestStore(opts: PostgresManifestStoreOptions): ManifestStore {
+  const db = opts.db
+
+  return {
+    async registerManifest(envelope) {
+      // Atomically: insert new row, flip is_current to false on every other
+      // row for this environment, mark this row is_current = true. Single
+      // transaction so concurrent registrations don't leave two rows current.
+      await db.transaction(async (tx) => {
+        await tx
+          .insert(workflowManifestsTable)
+          .values({
+            environment: envelope.environment,
+            versionId: envelope.versionId,
+            manifest: envelope.manifest,
+            isCurrent: true,
+          })
+          .onConflictDoNothing({
+            target: [workflowManifestsTable.environment, workflowManifestsTable.versionId],
+          })
+
+        // Demote any existing current row that isn't this versionId.
+        await tx
+          .update(workflowManifestsTable)
+          .set({ isCurrent: false })
+          .where(
+            and(
+              eq(workflowManifestsTable.environment, envelope.environment),
+              eq(workflowManifestsTable.isCurrent, true),
+              ne(workflowManifestsTable.versionId, envelope.versionId),
+            ),
+          )
+
+        // Promote the just-registered row in case it already existed
+        // (re-register of the same versionId).
+        await tx
+          .update(workflowManifestsTable)
+          .set({ isCurrent: true })
+          .where(
+            and(
+              eq(workflowManifestsTable.environment, envelope.environment),
+              eq(workflowManifestsTable.versionId, envelope.versionId),
+            ),
+          )
+      })
+      return { versionId: envelope.versionId }
+    },
+
+    async getCurrent(environment) {
+      const rows = await db
+        .select()
+        .from(workflowManifestsTable)
+        .where(
+          and(
+            eq(workflowManifestsTable.environment, environment),
+            eq(workflowManifestsTable.isCurrent, true),
+          ),
+        )
+        .limit(1)
+      const row = rows[0]
+      if (!row) return null
+      return {
+        environment: row.environment,
+        versionId: row.versionId,
+        manifest: row.manifest,
+      }
+    },
+
+    async pruneToVersions(environment, keep) {
+      if (keep < 1) {
+        throw new Error(`pruneToVersions: keep must be >= 1, got ${keep}`)
+      }
+      // Get the IDs of the latest `keep` rows for this environment, then
+      // delete everything else.
+      const newest = await db
+        .select({ versionId: workflowManifestsTable.versionId })
+        .from(workflowManifestsTable)
+        .where(eq(workflowManifestsTable.environment, environment))
+        .orderBy(desc(workflowManifestsTable.registeredAt))
+        .limit(keep)
+      const keepIds = newest.map((r) => r.versionId)
+      if (keepIds.length === 0) return { deleted: 0 }
+
+      const result = await db.execute(
+        sql`DELETE FROM ${workflowManifestsTable}
+            WHERE environment = ${environment}
+            AND version_id NOT IN (${sql.join(
+              keepIds.map((id) => sql`${id}`),
+              sql`, `,
+            )})`,
+      )
+      return { deleted: (result as { rowCount?: number }).rowCount ?? 0 }
+    },
+  }
+}