@voyantjs/workflows-orchestrator-node 0.28.3 → 0.29.0

package/src/postgres-run-record-store.ts

@@ -0,0 +1,187 @@
+// Postgres-backed RunRecordStore — implements the orchestrator's primary
+// state-store interface (`@voyantjs/workflows-orchestrator/RunRecordStore`)
+// against the existing `voyant_snapshot_runs` table.
+//
+// The snapshot table already carries a `run_record` JSONB column; this
+// store uses that for the full RunRecord plus the indexed columns
+// (`workflow_id`, `status`, `started_at`, etc.) for queries. Mode 2's
+// `createNodeStandaloneDriver` plugs this into the orchestrator core's
+// pure trigger/resume/cancel functions.
+//
+// The new `idempotency_key` column populated from `RunRecord.idempotencyKey`
+// is enforced by the unique partial index added in migration 0003 — the
+// orchestrator's deterministic-runId derivation from `idempotencyKey`
+// dedups via the row primary key as well, so this index is a defensive
+// safety net.
+
+import type {
+  OrchestratorRunStatus,
+  RunRecord,
+  RunRecordStore,
+} from "@voyantjs/workflows-orchestrator"
+import { and, desc, eq } from "drizzle-orm"
+import type { drizzle } from "drizzle-orm/node-postgres"
+
+import { snapshotRunsTable } from "./postgres-schema.js"
+
+type SnapshotDb = ReturnType<typeof drizzle>
+
+export interface PostgresRunRecordStoreOptions {
+  db: SnapshotDb
+}
+
+export function createPostgresRunRecordStore(opts: PostgresRunRecordStoreOptions): RunRecordStore {
+  return {
+    async get(id) {
+      const rows = await opts.db
+        .select()
+        .from(snapshotRunsTable)
+        .where(eq(snapshotRunsTable.id, id))
+        .limit(1)
+      const row = rows[0]
+      if (!row) return undefined
+      // The full state lives on `run_record`. Older rows persisted by
+      // `createPostgresSnapshotRunStore` may lack it; fall back to the
+      // denormalized columns so reads stay backwards-compatible.
+      const stored = asRunRecord(row.runRecord)
+      if (stored) return stored
+      return undefined
+    },
+
+    async save(record) {
+      const values = recordToValues(record)
+      // Upsert by id — last-write-wins. Used for state mutations after
+      // the run is created (resume / cancel / drive). Idempotency on
+      // *creation* is enforced separately via `tryInsert` below; this
+      // path is the steady-state save.
+      await opts.db.insert(snapshotRunsTable).values(values).onConflictDoUpdate({
+        target: snapshotRunsTable.id,
+        set: values,
+      })
+      return record
+    },
+
+    async tryInsert(record) {
+      const values = recordToValues(record)
+      // Atomic at the DB level: INSERT … ON CONFLICT DO NOTHING returns
+      // the row only if it was created, empty otherwise. When empty, we
+      // re-SELECT to load the existing record. This closes the race
+      // window between `get(id)` and `save(record)` that the previous
+      // get-then-upsert pattern left open — concurrent triggers with
+      // the same idempotency-derived runId now see deterministic
+      // "first writer wins" semantics.
+      const inserted = await opts.db
+        .insert(snapshotRunsTable)
+        .values(values)
+        .onConflictDoNothing({ target: snapshotRunsTable.id })
+        .returning({ id: snapshotRunsTable.id })
+
+      if (inserted.length > 0) {
+        return { record, created: true }
+      }
+      // Conflict — load whoever won the race.
+      const existingRows = await opts.db
+        .select()
+        .from(snapshotRunsTable)
+        .where(eq(snapshotRunsTable.id, record.id))
+        .limit(1)
+      const existingRow = existingRows[0]
+      if (!existingRow) {
+        // Pathological case: the conflict happened but we can't read it
+        // back. Surface as a write that became a no-op so the caller
+        // doesn't proceed to drive a non-existent run.
+        return { record, created: false }
+      }
+      const existing = asRunRecord(existingRow.runRecord)
+      return {
+        record: existing ?? record,
+        created: false,
+      }
+    },
+
+    async list(filter = {}) {
+      const conditions = []
+      if (filter.workflowId) {
+        conditions.push(eq(snapshotRunsTable.workflowId, filter.workflowId))
+      }
+      if (filter.status) {
+        conditions.push(eq(snapshotRunsTable.status, filter.status))
+      }
+
+      let query = opts.db.select().from(snapshotRunsTable).$dynamic()
+      if (conditions.length === 1) {
+        query = query.where(conditions[0]!)
+      } else if (conditions.length > 1) {
+        query = query.where(and(...conditions))
+      }
+      query = query.orderBy(desc(snapshotRunsTable.startedAt))
+      if (filter.limit !== undefined) {
+        query = query.limit(filter.limit)
+      }
+      const rows = await query
+
+      const out: RunRecord[] = []
+      for (const row of rows) {
+        const stored = asRunRecord(row.runRecord)
+        if (stored) out.push(stored)
+      }
+      return out
+    },
+  }
+}
+
+// ---- Helpers (parallel to the snapshot store's; kept private here to
+//      avoid coupling between the two stores' representations) ----
+
+function recordToValues(record: RunRecord) {
+  return {
+    id: record.id,
+    workflowId: record.workflowId,
+    status: record.status,
+    startedAt: record.startedAt,
+    completedAt: record.completedAt ?? null,
+    durationMs: record.completedAt !== undefined ? record.completedAt - record.startedAt : null,
+    tags: [...record.tags],
+    // `result` mirrors the snapshot-store convention: the run's public
+    // outcome view. We snapshot output + error here so the dashboard's
+    // reads remain consistent across both stores.
+    result: normalizeRequiredJson({
+      status: record.status,
+      output: record.output,
+      error: record.error,
+      startedAt: record.startedAt,
+      completedAt: record.completedAt,
+    }),
+    input: normalizeJson(record.input),
+    runRecord: normalizeRequiredJson(record as unknown as Record<string, unknown>),
+    entryFile: null,
+    replayOf: null,
+    idempotencyKey: record.idempotencyKey ?? null,
+  }
+}
+
+function asRunRecord(value: unknown): RunRecord | undefined {
+  if (typeof value !== "object" || value === null) return undefined
+  // Sanity check: every RunRecord has at least { id, status, journal }.
+  const v = value as Record<string, unknown>
+  if (typeof v.id !== "string") return undefined
+  if (typeof v.status !== "string") return undefined
+  return value as RunRecord
+}
+
+function normalizeJson<T>(value: T): T | null {
+  if (value === undefined) return null
+  return JSON.parse(
+    JSON.stringify(value, (_key, nested) => (typeof nested === "bigint" ? Number(nested) : nested)),
+  ) as T
+}
+
+function normalizeRequiredJson<T>(value: T): T {
+  return JSON.parse(
+    JSON.stringify(value, (_key, nested) => (typeof nested === "bigint" ? Number(nested) : nested)),
+  ) as T
+}
+
+// Re-export for type discoverability — the orchestrator's status union
+// shows up frequently in consumer code that filters runs by status.
+export type { OrchestratorRunStatus }

package/src/postgres-schema.ts

@@ -1,4 +1,16 @@
-import { bigint, index, integer, jsonb, pgTable, text } from "drizzle-orm/pg-core"
+import { sql } from "drizzle-orm"
+import {
+  bigint,
+  boolean,
+  index,
+  integer,
+  jsonb,
+  pgTable,
+  primaryKey,
+  text,
+  timestamp,
+  uniqueIndex,
+} from "drizzle-orm/pg-core"
 
 export const snapshotRunsTable = pgTable(
   "voyant_snapshot_runs",
@@ -15,6 +27,13 @@ export const snapshotRunsTable = pgTable(
     runRecord: jsonb("run_record").$type<Record<string, unknown>>(),
     entryFile: text("entry_file"),
     replayOf: text("replay_of"),
+    /**
+     * Caller-supplied idempotency token, mirrored from
+     * `RunRecord.idempotencyKey` / `TriggerArgs.idempotencyKey`.
+     * The unique partial index below enforces dedup on
+     * `(workflow_id, idempotency_key)`; null values don't participate.
+     */
+    idempotencyKey: text("idempotency_key"),
   },
   (table) => ({
     workflowStartedIdx: index("voyant_snapshot_runs_workflow_started_idx").on(
@@ -25,6 +44,15 @@ export const snapshotRunsTable = pgTable(
       table.status,
       table.startedAt,
     ),
+    /**
+     * Unique partial index — enforces idempotency dedup on
+     * `(workflow_id, idempotency_key)` while letting null keys coexist.
+     * Read in `createPostgresSnapshotRunStore` via `INSERT … ON CONFLICT
+     * DO NOTHING RETURNING id`.
+     */
+    idempotencyIdx: uniqueIndex("voyant_snapshot_runs_idempotency_idx")
+      .on(table.workflowId, table.idempotencyKey)
+      .where(sql`${table.idempotencyKey} IS NOT NULL`),
   }),
 )
 
@@ -42,3 +70,30 @@ export const wakeupsTable = pgTable(
     leaseIdx: index("voyant_wakeups_lease_idx").on(table.leaseExpiresAt),
   }),
 )
+
+/**
+ * Manifest store. Holds workflow + event-filter manifests pushed at
+ * `createApp()` boot via `driver.registerManifest(...)`. Last N versions
+ * retained per environment; `is_current` points to the active version.
+ *
+ * One row is "current" per environment, enforced by the partial unique
+ * index `voyant_workflow_manifests_current_idx`.
+ *
+ * See architecture doc §14 for the manifest lifecycle.
+ */
+export const workflowManifestsTable = pgTable(
+  "voyant_workflow_manifests",
+  {
+    environment: text("environment").notNull(),
+    versionId: text("version_id").notNull(),
+    manifest: jsonb("manifest").$type<Record<string, unknown>>().notNull(),
+    registeredAt: timestamp("registered_at", { withTimezone: true }).notNull().defaultNow(),
+    isCurrent: boolean("is_current").notNull().default(false),
+  },
+  (table) => ({
+    pk: primaryKey({ columns: [table.environment, table.versionId] }),
+    currentIdx: uniqueIndex("voyant_workflow_manifests_current_idx")
+      .on(table.environment)
+      .where(sql`${table.isCurrent}`),
+  }),
+)