@voyantjs/workflows-orchestrator-node 0.28.3 → 0.29.0

package/dist/postgres-run-record-store.js

@@ -0,0 +1,161 @@
+// Postgres-backed RunRecordStore — implements the orchestrator's primary
+// state-store interface (`@voyantjs/workflows-orchestrator/RunRecordStore`)
+// against the existing `voyant_snapshot_runs` table.
+//
+// The snapshot table already carries a `run_record` JSONB column; this
+// store uses that for the full RunRecord plus the indexed columns
+// (`workflow_id`, `status`, `started_at`, etc.) for queries. Mode 2's
+// `createNodeStandaloneDriver` plugs this into the orchestrator core's
+// pure trigger/resume/cancel functions.
+//
+// The new `idempotency_key` column populated from `RunRecord.idempotencyKey`
+// is enforced by the unique partial index added in migration 0003 — the
+// orchestrator's deterministic-runId derivation from `idempotencyKey`
+// dedups via the row primary key as well, so this index is a defensive
+// safety net.
+import { and, desc, eq } from "drizzle-orm";
+import { snapshotRunsTable } from "./postgres-schema.js";
+export function createPostgresRunRecordStore(opts) {
+    return {
+        async get(id) {
+            const rows = await opts.db
+                .select()
+                .from(snapshotRunsTable)
+                .where(eq(snapshotRunsTable.id, id))
+                .limit(1);
+            const row = rows[0];
+            if (!row)
+                return undefined;
+            // The full state lives on `run_record`. Older rows persisted by
+            // `createPostgresSnapshotRunStore` may lack it; fall back to the
+            // denormalized columns so reads stay backwards-compatible.
+            const stored = asRunRecord(row.runRecord);
+            if (stored)
+                return stored;
+            return undefined;
+        },
+        async save(record) {
+            const values = recordToValues(record);
+            // Upsert by id — last-write-wins. Used for state mutations after
+            // the run is created (resume / cancel / drive). Idempotency on
+            // *creation* is enforced separately via `tryInsert` below; this
+            // path is the steady-state save.
+            await opts.db.insert(snapshotRunsTable).values(values).onConflictDoUpdate({
+                target: snapshotRunsTable.id,
+                set: values,
+            });
+            return record;
+        },
+        async tryInsert(record) {
+            const values = recordToValues(record);
+            // Atomic at the DB level: INSERT … ON CONFLICT DO NOTHING returns
+            // the row only if it was created, empty otherwise. When empty, we
+            // re-SELECT to load the existing record. This closes the race
+            // window between `get(id)` and `save(record)` that the previous
+            // get-then-upsert pattern left open — concurrent triggers with
+            // the same idempotency-derived runId now see deterministic
+            // "first writer wins" semantics.
+            const inserted = await opts.db
+                .insert(snapshotRunsTable)
+                .values(values)
+                .onConflictDoNothing({ target: snapshotRunsTable.id })
+                .returning({ id: snapshotRunsTable.id });
+            if (inserted.length > 0) {
+                return { record, created: true };
+            }
+            // Conflict — load whoever won the race.
+            const existingRows = await opts.db
+                .select()
+                .from(snapshotRunsTable)
+                .where(eq(snapshotRunsTable.id, record.id))
+                .limit(1);
+            const existingRow = existingRows[0];
+            if (!existingRow) {
+                // Pathological case: the conflict happened but we can't read it
+                // back. Surface as a write that became a no-op so the caller
+                // doesn't proceed to drive a non-existent run.
+                return { record, created: false };
+            }
+            const existing = asRunRecord(existingRow.runRecord);
+            return {
+                record: existing ?? record,
+                created: false,
+            };
+        },
+        async list(filter = {}) {
+            const conditions = [];
+            if (filter.workflowId) {
+                conditions.push(eq(snapshotRunsTable.workflowId, filter.workflowId));
+            }
+            if (filter.status) {
+                conditions.push(eq(snapshotRunsTable.status, filter.status));
+            }
+            let query = opts.db.select().from(snapshotRunsTable).$dynamic();
+            if (conditions.length === 1) {
+                query = query.where(conditions[0]);
+            }
+            else if (conditions.length > 1) {
+                query = query.where(and(...conditions));
+            }
+            query = query.orderBy(desc(snapshotRunsTable.startedAt));
+            if (filter.limit !== undefined) {
+                query = query.limit(filter.limit);
+            }
+            const rows = await query;
+            const out = [];
+            for (const row of rows) {
+                const stored = asRunRecord(row.runRecord);
+                if (stored)
+                    out.push(stored);
+            }
+            return out;
+        },
+    };
+}
+// ---- Helpers (parallel to the snapshot store's; kept private here to
+//      avoid coupling between the two stores' representations) ----
+function recordToValues(record) {
+    return {
+        id: record.id,
+        workflowId: record.workflowId,
+        status: record.status,
+        startedAt: record.startedAt,
+        completedAt: record.completedAt ?? null,
+        durationMs: record.completedAt !== undefined ? record.completedAt - record.startedAt : null,
+        tags: [...record.tags],
+        // `result` mirrors the snapshot-store convention: the run's public
+        // outcome view. We snapshot output + error here so the dashboard's
+        // reads remain consistent across both stores.
+        result: normalizeRequiredJson({
+            status: record.status,
+            output: record.output,
+            error: record.error,
+            startedAt: record.startedAt,
+            completedAt: record.completedAt,
+        }),
+        input: normalizeJson(record.input),
+        runRecord: normalizeRequiredJson(record),
+        entryFile: null,
+        replayOf: null,
+        idempotencyKey: record.idempotencyKey ?? null,
+    };
+}
+function asRunRecord(value) {
+    if (typeof value !== "object" || value === null)
+        return undefined;
+    // Sanity check: every RunRecord has at least { id, status, journal }.
+    const v = value;
+    if (typeof v.id !== "string")
+        return undefined;
+    if (typeof v.status !== "string")
+        return undefined;
+    return value;
+}
+function normalizeJson(value) {
+    if (value === undefined)
+        return null;
+    return JSON.parse(JSON.stringify(value, (_key, nested) => (typeof nested === "bigint" ? Number(nested) : nested)));
+}
+function normalizeRequiredJson(value) {
+    return JSON.parse(JSON.stringify(value, (_key, nested) => (typeof nested === "bigint" ? Number(nested) : nested)));
+}

package/dist/postgres-schema.d.ts

@@ -214,6 +214,23 @@ export declare const snapshotRunsTable: import("drizzle-orm/pg-core").PgTableWit
             identity: undefined;
             generated: undefined;
         }, {}, {}>;
+        idempotencyKey: import("drizzle-orm/pg-core").PgColumn<{
+            name: "idempotency_key";
+            tableName: "voyant_snapshot_runs";
+            dataType: "string";
+            columnType: "PgText";
+            data: string;
+            driverParam: string;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
     };
     dialect: "pg";
 }>;
@@ -309,4 +326,108 @@ export declare const wakeupsTable: import("drizzle-orm/pg-core").PgTableWithColu
     };
     dialect: "pg";
 }>;
+/**
+ * Manifest store. Holds workflow + event-filter manifests pushed at
+ * `createApp()` boot via `driver.registerManifest(...)`. Last N versions
+ * retained per environment; `is_current` points to the active version.
+ *
+ * One row is "current" per environment, enforced by the partial unique
+ * index `voyant_workflow_manifests_current_idx`.
+ *
+ * See architecture doc §14 for the manifest lifecycle.
+ */
+export declare const workflowManifestsTable: import("drizzle-orm/pg-core").PgTableWithColumns<{
+    name: "voyant_workflow_manifests";
+    schema: undefined;
+    columns: {
+        environment: import("drizzle-orm/pg-core").PgColumn<{
+            name: "environment";
+            tableName: "voyant_workflow_manifests";
+            dataType: "string";
+            columnType: "PgText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
+        versionId: import("drizzle-orm/pg-core").PgColumn<{
+            name: "version_id";
+            tableName: "voyant_workflow_manifests";
+            dataType: "string";
+            columnType: "PgText";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
+        manifest: import("drizzle-orm/pg-core").PgColumn<{
+            name: "manifest";
+            tableName: "voyant_workflow_manifests";
+            dataType: "json";
+            columnType: "PgJsonb";
+            data: Record<string, unknown>;
+            driverParam: unknown;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {
+            $type: Record<string, unknown>;
+        }>;
+        registeredAt: import("drizzle-orm/pg-core").PgColumn<{
+            name: "registered_at";
+            tableName: "voyant_workflow_manifests";
+            dataType: "date";
+            columnType: "PgTimestamp";
+            data: Date;
+            driverParam: string;
+            notNull: true;
+            hasDefault: true;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
+        isCurrent: import("drizzle-orm/pg-core").PgColumn<{
+            name: "is_current";
+            tableName: "voyant_workflow_manifests";
+            dataType: "boolean";
+            columnType: "PgBoolean";
+            data: boolean;
+            driverParam: boolean;
+            notNull: true;
+            hasDefault: true;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
+    };
+    dialect: "pg";
+}>;
 //# sourceMappingURL=postgres-schema.d.ts.map

package/dist/postgres-schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgres-schema.d.ts","sourceRoot":"","sources":["../src/postgres-schema.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA0B7B,CAAA;AAED,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAaxB,CAAA"}
+{"version":3,"file":"postgres-schema.d.ts","sourceRoot":"","sources":["../src/postgres-schema.ts"],"names":[],"mappings":"AAcA,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA0C7B,CAAA;AAED,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAaxB,CAAA;AAED;;;;;;;;;GASG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAelC,CAAA"}

package/dist/postgres-schema.js

@@ -1,4 +1,5 @@
-import { bigint, index, integer, jsonb, pgTable, text } from "drizzle-orm/pg-core";
+import { sql } from "drizzle-orm";
+import { bigint, boolean, index, integer, jsonb, pgTable, primaryKey, text, timestamp, uniqueIndex, } from "drizzle-orm/pg-core";
 export const snapshotRunsTable = pgTable("voyant_snapshot_runs", {
     id: text("id").primaryKey(),
     workflowId: text("workflow_id").notNull(),
@@ -12,9 +13,25 @@ export const snapshotRunsTable = pgTable("voyant_snapshot_runs", {
     runRecord: jsonb("run_record").$type(),
     entryFile: text("entry_file"),
     replayOf: text("replay_of"),
+    /**
+     * Caller-supplied idempotency token, mirrored from
+     * `RunRecord.idempotencyKey` / `TriggerArgs.idempotencyKey`.
+     * The unique partial index below enforces dedup on
+     * `(workflow_id, idempotency_key)`; null values don't participate.
+     */
+    idempotencyKey: text("idempotency_key"),
 }, (table) => ({
     workflowStartedIdx: index("voyant_snapshot_runs_workflow_started_idx").on(table.workflowId, table.startedAt),
     statusStartedIdx: index("voyant_snapshot_runs_status_started_idx").on(table.status, table.startedAt),
+    /**
+     * Unique partial index — enforces idempotency dedup on
+     * `(workflow_id, idempotency_key)` while letting null keys coexist.
+     * Read in `createPostgresSnapshotRunStore` via `INSERT … ON CONFLICT
+     * DO NOTHING RETURNING id`.
+     */
+    idempotencyIdx: uniqueIndex("voyant_snapshot_runs_idempotency_idx")
+        .on(table.workflowId, table.idempotencyKey)
+        .where(sql `${table.idempotencyKey} IS NOT NULL`),
 }));
 export const wakeupsTable = pgTable("voyant_wakeups", {
     runId: text("run_id").primaryKey(),
@@ -26,3 +43,25 @@ export const wakeupsTable = pgTable("voyant_wakeups", {
     dueIdx: index("voyant_wakeups_due_idx").on(table.wakeAt),
     leaseIdx: index("voyant_wakeups_lease_idx").on(table.leaseExpiresAt),
 }));
+/**
+ * Manifest store. Holds workflow + event-filter manifests pushed at
+ * `createApp()` boot via `driver.registerManifest(...)`. Last N versions
+ * retained per environment; `is_current` points to the active version.
+ *
+ * One row is "current" per environment, enforced by the partial unique
+ * index `voyant_workflow_manifests_current_idx`.
+ *
+ * See architecture doc §14 for the manifest lifecycle.
+ */
+export const workflowManifestsTable = pgTable("voyant_workflow_manifests", {
+    environment: text("environment").notNull(),
+    versionId: text("version_id").notNull(),
+    manifest: jsonb("manifest").$type().notNull(),
+    registeredAt: timestamp("registered_at", { withTimezone: true }).notNull().defaultNow(),
+    isCurrent: boolean("is_current").notNull().default(false),
+}, (table) => ({
+    pk: primaryKey({ columns: [table.environment, table.versionId] }),
+    currentIdx: uniqueIndex("voyant_workflow_manifests_current_idx")
+        .on(table.environment)
+        .where(sql `${table.isCurrent}`),
+}));

package/drizzle/0003_idempotency_key.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "voyant_snapshot_runs" ADD COLUMN "idempotency_key" text;--> statement-breakpoint
+CREATE UNIQUE INDEX "voyant_snapshot_runs_idempotency_idx" ON "voyant_snapshot_runs" USING btree ("workflow_id","idempotency_key") WHERE "voyant_snapshot_runs"."idempotency_key" IS NOT NULL;

package/drizzle/0004_workflow_manifests.sql

@@ -0,0 +1,10 @@
+CREATE TABLE "voyant_workflow_manifests" (
+	"environment" text NOT NULL,
+	"version_id" text NOT NULL,
+	"manifest" jsonb NOT NULL,
+	"registered_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"is_current" boolean DEFAULT false NOT NULL,
+	CONSTRAINT "voyant_workflow_manifests_environment_version_id_pk" PRIMARY KEY("environment","version_id")
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "voyant_workflow_manifests_current_idx" ON "voyant_workflow_manifests" USING btree ("environment") WHERE "voyant_workflow_manifests"."is_current";

package/drizzle/meta/_journal.json

@@ -22,6 +22,20 @@
       "when": 1776506061874,
       "tag": "0002_allow_null_input",
       "breakpoints": true
+    },
+    {
+      "idx": 3,
+      "version": "7",
+      "when": 1779000000000,
+      "tag": "0003_idempotency_key",
+      "breakpoints": true
+    },
+    {
+      "idx": 4,
+      "version": "7",
+      "when": 1779000001000,
+      "tag": "0004_workflow_manifests",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voyantjs/workflows-orchestrator-node",
-  "version": "0.28.3",
+  "version": "0.29.0",
   "description": "Node/Docker runtime primitives for @voyantjs/workflows-orchestrator, including a file-backed run store and local scheduler.",
   "license": "Apache-2.0",
   "repository": {
@@ -29,8 +29,8 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "pg": "^8.20.0",
-    "@voyantjs/workflows-orchestrator": "0.28.3",
-    "@voyantjs/workflows": "0.28.3"
+    "@voyantjs/workflows-orchestrator": "0.29.0",
+    "@voyantjs/workflows": "0.29.0"
   },
   "devDependencies": {
     "@types/node": "^20.12.0",

package/src/fs-run-record-store.ts

@@ -25,6 +25,27 @@ export function createFsRunRecordStore(opts: FsRunRecordStoreOptions = {}): RunR
       return record
     },
 
+    async tryInsert(record) {
+      // Single-process FS store: atomicity comes from `mkdir` without
+      // `recursive: true` — succeeds exactly once per path, EEXIST
+      // otherwise. The orchestrator only enters this branch for
+      // idempotency-derived runIds, so the directory's prior existence
+      // means another caller already created the run.
+      try {
+        await mkdir(rootDir, { recursive: true })
+        await mkdir(join(rootDir, record.id))
+      } catch (err) {
+        const code = (err as NodeJS.ErrnoException).code
+        if (code !== "EEXIST") throw err
+        const existing = await readRunFile(join(rootDir, record.id, "run-record.json"))
+        if (existing) return { record: existing, created: false }
+        // Dir exists but no record file — partial-write race window;
+        // fall through and write our record.
+      }
+      await writeFile(join(rootDir, record.id, "run-record.json"), JSON.stringify(record, null, 2))
+      return { record, created: true }
+    },
+
     async list(filter = {}) {
       const entries = await safeReaddir(rootDir)
       const runs: RunRecord[] = []

package/src/index.ts

@@ -39,6 +39,10 @@ export {
   type RunPostgresMigrationsOptions,
   runPostgresMigrations,
 } from "./migrate.js"
+export {
+  createNodeStandaloneDriver,
+  type NodeStandaloneDriverOptions,
+} from "./node-standalone-driver.js"
 export {
   createPersistentWakeupManager,
   type PersistentWakeupManager,
@@ -49,9 +53,20 @@ export {
   createPostgresConnection,
   type PostgresConnection,
 } from "./postgres.js"
+export {
+  createPostgresManifestStore,
+  type ManifestEnvelope,
+  type ManifestStore,
+  type PostgresManifestStoreOptions,
+} from "./postgres-manifest-store.js"
+export {
+  createPostgresRunRecordStore,
+  type PostgresRunRecordStoreOptions,
+} from "./postgres-run-record-store.js"
 export {
   snapshotRunsTable,
   wakeupsTable,
+  workflowManifestsTable,
 } from "./postgres-schema.js"
 export {
   createPostgresSnapshotRunStore,