@sprqvntrs/workflows 0.2.4

package/index.ts

@@ -0,0 +1,234 @@
+/**
+ * @sprqvntrs/workflows
+ *
+ * PostgreSQL-backed workflow orchestration with pg-boss job queuing.
+ *
+ * This package provides a declarative, template-driven approach to executing
+ * multi-stage background workflows with reliable persistence and job processing.
+ *
+ * ## Quick Start
+ *
+ * ```typescript
+ * import { createWorkflowOrchestrator } from '@sprqvntrs/workflows';
+ * import { drizzle } from 'drizzle-orm/node-postgres';
+ * import { Pool } from 'pg';
+ *
+ * // Set up database
+ * const pool = new Pool({ connectionString: process.env.DATABASE_URL });
+ * const db = drizzle(pool);
+ *
+ * // Create orchestrator
+ * const orchestrator = await createWorkflowOrchestrator({
+ *   connectionString: process.env.DATABASE_URL,
+ *   db,
+ *   queues: [
+ *     { name: 'default', workers: 5 },
+ *     { name: 'sequential', workers: 1 },
+ *   ],
+ * });
+ *
+ * // Register a workflow template
+ * orchestrator.registerTemplate({
+ *   type: 'content-generation',
+ *   queue: 'default',
+ *   version: '1.0.0',
+ *   stages: [
+ *     { name: 'gather', operations: [{ type: 'gather.data' }] },
+ *     { name: 'analyze', parallel: true, operations: [
+ *       { type: 'analyze.content' },
+ *       { type: 'analyze.market' },
+ *     ]},
+ *     { name: 'generate', operations: [{ type: 'generate.report' }] },
+ *   ],
+ * });
+ *
+ * // Register operation handlers
+ * orchestrator.registerOperations({
+ *   'gather.data': async (ctx) => {
+ *     const data = await fetchData(ctx.previousResults.url);
+ *     return { status: 'completed', data: { fetchedData: data } };
+ *   },
+ *   // ... more handlers
+ * });
+ *
+ * // Start a workflow
+ * const { workflowId } = await orchestrator.start({
+ *   type: 'content-generation',
+ *   context: { url: 'https://example.com' },
+ * });
+ * ```
+ *
+ * ## Architecture
+ *
+ * The package follows a three-layer architecture:
+ *
+ * 1. **Orchestrator** - Public API for starting, managing, and querying workflows
+ * 2. **Execution Engine** - Interprets templates and executes stages/operations
+ * 3. **Infrastructure** - pg-boss job queue and Drizzle database persistence
+ *
+ * ## Features
+ *
+ * - **Declarative Templates** - Define workflows as configuration objects
+ * - **Parallel Execution** - Run operations concurrently within stages
+ * - **Checkpoints** - Pause workflows for manual intervention
+ * - **Workflow Chaining** - Trigger follow-up workflows on completion
+ * - **Fix-Verify Loops** - Automatic retry cycles for data integrity
+ * - **Entity Locking** - Prevent race conditions between workflows
+ * - **Type Safety** - Full TypeScript support with generics
+ *
+ * @packageDocumentation
+ */
+
+// =============================================================================
+// Main Orchestrator
+// =============================================================================
+
+export { createWorkflowOrchestrator, type WorkflowOrchestrator, type CreateOrchestratorConfig } from './src/orchestrator';
+
+// =============================================================================
+// Core Types
+// =============================================================================
+
+export type {
+  // Status types
+  WorkflowStatus,
+  OperationStatus,
+
+  // Template types
+  WorkflowTemplate,
+  StageTemplate,
+  OperationTemplate,
+  CheckpointTemplate,
+  CoordinationConfig,
+  QueueConfig,
+
+  // Operation types
+  OperationHandler,
+  OperationContext,
+  OperationResult,
+
+  // Record types
+  WorkflowRecord,
+  OperationRecord,
+  WorkflowContext,
+
+  // Configuration types
+  OrchestratorConfig,
+  QueueDefinition,
+
+  // API types
+  StartWorkflowOptions,
+  StartWorkflowResult,
+  ScheduleOptions,
+  WorkflowStatusDetails,
+} from './src/types';
+
+// Error types
+export { WorkflowError, TemplateError, OperationError, TimeoutError, CoordinationError } from './src/types';
+
+// =============================================================================
+// Registries
+// =============================================================================
+
+export {
+  createTemplateRegistry,
+  validateTemplate,
+  getOperationTypes,
+  getQueueNames,
+  type TemplateRegistry,
+  type ValidationError,
+} from './src/templates/registry';
+
+export {
+  createOperationRegistry,
+  withTimeout,
+  withRetry,
+  withErrorBoundary,
+  compose,
+  defineHandler,
+  type OperationRegistry,
+  type TypedOperationHandler,
+} from './src/operations/registry';
+
+// =============================================================================
+// Execution Engine
+// =============================================================================
+
+export {
+  createExecutionEngine,
+  type ExecutionEngine,
+  type ExecutionEngineConfig,
+} from './src/engine/execution-engine';
+
+// =============================================================================
+// Infrastructure
+// =============================================================================
+
+// pg-boss utilities
+export {
+  createBoss,
+  createSendOptions,
+  createWorkOptions,
+  registerWorker,
+  registerBatchWorker,
+  scheduleRecurring,
+  unschedule,
+  getJob,
+  cancelJob,
+  resumeJob,
+  setupGracefulShutdown,
+  setupEventListeners,
+  DEFAULT_QUEUE_CONFIG,
+  DEFAULT_PGBOSS_CONFIG,
+  type PgBossConfig,
+  type WorkflowJobData,
+  type JobHandler,
+  type BatchJobHandler,
+  type PgBossEvent,
+} from './src/infrastructure/pg-boss';
+
+// Database state
+export {
+  createDbState,
+  type DbState,
+  type Database,
+  type CreateWorkflowData,
+  type CreateOperationData,
+  type ListWorkflowsOptions,
+} from './src/infrastructure/db-state';
+
+// pg-boss introspection queries
+export {
+  createPgBossQueries,
+  type PgBossQueries,
+  type PgBossQueriesConfig,
+  type PgBossJobState,
+  type PgBossArchiveState,
+  type PgBossJobItem,
+  type PgBossJobStats,
+  type PgBossScheduleItem,
+  type PgBossArchiveItem,
+  type DeleteWorkflowJobsResult,
+  type PendingJobsResult,
+} from './src/infrastructure/pgboss-queries';
+
+// =============================================================================
+// Coordination
+// =============================================================================
+
+export {
+  createLockManager,
+  withLock,
+  withLockWait,
+  type LockManager,
+  type LockOptions,
+  type WaitOptions,
+  type LockInfo,
+} from './src/coordination/lock-manager';
+
+// =============================================================================
+// Re-exports for convenience
+// =============================================================================
+
+// Re-export PgBoss for advanced usage
+export { default as PgBoss } from 'pg-boss';

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@sprqvntrs/workflows",
+  "version": "0.2.4",
+  "description": "PostgreSQL-backed workflow orchestration with pg-boss job queuing",
+  "type": "module",
+  "main": "./index.ts",
+  "types": "./index.ts",
+  "exports": {
+    ".": "./index.ts",
+    "./worker": "./src/worker/index.ts",
+    "./schema": "./src/infrastructure/schema.ts",
+    "./testing": "./src/testing/index.ts"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/SPRQVNTRS/platform.git",
+    "directory": "packages/workflows"
+  },
+  "files": [
+    "src/**/*",
+    "index.ts",
+    "LICENSE"
+  ],
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "pg-boss": "^10.1.5"
+  },
+  "peerDependencies": {
+    "drizzle-orm": "^0.39.0",
+    "pg": "^8.11.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "@types/pg": "^8.11.0",
+    "drizzle-orm": "^0.39.0",
+    "pg": "^8.11.0",
+    "tsx": "^4.20.6",
+    "typescript": "^5.6.0",
+    "vitest": "^3.2.4"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "homepage": "https://github.com/SPRQVNTRS/platform/tree/main/packages/workflows#readme",
+  "bugs": {
+    "url": "https://github.com/SPRQVNTRS/platform/issues"
+  }
+}

package/src/coordination/lock-manager.ts

@@ -0,0 +1,431 @@
+/**
+ * Lock manager for workflow coordination.
+ *
+ * This module provides entity locking to prevent race conditions when
+ * multiple workflows operate on the same entity.
+ *
+ * @example
+ * ```typescript
+ * import { createLockManager } from '@sprqvntrs/workflows';
+ *
+ * const lockManager = createLockManager(dbState);
+ *
+ * // Acquire lock before processing
+ * const acquired = await lockManager.acquire('document', '123', workflowId);
+ * if (!acquired) {
+ *   throw new Error('Entity is locked by another workflow');
+ * }
+ *
+ * try {
+ *   // Process entity...
+ * } finally {
+ *   await lockManager.release('document', '123');
+ * }
+ * ```
+ */
+
+import type { DbState } from '../infrastructure/db-state';
+import { CoordinationError } from '../types';
+
+// =============================================================================
+// Types
+// =============================================================================
+
+/**
+ * Lock manager interface.
+ */
+export interface LockManager {
+  /**
+   * Acquires a lock on an entity.
+   *
+   * @param entityType - Type of entity (e.g., 'document', 'user')
+   * @param entityId - Entity identifier
+   * @param workflowId - Workflow requesting the lock
+   * @param options - Lock options
+   * @returns True if lock was acquired
+   *
+   * @example
+   * ```typescript
+   * const acquired = await lockManager.acquire('document', '123', workflowId, {
+   *   ttlMs: 600000, // 10 minute TTL
+   * });
+   * ```
+   */
+  acquire: (
+    entityType: string,
+    entityId: string,
+    workflowId: string,
+    options?: LockOptions,
+  ) => Promise<boolean>;
+
+  /**
+   * Releases a lock on an entity.
+   *
+   * @param entityType - Type of entity
+   * @param entityId - Entity identifier
+   * @returns True if lock was released
+   *
+   * @example
+   * ```typescript
+   * await lockManager.release('document', '123');
+   * ```
+   */
+  release: (entityType: string, entityId: string) => Promise<boolean>;
+
+  /**
+   * Releases all locks held by a workflow.
+   *
+   * @param workflowId - Workflow ID
+   * @returns Number of locks released
+   *
+   * @example
+   * ```typescript
+   * const count = await lockManager.releaseAll(workflowId);
+   * console.log(`Released ${count} locks`);
+   * ```
+   */
+  releaseAll: (workflowId: string) => Promise<number>;
+
+  /**
+   * Checks if an entity is locked.
+   *
+   * @param entityType - Type of entity
+   * @param entityId - Entity identifier
+   * @returns Lock info or null if not locked
+   *
+   * @example
+   * ```typescript
+   * const lock = await lockManager.check('document', '123');
+   * if (lock) {
+   *   console.log(`Locked by workflow ${lock.workflowId}`);
+   * }
+   * ```
+   */
+  check: (entityType: string, entityId: string) => Promise<LockInfo | null>;
+
+  /**
+   * Waits for a lock to become available.
+   *
+   * @param entityType - Type of entity
+   * @param entityId - Entity identifier
+   * @param workflowId - Workflow requesting the lock
+   * @param options - Wait options
+   * @returns True if lock was acquired
+   *
+   * @example
+   * ```typescript
+   * const acquired = await lockManager.waitAndAcquire('document', '123', workflowId, {
+   *   timeoutMs: 30000,
+   *   pollIntervalMs: 1000,
+   * });
+   * ```
+   */
+  waitAndAcquire: (
+    entityType: string,
+    entityId: string,
+    workflowId: string,
+    options?: WaitOptions,
+  ) => Promise<boolean>;
+
+  /**
+   * Cleans up expired locks.
+   *
+   * Should be called periodically to release stale locks.
+   *
+   * @returns Number of locks cleaned up
+   */
+  cleanup: () => Promise<number>;
+}
+
+/**
+ * Options for acquiring a lock.
+ */
+export interface LockOptions {
+  /**
+   * Time-to-live for the lock in milliseconds.
+   * If not provided, lock has no expiration.
+   */
+  ttlMs?: number;
+}
+
+/**
+ * Options for waiting for a lock.
+ */
+export interface WaitOptions extends LockOptions {
+  /**
+   * Maximum time to wait in milliseconds.
+   * @default 60000 (1 minute)
+   */
+  timeoutMs?: number;
+
+  /**
+   * Interval between lock checks in milliseconds.
+   * @default 1000 (1 second)
+   */
+  pollIntervalMs?: number;
+}
+
+/**
+ * Information about a held lock.
+ */
+export interface LockInfo {
+  /**
+   * Type of locked entity.
+   */
+  entityType: string;
+
+  /**
+   * ID of locked entity.
+   */
+  entityId: string;
+
+  /**
+   * Workflow holding the lock.
+   */
+  workflowId: string;
+
+  /**
+   * When lock was acquired.
+   */
+  acquiredAt: Date;
+
+  /**
+   * When lock expires (if TTL was set).
+   */
+  expiresAt: Date | null;
+}
+
+// =============================================================================
+// Factory Function
+// =============================================================================
+
+/**
+ * Creates a lock manager instance.
+ *
+ * The lock manager provides entity-level locking to prevent race conditions
+ * when multiple workflows operate on the same entity.
+ *
+ * @param dbState - Database state manager
+ * @returns Lock manager instance
+ *
+ * @example
+ * ```typescript
+ * const lockManager = createLockManager(dbState);
+ *
+ * // In your workflow operation
+ * async function processDocument(context: OperationContext) {
+ *   const { workflowId, previousResults } = context;
+ *   const documentId = previousResults.documentId as string;
+ *
+ *   // Acquire lock with TTL
+ *   const acquired = await lockManager.acquire('document', documentId, workflowId, {
+ *     ttlMs: 600000, // 10 minutes
+ *   });
+ *
+ *   if (!acquired) {
+ *     return { status: 'failed', reason: 'Document is being processed by another workflow' };
+ *   }
+ *
+ *   try {
+ *     // Process document...
+ *     return { status: 'completed', data: { processed: true } };
+ *   } finally {
+ *     // Always release in finally block
+ *     await lockManager.release('document', documentId);
+ *   }
+ * }
+ * ```
+ */
+export function createLockManager(dbState: DbState): LockManager {
+  return {
+    async acquire(
+      entityType: string,
+      entityId: string,
+      workflowId: string,
+      options?: LockOptions,
+    ): Promise<boolean> {
+      const expiresAt = options?.ttlMs ? new Date(Date.now() + options.ttlMs) : undefined;
+
+      return dbState.acquireLock(entityType, entityId, workflowId, expiresAt);
+    },
+
+    async release(entityType: string, entityId: string): Promise<boolean> {
+      return dbState.releaseLock(entityType, entityId);
+    },
+
+    async releaseAll(workflowId: string): Promise<number> {
+      return dbState.releaseLocksByWorkflow(workflowId);
+    },
+
+    async check(entityType: string, entityId: string): Promise<LockInfo | null> {
+      const lock = await dbState.getLock(entityType, entityId);
+      if (!lock) {
+        return null;
+      }
+
+      return {
+        entityType: lock.entityType,
+        entityId: lock.entityId,
+        workflowId: lock.workflowId,
+        acquiredAt: lock.acquiredAt,
+        expiresAt: lock.expiresAt,
+      };
+    },
+
+    async waitAndAcquire(
+      entityType: string,
+      entityId: string,
+      workflowId: string,
+      options?: WaitOptions,
+    ): Promise<boolean> {
+      const { timeoutMs = 60000, pollIntervalMs = 1000, ttlMs } = options ?? {};
+      const deadline = Date.now() + timeoutMs;
+
+      while (Date.now() < deadline) {
+        // Try to acquire
+        const expiresAt = ttlMs ? new Date(Date.now() + ttlMs) : undefined;
+        const acquired = await dbState.acquireLock(entityType, entityId, workflowId, expiresAt);
+
+        if (acquired) {
+          return true;
+        }
+
+        // Check if existing lock has expired
+        const existingLock = await dbState.getLock(entityType, entityId);
+        if (existingLock?.expiresAt && existingLock.expiresAt < new Date()) {
+          // Lock has expired, clean it up and try again
+          await dbState.releaseLock(entityType, entityId);
+          continue;
+        }
+
+        // Wait before next attempt
+        if (Date.now() + pollIntervalMs < deadline) {
+          await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+        } else {
+          break;
+        }
+      }
+
+      return false;
+    },
+
+    async cleanup(): Promise<number> {
+      return dbState.cleanupExpiredLocks();
+    },
+  };
+}
+
+// =============================================================================
+// Helper Functions
+// =============================================================================
+
+/**
+ * Executes a function with a lock, automatically releasing on completion.
+ *
+ * This helper ensures the lock is always released, even if the function throws.
+ *
+ * @param lockManager - Lock manager instance
+ * @param entityType - Type of entity to lock
+ * @param entityId - Entity identifier
+ * @param workflowId - Workflow ID
+ * @param fn - Function to execute while holding the lock
+ * @param options - Lock options
+ * @returns Result of the function
+ * @throws CoordinationError if lock cannot be acquired
+ *
+ * @example
+ * ```typescript
+ * const result = await withLock(
+ *   lockManager,
+ *   'document',
+ *   '123',
+ *   workflowId,
+ *   async () => {
+ *     // This runs while holding the lock
+ *     return await processDocument();
+ *   },
+ *   { ttlMs: 300000 }
+ * );
+ * ```
+ */
+export async function withLock<T>(
+  lockManager: LockManager,
+  entityType: string,
+  entityId: string,
+  workflowId: string,
+  fn: () => Promise<T>,
+  options?: LockOptions,
+): Promise<T> {
+  const acquired = await lockManager.acquire(entityType, entityId, workflowId, options);
+
+  if (!acquired) {
+    throw new CoordinationError(`Failed to acquire lock on ${entityType}:${entityId}`, {
+      entityType,
+      entityId,
+      workflowId,
+    });
+  }
+
+  try {
+    return await fn();
+  } finally {
+    await lockManager.release(entityType, entityId);
+  }
+}
+
+/**
+ * Executes a function with a lock, waiting if necessary.
+ *
+ * Like `withLock`, but waits for the lock to become available.
+ *
+ * @param lockManager - Lock manager instance
+ * @param entityType - Type of entity to lock
+ * @param entityId - Entity identifier
+ * @param workflowId - Workflow ID
+ * @param fn - Function to execute while holding the lock
+ * @param options - Wait options
+ * @returns Result of the function
+ * @throws CoordinationError if lock cannot be acquired within timeout
+ *
+ * @example
+ * ```typescript
+ * const result = await withLockWait(
+ *   lockManager,
+ *   'document',
+ *   '123',
+ *   workflowId,
+ *   async () => {
+ *     return await processDocument();
+ *   },
+ *   { timeoutMs: 60000, ttlMs: 300000 }
+ * );
+ * ```
+ */
+export async function withLockWait<T>(
+  lockManager: LockManager,
+  entityType: string,
+  entityId: string,
+  workflowId: string,
+  fn: () => Promise<T>,
+  options?: WaitOptions,
+): Promise<T> {
+  const acquired = await lockManager.waitAndAcquire(entityType, entityId, workflowId, options);
+
+  if (!acquired) {
+    throw new CoordinationError(
+      `Timed out waiting for lock on ${entityType}:${entityId}`,
+      {
+        entityType,
+        entityId,
+        workflowId,
+        timeoutMs: options?.timeoutMs,
+      },
+    );
+  }
+
+  try {
+    return await fn();
+  } finally {
+    await lockManager.release(entityType, entityId);
+  }
+}