@bratsos/workflow-engine 0.1.0 → 0.2.1

package/skills/workflow-engine/references/08-common-patterns.md

@@ -1,503 +1,200 @@
 # Common Patterns
 
-Best practices, recipes, and troubleshooting for the workflow engine.
+Best practices, recipes, and patterns for the command kernel.
 
-## Document Processing Pipeline
+## Idempotency
 
-A common pattern for processing documents through multiple stages:
+The `run.create` and `job.execute` commands support idempotency keys. Replaying a command with the same key returns the cached result without re-executing:
 
 ```typescript
-import { WorkflowBuilder, defineStage } from "@bratsos/workflow-engine";
-import { z } from "zod";
-
-// Stage 1: Extract content
-const extractStage = defineStage({
-  id: "extract",
-  name: "Extract Content",
-  schemas: {
-    input: z.object({ documentUrl: z.string().url() }),
-    output: z.object({
-      text: z.string(),
-      metadata: z.object({
-        title: z.string().optional(),
-        pageCount: z.number(),
-      }),
-    }),
-    config: z.object({
-      maxPages: z.number().default(100),
-    }),
-  },
-  async execute(ctx) {
-    const content = await fetchAndExtract(ctx.input.documentUrl, ctx.config);
-    return { output: content };
-  },
-});
-
-// Stage 2: Analyze (parallel)
-const classifyStage = defineStage({
-  id: "classify",
-  name: "Classify",
-  dependencies: ["extract"],
-  schemas: {
-    input: "none",
-    output: z.object({ categories: z.array(z.string()) }),
-    config: z.object({ model: z.string().default("gemini-2.5-flash") }),
-  },
-  async execute(ctx) {
-    const { text } = ctx.require("extract");
-    const ai = createAIHelper("classify", aiLogger);
-    const { object } = await ai.generateObject(ctx.config.model, text, ctx.schemas.output);
-    return { output: object };
-  },
-});
+const cmd = {
+  type: "run.create" as const,
+  idempotencyKey: "order-123-workflow",
+  workflowId: "process-order",
+  input: { orderId: "123" },
+};
 
-const summarizeStage = defineStage({
-  id: "summarize",
-  name: "Summarize",
-  dependencies: ["extract"],
-  schemas: {
-    input: "none",
-    output: z.object({ summary: z.string() }),
-    config: z.object({
-      maxWords: z.number().default(200),
-      model: z.string().default("gemini-2.5-flash"),
-    }),
-  },
-  async execute(ctx) {
-    const { text } = ctx.require("extract");
-    const ai = createAIHelper("summarize", aiLogger);
-    const { text: summary } = await ai.generateText(
-      ctx.config.model,
-      `Summarize in ${ctx.config.maxWords} words:\n\n${text}`
-    );
-    return { output: { summary } };
-  },
-});
+// First call creates the run
+const first = await kernel.dispatch(cmd);
 
-// Stage 3: Merge results
-const mergeStage = defineStage({
-  id: "merge",
-  name: "Merge Results",
-  dependencies: ["classify", "summarize"],
-  schemas: {
-    input: "none",
-    output: z.object({
-      title: z.string().optional(),
-      summary: z.string(),
-      categories: z.array(z.string()),
-    }),
-    config: z.object({}),
-  },
-  async execute(ctx) {
-    const extraction = ctx.require("extract");
-    const classification = ctx.require("classify");
-    const summary = ctx.require("summarize");
-
-    return {
-      output: {
-        title: extraction.metadata.title,
-        summary: summary.summary,
-        categories: classification.categories,
-      },
-    };
-  },
-});
-
-// Build workflow
-const documentPipeline = new WorkflowBuilder(
-  "document-pipeline",
-  "Document Pipeline",
-  "Extract, classify, and summarize documents",
-  z.object({ documentUrl: z.string().url() }),
-  z.object({ title: z.string().optional(), summary: z.string(), categories: z.array(z.string()) })
-)
-  .pipe(extractStage)
-  .parallel([classifyStage, summarizeStage])
-  .pipe(mergeStage)
-  .build();
+// Second call returns cached result (no duplicate run)
+const second = await kernel.dispatch(cmd);
+// first.workflowRunId === second.workflowRunId
 ```
 
-## AI Classification Workflow
+Use deterministic keys derived from domain data (e.g., `order-${orderId}`) to prevent duplicate processing.
 
-Pattern for multi-label classification with confidence scores:
+If the same key is currently executing, dispatch throws `IdempotencyInProgressError`. Retry with backoff instead of issuing parallel same-key commands.
 
-```typescript
-const ClassificationSchema = z.object({
-  labels: z.array(z.object({
-    name: z.string(),
-    confidence: z.number().min(0).max(1),
-    reasoning: z.string(),
-  })),
-  primaryLabel: z.string(),
-});
+## Transactional Outbox
 
-const classificationStage = defineStage({
-  id: "classification",
-  name: "AI Classification",
-  schemas: {
-    input: z.object({ text: z.string() }),
-    output: ClassificationSchema,
-    config: z.object({
-      model: z.string().default("gemini-2.5-flash"),
-      labels: z.array(z.string()),
-      minConfidence: z.number().default(0.7),
-    }),
-  },
-  async execute(ctx) {
-    const ai = createAIHelper("classification", aiLogger);
+Events are not emitted directly. Instead, handlers write events to a transactional outbox table. The `outbox.flush` command publishes pending events through the EventSink:
 
-    const prompt = `Classify the following text into these categories: ${ctx.config.labels.join(", ")}
+```typescript
+// Events accumulate in the outbox as commands execute
+await kernel.dispatch({ type: "run.create", ... });
+await kernel.dispatch({ type: "job.execute", ... });
 
-For each applicable label, provide:
-- The label name
-- A confidence score (0-1)
-- Brief reasoning
+// Flush publishes all pending events
+await kernel.dispatch({ type: "outbox.flush", maxEvents: 100 });
+```
 
-Text to classify:
-${ctx.input.text}`;
+This ensures events are only published after the underlying database transaction succeeds, preventing lost or phantom events.
 
-    const { object } = await ai.generateObject(ctx.config.model, prompt, ClassificationSchema);
+### Multi-phase transactions for `job.execute`
 
-    // Filter by minimum confidence
-    const filtered = {
-      ...object,
-      labels: object.labels.filter(l => l.confidence >= ctx.config.minConfidence),
-    };
+Most commands execute inside a single database transaction (handler logic + outbox event writes). `job.execute` is the exception — it uses a multi-phase transaction pattern to avoid holding a database connection open during long-running stage execution:
 
-    return { output: filtered };
-  },
-});
-```
+1. **Phase 1 (Start):** Upsert stage to `RUNNING` + write `stage:started` outbox event in one transaction. Commits immediately so `RUNNING` status is visible to observers.
+2. **Phase 2 (Execute):** `stageDef.execute()` runs outside any database transaction. Progress events are collected in memory.
+3. **Phase 3 (Complete):** Update stage to `COMPLETED`/`SUSPENDED`/`FAILED` + write completion and progress outbox events in one transaction.
 
-## Error Recovery Pattern
+If the process crashes between Phase 1 and Phase 3, the stage stays in `RUNNING` and `lease.reapStale` will eventually retry the job.
 
-Graceful error handling with fallbacks:
+The outbox includes retry logic with a dead-letter queue (DLQ). Events that fail to publish are retried up to 3 times before being moved to the DLQ. Use `plugin.replayDLQ` to reprocess them:
 
 ```typescript
-const robustStage = defineStage({
-  id: "robust-stage",
-  name: "Robust Stage",
-  schemas: {
-    input: z.object({ data: z.any() }),
-    output: z.object({
-      result: z.any(),
-      usedFallback: z.boolean(),
-      error: z.string().optional(),
-    }),
-    config: z.object({
-      primaryModel: z.string().default("claude-sonnet-4-20250514"),
-      fallbackModel: z.string().default("gemini-2.5-flash"),
-      maxRetries: z.number().default(3),
-    }),
-  },
-  async execute(ctx) {
-    const ai = createAIHelper("robust", aiLogger);
-
-    // Try primary model
-    for (let attempt = 1; attempt <= ctx.config.maxRetries; attempt++) {
-      try {
-        const result = await ai.generateText(ctx.config.primaryModel, ctx.input.data);
-        return {
-          output: { result: result.text, usedFallback: false },
-        };
-      } catch (error) {
-        await ctx.log("WARN", `Primary model failed (attempt ${attempt})`, {
-          error: error.message,
-        });
-
-        if (attempt === ctx.config.maxRetries) break;
-        await sleep(1000 * attempt); // Exponential backoff
-      }
-    }
-
-    // Try fallback model
-    try {
-      await ctx.log("INFO", "Using fallback model");
-      const result = await ai.generateText(ctx.config.fallbackModel, ctx.input.data);
-      return {
-        output: {
-          result: result.text,
-          usedFallback: true,
-          error: "Primary model failed, used fallback",
-        },
-      };
-    } catch (error) {
-      await ctx.log("ERROR", "Fallback model also failed");
-      throw new Error(`All models failed: ${error.message}`);
-    }
-  },
-});
+await kernel.dispatch({ type: "plugin.replayDLQ", maxEvents: 50 });
 ```
 
-## Cost Optimization Strategies
+## Stale Lease Recovery
 
-### 1. Use Batch Operations
+When a worker crashes, its job leases become stale. The `lease.reapStale` command releases them:
 
 ```typescript
-// Instead of many individual calls
-for (const item of items) {
-  await ai.generateText(model, item.prompt);  // Expensive!
-}
-
-// Use batch for 50% savings
-const batch = ai.batch(model, "anthropic");
-const handle = await batch.submit(items.map((item, i) => ({
-  id: `item-${i}`,
-  prompt: item.prompt,
-})));
+await kernel.dispatch({
+  type: "lease.reapStale",
+  staleThresholdMs: 60_000, // Release jobs locked > 60s
+});
 ```
 
-### 2. Cache Expensive Results
-
-```typescript
-async execute(ctx) {
-  const cacheKey = `result-${hash(ctx.input)}`;
-
-  // Check cache
-  if (await ctx.storage.exists(cacheKey)) {
-    return { output: await ctx.storage.load(cacheKey) };
-  }
-
-  // Compute expensive result
-  const result = await expensiveOperation(ctx.input);
+In the Node host, this runs automatically on each orchestration tick. For serverless, include it in your maintenance cron.
 
-  // Cache for future runs
-  await ctx.storage.save(cacheKey, result);
+## Rerun From Stage
 
-  return { output: result };
-}
-```
-
-### 3. Use Appropriate Models
+Rerun a workflow from a specific stage, keeping outputs from earlier stages:
 
 ```typescript
-// Quick classification - use fast model
-const { object } = await ai.generateObject("gemini-2.5-flash", prompt, schema);
-
-// Complex reasoning - use powerful model
-const { text } = await ai.generateText("claude-sonnet-4-20250514", complexPrompt, {
-  maxTokens: 4000,
+const { deletedStages } = await kernel.dispatch({
+  type: "run.rerunFrom",
+  workflowRunId: "run-123",
+  fromStageId: "summarize",
 });
-```
 
-### 4. Optimize Token Usage
-
-```typescript
-// Truncate long inputs
-const truncatedText = text.slice(0, 50000);
-
-// Use structured output to reduce tokens
-const { object } = await ai.generateObject(model, prompt, schema);
-// vs: const { text } = await ai.generateText(model, prompt); JSON.parse(text);
+// Stages from "summarize" onward are deleted and re-queued
+// Earlier stages (e.g., "extract") keep their outputs
 ```
 
-## Logging Best Practices
+## Plugin System
 
-### Structured Logging
+Plugins react to kernel events published through the outbox:
 
 ```typescript
-async execute(ctx) {
-  await ctx.log("INFO", "Stage started", {
-    inputSize: JSON.stringify(ctx.input).length,
-    config: ctx.config,
-  });
-
-  try {
-    const result = await processData(ctx.input);
-
-    await ctx.log("INFO", "Processing complete", {
-      itemsProcessed: result.items.length,
-      duration: Date.now() - startTime,
-    });
+import { definePlugin, createPluginRunner } from "@bratsos/workflow-engine/kernel";
 
-    return { output: result };
-  } catch (error) {
-    await ctx.log("ERROR", "Processing failed", {
-      error: error.message,
-      stack: error.stack,
-    });
-    throw error;
-  }
-}
-```
-
-### Log Levels
-
-| Level | Use For |
-|-------|---------|
-| DEBUG | Detailed debugging info |
-| INFO | Normal operations |
-| WARN | Recoverable issues |
-| ERROR | Failures |
-
-## Type-Safe Context Passing
-
-### Define Workflow Context Type
-
-```typescript
-// Define exact shape of workflow context
-type MyWorkflowContext = {
-  "extract": { text: string; metadata: { title: string } };
-  "classify": { categories: string[] };
-  "summarize": { summary: string };
-};
-
-// Use in stage definition
-const mergeStage = defineStage<
-  "none",
-  typeof OutputSchema,
-  typeof ConfigSchema,
-  MyWorkflowContext
->({
-  id: "merge",
-  // ctx.require("extract") is now typed as { text: string; metadata: { title: string } }
-  async execute(ctx) {
-    const extract = ctx.require("extract"); // Typed!
-    const classify = ctx.require("classify"); // Typed!
+const metricsPlugin = definePlugin({
+  name: "metrics",
+  handlers: {
+    "workflow:completed": async (event) => {
+      await recordMetric("workflow_completed", { workflowId: event.workflowId });
+    },
+    "stage:failed": async (event) => {
+      await alertOnFailure(event);
+    },
   },
 });
-```
 
-### Infer from Workflow
-
-```typescript
-import type { InferWorkflowContext } from "@bratsos/workflow-engine";
-
-const workflow = new WorkflowBuilder(...)
-  .pipe(extractStage)
-  .pipe(classifyStage)
-  .build();
+const runner = createPluginRunner({
+  plugins: [metricsPlugin],
+  eventSink: myEventSink,
+});
 
-type WorkflowContext = InferWorkflowContext<typeof workflow>;
+// Process events from the outbox
+await runner.processEvents(events);
 ```
 
-## Troubleshooting Guide
+## Multi-Worker Coordination
 
-### Stage Not Found in Context
+Multiple workers can process jobs from the same queue safely:
 
-**Error**: `Missing required stage output: "stage-id"`
+- **Run claiming** uses `FOR UPDATE SKIP LOCKED` in PostgreSQL -- no duplicate claims
+- **Job dequeuing** uses atomic `UPDATE ... WHERE status = 'PENDING'` -- no duplicate execution
+- **Stale lease recovery** releases jobs from crashed workers
 
-**Cause**: Stage dependency not in workflow or hasn't run yet.
-
-**Fix**:
-1. Check `dependencies` array includes the stage
-2. Verify stage is added before dependent stage in workflow
-3. Use `ctx.optional()` if stage is truly optional
-
-### Batch Never Completes
-
-**Symptoms**: Stage stays SUSPENDED forever
-
-**Causes**:
-1. `nextPollAt` not being updated
-2. `maxWaitTime` too short
-3. Batch provider issues
-
-**Fix**:
 ```typescript
-// In checkCompletion
-return {
-  ready: false,
-  nextCheckIn: 60000,  // Make sure this is set
-};
-
-// Check batch status manually
-const batch = ai.batch(model, provider);
-const status = await batch.getStatus(batchId);
-console.log("Batch status:", status);
+// Worker 1 and Worker 2 can run simultaneously
+const host1 = createNodeHost({ kernel, jobTransport, workerId: "worker-1" });
+const host2 = createNodeHost({ kernel, jobTransport, workerId: "worker-2" });
 ```
 
-### Type Mismatch in Workflow
-
-**Error**: TypeScript errors about incompatible types
-
-**Cause**: Output of one stage doesn't match input of next.
-
-**Fix**:
-1. Use `input: "none"` for stages that only use context
-2. Ensure schemas match across stages
-3. Use `.passthrough()` on Zod schemas for flexibility
-
-### Prisma Enum Errors
-
-**Error**: `Invalid value for argument 'status'. Expected Status`
-
-**Cause**: Prisma 7.x requires typed enums, not strings
-
-**Fix**: The library handles this automatically via the enum compatibility layer. If you see this error:
-1. Ensure you're using `createPrismaWorkflowPersistence(prisma)`
-2. Check you're not bypassing the persistence layer with direct Prisma calls
+## Optimistic Concurrency
 
-### Memory Issues with Large Batches
+The persistence layer uses version fields on records to detect concurrent modifications:
 
-**Symptoms**: Process crashes or slows down
-
-**Fix**:
 ```typescript
-// Process in smaller batches
-const CHUNK_SIZE = 100;
-for (let i = 0; i < items.length; i += CHUNK_SIZE) {
-  const chunk = items.slice(i, i + CHUNK_SIZE);
-  await processChunk(chunk);
-}
-
-// Stream instead of loading all at once
-for await (const chunk of streamResults(batchId)) {
-  yield processChunk(chunk);
-}
+// If two workers try to update the same run simultaneously,
+// one will get a StaleVersionError and retry
+import { StaleVersionError } from "@bratsos/workflow-engine";
 ```
 
-### Workflow Stuck in RUNNING
-
-**Symptoms**: Workflow never completes
+## Document Processing Pipeline
 
-**Causes**:
-1. Stage threw unhandled error
-2. Job queue not processing
-3. Worker crashed
+A common pattern combining sequential and parallel stages:
 
-**Fix**:
 ```typescript
-// Check for failed stages
-const stages = await persistence.getStagesByRun(runId);
-const failed = stages.filter(s => s.status === "FAILED");
-if (failed.length > 0) {
-  console.log("Failed stages:", failed.map(s => ({ id: s.stageId, error: s.errorMessage })));
-}
-
-// Release stale jobs
-await jobQueue.releaseStaleJobs(60000);
+const workflow = new WorkflowBuilder(
+  "doc-processor", "Document Processor", "Process documents",
+  InputSchema, OutputSchema,
+)
+  .pipe(extractTextStage)              // Stage 1: Extract
+  .parallel([
+    sentimentAnalysisStage,            // Stage 2a: Analyze sentiment
+    keywordExtractionStage,            // Stage 2b: Extract keywords
+  ])
+  .pipe(aggregateResultsStage)         // Stage 3: Combine results
+  .build();
 ```
 
-## Configuration Recipes
-
-### Development Config
+Subsequent stages access parallel outputs by stage ID:
 
 ```typescript
-const devConfig = {
-  pollIntervalMs: 2000,
-  jobPollIntervalMs: 500,
-  staleJobThresholdMs: 30000,
-};
+async execute(ctx) {
+  const sentiment = ctx.require("sentiment-analysis");
+  const keywords = ctx.require("keyword-extraction");
+  // ...
+}
 ```
 
-### Production Config
+## Error Handling in Stages
 
 ```typescript
-const prodConfig = {
-  pollIntervalMs: 10000,
-  jobPollIntervalMs: 1000,
-  staleJobThresholdMs: 120000,
-  workerId: `worker-${process.env.HOSTNAME}`,
-};
+async execute(ctx) {
+  try {
+    const result = await processDocument(ctx.input);
+    return { output: result };
+  } catch (error) {
+    ctx.log("ERROR", "Processing failed", {
+      error: error instanceof Error ? error.message : String(error),
+    });
+    throw error; // Re-throw to mark stage as FAILED
+  }
+}
 ```
 
-### High-Throughput Config
+Failed stages trigger the `stage:failed` event. The host's maintenance tick detects failure through `run.transition`, which marks the workflow as FAILED.
+
+## Progress Reporting
 
 ```typescript
-const throughputConfig = {
-  pollIntervalMs: 5000,
-  jobPollIntervalMs: 100,
-  staleJobThresholdMs: 300000,
-};
+async execute(ctx) {
+  for (const [index, item] of items.entries()) {
+    ctx.onProgress({
+      progress: (index + 1) / items.length,
+      message: `Processing item ${index + 1}/${items.length}`,
+      details: { currentItem: item.id },
+    });
+    await processItem(item);
+  }
+  return { output: { processedCount: items.length } };
+}
 ```