@doclo/flows 0.1.5 → 0.1.7

package/dist/index.d.ts

@@ -468,7 +468,7 @@ type SerializableFlow = {
 /**
  * Serializable step definition
  */
-type SerializableStep = SerializableStandardStep | SerializableConditionalStep | SerializableForEachStep;
+type SerializableStep = SerializableStandardStep | SerializableConditionalStep | SerializableForEachStep | SerializableRouteStep;
 /**
  * Standard sequential step
  */
@@ -514,6 +514,56 @@ type SerializableForEachStep = {
     config: SplitConfig;
     itemFlow: SerializableFlow | FlowReference;
 };
+/**
+ * Route branch configuration
+ * Maps multiple MIME types to a single branch
+ */
+type RouteBranchConfig = {
+    /**
+     * MIME types that route to this branch.
+     * Supports exact matches (e.g., 'application/pdf')
+     * and glob patterns (e.g., 'image/*')
+     */
+    mimeTypes: string[];
+    /**
+     * Optional description for documentation/debugging
+     */
+    description?: string;
+};
+/**
+ * Route step configuration
+ * Routes documents based on detected MIME type (no provider required)
+ */
+type RouteConfig = {
+    type: 'route';
+    /**
+     * Branch definitions mapping branch names to MIME type configurations.
+     * Order matters for matching - first matching branch wins.
+     */
+    branches: Record<string, RouteBranchConfig>;
+};
+/**
+ * Route step (MIME-based routing + branches)
+ *
+ * Unlike conditional (which uses VLM categorization), route uses
+ * deterministic MIME detection from magic bytes - no provider needed.
+ *
+ * Branches can be either inline flows or references to separate flows.
+ * Use references to avoid hitting database JSON nesting limits.
+ */
+type SerializableRouteStep = {
+    type: 'route';
+    id: string;
+    name?: string;
+    nodeType: 'route';
+    config: RouteConfig;
+    branches: Record<string, SerializableFlow | FlowReference>;
+    /**
+     * Fallback branch for unmatched MIME types.
+     * If not provided and no branch matches, throws error with details.
+     */
+    others?: SerializableFlow | FlowReference;
+};
 /**
  * Input mapping configuration for trigger nodes
  * Declarative alternatives to mapInput functions (for serialization)
@@ -555,6 +605,9 @@ type ParseConfig = {
         onTie?: 'random' | 'fail' | 'retry';
     };
     maxTokens?: number;
+    promptRef?: string;
+    promptVariables?: Record<string, any>;
+    additionalInstructions?: string;
 };
 type ExtractConfig = {
     type: 'extract';
@@ -571,6 +624,9 @@ type ExtractConfig = {
         max_tokens?: number;
     };
     maxTokens?: number;
+    promptRef?: string;
+    promptVariables?: Record<string, any>;
+    additionalInstructions?: string;
 };
 type SplitConfig = {
     type: 'split';
@@ -606,6 +662,8 @@ type CategorizeConfig = {
         onTie?: 'random' | 'fail' | 'retry';
     };
     promptRef?: string;
+    promptVariables?: Record<string, any>;
+    additionalInstructions?: string;
     maxTokens?: number;
 };
 type TriggerConfig = {
@@ -781,6 +839,26 @@ interface ForEachCompositeConfig {
  * Includes full observability, metrics merging, and error context.
  */
 declare function createForEachCompositeNode(config: ForEachCompositeConfig): NodeDef<FlowInput, unknown[]>;
+/**
+ * Configuration for route composite node
+ */
+interface RouteCompositeConfig {
+    stepId: string;
+    routeConfig: RouteConfig;
+    branches: Record<string, SerializableFlow | FlowReference>;
+    others?: SerializableFlow | FlowReference;
+    providers: ProviderRegistry;
+    flows: FlowRegistry;
+}
+/**
+ * Creates a composite node that:
+ * 1. Detects MIME type from input (base64 or URL)
+ * 2. Routes to appropriate branch based on MIME type
+ * 3. Returns the branch flow's output
+ *
+ * No provider required - uses deterministic MIME detection from magic bytes.
+ */
+declare function createRouteCompositeNode(config: RouteCompositeConfig): NodeDef<FlowInput, unknown>;
 
 /**
  * Flow Validation
@@ -961,4 +1039,4 @@ declare function buildTwoProviderFlow(opts: {
     }>;
 };
 
-export { type BatchFlowResult, type BuiltFlow, type CategorizeConfig, type ConditionalCompositeConfig, type ExtractConfig, FLOW_REGISTRY, type FieldMapping, type FlowBuilder, type FlowOptions, type FlowProgressCallbacks, type FlowReference, type FlowRegistry$1 as FlowRegistry, type FlowRunOptions, FlowSerializationError, type FlowValidationResult, type ForEachCompositeConfig, type InputMappingConfig, type NodeConfig, type OutputConfig, type ParseConfig, type ProviderRegistry, type SerializableConditionalStep, type SerializableFlow, type SerializableForEachStep, type SerializableInputValidation, type SerializableStandardStep, type SerializableStep, type SplitConfig, type TriggerConfig, type ValidationError, type ValidationOptions, type ValidationResult, type ValidationWarning, buildFlowFromConfig, buildMultiProviderFlow, buildTwoProviderFlow, buildVLMDirectFlow, clearRegistry, createConditionalCompositeNode, createFlow, createForEachCompositeNode, defineFlowConfig, extractNodeMetadata, getFlow, getFlowCount, hasFlow, isBatchFlowResult, isFlowReference, isSingleFlowResult, listFlows, registerFlow, resolveFlowReference, unregisterFlow, validateFlow, validateFlowOrThrow };
+export { type BatchFlowResult, type BuiltFlow, type CategorizeConfig, type ConditionalCompositeConfig, type ExtractConfig, FLOW_REGISTRY, type FieldMapping, type FlowBuilder, type FlowOptions, type FlowProgressCallbacks, type FlowReference, type FlowRegistry$1 as FlowRegistry, type FlowRunOptions, FlowSerializationError, type FlowValidationResult, type ForEachCompositeConfig, type InputMappingConfig, type NodeConfig, type OutputConfig, type ParseConfig, type ProviderRegistry, type RouteBranchConfig, type RouteCompositeConfig, type RouteConfig, type SerializableConditionalStep, type SerializableFlow, type SerializableForEachStep, type SerializableInputValidation, type SerializableRouteStep, type SerializableStandardStep, type SerializableStep, type SplitConfig, type TriggerConfig, type ValidationError, type ValidationOptions, type ValidationResult, type ValidationWarning, buildFlowFromConfig, buildMultiProviderFlow, buildTwoProviderFlow, buildVLMDirectFlow, clearRegistry, createConditionalCompositeNode, createFlow, createForEachCompositeNode, createRouteCompositeNode, defineFlowConfig, extractNodeMetadata, getFlow, getFlowCount, hasFlow, isBatchFlowResult, isFlowReference, isSingleFlowResult, listFlows, registerFlow, resolveFlowReference, unregisterFlow, validateFlow, validateFlowOrThrow };

package/dist/index.js

@@ -53,6 +53,15 @@ function normalizeFlowInput(input) {
   }
   return input;
 }
+function filterInternalArtifacts(artifacts) {
+  const filtered = {};
+  for (const [key, value] of Object.entries(artifacts)) {
+    if (!key.startsWith("__")) {
+      filtered[key] = value;
+    }
+  }
+  return filtered;
+}
 var Flow = class {
   steps = [];
   observability;
@@ -1051,7 +1060,7 @@ Correct: .conditional('step', () => parse({ provider }))`
               results: flowResults,
               metrics,
               aggregated: aggregateMetrics(metrics),
-              artifacts
+              artifacts: filterInternalArtifacts(artifacts)
             };
           }
         } catch (error) {
@@ -1115,7 +1124,7 @@ Correct: .conditional('step', () => parse({ provider }))`
             outputs,
             metrics,
             aggregated: aggregateMetrics(metrics),
-            artifacts
+            artifacts: filterInternalArtifacts(artifacts)
           };
         } else {
           result = {
@@ -1124,7 +1133,7 @@ Correct: .conditional('step', () => parse({ provider }))`
             outputs,
             metrics,
             aggregated: aggregateMetrics(metrics),
-            artifacts
+            artifacts: filterInternalArtifacts(artifacts)
           };
         }
       } else {
@@ -1132,7 +1141,7 @@ Correct: .conditional('step', () => parse({ provider }))`
           output: currentData,
           metrics,
           aggregated: aggregateMetrics(metrics),
-          artifacts
+          artifacts: filterInternalArtifacts(artifacts)
         };
       }
       if (this.observability && sampled && traceContext && executionId) {
@@ -1243,6 +1252,7 @@ import { parse, extract, split as split2, categorize as categorize2, trigger, ou
 
 // src/composite-nodes.ts
 import { FlowExecutionError as FlowExecutionError2 } from "@doclo/core";
+import { detectMimeTypeFromBase64Async, detectMimeTypeFromBase64 } from "@doclo/core";
 import { categorize, split } from "@doclo/nodes";
 function parseProviderName(name) {
   const colonIndex = name.indexOf(":");
@@ -1622,6 +1632,214 @@ function resolveBranchFlow(flowOrRef, flows) {
   }
   return flowOrRef;
 }
+function matchesMimePattern(mimeType, pattern) {
+  if (pattern === mimeType) return true;
+  if (pattern.endsWith("/*")) {
+    const prefix = pattern.slice(0, -2);
+    return mimeType.startsWith(prefix + "/");
+  }
+  return false;
+}
+function findMatchingBranch(mimeType, branches) {
+  for (const [branchName, config] of Object.entries(branches)) {
+    for (const pattern of config.mimeTypes) {
+      if (matchesMimePattern(mimeType, pattern)) {
+        return branchName;
+      }
+    }
+  }
+  return void 0;
+}
+function extractBase64Data(input) {
+  if (input.startsWith("data:")) {
+    const commaIndex = input.indexOf(",");
+    if (commaIndex !== -1) {
+      return input.substring(commaIndex + 1);
+    }
+  }
+  return input;
+}
+function createRouteCompositeNode(config) {
+  const { stepId, routeConfig, branches, others, providers, flows } = config;
+  return {
+    key: "route-composite",
+    run: async (input, ctx) => {
+      const t0 = Date.now();
+      let detectedMimeType;
+      let selectedBranch;
+      let phase = "detect";
+      try {
+        let base64Data;
+        if (input.base64) {
+          base64Data = extractBase64Data(input.base64);
+        } else if (input.url) {
+          const response = await fetch(input.url);
+          if (!response.ok) {
+            throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`);
+          }
+          const arrayBuffer = await response.arrayBuffer();
+          const bytes = new Uint8Array(arrayBuffer);
+          let binary = "";
+          for (let i = 0; i < bytes.byteLength; i++) {
+            binary += String.fromCharCode(bytes[i]);
+          }
+          base64Data = btoa(binary);
+        } else {
+          throw new Error(
+            "Route node requires either url or base64 input for MIME detection"
+          );
+        }
+        try {
+          detectedMimeType = await detectMimeTypeFromBase64Async(base64Data);
+        } catch {
+          try {
+            detectedMimeType = detectMimeTypeFromBase64(base64Data);
+          } catch (syncErr) {
+            throw new Error(
+              `Unable to detect MIME type from input. Ensure the document is a supported format. Error: ${syncErr instanceof Error ? syncErr.message : String(syncErr)}`
+            );
+          }
+        }
+        if (ctx?.emit) {
+          ctx.emit(`${stepId}:detectedMimeType`, detectedMimeType);
+        }
+        phase = "route";
+        selectedBranch = findMatchingBranch(detectedMimeType, routeConfig.branches);
+        if (!selectedBranch && others) {
+          selectedBranch = "others";
+        }
+        if (!selectedBranch) {
+          const availableBranches = Object.entries(routeConfig.branches).map(([name, cfg]) => `${name}: [${cfg.mimeTypes.join(", ")}]`).join("; ");
+          throw new Error(
+            `No branch matches MIME type "${detectedMimeType}". Available branches: ${availableBranches}. Add an 'others' fallback branch to handle unmatched types.`
+          );
+        }
+        if (ctx?.emit) {
+          ctx.emit(`${stepId}:selectedBranch`, selectedBranch);
+        }
+        phase = "branch";
+        const branchFlowOrRef = selectedBranch === "others" ? others : branches[selectedBranch];
+        if (!branchFlowOrRef) {
+          throw new Error(
+            `Branch "${selectedBranch}" not found in route configuration`
+          );
+        }
+        const branchFlowDef = resolveBranchFlow(branchFlowOrRef, flows);
+        const branchFlow = buildFlowFromConfig(
+          branchFlowDef,
+          providers,
+          flows,
+          ctx?.observability?.config ? {
+            observability: ctx.observability.config,
+            metadata: {
+              ...ctx.observability?.metadata,
+              parentNode: stepId,
+              phase: "branch",
+              detectedMimeType,
+              selectedBranch
+            }
+          } : void 0
+        );
+        const branchT0 = Date.now();
+        const branchResultRaw = await branchFlow.run(input);
+        if (!isSingleFlowResult(branchResultRaw)) {
+          throw new Error("Branch flow returned batch result instead of single result");
+        }
+        const branchResult = branchResultRaw;
+        if (ctx?.metrics && branchResult.metrics) {
+          const branchMetrics = flattenMetrics(
+            branchResult.metrics,
+            `${stepId}.branch.${selectedBranch}`
+          );
+          branchMetrics.forEach((m) => ctx.metrics.push(m));
+        }
+        if (ctx?.emit) {
+          ctx.emit(`${stepId}:branchOutput`, branchResult.output);
+          ctx.emit(`${stepId}:branchArtifacts`, branchResult.artifacts);
+        }
+        const branchCost = branchResult.metrics ? branchResult.metrics.reduce((sum, m) => sum + (m.costUSD ?? 0), 0) : 0;
+        const totalMs = Date.now() - t0;
+        const branchMs = branchResult.metrics ? branchResult.metrics.reduce((sum, m) => sum + (m.ms ?? 0), 0) : 0;
+        const overheadMs = totalMs - branchMs;
+        if (ctx?.metrics) {
+          const wrapperMetric = {
+            step: stepId,
+            configStepId: ctx.stepId,
+            startMs: t0,
+            provider: "internal",
+            // No external provider - uses built-in MIME detection
+            model: "mime-detection",
+            ms: totalMs,
+            costUSD: branchCost,
+            // Total cost from branch (MIME detection is free)
+            attemptNumber: 1,
+            // Composite wrappers don't retry, always 1
+            metadata: {
+              kind: "wrapper",
+              // Distinguish wrapper from leaf metrics
+              type: "route",
+              rollup: true,
+              // Duration includes child work
+              overheadMs,
+              // Pure wrapper overhead (MIME detection + routing)
+              detectedMimeType,
+              selectedBranch,
+              branchStepCount: branchResult.metrics?.length || 0,
+              branchFlowId: selectedBranch === "others" ? others && typeof others === "object" && "flowRef" in others ? others.flowRef : "inline" : typeof branches[selectedBranch] === "object" && "flowRef" in branches[selectedBranch] ? branches[selectedBranch].flowRef : "inline"
+            }
+          };
+          ctx.metrics.push(wrapperMetric);
+        }
+        return branchResult.output;
+      } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        const isNestedFlowError = err instanceof FlowExecutionError2;
+        if (ctx?.metrics) {
+          ctx.metrics.push({
+            step: stepId,
+            configStepId: ctx.stepId,
+            startMs: t0,
+            ms: Date.now() - t0,
+            costUSD: 0,
+            attemptNumber: 1,
+            // @ts-ignore - Add error field
+            error: err.message,
+            metadata: {
+              kind: "wrapper",
+              type: "route",
+              failed: true,
+              detectedMimeType,
+              selectedBranch,
+              failedPhase: phase
+            }
+          });
+        }
+        const flowPath = [{
+          stepId,
+          stepIndex: 0,
+          stepType: "route",
+          branch: selectedBranch || void 0
+        }];
+        if (isNestedFlowError && err.flowPath) {
+          flowPath.push(...err.flowPath);
+        }
+        const rootCauseMessage = isNestedFlowError ? err.getRootCause().message : err.message;
+        throw new FlowExecutionError2(
+          `Route step "${stepId}" failed${detectedMimeType ? ` (mimeType: ${detectedMimeType})` : ""}${selectedBranch ? ` (branch: ${selectedBranch})` : ""} in phase: ${phase}
+  Error: ${rootCauseMessage}`,
+          stepId,
+          0,
+          "route",
+          [],
+          isNestedFlowError ? err.originalError : err,
+          void 0,
+          flowPath,
+          isNestedFlowError ? err.allCompletedSteps : void 0
+        );
+      }
+    }
+  };
+}
 
 // src/serialization.ts
 function extractNodeMetadata(node2) {
@@ -1684,6 +1902,16 @@ function buildFlowFromConfig(flowDef, providers, flows, options) {
         flows: flows || {}
       });
       flow = flow.step(step.id, node2, step.name);
+    } else if (step.type === "route") {
+      const node2 = createRouteCompositeNode({
+        stepId: step.id,
+        routeConfig: step.config,
+        branches: step.branches,
+        others: step.others,
+        providers,
+        flows: flows || {}
+      });
+      flow = flow.step(step.id, node2, step.name);
     } else {
       const exhaustiveCheck = step;
       throw new FlowSerializationError(`Unknown step type: ${exhaustiveCheck.type}`);
@@ -1808,7 +2036,10 @@ function createNodeFromConfig(nodeType, config, providers, flows) {
       return parse({
         provider,
         consensus: cfg.consensus,
-        maxTokens: cfg.maxTokens
+        maxTokens: cfg.maxTokens,
+        promptRef: cfg.promptRef,
+        promptVariables: cfg.promptVariables,
+        additionalInstructions: cfg.additionalInstructions
       });
     }
     case "extract": {
@@ -1818,7 +2049,10 @@ function createNodeFromConfig(nodeType, config, providers, flows) {
         schema: cfg.schema,
         consensus: cfg.consensus,
         reasoning: cfg.reasoning,
-        maxTokens: cfg.maxTokens
+        maxTokens: cfg.maxTokens,
+        promptRef: cfg.promptRef,
+        promptVariables: cfg.promptVariables,
+        additionalInstructions: cfg.additionalInstructions
       });
     }
     case "split": {
@@ -1840,7 +2074,10 @@ function createNodeFromConfig(nodeType, config, providers, flows) {
         provider,
         categories: cfg.categories,
         consensus: cfg.consensus,
-        maxTokens: cfg.maxTokens
+        maxTokens: cfg.maxTokens,
+        promptRef: cfg.promptRef,
+        promptVariables: cfg.promptVariables,
+        additionalInstructions: cfg.additionalInstructions
       });
     }
     default:
@@ -1916,6 +2153,26 @@ function calculateFlowNestingDepth(flow, currentDepth = 1) {
           }
         }
       }
+    } else if (step.type === "route") {
+      const routeStep = step;
+      if (routeStep.branches) {
+        for (const branchFlowOrRef of Object.values(routeStep.branches)) {
+          if ("flowRef" in branchFlowOrRef) {
+            maxDepth = Math.max(maxDepth, stepDepth + 3);
+          } else {
+            const branchDepth = calculateFlowNestingDepth(branchFlowOrRef, stepDepth + 2);
+            maxDepth = Math.max(maxDepth, branchDepth);
+          }
+        }
+      }
+      if (routeStep.others) {
+        if ("flowRef" in routeStep.others) {
+          maxDepth = Math.max(maxDepth, stepDepth + 3);
+        } else {
+          const othersDepth = calculateFlowNestingDepth(routeStep.others, stepDepth + 2);
+          maxDepth = Math.max(maxDepth, othersDepth);
+        }
+      }
     } else if (step.type === "forEach") {
       const forEachStep = step;
       if (forEachStep.itemFlow) {
@@ -2076,8 +2333,111 @@ function validateFlow(flowDef, options = {}) {
           }
         }
       }
+    } else if (step.type === "route") {
+      const routeStep = step;
+      if (!routeStep.branches || typeof routeStep.branches !== "object") {
+        errors.push({
+          type: "invalid_config",
+          stepId,
+          message: "Route step missing or invalid branches field"
+        });
+      } else if (Object.keys(routeStep.branches).length === 0) {
+        errors.push({
+          type: "invalid_config",
+          stepId,
+          message: "Route step must have at least one branch"
+        });
+      }
+      if (routeStep.config?.branches && routeStep.branches) {
+        for (const branchName of Object.keys(routeStep.config.branches)) {
+          if (!routeStep.branches[branchName]) {
+            errors.push({
+              type: "invalid_config",
+              stepId,
+              message: `Branch "${branchName}" defined in config but missing flow definition`
+            });
+          }
+        }
+        for (const [branchName, branchConfig] of Object.entries(routeStep.config.branches)) {
+          if (!branchConfig.mimeTypes || branchConfig.mimeTypes.length === 0) {
+            errors.push({
+              type: "invalid_config",
+              stepId: `${stepId}.${branchName}`,
+              message: `Branch "${branchName}" has no MIME types defined`
+            });
+          } else {
+            for (const mimeType of branchConfig.mimeTypes) {
+              if (!mimeType.includes("/")) {
+                errors.push({
+                  type: "invalid_config",
+                  stepId: `${stepId}.${branchName}`,
+                  message: `Invalid MIME type pattern: "${mimeType}". Expected format: "type/subtype" or "type/*"`
+                });
+              }
+            }
+          }
+        }
+      }
+      if (routeStep.branches) {
+        for (const [branchName, branchFlowOrRef] of Object.entries(routeStep.branches)) {
+          if ("flowRef" in branchFlowOrRef) {
+            const flowRef = branchFlowOrRef.flowRef;
+            if (typeof flowRef !== "string" || flowRef.trim() === "") {
+              errors.push({
+                type: "invalid_config",
+                stepId: `${stepId}.${branchName}`,
+                message: `Branch "${branchName}": flowRef must be a non-empty string`
+              });
+            }
+          } else {
+            const branchResult = validateFlow(branchFlowOrRef, options);
+            for (const error of branchResult.errors) {
+              errors.push({
+                ...error,
+                stepId: `${stepId}.${branchName}`,
+                message: `Branch "${branchName}": ${error.message}`
+              });
+            }
+            for (const warning of branchResult.warnings) {
+              warnings.push({
+                ...warning,
+                stepId: `${stepId}.${branchName}`,
+                message: `Branch "${branchName}": ${warning.message}`
+              });
+            }
+          }
+        }
+      }
+      if (routeStep.others) {
+        if ("flowRef" in routeStep.others) {
+          const flowRef = routeStep.others.flowRef;
+          if (typeof flowRef !== "string" || flowRef.trim() === "") {
+            errors.push({
+              type: "invalid_config",
+              stepId: `${stepId}.others`,
+              message: "Others branch flowRef must be a non-empty string"
+            });
+          }
+        } else {
+          const othersResult = validateFlow(routeStep.others, options);
+          for (const error of othersResult.errors) {
+            errors.push({
+              ...error,
+              stepId: `${stepId}.others`,
+              message: `Others branch: ${error.message}`
+            });
+          }
+          for (const warning of othersResult.warnings) {
+            warnings.push({
+              ...warning,
+              stepId: `${stepId}.others`,
+              message: `Others branch: ${warning.message}`
+            });
+          }
+        }
+      }
     }
-    const validNodeTypes = ["parse", "extract", "split", "categorize", "trigger", "output"];
+    const validNodeTypes = ["parse", "extract", "split", "categorize", "trigger", "output", "route"];
     if (!step.nodeType || !validNodeTypes.includes(step.nodeType)) {
       errors.push({
         type: "invalid_config",
@@ -2099,7 +2459,7 @@ function validateFlow(flowDef, options = {}) {
     const hasProviderRef2 = (cfg) => {
       return "providerRef" in cfg && typeof cfg.providerRef === "string";
     };
-    if (step.nodeType !== "trigger" && step.nodeType !== "output") {
+    if (step.nodeType !== "trigger" && step.nodeType !== "output" && step.nodeType !== "route") {
       if (!hasProviderRef2(config)) {
         errors.push({
           type: "missing_provider",
@@ -2609,6 +2969,7 @@ export {
   createConditionalCompositeNode,
   createFlow,
   createForEachCompositeNode,
+  createRouteCompositeNode,
   defineFlowConfig,
   extract2 as extract,
   extractNodeMetadata,