@uploadista/core 0.0.17 → 0.0.18-beta.10

package/src/flow/flow.ts

@@ -16,14 +16,18 @@ import { Effect, Stream } from "effect";
 import { z } from "zod";
 
 import { UploadistaError } from "../errors";
+import { CircuitBreakerStoreService } from "../types/circuit-breaker-store";
 import { UploadFileDataStores } from "../types/data-store";
 import type { UploadFile } from "../types/upload-file";
+import type { CircuitBreakerConfig } from "./circuit-breaker";
+import { DistributedCircuitBreakerRegistry } from "./distributed-circuit-breaker";
 import type { FlowEdge } from "./edge";
 import { EventType } from "./event";
 import { getNodeData } from "./node";
 import { ParallelScheduler } from "./parallel-scheduler";
 import { isUploadFile } from "./type-guards";
 import type {
+  FlowCircuitBreakerConfig,
   FlowConfig,
   FlowNode,
   FlowNodeData,
@@ -325,10 +329,44 @@ export function createFlowWithSchema<
       inputSchema,
       outputSchema,
       typeChecker,
+      circuitBreaker: circuitBreakerConfig,
     } = config;
     const nodes = resolvedNodes;
     const typeValidator = new FlowTypeValidator(typeChecker);
 
+    /**
+     * Gets the circuit breaker config for a specific node.
+     * Priority: node config > flow nodeTypeOverrides > flow defaults
+     */
+    const getCircuitBreakerConfigForNode = (
+      node: FlowNode<any, any, UploadistaError>,
+    ): CircuitBreakerConfig | undefined => {
+      // Get node-level config from the resolved node
+      const nodeConfig = node.circuitBreaker as
+        | FlowCircuitBreakerConfig
+        | undefined;
+
+      // Get flow-level config for this node type (using nodeTypeId for stable identification)
+      const flowNodeTypeConfig = node.nodeTypeId
+        ? circuitBreakerConfig?.nodeTypeOverrides?.[node.nodeTypeId]
+        : undefined;
+
+      // Get flow defaults
+      const flowDefaults = circuitBreakerConfig?.defaults;
+
+      // If nothing is configured, return undefined (circuit breaker disabled)
+      if (!nodeConfig && !flowNodeTypeConfig && !flowDefaults) {
+        return undefined;
+      }
+
+      // Merge configs with priority: node > nodeTypeOverrides > defaults
+      return {
+        ...flowDefaults,
+        ...flowNodeTypeConfig,
+        ...nodeConfig,
+      } as CircuitBreakerConfig;
+    };
+
     // Build adjacency list for topological sorting
     const buildGraph = () => {
       const graph: Record<string, string[]> = {};
@@ -502,13 +540,13 @@ export function createFlowWithSchema<
       outputNodes.forEach((node: any) => {
         const result = nodeResults.get(node.id);
         if (result !== undefined) {
-          // Get the nodeType from the nodeTypes map
-          const nodeTypeId = nodeTypesMap.get(node.id);
+          // Get the outputTypeId from the node types map (set from node execution results)
+          const outputTypeId = nodeTypesMap.get(node.id);
 
           // Create TypedOutput with metadata
           typedOutputs.push({
             nodeId: node.id,
-            nodeType: nodeTypeId,
+            nodeType: outputTypeId,
             data: result,
             timestamp: new Date().toISOString(),
           });
@@ -581,6 +619,7 @@ export function createFlowWithSchema<
       nodeMap: Map<string, FlowNode<any, any, UploadistaError>>,
       jobId: string,
       clientId: string | null,
+      circuitBreakerRegistry: DistributedCircuitBreakerRegistry | null,
     ): Effect.Effect<
       {
         nodeId: string;
@@ -634,6 +673,85 @@ export function createFlowWithSchema<
         const baseDelay = node.retry?.retryDelay ?? 1000;
         const useExponentialBackoff = node.retry?.exponentialBackoff ?? true;
 
+        // Get circuit breaker configuration for this node
+        const cbConfig = getCircuitBreakerConfigForNode(node);
+        const circuitBreaker =
+          cbConfig?.enabled && node.nodeTypeId && circuitBreakerRegistry
+            ? circuitBreakerRegistry.getOrCreate(node.nodeTypeId, cbConfig)
+            : null;
+
+        // Check circuit breaker before attempting execution
+        if (circuitBreaker) {
+          const {
+            allowed,
+            state: cbState,
+            failureCount: cbFailureCount,
+          } = yield* circuitBreaker.allowRequest();
+
+          if (!allowed) {
+            const fallback = circuitBreaker.getFallback();
+
+            yield* Effect.logWarning(
+              `Circuit breaker OPEN for node type "${node.nodeTypeId}" - applying fallback`,
+            );
+
+            // Handle fallback based on configuration
+            if (fallback.type === "skip") {
+              // Skip the node but continue flow execution
+              if (onEvent) {
+                yield* onEvent({
+                  jobId,
+                  flowId,
+                  nodeId,
+                  eventType: EventType.NodeEnd,
+                  nodeName: node.name,
+                });
+              }
+
+              // For skip fallback, we need to pass through some value
+              // Get the first input as pass-through data
+              const passThruInput = nodeInputs[nodeId];
+              return {
+                nodeId,
+                result: passThruInput,
+                success: true,
+                waiting: false,
+              };
+            }
+
+            if (fallback.type === "default") {
+              // Return configured default value
+              if (onEvent) {
+                yield* onEvent({
+                  jobId,
+                  flowId,
+                  nodeId,
+                  eventType: EventType.NodeEnd,
+                  nodeName: node.name,
+                  result: fallback.value,
+                });
+              }
+              return {
+                nodeId,
+                result: fallback.value,
+                success: true,
+                waiting: false,
+              };
+            }
+
+            // Default: fail immediately
+            return yield* UploadistaError.fromCode("CIRCUIT_BREAKER_OPEN", {
+              body: `Circuit breaker is open for node type "${node.name}"`,
+              details: {
+                nodeType: node.name,
+                nodeId,
+                state: cbState,
+                failureCount: cbFailureCount,
+              },
+            }).toEffect();
+          }
+        }
+
         let retryCount = 0;
         let lastError: UploadistaError | null = null;
 
@@ -778,6 +896,11 @@ export function createFlowWithSchema<
               }
             }
 
+            // Record success with circuit breaker
+            if (circuitBreaker) {
+              yield* circuitBreaker.recordSuccess();
+            }
+
             // Emit NodeEnd event with result
             if (onEvent) {
               yield* onEvent({
@@ -804,6 +927,11 @@ export function createFlowWithSchema<
                 ? error
                 : UploadistaError.fromCode("FLOW_NODE_ERROR", { cause: error });
 
+            // Record failure with circuit breaker (on each retry attempt)
+            if (circuitBreaker) {
+              yield* circuitBreaker.recordFailure(lastError.body);
+            }
+
             // Check if we should retry
             if (retryCount < maxRetries) {
               retryCount++;
@@ -890,6 +1018,14 @@ export function createFlowWithSchema<
       UploadFileDataStores
     > => {
       return Effect.gen(function* () {
+        // Get circuit breaker store from context (optional - if not provided, circuit breakers are disabled)
+        const circuitBreakerStore = yield* Effect.serviceOption(
+          CircuitBreakerStoreService,
+        );
+        const circuitBreakerRegistry = circuitBreakerStore._tag === "Some"
+          ? new DistributedCircuitBreakerRegistry(circuitBreakerStore.value)
+          : null;
+
         // Emit FlowStart event only if starting fresh
         if (!resumeFrom && onEvent) {
           yield* onEvent({
@@ -1005,6 +1141,7 @@ export function createFlowWithSchema<
                     nodeMap,
                     jobId,
                     clientId,
+                    circuitBreakerRegistry,
                   );
 
                   return { nodeId, nodeResult };
@@ -1082,6 +1219,7 @@ export function createFlowWithSchema<
               nodeMap,
               jobId,
               clientId,
+              circuitBreakerRegistry,
             );
 
             if (nodeResult.waiting) {

package/src/flow/index.ts

@@ -1,11 +1,17 @@
+// Circuit breaker
+export * from "./circuit-breaker";
+export * from "./circuit-breaker-store";
+export * from "./distributed-circuit-breaker";
+
 // Edge types
 export type { FlowEdge } from "./edge";
 export * from "./edge";
 
 export * from "./event";
 export type { Flow, FlowData } from "./flow";
-// Type registry
-export * from "./type-registry";
+// Type registries (separate registries for input and output types)
+export * from "./input-type-registry";
+export * from "./output-type-registry";
 // Built-in node types (auto-registers on import)
 import "./node-types";
 
@@ -38,5 +44,11 @@ export * from "./types/flow-file";
 export * from "./types/flow-job";
 export * from "./types/flow-types";
 export * from "./types/run-args";
+// Dead Letter Queue types and service
+export * from "./types/dead-letter-item";
+export * from "./types/retry-policy";
+export * from "./dead-letter-queue";
 export * from "./types/type-utils";
 export * from "./utils/resolve-upload-metadata";
+// File naming utilities
+export * from "./utils/file-naming";

package/src/flow/input-type-registry.ts

@@ -0,0 +1,229 @@
+/**
+ * Input type registry for flow entry point nodes.
+ *
+ * This module provides a registry for input node type definitions. Input types
+ * describe how data enters the flow system from external sources (e.g., streaming
+ * uploads, URL fetches, webhook triggers).
+ *
+ * Input types are distinct from output types - they describe the external interface
+ * that clients use to interact with input nodes, not the data shape that flows
+ * through the system.
+ *
+ * @module flow/input-type-registry
+ * @see {@link outputTypeRegistry} for output types
+ *
+ * @example
+ * ```typescript
+ * import { inputTypeRegistry } from "@uploadista/core/flow";
+ * import { z } from "zod";
+ *
+ * // Register a custom input type
+ * inputTypeRegistry.register({
+ *   id: "webhook-input-v1",
+ *   schema: z.object({
+ *     payload: z.unknown(),
+ *     headers: z.record(z.string()),
+ *   }),
+ *   version: "1.0.0",
+ *   description: "Webhook-triggered file input",
+ * });
+ * ```
+ */
+
+import type { z } from "zod";
+import { UploadistaError } from "../errors";
+
+/**
+ * Defines a registered input type with its schema and metadata.
+ *
+ * Input type definitions describe how external clients interact with input nodes.
+ * Unlike output types, input types define the external interface (e.g., init/finalize
+ * operations for streaming uploads).
+ *
+ * @template TSchema - The Zod schema type for this input's data
+ *
+ * @property id - Unique identifier (e.g., "streaming-input-v1")
+ * @property schema - Zod schema for validating input data from clients
+ * @property version - Semantic version (e.g., "1.0.0") for tracking type evolution
+ * @property description - Human-readable explanation of what this input type does
+ */
+export interface InputTypeDefinition<TSchema = unknown> {
+  id: string;
+  schema: z.ZodSchema<TSchema>;
+  version: string;
+  description: string;
+}
+
+/**
+ * Result type for input validation operations.
+ *
+ * @template T - The expected type on successful validation
+ */
+export type InputValidationResult<T> =
+  | { success: true; data: T }
+  | { success: false; error: UploadistaError };
+
+/**
+ * Registry for input node type definitions.
+ *
+ * The InputTypeRegistry maintains a global registry of input types with their schemas
+ * and metadata. Input types describe how data enters the flow from external sources.
+ *
+ * @remarks
+ * - Use the exported `inputTypeRegistry` singleton instance
+ * - Types cannot be unregistered or modified after registration
+ * - Duplicate type IDs are rejected
+ *
+ * @example
+ * ```typescript
+ * // Register a new input type
+ * inputTypeRegistry.register({
+ *   id: "form-input-v1",
+ *   schema: formInputSchema,
+ *   version: "1.0.0",
+ *   description: "Form-based file input",
+ * });
+ *
+ * // Check if type exists
+ * if (inputTypeRegistry.has("streaming-input-v1")) {
+ *   const def = inputTypeRegistry.get("streaming-input-v1");
+ * }
+ * ```
+ */
+export class InputTypeRegistry {
+  private readonly types: Map<string, InputTypeDefinition<unknown>>;
+
+  constructor() {
+    this.types = new Map();
+  }
+
+  /**
+   * Register a new input type in the registry.
+   *
+   * @template T - The TypeScript type inferred from the Zod schema
+   * @param definition - The complete type definition including schema and metadata
+   * @throws {UploadistaError} If a type with the same ID is already registered
+   */
+  register<T>(definition: InputTypeDefinition<T>): void {
+    if (this.types.has(definition.id)) {
+      throw UploadistaError.fromCode("VALIDATION_ERROR", {
+        body: `Input type "${definition.id}" is already registered. Types cannot be modified or re-registered.`,
+        details: { typeId: definition.id },
+      });
+    }
+
+    this.types.set(definition.id, definition as InputTypeDefinition<unknown>);
+  }
+
+  /**
+   * Retrieve a registered type definition by its ID.
+   *
+   * @param id - The unique type identifier (e.g., "streaming-input-v1")
+   * @returns The type definition if found, undefined otherwise
+   */
+  get(id: string): InputTypeDefinition<unknown> | undefined {
+    return this.types.get(id);
+  }
+
+  /**
+   * List all registered input types.
+   *
+   * @returns Array of all input type definitions
+   */
+  list(): InputTypeDefinition<unknown>[] {
+    return Array.from(this.types.values());
+  }
+
+  /**
+   * Validate data against a registered type's schema.
+   *
+   * @template T - The expected TypeScript type after validation
+   * @param typeId - The ID of the registered type to validate against
+   * @param data - The data to validate
+   * @returns A result object with either the validated data or an error
+   */
+  validate<T>(typeId: string, data: unknown): InputValidationResult<T> {
+    const typeDef = this.types.get(typeId);
+
+    if (!typeDef) {
+      return {
+        success: false,
+        error: UploadistaError.fromCode("VALIDATION_ERROR", {
+          body: `Input type "${typeId}" is not registered`,
+          details: { typeId },
+        }),
+      };
+    }
+
+    try {
+      const parsed = typeDef.schema.parse(data);
+      return { success: true, data: parsed as T };
+    } catch (error) {
+      return {
+        success: false,
+        error: UploadistaError.fromCode("VALIDATION_ERROR", {
+          body: `Data validation failed for input type "${typeId}"`,
+          cause: error,
+          details: { typeId, validationErrors: error },
+        }),
+      };
+    }
+  }
+
+  /**
+   * Check if a type is registered.
+   *
+   * @param id - The unique type identifier to check
+   * @returns True if the type is registered, false otherwise
+   */
+  has(id: string): boolean {
+    return this.types.has(id);
+  }
+
+  /**
+   * Get the total number of registered types.
+   *
+   * @returns The count of registered types
+   */
+  size(): number {
+    return this.types.size;
+  }
+}
+
+/**
+ * Global singleton instance of the input type registry.
+ *
+ * Use this instance to register and access input node type definitions.
+ * Input types describe how data enters the flow from external sources.
+ *
+ * @example
+ * ```typescript
+ * import { inputTypeRegistry } from "@uploadista/core/flow";
+ *
+ * // Register a type
+ * inputTypeRegistry.register({
+ *   id: "my-input-v1",
+ *   schema: myInputSchema,
+ *   version: "1.0.0",
+ *   description: "My custom input type",
+ * });
+ *
+ * // Validate data
+ * const result = inputTypeRegistry.validate("my-input-v1", data);
+ * ```
+ */
+export const inputTypeRegistry = new InputTypeRegistry();
+
+/**
+ * Validates flow input data against a registered input type.
+ *
+ * @param typeId - The registered type ID (e.g., "streaming-input-v1")
+ * @param data - The input data to validate
+ * @returns A validation result with either the typed data or an error
+ */
+export function validateFlowInput<T = unknown>(
+  typeId: string,
+  data: unknown,
+): InputValidationResult<T> {
+  return inputTypeRegistry.validate<T>(typeId, data);
+}

package/src/flow/node-types/index.ts

@@ -4,6 +4,10 @@
  * This module automatically registers the standard input and output node types
  * when imported. These types enable type-safe result consumption in clients.
  *
+ * Input types are registered in `inputTypeRegistry` and describe how data enters
+ * the flow from external sources. Output types are registered in `outputTypeRegistry`
+ * and describe the data shapes produced by nodes.
+ *
  * @module flow/node-types
  *
  * @remarks
@@ -14,18 +18,22 @@
  * ```typescript
  * // Types are automatically registered on import
  * import "@uploadista/core/flow";
- * import { flowTypeRegistry } from "@uploadista/core/flow";
+ * import { inputTypeRegistry, outputTypeRegistry } from "@uploadista/core/flow";
  *
  * // Check registered types
- * const inputTypes = flowTypeRegistry.listByCategory("input");
- * console.log(inputTypes.map(t => t.id)); // ["storage-output-v1"]
+ * const inputTypes = inputTypeRegistry.list();
+ * console.log(inputTypes.map(t => t.id)); // ["streaming-input-v1"]
+ *
+ * const outputTypes = outputTypeRegistry.list();
+ * console.log(outputTypes.map(t => t.id)); // ["storage-output-v1", "ocr-output-v1", ...]
  * ```
  */
 
 import { z } from "zod";
 import { uploadFileSchema } from "../../types/upload-file";
+import { inputTypeRegistry } from "../input-type-registry";
 import { inputDataSchema } from "../nodes/input-node";
-import { flowTypeRegistry } from "../type-registry";
+import { outputTypeRegistry } from "../output-type-registry";
 
 /**
  * Type ID constants for built-in node types.
@@ -35,11 +43,12 @@ import { flowTypeRegistry } from "../type-registry";
  *
  * @example
  * ```typescript
- * import { STREAMING_INPUT_TYPE_ID } from "@uploadista/core/flow";
+ * import { STREAMING_INPUT_TYPE_ID, STORAGE_OUTPUT_TYPE_ID } from "@uploadista/core/flow";
  *
  * const inputNode = createFlowNode({
  *   // ... other config
- *   nodeTypeId: STREAMING_INPUT_TYPE_ID
+ *   inputTypeId: STREAMING_INPUT_TYPE_ID,
+ *   outputTypeId: STORAGE_OUTPUT_TYPE_ID,
  * });
  * ```
  */
@@ -88,7 +97,7 @@ export type ImageDescriptionOutput = z.infer<
 >;
 
 /**
- * Register streaming input node type.
+ * Register streaming input node type in inputTypeRegistry.
  *
  * This is the standard input type for flows that accept file uploads via
  * streaming chunks or direct URL fetches. It supports three operations:
@@ -96,9 +105,8 @@ export type ImageDescriptionOutput = z.infer<
  * - finalize: Complete the upload after all chunks are uploaded
  * - url: Fetch a file directly from a URL
  */
-flowTypeRegistry.register({
+inputTypeRegistry.register({
   id: STREAMING_INPUT_TYPE_ID,
-  category: "input",
   schema: inputDataSchema,
   version: "1.0.0",
   description:
@@ -106,14 +114,13 @@ flowTypeRegistry.register({
 });
 
 /**
- * Register storage output node type.
+ * Register storage output node type in outputTypeRegistry.
  *
  * This is the standard output type for flows that save files to storage backends
  * (S3, Azure, GCS, etc.). It produces UploadFile objects with final storage URLs.
  */
-flowTypeRegistry.register({
+outputTypeRegistry.register({
   id: STORAGE_OUTPUT_TYPE_ID,
-  category: "output",
   schema: uploadFileSchema,
   version: "1.0.0",
   description:
@@ -121,14 +128,13 @@ flowTypeRegistry.register({
 });
 
 /**
- * Register OCR output node type.
+ * Register OCR output node type in outputTypeRegistry.
  *
  * This output type is for document text extraction nodes that use AI/OCR to
  * extract structured text from images or PDFs.
  */
-flowTypeRegistry.register({
+outputTypeRegistry.register({
   id: OCR_OUTPUT_TYPE_ID,
-  category: "output",
   schema: ocrOutputSchema,
   version: "1.0.0",
   description:
@@ -136,19 +142,19 @@ flowTypeRegistry.register({
 });
 
 /**
- * Register image description output node type.
+ * Register image description output node type in outputTypeRegistry.
  *
  * This output type is for AI-powered image analysis nodes that generate
  * textual descriptions of image content.
  */
-flowTypeRegistry.register({
+outputTypeRegistry.register({
   id: IMAGE_DESCRIPTION_OUTPUT_TYPE_ID,
-  category: "output",
   schema: imageDescriptionOutputSchema,
   version: "1.0.0",
   description:
     "Image description output node that generates AI-powered descriptions of images",
 });
 
-// Export the registry for convenience
-export { flowTypeRegistry } from "../type-registry";
+// Export the registries for convenience
+export { inputTypeRegistry } from "../input-type-registry";
+export { outputTypeRegistry } from "../output-type-registry";