@batchactions/distributed 0.0.1

package/dist/index.d.cts

@@ -0,0 +1,143 @@
+import { StateStore, JobHooks, DataSource, RecordProcessorFn, EventType, EventPayload, DomainEvent } from '@batchactions/core';
+export { BatchReservation, ClaimBatchFailureReason, ClaimBatchResult, DistributedJobStatus, DistributedStateStore, isDistributedStateStore } from '@batchactions/core';
+import { SchemaDefinition, DuplicateChecker, SourceParser } from '@batchactions/import';
+
+/** Result of the prepare phase for distributed processing. */
+interface PrepareResult {
+    /** Unique job identifier. Use this to dispatch workers. */
+    readonly jobId: string;
+    /** Total number of records found in the source. */
+    readonly totalRecords: number;
+    /** Total number of batches created. */
+    readonly totalBatches: number;
+}
+
+/** Result of processing a single distributed batch. */
+interface DistributedBatchResult {
+    /** Whether a batch was successfully claimed. */
+    readonly claimed: boolean;
+    /** Batch ID that was processed (only if claimed). */
+    readonly batchId?: string;
+    /** Batch index that was processed (only if claimed). */
+    readonly batchIndex?: number;
+    /** Records successfully processed in this batch. */
+    readonly processedCount: number;
+    /** Records that failed in this batch. */
+    readonly failedCount: number;
+    /** Whether this worker finalized the entire job. */
+    readonly jobComplete: boolean;
+    /** The job identifier. */
+    readonly jobId: string;
+}
+/** Configuration for the distributed batch processor. */
+interface DistributedBatchConfig {
+    readonly schema: SchemaDefinition;
+    readonly stateStore: StateStore;
+    readonly continueOnError?: boolean;
+    readonly maxRetries?: number;
+    readonly retryDelayMs?: number;
+    readonly hooks?: JobHooks;
+    readonly duplicateChecker?: DuplicateChecker;
+}
+
+/** Configuration for distributed import processing. */
+interface DistributedImportConfig {
+    /** Schema definition for validation. */
+    readonly schema: SchemaDefinition;
+    /** Number of records per batch. Default: 100. */
+    readonly batchSize?: number;
+    /** Whether to continue processing on record errors. Default: true. */
+    readonly continueOnError?: boolean;
+    /**
+     * State store that implements `DistributedStateStore`.
+     * Required — must support atomic batch claiming.
+     */
+    readonly stateStore: StateStore;
+    /** Maximum retry attempts for processor failures. Default: 0. */
+    readonly maxRetries?: number;
+    /** Base delay in ms for retry backoff. Default: 1000. */
+    readonly retryDelayMs?: number;
+    /** Optional lifecycle hooks. */
+    readonly hooks?: JobHooks;
+    /** Optional external duplicate detection. */
+    readonly duplicateChecker?: DuplicateChecker;
+    /**
+     * Timeout in ms for stale batch reclamation. Default: 900000 (15 min).
+     * Batches stuck in PROCESSING longer than this are reclaimed for other workers.
+     */
+    readonly staleBatchTimeoutMs?: number;
+}
+/**
+ * Facade for distributed parallel batch processing.
+ *
+ * Two-phase processing model:
+ * 1. **Prepare** (single orchestrator): streams the source file, materializes
+ *    records in the StateStore, and registers batch boundaries.
+ * 2. **Process** (N parallel workers): each worker calls `processWorkerBatch()`
+ *    in a loop to claim and process batches until none remain.
+ *
+ * @example
+ * ```typescript
+ * // === Orchestrator Lambda ===
+ * const di = new DistributedImport(config);
+ * const { jobId, totalBatches } = await di.prepare(source, parser);
+ * // Fan out: send { jobId } to N worker Lambdas via SQS
+ *
+ * // === Worker Lambda ===
+ * const di = new DistributedImport(config);
+ * const workerId = context.awsRequestId;
+ * while (true) {
+ *   const result = await di.processWorkerBatch(jobId, processor, workerId);
+ *   if (!result.claimed || result.jobComplete) break;
+ * }
+ * ```
+ */
+declare class DistributedImport {
+    private readonly config;
+    private readonly eventBus;
+    constructor(config: DistributedImportConfig);
+    /**
+     * Phase 1: Prepare the job for distributed processing.
+     *
+     * Streams the entire source, materializes all records in the StateStore,
+     * and creates batch metadata. Call this from a single orchestrator.
+     *
+     * @param source - Data source to read from.
+     * @param parser - Parser for the source format.
+     * @returns Preparation result with jobId, totalRecords, totalBatches.
+     */
+    prepare(source: DataSource, parser: SourceParser): Promise<PrepareResult>;
+    /**
+     * Phase 2: Claim and process the next available batch.
+     *
+     * Atomically claims an unclaimed batch, loads its records from the
+     * StateStore, validates and processes them. Returns immediately if
+     * no batches are available.
+     *
+     * Before claiming, reclaims any stale batches that have been stuck
+     * in PROCESSING longer than `staleBatchTimeoutMs`.
+     *
+     * Call this from each worker in a loop until `claimed` is `false`
+     * or `jobComplete` is `true`.
+     *
+     * @param jobId - The job ID returned by `prepare()`.
+     * @param processor - Callback invoked for each valid record. Must be idempotent.
+     * @param workerId - Unique identifier for this worker (e.g. Lambda request ID).
+     * @returns Result with batch details, counts, and completion status.
+     */
+    processWorkerBatch(jobId: string, processor: RecordProcessorFn, workerId: string): Promise<DistributedBatchResult>;
+    /**
+     * Subscribe to a specific domain event type.
+     *
+     * Events are local to this `DistributedImport` instance. Each worker
+     * has its own event bus. The `job:completed` event is only emitted
+     * by the worker that finalizes the job (exactly-once).
+     */
+    on<T extends EventType>(type: T, handler: (event: EventPayload<T>) => void): this;
+    /** Subscribe to all domain events. */
+    onAny(handler: (event: DomainEvent) => void): this;
+    /** Unsubscribe a wildcard handler. */
+    offAny(handler: (event: DomainEvent) => void): this;
+}
+
+export { type DistributedBatchConfig, type DistributedBatchResult, DistributedImport, type DistributedImportConfig, type PrepareResult };

package/dist/index.d.ts

@@ -0,0 +1,143 @@
+import { StateStore, JobHooks, DataSource, RecordProcessorFn, EventType, EventPayload, DomainEvent } from '@batchactions/core';
+export { BatchReservation, ClaimBatchFailureReason, ClaimBatchResult, DistributedJobStatus, DistributedStateStore, isDistributedStateStore } from '@batchactions/core';
+import { SchemaDefinition, DuplicateChecker, SourceParser } from '@batchactions/import';
+
+/** Result of the prepare phase for distributed processing. */
+interface PrepareResult {
+    /** Unique job identifier. Use this to dispatch workers. */
+    readonly jobId: string;
+    /** Total number of records found in the source. */
+    readonly totalRecords: number;
+    /** Total number of batches created. */
+    readonly totalBatches: number;
+}
+
+/** Result of processing a single distributed batch. */
+interface DistributedBatchResult {
+    /** Whether a batch was successfully claimed. */
+    readonly claimed: boolean;
+    /** Batch ID that was processed (only if claimed). */
+    readonly batchId?: string;
+    /** Batch index that was processed (only if claimed). */
+    readonly batchIndex?: number;
+    /** Records successfully processed in this batch. */
+    readonly processedCount: number;
+    /** Records that failed in this batch. */
+    readonly failedCount: number;
+    /** Whether this worker finalized the entire job. */
+    readonly jobComplete: boolean;
+    /** The job identifier. */
+    readonly jobId: string;
+}
+/** Configuration for the distributed batch processor. */
+interface DistributedBatchConfig {
+    readonly schema: SchemaDefinition;
+    readonly stateStore: StateStore;
+    readonly continueOnError?: boolean;
+    readonly maxRetries?: number;
+    readonly retryDelayMs?: number;
+    readonly hooks?: JobHooks;
+    readonly duplicateChecker?: DuplicateChecker;
+}
+
+/** Configuration for distributed import processing. */
+interface DistributedImportConfig {
+    /** Schema definition for validation. */
+    readonly schema: SchemaDefinition;
+    /** Number of records per batch. Default: 100. */
+    readonly batchSize?: number;
+    /** Whether to continue processing on record errors. Default: true. */
+    readonly continueOnError?: boolean;
+    /**
+     * State store that implements `DistributedStateStore`.
+     * Required — must support atomic batch claiming.
+     */
+    readonly stateStore: StateStore;
+    /** Maximum retry attempts for processor failures. Default: 0. */
+    readonly maxRetries?: number;
+    /** Base delay in ms for retry backoff. Default: 1000. */
+    readonly retryDelayMs?: number;
+    /** Optional lifecycle hooks. */
+    readonly hooks?: JobHooks;
+    /** Optional external duplicate detection. */
+    readonly duplicateChecker?: DuplicateChecker;
+    /**
+     * Timeout in ms for stale batch reclamation. Default: 900000 (15 min).
+     * Batches stuck in PROCESSING longer than this are reclaimed for other workers.
+     */
+    readonly staleBatchTimeoutMs?: number;
+}
+/**
+ * Facade for distributed parallel batch processing.
+ *
+ * Two-phase processing model:
+ * 1. **Prepare** (single orchestrator): streams the source file, materializes
+ *    records in the StateStore, and registers batch boundaries.
+ * 2. **Process** (N parallel workers): each worker calls `processWorkerBatch()`
+ *    in a loop to claim and process batches until none remain.
+ *
+ * @example
+ * ```typescript
+ * // === Orchestrator Lambda ===
+ * const di = new DistributedImport(config);
+ * const { jobId, totalBatches } = await di.prepare(source, parser);
+ * // Fan out: send { jobId } to N worker Lambdas via SQS
+ *
+ * // === Worker Lambda ===
+ * const di = new DistributedImport(config);
+ * const workerId = context.awsRequestId;
+ * while (true) {
+ *   const result = await di.processWorkerBatch(jobId, processor, workerId);
+ *   if (!result.claimed || result.jobComplete) break;
+ * }
+ * ```
+ */
+declare class DistributedImport {
+    private readonly config;
+    private readonly eventBus;
+    constructor(config: DistributedImportConfig);
+    /**
+     * Phase 1: Prepare the job for distributed processing.
+     *
+     * Streams the entire source, materializes all records in the StateStore,
+     * and creates batch metadata. Call this from a single orchestrator.
+     *
+     * @param source - Data source to read from.
+     * @param parser - Parser for the source format.
+     * @returns Preparation result with jobId, totalRecords, totalBatches.
+     */
+    prepare(source: DataSource, parser: SourceParser): Promise<PrepareResult>;
+    /**
+     * Phase 2: Claim and process the next available batch.
+     *
+     * Atomically claims an unclaimed batch, loads its records from the
+     * StateStore, validates and processes them. Returns immediately if
+     * no batches are available.
+     *
+     * Before claiming, reclaims any stale batches that have been stuck
+     * in PROCESSING longer than `staleBatchTimeoutMs`.
+     *
+     * Call this from each worker in a loop until `claimed` is `false`
+     * or `jobComplete` is `true`.
+     *
+     * @param jobId - The job ID returned by `prepare()`.
+     * @param processor - Callback invoked for each valid record. Must be idempotent.
+     * @param workerId - Unique identifier for this worker (e.g. Lambda request ID).
+     * @returns Result with batch details, counts, and completion status.
+     */
+    processWorkerBatch(jobId: string, processor: RecordProcessorFn, workerId: string): Promise<DistributedBatchResult>;
+    /**
+     * Subscribe to a specific domain event type.
+     *
+     * Events are local to this `DistributedImport` instance. Each worker
+     * has its own event bus. The `job:completed` event is only emitted
+     * by the worker that finalizes the job (exactly-once).
+     */
+    on<T extends EventType>(type: T, handler: (event: EventPayload<T>) => void): this;
+    /** Subscribe to all domain events. */
+    onAny(handler: (event: DomainEvent) => void): this;
+    /** Unsubscribe a wildcard handler. */
+    offAny(handler: (event: DomainEvent) => void): this;
+}
+
+export { type DistributedBatchConfig, type DistributedBatchResult, DistributedImport, type DistributedImportConfig, type PrepareResult };