@batchactions/distributed 0.0.2 → 0.0.4

package/README.md

@@ -1,329 +1,88 @@
 # @batchactions/distributed
 
-Distributed parallel batch processing for [@batchactions/core](https://www.npmjs.com/package/@batchactions/core). Fan out N workers (AWS Lambda, Cloud Functions, ECS tasks, etc.) to process batches concurrently with atomic claiming, crash recovery, and exactly-once completion.
+Distributed orchestration for `@batchactions` imports.
 
-## When to Use This
+Use this package when one process is not enough and you need multiple workers (Lambda, containers, queue workers) claiming and processing batches in parallel.
 
-Use `@batchactions/distributed` when:
-
-- You need to import **hundreds of thousands or millions of records** and a single process is too slow.
-- You are running in **serverless** (AWS Lambda, Google Cloud Functions) and want to parallelize across multiple invocations.
-- You need **crash resilience** — if a worker dies, another worker picks up the batch.
-
-For simpler scenarios (< 100k records, single server), `@batchactions/core` alone is sufficient. Use `processChunk()` for serverless with time limits, or `maxConcurrentBatches` for in-process parallelism.
-
-## Installation
+## Install
 
 ```bash
-npm install @batchactions/distributed
+npm install @batchactions/distributed @batchactions/core @batchactions/import
 ```
 
-**Peer dependencies:**
-
-- `@batchactions/core` >= 0.4.0
-
-You also need a `DistributedStateStore` implementation. The official one is [`@batchactions/state-sequelize`](https://www.npmjs.com/package/@batchactions/state-sequelize):
+You also need a `DistributedStateStore` implementation. Choose one:
 
 ```bash
-npm install @batchactions/state-sequelize sequelize pg
-```
-
-## How It Works
-
-A two-phase processing model:
+# Option A: Sequelize
+npm install @batchactions/state-sequelize sequelize
 
-```
-                  Phase 1: PREPARE (single orchestrator)
-                  ┌─────────────────────────────────┐
-                  │ Stream source file               │
-                  │ Validate & materialize records    │
-                  │ Create batch boundaries           │
-                  │ Save everything to StateStore     │
-                  └──────────┬──────────────────────-─┘
-                             │
-                    { jobId, totalBatches }
-                             │
-              ┌──────────────┼──────────────┐
-              ▼              ▼              ▼
-    Phase 2: PROCESS (N parallel workers)
-    ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
-    │  Worker 1    │ │  Worker 2    │ │  Worker N    │
-    │  claimBatch  │ │  claimBatch  │ │  claimBatch  │
-    │  process     │ │  process     │ │  process     │
-    │  next batch  │ │  next batch  │ │  next batch  │
-    │  ...         │ │  ...         │ │  ...         │
-    └──────────────┘ └──────────────┘ └──────────────┘
-              │              │              │
-              └──────────────┼──────────────┘
-                             │
-                    tryFinalizeJob()
-                    (exactly-once)
+# Option B: Prisma (v6 or v7)
+npm install @batchactions/state-prisma
 ```
 
-1. **Prepare** (orchestrator): Streams the source file, validates field names, materializes all records in the `StateStore`, and registers batch boundaries. Returns `{ jobId, totalBatches }`.
+## Processing Model
 
-2. **Process** (workers): Each worker calls `processWorkerBatch()` in a loop. The method atomically claims the next available batch (no two workers get the same batch), loads its records, runs the full validation + hooks + duplicate-check + processor pipeline, and marks the batch as completed or failed. When the last batch finishes, `tryFinalizeJob()` transitions the job to COMPLETED (exactly once).
+1. `prepare(source, parser)` runs once in an orchestrator process.
+2. `processWorkerBatch(jobId, processor, workerId)` runs in N workers until no batches remain.
 
 ## Quick Start
 
-### Orchestrator (Phase 1)
-
 ```typescript
 import { DistributedImport } from '@batchactions/distributed';
 import { CsvParser } from '@batchactions/import';
 import { UrlSource } from '@batchactions/core';
 import { SequelizeStateStore } from '@batchactions/state-sequelize';
-import { Sequelize } from 'sequelize';
-
-const sequelize = new Sequelize(process.env.DATABASE_URL!);
-const stateStore = new SequelizeStateStore(sequelize);
-await stateStore.initialize();
 
 const di = new DistributedImport({
   schema: {
     fields: [
       { name: 'email', type: 'email', required: true },
       { name: 'name', type: 'string', required: true },
-      { name: 'role', type: 'string', required: false, defaultValue: 'user' },
     ],
   },
   batchSize: 500,
-  stateStore,
+  stateStore: new SequelizeStateStore(sequelize),
   continueOnError: true,
 });
 
-// Phase 1: Prepare
 const source = new UrlSource('https://storage.example.com/users.csv');
-const { jobId, totalBatches, totalRecords } = await di.prepare(source, new CsvParser());
-
-console.log(`Job ${jobId}: ${totalRecords} records in ${totalBatches} batches`);
-
-// Fan out: send { jobId } to N workers via SQS, SNS, EventBridge, etc.
-await sqs.sendMessage({
-  QueueUrl: WORKER_QUEUE_URL,
-  MessageBody: JSON.stringify({ jobId }),
-});
-```
-
-### Worker (Phase 2)
-
-```typescript
-import { DistributedImport } from '@batchactions/distributed';
-import { SequelizeStateStore } from '@batchactions/state-sequelize';
-import { Sequelize } from 'sequelize';
-
-// Lambda handler
-export async function handler(event: SQSEvent, context: Context) {
-  const { jobId } = JSON.parse(event.Records[0].body);
-  const workerId = context.awsRequestId;
-
-  const sequelize = new Sequelize(process.env.DATABASE_URL!);
-  const stateStore = new SequelizeStateStore(sequelize);
-  await stateStore.initialize();
-
-  const di = new DistributedImport({
-    schema: {
-      fields: [
-        { name: 'email', type: 'email', required: true },
-        { name: 'name', type: 'string', required: true },
-        { name: 'role', type: 'string', required: false, defaultValue: 'user' },
-      ],
-    },
-    batchSize: 500,
-    stateStore,
-    continueOnError: true,
-  });
-
-  // Process batches until none remain
-  while (true) {
-    const result = await di.processWorkerBatch(jobId, async (record) => {
-      await db.query(
-        'INSERT INTO users (email, name, role) VALUES ($1, $2, $3)',
-        [record.email, record.name, record.role],
-      );
-    }, workerId);
-
-    if (!result.claimed) {
-      console.log('No more batches to process');
-      break;
-    }
-
-    console.log(`Batch ${result.batchIndex}: ${result.processedCount} processed, ${result.failedCount} failed`);
-
-    if (result.jobComplete) {
-      console.log('Job finalized by this worker!');
-      break;
-    }
-  }
-}
-```
-
-## Configuration
-
-### `DistributedImportConfig`
-
-| Property | Type | Default | Description |
-|---|---|---|---|
-| `schema` | `SchemaDefinition` | required | Field definitions and validation rules |
-| `batchSize` | `number` | `100` | Records per batch |
-| `continueOnError` | `boolean` | `true` | Continue when records fail validation or processing |
-| `stateStore` | `StateStore` | required | Must implement `DistributedStateStore` (e.g. `SequelizeStateStore`) |
-| `maxRetries` | `number` | `0` | Retry attempts for processor failures (exponential backoff) |
-| `retryDelayMs` | `number` | `1000` | Base delay in ms between retry attempts |
-| `hooks` | `JobHooks` | -- | Lifecycle hooks (`beforeValidate`, `afterValidate`, `beforeProcess`, `afterProcess`) |
-| `duplicateChecker` | `DuplicateChecker` | -- | External duplicate detection |
-| `staleBatchTimeoutMs` | `number` | `900000` | Timeout in ms before stale batches are reclaimed (15 min default) |
-
-## API Reference
-
-### `DistributedImport`
-
-| Method | Description |
-|---|---|
-| `prepare(source, parser)` | Phase 1: Stream source, materialize records, create batches. Returns `PrepareResult`. |
-| `processWorkerBatch(jobId, processor, workerId)` | Phase 2: Claim next batch, process records, finalize if last. Returns `DistributedBatchResult`. |
-| `on(event, handler)` | Subscribe to a domain event. |
-| `onAny(handler)` | Subscribe to all events. |
-| `offAny(handler)` | Unsubscribe a wildcard handler. |
-
-### `PrepareResult`
-
-| Field | Type | Description |
-|---|---|---|
-| `jobId` | `string` | Unique job identifier. Pass this to workers. |
-| `totalRecords` | `number` | Total records found in the source. |
-| `totalBatches` | `number` | Number of batches created. |
+const { jobId } = await di.prepare(source, new CsvParser());
 
-### `DistributedBatchResult`
-
-| Field | Type | Description |
-|---|---|---|
-| `claimed` | `boolean` | Whether a batch was successfully claimed. `false` means no batches remain. |
-| `batchId` | `string?` | ID of the batch that was processed. |
-| `batchIndex` | `number?` | Index of the batch that was processed. |
-| `processedCount` | `number` | Records successfully processed in this batch. |
-| `failedCount` | `number` | Records that failed in this batch. |
-| `jobComplete` | `boolean` | `true` if this worker finalized the entire job. |
-| `jobId` | `string` | The job identifier. |
-
-## Crash Recovery
-
-If a worker crashes or times out, its claimed batch becomes "stale". The next `processWorkerBatch()` call automatically reclaims stale batches (based on `staleBatchTimeoutMs`) before claiming new ones.
-
-**Requirements:**
-
-- Your **processor callback must be idempotent**. If a batch is re-processed after a crash, records may be sent to the processor again.
-- Use `ON CONFLICT DO NOTHING` / `INSERT ... IGNORE` or similar patterns in your database writes.
-
-You can also manually reclaim stale batches:
-
-```typescript
-import { isDistributedStateStore } from '@batchactions/distributed';
-
-if (isDistributedStateStore(stateStore)) {
-  const reclaimed = await stateStore.reclaimStaleBatches(jobId, 60_000); // 1 min timeout
-  console.log(`Reclaimed ${reclaimed} stale batches`);
+while (true) {
+  const result = await di.processWorkerBatch(jobId, processRecord, workerId);
+  if (!result.claimed || result.jobComplete) break;
 }
 ```
 
-## Events
-
-Each worker has its own local event bus. Subscribe to events for logging, metrics, or progress tracking:
+## Main Exports
 
-```typescript
-di.on('batch:claimed', (e) => {
-  console.log(`Worker claimed batch ${e.batchIndex} of job ${e.jobId}`);
-});
-
-di.on('record:failed', (e) => {
-  console.error(`Record ${e.recordIndex} failed: ${e.error}`);
-});
-
-di.on('import:completed', (e) => {
-  // Only emitted by the worker that finalizes the job
-  console.log(`Job complete! ${e.summary.processed} processed, ${e.summary.failed} failed`);
-});
-
-// Forward all events (e.g. to CloudWatch, Datadog)
-di.onAny((event) => {
-  metrics.emit(event.type, event);
-});
-```
+- `DistributedImport`
+- `PrepareResult`
+- `DistributedBatchResult`, `DistributedBatchConfig`
+- `DistributedStateStore` related types (re-exported)
+- `isDistributedStateStore`
 
-**Note:** `import:completed` is emitted only by the worker that finalizes the job (exactly once).
+For full typed exports, see `packages/distributed/src/index.ts`.
 
-## Architecture
+## Compatibility
 
-```
-@batchactions/distributed
-├── DistributedImport.ts          # Facade (composition root)
-├── PrepareDistributedImport.ts   # Phase 1 use case
-├── ProcessDistributedBatch.ts    # Phase 2 use case
-└── index.ts                      # Public API
-
-Depends on:
-└── @batchactions/core
-    ├── DistributedStateStore     # Port interface (extended StateStore)
-    ├── BatchReservation          # Domain types
-    ├── SchemaValidator           # Validation pipeline
-    └── EventBus                  # Event system
-
-Implemented by:
-└── @batchactions/state-sequelize
-    └── SequelizeStateStore       # Concrete DistributedStateStore
-        ├── bulkimport_jobs       # Job state table
-        ├── bulkimport_records    # Record data table
-        └── bulkimport_batches    # Batch metadata table (distributed)
-```
-
-## Implementing a Custom `DistributedStateStore`
-
-If you don't use Sequelize, you can implement the `DistributedStateStore` interface:
-
-```typescript
-import type { DistributedStateStore, ClaimBatchResult, DistributedJobStatus, ProcessedRecord } from '@batchactions/distributed';
-
-class MyDistributedStore implements DistributedStateStore {
-  // ... all StateStore methods plus:
-
-  async claimBatch(jobId: string, workerId: string): Promise<ClaimBatchResult> {
-    // Atomic: find first PENDING batch, set to PROCESSING with workerId
-    // Use SELECT FOR UPDATE SKIP LOCKED or similar
-  }
-
-  async releaseBatch(jobId: string, batchId: string, workerId: string): Promise<void> {
-    // Reset batch to PENDING (only if claimed by this worker)
-  }
-
-  async reclaimStaleBatches(jobId: string, timeoutMs: number): Promise<number> {
-    // Find PROCESSING batches with claimedAt older than timeout, reset to PENDING
-  }
-
-  async saveBatchRecords(jobId: string, batchId: string, records: readonly ProcessedRecord[]): Promise<void> {
-    // Bulk insert records for a batch
-  }
-
-  async getBatchRecords(jobId: string, batchId: string): Promise<readonly ProcessedRecord[]> {
-    // Load all records for a batch
-  }
+- Node.js >= 20.0.0
+- Peer dependencies:
+- `@batchactions/core` >= 0.0.1
+- `@batchactions/import` >= 0.0.1
 
-  async getDistributedStatus(jobId: string): Promise<DistributedJobStatus> {
-    // Aggregate: count batches by status
-  }
+## Operational Notes
 
-  async tryFinalizeJob(jobId: string): Promise<boolean> {
-    // Atomic: if all batches are terminal, set job to COMPLETED/FAILED
-    // Return true if THIS call finalized (exactly-once)
-  }
-}
-```
+- Worker processors must be idempotent.
+- Stale claimed batches are reclaimed automatically based on `staleBatchTimeoutMs`.
+- Job finalization is exactly-once via `tryFinalizeJob()` in the store.
 
-## Requirements
+## Links
 
-- Node.js >= 20.0.0
-- `@batchactions/core` >= 0.4.0
-- A `DistributedStateStore` implementation (e.g. `@batchactions/state-sequelize` >= 0.1.2)
+- Repository: https://github.com/vgpastor/batchactions/tree/main/packages/distributed
+- Issues: https://github.com/vgpastor/batchactions/issues
+- Contributing guide: https://github.com/vgpastor/batchactions/blob/main/CONTRIBUTING.md
 
 ## License
 
-[MIT](../../LICENSE)
+MIT

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@batchactions/distributed",
-  "version": "0.0.2",
-  "description": "Distributed parallel batch processing for @batchactions/core — fan-out N workers to process batches concurrently",
+  "version": "0.0.4",
+  "description": "Distributed worker orchestration for TypeScript batch imports with parallel processing, serverless fan-out, and atomic batch claiming",
   "type": "module",
   "exports": {
     ".": {
@@ -22,19 +22,30 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "typecheck": "tsc --noEmit",
-    "lint": "eslint src/ tests/"
+    "lint": "eslint src/ tests/",
+    "lint:fix": "eslint src/ tests/ --fix",
+    "format": "prettier --write .",
+    "format:check": "prettier --check ."
   },
   "keywords": [
-    "batchactions",
-    "distributed",
-    "parallel",
+    "distributed-processing",
+    "distributed-workers",
+    "worker-orchestration",
     "batch-processing",
+    "parallel-processing",
     "serverless",
     "lambda",
-    "worker"
+    "queue-workers",
+    "typescript",
+    "nodejs",
+    "etl"
   ],
   "license": "MIT",
-  "author": "Víctor Garcia <vgpastor@ingenierosweb.co>",
+  "author": "Victor Garcia <vgpastor@ingenierosweb.co>",
+  "homepage": "https://github.com/vgpastor/batchactions/tree/main/packages/distributed#readme",
+  "bugs": {
+    "url": "https://github.com/vgpastor/batchactions/issues"
+  },
   "repository": {
     "type": "git",
     "url": "https://github.com/vgpastor/batchactions",
@@ -51,12 +62,12 @@
     "@batchactions/core": "*",
     "@batchactions/import": "*",
     "@types/node": "^22.13.4",
-    "@typescript-eslint/eslint-plugin": "^8.24.1",
-    "@typescript-eslint/parser": "^8.24.1",
-    "eslint": "^9.20.0",
+    "@typescript-eslint/eslint-plugin": "^8.56.0",
+    "@typescript-eslint/parser": "^8.56.0",
+    "eslint": "^9.21.0",
     "eslint-config-prettier": "^10.0.1",
-    "tsup": "^8.3.6",
+    "tsup": "^8.5.1",
     "typescript": "^5.7.3",
-    "vitest": "^3.0.6"
+    "vitest": "^4.0.0"
   }
 }