@batchactions/state-sequelize 0.0.2 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +41 -68
  3. package/package.json +30 -15
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 @bulkimport/core contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,108 +1,81 @@
1
1
  # @batchactions/state-sequelize
2
2
 
3
- Sequelize-based `StateStore` and `DistributedStateStore` adapter for [@batchactions/core](https://www.npmjs.com/package/@batchactions/core).
3
+ Sequelize adapter that implements both `StateStore` and `DistributedStateStore` for `@batchactions`.
4
4
 
5
- Persists import job state, processed records, and distributed batch metadata to any relational database supported by Sequelize v6 (PostgreSQL, MySQL, MariaDB, SQLite, MS SQL Server).
5
+ Use this package to persist job state, records, and distributed batch metadata in SQL databases supported by Sequelize v6.
6
6
 
7
- ## Installation
7
+ ## Install
8
8
 
9
9
  ```bash
10
- npm install @batchactions/state-sequelize
10
+ npm install @batchactions/state-sequelize @batchactions/core sequelize
11
11
  ```
12
12
 
13
- **Peer dependencies:** `@batchactions/core` (>=0.1.0) and `sequelize` (^6.0.0) must be installed in your project.
13
+ For distributed mode:
14
14
 
15
- ## Usage
15
+ ```bash
16
+ npm install @batchactions/distributed @batchactions/import
17
+ ```
18
+
19
+ ## Quick Start
16
20
 
17
21
  ```typescript
18
- import { BulkImport, CsvParser } from '@batchactions/import';
19
- import { BufferSource } from '@batchactions/core';
20
- import { SequelizeStateStore } from '@batchactions/state-sequelize';
21
22
  import { Sequelize } from 'sequelize';
23
+ import { SequelizeStateStore } from '@batchactions/state-sequelize';
24
+ import { BulkImport, CsvParser, BufferSource } from '@batchactions/import';
22
25
 
23
- // Use your existing Sequelize instance
24
- const sequelize = new Sequelize('postgres://user:pass@localhost:5432/mydb');
25
-
26
- // Create and initialize the store (creates tables if they don't exist)
26
+ const sequelize = new Sequelize(process.env.DATABASE_URL!);
27
27
  const stateStore = new SequelizeStateStore(sequelize);
28
28
  await stateStore.initialize();
29
29
 
30
- // Pass it to BulkImport
31
30
  const importer = new BulkImport({
32
- schema: { fields: [/* ... */] },
33
- batchSize: 500,
34
- continueOnError: true,
31
+ schema: { fields: [{ name: 'email', type: 'email', required: true }] },
35
32
  stateStore,
36
33
  });
37
34
 
38
- importer.from(new BufferSource(csvString), new CsvParser());
39
-
35
+ importer.from(new BufferSource('email\nuser@example.com'), new CsvParser());
40
36
  await importer.start(async (record) => {
41
- await sequelize.models.User.create(record);
37
+ await saveUser(record);
42
38
  });
43
39
  ```
44
40
 
45
- ## Database Tables
46
-
47
- The adapter creates three tables:
48
-
49
- - **`batchactions_jobs`** -- Import job state (status, config, batches as JSON, distributed flag)
50
- - **`batchactions_records`** -- Individual processed records (status, raw/parsed data, errors)
51
- - **`batchactions_batches`** -- Batch metadata for distributed processing (status, workerId, version for optimistic locking)
52
-
53
- Tables are created automatically when you call `initialize()`. The call is idempotent.
54
-
55
- ## Distributed Processing
41
+ ## Tables Created
56
42
 
57
- `SequelizeStateStore` fully implements the `DistributedStateStore` interface, enabling multi-worker parallel processing with [`@batchactions/distributed`](https://www.npmjs.com/package/@batchactions/distributed).
43
+ - `batchactions_jobs`
44
+ - `batchactions_records`
45
+ - `batchactions_batches`
58
46
 
59
- ```bash
60
- npm install @batchactions/distributed
61
- ```
47
+ `initialize()` is idempotent and can be called safely on startup.
62
48
 
63
- ```typescript
64
- import { DistributedImport } from '@batchactions/distributed';
65
- import { SequelizeStateStore } from '@batchactions/state-sequelize';
49
+ ## Distributed Support
66
50
 
67
- const stateStore = new SequelizeStateStore(sequelize);
68
- await stateStore.initialize();
51
+ `SequelizeStateStore` supports:
69
52
 
70
- const di = new DistributedImport({
71
- schema: { fields: [/* ... */] },
72
- batchSize: 500,
73
- stateStore,
74
- });
53
+ - Atomic batch claiming (`claimBatch`)
54
+ - Stale batch reclaiming (`reclaimStaleBatches`)
55
+ - Batch-level record persistence
56
+ - Exactly-once job finalization (`tryFinalizeJob`)
75
57
 
76
- // Orchestrator: prepare the job
77
- const { jobId, totalBatches } = await di.prepare(source, parser);
58
+ ## Limitations
78
59
 
79
- // Worker: claim and process batches
80
- const result = await di.processWorkerBatch(jobId, processor, workerId);
81
- ```
60
+ - Non-serializable schema fields (`customValidator`, `transform`, `pattern`) are stripped before persistence and must be re-injected when restoring jobs.
61
+ - SQLite is suitable for development/tests, but not recommended for high-concurrency distributed processing.
82
62
 
83
- ### Distributed Features
63
+ ## Compatibility
84
64
 
85
- | Feature | Description |
86
- |---|---|
87
- | **Atomic batch claiming** | `claimBatch()` uses transactions + optimistic locking (`version` column) to ensure no two workers claim the same batch |
88
- | **Stale batch recovery** | `reclaimStaleBatches(timeoutMs)` resets batches stuck in PROCESSING beyond the timeout |
89
- | **Exactly-once finalization** | `tryFinalizeJob()` atomically transitions the job to COMPLETED/FAILED only once |
90
- | **Batch record storage** | `saveBatchRecords()` / `getBatchRecords()` for bulk record persistence |
91
- | **Distributed status** | `getDistributedStatus()` aggregates batch counts by status |
65
+ - Node.js >= 20.0.0
66
+ - Peer dependencies:
67
+ - `@batchactions/core` >= 0.0.1
68
+ - `sequelize` ^6.0.0
92
69
 
93
- ### Recommended Databases for Distributed Mode
70
+ ## Alternatives
94
71
 
95
- | Database | Row Locking | Recommended |
96
- |---|---|---|
97
- | PostgreSQL | `FOR UPDATE SKIP LOCKED` | Yes |
98
- | MySQL 8+ | `FOR UPDATE SKIP LOCKED` | Yes |
99
- | MariaDB 10.6+ | `FOR UPDATE SKIP LOCKED` | Yes |
100
- | SQLite | Single-writer (no concurrent transactions) | Dev/test only |
72
+ If you prefer Prisma over Sequelize, use [`@batchactions/state-prisma`](../state-prisma/README.md) which implements the same `StateStore` and `DistributedStateStore` interfaces with Prisma v6/v7.
101
73
 
102
- ## Limitations
74
+ ## Links
103
75
 
104
- - Schema fields containing non-serializable values (`customValidator`, `transform`, `pattern`) are stripped when saving to the database. When restoring a job, the consumer must re-inject these fields.
105
- - SQLite does not support concurrent transactions, so distributed batch claiming is limited to sequential use in tests. Use PostgreSQL or MySQL for production distributed processing.
76
+ - Repository: https://github.com/vgpastor/batchactions/tree/main/packages/state-sequelize
77
+ - Issues: https://github.com/vgpastor/batchactions/issues
78
+ - Contributing guide: https://github.com/vgpastor/batchactions/blob/main/CONTRIBUTING.md
106
79
 
107
80
  ## License
108
81
 
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@batchactions/state-sequelize",
3
- "version": "0.0.2",
4
- "description": "Sequelize-based StateStore adapter for @batchactions/core",
3
+ "version": "0.0.4",
4
+ "description": "Sequelize persistence adapter for batch jobs and distributed workers with state storage, atomic claims, and job recovery",
5
5
  "type": "module",
6
6
  "exports": {
7
7
  ".": {
@@ -22,24 +22,38 @@
22
22
  "test": "vitest run",
23
23
  "test:watch": "vitest",
24
24
  "typecheck": "tsc --noEmit",
25
- "lint": "eslint src/ tests/"
25
+ "lint": "eslint src/ tests/",
26
+ "lint:fix": "eslint src/ tests/ --fix",
27
+ "format": "prettier --write .",
28
+ "format:check": "prettier --check ."
26
29
  },
27
30
  "keywords": [
28
- "batchactions",
29
31
  "sequelize",
30
32
  "state-store",
31
- "persistence",
32
- "database"
33
+ "distributed-state",
34
+ "job-persistence",
35
+ "batch-processing",
36
+ "distributed-workers",
37
+ "sql",
38
+ "mysql",
39
+ "postgresql",
40
+ "sqlite",
41
+ "typescript",
42
+ "nodejs"
33
43
  ],
34
44
  "license": "MIT",
35
- "author": "Víctor Garcia <vgpastor@ingenierosweb.co>",
45
+ "author": "Victor Garcia <vgpastor@ingenierosweb.co>",
46
+ "homepage": "https://github.com/vgpastor/batchactions/tree/main/packages/state-sequelize#readme",
47
+ "bugs": {
48
+ "url": "https://github.com/vgpastor/batchactions/issues"
49
+ },
36
50
  "repository": {
37
51
  "type": "git",
38
52
  "url": "https://github.com/vgpastor/batchactions",
39
53
  "directory": "packages/state-sequelize"
40
54
  },
41
55
  "engines": {
42
- "node": ">=16.7.0"
56
+ "node": ">=20.0.0"
43
57
  },
44
58
  "peerDependencies": {
45
59
  "@batchactions/core": ">=0.0.1",
@@ -47,15 +61,16 @@
47
61
  },
48
62
  "devDependencies": {
49
63
  "@batchactions/core": "*",
64
+ "@types/better-sqlite3": "^7.6.13",
50
65
  "@types/node": "^22.13.4",
51
- "@typescript-eslint/eslint-plugin": "^8.24.1",
52
- "@typescript-eslint/parser": "^8.24.1",
53
- "eslint": "^9.20.0",
66
+ "@typescript-eslint/eslint-plugin": "^8.56.0",
67
+ "@typescript-eslint/parser": "^8.56.0",
68
+ "better-sqlite3": "^12.6.2",
69
+ "eslint": "^9.21.0",
54
70
  "eslint-config-prettier": "^10.0.1",
55
- "sequelize": "^6.37.0",
56
- "sqlite3": "^5.1.7",
57
- "tsup": "^8.3.6",
71
+ "sequelize": "^6.37.5",
72
+ "tsup": "^8.5.1",
58
73
  "typescript": "^5.7.3",
59
- "vitest": "^3.0.6"
74
+ "vitest": "^4.0.0"
60
75
  }
61
76
  }