@batchactions/state-sequelize 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +39 -70
  3. package/package.json +21 -12
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 @bulkimport/core contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,108 +1,77 @@
1
1
  # @batchactions/state-sequelize
2
2
 
3
- Sequelize-based `StateStore` and `DistributedStateStore` adapter for [@batchactions/core](https://www.npmjs.com/package/@batchactions/core).
3
+ Sequelize adapter that implements both `StateStore` and `DistributedStateStore` for `@batchactions`.
4
4
 
5
- Persists import job state, processed records, and distributed batch metadata to any relational database supported by Sequelize v6 (PostgreSQL, MySQL, MariaDB, SQLite, MS SQL Server).
5
+ Use this package to persist job state, records, and distributed batch metadata in SQL databases supported by Sequelize v6.
6
6
 
7
- ## Installation
7
+ ## Install
8
8
 
9
9
  ```bash
10
- npm install @batchactions/state-sequelize
10
+ npm install @batchactions/state-sequelize @batchactions/core sequelize
11
11
  ```
12
12
 
13
- **Peer dependencies:** `@batchactions/core` (>=0.1.0) and `sequelize` (^6.0.0) must be installed in your project.
13
+ For distributed mode:
14
14
 
15
- ## Usage
15
+ ```bash
16
+ npm install @batchactions/distributed @batchactions/import
17
+ ```
18
+
19
+ ## Quick Start
16
20
 
17
21
  ```typescript
18
- import { BulkImport, CsvParser } from '@batchactions/import';
19
- import { BufferSource } from '@batchactions/core';
20
- import { SequelizeStateStore } from '@batchactions/state-sequelize';
21
22
  import { Sequelize } from 'sequelize';
23
+ import { SequelizeStateStore } from '@batchactions/state-sequelize';
24
+ import { BulkImport, CsvParser, BufferSource } from '@batchactions/import';
22
25
 
23
- // Use your existing Sequelize instance
24
- const sequelize = new Sequelize('postgres://user:pass@localhost:5432/mydb');
25
-
26
- // Create and initialize the store (creates tables if they don't exist)
26
+ const sequelize = new Sequelize(process.env.DATABASE_URL!);
27
27
  const stateStore = new SequelizeStateStore(sequelize);
28
28
  await stateStore.initialize();
29
29
 
30
- // Pass it to BulkImport
31
30
  const importer = new BulkImport({
32
- schema: { fields: [/* ... */] },
33
- batchSize: 500,
34
- continueOnError: true,
31
+ schema: { fields: [{ name: 'email', type: 'email', required: true }] },
35
32
  stateStore,
36
33
  });
37
34
 
38
- importer.from(new BufferSource(csvString), new CsvParser());
39
-
35
+ importer.from(new BufferSource('email\nuser@example.com'), new CsvParser());
40
36
  await importer.start(async (record) => {
41
- await sequelize.models.User.create(record);
37
+ await saveUser(record);
42
38
  });
43
39
  ```
44
40
 
45
- ## Database Tables
46
-
47
- The adapter creates three tables:
48
-
49
- - **`batchactions_jobs`** -- Import job state (status, config, batches as JSON, distributed flag)
50
- - **`batchactions_records`** -- Individual processed records (status, raw/parsed data, errors)
51
- - **`batchactions_batches`** -- Batch metadata for distributed processing (status, workerId, version for optimistic locking)
52
-
53
- Tables are created automatically when you call `initialize()`. The call is idempotent.
54
-
55
- ## Distributed Processing
56
-
57
- `SequelizeStateStore` fully implements the `DistributedStateStore` interface, enabling multi-worker parallel processing with [`@batchactions/distributed`](https://www.npmjs.com/package/@batchactions/distributed).
41
+ ## Tables Created
58
42
 
59
- ```bash
60
- npm install @batchactions/distributed
61
- ```
62
-
63
- ```typescript
64
- import { DistributedImport } from '@batchactions/distributed';
65
- import { SequelizeStateStore } from '@batchactions/state-sequelize';
43
+ - `batchactions_jobs`
44
+ - `batchactions_records`
45
+ - `batchactions_batches`
66
46
 
67
- const stateStore = new SequelizeStateStore(sequelize);
68
- await stateStore.initialize();
47
+ `initialize()` is idempotent and can be called safely on startup.
69
48
 
70
- const di = new DistributedImport({
71
- schema: { fields: [/* ... */] },
72
- batchSize: 500,
73
- stateStore,
74
- });
49
+ ## Distributed Support
75
50
 
76
- // Orchestrator: prepare the job
77
- const { jobId, totalBatches } = await di.prepare(source, parser);
51
+ `SequelizeStateStore` supports:
78
52
 
79
- // Worker: claim and process batches
80
- const result = await di.processWorkerBatch(jobId, processor, workerId);
81
- ```
53
+ - Atomic batch claiming (`claimBatch`)
54
+ - Stale batch reclaiming (`reclaimStaleBatches`)
55
+ - Batch-level record persistence
56
+ - Exactly-once job finalization (`tryFinalizeJob`)
82
57
 
83
- ### Distributed Features
58
+ ## Limitations
84
59
 
85
- | Feature | Description |
86
- |---|---|
87
- | **Atomic batch claiming** | `claimBatch()` uses transactions + optimistic locking (`version` column) to ensure no two workers claim the same batch |
88
- | **Stale batch recovery** | `reclaimStaleBatches(timeoutMs)` resets batches stuck in PROCESSING beyond the timeout |
89
- | **Exactly-once finalization** | `tryFinalizeJob()` atomically transitions the job to COMPLETED/FAILED only once |
90
- | **Batch record storage** | `saveBatchRecords()` / `getBatchRecords()` for bulk record persistence |
91
- | **Distributed status** | `getDistributedStatus()` aggregates batch counts by status |
60
+ - Non-serializable schema fields (`customValidator`, `transform`, `pattern`) are stripped before persistence and must be re-injected when restoring jobs.
61
+ - SQLite is suitable for development/tests, but not recommended for high-concurrency distributed processing.
92
62
 
93
- ### Recommended Databases for Distributed Mode
63
+ ## Compatibility
94
64
 
95
- | Database | Row Locking | Recommended |
96
- |---|---|---|
97
- | PostgreSQL | `FOR UPDATE SKIP LOCKED` | Yes |
98
- | MySQL 8+ | `FOR UPDATE SKIP LOCKED` | Yes |
99
- | MariaDB 10.6+ | `FOR UPDATE SKIP LOCKED` | Yes |
100
- | SQLite | Single-writer (no concurrent transactions) | Dev/test only |
65
+ - Node.js >= 20.0.0
66
+ - Peer dependencies:
67
+ - `@batchactions/core` >= 0.0.1
68
+ - `sequelize` ^6.0.0
101
69
 
102
- ## Limitations
70
+ ## Links
103
71
 
104
- - Schema fields containing non-serializable values (`customValidator`, `transform`, `pattern`) are stripped when saving to the database. When restoring a job, the consumer must re-inject these fields.
105
- - SQLite does not support concurrent transactions, so distributed batch claiming is limited to sequential use in tests. Use PostgreSQL or MySQL for production distributed processing.
72
+ - Repository: https://github.com/vgpastor/batchactions/tree/main/packages/state-sequelize
73
+ - Issues: https://github.com/vgpastor/batchactions/issues
74
+ - Contributing guide: https://github.com/vgpastor/batchactions/blob/main/CONTRIBUTING.md
106
75
 
107
76
  ## License
108
77
 
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@batchactions/state-sequelize",
3
- "version": "0.0.2",
4
- "description": "Sequelize-based StateStore adapter for @batchactions/core",
3
+ "version": "0.0.3",
4
+ "description": "Sequelize StateStore and DistributedStateStore adapter for @batchactions/core",
5
5
  "type": "module",
6
6
  "exports": {
7
7
  ".": {
@@ -22,24 +22,32 @@
22
22
  "test": "vitest run",
23
23
  "test:watch": "vitest",
24
24
  "typecheck": "tsc --noEmit",
25
- "lint": "eslint src/ tests/"
25
+ "lint": "eslint src/ tests/",
26
+ "lint:fix": "eslint src/ tests/ --fix",
27
+ "format": "prettier --write .",
28
+ "format:check": "prettier --check ."
26
29
  },
27
30
  "keywords": [
28
31
  "batchactions",
29
32
  "sequelize",
30
33
  "state-store",
34
+ "distributed",
31
35
  "persistence",
32
36
  "database"
33
37
  ],
34
38
  "license": "MIT",
35
- "author": "Víctor Garcia <vgpastor@ingenierosweb.co>",
39
+ "author": "Victor Garcia <vgpastor@ingenierosweb.co>",
40
+ "homepage": "https://github.com/vgpastor/batchactions/tree/main/packages/state-sequelize#readme",
41
+ "bugs": {
42
+ "url": "https://github.com/vgpastor/batchactions/issues"
43
+ },
36
44
  "repository": {
37
45
  "type": "git",
38
46
  "url": "https://github.com/vgpastor/batchactions",
39
47
  "directory": "packages/state-sequelize"
40
48
  },
41
49
  "engines": {
42
- "node": ">=16.7.0"
50
+ "node": ">=20.0.0"
43
51
  },
44
52
  "peerDependencies": {
45
53
  "@batchactions/core": ">=0.0.1",
@@ -47,15 +55,16 @@
47
55
  },
48
56
  "devDependencies": {
49
57
  "@batchactions/core": "*",
58
+ "@types/better-sqlite3": "^7.6.13",
50
59
  "@types/node": "^22.13.4",
51
- "@typescript-eslint/eslint-plugin": "^8.24.1",
52
- "@typescript-eslint/parser": "^8.24.1",
53
- "eslint": "^9.20.0",
60
+ "@typescript-eslint/eslint-plugin": "^8.56.0",
61
+ "@typescript-eslint/parser": "^8.56.0",
62
+ "better-sqlite3": "^12.6.2",
63
+ "eslint": "^9.21.0",
54
64
  "eslint-config-prettier": "^10.0.1",
55
- "sequelize": "^6.37.0",
56
- "sqlite3": "^5.1.7",
57
- "tsup": "^8.3.6",
65
+ "sequelize": "^6.37.5",
66
+ "tsup": "^8.5.1",
58
67
  "typescript": "^5.7.3",
59
- "vitest": "^3.0.6"
68
+ "vitest": "^4.0.0"
60
69
  }
61
70
  }