@caracal-lynx/sluice 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. package/CLAUDE.md +1822 -1822
  2. package/LICENCE-FAQ.md +74 -74
  3. package/LICENSE +92 -92
  4. package/PLUGINS.md +294 -294
  5. package/README.md +681 -681
  6. package/dist/multi-source-runner.js +16 -16
  7. package/dist/runner.js +10 -10
  8. package/package.json +98 -93
  9. package/dist/adapters/source/rest.types.d.ts +0 -15
  10. package/dist/adapters/source/rest.types.d.ts.map +0 -1
  11. package/dist/adapters/source/rest.types.js +0 -6
  12. package/dist/adapters/source/rest.types.js.map +0 -1
  13. package/dist/merge/strategies/registry.d.ts +0 -8
  14. package/dist/merge/strategies/registry.d.ts.map +0 -1
  15. package/dist/merge/strategies/registry.js +0 -19
  16. package/dist/merge/strategies/registry.js.map +0 -1
package/dist/multi-source-runner.js CHANGED
@@ -307,20 +307,20 @@ export class MultiSourcePipelineRunner extends PipelineRunner {
307
307
  if (!sinceCheck[0]?.ts) {
308
308
  throw new ConfigError(`run.incrementalSince "${incrementalSince}" is not a valid timestamp`);
309
309
  }
310
- const invalidRows = await store.query(`
311
- SELECT count(*) AS n
312
- FROM ${quoteIdent(tableName)}
313
- WHERE ${quoteIdent(incrementalField)} IS NOT NULL
314
- AND TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) IS NULL
310
+ const invalidRows = await store.query(`
311
+ SELECT count(*) AS n
312
+ FROM ${quoteIdent(tableName)}
313
+ WHERE ${quoteIdent(incrementalField)} IS NOT NULL
314
+ AND TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) IS NULL
315
315
  `);
316
316
  const invalidCount = Number(invalidRows[0]?.n ?? 0);
317
317
  if (invalidCount > 0) {
318
- const examples = await store.query(`
319
- SELECT DISTINCT ${quoteIdent(incrementalField)} AS v
320
- FROM ${quoteIdent(tableName)}
321
- WHERE ${quoteIdent(incrementalField)} IS NOT NULL
322
- AND TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) IS NULL
323
- LIMIT 5
318
+ const examples = await store.query(`
319
+ SELECT DISTINCT ${quoteIdent(incrementalField)} AS v
320
+ FROM ${quoteIdent(tableName)}
321
+ WHERE ${quoteIdent(incrementalField)} IS NOT NULL
322
+ AND TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) IS NULL
323
+ LIMIT 5
324
324
  `);
325
325
  const sampleValues = examples
326
326
  .map((e) => (e.v === null || e.v === undefined ? '' : String(e.v)))
@@ -329,11 +329,11 @@ export class MultiSourcePipelineRunner extends PipelineRunner {
329
329
  throw new ConfigError(`incrementalField "${incrementalField}" contains ${invalidCount} non-parseable timestamp value(s)` +
330
330
  (sampleValues ? ` (examples: ${sampleValues})` : ''));
331
331
  }
332
- await store.query(`
333
- CREATE OR REPLACE TABLE ${quoteIdent(tableName)} AS
334
- SELECT *
335
- FROM ${quoteIdent(tableName)}
336
- WHERE TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) >= TRY_CAST(? AS TIMESTAMP)
332
+ await store.query(`
333
+ CREATE OR REPLACE TABLE ${quoteIdent(tableName)} AS
334
+ SELECT *
335
+ FROM ${quoteIdent(tableName)}
336
+ WHERE TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) >= TRY_CAST(? AS TIMESTAMP)
337
337
  `, [incrementalSince]);
338
338
  }
339
339
  async buildProfileFromTable(store, tableName, columns) {
package/dist/runner.js CHANGED
@@ -454,21 +454,21 @@ export class PipelineRunner {
454
454
  if (!sinceCheck[0]?.ts) {
455
455
  throw new ConfigError(`run.incrementalSince "${incrementalSince}" is not a valid timestamp`);
456
456
  }
457
- const invalidRows = await store.query(`
458
- SELECT count(*) AS n
459
- FROM ${quoteIdent(tableName)}
460
- WHERE ${quoteIdent(incrementalField)} IS NOT NULL
461
- AND TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) IS NULL
457
+ const invalidRows = await store.query(`
458
+ SELECT count(*) AS n
459
+ FROM ${quoteIdent(tableName)}
460
+ WHERE ${quoteIdent(incrementalField)} IS NOT NULL
461
+ AND TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) IS NULL
462
462
  `);
463
463
  const invalidCount = Number(invalidRows[0]?.n ?? 0);
464
464
  if (invalidCount > 0) {
465
465
  throw new ConfigError(`incrementalField "${incrementalField}" contains ${invalidCount} non-parseable timestamp value(s)`);
466
466
  }
467
- await store.query(`
468
- CREATE OR REPLACE TABLE ${quoteIdent(tableName)} AS
469
- SELECT *
470
- FROM ${quoteIdent(tableName)}
471
- WHERE TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) >= TRY_CAST(? AS TIMESTAMP)
467
+ await store.query(`
468
+ CREATE OR REPLACE TABLE ${quoteIdent(tableName)} AS
469
+ SELECT *
470
+ FROM ${quoteIdent(tableName)}
471
+ WHERE TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) >= TRY_CAST(? AS TIMESTAMP)
472
472
  `, [incrementalSince]);
473
473
  }
474
474
  async materializeAcceptedRows(store, tableName, columns, rejectedRowIndices) {
package/package.json CHANGED
@@ -1,93 +1,98 @@
1
- {
2
- "name": "@caracal-lynx/sluice",
3
- "version": "0.1.2",
4
- "description": "Config-driven ETL toolkit for ERP data migrations",
5
- "license": "Elastic-2.0",
6
- "author": "Caracal Lynx Ltd. <michael.scott@caracallynx.com> (https://caracallynx.com)",
7
- "homepage": "https://github.com/caracal-lynx/sluice#readme",
8
- "repository": {
9
- "type": "git",
10
- "url": "git+https://github.com/caracal-lynx/sluice.git"
11
- },
12
- "bugs": {
13
- "url": "https://github.com/caracal-lynx/sluice/issues"
14
- },
15
- "keywords": [
16
- "etl",
17
- "data-migration",
18
- "erp",
19
- "yaml",
20
- "duckdb",
21
- "cli",
22
- "data-quality",
23
- "pipeline",
24
- "typescript"
25
- ],
26
- "type": "module",
27
- "main": "dist/index.js",
28
- "types": "dist/index.d.ts",
29
- "bin": {
30
- "sluice": "dist/cli.js"
31
- },
32
- "files": [
33
- "dist",
34
- "README.md",
35
- "LICENSE",
36
- "LICENCE-FAQ.md",
37
- "CLAUDE.md",
38
- "PLUGINS.md"
39
- ],
40
- "publishConfig": {
41
- "access": "public"
42
- },
43
- "engines": {
44
- "node": ">=24.0.0"
45
- },
46
- "scripts": {
47
- "build": "tsc -p tsconfig.json",
48
- "dev": "tsx watch src/cli.ts",
49
- "lint": "eslint src tests",
50
- "format": "prettier --write src tests",
51
- "test": "vitest run",
52
- "test:watch": "vitest",
53
- "test:cov": "vitest run --coverage",
54
- "typecheck": "tsc --noEmit",
55
- "typecheck:tsgo": "tsgo --noEmit",
56
- "sluice": "tsx src/cli.ts"
57
- },
58
- "dependencies": {
59
- "@duckdb/node-api": "^1.5.0-r.1",
60
- "axios": "^1.7.0",
61
- "axios-retry": "^4.5.0",
62
- "cli-progress": "^3.12.0",
63
- "commander": "^12.1.0",
64
- "csv-parse": "^5.5.6",
65
- "csv-stringify": "^6.5.1",
66
- "dayjs": "^1.11.13",
67
- "dotenv": "^16.4.5",
68
- "expr-eval": "^2.0.2",
69
- "js-yaml": "^4.1.0",
70
- "mssql": "^11.0.1",
71
- "pg": "^8.13.0",
72
- "picocolors": "^1.1.1",
73
- "pino": "^9.5.0",
74
- "xlsx": "^0.18.5",
75
- "zod": "^3.23.8"
76
- },
77
- "devDependencies": {
78
- "@types/cli-progress": "^3.11.6",
79
- "@types/js-yaml": "^4.0.9",
80
- "@types/mssql": "^9.1.5",
81
- "@types/node": "^24.12.2",
82
- "@types/pg": "^8.11.10",
83
- "@typescript/native-preview": "^7.0.0-dev.20260504.1",
84
- "@vitest/coverage-v8": "^2.1.0",
85
- "eslint": "^9.17.0",
86
- "pino-pretty": "^13.0.0",
87
- "prettier": "^3.4.0",
88
- "tsx": "^4.19.0",
89
- "typescript": "^6.0.3",
90
- "typescript-eslint": "^8.59.1",
91
- "vitest": "^2.1.0"
92
- }
93
- }
1
+ {
2
+ "name": "@caracal-lynx/sluice",
3
+ "version": "0.1.3",
4
+ "description": "Config-driven ETL toolkit for ERP data migrations",
5
+ "license": "Elastic-2.0",
6
+ "author": "Caracal Lynx Ltd. <michael.scott@caracallynx.com> (https://caracallynx.com)",
7
+ "homepage": "https://github.com/caracal-lynx/sluice#readme",
8
+ "repository": {
9
+ "type": "git",
10
+ "url": "git+https://github.com/caracal-lynx/sluice.git"
11
+ },
12
+ "bugs": {
13
+ "url": "https://github.com/caracal-lynx/sluice/issues"
14
+ },
15
+ "keywords": [
16
+ "etl",
17
+ "data-migration",
18
+ "erp",
19
+ "yaml",
20
+ "duckdb",
21
+ "cli",
22
+ "data-quality",
23
+ "pipeline",
24
+ "typescript"
25
+ ],
26
+ "type": "module",
27
+ "main": "dist/index.js",
28
+ "types": "dist/index.d.ts",
29
+ "bin": {
30
+ "sluice": "dist/cli.js"
31
+ },
32
+ "files": [
33
+ "dist",
34
+ "README.md",
35
+ "LICENSE",
36
+ "LICENCE-FAQ.md",
37
+ "CLAUDE.md",
38
+ "PLUGINS.md"
39
+ ],
40
+ "publishConfig": {
41
+ "access": "public"
42
+ },
43
+ "engines": {
44
+ "node": ">=24.0.0"
45
+ },
46
+ "scripts": {
47
+ "build": "tsc -p tsconfig.json",
48
+ "dev": "tsx watch src/cli.ts",
49
+ "lint": "eslint src tests",
50
+ "format": "prettier --write src tests",
51
+ "test": "vitest run",
52
+ "test:watch": "vitest",
53
+ "test:cov": "vitest run --coverage",
54
+ "typecheck": "tsc --noEmit",
55
+ "typecheck:tsgo": "tsgo --noEmit",
56
+ "sluice": "tsx src/cli.ts",
57
+ "changeset": "changeset",
58
+ "version": "changeset version && npm install --package-lock-only",
59
+ "release": "npm run build && changeset publish"
60
+ },
61
+ "dependencies": {
62
+ "@duckdb/node-api": "^1.5.0-r.1",
63
+ "axios": "^1.7.0",
64
+ "axios-retry": "^4.5.0",
65
+ "cli-progress": "^3.12.0",
66
+ "commander": "^12.1.0",
67
+ "csv-parse": "^5.5.6",
68
+ "csv-stringify": "^6.5.1",
69
+ "dayjs": "^1.11.13",
70
+ "dotenv": "^16.4.5",
71
+ "expr-eval": "^2.0.2",
72
+ "js-yaml": "^4.1.0",
73
+ "mssql": "^11.0.1",
74
+ "pg": "^8.13.0",
75
+ "picocolors": "^1.1.1",
76
+ "pino": "^9.5.0",
77
+ "xlsx": "^0.18.5",
78
+ "zod": "^3.23.8"
79
+ },
80
+ "devDependencies": {
81
+ "@changesets/changelog-github": "^0.6.0",
82
+ "@changesets/cli": "^2.31.0",
83
+ "@types/cli-progress": "^3.11.6",
84
+ "@types/js-yaml": "^4.0.9",
85
+ "@types/mssql": "^9.1.5",
86
+ "@types/node": "^24.12.2",
87
+ "@types/pg": "^8.11.10",
88
+ "@typescript/native-preview": "^7.0.0-dev.20260504.1",
89
+ "@vitest/coverage-v8": "^2.1.0",
90
+ "eslint": "^9.17.0",
91
+ "pino-pretty": "^13.0.0",
92
+ "prettier": "^3.4.0",
93
+ "tsx": "^4.19.0",
94
+ "typescript": "^6.0.3",
95
+ "typescript-eslint": "^8.59.1",
96
+ "vitest": "^2.1.0"
97
+ }
98
+ }
package/dist/adapters/source/rest.types.d.ts DELETED
@@ -1,15 +0,0 @@
1
- /**
2
- * Local re-exports of the config types most used in rest.ts. Keeps rest.ts
3
- * imports readable without pulling broadly from config.
4
- */
5
- export type { RunConfig, SourceConfig } from '../../config/types.js';
6
- export interface PaginationConfig {
7
- type: 'offset' | 'cursor' | 'page';
8
- pageSize: number;
9
- pageParam?: string;
10
- totalField?: string;
11
- dataField?: string;
12
- cursorField?: string;
13
- cursorParam?: string;
14
- }
15
- //# sourceMappingURL=rest.types.d.ts.map
package/dist/adapters/source/rest.types.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"rest.types.d.ts","sourceRoot":"","sources":["../../../src/adapters/source/rest.types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErE,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}
package/dist/adapters/source/rest.types.js DELETED
@@ -1,6 +0,0 @@
1
- /**
2
- * Local re-exports of the config types most used in rest.ts. Keeps rest.ts
3
- * imports readable without pulling broadly from config.
4
- */
5
- export {};
6
- //# sourceMappingURL=rest.types.js.map
package/dist/adapters/source/rest.types.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"rest.types.js","sourceRoot":"","sources":["../../../src/adapters/source/rest.types.ts"],"names":[],"mappings":"AAAA;;;GAGG"}
package/dist/merge/strategies/registry.d.ts DELETED
@@ -1,8 +0,0 @@
1
- /**
2
- * Merge strategy registry.
3
- * Loads all built-in merge strategies.
4
- */
5
- import type { MergeStrategyPlugin } from '../types.js';
6
- export declare function getStrategy(id: string): MergeStrategyPlugin | undefined;
7
- export declare function listStrategies(): string[];
8
- //# sourceMappingURL=registry.d.ts.map
package/dist/merge/strategies/registry.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../../src/merge/strategies/registry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAQvD,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,mBAAmB,GAAG,SAAS,CAEvE;AAED,wBAAgB,cAAc,IAAI,MAAM,EAAE,CAEzC"}
package/dist/merge/strategies/registry.js DELETED
@@ -1,19 +0,0 @@
1
- /**
2
- * Merge strategy registry.
3
- * Loads all built-in merge strategies.
4
- */
5
- import { coalesceStrategy } from './coalesce.js';
6
- import { unionStrategy } from './union.js';
7
- import { intersectStrategy } from './intersect.js';
8
- const BUILT_IN_STRATEGIES = {
9
- coalesce: coalesceStrategy,
10
- union: unionStrategy,
11
- intersect: intersectStrategy,
12
- };
13
- export function getStrategy(id) {
14
- return BUILT_IN_STRATEGIES[id];
15
- }
16
- export function listStrategies() {
17
- return Object.keys(BUILT_IN_STRATEGIES);
18
- }
19
- //# sourceMappingURL=registry.js.map
package/dist/merge/strategies/registry.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"registry.js","sourceRoot":"","sources":["../../../src/merge/strategies/registry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGnD,MAAM,mBAAmB,GAAwC;IAC/D,QAAQ,EAAE,gBAAgB;IAC1B,KAAK,EAAE,aAAa;IACpB,SAAS,EAAE,iBAAiB;CAC7B,CAAC;AAEF,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,OAAO,mBAAmB,CAAC,EAAE,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;AAC1C,CAAC"}