@caracal-lynx/sluice 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +1822 -0
- package/LICENCE-FAQ.md +74 -0
- package/LICENSE +92 -0
- package/README.md +582 -0
- package/dist/adapters/source/csv.d.ts +10 -0
- package/dist/adapters/source/csv.d.ts.map +1 -0
- package/dist/adapters/source/csv.js +110 -0
- package/dist/adapters/source/csv.js.map +1 -0
- package/dist/adapters/source/index.d.ts +9 -0
- package/dist/adapters/source/index.d.ts.map +1 -0
- package/dist/adapters/source/index.js +26 -0
- package/dist/adapters/source/index.js.map +1 -0
- package/dist/adapters/source/mssql.d.ts +11 -0
- package/dist/adapters/source/mssql.d.ts.map +1 -0
- package/dist/adapters/source/mssql.js +230 -0
- package/dist/adapters/source/mssql.js.map +1 -0
- package/dist/adapters/source/pg.d.ts +11 -0
- package/dist/adapters/source/pg.d.ts.map +1 -0
- package/dist/adapters/source/pg.js +88 -0
- package/dist/adapters/source/pg.js.map +1 -0
- package/dist/adapters/source/registry.d.ts +10 -0
- package/dist/adapters/source/registry.d.ts.map +1 -0
- package/dist/adapters/source/registry.js +36 -0
- package/dist/adapters/source/registry.js.map +1 -0
- package/dist/adapters/source/rest.d.ts +16 -0
- package/dist/adapters/source/rest.d.ts.map +1 -0
- package/dist/adapters/source/rest.js +182 -0
- package/dist/adapters/source/rest.js.map +1 -0
- package/dist/adapters/source/rest.types.d.ts +15 -0
- package/dist/adapters/source/rest.types.d.ts.map +1 -0
- package/dist/adapters/source/rest.types.js +6 -0
- package/dist/adapters/source/rest.types.js.map +1 -0
- package/dist/adapters/source/types.d.ts +23 -0
- package/dist/adapters/source/types.d.ts.map +1 -0
- package/dist/adapters/source/types.js +4 -0
- package/dist/adapters/source/types.js.map +1 -0
- package/dist/adapters/source/xlsx.d.ts +10 -0
- package/dist/adapters/source/xlsx.d.ts.map +1 -0
- package/dist/adapters/source/xlsx.js +71 -0
- package/dist/adapters/source/xlsx.js.map +1 -0
- package/dist/adapters/target/bc.d.ts +21 -0
- package/dist/adapters/target/bc.d.ts.map +1 -0
- package/dist/adapters/target/bc.js +188 -0
- package/dist/adapters/target/bc.js.map +1 -0
- package/dist/adapters/target/bluecherry.d.ts +10 -0
- package/dist/adapters/target/bluecherry.d.ts.map +1 -0
- package/dist/adapters/target/bluecherry.js +127 -0
- package/dist/adapters/target/bluecherry.js.map +1 -0
- package/dist/adapters/target/csv.d.ts +10 -0
- package/dist/adapters/target/csv.d.ts.map +1 -0
- package/dist/adapters/target/csv.js +40 -0
- package/dist/adapters/target/csv.js.map +1 -0
- package/dist/adapters/target/ifs.d.ts +10 -0
- package/dist/adapters/target/ifs.d.ts.map +1 -0
- package/dist/adapters/target/ifs.js +55 -0
- package/dist/adapters/target/ifs.js.map +1 -0
- package/dist/adapters/target/index.d.ts +8 -0
- package/dist/adapters/target/index.d.ts.map +1 -0
- package/dist/adapters/target/index.js +22 -0
- package/dist/adapters/target/index.js.map +1 -0
- package/dist/adapters/target/pg.d.ts +11 -0
- package/dist/adapters/target/pg.d.ts.map +1 -0
- package/dist/adapters/target/pg.js +103 -0
- package/dist/adapters/target/pg.js.map +1 -0
- package/dist/adapters/target/registry.d.ts +9 -0
- package/dist/adapters/target/registry.d.ts.map +1 -0
- package/dist/adapters/target/registry.js +29 -0
- package/dist/adapters/target/registry.js.map +1 -0
- package/dist/adapters/target/types.d.ts +15 -0
- package/dist/adapters/target/types.d.ts.map +1 -0
- package/dist/adapters/target/types.js +4 -0
- package/dist/adapters/target/types.js.map +1 -0
- package/dist/cli.d.ts +25 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +354 -0
- package/dist/cli.js.map +1 -0
- package/dist/config/index.d.ts +4 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +6 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/loader.d.ts +5 -0
- package/dist/config/loader.d.ts.map +1 -0
- package/dist/config/loader.js +135 -0
- package/dist/config/loader.js.map +1 -0
- package/dist/config/schema.d.ts +4162 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +263 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/config/types.d.ts +3 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +4 -0
- package/dist/config/types.js.map +1 -0
- package/dist/dq/engine.d.ts +10 -0
- package/dist/dq/engine.d.ts.map +1 -0
- package/dist/dq/engine.js +114 -0
- package/dist/dq/engine.js.map +1 -0
- package/dist/dq/index.d.ts +6 -0
- package/dist/dq/index.d.ts.map +1 -0
- package/dist/dq/index.js +6 -0
- package/dist/dq/index.js.map +1 -0
- package/dist/dq/reporter.d.ts +5 -0
- package/dist/dq/reporter.d.ts.map +1 -0
- package/dist/dq/reporter.js +41 -0
- package/dist/dq/reporter.js.map +1 -0
- package/dist/dq/rules/allowedValues.d.ts +7 -0
- package/dist/dq/rules/allowedValues.d.ts.map +1 -0
- package/dist/dq/rules/allowedValues.js +26 -0
- package/dist/dq/rules/allowedValues.js.map +1 -0
- package/dist/dq/rules/email.d.ts +7 -0
- package/dist/dq/rules/email.d.ts.map +1 -0
- package/dist/dq/rules/email.js +24 -0
- package/dist/dq/rules/email.js.map +1 -0
- package/dist/dq/rules/index.d.ts +15 -0
- package/dist/dq/rules/index.d.ts.map +1 -0
- package/dist/dq/rules/index.js +30 -0
- package/dist/dq/rules/index.js.map +1 -0
- package/dist/dq/rules/maxLength.d.ts +7 -0
- package/dist/dq/rules/maxLength.d.ts.map +1 -0
- package/dist/dq/rules/maxLength.js +25 -0
- package/dist/dq/rules/maxLength.js.map +1 -0
- package/dist/dq/rules/minMax.d.ts +11 -0
- package/dist/dq/rules/minMax.d.ts.map +1 -0
- package/dist/dq/rules/minMax.js +52 -0
- package/dist/dq/rules/minMax.js.map +1 -0
- package/dist/dq/rules/notNull.d.ts +7 -0
- package/dist/dq/rules/notNull.d.ts.map +1 -0
- package/dist/dq/rules/notNull.js +21 -0
- package/dist/dq/rules/notNull.js.map +1 -0
- package/dist/dq/rules/pattern.d.ts +7 -0
- package/dist/dq/rules/pattern.d.ts.map +1 -0
- package/dist/dq/rules/pattern.js +31 -0
- package/dist/dq/rules/pattern.js.map +1 -0
- package/dist/dq/rules/types.d.ts +6 -0
- package/dist/dq/rules/types.d.ts.map +1 -0
- package/dist/dq/rules/types.js +4 -0
- package/dist/dq/rules/types.js.map +1 -0
- package/dist/dq/rules/ukPostcode.d.ts +7 -0
- package/dist/dq/rules/ukPostcode.d.ts.map +1 -0
- package/dist/dq/rules/ukPostcode.js +24 -0
- package/dist/dq/rules/ukPostcode.js.map +1 -0
- package/dist/dq/rules/unique.d.ts +14 -0
- package/dist/dq/rules/unique.d.ts.map +1 -0
- package/dist/dq/rules/unique.js +9 -0
- package/dist/dq/rules/unique.js.map +1 -0
- package/dist/dq/types.d.ts +29 -0
- package/dist/dq/types.d.ts.map +1 -0
- package/dist/dq/types.js +4 -0
- package/dist/dq/types.js.map +1 -0
- package/dist/enrich/types.d.ts +87 -0
- package/dist/enrich/types.d.ts.map +1 -0
- package/dist/enrich/types.js +4 -0
- package/dist/enrich/types.js.map +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +17 -0
- package/dist/index.js.map +1 -0
- package/dist/merge/conflict-log.d.ts +9 -0
- package/dist/merge/conflict-log.d.ts.map +1 -0
- package/dist/merge/conflict-log.js +28 -0
- package/dist/merge/conflict-log.js.map +1 -0
- package/dist/merge/engine.d.ts +7 -0
- package/dist/merge/engine.d.ts.map +1 -0
- package/dist/merge/engine.js +19 -0
- package/dist/merge/engine.js.map +1 -0
- package/dist/merge/index.d.ts +11 -0
- package/dist/merge/index.d.ts.map +1 -0
- package/dist/merge/index.js +34 -0
- package/dist/merge/index.js.map +1 -0
- package/dist/merge/sql-builder.d.ts +19 -0
- package/dist/merge/sql-builder.d.ts.map +1 -0
- package/dist/merge/sql-builder.js +148 -0
- package/dist/merge/sql-builder.js.map +1 -0
- package/dist/merge/strategies/coalesce.d.ts +17 -0
- package/dist/merge/strategies/coalesce.d.ts.map +1 -0
- package/dist/merge/strategies/coalesce.js +77 -0
- package/dist/merge/strategies/coalesce.js.map +1 -0
- package/dist/merge/strategies/index.d.ts +5 -0
- package/dist/merge/strategies/index.d.ts.map +1 -0
- package/dist/merge/strategies/index.js +7 -0
- package/dist/merge/strategies/index.js.map +1 -0
- package/dist/merge/strategies/intersect.d.ts +17 -0
- package/dist/merge/strategies/intersect.d.ts.map +1 -0
- package/dist/merge/strategies/intersect.js +75 -0
- package/dist/merge/strategies/intersect.js.map +1 -0
- package/dist/merge/strategies/priority-override.d.ts +16 -0
- package/dist/merge/strategies/priority-override.d.ts.map +1 -0
- package/dist/merge/strategies/priority-override.js +78 -0
- package/dist/merge/strategies/priority-override.js.map +1 -0
- package/dist/merge/strategies/registry.d.ts +8 -0
- package/dist/merge/strategies/registry.d.ts.map +1 -0
- package/dist/merge/strategies/registry.js +19 -0
- package/dist/merge/strategies/registry.js.map +1 -0
- package/dist/merge/strategies/union.d.ts +15 -0
- package/dist/merge/strategies/union.d.ts.map +1 -0
- package/dist/merge/strategies/union.js +75 -0
- package/dist/merge/strategies/union.js.map +1 -0
- package/dist/merge/types.d.ts +24 -0
- package/dist/merge/types.d.ts.map +1 -0
- package/dist/merge/types.js +4 -0
- package/dist/merge/types.js.map +1 -0
- package/dist/multi-source-runner.d.ts +22 -0
- package/dist/multi-source-runner.d.ts.map +1 -0
- package/dist/multi-source-runner.js +398 -0
- package/dist/multi-source-runner.js.map +1 -0
- package/dist/plugins/index.d.ts +4 -0
- package/dist/plugins/index.d.ts.map +1 -0
- package/dist/plugins/index.js +5 -0
- package/dist/plugins/index.js.map +1 -0
- package/dist/plugins/loader.d.ts +22 -0
- package/dist/plugins/loader.d.ts.map +1 -0
- package/dist/plugins/loader.js +151 -0
- package/dist/plugins/loader.js.map +1 -0
- package/dist/plugins/registry.d.ts +25 -0
- package/dist/plugins/registry.d.ts.map +1 -0
- package/dist/plugins/registry.js +42 -0
- package/dist/plugins/registry.js.map +1 -0
- package/dist/plugins/types.d.ts +61 -0
- package/dist/plugins/types.d.ts.map +1 -0
- package/dist/plugins/types.js +4 -0
- package/dist/plugins/types.js.map +1 -0
- package/dist/runner.d.ts +97 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +520 -0
- package/dist/runner.js.map +1 -0
- package/dist/staging/index.d.ts +3 -0
- package/dist/staging/index.d.ts.map +1 -0
- package/dist/staging/index.js +5 -0
- package/dist/staging/index.js.map +1 -0
- package/dist/staging/schema.d.ts +19 -0
- package/dist/staging/schema.d.ts.map +1 -0
- package/dist/staging/schema.js +15 -0
- package/dist/staging/schema.js.map +1 -0
- package/dist/staging/store.d.ts +71 -0
- package/dist/staging/store.d.ts.map +1 -0
- package/dist/staging/store.js +270 -0
- package/dist/staging/store.js.map +1 -0
- package/dist/transform/cleanse.d.ts +2 -0
- package/dist/transform/cleanse.d.ts.map +1 -0
- package/dist/transform/cleanse.js +59 -0
- package/dist/transform/cleanse.js.map +1 -0
- package/dist/transform/engine.d.ts +10 -0
- package/dist/transform/engine.d.ts.map +1 -0
- package/dist/transform/engine.js +225 -0
- package/dist/transform/engine.js.map +1 -0
- package/dist/transform/expression.d.ts +5 -0
- package/dist/transform/expression.d.ts.map +1 -0
- package/dist/transform/expression.js +52 -0
- package/dist/transform/expression.js.map +1 -0
- package/dist/transform/index.d.ts +6 -0
- package/dist/transform/index.d.ts.map +1 -0
- package/dist/transform/index.js +7 -0
- package/dist/transform/index.js.map +1 -0
- package/dist/transform/lookup.d.ts +10 -0
- package/dist/transform/lookup.d.ts.map +1 -0
- package/dist/transform/lookup.js +66 -0
- package/dist/transform/lookup.js.map +1 -0
- package/dist/transform/types.d.ts +10 -0
- package/dist/transform/types.d.ts.map +1 -0
- package/dist/transform/types.js +4 -0
- package/dist/transform/types.js.map +1 -0
- package/dist/utils/env.d.ts +3 -0
- package/dist/utils/env.d.ts.map +1 -0
- package/dist/utils/env.js +26 -0
- package/dist/utils/env.js.map +1 -0
- package/dist/utils/errors.d.ts +26 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +39 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/index.d.ts +5 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +7 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/logger.d.ts +14 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +16 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/progress.d.ts +66 -0
- package/dist/utils/progress.d.ts.map +1 -0
- package/dist/utils/progress.js +283 -0
- package/dist/utils/progress.js.map +1 -0
- package/package.json +92 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Elastic-2.0
|
|
2
|
+
// Copyright (c) 2026 Caracal Lynx Ltd.
|
|
3
|
+
/**
|
|
4
|
+
* BlueCherry (CGS) ERP target adapter.
|
|
5
|
+
*
|
|
6
|
+
* Validates that every REQUIRED_COLUMN for the target entity exists in
|
|
7
|
+
* stg_transformed, then writes a CSV honouring:
|
|
8
|
+
* - default includeHeader: true
|
|
9
|
+
* - default dateFormat: MM/DD/YYYY
|
|
10
|
+
* - any column whose name ends with `Date` (case-insensitive) auto-formatted
|
|
11
|
+
* - nullValue for null/undefined cells
|
|
12
|
+
* - template: 'default' → REQUIRED_COLUMNS[entity] + any extra stg_transformed columns
|
|
13
|
+
* - template: '<path>' → first line of the CSV file is the column order
|
|
14
|
+
*
|
|
15
|
+
* Per plan risk #5: the required-column check runs at the top of `load()`
|
|
16
|
+
* (the `connect()` interface doesn't carry a StagingStore, so the check can't
|
|
17
|
+
* live there without changing the interface).
|
|
18
|
+
*/
|
|
19
|
+
import * as fs from 'node:fs/promises';
|
|
20
|
+
import { readFileSync } from 'node:fs';
|
|
21
|
+
import * as path from 'node:path';
|
|
22
|
+
import { stringify } from 'csv-stringify/sync';
|
|
23
|
+
import dayjs from 'dayjs';
|
|
24
|
+
import customParseFormat from 'dayjs/plugin/customParseFormat.js';
|
|
25
|
+
import { ConfigError, LoadError } from '../../utils/errors.js';
|
|
26
|
+
import { logger } from '../../utils/logger.js';
|
|
27
|
+
dayjs.extend(customParseFormat);
|
|
28
|
+
const STAGING_TABLE = 'stg_transformed';
|
|
29
|
+
// These column names are internal conventions. Verify against actual
|
|
30
|
+
// BlueCherry import documentation before running a live migration.
|
|
31
|
+
const REQUIRED_COLUMNS = Object.freeze({
|
|
32
|
+
Style: ['StyleNo', 'StyleDesc', 'Division', 'Season', 'CostPrice', 'RetailPrice', 'ActiveFlag'],
|
|
33
|
+
Vendor: ['VendorNo', 'VendorName', 'Country', 'CurrencyCode'],
|
|
34
|
+
PurchaseOrder: ['PONumber', 'VendorNo', 'Season', 'OrderDate', 'DeliveryDate'],
|
|
35
|
+
PODetail: ['PONumber', 'StyleNo', 'ColourCode', 'SizeCode', 'Quantity', 'CostPrice'],
|
|
36
|
+
Season: ['SeasonCode', 'SeasonDesc', 'StartDate', 'EndDate'],
|
|
37
|
+
ColourSize: ['StyleNo', 'ColourCode', 'ColourDesc', 'SizeCode', 'SizeDesc'],
|
|
38
|
+
});
|
|
39
|
+
function isDateColumn(name) {
|
|
40
|
+
return name.toLowerCase().endsWith('date');
|
|
41
|
+
}
|
|
42
|
+
function reformatIfDate(colName, value, outputFormat) {
|
|
43
|
+
if (value === null || value === undefined || value === '')
|
|
44
|
+
return null;
|
|
45
|
+
if (!isDateColumn(colName))
|
|
46
|
+
return String(value);
|
|
47
|
+
const parsed = dayjs(String(value));
|
|
48
|
+
if (!parsed.isValid())
|
|
49
|
+
return String(value);
|
|
50
|
+
return parsed.format(outputFormat);
|
|
51
|
+
}
|
|
52
|
+
export class BlueCherryTargetAdapter {
|
|
53
|
+
id = 'bluecherry';
|
|
54
|
+
async connect(config) {
|
|
55
|
+
// Basic config sanity — the full column check needs access to the store,
|
|
56
|
+
// so it runs inside load().
|
|
57
|
+
if (!config.entity) {
|
|
58
|
+
throw new ConfigError('bluecherry target requires `entity`');
|
|
59
|
+
}
|
|
60
|
+
if (!REQUIRED_COLUMNS[config.entity]) {
|
|
61
|
+
throw new ConfigError(`bluecherry: unknown entity "${config.entity}". Known: ${Object.keys(REQUIRED_COLUMNS).join(', ')}`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async disconnect() {
|
|
65
|
+
// nothing to release
|
|
66
|
+
}
|
|
67
|
+
async load(config, store, _runConfig, onProgress) {
|
|
68
|
+
if (!config.output) {
|
|
69
|
+
throw new LoadError('bluecherry target requires `output`');
|
|
70
|
+
}
|
|
71
|
+
if (!config.entity) {
|
|
72
|
+
throw new ConfigError('bluecherry target requires `entity`');
|
|
73
|
+
}
|
|
74
|
+
const required = REQUIRED_COLUMNS[config.entity];
|
|
75
|
+
if (!required) {
|
|
76
|
+
throw new ConfigError(`bluecherry: unknown entity "${config.entity}"`);
|
|
77
|
+
}
|
|
78
|
+
const staged = await store.columnNames(STAGING_TABLE);
|
|
79
|
+
const missing = required.filter((c) => !staged.includes(c));
|
|
80
|
+
if (missing.length > 0) {
|
|
81
|
+
throw new ConfigError(`bluecherry: stg_transformed is missing required columns for entity "${config.entity}": ${missing.join(', ')}`);
|
|
82
|
+
}
|
|
83
|
+
// Resolve column order from `template`
|
|
84
|
+
let columnOrder;
|
|
85
|
+
if (config.template === 'default' || config.template === undefined) {
|
|
86
|
+
const extras = staged.filter((c) => !required.includes(c));
|
|
87
|
+
columnOrder = [...required, ...extras];
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
try {
|
|
91
|
+
const firstLine = readFileSync(config.template, 'utf-8').split(/\r?\n/)[0] ?? '';
|
|
92
|
+
columnOrder = firstLine
|
|
93
|
+
.split(config.delimiter ?? ',')
|
|
94
|
+
.map((c) => c.trim())
|
|
95
|
+
.filter(Boolean);
|
|
96
|
+
}
|
|
97
|
+
catch (err) {
|
|
98
|
+
throw new ConfigError(`bluecherry: failed to read template "${config.template}": ${err instanceof Error ? err.message : String(err)}`, err);
|
|
99
|
+
}
|
|
100
|
+
const templateMissing = required.filter((c) => !columnOrder.includes(c));
|
|
101
|
+
if (templateMissing.length > 0) {
|
|
102
|
+
throw new ConfigError(`bluecherry: template is missing required columns: ${templateMissing.join(', ')}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const outputPath = path.resolve(config.output);
|
|
106
|
+
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
107
|
+
const dateFormat = config.dateFormat ?? 'MM/DD/YYYY';
|
|
108
|
+
const nullValue = config.nullValue;
|
|
109
|
+
const includeHeader = config.includeHeader ?? true;
|
|
110
|
+
const rows = await store.query(`SELECT * FROM "${STAGING_TABLE}"`);
|
|
111
|
+
const csvRows = [];
|
|
112
|
+
if (includeHeader)
|
|
113
|
+
csvRows.push(columnOrder);
|
|
114
|
+
for (const row of rows) {
|
|
115
|
+
csvRows.push(columnOrder.map((col) => {
|
|
116
|
+
const formatted = reformatIfDate(col, row[col], dateFormat);
|
|
117
|
+
return formatted === null ? nullValue : formatted;
|
|
118
|
+
}));
|
|
119
|
+
}
|
|
120
|
+
const csv = stringify(csvRows, { delimiter: config.delimiter ?? ',' });
|
|
121
|
+
await fs.writeFile(outputPath, csv, 'utf-8');
|
|
122
|
+
onProgress(rows.length);
|
|
123
|
+
logger.debug({ outputPath, rowsLoaded: rows.length, entity: config.entity, columns: columnOrder.length }, 'bluecherry target: load complete');
|
|
124
|
+
return { rowsLoaded: rows.length, rowsFailed: 0, outputPath };
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
//# sourceMappingURL=bluecherry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bluecherry.js","sourceRoot":"","sources":["../../../src/adapters/target/bluecherry.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,iBAAiB,MAAM,mCAAmC,CAAC;AAIlE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAG/C,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;AAEhC,MAAM,aAAa,GAAG,iBAAiB,CAAC;AAExC,qEAAqE;AACrE,mEAAmE;AACnE,MAAM,gBAAgB,GAAuC,MAAM,CAAC,MAAM,CAAC;IACzE,KAAK,EAAE,CAAC,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,CAAC;IAC/F,MAAM,EAAE,CAAC,UAAU,EAAE,YAAY,EAAE,SAAS,EAAE,cAAc,CAAC;IAC7D,aAAa,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,CAAC;IAC9E,QAAQ,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,CAAC;IACpF,MAAM,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,CAAC;IAC5D,UAAU,EAAE,CAAC,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,CAAC;CAC5E,CAAC,CAAC;AAEH,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,cAAc,CAAC,OAAe,EAAE,KAAc,EAAE,YAAoB;IAC3E,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IACvE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACpC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,OAAO,uBAAuB;IACzB,EAAE,GAAG,YAAY,CAAC;IAE3B,KAAK,CAAC,OAAO,CAAC,MAAoB;QAChC,yEAAyE;QACzE,4BAA4B;QAC5B,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,WAAW,CAAC,qCAAqC,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,WAAW,CACnB,+BAA+B,MAAM,CAAC,MAAM,aAAa,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpG,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU;QACd,qBAAqB;IACvB,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAAoB,EACpB,KAAmB,EACnB,UAAqB,EACrB,UAAkC;QAElC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,WAAW,CAAC,qCAAqC,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,WAAW,CAAC,+BAA+B,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,WAAW,CACnB,uEAAuE,MAAM,CAAC,MAAM,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC/G,CAAC;QACJ,CAAC;QAED,uCAAuC;QACvC,IAAI,WAAqB,CAAC;QAC1B,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YACnE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3D,WAAW,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACjF,WAAW,GAAG,SAAS;qBACpB,KAAK,CAAC,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC;qBAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;qBACpB,MAAM,CAAC,OAAO,CAAC,CAAC;YACrB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,WAAW,CACnB,wCAAwC,MAAM,CAAC,QAAQ,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/G,GAAG,CACJ,CAAC;YACJ,CAAC;YACD,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,WAAW,CACnB,qDAAqD,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE9D,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,YAAY,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC;QAEnD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAA0B,kBAAkB,aAAa,GAAG,CAAC,CAAC;QAC5F,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,IAAI,aAAa;YAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CACV,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;gBACtB,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,UAAU,CAAC,CAAC;gBAC5D,OAAO,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;YACpD,CAAC,CAAC,CACH,CAAC;QACJ,CAAC;QAED,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,GAAG,EAAE,CAAC,CAAC;QACvE,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QAE7C,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxB,MAAM,CAAC,KAAK,CACV,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,EAC3F,kCAAkC,CACnC,CAAC;QAEF,OAAO,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;IAChE,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { RunConfig, TargetConfig } from '../../config/types.js';
|
|
2
|
+
import type { StagingStore } from '../../staging/index.js';
|
|
3
|
+
import type { LoadResult, TargetAdapter } from './types.js';
|
|
4
|
+
export declare class CsvTargetAdapter implements TargetAdapter {
|
|
5
|
+
readonly id = "csv";
|
|
6
|
+
connect(_config: TargetConfig): Promise<void>;
|
|
7
|
+
disconnect(): Promise<void>;
|
|
8
|
+
load(config: TargetConfig, store: StagingStore, _runConfig: RunConfig, onProgress: (rows: number) => void): Promise<LoadResult>;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=csv.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../../src/adapters/target/csv.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAG3D,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAI5D,qBAAa,gBAAiB,YAAW,aAAa;IACpD,QAAQ,CAAC,EAAE,SAAS;IAEd,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7C,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,IAAI,CACR,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,YAAY,EACnB,UAAU,EAAE,SAAS,EACrB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GACjC,OAAO,CAAC,UAAU,CAAC;CAqBvB"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Elastic-2.0
|
|
2
|
+
// Copyright (c) 2026 Caracal Lynx Ltd.
|
|
3
|
+
/**
|
|
4
|
+
* Generic CSV target adapter.
|
|
5
|
+
*
|
|
6
|
+
* Reads `stg_transformed` via StagingStore.exportToCsv (DuckDB COPY).
|
|
7
|
+
* Honours target.includeHeader (default true), delimiter, nullValue.
|
|
8
|
+
*/
|
|
9
|
+
import * as fs from 'node:fs/promises';
|
|
10
|
+
import * as path from 'node:path';
|
|
11
|
+
import { LoadError } from '../../utils/errors.js';
|
|
12
|
+
import { logger } from '../../utils/logger.js';
|
|
13
|
+
const STAGING_TABLE = 'stg_transformed';
|
|
14
|
+
export class CsvTargetAdapter {
|
|
15
|
+
id = 'csv';
|
|
16
|
+
async connect(_config) {
|
|
17
|
+
// No persistent connection needed.
|
|
18
|
+
}
|
|
19
|
+
async disconnect() {
|
|
20
|
+
// Nothing to release.
|
|
21
|
+
}
|
|
22
|
+
async load(config, store, _runConfig, onProgress) {
|
|
23
|
+
if (!config.output) {
|
|
24
|
+
throw new LoadError('csv target requires `output`');
|
|
25
|
+
}
|
|
26
|
+
const outputPath = path.resolve(config.output);
|
|
27
|
+
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
28
|
+
const rowsLoaded = await store.rowCount(STAGING_TABLE);
|
|
29
|
+
const includeHeader = config.includeHeader ?? true;
|
|
30
|
+
await store.exportToCsv(STAGING_TABLE, outputPath, {
|
|
31
|
+
delimiter: config.delimiter,
|
|
32
|
+
header: includeHeader,
|
|
33
|
+
nullValue: config.nullValue,
|
|
34
|
+
});
|
|
35
|
+
onProgress(rowsLoaded);
|
|
36
|
+
logger.debug({ outputPath, rowsLoaded }, 'csv target: load complete');
|
|
37
|
+
return { rowsLoaded, rowsFailed: 0, outputPath };
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=csv.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../../src/adapters/target/csv.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAIlC,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAG/C,MAAM,aAAa,GAAG,iBAAiB,CAAC;AAExC,MAAM,OAAO,gBAAgB;IAClB,EAAE,GAAG,KAAK,CAAC;IAEpB,KAAK,CAAC,OAAO,CAAC,OAAqB;QACjC,mCAAmC;IACrC,CAAC;IAED,KAAK,CAAC,UAAU;QACd,sBAAsB;IACxB,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAAoB,EACpB,KAAmB,EACnB,UAAqB,EACrB,UAAkC;QAElC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,SAAS,CAAC,8BAA8B,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE9D,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QACvD,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC;QAEnD,MAAM,KAAK,CAAC,WAAW,CAAC,aAAa,EAAE,UAAU,EAAE;YACjD,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,aAAa;YACrB,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B,CAAC,CAAC;QAEH,UAAU,CAAC,UAAU,CAAC,CAAC;QACvB,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,EAAE,2BAA2B,CAAC,CAAC;QAEtE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;IACnD,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { RunConfig, TargetConfig } from '../../config/types.js';
|
|
2
|
+
import { type StagingStore } from '../../staging/index.js';
|
|
3
|
+
import type { LoadResult, TargetAdapter } from './types.js';
|
|
4
|
+
export declare class IfsTargetAdapter implements TargetAdapter {
|
|
5
|
+
readonly id = "ifs";
|
|
6
|
+
connect(_config: TargetConfig): Promise<void>;
|
|
7
|
+
disconnect(): Promise<void>;
|
|
8
|
+
load(config: TargetConfig, store: StagingStore, _runConfig: RunConfig, onProgress: (rows: number) => void): Promise<LoadResult>;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=ifs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ifs.d.ts","sourceRoot":"","sources":["../../../src/adapters/target/ifs.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,EAAc,KAAK,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGvE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAI5D,qBAAa,gBAAiB,YAAW,aAAa;IACpD,QAAQ,CAAC,EAAE,SAAS;IAEd,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7C,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,IAAI,CACR,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,YAAY,EACnB,UAAU,EAAE,SAAS,EACrB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GACjC,OAAO,CAAC,UAAU,CAAC;CAyCvB"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Elastic-2.0
|
|
2
|
+
// Copyright (c) 2026 Caracal Lynx Ltd.
|
|
3
|
+
/**
|
|
4
|
+
* IFS ERP target adapter.
|
|
5
|
+
*
|
|
6
|
+
* IFS bulk import expects a UTF-8 CSV, typically without a header row, with
|
|
7
|
+
* columns in a fixed order the downstream loader understands. This adapter
|
|
8
|
+
* emits the CSV via DuckDB's COPY with an explicit SELECT list when
|
|
9
|
+
* `columnOrder` is set.
|
|
10
|
+
*/
|
|
11
|
+
import * as fs from 'node:fs/promises';
|
|
12
|
+
import * as path from 'node:path';
|
|
13
|
+
import { quoteIdent } from '../../staging/index.js';
|
|
14
|
+
import { LoadError } from '../../utils/errors.js';
|
|
15
|
+
import { logger } from '../../utils/logger.js';
|
|
16
|
+
const STAGING_TABLE = 'stg_transformed';
|
|
17
|
+
export class IfsTargetAdapter {
|
|
18
|
+
id = 'ifs';
|
|
19
|
+
async connect(_config) {
|
|
20
|
+
// Nothing to establish.
|
|
21
|
+
}
|
|
22
|
+
async disconnect() {
|
|
23
|
+
// Nothing to release.
|
|
24
|
+
}
|
|
25
|
+
async load(config, store, _runConfig, onProgress) {
|
|
26
|
+
if (!config.output) {
|
|
27
|
+
throw new LoadError('ifs target requires `output`');
|
|
28
|
+
}
|
|
29
|
+
const outputPath = path.resolve(config.output);
|
|
30
|
+
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
31
|
+
const includeHeader = config.includeHeader ?? false; // IFS default: no header
|
|
32
|
+
const rowsLoaded = await store.rowCount(STAGING_TABLE);
|
|
33
|
+
const allColumns = await store.columnNames(STAGING_TABLE);
|
|
34
|
+
let orderedColumns;
|
|
35
|
+
if (config.columnOrder && config.columnOrder.length > 0) {
|
|
36
|
+
const missing = config.columnOrder.filter((c) => !allColumns.includes(c));
|
|
37
|
+
if (missing.length > 0) {
|
|
38
|
+
throw new LoadError(`ifs: columnOrder references columns not present in stg_transformed: ${missing.join(', ')}`);
|
|
39
|
+
}
|
|
40
|
+
orderedColumns = config.columnOrder;
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
orderedColumns = allColumns;
|
|
44
|
+
}
|
|
45
|
+
const safePath = outputPath.replace(/\\/g, '/').replace(/'/g, "''");
|
|
46
|
+
const safeDelim = (config.delimiter ?? ',').replace(/'/g, "''");
|
|
47
|
+
const safeNull = config.nullValue.replace(/'/g, "''");
|
|
48
|
+
const selectList = orderedColumns.map(quoteIdent).join(', ');
|
|
49
|
+
await store.query(`COPY (SELECT ${selectList} FROM ${quoteIdent(STAGING_TABLE)}) TO '${safePath}' (HEADER ${includeHeader ? 'TRUE' : 'FALSE'}, DELIMITER '${safeDelim}', NULL '${safeNull}')`);
|
|
50
|
+
onProgress(rowsLoaded);
|
|
51
|
+
logger.debug({ outputPath, rowsLoaded, columns: orderedColumns.length }, 'ifs target: load complete');
|
|
52
|
+
return { rowsLoaded, rowsFailed: 0, outputPath };
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=ifs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ifs.js","sourceRoot":"","sources":["../../../src/adapters/target/ifs.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAGlC,OAAO,EAAE,UAAU,EAAqB,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAG/C,MAAM,aAAa,GAAG,iBAAiB,CAAC;AAExC,MAAM,OAAO,gBAAgB;IAClB,EAAE,GAAG,KAAK,CAAC;IAEpB,KAAK,CAAC,OAAO,CAAC,OAAqB;QACjC,wBAAwB;IAC1B,CAAC;IAED,KAAK,CAAC,UAAU;QACd,sBAAsB;IACxB,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAAoB,EACpB,KAAmB,EACnB,UAAqB,EACrB,UAAkC;QAElC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,SAAS,CAAC,8BAA8B,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE9D,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC,yBAAyB;QAC9E,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAE1D,IAAI,cAAwB,CAAC;QAC7B,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,SAAS,CACjB,uEAAuE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5F,CAAC;YACJ,CAAC;YACD,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,UAAU,CAAC;QAC9B,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACtD,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7D,MAAM,KAAK,CAAC,KAAK,CACf,gBAAgB,UAAU,SAAS,UAAU,CAAC,aAAa,CAAC,SAAS,QAAQ,aAAa,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,gBAAgB,SAAS,YAAY,QAAQ,IAAI,CAC5K,CAAC;QAEF,UAAU,CAAC,UAAU,CAAC,CAAC;QACvB,MAAM,CAAC,KAAK,CACV,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,cAAc,CAAC,MAAM,EAAE,EAC1D,2BAA2B,CAC5B,CAAC;QAEF,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;IACnD,CAAC;CACF"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { BcTargetAdapter, BcTokenManager } from './bc.js';
|
|
2
|
+
export { BlueCherryTargetAdapter } from './bluecherry.js';
|
|
3
|
+
export { CsvTargetAdapter } from './csv.js';
|
|
4
|
+
export { IfsTargetAdapter } from './ifs.js';
|
|
5
|
+
export { PgTargetAdapter } from './pg.js';
|
|
6
|
+
export { TargetAdapterRegistry } from './registry.js';
|
|
7
|
+
export type { LoadResult, TargetAdapter } from './types.js';
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/adapters/target/index.ts"],"names":[],"mappings":"AAkBA,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACtD,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Elastic-2.0
|
|
2
|
+
// Copyright (c) 2026 Caracal Lynx Ltd.
|
|
3
|
+
import { BcTargetAdapter } from './bc.js';
|
|
4
|
+
import { BlueCherryTargetAdapter } from './bluecherry.js';
|
|
5
|
+
import { CsvTargetAdapter } from './csv.js';
|
|
6
|
+
import { IfsTargetAdapter } from './ifs.js';
|
|
7
|
+
import { PgTargetAdapter } from './pg.js';
|
|
8
|
+
import { TargetAdapterRegistry } from './registry.js';
|
|
9
|
+
if (!TargetAdapterRegistry.has('csv')) {
|
|
10
|
+
TargetAdapterRegistry.register(new CsvTargetAdapter());
|
|
11
|
+
TargetAdapterRegistry.register(new IfsTargetAdapter());
|
|
12
|
+
TargetAdapterRegistry.register(new BlueCherryTargetAdapter());
|
|
13
|
+
TargetAdapterRegistry.register(new PgTargetAdapter());
|
|
14
|
+
TargetAdapterRegistry.register(new BcTargetAdapter());
|
|
15
|
+
}
|
|
16
|
+
export { BcTargetAdapter, BcTokenManager } from './bc.js';
|
|
17
|
+
export { BlueCherryTargetAdapter } from './bluecherry.js';
|
|
18
|
+
export { CsvTargetAdapter } from './csv.js';
|
|
19
|
+
export { IfsTargetAdapter } from './ifs.js';
|
|
20
|
+
export { PgTargetAdapter } from './pg.js';
|
|
21
|
+
export { TargetAdapterRegistry } from './registry.js';
|
|
22
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/adapters/target/index.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAEtD,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;IACtC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,gBAAgB,EAAE,CAAC,CAAC;IACvD,qBAAqB,CAAC,QAAQ,CAAC,IAAI,gBAAgB,EAAE,CAAC,CAAC;IACvD,qBAAqB,CAAC,QAAQ,CAAC,IAAI,uBAAuB,EAAE,CAAC,CAAC;IAC9D,qBAAqB,CAAC,QAAQ,CAAC,IAAI,eAAe,EAAE,CAAC,CAAC;IACtD,qBAAqB,CAAC,QAAQ,CAAC,IAAI,eAAe,EAAE,CAAC,CAAC;AACxD,CAAC;AAED,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { RunConfig, TargetConfig } from '../../config/types.js';
|
|
2
|
+
import { type StagingStore } from '../../staging/index.js';
|
|
3
|
+
import type { LoadResult, TargetAdapter } from './types.js';
|
|
4
|
+
export declare class PgTargetAdapter implements TargetAdapter {
|
|
5
|
+
readonly id = "pg";
|
|
6
|
+
private pool;
|
|
7
|
+
connect(config: TargetConfig): Promise<void>;
|
|
8
|
+
disconnect(): Promise<void>;
|
|
9
|
+
load(config: TargetConfig, store: StagingStore, runConfig: RunConfig, onProgress: (rows: number) => void): Promise<LoadResult>;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=pg.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pg.d.ts","sourceRoot":"","sources":["../../../src/adapters/target/pg.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,EAAc,KAAK,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGvE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAI5D,qBAAa,eAAgB,YAAW,aAAa;IACnD,QAAQ,CAAC,EAAE,QAAQ;IACnB,OAAO,CAAC,IAAI,CAAwB;IAE9B,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB5C,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,IAAI,CACR,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,YAAY,EACnB,SAAS,EAAE,SAAS,EACpB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GACjC,OAAO,CAAC,UAAU,CAAC;CAuEvB"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Elastic-2.0
|
|
2
|
+
// Copyright (c) 2026 Caracal Lynx Ltd.
|
|
3
|
+
/**
|
|
4
|
+
* PostgreSQL target adapter. Inserts rows from stg_transformed into
|
|
5
|
+
* `{schema}.{table}`, honouring `onConflict`:
|
|
6
|
+
* - fail → plain INSERT (PG raises on unique violations)
|
|
7
|
+
* - upsert → INSERT … ON CONFLICT (upsertKey) DO UPDATE SET …
|
|
8
|
+
* - ignore → INSERT … ON CONFLICT DO NOTHING
|
|
9
|
+
*/
|
|
10
|
+
import pg from 'pg';
|
|
11
|
+
import { quoteIdent } from '../../staging/index.js';
|
|
12
|
+
import { ConfigError, LoadError } from '../../utils/errors.js';
|
|
13
|
+
import { logger } from '../../utils/logger.js';
|
|
14
|
+
const STAGING_TABLE = 'stg_transformed';
|
|
15
|
+
export class PgTargetAdapter {
|
|
16
|
+
id = 'pg';
|
|
17
|
+
pool = null;
|
|
18
|
+
async connect(config) {
|
|
19
|
+
if (!config.connection) {
|
|
20
|
+
throw new ConfigError('pg target requires `connection`');
|
|
21
|
+
}
|
|
22
|
+
this.pool = new pg.Pool({ connectionString: config.connection });
|
|
23
|
+
try {
|
|
24
|
+
const client = await this.pool.connect();
|
|
25
|
+
client.release();
|
|
26
|
+
}
|
|
27
|
+
catch (err) {
|
|
28
|
+
this.pool = null;
|
|
29
|
+
throw new LoadError(`pg target: connect failed: ${err instanceof Error ? err.message : String(err)}`, err);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
async disconnect() {
|
|
33
|
+
if (this.pool) {
|
|
34
|
+
const p = this.pool;
|
|
35
|
+
this.pool = null;
|
|
36
|
+
await p.end();
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
async load(config, store, runConfig, onProgress) {
|
|
40
|
+
if (!this.pool)
|
|
41
|
+
throw new LoadError('pg target: not connected');
|
|
42
|
+
if (!config.table)
|
|
43
|
+
throw new ConfigError('pg target requires `table`');
|
|
44
|
+
const schema = config.schema;
|
|
45
|
+
const table = config.table;
|
|
46
|
+
const qualified = `${quoteIdent(schema)}.${quoteIdent(table)}`;
|
|
47
|
+
const columns = await store.columnNames(STAGING_TABLE);
|
|
48
|
+
if (columns.length === 0) {
|
|
49
|
+
throw new LoadError('pg target: stg_transformed has no columns');
|
|
50
|
+
}
|
|
51
|
+
const rows = await store.query(`SELECT * FROM "${STAGING_TABLE}"`);
|
|
52
|
+
const colList = columns.map(quoteIdent).join(', ');
|
|
53
|
+
let conflictClause = '';
|
|
54
|
+
if (config.onConflict === 'upsert') {
|
|
55
|
+
const key = config.upsertKey;
|
|
56
|
+
if (!key || key.length === 0) {
|
|
57
|
+
throw new ConfigError('pg target: onConflict: upsert requires upsertKey');
|
|
58
|
+
}
|
|
59
|
+
const updateSet = columns
|
|
60
|
+
.filter((c) => !key.includes(c))
|
|
61
|
+
.map((c) => `${quoteIdent(c)} = EXCLUDED.${quoteIdent(c)}`)
|
|
62
|
+
.join(', ');
|
|
63
|
+
const conflictCols = key.map(quoteIdent).join(', ');
|
|
64
|
+
conflictClause = updateSet.length > 0
|
|
65
|
+
? `ON CONFLICT (${conflictCols}) DO UPDATE SET ${updateSet}`
|
|
66
|
+
: `ON CONFLICT (${conflictCols}) DO NOTHING`;
|
|
67
|
+
}
|
|
68
|
+
else if (config.onConflict === 'ignore') {
|
|
69
|
+
conflictClause = 'ON CONFLICT DO NOTHING';
|
|
70
|
+
}
|
|
71
|
+
let rowsLoaded = 0;
|
|
72
|
+
let rowsFailed = 0;
|
|
73
|
+
for (let i = 0; i < rows.length; i += runConfig.batchSize) {
|
|
74
|
+
const batch = rows.slice(i, i + runConfig.batchSize);
|
|
75
|
+
const placeholderRows = [];
|
|
76
|
+
const values = [];
|
|
77
|
+
let ph = 1;
|
|
78
|
+
for (const row of batch) {
|
|
79
|
+
const placeholders = [];
|
|
80
|
+
for (const col of columns) {
|
|
81
|
+
placeholders.push(`$${ph++}`);
|
|
82
|
+
values.push(row[col] === undefined ? null : row[col]);
|
|
83
|
+
}
|
|
84
|
+
placeholderRows.push(`(${placeholders.join(', ')})`);
|
|
85
|
+
}
|
|
86
|
+
const sql = `INSERT INTO ${qualified} (${colList}) VALUES ${placeholderRows.join(', ')} ${conflictClause}`.trim();
|
|
87
|
+
try {
|
|
88
|
+
const res = await this.pool.query(sql, values);
|
|
89
|
+
rowsLoaded += res.rowCount ?? batch.length;
|
|
90
|
+
onProgress(rowsLoaded);
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
rowsFailed += batch.length;
|
|
94
|
+
if (runConfig.onError === 'stop') {
|
|
95
|
+
throw new LoadError(`pg target: insert failed: ${err instanceof Error ? err.message : String(err)}`, err);
|
|
96
|
+
}
|
|
97
|
+
logger.warn({ err: err instanceof Error ? err.message : String(err), batchStart: i }, 'pg target: batch failed (onError: continue)');
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return { rowsLoaded, rowsFailed };
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
//# sourceMappingURL=pg.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pg.js","sourceRoot":"","sources":["../../../src/adapters/target/pg.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;;;GAMG;AAEH,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,UAAU,EAAqB,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAG/C,MAAM,aAAa,GAAG,iBAAiB,CAAC;AAExC,MAAM,OAAO,eAAe;IACjB,EAAE,GAAG,IAAI,CAAC;IACX,IAAI,GAAmB,IAAI,CAAC;IAEpC,KAAK,CAAC,OAAO,CAAC,MAAoB;QAChC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YACvB,MAAM,IAAI,WAAW,CAAC,iCAAiC,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACzC,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,MAAM,IAAI,SAAS,CACjB,8BAA8B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAChF,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;YACpB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAAoB,EACpB,KAAmB,EACnB,SAAoB,EACpB,UAAkC;QAElC,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,SAAS,CAAC,0BAA0B,CAAC,CAAC;QAChE,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,MAAM,IAAI,WAAW,CAAC,4BAA4B,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC3B,MAAM,SAAS,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAE/D,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QACvD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,SAAS,CAAC,2CAA2C,CAAC,CAAC;QACnE,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAA0B,kBAAkB,aAAa,GAAG,CAAC,CAAC;QAE5F,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,cAAc,GAAG,EAAE,CAAC;QACxB,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC;YAC7B,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,WAAW,CAAC,kDAAkD,CAAC,CAAC;YAC5E,CAAC;YACD,MAAM,SAAS,GAAG,OAAO;iBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;iBAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,eAAe,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC1D,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpD,cAAc,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC;gBACnC,CAAC,CAAC,gBAAgB,YAAY,mBAAmB,SAAS,EAAE;gBAC5D,CAAC,CAAC,gBAAgB,YAAY,cAAc,CAAC;QACjD,CAAC;aAAM,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YAC1C,cAAc,GAAG,wBAAwB,CAAC;QAC5C,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;YACrD,MAAM,eAAe,GAAa,EAAE,CAAC;YACrC,MAAM,MAAM,GAAc,EAAE,CAAC;YAC7B,IAAI,EAAE,GAAG,CAAC,CAAC;YACX,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,MAAM,YAAY,GAAa,EAAE,CAAC;gBAClC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;oBAC1B,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;oBAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACxD,CAAC;gBACD,eAAe,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvD,CAAC;YACD,MAAM,GAAG,GAAG,eAAe,SAAS,KAAK,OAAO,YAAY,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,cAAc,EAAE,CAAC,IAAI,EAAE,CAAC;YAClH,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC/C,UAAU,IAAI,GAAG,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;gBAC3C,UAAU,CAAC,UAAU,CAAC,CAAC;YACzB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;gBAC3B,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;oBACjC,MAAM,IAAI,SAAS,CACjB,6BAA6B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/E,GAAG,CACJ,CAAC;gBACJ,CAAC;gBACD,MAAM,CAAC,IAAI,CACT,EAAE,GAAG,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,EACxE,6CAA6C,CAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;IACpC,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { TargetAdapter } from './types.js';
|
|
2
|
+
export declare class TargetAdapterRegistry {
|
|
3
|
+
static register(adapter: TargetAdapter): void;
|
|
4
|
+
static get(id: string): TargetAdapter;
|
|
5
|
+
static has(id: string): boolean;
|
|
6
|
+
static list(): string[];
|
|
7
|
+
static unregister(id: string): void;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=registry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../../src/adapters/target/registry.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAIhD,qBAAa,qBAAqB;IAChC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI;IAO7C,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,aAAa;IAUrC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAI/B,MAAM,CAAC,IAAI,IAAI,MAAM,EAAE;IAIvB,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;CAGpC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Elastic-2.0
|
|
2
|
+
// Copyright (c) 2026 Caracal Lynx Ltd.
|
|
3
|
+
import { ConfigError } from '../../utils/errors.js';
|
|
4
|
+
const adapters = new Map();
|
|
5
|
+
export class TargetAdapterRegistry {
|
|
6
|
+
static register(adapter) {
|
|
7
|
+
if (adapters.has(adapter.id)) {
|
|
8
|
+
throw new ConfigError(`Duplicate target adapter id "${adapter.id}"`);
|
|
9
|
+
}
|
|
10
|
+
adapters.set(adapter.id, adapter);
|
|
11
|
+
}
|
|
12
|
+
static get(id) {
|
|
13
|
+
const adapter = adapters.get(id);
|
|
14
|
+
if (!adapter) {
|
|
15
|
+
throw new ConfigError(`No target adapter registered for "${id}". Known adapters: ${[...adapters.keys()].join(', ') || '(none)'}`);
|
|
16
|
+
}
|
|
17
|
+
return adapter;
|
|
18
|
+
}
|
|
19
|
+
static has(id) {
|
|
20
|
+
return adapters.has(id);
|
|
21
|
+
}
|
|
22
|
+
static list() {
|
|
23
|
+
return [...adapters.keys()];
|
|
24
|
+
}
|
|
25
|
+
static unregister(id) {
|
|
26
|
+
adapters.delete(id);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=registry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../../src/adapters/target/registry.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAGpD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAyB,CAAC;AAElD,MAAM,OAAO,qBAAqB;IAChC,MAAM,CAAC,QAAQ,CAAC,OAAsB;QACpC,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,WAAW,CAAC,gCAAgC,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;QACvE,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,EAAU;QACnB,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,WAAW,CACnB,qCAAqC,EAAE,sBAAsB,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,EAAE,CAC3G,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,EAAU;QACnB,OAAO,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,CAAC,IAAI;QACT,OAAO,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,CAAC,UAAU,CAAC,EAAU;QAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACtB,CAAC;CACF"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { RunConfig, TargetConfig } from '../../config/types.js';
|
|
2
|
+
import type { StagingStore } from '../../staging/index.js';
|
|
3
|
+
export interface TargetAdapter {
|
|
4
|
+
readonly id: string;
|
|
5
|
+
connect(config: TargetConfig): Promise<void>;
|
|
6
|
+
load(config: TargetConfig, store: StagingStore, runConfig: RunConfig, onProgress: (rows: number) => void): Promise<LoadResult>;
|
|
7
|
+
disconnect(): Promise<void>;
|
|
8
|
+
}
|
|
9
|
+
export interface LoadResult {
|
|
10
|
+
rowsLoaded: number;
|
|
11
|
+
rowsFailed: number;
|
|
12
|
+
/** Set for file-based targets (csv, ifs, bluecherry). */
|
|
13
|
+
outputPath?: string;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/adapters/target/types.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7C,IAAI,CACF,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,YAAY,EACnB,SAAS,EAAE,SAAS,EACpB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GACjC,OAAO,CAAC,UAAU,CAAC,CAAC;IAEvB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7B;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/adapters/target/types.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC"}
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Sluice CLI.
|
|
4
|
+
*
|
|
5
|
+
* Commands (per CLAUDE.md):
|
|
6
|
+
* sluice run <pipeline.yaml> Full pipeline run
|
|
7
|
+
* sluice validate <pipeline.yaml> DQ + transform only; no load
|
|
8
|
+
* sluice profile <pipeline.yaml> Extract + column profiling; no DQ
|
|
9
|
+
* sluice check <pipeline.yaml> Config validation only; no execution
|
|
10
|
+
*
|
|
11
|
+
* Exit codes:
|
|
12
|
+
* 0 success
|
|
13
|
+
* 1 pipeline error
|
|
14
|
+
* 2 DQ critical violations
|
|
15
|
+
* 3 config error
|
|
16
|
+
* 4 enrich error (Phase 4)
|
|
17
|
+
*/
|
|
18
|
+
import { Command } from 'commander';
|
|
19
|
+
import { MultiSourcePipelineRunner } from './multi-source-runner.js';
|
|
20
|
+
import { PipelineRunner } from './runner.js';
|
|
21
|
+
export declare function resolvePluginDirs(cwd: string, pluginDirs?: string[]): string[];
|
|
22
|
+
export declare function exitCodeFor(err: unknown): number;
|
|
23
|
+
export declare function createRunnerForPipeline(yaml: string): Promise<PipelineRunner | MultiSourcePipelineRunner>;
|
|
24
|
+
export declare function buildProgram(): Command;
|
|
25
|
+
//# sourceMappingURL=cli.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAIA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EACL,cAAc,EAGf,MAAM,aAAa,CAAC;AAiBrB,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,GAAE,MAAM,EAAO,GAAG,MAAM,EAAE,CAKlF;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAMhD;AA0DD,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,cAAc,GAAG,yBAAyB,CAAC,CAKrD;AA6ND,wBAAgB,YAAY,IAAI,OAAO,CAiEtC"}
|