@caracal-lynx/sluice 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +1822 -0
- package/LICENCE-FAQ.md +74 -0
- package/LICENSE +92 -0
- package/README.md +582 -0
- package/dist/adapters/source/csv.d.ts +10 -0
- package/dist/adapters/source/csv.d.ts.map +1 -0
- package/dist/adapters/source/csv.js +110 -0
- package/dist/adapters/source/csv.js.map +1 -0
- package/dist/adapters/source/index.d.ts +9 -0
- package/dist/adapters/source/index.d.ts.map +1 -0
- package/dist/adapters/source/index.js +26 -0
- package/dist/adapters/source/index.js.map +1 -0
- package/dist/adapters/source/mssql.d.ts +11 -0
- package/dist/adapters/source/mssql.d.ts.map +1 -0
- package/dist/adapters/source/mssql.js +230 -0
- package/dist/adapters/source/mssql.js.map +1 -0
- package/dist/adapters/source/pg.d.ts +11 -0
- package/dist/adapters/source/pg.d.ts.map +1 -0
- package/dist/adapters/source/pg.js +88 -0
- package/dist/adapters/source/pg.js.map +1 -0
- package/dist/adapters/source/registry.d.ts +10 -0
- package/dist/adapters/source/registry.d.ts.map +1 -0
- package/dist/adapters/source/registry.js +36 -0
- package/dist/adapters/source/registry.js.map +1 -0
- package/dist/adapters/source/rest.d.ts +16 -0
- package/dist/adapters/source/rest.d.ts.map +1 -0
- package/dist/adapters/source/rest.js +182 -0
- package/dist/adapters/source/rest.js.map +1 -0
- package/dist/adapters/source/rest.types.d.ts +15 -0
- package/dist/adapters/source/rest.types.d.ts.map +1 -0
- package/dist/adapters/source/rest.types.js +6 -0
- package/dist/adapters/source/rest.types.js.map +1 -0
- package/dist/adapters/source/types.d.ts +23 -0
- package/dist/adapters/source/types.d.ts.map +1 -0
- package/dist/adapters/source/types.js +4 -0
- package/dist/adapters/source/types.js.map +1 -0
- package/dist/adapters/source/xlsx.d.ts +10 -0
- package/dist/adapters/source/xlsx.d.ts.map +1 -0
- package/dist/adapters/source/xlsx.js +71 -0
- package/dist/adapters/source/xlsx.js.map +1 -0
- package/dist/adapters/target/bc.d.ts +21 -0
- package/dist/adapters/target/bc.d.ts.map +1 -0
- package/dist/adapters/target/bc.js +188 -0
- package/dist/adapters/target/bc.js.map +1 -0
- package/dist/adapters/target/bluecherry.d.ts +10 -0
- package/dist/adapters/target/bluecherry.d.ts.map +1 -0
- package/dist/adapters/target/bluecherry.js +127 -0
- package/dist/adapters/target/bluecherry.js.map +1 -0
- package/dist/adapters/target/csv.d.ts +10 -0
- package/dist/adapters/target/csv.d.ts.map +1 -0
- package/dist/adapters/target/csv.js +40 -0
- package/dist/adapters/target/csv.js.map +1 -0
- package/dist/adapters/target/ifs.d.ts +10 -0
- package/dist/adapters/target/ifs.d.ts.map +1 -0
- package/dist/adapters/target/ifs.js +55 -0
- package/dist/adapters/target/ifs.js.map +1 -0
- package/dist/adapters/target/index.d.ts +8 -0
- package/dist/adapters/target/index.d.ts.map +1 -0
- package/dist/adapters/target/index.js +22 -0
- package/dist/adapters/target/index.js.map +1 -0
- package/dist/adapters/target/pg.d.ts +11 -0
- package/dist/adapters/target/pg.d.ts.map +1 -0
- package/dist/adapters/target/pg.js +103 -0
- package/dist/adapters/target/pg.js.map +1 -0
- package/dist/adapters/target/registry.d.ts +9 -0
- package/dist/adapters/target/registry.d.ts.map +1 -0
- package/dist/adapters/target/registry.js +29 -0
- package/dist/adapters/target/registry.js.map +1 -0
- package/dist/adapters/target/types.d.ts +15 -0
- package/dist/adapters/target/types.d.ts.map +1 -0
- package/dist/adapters/target/types.js +4 -0
- package/dist/adapters/target/types.js.map +1 -0
- package/dist/cli.d.ts +25 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +354 -0
- package/dist/cli.js.map +1 -0
- package/dist/config/index.d.ts +4 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +6 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/loader.d.ts +5 -0
- package/dist/config/loader.d.ts.map +1 -0
- package/dist/config/loader.js +135 -0
- package/dist/config/loader.js.map +1 -0
- package/dist/config/schema.d.ts +4162 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +263 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/config/types.d.ts +3 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +4 -0
- package/dist/config/types.js.map +1 -0
- package/dist/dq/engine.d.ts +10 -0
- package/dist/dq/engine.d.ts.map +1 -0
- package/dist/dq/engine.js +114 -0
- package/dist/dq/engine.js.map +1 -0
- package/dist/dq/index.d.ts +6 -0
- package/dist/dq/index.d.ts.map +1 -0
- package/dist/dq/index.js +6 -0
- package/dist/dq/index.js.map +1 -0
- package/dist/dq/reporter.d.ts +5 -0
- package/dist/dq/reporter.d.ts.map +1 -0
- package/dist/dq/reporter.js +41 -0
- package/dist/dq/reporter.js.map +1 -0
- package/dist/dq/rules/allowedValues.d.ts +7 -0
- package/dist/dq/rules/allowedValues.d.ts.map +1 -0
- package/dist/dq/rules/allowedValues.js +26 -0
- package/dist/dq/rules/allowedValues.js.map +1 -0
- package/dist/dq/rules/email.d.ts +7 -0
- package/dist/dq/rules/email.d.ts.map +1 -0
- package/dist/dq/rules/email.js +24 -0
- package/dist/dq/rules/email.js.map +1 -0
- package/dist/dq/rules/index.d.ts +15 -0
- package/dist/dq/rules/index.d.ts.map +1 -0
- package/dist/dq/rules/index.js +30 -0
- package/dist/dq/rules/index.js.map +1 -0
- package/dist/dq/rules/maxLength.d.ts +7 -0
- package/dist/dq/rules/maxLength.d.ts.map +1 -0
- package/dist/dq/rules/maxLength.js +25 -0
- package/dist/dq/rules/maxLength.js.map +1 -0
- package/dist/dq/rules/minMax.d.ts +11 -0
- package/dist/dq/rules/minMax.d.ts.map +1 -0
- package/dist/dq/rules/minMax.js +52 -0
- package/dist/dq/rules/minMax.js.map +1 -0
- package/dist/dq/rules/notNull.d.ts +7 -0
- package/dist/dq/rules/notNull.d.ts.map +1 -0
- package/dist/dq/rules/notNull.js +21 -0
- package/dist/dq/rules/notNull.js.map +1 -0
- package/dist/dq/rules/pattern.d.ts +7 -0
- package/dist/dq/rules/pattern.d.ts.map +1 -0
- package/dist/dq/rules/pattern.js +31 -0
- package/dist/dq/rules/pattern.js.map +1 -0
- package/dist/dq/rules/types.d.ts +6 -0
- package/dist/dq/rules/types.d.ts.map +1 -0
- package/dist/dq/rules/types.js +4 -0
- package/dist/dq/rules/types.js.map +1 -0
- package/dist/dq/rules/ukPostcode.d.ts +7 -0
- package/dist/dq/rules/ukPostcode.d.ts.map +1 -0
- package/dist/dq/rules/ukPostcode.js +24 -0
- package/dist/dq/rules/ukPostcode.js.map +1 -0
- package/dist/dq/rules/unique.d.ts +14 -0
- package/dist/dq/rules/unique.d.ts.map +1 -0
- package/dist/dq/rules/unique.js +9 -0
- package/dist/dq/rules/unique.js.map +1 -0
- package/dist/dq/types.d.ts +29 -0
- package/dist/dq/types.d.ts.map +1 -0
- package/dist/dq/types.js +4 -0
- package/dist/dq/types.js.map +1 -0
- package/dist/enrich/types.d.ts +87 -0
- package/dist/enrich/types.d.ts.map +1 -0
- package/dist/enrich/types.js +4 -0
- package/dist/enrich/types.js.map +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +17 -0
- package/dist/index.js.map +1 -0
- package/dist/merge/conflict-log.d.ts +9 -0
- package/dist/merge/conflict-log.d.ts.map +1 -0
- package/dist/merge/conflict-log.js +28 -0
- package/dist/merge/conflict-log.js.map +1 -0
- package/dist/merge/engine.d.ts +7 -0
- package/dist/merge/engine.d.ts.map +1 -0
- package/dist/merge/engine.js +19 -0
- package/dist/merge/engine.js.map +1 -0
- package/dist/merge/index.d.ts +11 -0
- package/dist/merge/index.d.ts.map +1 -0
- package/dist/merge/index.js +34 -0
- package/dist/merge/index.js.map +1 -0
- package/dist/merge/sql-builder.d.ts +19 -0
- package/dist/merge/sql-builder.d.ts.map +1 -0
- package/dist/merge/sql-builder.js +148 -0
- package/dist/merge/sql-builder.js.map +1 -0
- package/dist/merge/strategies/coalesce.d.ts +17 -0
- package/dist/merge/strategies/coalesce.d.ts.map +1 -0
- package/dist/merge/strategies/coalesce.js +77 -0
- package/dist/merge/strategies/coalesce.js.map +1 -0
- package/dist/merge/strategies/index.d.ts +5 -0
- package/dist/merge/strategies/index.d.ts.map +1 -0
- package/dist/merge/strategies/index.js +7 -0
- package/dist/merge/strategies/index.js.map +1 -0
- package/dist/merge/strategies/intersect.d.ts +17 -0
- package/dist/merge/strategies/intersect.d.ts.map +1 -0
- package/dist/merge/strategies/intersect.js +75 -0
- package/dist/merge/strategies/intersect.js.map +1 -0
- package/dist/merge/strategies/priority-override.d.ts +16 -0
- package/dist/merge/strategies/priority-override.d.ts.map +1 -0
- package/dist/merge/strategies/priority-override.js +78 -0
- package/dist/merge/strategies/priority-override.js.map +1 -0
- package/dist/merge/strategies/registry.d.ts +8 -0
- package/dist/merge/strategies/registry.d.ts.map +1 -0
- package/dist/merge/strategies/registry.js +19 -0
- package/dist/merge/strategies/registry.js.map +1 -0
- package/dist/merge/strategies/union.d.ts +15 -0
- package/dist/merge/strategies/union.d.ts.map +1 -0
- package/dist/merge/strategies/union.js +75 -0
- package/dist/merge/strategies/union.js.map +1 -0
- package/dist/merge/types.d.ts +24 -0
- package/dist/merge/types.d.ts.map +1 -0
- package/dist/merge/types.js +4 -0
- package/dist/merge/types.js.map +1 -0
- package/dist/multi-source-runner.d.ts +22 -0
- package/dist/multi-source-runner.d.ts.map +1 -0
- package/dist/multi-source-runner.js +398 -0
- package/dist/multi-source-runner.js.map +1 -0
- package/dist/plugins/index.d.ts +4 -0
- package/dist/plugins/index.d.ts.map +1 -0
- package/dist/plugins/index.js +5 -0
- package/dist/plugins/index.js.map +1 -0
- package/dist/plugins/loader.d.ts +22 -0
- package/dist/plugins/loader.d.ts.map +1 -0
- package/dist/plugins/loader.js +151 -0
- package/dist/plugins/loader.js.map +1 -0
- package/dist/plugins/registry.d.ts +25 -0
- package/dist/plugins/registry.d.ts.map +1 -0
- package/dist/plugins/registry.js +42 -0
- package/dist/plugins/registry.js.map +1 -0
- package/dist/plugins/types.d.ts +61 -0
- package/dist/plugins/types.d.ts.map +1 -0
- package/dist/plugins/types.js +4 -0
- package/dist/plugins/types.js.map +1 -0
- package/dist/runner.d.ts +97 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +520 -0
- package/dist/runner.js.map +1 -0
- package/dist/staging/index.d.ts +3 -0
- package/dist/staging/index.d.ts.map +1 -0
- package/dist/staging/index.js +5 -0
- package/dist/staging/index.js.map +1 -0
- package/dist/staging/schema.d.ts +19 -0
- package/dist/staging/schema.d.ts.map +1 -0
- package/dist/staging/schema.js +15 -0
- package/dist/staging/schema.js.map +1 -0
- package/dist/staging/store.d.ts +71 -0
- package/dist/staging/store.d.ts.map +1 -0
- package/dist/staging/store.js +270 -0
- package/dist/staging/store.js.map +1 -0
- package/dist/transform/cleanse.d.ts +2 -0
- package/dist/transform/cleanse.d.ts.map +1 -0
- package/dist/transform/cleanse.js +59 -0
- package/dist/transform/cleanse.js.map +1 -0
- package/dist/transform/engine.d.ts +10 -0
- package/dist/transform/engine.d.ts.map +1 -0
- package/dist/transform/engine.js +225 -0
- package/dist/transform/engine.js.map +1 -0
- package/dist/transform/expression.d.ts +5 -0
- package/dist/transform/expression.d.ts.map +1 -0
- package/dist/transform/expression.js +52 -0
- package/dist/transform/expression.js.map +1 -0
- package/dist/transform/index.d.ts +6 -0
- package/dist/transform/index.d.ts.map +1 -0
- package/dist/transform/index.js +7 -0
- package/dist/transform/index.js.map +1 -0
- package/dist/transform/lookup.d.ts +10 -0
- package/dist/transform/lookup.d.ts.map +1 -0
- package/dist/transform/lookup.js +66 -0
- package/dist/transform/lookup.js.map +1 -0
- package/dist/transform/types.d.ts +10 -0
- package/dist/transform/types.d.ts.map +1 -0
- package/dist/transform/types.js +4 -0
- package/dist/transform/types.js.map +1 -0
- package/dist/utils/env.d.ts +3 -0
- package/dist/utils/env.d.ts.map +1 -0
- package/dist/utils/env.js +26 -0
- package/dist/utils/env.js.map +1 -0
- package/dist/utils/errors.d.ts +26 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +39 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/index.d.ts +5 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +7 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/logger.d.ts +14 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +16 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/progress.d.ts +66 -0
- package/dist/utils/progress.d.ts.map +1 -0
- package/dist/utils/progress.js +283 -0
- package/dist/utils/progress.js.map +1 -0
- package/package.json +92 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../../src/merge/strategies/registry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAQvD,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,mBAAmB,GAAG,SAAS,CAEvE;AAED,wBAAgB,cAAc,IAAI,MAAM,EAAE,CAEzC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Merge strategy registry.
|
|
3
|
+
* Loads all built-in merge strategies.
|
|
4
|
+
*/
|
|
5
|
+
import { coalesceStrategy } from './coalesce.js';
|
|
6
|
+
import { unionStrategy } from './union.js';
|
|
7
|
+
import { intersectStrategy } from './intersect.js';
|
|
8
|
+
const BUILT_IN_STRATEGIES = {
|
|
9
|
+
coalesce: coalesceStrategy,
|
|
10
|
+
union: unionStrategy,
|
|
11
|
+
intersect: intersectStrategy,
|
|
12
|
+
};
|
|
13
|
+
export function getStrategy(id) {
|
|
14
|
+
return BUILT_IN_STRATEGIES[id];
|
|
15
|
+
}
|
|
16
|
+
export function listStrategies() {
|
|
17
|
+
return Object.keys(BUILT_IN_STRATEGIES);
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=registry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../../src/merge/strategies/registry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGnD,MAAM,mBAAmB,GAAwC;IAC/D,QAAQ,EAAE,gBAAgB;IAC1B,KAAK,EAAE,aAAa;IACpB,SAAS,EAAE,iBAAiB;CAC7B,CAAC;AAEF,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,OAAO,mBAAmB,CAAC,EAAE,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Merge strategy: UNION
|
|
3
|
+
*
|
|
4
|
+
* Includes all rows from all sources (outer union).
|
|
5
|
+
* For each field, uses coalesce semantics (first non-null value respecting priority).
|
|
6
|
+
* This is equivalent to coalesce with onUnmatched: include.
|
|
7
|
+
*
|
|
8
|
+
* Example:
|
|
9
|
+
* Source A: {ID: "A", VAL: "from-a"}
|
|
10
|
+
* Source B: {ID: "B", VAL: "from-b"}
|
|
11
|
+
* → Result: includes both {ID: "A"} and {ID: "B"} rows
|
|
12
|
+
*/
|
|
13
|
+
import type { MergeStrategyPlugin } from '../types.js';
|
|
14
|
+
export declare const unionStrategy: MergeStrategyPlugin;
|
|
15
|
+
//# sourceMappingURL=union.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"union.d.ts","sourceRoot":"","sources":["../../../src/merge/strategies/union.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAiBvD,eAAO,MAAM,aAAa,EAAE,mBAkG3B,CAAC"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Elastic-2.0
|
|
2
|
+
// Copyright (c) 2026 Caracal Lynx Ltd.
|
|
3
|
+
import { quoteIdent } from '../../staging/index.js';
|
|
4
|
+
import { logger } from '../../utils/logger.js';
|
|
5
|
+
import { buildJoinedTableSql, buildMergedTableSql, buildPresentCountExpr, normalizeKeyColumns, } from '../sql-builder.js';
|
|
6
|
+
import { buildConflictLog } from '../conflict-log.js';
|
|
7
|
+
import { ConfigError } from '../../utils/errors.js';
|
|
8
|
+
export const unionStrategy = {
|
|
9
|
+
id: 'union',
|
|
10
|
+
description: 'All rows from all sources. Includes all rows from every source; deduplicates by key. Useful for union-ing independent sources.',
|
|
11
|
+
async merge(store, rawSources, config) {
|
|
12
|
+
if (rawSources.length < 2) {
|
|
13
|
+
throw new ConfigError('merge requires at least 2 sources');
|
|
14
|
+
}
|
|
15
|
+
const sources = [...rawSources].sort((a, b) => a.priority - b.priority);
|
|
16
|
+
const keyColumns = normalizeKeyColumns(config.key);
|
|
17
|
+
const sourceColumns = {};
|
|
18
|
+
for (const source of sources) {
|
|
19
|
+
sourceColumns[source.id] = await store.columnNames(source.tableName);
|
|
20
|
+
for (const key of keyColumns) {
|
|
21
|
+
if (!sourceColumns[source.id].includes(key)) {
|
|
22
|
+
throw new ConfigError(`merge key column '${key}' is missing from source '${source.id}' table '${source.tableName}'`);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
const context = { sources, keyColumns, sourceColumns };
|
|
27
|
+
// Build the joined table (FULL OUTER JOIN on keys)
|
|
28
|
+
const joinedTable = 'stg_merge_joined';
|
|
29
|
+
await store.query(buildJoinedTableSql(joinedTable, context));
|
|
30
|
+
// Check for unmatched rows (informational only for union strategy)
|
|
31
|
+
const presentCountExpr = buildPresentCountExpr(sources, keyColumns);
|
|
32
|
+
const unmatchedRows = await store.query(`SELECT count(*) AS n FROM ${quoteIdent(joinedTable)} WHERE (${presentCountExpr}) = 1`);
|
|
33
|
+
const unmatched = Number(unmatchedRows[0]?.n ?? 0);
|
|
34
|
+
if (unmatched > 0 && config.onUnmatched === 'warn') {
|
|
35
|
+
logger.warn({ unmatched }, 'merge: unmatched rows included in union');
|
|
36
|
+
}
|
|
37
|
+
// Build output columns in order: keys first, then all non-key columns
|
|
38
|
+
const seen = new Set();
|
|
39
|
+
const outputColumns = [];
|
|
40
|
+
for (const key of keyColumns) {
|
|
41
|
+
if (!seen.has(key)) {
|
|
42
|
+
seen.add(key);
|
|
43
|
+
outputColumns.push(key);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
for (const source of sources) {
|
|
47
|
+
for (const col of sourceColumns[source.id] ?? []) {
|
|
48
|
+
if (!seen.has(col)) {
|
|
49
|
+
seen.add(col);
|
|
50
|
+
outputColumns.push(col);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
// Merge: union includes all rows, coalesce field values
|
|
55
|
+
const mergedTable = 'stg_merged';
|
|
56
|
+
const cfgOverride = { ...config, strategy: 'coalesce', onUnmatched: 'include' };
|
|
57
|
+
await store.query(buildMergedTableSql(mergedTable, joinedTable, context, cfgOverride, outputColumns));
|
|
58
|
+
const rowsMerged = await store.rowCount(mergedTable);
|
|
59
|
+
const conflictLog = await buildConflictLog(store, joinedTable, context, cfgOverride, outputColumns, 'stg_merge_conflicts');
|
|
60
|
+
logger.info({
|
|
61
|
+
strategy: 'union',
|
|
62
|
+
sources: sources.length,
|
|
63
|
+
rowsMerged,
|
|
64
|
+
conflicts: conflictLog.count,
|
|
65
|
+
unmatched,
|
|
66
|
+
}, 'merge: complete');
|
|
67
|
+
return {
|
|
68
|
+
rowsMerged,
|
|
69
|
+
conflicts: conflictLog.count,
|
|
70
|
+
unmatched,
|
|
71
|
+
tableName: mergedTable,
|
|
72
|
+
};
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
//# sourceMappingURL=union.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"union.js","sourceRoot":"","sources":["../../../src/merge/strategies/union.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAiBvC,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACrB,mBAAmB,GAEpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAGtD,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEpD,MAAM,CAAC,MAAM,aAAa,GAAwB;IAChD,EAAE,EAAE,OAAO;IACX,WAAW,EAAE,gIAAgI;IAE7I,KAAK,CAAC,KAAK,CACT,KAAmB,EACnB,UAA6B,EAC7B,MAAmB;QAEnB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,WAAW,CAAC,mCAAmC,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,OAAO,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAEnD,MAAM,aAAa,GAA6B,EAAE,CAAC;QACnD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACrE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC7C,MAAM,IAAI,WAAW,CACnB,qBAAqB,GAAG,6BAA6B,MAAM,CAAC,EAAE,YAAY,MAAM,CAAC,SAAS,GAAG,CAC9F,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAsB,EAAE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC;QAE1E,mDAAmD;QACnD,MAAM,WAAW,GAAG,kBAAkB,CAAC;QACvC,MAAM,KAAK,CAAC,KAAK,CAAC,mBAAmB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;QAE7D,mEAAmE;QACnE,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACpE,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,KAAK,CACrC,6BAA6B,UAAU,CAAC,WAAW,CAAC,WAAW,gBAAgB,OAAO,CACvF,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QAEnD,IAAI,SAAS,GAAG,CAAC,IAAI,MAAM,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;YACnD,MAAM,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,yCAAyC,CAAC,CAAC;QACxE,CAAC;QAED,sEAAsE;QACtE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,aAAa,GAAa,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;gBACjD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACd,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QAED,wDAAwD;QACxD,MAAM,WAAW,GAAG,YAAY,CAAC;QACjC,MAAM,WAAW,GAAgB,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC;QAC7F,MAAM,KAAK,CAAC,KAAK,CACf,mBAAmB,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,CAAC,CACnF,CAAC;QAEF,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QACrD,MAAM,WAAW,GAAG,MAAM,gBAAgB,CACxC,KAAK,EACL,WAAW,EACX,OAAO,EACP,WAAW,EACX,aAAa,EACb,qBAAqB,CACtB,CAAC;QAEF,MAAM,CAAC,IAAI,CACT;YACE,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,UAAU;YACV,SAAS,EAAE,WAAW,CAAC,KAAK;YAC5B,SAAS;SACV,EACD,iBAAiB,CAClB,CAAC;QAEF,OAAO;YACL,UAAU;YACV,SAAS,EAAE,WAAW,CAAC,KAAK;YAC5B,SAAS;YACT,SAAS,EAAE,WAAW;SACvB,CAAC;IACJ,CAAC;CACF,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { StagingStore } from '../staging/store.js';
|
|
2
|
+
import type { MergeConfig } from '../config/types.js';
|
|
3
|
+
export interface MergeSourceMeta {
|
|
4
|
+
id: string;
|
|
5
|
+
priority: number;
|
|
6
|
+
tableName: string;
|
|
7
|
+
}
|
|
8
|
+
export interface MergeResult {
|
|
9
|
+
rowsMerged: number;
|
|
10
|
+
conflicts: number;
|
|
11
|
+
unmatched: number;
|
|
12
|
+
tableName: 'stg_merged';
|
|
13
|
+
}
|
|
14
|
+
export interface MergeStrategyPlugin {
|
|
15
|
+
readonly id: string;
|
|
16
|
+
readonly description?: string;
|
|
17
|
+
/**
|
|
18
|
+
* Merges N source staging tables into stg_merged.
|
|
19
|
+
* Sources are passed in priority order (priority 1 first).
|
|
20
|
+
* The implementation is responsible for creating stg_merged in the store.
|
|
21
|
+
*/
|
|
22
|
+
merge(store: StagingStore, sources: MergeSourceMeta[], config: MergeConfig): Promise<MergeResult>;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/merge/types.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAS,MAAM,CAAC;IAClB,QAAQ,EAAG,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAG,MAAM,CAAC;IACpB,SAAS,EAAI,MAAM,CAAC;IACpB,SAAS,EAAI,MAAM,CAAC;IACpB,SAAS,EAAI,YAAY,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,KAAK,CACH,KAAK,EAAI,YAAY,EACrB,OAAO,EAAE,eAAe,EAAE,EAC1B,MAAM,EAAG,WAAW,GACnB,OAAO,CAAC,WAAW,CAAC,CAAC;CACzB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/merge/types.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type ExtractResult } from './adapters/source/index.js';
|
|
2
|
+
import { type Pipeline } from './config/types.js';
|
|
3
|
+
import type { DQSummary } from './dq/index.js';
|
|
4
|
+
import type { EnrichSummary } from './enrich/types.js';
|
|
5
|
+
import { PipelineRunner, type RunOverrides, type RunResult } from './runner.js';
|
|
6
|
+
import { type LoadResult } from './adapters/target/index.js';
|
|
7
|
+
export declare class MultiSourcePipelineRunner extends PipelineRunner {
|
|
8
|
+
private readonly mergeEngine;
|
|
9
|
+
private sourceExtracts;
|
|
10
|
+
private sourceIncrementalSince;
|
|
11
|
+
private sortSourcesForMerge;
|
|
12
|
+
profile(yamlPath: string, overrides?: RunOverrides): Promise<RunResult>;
|
|
13
|
+
run(yamlPath: string, overrides?: RunOverrides): Promise<RunResult>;
|
|
14
|
+
protected writeStateFile(config: Pipeline, extract: ExtractResult, dq: DQSummary, load: LoadResult, enrichSummary?: EnrichSummary): Promise<string>;
|
|
15
|
+
private extractOneSource;
|
|
16
|
+
private resolveIncrementalSinceForSource;
|
|
17
|
+
private applyIncrementalFilterForSource;
|
|
18
|
+
private buildProfileFromTable;
|
|
19
|
+
private buildSingleSourceConfig;
|
|
20
|
+
private buildPostMergeConfig;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=multi-source-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multi-source-runner.d.ts","sourceRoot":"","sources":["../src/multi-source-runner.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEhE,OAAO,EAAwC,KAAK,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AACxF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAEhF,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAqB7D,qBAAa,yBAA0B,SAAQ,cAAc;IAC3D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,OAAO,CAAC,cAAc,CAA2B;IACjD,OAAO,CAAC,sBAAsB,CAA8B;IAG5D,OAAO,CAAC,mBAAmB;IAWZ,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,GAAE,YAAiB,GAAG,OAAO,CAAC,SAAS,CAAC;IA4E3E,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,GAAE,YAAiB,GAAG,OAAO,CAAC,SAAS,CAAC;cA+L7D,cAAc,CACrC,MAAM,EAAE,QAAQ,EAChB,OAAO,EAAE,aAAa,EACtB,EAAE,EAAE,SAAS,EACb,IAAI,EAAE,UAAU,EAChB,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,MAAM,CAAC;YAoCJ,gBAAgB;YAyFhB,gCAAgC;YAmBhC,+BAA+B;YA+D/B,qBAAqB;IA8DnC,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,oBAAoB;CAe7B"}
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Elastic-2.0
|
|
2
|
+
// Copyright (c) 2026 Caracal Lynx Ltd.
|
|
3
|
+
import * as fs from 'node:fs/promises';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
import { ConfigLoader } from './config/loader.js';
|
|
6
|
+
import { isMultiSource } from './config/types.js';
|
|
7
|
+
import { MergeEngine } from './merge/index.js';
|
|
8
|
+
import { PipelineRunner } from './runner.js';
|
|
9
|
+
import { StagingStore, quoteIdent } from './staging/index.js';
|
|
10
|
+
import { ConfigError, PipelineDQError } from './utils/errors.js';
|
|
11
|
+
import { logger } from './utils/logger.js';
|
|
12
|
+
function asColumns(names) {
|
|
13
|
+
return names.map((name) => ({ name, duckDbType: 'VARCHAR' }));
|
|
14
|
+
}
|
|
15
|
+
function sourceRejectionPath(config, sourceId) {
|
|
16
|
+
const configured = config.dq.rejectionFile;
|
|
17
|
+
if (!configured)
|
|
18
|
+
return undefined;
|
|
19
|
+
const parsed = path.parse(configured);
|
|
20
|
+
return path.join(parsed.dir, `${parsed.name}-${sourceId}${parsed.ext || '.csv'}`);
|
|
21
|
+
}
|
|
22
|
+
export class MultiSourcePipelineRunner extends PipelineRunner {
|
|
23
|
+
mergeEngine = new MergeEngine();
|
|
24
|
+
sourceExtracts = [];
|
|
25
|
+
sourceIncrementalSince = {};
|
|
26
|
+
// Keep merge ordering deterministic when two sources share the same priority.
|
|
27
|
+
sortSourcesForMerge(sources) {
|
|
28
|
+
return sources
|
|
29
|
+
.map((source, index) => ({ source, index }))
|
|
30
|
+
.sort((a, b) => {
|
|
31
|
+
const byPriority = a.source.priority - b.source.priority;
|
|
32
|
+
if (byPriority !== 0)
|
|
33
|
+
return byPriority;
|
|
34
|
+
return a.index - b.index;
|
|
35
|
+
})
|
|
36
|
+
.map((entry) => entry.source);
|
|
37
|
+
}
|
|
38
|
+
async profile(yamlPath, overrides = {}) {
|
|
39
|
+
if (overrides.progress)
|
|
40
|
+
this.progress = overrides.progress;
|
|
41
|
+
const loaded = await ConfigLoader.load(yamlPath);
|
|
42
|
+
const config = this.applyOverrides(loaded, overrides);
|
|
43
|
+
await this.loadAllPlugins(path.dirname(path.resolve(yamlPath)), overrides.pluginDirs ?? []);
|
|
44
|
+
if (!isMultiSource(config)) {
|
|
45
|
+
throw new ConfigError(`Pipeline "${config.pipeline.name}" is not multi-source. Use PipelineRunner for single-source pipelines.`);
|
|
46
|
+
}
|
|
47
|
+
if (config.run.mode === 'incremental') {
|
|
48
|
+
throw new ConfigError('incremental mode is not implemented for multi-source profile');
|
|
49
|
+
}
|
|
50
|
+
const outputDir = path.resolve(config.run.outputDir);
|
|
51
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
52
|
+
const store = new StagingStore(this.resolveStagingDb(config));
|
|
53
|
+
try {
|
|
54
|
+
await store.open();
|
|
55
|
+
this.sourceExtracts = [];
|
|
56
|
+
const sortedSources = this.sortSourcesForMerge(config.sources);
|
|
57
|
+
for (const sourceEntry of sortedSources) {
|
|
58
|
+
await this.extractOneSource(config, store, sourceEntry, false);
|
|
59
|
+
}
|
|
60
|
+
const mergeResult = await this.mergeEngine.run(store, sortedSources.map((s) => ({
|
|
61
|
+
id: s.id,
|
|
62
|
+
priority: s.priority,
|
|
63
|
+
tableName: `stg_raw_${s.id}`,
|
|
64
|
+
})), config.merge);
|
|
65
|
+
const mergedColumns = asColumns(await store.columnNames('stg_merged'));
|
|
66
|
+
const profile = await this.buildProfileFromTable(store, 'stg_merged', mergedColumns);
|
|
67
|
+
const profilePath = path.join(outputDir, `${config.pipeline.name}-profile.json`);
|
|
68
|
+
await fs.writeFile(profilePath, `${JSON.stringify(profile, null, 2)}\n`, 'utf-8');
|
|
69
|
+
logger.info({ profilePath, columns: profile.length }, 'pipeline: profile complete');
|
|
70
|
+
const extractResult = {
|
|
71
|
+
rowsExtracted: this.sourceExtracts.reduce((sum, s) => sum + s.rowsExtracted, 0),
|
|
72
|
+
tableName: mergeResult.tableName,
|
|
73
|
+
columns: mergedColumns,
|
|
74
|
+
};
|
|
75
|
+
return {
|
|
76
|
+
pipeline: config.pipeline.name,
|
|
77
|
+
mode: config.run.mode,
|
|
78
|
+
extract: extractResult,
|
|
79
|
+
dq: {
|
|
80
|
+
pipeline: config.pipeline.name,
|
|
81
|
+
runAt: new Date().toISOString(),
|
|
82
|
+
rowsChecked: mergeResult.rowsMerged,
|
|
83
|
+
rowsPassed: mergeResult.rowsMerged,
|
|
84
|
+
rowsRejected: 0,
|
|
85
|
+
violations: { critical: 0, warning: 0, info: 0 },
|
|
86
|
+
byField: {},
|
|
87
|
+
reportPath: '',
|
|
88
|
+
},
|
|
89
|
+
transform: null,
|
|
90
|
+
load: null,
|
|
91
|
+
profilePath,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
await store.close();
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
async run(yamlPath, overrides = {}) {
|
|
99
|
+
if (overrides.progress)
|
|
100
|
+
this.progress = overrides.progress;
|
|
101
|
+
const runStartMs = Date.now();
|
|
102
|
+
const loaded = await ConfigLoader.load(yamlPath);
|
|
103
|
+
const config = this.applyOverrides(loaded, overrides);
|
|
104
|
+
await this.loadAllPlugins(path.dirname(path.resolve(yamlPath)), overrides.pluginDirs ?? []);
|
|
105
|
+
if (!isMultiSource(config)) {
|
|
106
|
+
throw new ConfigError(`Pipeline "${config.pipeline.name}" is not multi-source. Use PipelineRunner for single-source pipelines.`);
|
|
107
|
+
}
|
|
108
|
+
if (config.run.mode === 'incremental' && !config.run.incrementalField) {
|
|
109
|
+
throw new ConfigError('run.incrementalField is required for incremental multi-source pipelines');
|
|
110
|
+
}
|
|
111
|
+
const outputDir = path.resolve(config.run.outputDir);
|
|
112
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
113
|
+
const store = new StagingStore(this.resolveStagingDb(config));
|
|
114
|
+
try {
|
|
115
|
+
await store.open();
|
|
116
|
+
this.sourceExtracts = [];
|
|
117
|
+
this.sourceIncrementalSince = Object.fromEntries(config.sources.map((s) => [s.id, '']));
|
|
118
|
+
const sortedSources = this.sortSourcesForMerge(config.sources);
|
|
119
|
+
const incrementalSourceId = config.run.mode === 'incremental'
|
|
120
|
+
? config.merge.incrementalSource
|
|
121
|
+
: undefined;
|
|
122
|
+
const incrementalSince = incrementalSourceId
|
|
123
|
+
? await this.resolveIncrementalSinceForSource(config, incrementalSourceId)
|
|
124
|
+
: '';
|
|
125
|
+
if (incrementalSourceId) {
|
|
126
|
+
this.sourceIncrementalSince[incrementalSourceId] = incrementalSince;
|
|
127
|
+
}
|
|
128
|
+
for (const sourceEntry of sortedSources) {
|
|
129
|
+
await this.extractOneSource(config, store, sourceEntry, true, incrementalSourceId === sourceEntry.id ? incrementalSince : undefined);
|
|
130
|
+
}
|
|
131
|
+
this.progress.startPhase('merge', `Merge (${config.merge.strategy ?? 'coalesce'})`);
|
|
132
|
+
let mergeResult;
|
|
133
|
+
try {
|
|
134
|
+
mergeResult = await this.mergeEngine.run(store, sortedSources.map((s) => ({
|
|
135
|
+
id: s.id,
|
|
136
|
+
priority: s.priority,
|
|
137
|
+
tableName: `stg_raw_${s.id}`,
|
|
138
|
+
})), config.merge);
|
|
139
|
+
this.progress.update(mergeResult.rowsMerged);
|
|
140
|
+
this.progress.endPhase({
|
|
141
|
+
state: mergeResult.conflicts > 0 ? 'warn' : 'success',
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
this.progress.endPhase({ state: 'fail' });
|
|
146
|
+
throw err;
|
|
147
|
+
}
|
|
148
|
+
const postMergeConfig = this.buildPostMergeConfig(config);
|
|
149
|
+
const enrichSummary = await this.runEnrich(postMergeConfig, store, path.dirname(path.resolve(yamlPath)), overrides);
|
|
150
|
+
const dqSummary = await this.runDQ(postMergeConfig, store, 'stg_merged', undefined, 'Data quality (post-merge)');
|
|
151
|
+
if (postMergeConfig.dq.stopOnCritical && dqSummary.violations.critical > 0) {
|
|
152
|
+
throw new PipelineDQError(dqSummary.violations.critical, dqSummary.reportPath);
|
|
153
|
+
}
|
|
154
|
+
const mergedColumns = asColumns(await store.columnNames('stg_merged'));
|
|
155
|
+
const transformSourceTable = await this.materializeAcceptedRows(store, 'stg_merged', mergedColumns, dqSummary.rejectedRowIndices ?? []);
|
|
156
|
+
const transformResult = await this.runTransform(postMergeConfig, store, transformSourceTable, 'stg_transformed');
|
|
157
|
+
const extractResult = {
|
|
158
|
+
rowsExtracted: this.sourceExtracts.reduce((sum, s) => sum + s.rowsExtracted, 0),
|
|
159
|
+
tableName: mergeResult.tableName,
|
|
160
|
+
columns: mergedColumns,
|
|
161
|
+
};
|
|
162
|
+
const baseRunResult = this.buildRunResult(postMergeConfig, extractResult, dqSummary, transformResult, null, enrichSummary);
|
|
163
|
+
const mergeSummary = {
|
|
164
|
+
rowsMerged: mergeResult.rowsMerged,
|
|
165
|
+
conflicts: mergeResult.conflicts,
|
|
166
|
+
unmatched: mergeResult.unmatched,
|
|
167
|
+
};
|
|
168
|
+
if (postMergeConfig.run.dryRun || postMergeConfig.run.mode === 'validate-only') {
|
|
169
|
+
logger.info({ dryRun: postMergeConfig.run.dryRun, mode: postMergeConfig.run.mode }, 'pipeline: stopping before load');
|
|
170
|
+
this.progress.summary({
|
|
171
|
+
pipeline: postMergeConfig.pipeline.name,
|
|
172
|
+
elapsedMs: Date.now() - runStartMs,
|
|
173
|
+
rowsExtracted: extractResult.rowsExtracted,
|
|
174
|
+
warnings: dqSummary.violations.warning,
|
|
175
|
+
state: dqSummary.violations.warning > 0 || mergeResult.conflicts > 0 ? 'warn' : 'success',
|
|
176
|
+
});
|
|
177
|
+
return {
|
|
178
|
+
...baseRunResult,
|
|
179
|
+
merge: mergeSummary,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
const loadResult = await this.runLoad(postMergeConfig, store);
|
|
183
|
+
const stateFilePath = await this.writeStateFile(postMergeConfig, extractResult, dqSummary, loadResult, enrichSummary);
|
|
184
|
+
logger.info({
|
|
185
|
+
pipeline: postMergeConfig.pipeline.name,
|
|
186
|
+
rowsMerged: mergeResult.rowsMerged,
|
|
187
|
+
rowsLoaded: loadResult.rowsLoaded,
|
|
188
|
+
}, 'pipeline: done (multi-source)');
|
|
189
|
+
this.progress.summary({
|
|
190
|
+
pipeline: postMergeConfig.pipeline.name,
|
|
191
|
+
elapsedMs: Date.now() - runStartMs,
|
|
192
|
+
rowsExtracted: extractResult.rowsExtracted,
|
|
193
|
+
rowsLoaded: loadResult.rowsLoaded,
|
|
194
|
+
warnings: dqSummary.violations.warning,
|
|
195
|
+
state: dqSummary.violations.warning > 0 || mergeResult.conflicts > 0 ? 'warn' : 'success',
|
|
196
|
+
});
|
|
197
|
+
return {
|
|
198
|
+
...this.buildRunResult(postMergeConfig, extractResult, dqSummary, transformResult, loadResult, enrichSummary),
|
|
199
|
+
merge: mergeSummary,
|
|
200
|
+
stateFilePath,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
finally {
|
|
204
|
+
await store.close();
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
async writeStateFile(config, extract, dq, load, enrichSummary) {
|
|
208
|
+
const outputDir = path.resolve(config.run.outputDir);
|
|
209
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
210
|
+
const stateFilePath = path.join(outputDir, `${config.pipeline.name}-state.json`);
|
|
211
|
+
const sourcesBlock = Object.fromEntries(this.sourceExtracts.map((s) => [
|
|
212
|
+
s.sourceId,
|
|
213
|
+
{
|
|
214
|
+
lastRunAt: new Date().toISOString(),
|
|
215
|
+
rowsExtracted: s.rowsExtracted,
|
|
216
|
+
incrementalSince: this.sourceIncrementalSince[s.sourceId] ?? '',
|
|
217
|
+
},
|
|
218
|
+
]));
|
|
219
|
+
const state = {
|
|
220
|
+
pipeline: config.pipeline.name,
|
|
221
|
+
lastRunAt: new Date().toISOString(),
|
|
222
|
+
lastMode: config.run.mode,
|
|
223
|
+
rowsMerged: dq.rowsChecked,
|
|
224
|
+
rowsLoaded: load.rowsLoaded,
|
|
225
|
+
criticalViolations: dq.violations.critical,
|
|
226
|
+
warnings: dq.violations.warning,
|
|
227
|
+
incrementalSince: config.run.incrementalSince ?? '',
|
|
228
|
+
sources: sourcesBlock,
|
|
229
|
+
};
|
|
230
|
+
if (enrichSummary !== undefined) {
|
|
231
|
+
state['enrichSummary'] = enrichSummary;
|
|
232
|
+
}
|
|
233
|
+
await fs.writeFile(stateFilePath, `${JSON.stringify(state, null, 2)}\n`, 'utf-8');
|
|
234
|
+
logger.debug({ stateFilePath }, 'pipeline: state file written');
|
|
235
|
+
return stateFilePath;
|
|
236
|
+
}
|
|
237
|
+
async extractOneSource(config, store, sourceEntry, runSourceDq = true, incrementalSince) {
|
|
238
|
+
const tableName = `stg_raw_${sourceEntry.id}`;
|
|
239
|
+
const sourceScopedConfig = this.buildSingleSourceConfig(config, sourceEntry);
|
|
240
|
+
const extractResult = await this.runExtract(sourceScopedConfig, store, tableName, `Extract (${sourceEntry.id})`);
|
|
241
|
+
this.sourceExtracts.push({
|
|
242
|
+
sourceId: sourceEntry.id,
|
|
243
|
+
rowsExtracted: extractResult.rowsExtracted,
|
|
244
|
+
tableName,
|
|
245
|
+
});
|
|
246
|
+
if (sourceEntry.rename && Object.keys(sourceEntry.rename).length > 0) {
|
|
247
|
+
await store.renameColumns(tableName, sourceEntry.rename);
|
|
248
|
+
}
|
|
249
|
+
if (incrementalSince && config.run.incrementalField) {
|
|
250
|
+
await this.applyIncrementalFilterForSource(store, tableName, config.run.incrementalField, incrementalSince);
|
|
251
|
+
const filteredRows = await store.rowCount(tableName);
|
|
252
|
+
const current = this.sourceExtracts.find((s) => s.sourceId === sourceEntry.id);
|
|
253
|
+
if (current)
|
|
254
|
+
current.rowsExtracted = filteredRows;
|
|
255
|
+
logger.info({
|
|
256
|
+
sourceId: sourceEntry.id,
|
|
257
|
+
tableName,
|
|
258
|
+
incrementalField: config.run.incrementalField,
|
|
259
|
+
incrementalSince,
|
|
260
|
+
rowsAfterFilter: filteredRows,
|
|
261
|
+
}, 'pipeline: incremental filter applied');
|
|
262
|
+
}
|
|
263
|
+
if (!runSourceDq)
|
|
264
|
+
return;
|
|
265
|
+
const sourceRules = config.dq.rules.filter((r) => r.sourceId === sourceEntry.id);
|
|
266
|
+
if (sourceRules.length === 0)
|
|
267
|
+
return;
|
|
268
|
+
const perSourceDqConfig = {
|
|
269
|
+
...sourceScopedConfig,
|
|
270
|
+
dq: {
|
|
271
|
+
...sourceScopedConfig.dq,
|
|
272
|
+
rejectionFile: sourceRejectionPath(config, sourceEntry.id),
|
|
273
|
+
rules: sourceRules,
|
|
274
|
+
},
|
|
275
|
+
};
|
|
276
|
+
const sourceSummary = await this.runDQ(perSourceDqConfig, store, tableName, sourceEntry.id, `Data quality (${sourceEntry.id})`);
|
|
277
|
+
if (config.dq.stopOnCritical && sourceSummary.violations.critical > 0) {
|
|
278
|
+
throw new PipelineDQError(sourceSummary.violations.critical, sourceSummary.reportPath);
|
|
279
|
+
}
|
|
280
|
+
const currentColumns = asColumns(await store.columnNames(tableName));
|
|
281
|
+
const acceptedTable = await this.materializeAcceptedRows(store, tableName, currentColumns, sourceSummary.rejectedRowIndices ?? []);
|
|
282
|
+
if (acceptedTable !== tableName) {
|
|
283
|
+
await store.query(`CREATE OR REPLACE TABLE ${quoteIdent(tableName)} AS SELECT * FROM ${quoteIdent(acceptedTable)}`);
|
|
284
|
+
await store.dropTable(acceptedTable);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
async resolveIncrementalSinceForSource(config, incrementalSourceId) {
|
|
288
|
+
if (config.run.incrementalSince)
|
|
289
|
+
return config.run.incrementalSince;
|
|
290
|
+
const statePath = path.join(path.resolve(config.run.outputDir), `${config.pipeline.name}-state.json`);
|
|
291
|
+
try {
|
|
292
|
+
const raw = await fs.readFile(statePath, 'utf-8');
|
|
293
|
+
const parsed = JSON.parse(raw);
|
|
294
|
+
const lastRunAt = parsed.sources?.[incrementalSourceId]?.lastRunAt;
|
|
295
|
+
return typeof lastRunAt === 'string' ? lastRunAt : '';
|
|
296
|
+
}
|
|
297
|
+
catch {
|
|
298
|
+
return '';
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
async applyIncrementalFilterForSource(store, tableName, incrementalField, incrementalSince) {
|
|
302
|
+
const columns = await store.columnNames(tableName);
|
|
303
|
+
if (!columns.includes(incrementalField)) {
|
|
304
|
+
throw new ConfigError(`incrementalField "${incrementalField}" was not found in source table "${tableName}"`);
|
|
305
|
+
}
|
|
306
|
+
const sinceCheck = await store.query('SELECT TRY_CAST(? AS TIMESTAMP) AS ts', [incrementalSince]);
|
|
307
|
+
if (!sinceCheck[0]?.ts) {
|
|
308
|
+
throw new ConfigError(`run.incrementalSince "${incrementalSince}" is not a valid timestamp`);
|
|
309
|
+
}
|
|
310
|
+
const invalidRows = await store.query(`
|
|
311
|
+
SELECT count(*) AS n
|
|
312
|
+
FROM ${quoteIdent(tableName)}
|
|
313
|
+
WHERE ${quoteIdent(incrementalField)} IS NOT NULL
|
|
314
|
+
AND TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) IS NULL
|
|
315
|
+
`);
|
|
316
|
+
const invalidCount = Number(invalidRows[0]?.n ?? 0);
|
|
317
|
+
if (invalidCount > 0) {
|
|
318
|
+
const examples = await store.query(`
|
|
319
|
+
SELECT DISTINCT ${quoteIdent(incrementalField)} AS v
|
|
320
|
+
FROM ${quoteIdent(tableName)}
|
|
321
|
+
WHERE ${quoteIdent(incrementalField)} IS NOT NULL
|
|
322
|
+
AND TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) IS NULL
|
|
323
|
+
LIMIT 5
|
|
324
|
+
`);
|
|
325
|
+
const sampleValues = examples
|
|
326
|
+
.map((e) => (e.v === null || e.v === undefined ? '' : String(e.v)))
|
|
327
|
+
.filter((v) => v.length > 0)
|
|
328
|
+
.join(', ');
|
|
329
|
+
throw new ConfigError(`incrementalField "${incrementalField}" contains ${invalidCount} non-parseable timestamp value(s)` +
|
|
330
|
+
(sampleValues ? ` (examples: ${sampleValues})` : ''));
|
|
331
|
+
}
|
|
332
|
+
await store.query(`
|
|
333
|
+
CREATE OR REPLACE TABLE ${quoteIdent(tableName)} AS
|
|
334
|
+
SELECT *
|
|
335
|
+
FROM ${quoteIdent(tableName)}
|
|
336
|
+
WHERE TRY_CAST(${quoteIdent(incrementalField)} AS TIMESTAMP) >= TRY_CAST(? AS TIMESTAMP)
|
|
337
|
+
`, [incrementalSince]);
|
|
338
|
+
}
|
|
339
|
+
async buildProfileFromTable(store, tableName, columns) {
|
|
340
|
+
const result = [];
|
|
341
|
+
const rowCountRow = await store.query(`SELECT count(*) AS n FROM ${quoteIdent(tableName)}`);
|
|
342
|
+
const rowCount = Number(rowCountRow[0]?.n ?? 0);
|
|
343
|
+
for (const col of columns) {
|
|
344
|
+
const qCol = quoteIdent(col.name);
|
|
345
|
+
const [nullCountRow] = await store.query(`SELECT count(*) AS n FROM ${quoteIdent(tableName)} WHERE ${qCol} IS NULL`);
|
|
346
|
+
const [distinctRow] = await store.query(`SELECT count(DISTINCT ${qCol}) AS n FROM ${quoteIdent(tableName)}`);
|
|
347
|
+
const lenRow = await store.query(`SELECT min(length(CAST(${qCol} AS VARCHAR))) AS mn, max(length(CAST(${qCol} AS VARCHAR))) AS mx FROM ${quoteIdent(tableName)}`);
|
|
348
|
+
const sampleRows = await store.query(`SELECT DISTINCT ${qCol} AS v FROM ${quoteIdent(tableName)} WHERE ${qCol} IS NOT NULL LIMIT 5`);
|
|
349
|
+
result.push({
|
|
350
|
+
name: col.name,
|
|
351
|
+
duckDbType: col.duckDbType,
|
|
352
|
+
rowCount,
|
|
353
|
+
nullCount: Number(nullCountRow?.n ?? 0),
|
|
354
|
+
distinctCount: Number(distinctRow?.n ?? 0),
|
|
355
|
+
minLen: lenRow[0]?.mn === null || lenRow[0]?.mn === undefined ? null : Number(lenRow[0].mn),
|
|
356
|
+
maxLen: lenRow[0]?.mx === null || lenRow[0]?.mx === undefined ? null : Number(lenRow[0].mx),
|
|
357
|
+
sample: sampleRows.map((r) => r.v === null || r.v === undefined ? null : String(r.v)),
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
return result;
|
|
361
|
+
}
|
|
362
|
+
buildSingleSourceConfig(config, sourceEntry) {
|
|
363
|
+
const next = {
|
|
364
|
+
...config,
|
|
365
|
+
source: {
|
|
366
|
+
adapter: sourceEntry.adapter,
|
|
367
|
+
connection: sourceEntry.connection,
|
|
368
|
+
query: sourceEntry.query,
|
|
369
|
+
file: sourceEntry.file,
|
|
370
|
+
endpoint: sourceEntry.endpoint,
|
|
371
|
+
headers: sourceEntry.headers,
|
|
372
|
+
delimiter: sourceEntry.delimiter,
|
|
373
|
+
encoding: sourceEntry.encoding,
|
|
374
|
+
sheet: sourceEntry.sheet,
|
|
375
|
+
pagination: sourceEntry.pagination,
|
|
376
|
+
},
|
|
377
|
+
};
|
|
378
|
+
delete next['sources'];
|
|
379
|
+
delete next['merge'];
|
|
380
|
+
return next;
|
|
381
|
+
}
|
|
382
|
+
buildPostMergeConfig(config) {
|
|
383
|
+
// The merge phase has already produced stg_merged; downstream phases
|
|
384
|
+
// (runDQ, runTransform, runLoad) never read config.source, so we drop
|
|
385
|
+
// source/sources/merge to get a single-source-shaped config without
|
|
386
|
+
// revalidating. Post-merge DQ rules are the ones without a sourceId.
|
|
387
|
+
const postMergeRules = config.dq.rules.filter((r) => r.sourceId === undefined);
|
|
388
|
+
const { source: _src, sources: _srcs, merge: _merge, ...rest } = config;
|
|
389
|
+
void _src;
|
|
390
|
+
void _srcs;
|
|
391
|
+
void _merge;
|
|
392
|
+
return {
|
|
393
|
+
...rest,
|
|
394
|
+
dq: { ...config.dq, rules: postMergeRules },
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
//# sourceMappingURL=multi-source-runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multi-source-runner.js","sourceRoot":"","sources":["../src/multi-source-runner.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAGlC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAwC,MAAM,mBAAmB,CAAC;AAGxF,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAqC,MAAM,aAAa,CAAC;AAChF,OAAO,EAAmB,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE/E,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACjE,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAQ3C,SAAS,SAAS,CAAC,KAAe;IAChC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAgB,EAAE,QAAgB;IAC7D,MAAM,UAAU,GAAG,MAAM,CAAC,EAAE,CAAC,aAAa,CAAC;IAC3C,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,IAAI,IAAI,QAAQ,GAAG,MAAM,CAAC,GAAG,IAAI,MAAM,EAAE,CAAC,CAAC;AACpF,CAAC;AAED,MAAM,OAAO,yBAA0B,SAAQ,cAAc;IAC1C,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IACzC,cAAc,GAAwB,EAAE,CAAC;IACzC,sBAAsB,GAA2B,EAAE,CAAC;IAE5D,8EAA8E;IACtE,mBAAmB,CAAC,OAA2B;QACrD,OAAO,OAAO;aACX,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACb,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;YACzD,IAAI,UAAU,KAAK,CAAC;gBAAE,OAAO,UAAU,CAAC;YACxC,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;QAC3B,CAAC,CAAC;aACD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAEQ,KAAK,CAAC,OAAO,CAAC,QAAgB,EAAE,YAA0B,EAAE;QACnE,IAAI,SAAS,CAAC,QAAQ;YAAE,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;QAC3D,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,IAAI,CAAC,cAAc,CACvB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EACpC,SAAS,CAAC,UAAU,IAAI,EAAE,CAC3B,CAAC;QAEF,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,WAAW,CACnB,aAAa,MAAM,CAAC,QAAQ,CAAC,IAAI,wEAAwE,CAC1G,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACtC,MAAM,IAAI,WAAW,CAAC,8DAA8D,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YAEnB,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;YACzB,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC/D,KAAK,MAAM,WAAW,IAAI,aAAa,EAAE,CAAC;gBACxC,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YACjE,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAC5C,KAAK,EACL,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,SAAS,EAAE,WAAW,CAAC,CAAC,EAAE,EAAE;aAC7B,CAAC,CAAC,EACH,MAAM,CAAC,KAAK,CACb,CAAC;YAEF,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;YACvE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC;YACrF,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,eAAe,CAAC,CAAC;YACjF,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAClF,MAAM,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,4BAA4B,CAAC,CAAC;YAEpF,MAAM,aAAa,GAAkB;gBACnC,aAAa,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC;gBAC/E,SAAS,EAAE,WAAW,CAAC,SAAS;gBAChC,OAAO,EAAE,aAAa;aACvB,CAAC;YAEF,OAAO;gBACL,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI;gBAC9B,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI;gBACrB,OAAO,EAAE,aAAa;gBACtB,EAAE,EAAE;oBACF,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI;oBAC9B,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC/B,WAAW,EAAE,WAAW,CAAC,UAAU;oBACnC,UAAU,EAAE,WAAW,CAAC,UAAU;oBAClC,YAAY,EAAE,CAAC;oBACf,UAAU,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE;oBAChD,OAAO,EAAE,EAAE;oBACX,UAAU,EAAE,EAAE;iBACf;gBACD,SAAS,EAAE,IAAI;gBACf,IAAI,EAAE,IAAI;gBACV,WAAW;aACZ,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAEQ,KAAK,CAAC,GAAG,CAAC,QAAgB,EAAE,YAA0B,EAAE;QAC/D,IAAI,SAAS,CAAC,QAAQ;YAAE,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;QAC3D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,IAAI,CAAC,cAAc,CACvB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EACpC,SAAS,CAAC,UAAU,IAAI,EAAE,CAC3B,CAAC;QAEF,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,WAAW,CACnB,aAAa,MAAM,CAAC,QAAQ,CAAC,IAAI,wEAAwE,CAC1G,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YACtE,MAAM,IAAI,WAAW,CAAC,yEAAyE,CAAC,CAAC;QACnG,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YAEnB,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC,sBAAsB,GAAG,MAAM,CAAC,WAAW,CAC9C,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CACZ,CAAC;YAC5B,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC/D,MAAM,mBAAmB,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,aAAa;gBAC3D,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB;gBAChC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,gBAAgB,GAAG,mBAAmB;gBAC1C,CAAC,CAAC,MAAM,IAAI,CAAC,gCAAgC,CAAC,MAAM,EAAE,mBAAmB,CAAC;gBAC1E,CAAC,CAAC,EAAE,CAAC;YAEP,IAAI,mBAAmB,EAAE,CAAC;gBACxB,IAAI,CAAC,sBAAsB,CAAC,mBAAmB,CAAC,GAAG,gBAAgB,CAAC;YACtE,CAAC;YAED,KAAK,MAAM,WAAW,IAAI,aAAa,EAAE,CAAC;gBACxC,MAAM,IAAI,CAAC,gBAAgB,CACzB,MAAM,EACN,KAAK,EACL,WAAW,EACX,IAAI,EACJ,mBAAmB,KAAK,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CACtE,CAAC;YACJ,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,EAAE,UAAU,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,UAAU,GAAG,CAAC,CAAC;YACpF,IAAI,WAAW,CAAC;YAChB,IAAI,CAAC;gBACH,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CACtC,KAAK,EACL,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACxB,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,SAAS,EAAE,WAAW,CAAC,CAAC,EAAE,EAAE;iBAC7B,CAAC,CAAC,EACH,MAAM,CAAC,KAAK,CACb,CAAC;gBACF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBAC7C,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACrB,KAAK,EAAE,WAAW,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;iBACtD,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;gBAC1C,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;YAE1D,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,SAAS,CACxC,eAAe,EACf,KAAK,EACL,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EACpC,SAAS,CACV,CAAC;YAEF,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAChC,eAAe,EACf,KAAK,EACL,YAAY,EACZ,SAAS,EACT,2BAA2B,CAC5B,CAAC;YAEF,IAAI,eAAe,CAAC,EAAE,CAAC,cAAc,IAAI,SAAS,CAAC,UAAU,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;gBAC3E,MAAM,IAAI,eAAe,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;YACjF,CAAC;YAED,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;YACvE,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC7D,KAAK,EACL,YAAY,EACZ,aAAa,EACb,SAAS,CAAC,kBAAkB,IAAI,EAAE,CACnC,CAAC;YACF,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,YAAY,CAC7C,eAAe,EACf,KAAK,EACL,oBAAoB,EACpB,iBAAiB,CAClB,CAAC;YAEF,MAAM,aAAa,GAAkB;gBACnC,aAAa,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC;gBAC/E,SAAS,EAAE,WAAW,CAAC,SAAS;gBAChC,OAAO,EAAE,aAAa;aACvB,CAAC;YAEF,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CACvC,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EACf,IAAI,EACJ,aAAa,CACd,CAAC;YACF,MAAM,YAAY,GAAG;gBACnB,UAAU,EAAE,WAAW,CAAC,UAAU;gBAClC,SAAS,EAAE,WAAW,CAAC,SAAS;gBAChC,SAAS,EAAE,WAAW,CAAC,SAAS;aACjC,CAAC;YAEF,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBAC/E,MAAM,CAAC,IAAI,CACT,EAAE,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,EACtE,gCAAgC,CACjC,CAAC;gBACF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;oBACpB,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC,IAAI;oBACvC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;oBAClC,aAAa,EAAE,aAAa,CAAC,aAAa;oBAC1C,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC,OAAO;oBACtC,KAAK,EAAE,SAAS,CAAC,UAAU,CAAC,OAAO,GAAG,CAAC,IAAI,WAAW,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;iBAC1F,CAAC,CAAC;gBACH,OAAO;oBACL,GAAG,aAAa;oBAChB,KAAK,EAAE,YAAY;iBACpB,CAAC;YACJ,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;YAC9D,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,cAAc,CAC7C,eAAe,EACf,aAAa,EACb,SAAS,EACT,UAAU,EACV,aAAa,CACd,CAAC;YAEF,MAAM,CAAC,IAAI,CACT;gBACE,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC,IAAI;gBACvC,UAAU,EAAE,WAAW,CAAC,UAAU;gBAClC,UAAU,EAAE,UAAU,CAAC,UAAU;aAClC,EACD,+BAA+B,CAChC,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACpB,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC,IAAI;gBACvC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;gBAClC,aAAa,EAAE,aAAa,CAAC,aAAa;gBAC1C,UAAU,EAAE,UAAU,CAAC,UAAU;gBACjC,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC,OAAO;gBACtC,KAAK,EAAE,SAAS,CAAC,UAAU,CAAC,OAAO,GAAG,CAAC,IAAI,WAAW,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;aAC1F,CAAC,CAAC;YAEH,OAAO;gBACL,GAAG,IAAI,CAAC,cAAc,CACpB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EACf,UAAU,EACV,aAAa,CACd;gBACD,KAAK,EAAE,YAAY;gBACnB,aAAa;aACd,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAEkB,KAAK,CAAC,cAAc,CACrC,MAAgB,EAChB,OAAsB,EACtB,EAAa,EACb,IAAgB,EAChB,aAA6B;QAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,aAAa,CAAC,CAAC;QAEjF,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CACrC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YAC7B,CAAC,CAAC,QAAQ;YACV;gBACE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE,CAAC,CAAC,aAAa;gBAC9B,gBAAgB,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE;aAChE;SACF,CAAC,CACH,CAAC;QAEF,MAAM,KAAK,GAA4B;YACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI;YAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI;YACzB,UAAU,EAAE,EAAE,CAAC,WAAW;YAC1B,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,kBAAkB,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ;YAC1C,QAAQ,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO;YAC/B,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,IAAI,EAAE;YACnD,OAAO,EAAE,YAAY;SACtB,CAAC;QACF,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAChC,KAAK,CAAC,eAAe,CAAC,GAAG,aAAa,CAAC;QACzC,CAAC;QAED,MAAM,EAAE,CAAC,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAClF,MAAM,CAAC,KAAK,CAAC,EAAE,aAAa,EAAE,EAAE,8BAA8B,CAAC,CAAC;QAChE,OAAO,aAAa,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,MAAgB,EAChB,KAAmB,EACnB,WAA6B,EAC7B,WAAW,GAAG,IAAI,EAClB,gBAAyB;QAEzB,MAAM,SAAS,GAAG,WAAW,WAAW,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,kBAAkB,GAAG,IAAI,CAAC,uBAAuB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAE7E,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,UAAU,CACzC,kBAAkB,EAClB,KAAK,EACL,SAAS,EACT,YAAY,WAAW,CAAC,EAAE,GAAG,CAC9B,CAAC;QACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;YACvB,QAAQ,EAAE,WAAW,CAAC,EAAE;YACxB,aAAa,EAAE,aAAa,CAAC,aAAa;YAC1C,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,WAAW,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrE,MAAM,KAAK,CAAC,aAAa,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,gBAAgB,IAAI,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YACpD,MAAM,IAAI,CAAC,+BAA+B,CACxC,KAAK,EACL,SAAS,EACT,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAC3B,gBAAgB,CACjB,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACrD,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,WAAW,CAAC,EAAE,CAAC,CAAC;YAC/E,IAAI,OAAO;gBAAE,OAAO,CAAC,aAAa,GAAG,YAAY,CAAC;YAClD,MAAM,CAAC,IAAI,CACT;gBACE,QAAQ,EAAE,WAAW,CAAC,EAAE;gBACxB,SAAS;gBACT,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB;gBAC7C,gBAAgB;gBAChB,eAAe,EAAE,YAAY;aAC9B,EACD,sCAAsC,CACvC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,WAAW;YAAE,OAAO;QAEzB,MAAM,WAAW,GAAG,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,WAAW,CAAC,EAAE,CAAC,CAAC;QACjF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAErC,MAAM,iBAAiB,GAAa;YAClC,GAAG,kBAAkB;YACrB,EAAE,EAAE;gBACF,GAAG,kBAAkB,CAAC,EAAE;gBACxB,aAAa,EAAE,mBAAmB,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,CAAC;gBAC1D,KAAK,EAAE,WAAW;aACnB;SACF,CAAC;QACF,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CACpC,iBAAiB,EACjB,KAAK,EACL,SAAS,EACT,WAAW,CAAC,EAAE,EACd,iBAAiB,WAAW,CAAC,EAAE,GAAG,CACnC,CAAC;QAEF,IAAI,MAAM,CAAC,EAAE,CAAC,cAAc,IAAI,aAAa,CAAC,UAAU,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,eAAe,CAAC,aAAa,CAAC,UAAU,CAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;QACzF,CAAC;QAED,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC;QACrE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACtD,KAAK,EACL,SAAS,EACT,cAAc,EACd,aAAa,CAAC,kBAAkB,IAAI,EAAE,CACvC,CAAC;QAEF,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,KAAK,CAAC,KAAK,CACf,2BAA2B,UAAU,CAAC,SAAS,CAAC,qBAAqB,UAAU,CAAC,aAAa,CAAC,EAAE,CACjG,CAAC;YACF,MAAM,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gCAAgC,CAAC,MAAgB,EAAE,mBAA2B;QAC1F,IAAI,MAAM,CAAC,GAAG,CAAC,gBAAgB;YAAE,OAAO,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC;QAEpE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CACzB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,EAClC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,aAAa,CACrC,CAAC;QACF,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAE5B,CAAC;YACF,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,mBAAmB,CAAC,EAAE,SAAS,CAAC;YACnE,OAAO,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,+BAA+B,CAC3C,KAAmB,EACnB,SAAiB,EACjB,gBAAwB,EACxB,gBAAwB;QAExB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,WAAW,CACnB,qBAAqB,gBAAgB,oCAAoC,SAAS,GAAG,CACtF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,KAAK,CAClC,uCAAuC,EACvC,CAAC,gBAAgB,CAAC,CACnB,CAAC;QACF,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,WAAW,CACnB,yBAAyB,gBAAgB,4BAA4B,CACtE,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,KAAK,CACnC;;aAEO,UAAU,CAAC,SAAS,CAAC;cACpB,UAAU,CAAC,gBAAgB,CAAC;uBACnB,UAAU,CAAC,gBAAgB,CAAC;OAC5C,CACF,CAAC;QACF,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAChC;0BACkB,UAAU,CAAC,gBAAgB,CAAC;eACvC,UAAU,CAAC,SAAS,CAAC;gBACpB,UAAU,CAAC,gBAAgB,CAAC;yBACnB,UAAU,CAAC,gBAAgB,CAAC;;SAE5C,CACF,CAAC;YACF,MAAM,YAAY,GAAG,QAAQ;iBAC1B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBAClE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;iBAC3B,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,MAAM,IAAI,WAAW,CACnB,qBAAqB,gBAAgB,cAAc,YAAY,mCAAmC;gBAClG,CAAC,YAAY,CAAC,CAAC,CAAC,eAAe,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CACrD,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,CACf;gCAC0B,UAAU,CAAC,SAAS,CAAC;;aAExC,UAAU,CAAC,SAAS,CAAC;uBACX,UAAU,CAAC,gBAAgB,CAAC;OAC5C,EACD,CAAC,gBAAgB,CAAC,CACnB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,qBAAqB,CACjC,KAAmB,EACnB,SAAiB,EACjB,OAAqB;QAWrB,MAAM,MAAM,GASP,EAAE,CAAC;QAER,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,KAAK,CACnC,6BAA6B,UAAU,CAAC,SAAS,CAAC,EAAE,CACrD,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QAEhD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,CAAC,YAAY,CAAC,GAAG,MAAM,KAAK,CAAC,KAAK,CACtC,6BAA6B,UAAU,CAAC,SAAS,CAAC,UAAU,IAAI,UAAU,CAC3E,CAAC;YACF,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,KAAK,CAAC,KAAK,CACrC,yBAAyB,IAAI,eAAe,UAAU,CAAC,SAAS,CAAC,EAAE,CACpE,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAC9B,0BAA0B,IAAI,yCAAyC,IAAI,6BAA6B,UAAU,CAAC,SAAS,CAAC,EAAE,CAChI,CAAC;YACF,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,KAAK,CAClC,mBAAmB,IAAI,cAAc,UAAU,CAAC,SAAS,CAAC,UAAU,IAAI,sBAAsB,CAC/F,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,QAAQ;gBACR,SAAS,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,IAAI,CAAC,CAAC;gBACvC,aAAa,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC;gBAC1C,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3F,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3F,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3B,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CACvD;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,uBAAuB,CAAC,MAAgB,EAAE,WAA6B;QAC7E,MAAM,IAAI,GAAG;YACX,GAAG,MAAM;YACT,MAAM,EAAE;gBACN,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,UAAU,EAAE,WAAW,CAAC,UAAU;gBAClC,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,QAAQ,EAAE,WAAW,CAAC,QAAQ;gBAC9B,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,SAAS,EAAE,WAAW,CAAC,SAAS;gBAChC,QAAQ,EAAE,WAAW,CAAC,QAAQ;gBAC9B,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,UAAU,EAAE,WAAW,CAAC,UAAU;aACnC;SACF,CAAC;QACF,OAAQ,IAAgC,CAAC,SAAS,CAAC,CAAC;QACpD,OAAQ,IAAgC,CAAC,OAAO,CAAC,CAAC;QAClD,OAAO,IAAgB,CAAC;IAC1B,CAAC;IAEO,oBAAoB,CAAC,MAAgB;QAC3C,qEAAqE;QACrE,sEAAsE;QACtE,oEAAoE;QACpE,qEAAqE;QACrE,MAAM,cAAc,GAAG,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;QAC/E,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;QACxE,KAAK,IAAI,CAAC;QACV,KAAK,KAAK,CAAC;QACX,KAAK,MAAM,CAAC;QACZ,OAAO;YACL,GAAG,IAAI;YACP,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE;SAChC,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { RuleRegistry, TransformRegistry } from './registry.js';
|
|
2
|
+
export { loadPlugins, loadNpmPlugins } from './loader.js';
|
|
3
|
+
export type { RulePlugin, RuleViolation, TransformPlugin, CustomFieldMapping, PluginPackage } from './types.js';
|
|
4
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/plugins/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC1D,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}
|