@caracal-lynx/sluice 0.1.0

package/dist/config/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAoCxB,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAGxB,CAAC;AAOF,eAAO,MAAM,SAAS,gHAGpB,CAAC;AAEH,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;EAKf,CAAC;AAEH,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAIhB,CAAC;AAEH,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKnB,CAAC;AAUH,QAAA,MAAM,wBAAwB;;;;;;yBAEP,CAAC;AAExB,QAAA,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAQtB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAIvB,CAAC;AAIH,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKhB,CAAC;AAeH,QAAA,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsBvB,CAAC;AAEF,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAG1B,CAAC;AAIH,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyBxB,CAAC;AAIF,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAcpB,CAAC;AAaH,eAAO,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAQtB,CAAC;AAEH,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EASlC,CAAC;AAYF,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiC1B,CAAC;AAIF,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;EAM9B,CAAC;AAMH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAQ9B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAGrC,CAAC;AAKH,MAAM,MAAM,QAAQ,GAAe,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAClE,MAAM,MAAM,YAAY,GAAW,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAChE,MAAM,MAAM,YAAY,GAAW,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAChE,MAAM,MAAM,SAAS,GAAc,CAAC,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,CAAC;AAC7D,MAAM,MAAM,YAAY,GAAW,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AACtE,MAAM,MAAM,MAAM,GAAiB,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAChE,MAAM,MAAM,WAAW,GAAY,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAC/D,MAAM,MAAM,MAAM,GAAiB,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAChE,MAAM,MAAM,aAAa,GAAU,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AACvE,MAAM,MAAM,aAAa,GAAU,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AACvE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,WAAW,GAAY,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAC/D,MAAM,MAAM,gBAAgB,GAAO,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAC1E,MAAM,MAAM,YAAY,GAAW,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAChE,MAAM,MAAM,kBAAkB,GAAK,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AACtE,MAAM,MAAM,kBAAkB,GAAK,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAI5E,kFAAkF;AAClF,wBAAgB,cAAc,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG;IAAE,MAAM,EAAE,YAAY,CAAA;CAAE,CAEpF;AAED,4FAA4F;AAC5F,wBAAgB,aAAa,CAC3B,CAAC,EAAE,QAAQ,GACV,CAAC,IAAI,QAAQ,GAAG;IAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAErE"}

package/dist/config/schema.js

@@ -0,0 +1,263 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+/**
+ * Sluice — pipeline config schema
+ * @caracal-lynx/sluice
+ *
+ * Canonical Zod schema for all pipeline YAML configs.
+ * All TypeScript types are inferred from these schemas — do not write
+ * manual interfaces for anything that maps to pipeline config.
+ *
+ * Usage:
+ *   import { PipelineSchema } from '@caracal-lynx/sluice';
+ *   const pipeline = PipelineSchema.parse(yaml.load(fs.readFileSync('customers.pipeline.yaml', 'utf8')));
+ *
+ * ${ENV_VAR} strings in connection/url fields are resolved by ConfigLoader
+ * before this schema is called — Zod sees plain strings, not tokens.
+ */
+import { z } from 'zod';
+// ── Primitives ────────────────────────────────────────────────────────────────
+const Severity = z.enum(['critical', 'warning', 'info']);
+const SourceAd = z.enum(['mssql', 'pg', 'csv', 'xlsx', 'rest']);
+const TargetAd = z.enum(['bc', 'ifs', 'bluecherry', 'csv', 'pg', 'rest']);
+const CleanseOps = z.string().regex(/^[a-zA-Z|:0-9]+$/);
+// ── Pagination (REST source) ──────────────────────────────────────────────────
+const PaginationSchema = z.object({
+    type: z.enum(['offset', 'cursor', 'page']),
+    pageSize: z.number().int().positive().default(100),
+    pageParam: z.string().optional(),
+    totalField: z.string().optional(),
+    dataField: z.string().optional(),
+    cursorField: z.string().optional(),
+    cursorParam: z.string().optional(),
+});
+// ── Source ────────────────────────────────────────────────────────────────────
+const SourceBaseSchema = z.object({
+    adapter: SourceAd,
+    connection: z.string().optional(),
+    query: z.string().optional(),
+    file: z.string().optional(),
+    endpoint: z.string().optional(),
+    headers: z.record(z.string()).optional(),
+    delimiter: z.string().default(','),
+    encoding: z.string().default('utf-8'),
+    sheet: z.union([z.string(), z.number()]).optional(),
+    pagination: PaginationSchema.optional(),
+});
+export const SourceSchema = SourceBaseSchema.refine(s => s.query || s.file || s.endpoint, { message: 'source must have query, file, or endpoint' });
+// ── DQ ────────────────────────────────────────────────────────────────────────
+// Exported because (a) ConfigLoader needs .options at runtime to discriminate
+// built-in checks from composite-rule references during expansion, and
+// (b) plugin authors may want to introspect the built-in check set.
+export const CheckType = z.enum([
+    'notNull', 'unique', 'pattern', 'email', 'ukPostcode',
+    'maxLength', 'min', 'max', 'allowedValues',
+]);
+const CheckSchema = z.object({
+    type: CheckType,
+    value: z.union([z.string(), z.number(), z.array(z.string())]).optional(),
+    severity: Severity,
+    message: z.string().optional(),
+});
+const DqRuleSchema = z.object({
+    field: z.string(),
+    sourceId: z.string().optional(), // scopes a rule to a named source in multi-source pipelines; ignored in single-source mode
+    checks: z.array(CheckSchema).min(1),
+});
+export const DqSchema = z.object({
+    rulesFile: z.string().optional(), // Phase 2: composite rule library path
+    stopOnCritical: z.boolean().default(true),
+    rejectionFile: z.string().optional(),
+    rules: z.array(DqRuleSchema).default([]),
+});
+// ── Enrich (Phase 4a) ─────────────────────────────────────────────────────────
+//
+// The framework that consumes these schemas — EnrichRegistry, EnrichmentRunner,
+// EnrichCache, plugin loader, and built-in providers — lives in the private
+// `@caracal-lynx/sluice-enrich` package and is not part of the open-source core.
+// What ships here is the schema, the public `EnrichPlugin` interface
+// (in src/enrich/types.ts), and the `registerEnrichPhase()` injection hook.
+const EnrichWriteColumnsSchema = z
+    .object({ valid: z.string() }) // 'valid' key is required
+    .catchall(z.string()); // additional data-field mappings are optional
+const EnrichLookupSchema = z.object({
+    field: z.string(),
+    provider: z.string(),
+    writeColumns: EnrichWriteColumnsSchema,
+    preValidate: z.string().optional(),
+    onError: z.enum(['flag', 'skip', 'fail']).optional(),
+    cache: z.boolean().optional(),
+    options: z.record(z.unknown()).optional(),
+});
+export const EnrichSchema = z.object({
+    cache: z.union([z.boolean(), z.literal('persist')]).default(true),
+    onError: z.enum(['flag', 'skip', 'fail']).default('flag'),
+    lookups: z.array(EnrichLookupSchema).min(1),
+});
+// ── Transform ─────────────────────────────────────────────────────────────────
+const LookupSchema = z.object({
+    name: z.string(),
+    source: SourceSchema,
+    key: z.string(),
+    value: z.string(),
+});
+const FieldType = z.enum([
+    'string', 'number', 'decimal', 'boolean', 'date',
+    'lookup', 'concat', 'constant', 'expression',
+    'custom', // Phase 2: delegates to a TransformPlugin via customOp
+]);
+// Field types that read from the source row — `from` must be set.
+// `constant` emits a fixed value, `expression` evaluates against the full row,
+// `custom` receives the whole row and chooses what to use — all three may omit `from`.
+const TYPES_REQUIRING_FROM = new Set([
+    'string', 'number', 'decimal', 'boolean', 'date', 'lookup', 'concat',
+]);
+const FieldMappingSchema = z.object({
+    from: z.union([z.string(), z.array(z.string())]).optional(),
+    to: z.string(),
+    type: FieldType,
+    max: z.number().optional(),
+    precision: z.number().optional(),
+    format: z.string().optional(),
+    cleanse: CleanseOps.optional(),
+    lookup: z.string().optional(),
+    separator: z.string().optional(),
+    value: z.union([z.string(), z.number(), z.boolean()]).optional(),
+    default: z.union([z.string(), z.number(), z.boolean(), z.null()]).optional(),
+    optional: z.boolean().default(false),
+    // Phase 2:
+    customOp: z.string().optional(),
+    options: z.record(z.unknown()).optional(),
+}).refine(m => m.type !== 'custom' || !!m.customOp, { message: 'type: custom requires customOp to be set', path: ['customOp'] }).refine(m => !TYPES_REQUIRING_FROM.has(m.type) || m.from !== undefined, { message: 'this field type requires "from" to be set', path: ['from'] });
+export const TransformSchema = z.object({
+    lookups: z.array(LookupSchema).default([]),
+    fields: z.array(FieldMappingSchema).min(1),
+});
+// ── Target ────────────────────────────────────────────────────────────────────
+export const TargetSchema = z.object({
+    adapter: TargetAd,
+    output: z.string().optional(),
+    entity: z.string().optional(),
+    connection: z.string().optional(),
+    includeHeader: z.boolean().optional(),
+    columnOrder: z.array(z.string()).optional(),
+    dateFormat: z.string().optional(),
+    delimiter: z.string().default(','),
+    encoding: z.string().default('utf-8'),
+    nullValue: z.string().default(''),
+    template: z.string().optional(),
+    // Business Central REST:
+    baseUrl: z.string().optional(),
+    company: z.string().optional(),
+    apiVersion: z.string().default('v2.0'),
+    onConflict: z.enum(['fail', 'upsert', 'ignore']).default('fail'),
+    upsertKey: z.array(z.string()).optional(),
+    batchEndpoint: z.boolean().default(true),
+    // PostgreSQL:
+    table: z.string().optional(),
+    schema: z.string().default('public'),
+}).refine(t => t.onConflict !== 'upsert' || (t.upsertKey !== undefined && t.upsertKey.length > 0), { message: 'upsertKey is required when onConflict is "upsert"', path: ['upsertKey'] });
+// ── Run ───────────────────────────────────────────────────────────────────────
+export const RunSchema = z.object({
+    mode: z.enum(['full', 'incremental', 'validate-only']).default('full'),
+    batchSize: z.number().int().positive().default(500),
+    onError: z.enum(['continue', 'stop']).default('continue'),
+    logLevel: z.enum(['debug', 'info', 'warn', 'error']).default('info'),
+    dryRun: z.boolean().default(false),
+    outputDir: z.string().default('./output'),
+    stagingDb: z.string().default(''),
+    // Phase 4a — per-run enrich tuning (consumed by @caracal-lynx/sluice-enrich)
+    enrichConcurrency: z.number().int().positive().default(5),
+    enrichTimeoutMs: z.number().int().positive().default(5000),
+    enrichMaxRetries: z.number().int().min(0).max(5).default(3),
+    incrementalField: z.string().optional(),
+    incrementalSince: z.string().optional(),
+});
+// ── Multi-source merge ────────────────────────────────────────────────────────
+const MergeFieldStrategySchema = z.object({
+    field: z.string(),
+    strategy: z.enum(['coalesce', 'priority-override']).optional(),
+    source: z.string().optional(), // named source id
+}).refine(s => s.strategy !== undefined || s.source !== undefined, { message: 'fieldStrategy must specify strategy, source, or both' });
+export const MergeSchema = z.object({
+    key: z.union([z.string(), z.array(z.string())]),
+    strategy: z.enum(['coalesce', 'priority-override', 'union', 'intersect'])
+        .default('coalesce'),
+    onUnmatched: z.enum(['include', 'exclude', 'warn', 'error']).default('include'),
+    fieldStrategies: z.array(MergeFieldStrategySchema).default([]),
+    conflictLog: z.string().optional(),
+    incrementalSource: z.string().optional(), // source id; required when run.mode = 'incremental'
+});
+export const MultiSourceEntrySchema = SourceBaseSchema.extend({
+    id: z.string().regex(/^[a-z0-9-]+$/, {
+        message: 'source id must be lowercase alphanumeric with hyphens only',
+    }),
+    priority: z.number().int().positive(),
+    rename: z.record(z.string()).optional(), // { 'old column': 'new column' }
+}).refine(s => s.query || s.file || s.endpoint, { message: 'source must have query, file, or endpoint' });
+// ── Pipeline root ─────────────────────────────────────────────────────────────
+const PipelineMetaSchema = z.object({
+    name: z.string().regex(/^[a-z0-9-]+$/, 'name must be lowercase-hyphenated'),
+    client: z.string(),
+    version: z.string(),
+    entity: z.string(),
+    description: z.string().optional(),
+});
+export const PipelineSchema = z.object({
+    pipeline: PipelineMetaSchema,
+    source: SourceSchema.optional(),
+    sources: z.array(MultiSourceEntrySchema).min(2).optional(),
+    merge: MergeSchema.optional(),
+    enrich: EnrichSchema.optional(), // Phase 4a — runs between Extract/Merge and DQ
+    dq: DqSchema,
+    transform: TransformSchema,
+    target: TargetSchema,
+    run: RunSchema.default({}),
+}).refine(p => (!!p.source && !p.sources && !p.merge) || (!p.source && !!p.sources && !!p.merge), { message: 'pipeline must have either source (single) or both sources and merge (multi)' }).refine(p => {
+    if (!p.sources)
+        return true;
+    const ids = p.sources.map(s => s.id);
+    return new Set(ids).size === ids.length;
+}, { message: 'duplicate source ids in sources array' }).refine(p => {
+    if (!p.sources || !p.merge || p.run?.mode !== 'incremental')
+        return true;
+    return !!p.merge.incrementalSource;
+}, { message: 'merge.incrementalSource is required when run.mode is incremental' }).refine(p => {
+    if (!p.sources || !p.merge || !p.merge.incrementalSource)
+        return true;
+    const ids = new Set(p.sources.map(s => s.id));
+    return ids.has(p.merge.incrementalSource);
+}, { message: 'merge.incrementalSource must match one of the source ids in sources' });
+// ── Phase 2: toolkit-level config (sluice.config.yaml) ───────────────────────
+export const ToolkitConfigSchema = z.object({
+    version: z.string(),
+    plugins: z.array(z.object({
+        package: z.string(),
+        options: z.record(z.unknown()).optional(),
+    })).default([]),
+});
+// ── Phase 2: composite rule library (shared/rules.yaml) ──────────────────────
+const BUILT_IN_CHECK_NAMES = new Set(CheckType.options);
+export const CompositeRuleSchema = z.object({
+    id: z.string()
+        .regex(/^[a-zA-Z][a-zA-Z0-9_-]*$/)
+        .refine(id => !BUILT_IN_CHECK_NAMES.has(id), {
+        message: 'composite rule id must not collide with a built-in check type',
+    }),
+    description: z.string().optional(),
+    checks: z.array(CheckSchema).min(1),
+});
+export const CompositeRuleLibrarySchema = z.object({
+    version: z.string(),
+    rules: z.array(CompositeRuleSchema).min(1),
+});
+// ── Type guards ───────────────────────────────────────────────────────────────
+/** Narrows a Pipeline to one with a defined `source` (single-source pipeline). */
+export function isSingleSource(p) {
+    return p.source !== undefined;
+}
+/** Narrows a Pipeline to one with defined `sources` and `merge` (multi-source pipeline). */
+export function isMultiSource(p) {
+    return p.sources !== undefined;
+}
+//# sourceMappingURL=schema.js.map

package/dist/config/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,iFAAiF;AAEjF,MAAM,QAAQ,GAAK,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;AAC3D,MAAM,QAAQ,GAAK,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAClE,MAAM,QAAQ,GAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5E,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;AAExD,iFAAiF;AAEjF,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,IAAI,EAAS,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACjD,QAAQ,EAAK,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACrD,SAAS,EAAI,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,SAAS,EAAI,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACnC,CAAC,CAAC;AAEH,iFAAiF;AAEjF,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,OAAO,EAAK,QAAQ;IACpB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,KAAK,EAAO,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,IAAI,EAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,QAAQ,EAAI,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,OAAO,EAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC3C,SAAS,EAAG,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACnC,QAAQ,EAAI,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;IACvC,KAAK,EAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxD,UAAU,EAAE,gBAAgB,CAAC,QAAQ,EAAE;CACxC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CACjD,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,EACpC,EAAE,OAAO,EAAE,2CAA2C,EAAE,CACzD,CAAC;AAEF,iFAAiF;AAEjF,8EAA8E;AAC9E,uEAAuE;AACvE,oEAAoE;AACpE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC;IAC9B,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY;IACrD,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe;CAC3C,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,IAAI,EAAM,SAAS;IACnB,KAAK,EAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC3E,QAAQ,EAAE,QAAQ;IAClB,OAAO,EAAG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,KAAK,EAAK,CAAC,CAAC,MAAM,EAAE;IACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAI,2FAA2F;IAC9H,MAAM,EAAI,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACtC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,SAAS,EAAO,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAI,uCAAuC;IAChF,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACzC,aAAa,EAAG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACrC,KAAK,EAAW,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CAClD,CAAC,CAAC;AAEH,iFAAiF;AACjF,EAAE;AACF,gFAAgF;AAChF,4EAA4E;AAC5E,iFAAiF;AACjF,qEAAqE;AACrE,4EAA4E;AAE5E,MAAM,wBAAwB,GAAG,CAAC;KAC/B,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAG,0BAA0B;KAC1D,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAW,8CAA8C;AAEjF,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAS,CAAC,CAAC,MAAM,EAAE;IACxB,QAAQ,EAAM,CAAC,CAAC,MAAM,EAAE;IACxB,YAAY,EAAE,wBAAwB;IACtC,WAAW,EAAG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,OAAO,EAAO,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACzD,KAAK,EAAS,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACpC,OAAO,EAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC/C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,KAAK,EAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACnE,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IACzD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAC5C,CAAC,CAAC;AAEH,iFAAiF;AAEjF,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,IAAI,EAAI,CAAC,CAAC,MAAM,EAAE;IAClB,MAAM,EAAE,YAAY;IACpB,GAAG,EAAK,CAAC,CAAC,MAAM,EAAE;IAClB,KAAK,EAAG,CAAC,CAAC,MAAM,EAAE;CACnB,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC;IACvB,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM;IAChD,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY;IAC5C,QAAQ,EAAK,uDAAuD;CACrE,CAAC,CAAC;AAEH,kEAAkE;AAClE,+EAA+E;AAC/E,uFAAuF;AACvF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAS;IAC3C,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ;CACrE,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,IAAI,EAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChE,EAAE,EAAS,CAAC,CAAC,MAAM,EAAE;IACrB,IAAI,EAAO,SAAS;IACpB,GAAG,EAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,MAAM,EAAK,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,OAAO,EAAI,UAAU,CAAC,QAAQ,EAAE;IAChC,MAAM,EAAK,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,KAAK,EAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IACpE,OAAO,EAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9E,QAAQ,EAAG,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACrC,WAAW;IACX,QAAQ,EAAG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,OAAO,EAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC,MAAM,CACP,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EACxC,EAAE,OAAO,EAAE,0CAA0C,EAAE,IAAI,EAAE,CAAC,UAAU,CAAC,EAAE,CAC5E,CAAC,MAAM,CACN,CAAC,CAAC,EAAE,CAAC,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,EAC9D,EAAE,OAAO,EAAE,2CAA2C,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,CACzE,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1C,MAAM,EAAG,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAC5C,CAAC,CAAC;AAEH,iFAAiF;AAEjF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,OAAO,EAAQ,QAAQ;IACvB,MAAM,EAAS,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,MAAM,EAAS,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,UAAU,EAAK,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACrC,WAAW,EAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC7C,UAAU,EAAK,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,SAAS,EAAM,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IACtC,QAAQ,EAAO,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;IAC1C,SAAS,EAAM,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACrC,QAAQ,EAAO,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,yBAAyB;IACzB,OAAO,EAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,OAAO,EAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,UAAU,EAAK,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IACzC,UAAU,EAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IACnE,SAAS,EAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC7C,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACxC,cAAc;IACd,KAAK,EAAU,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,MAAM,EAAS,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC;CAC5C,CAAC,CAAC,MAAM,CACP,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EACvF,EAAE,OAAO,EAAE,mDAAmD,EAAE,IAAI,EAAE,CAAC,WAAW,CAAC,EAAE,CACtF,CAAC;AAEF,iFAAiF;AAEjF,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,IAAI,EAAe,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IACnF,SAAS,EAAU,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IAC3D,OAAO,EAAY,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;IACnE,QAAQ,EAAW,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7E,MAAM,EAAa,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IAC7C,SAAS,EAAU,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC;IACjD,SAAS,EAAU,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACzC,6EAA6E;IAC7E,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACzD,eAAe,EAAI,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAC5D,gBAAgB,EAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5D,gBAAgB,EAAG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACxC,gBAAgB,EAAG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACzC,CAAC,CAAC;AAEH,iFAAiF;AAEjF,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAK,CAAC,CAAC,MAAM,EAAE;IACpB,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,mBAAmB,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9D,MAAM,EAAI,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAI,kBAAkB;CACtD,CAAC,CAAC,MAAM,CACP,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,EACvD,EAAE,OAAO,EAAE,sDAAsD,EAAE,CACpE,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,GAAG,EAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC7D,QAAQ,EAAW,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,mBAAmB,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;SAC5D,OAAO,CAAC,UAAU,CAAC;IACzC,WAAW,EAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IACrF,eAAe,EAAI,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAChE,WAAW,EAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACxC,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAG,oDAAoD;CAChG,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,MAAM,CAAC;IAC5D,EAAE,EAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,cAAc,EAAE;QACzC,OAAO,EAAE,4DAA4D;KACtE,CAAC;IACF,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACrC,MAAM,EAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAG,iCAAiC;CAC9E,CAAC,CAAC,MAAM,CACP,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,EACpC,EAAE,OAAO,EAAE,2CAA2C,EAAE,CACzD,CAAC;AAEF,iFAAiF;AAEjF,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,IAAI,EAAS,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,cAAc,EAAE,mCAAmC,CAAC;IAClF,MAAM,EAAO,CAAC,CAAC,MAAM,EAAE;IACvB,OAAO,EAAM,CAAC,CAAC,MAAM,EAAE;IACvB,MAAM,EAAO,CAAC,CAAC,MAAM,EAAE;IACvB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACnC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,QAAQ,EAAG,kBAAkB;IAC7B,MAAM,EAAK,YAAY,CAAC,QAAQ,EAAE;IAClC,OAAO,EAAI,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5D,KAAK,EAAM,WAAW,CAAC,QAAQ,EAAE;IACjC,MAAM,EAAK,YAAY,CAAC,QAAQ,EAAE,EAAI,+CAA+C;IACrF,EAAE,EAAS,QAAQ;IACnB,SAAS,EAAE,eAAe;IAC1B,MAAM,EAAK,YAAY;IACvB,GAAG,EAAQ,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;CACjC,CAAC,CAAC,MAAM,CACP,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EACtF,EAAE,OAAO,EAAE,6EAA6E,EAAE,CAC3F,CAAC,MAAM,CACN,CAAC,CAAC,EAAE;IACF,IAAI,CAAC,CAAC,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC5B,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACrC,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC;AAC1C,CAAC,EACD,EAAE,OAAO,EAAE,uCAAuC,EAAE,CACrD,CAAC,MAAM,CACN,CAAC,CAAC,EAAE;IACF,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,KAAK,aAAa;QAAE,OAAO,IAAI,CAAC;IACzE,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC;AACrC,CAAC,EACD,EAAE,OAAO,EAAE,kEAAkE,EAAE,CAChF,CAAC,MAAM,CACN,CAAC,CAAC,EAAE;IACF,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB;QAAE,OAAO,IAAI,CAAC;IACtE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9C,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;AAC5C,CAAC,EACD,EAAE,OAAO,EAAE,qEAAqE,EAAE,CACnF,CAAC;AAEF,gFAAgF;AAEhF,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACxB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;KAC1C,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CAChB,CAAC,CAAC;AAEH,gFAAgF;AAEhF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAS,SAAS,CAAC,OAAO,CAAC,CAAC;AAEhE,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;SACX,KAAK,CAAC,0BAA0B,CAAC;SACjC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAC3C,OAAO,EAAE,+DAA+D;KACzE,CAAC;IACJ,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,MAAM,EAAO,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACzC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,KAAK,EAAI,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAC7C,CAAC,CAAC;AAsBH,iFAAiF;AAEjF,kFAAkF;AAClF,MAAM,UAAU,cAAc,CAAC,CAAW;IACxC,OAAO,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC;AAChC,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,aAAa,CAC3B,CAAW;IAEX,OAAO,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC;AACjC,CAAC"}

package/dist/config/types.d.ts

@@ -0,0 +1,3 @@
+export type { Pipeline, SourceConfig, TargetConfig, RunConfig, FieldMapping, DqRule, CheckConfig, Lookup, ToolkitConfig, CompositeRule, CompositeRuleLibrary, MergeConfig, MultiSourceEntry, EnrichConfig, EnrichLookupConfig, EnrichWriteColumns, } from './schema.js';
+export { isSingleSource, isMultiSource } from './schema.js';
+//# sourceMappingURL=types.d.ts.map

package/dist/config/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/config/types.ts"],"names":[],"mappings":"AAGA,YAAY,EACV,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,MAAM,EACN,WAAW,EACX,MAAM,EACN,aAAa,EACb,aAAa,EACb,oBAAoB,EACpB,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC"}

package/dist/config/types.js

@@ -0,0 +1,4 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+export { isSingleSource, isMultiSource } from './schema.js';
+//# sourceMappingURL=types.js.map

package/dist/config/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/config/types.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAqBvC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC"}

package/dist/dq/engine.d.ts

@@ -0,0 +1,10 @@
+import type { Pipeline } from '../config/types.js';
+import type { RuleRegistry } from '../plugins/registry.js';
+import { StagingStore } from '../staging/index.js';
+import type { DQSummary } from './types.js';
+export declare class DQEngine {
+    private readonly ruleRegistry?;
+    constructor(ruleRegistry?: RuleRegistry | undefined);
+    run(config: Pipeline, store: StagingStore, tableName?: string, onProgress?: (rows: number) => void): Promise<DQSummary>;
+}
+//# sourceMappingURL=engine.d.ts.map

package/dist/dq/engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/dq/engine.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAc,MAAM,qBAAqB,CAAC;AAK/D,OAAO,KAAK,EAAE,SAAS,EAAmB,MAAM,YAAY,CAAC;AAE7D,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAAb,YAAY,CAAC,EAAE,YAAY,YAAA;IAElD,GAAG,CACP,MAAM,EAAE,QAAQ,EAChB,KAAK,EAAE,YAAY,EACnB,SAAS,SAAY,EACrB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAClC,OAAO,CAAC,SAAS,CAAC;CA4GtB"}

package/dist/dq/engine.js

@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+/**
+ * DQ engine.
+ *
+ * `run()` takes `tableName` so callers can point the engine at `stg_raw`,
+ * `stg_merged`, or `stg_raw_{sourceId}` — the engine itself is table-agnostic.
+ *
+ * `DqRule.sourceId` is parsed by the schema but not consulted here. The
+ * MultiSourcePipelineRunner filters rules by `sourceId` before invoking
+ * this engine.
+ */
+import * as path from 'node:path';
+import { quoteIdent } from '../staging/index.js';
+import { ConfigError } from '../utils/errors.js';
+import { writeRejectionCsv, writeSummaryJson } from './reporter.js';
+import { BUILT_IN_RULES } from './rules/index.js';
+export class DQEngine {
+    ruleRegistry;
+    constructor(ruleRegistry) {
+        this.ruleRegistry = ruleRegistry;
+    }
+    async run(config, store, tableName = 'stg_raw', onProgress) {
+        const rules = config.dq.rules;
+        // Short-circuit when there are no rules: still write a summary so callers
+        // have a consistent report-path contract.
+        const rows = await store.query(`SELECT * FROM ${quoteIdent(tableName)}`);
+        // ── Pre-pass: compute duplicate sets for every field carrying a `unique` check ──
+        const duplicatesByField = new Map();
+        for (const rule of rules) {
+            const hasUnique = rule.checks.some((c) => c.type === 'unique');
+            if (!hasUnique || duplicatesByField.has(rule.field))
+                continue;
+            const qField = quoteIdent(rule.field);
+            const dupRows = await store.query(`SELECT ${qField} AS v, count(*) AS n FROM ${quoteIdent(tableName)} WHERE ${qField} IS NOT NULL GROUP BY v HAVING count(*) > 1`);
+            duplicatesByField.set(rule.field, new Set(dupRows.map((r) => String(r.v))));
+        }
+        // ── Main pass: validate every (row, rule, check) combination ──
+        const violations = [];
+        const progressEvery = 500;
+        for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) {
+            if (onProgress && rowIndex > 0 && rowIndex % progressEvery === 0) {
+                onProgress(rowIndex);
+            }
+            const row = rows[rowIndex];
+            for (const rule of rules) {
+                const value = row[rule.field];
+                for (const check of rule.checks) {
+                    if (check.type === 'unique') {
+                        const dups = duplicatesByField.get(rule.field);
+                        if (value !== null &&
+                            value !== undefined &&
+                            value !== '' &&
+                            dups?.has(String(value))) {
+                            violations.push({
+                                field: rule.field,
+                                rowIndex,
+                                value,
+                                rule: 'unique',
+                                severity: check.severity,
+                                message: check.message ?? `${rule.field} value "${String(value)}" is not unique`,
+                            });
+                        }
+                        continue;
+                    }
+                    const impl = BUILT_IN_RULES[check.type] ??
+                        this.ruleRegistry?.get(check.type);
+                    if (!impl) {
+                        throw new ConfigError(`Unknown DQ rule type "${String(check.type)}". ` +
+                            `Built-in types: ${Object.keys(BUILT_IN_RULES).join(', ')}`);
+                    }
+                    const violation = impl.validate(value, check, rowIndex, rule.field);
+                    if (violation)
+                        violations.push(violation);
+                }
+            }
+        }
+        // ── Aggregate ──
+        const counts = { critical: 0, warning: 0, info: 0 };
+        const byField = {};
+        const criticalRows = new Set();
+        for (const v of violations) {
+            counts[v.severity]++;
+            if (v.severity === 'critical')
+                criticalRows.add(v.rowIndex);
+            const bf = (byField[v.field] ??= { critical: 0, warning: 0, info: 0 });
+            bf[v.severity]++;
+        }
+        const rowsChecked = rows.length;
+        const rowsRejected = criticalRows.size;
+        const rowsPassed = rowsChecked - rowsRejected;
+        if (onProgress)
+            onProgress(rowsChecked);
+        // ── Write reports ──
+        const outputDir = path.resolve(config.run.outputDir);
+        const rejectionFile = config.dq.rejectionFile ?? path.join(outputDir, `${config.pipeline.name}-rejected.csv`);
+        const summaryFile = path.join(outputDir, `${config.pipeline.name}-dq-summary.json`);
+        await writeRejectionCsv(rejectionFile, violations);
+        const summary = {
+            pipeline: config.pipeline.name,
+            runAt: new Date().toISOString(),
+            rowsChecked,
+            rowsPassed,
+            rowsRejected,
+            violations: counts,
+            byField,
+            reportPath: summaryFile,
+            rejectedRowIndices: [...criticalRows].sort((a, b) => a - b),
+        };
+        await writeSummaryJson(summaryFile, summary);
+        return summary;
+    }
+}
+//# sourceMappingURL=engine.js.map

package/dist/dq/engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../src/dq/engine.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;;;;;;GASG;AAEH,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAIlC,OAAO,EAAgB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAIlD,MAAM,OAAO,QAAQ;IACU;IAA7B,YAA6B,YAA2B;QAA3B,iBAAY,GAAZ,YAAY,CAAe;IAAG,CAAC;IAE5D,KAAK,CAAC,GAAG,CACP,MAAgB,EAChB,KAAmB,EACnB,SAAS,GAAG,SAAS,EACrB,UAAmC;QAEnC,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC;QAE9B,0EAA0E;QAC1E,0CAA0C;QAC1C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAC5B,iBAAiB,UAAU,CAAC,SAAS,CAAC,EAAE,CACzC,CAAC;QAEF,mFAAmF;QACnF,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;YAC/D,IAAI,CAAC,SAAS,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,SAAS;YAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAC/B,UAAU,MAAM,6BAA6B,UAAU,CAAC,SAAS,CAAC,UAAU,MAAM,6CAA6C,CAChI,CAAC;YACF,iBAAiB,CAAC,GAAG,CACnB,IAAI,CAAC,KAAK,EACV,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACzC,CAAC;QACJ,CAAC;QAED,iEAAiE;QACjE,MAAM,UAAU,GAAoB,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,GAAG,CAAC;QAC1B,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC1D,IAAI,UAAU,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,aAAa,KAAK,CAAC,EAAE,CAAC;gBACjE,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAE,CAAC;YAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC9B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC5B,MAAM,IAAI,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC/C,IACE,KAAK,KAAK,IAAI;4BACd,KAAK,KAAK,SAAS;4BACnB,KAAK,KAAK,EAAE;4BACZ,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EACxB,CAAC;4BACD,UAAU,CAAC,IAAI,CAAC;gCACd,KAAK,EAAE,IAAI,CAAC,KAAK;gCACjB,QAAQ;gCACR,KAAK;gCACL,IAAI,EAAE,QAAQ;gCACd,QAAQ,EAAE,KAAK,CAAC,QAAQ;gCACxB,OAAO,EACL,KAAK,CAAC,OAAO,IAAI,GAAG,IAAI,CAAC,KAAK,WAAW,MAAM,CAAC,KAAK,CAAC,iBAAiB;6BAC1E,CAAC,CAAC;wBACL,CAAC;wBACD,SAAS;oBACX,CAAC;oBACD,MAAM,IAAI,GACP,cAAuC,CAAC,KAAK,CAAC,IAAI,CAAC;wBACpD,IAAI,CAAC,YAAY,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACrC,IAAI,CAAC,IAAI,EAAE,CAAC;wBACV,MAAM,IAAI,WAAW,CACnB,yBAAyB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK;4BAC9C,mBAAmB,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC9D,CAAC;oBACJ,CAAC;oBACD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpE,IAAI,SAAS;wBAAE,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,MAAM,MAAM,GAAoB,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACrE,MAAM,OAAO,GAAoC,EAAE,CAAC;QACpD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,CAAC,QAAQ,KAAK,UAAU;gBAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC5D,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;YACvE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnB,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC;QAChC,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC;QACvC,MAAM,UAAU,GAAG,WAAW,GAAG,YAAY,CAAC;QAC9C,IAAI,UAAU;YAAE,UAAU,CAAC,WAAW,CAAC,CAAC;QAExC,sBAAsB;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,aAAa,GACjB,MAAM,CAAC,EAAE,CAAC,aAAa,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,eAAe,CAAC,CAAC;QAC1F,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,kBAAkB,CAAC,CAAC;QAEpF,MAAM,iBAAiB,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;QAEnD,MAAM,OAAO,GAAc;YACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI;YAC9B,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC/B,WAAW;YACX,UAAU;YACV,YAAY;YACZ,UAAU,EAAE,MAAM;YAClB,OAAO;YACP,UAAU,EAAE,WAAW;YACvB,kBAAkB,EAAE,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;SAC5D,CAAC;QACF,MAAM,gBAAgB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAE7C,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}

package/dist/dq/index.d.ts

@@ -0,0 +1,6 @@
+export { DQEngine } from './engine.js';
+export { writeRejectionCsv, writeSummaryJson } from './reporter.js';
+export { BUILT_IN_RULES } from './rules/index.js';
+export type { Rule, RuleViolation } from './rules/types.js';
+export type { DQSummary, ViolationCounts } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/dq/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/dq/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,YAAY,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC5D,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC"}

package/dist/dq/index.js

@@ -0,0 +1,6 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+export { DQEngine } from './engine.js';
+export { writeRejectionCsv, writeSummaryJson } from './reporter.js';
+export { BUILT_IN_RULES } from './rules/index.js';
+//# sourceMappingURL=index.js.map

package/dist/dq/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/dq/index.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/dq/reporter.d.ts

@@ -0,0 +1,5 @@
+import type { RuleViolation } from './rules/types.js';
+import type { DQSummary } from './types.js';
+export declare function writeRejectionCsv(outputPath: string, violations: RuleViolation[]): Promise<void>;
+export declare function writeSummaryJson(outputPath: string, summary: DQSummary): Promise<void>;
+//# sourceMappingURL=reporter.d.ts.map

package/dist/dq/reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/dq/reporter.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAW5C,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,aAAa,EAAE,GAC1B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,SAAS,GACjB,OAAO,CAAC,IAAI,CAAC,CAWf"}

package/dist/dq/reporter.js

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+/**
+ * DQ reporter — writes rejection CSV + summary JSON.
+ * Pure file I/O; no DuckDB access.
+ */
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { stringify } from 'csv-stringify/sync';
+const REJECTION_CSV_HEADER = [
+    'row_index',
+    'field',
+    'value',
+    'rule',
+    'severity',
+    'message',
+];
+export async function writeRejectionCsv(outputPath, violations) {
+    const rows = violations.map((v) => [
+        String(v.rowIndex),
+        v.field,
+        v.value === null || v.value === undefined ? '' : String(v.value),
+        v.rule,
+        v.severity,
+        v.message,
+    ]);
+    const csv = stringify([[...REJECTION_CSV_HEADER], ...rows]);
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    await fs.writeFile(outputPath, csv, 'utf-8');
+}
+export async function writeSummaryJson(outputPath, summary) {
+    // Strip the internal `rejectedRowIndices` field from the serialised JSON —
+    // CLAUDE.md's published summary shape does not include it, and the list
+    // could be huge for large datasets. It stays on the in-memory summary for
+    // the runner to consume when filtering rejected rows pre-transform.
+    const { rejectedRowIndices: _omit, ...publicShape } = summary;
+    void _omit;
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    await fs.writeFile(outputPath, `${JSON.stringify(publicShape, null, 2)}\n`, 'utf-8');
+}
+//# sourceMappingURL=reporter.js.map

package/dist/dq/reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/dq/reporter.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAK/C,MAAM,oBAAoB,GAAG;IAC3B,WAAW;IACX,OAAO;IACP,OAAO;IACP,MAAM;IACN,UAAU;IACV,SAAS;CACD,CAAC;AAEX,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,UAAkB,EAClB,UAA2B;IAE3B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;QAClB,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;QAChE,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC,OAAO;KACV,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,oBAAoB,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;IAC5D,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,UAAkB,EAClB,OAAkB;IAElB,2EAA2E;IAC3E,wEAAwE;IACxE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,GAAG,WAAW,EAAE,GAAG,OAErD,CAAC;IACF,KAAK,KAAK,CAAC;IACX,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AACvF,CAAC"}

package/dist/dq/rules/allowedValues.d.ts

@@ -0,0 +1,7 @@
+import type { CheckConfig } from '../../config/types.js';
+import type { Rule, RuleViolation } from './types.js';
+export declare class AllowedValuesRule implements Rule {
+    readonly id = "allowedValues";
+    validate(value: unknown, config: CheckConfig, rowIndex: number, field: string): RuleViolation | null;
+}
+//# sourceMappingURL=allowedValues.d.ts.map

package/dist/dq/rules/allowedValues.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"allowedValues.d.ts","sourceRoot":"","sources":["../../../src/dq/rules/allowedValues.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEzD,OAAO,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEtD,qBAAa,iBAAkB,YAAW,IAAI;IAC5C,QAAQ,CAAC,EAAE,mBAAmB;IAE9B,QAAQ,CACN,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,aAAa,GAAG,IAAI;CAoBxB"}

package/dist/dq/rules/allowedValues.js

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+import { DQError } from '../../utils/errors.js';
+export class AllowedValuesRule {
+    id = 'allowedValues';
+    validate(value, config, rowIndex, field) {
+        if (value === null || value === undefined || value === '')
+            return null;
+        if (!Array.isArray(config.value)) {
+            throw new DQError(`allowedValues rule on field "${field}" requires an array of permitted strings`);
+        }
+        const s = String(value);
+        if (config.value.includes(s))
+            return null;
+        return {
+            field,
+            rowIndex,
+            value,
+            rule: 'allowedValues',
+            severity: config.severity,
+            message: config.message ??
+                `${field} value "${s}" not in allowed list [${config.value.join(', ')}]`,
+        };
+    }
+}
+//# sourceMappingURL=allowedValues.js.map

package/dist/dq/rules/allowedValues.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"allowedValues.js","sourceRoot":"","sources":["../../../src/dq/rules/allowedValues.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAGvC,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAGhD,MAAM,OAAO,iBAAiB;IACnB,EAAE,GAAG,eAAe,CAAC;IAE9B,QAAQ,CACN,KAAc,EACd,MAAmB,EACnB,QAAgB,EAChB,KAAa;QAEb,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC;QACvE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,OAAO,CACf,gCAAgC,KAAK,0CAA0C,CAChF,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,OAAO;YACL,KAAK;YACL,QAAQ;YACR,KAAK;YACL,IAAI,EAAE,eAAe;YACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,OAAO,EACL,MAAM,CAAC,OAAO;gBACd,GAAG,KAAK,WAAW,CAAC,0BAA0B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;SAC3E,CAAC;IACJ,CAAC;CACF"}

package/dist/dq/rules/email.d.ts

@@ -0,0 +1,7 @@
+import type { CheckConfig } from '../../config/types.js';
+import type { Rule, RuleViolation } from './types.js';
+export declare class EmailRule implements Rule {
+    readonly id = "email";
+    validate(value: unknown, config: CheckConfig, rowIndex: number, field: string): RuleViolation | null;
+}
+//# sourceMappingURL=email.d.ts.map

package/dist/dq/rules/email.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"email.d.ts","sourceRoot":"","sources":["../../../src/dq/rules/email.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAQtD,qBAAa,SAAU,YAAW,IAAI;IACpC,QAAQ,CAAC,EAAE,WAAW;IAEtB,QAAQ,CACN,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,aAAa,GAAG,IAAI;CAYxB"}

package/dist/dq/rules/email.js

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+// Pragmatic RFC 5322-ish regex — matches the vast majority of real addresses
+// without the full 400-line spec. Rejects leading/trailing dots, multiple @,
+// and obviously-malformed domains.
+const EMAIL_RE = /^[A-Za-z0-9._%+-]+@[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+$/;
+export class EmailRule {
+    id = 'email';
+    validate(value, config, rowIndex, field) {
+        if (value === null || value === undefined || value === '')
+            return null;
+        if (EMAIL_RE.test(String(value)))
+            return null;
+        return {
+            field,
+            rowIndex,
+            value,
+            rule: 'email',
+            severity: config.severity,
+            message: config.message ?? `${field} is not a valid email address`,
+        };
+    }
+}
+//# sourceMappingURL=email.js.map