@caracal-lynx/sluice 0.1.0

package/dist/staging/store.d.ts

@@ -0,0 +1,71 @@
+import { type ColumnMeta } from './schema.js';
+export interface ExportToCsvOptions {
+    delimiter?: string;
+    header?: boolean;
+    /** Token to emit for NULL values. Default: DuckDB's default (empty string). */
+    nullValue?: string;
+    /** Accepted for API symmetry; DuckDB always writes UTF-8. */
+    encoding?: string;
+}
+export declare class StagingStore {
+    private readonly dbPath;
+    private instance;
+    private conn;
+    /** `:memory:` for dryRun/tests, else a filesystem path. */
+    constructor(dbPath: string);
+    open(): Promise<void>;
+    close(): Promise<void>;
+    createTable(name: string, columns: ColumnMeta[]): Promise<void>;
+    /**
+     * Bulk-insert rows. Column schema is taken from the first row; rows missing
+     * a column get `null`. All rows must share the first row's column set.
+     *
+     * Builds a single multi-row prepared INSERT for efficiency:
+     *   `INSERT INTO "t" ("a","b") VALUES ($1,$2), ($3,$4), ($5,$6)`
+     * and binds all params positionally.
+     */
+    insertBatch(table: string, rows: Record<string, unknown>[]): Promise<void>;
+    query<T>(sql: string, params?: unknown[]): Promise<T[]>;
+    tableExists(name: string): Promise<boolean>;
+    dropTable(name: string): Promise<void>;
+    rowCount(table: string): Promise<number>;
+    columnNames(table: string): Promise<string[]>;
+    /**
+     * Export a table to a CSV file. DuckDB always emits UTF-8; the `encoding`
+     * option is accepted for API symmetry and currently ignored.
+     */
+    exportToCsv(table: string, outputPath: string, options?: ExportToCsvOptions): Promise<void>;
+    /**
+     * Rename columns in place. Implemented as CREATE OR REPLACE TABLE ... AS
+     * SELECT to sidestep DuckDB's column-rename limitations. Unknown keys are
+     * warned and skipped (not an error). No-op when `renames` is empty.
+     *
+     * Used by `MultiSourcePipelineRunner` to apply per-source rename maps
+     * after extract and before merge.
+     */
+    renameColumns(tableName: string, renames: Record<string, string>): Promise<void>;
+    /**
+     * Returns distinct non-null, non-empty values for a column in stg_raw.
+     * Used by the Phase 4a enrich runner to deduplicate API calls.
+     */
+    selectDistinct(_field: string): Promise<string[]>;
+    /** Adds a column to stg_raw if it does not already exist. */
+    addColumnIfNotExists(_column: string, _type: 'BOOLEAN' | 'VARCHAR'): Promise<void>;
+    /**
+     * Batch-updates stg_raw. `updates` is keyed by rowid and maps each row to a
+     * partial column-value record to UPDATE.
+     */
+    batchUpdateColumns(_updates: Map<number, Record<string, unknown>>): Promise<void>;
+    /**
+     * Run any SQL statement. Returns native-JS row objects (Date for TIMESTAMP,
+     * bigint for BIGINT, null for NULL) for SELECTs; for DDL/DML the returned
+     * array is empty or a `[{ Count: <n>n }]` summary that callers ignore.
+     */
+    private exec;
+    /**
+     * Run a parameterised statement via a prepared statement. Translates `?`
+     * placeholders to `$N` and binds positional params with type sniffing.
+     */
+    private execPrepared;
+}
+//# sourceMappingURL=store.d.ts.map

package/dist/staging/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/staging/store.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAmC,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAE/E,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAqDD,qBAAa,YAAY;IAKX,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJnC,OAAO,CAAC,QAAQ,CAA+B;IAC/C,OAAO,CAAC,IAAI,CAAiC;IAE7C,2DAA2D;gBAC9B,MAAM,EAAE,MAAM;IAErC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAerB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBtB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAIrE;;;;;;;OAOG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAmB1E,KAAK,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAK3D,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAS3C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAItC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAOxC,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAQnD;;;OAGG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBjG;;;;;;;OAOG;IACG,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6BtF;;;OAGG;IACG,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAMvD,6DAA6D;IACvD,oBAAoB,CACxB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,SAAS,GAAG,SAAS,GAC3B,OAAO,CAAC,IAAI,CAAC;IAMhB;;;OAGG;IACG,kBAAkB,CACtB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAC7C,OAAO,CAAC,IAAI,CAAC;IAQhB;;;;OAIG;YACW,IAAI;IAkBlB;;;OAGG;YACW,YAAY;CA6B3B"}

package/dist/staging/store.js

@@ -0,0 +1,270 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+/**
+ * StagingStore — thin promisified wrapper around DuckDB.
+ *
+ * The only place in `src/` that imports `@duckdb/node-api` directly.
+ * PipelineRunner owns the single instance.
+ *
+ * Public API is preserved verbatim across the Node 20→24 / `duckdb` →
+ * `@duckdb/node-api` migration: callers still do `new StagingStore(path)`
+ * followed by `await store.open()`.
+ */
+import { DuckDBInstance, } from '@duckdb/node-api';
+import { StagingError } from '../utils/errors.js';
+import { logger } from '../utils/logger.js';
+import { buildCreateTableSql, quoteIdent } from './schema.js';
+/**
+ * Translate `?` positional placeholders (the convention inherited from the
+ * old `duckdb` driver) into the `$N` form that `@duckdb/node-api` expects.
+ * Quoted identifiers and string literals containing `?` are not corrupted
+ * because internal callers in this file never put `?` inside a quoted
+ * literal — only as a positional bind marker.
+ */
+function translatePlaceholders(sql) {
+    let i = 0;
+    return sql.replace(/\?/g, () => `$${++i}`);
+}
+/**
+ * Bind a single positional parameter, sniffing the JS type and routing to the
+ * appropriate typed bind method. DuckDB coerces between numeric column types
+ * (e.g. INTEGER value into BIGINT column) so we don't need to know the column
+ * type up-front — only enough about the JS value to pick a sensible bind.
+ *
+ * `null` and `undefined` both map to bindNull (matches the old driver, where
+ * `insertBatch` rewrote `undefined → null` before binding).
+ */
+function bindParam(prep, pos, value) {
+    if (value === null || value === undefined) {
+        prep.bindNull(pos);
+        return;
+    }
+    if (typeof value === 'boolean') {
+        prep.bindBoolean(pos, value);
+        return;
+    }
+    if (typeof value === 'bigint') {
+        prep.bindBigInt(pos, value);
+        return;
+    }
+    if (typeof value === 'number') {
+        if (Number.isInteger(value) && value >= -2_147_483_648 && value <= 2_147_483_647) {
+            prep.bindInteger(pos, value);
+        }
+        else {
+            prep.bindDouble(pos, value);
+        }
+        return;
+    }
+    // Strings, Dates (ISO-stringified), and everything else go through varchar.
+    // DuckDB coerces e.g. '2026-04-19 12:00:00' into a TIMESTAMP column.
+    if (value instanceof Date) {
+        prep.bindVarchar(pos, value.toISOString());
+        return;
+    }
+    prep.bindVarchar(pos, String(value));
+}
+export class StagingStore {
+    dbPath;
+    instance = null;
+    conn = null;
+    /** `:memory:` for dryRun/tests, else a filesystem path. */
+    constructor(dbPath) {
+        this.dbPath = dbPath;
+    }
+    async open() {
+        if (this.instance)
+            return;
+        try {
+            this.instance = await DuckDBInstance.create(this.dbPath);
+            this.conn = await this.instance.connect();
+        }
+        catch (err) {
+            this.instance = null;
+            this.conn = null;
+            throw new StagingError(`failed to open DuckDB at ${this.dbPath}: ${err instanceof Error ? err.message : String(err)}`, err);
+        }
+    }
+    async close() {
+        if (!this.instance)
+            return;
+        const conn = this.conn;
+        const instance = this.instance;
+        this.conn = null;
+        this.instance = null;
+        try {
+            conn?.disconnectSync();
+            instance.closeSync();
+        }
+        catch (err) {
+            throw new StagingError(`failed to close DuckDB: ${err instanceof Error ? err.message : String(err)}`, err);
+        }
+    }
+    async createTable(name, columns) {
+        await this.exec(buildCreateTableSql(name, columns));
+    }
+    /**
+     * Bulk-insert rows. Column schema is taken from the first row; rows missing
+     * a column get `null`. All rows must share the first row's column set.
+     *
+     * Builds a single multi-row prepared INSERT for efficiency:
+     *   `INSERT INTO "t" ("a","b") VALUES ($1,$2), ($3,$4), ($5,$6)`
+     * and binds all params positionally.
+     */
+    async insertBatch(table, rows) {
+        if (rows.length === 0)
+            return;
+        const first = rows[0];
+        const cols = Object.keys(first);
+        if (cols.length === 0) {
+            throw new StagingError('insertBatch: rows have no columns');
+        }
+        const colList = cols.map(quoteIdent).join(', ');
+        let n = 0;
+        const rowPlaceholders = rows
+            .map(() => `(${cols.map(() => `$${++n}`).join(', ')})`)
+            .join(', ');
+        const flatParams = rows.flatMap((r) => cols.map((c) => (r[c] === undefined ? null : r[c])));
+        const sql = `INSERT INTO ${quoteIdent(table)} (${colList}) VALUES ${rowPlaceholders}`;
+        await this.execPrepared(sql, flatParams);
+    }
+    async query(sql, params = []) {
+        const rows = await this.exec(sql, params);
+        return rows;
+    }
+    async tableExists(name) {
+        const rows = await this.query('SELECT count(*) AS n FROM information_schema.tables WHERE table_name = ?', [name]);
+        const n = rows[0]?.n ?? 0;
+        return Number(n) > 0;
+    }
+    async dropTable(name) {
+        await this.exec(`DROP TABLE IF EXISTS ${quoteIdent(name)}`);
+    }
+    async rowCount(table) {
+        const rows = await this.query(`SELECT count(*) AS n FROM ${quoteIdent(table)}`);
+        return Number(rows[0]?.n ?? 0);
+    }
+    async columnNames(table) {
+        const rows = await this.query('SELECT column_name FROM information_schema.columns WHERE table_name = ? ORDER BY ordinal_position', [table]);
+        return rows.map((r) => r.column_name);
+    }
+    /**
+     * Export a table to a CSV file. DuckDB always emits UTF-8; the `encoding`
+     * option is accepted for API symmetry and currently ignored.
+     */
+    async exportToCsv(table, outputPath, options) {
+        const delim = options?.delimiter ?? ',';
+        const header = options?.header ?? true;
+        const safePath = outputPath.replace(/\\/g, '/').replace(/'/g, "''");
+        const safeDelim = delim.replace(/'/g, "''");
+        const clauses = [
+            `HEADER ${header ? 'TRUE' : 'FALSE'}`,
+            `DELIMITER '${safeDelim}'`,
+        ];
+        if (options?.nullValue !== undefined) {
+            clauses.push(`NULL '${options.nullValue.replace(/'/g, "''")}'`);
+        }
+        await this.exec(`COPY ${quoteIdent(table)} TO '${safePath}' (${clauses.join(', ')})`);
+    }
+    /**
+     * Rename columns in place. Implemented as CREATE OR REPLACE TABLE ... AS
+     * SELECT to sidestep DuckDB's column-rename limitations. Unknown keys are
+     * warned and skipped (not an error). No-op when `renames` is empty.
+     *
+     * Used by `MultiSourcePipelineRunner` to apply per-source rename maps
+     * after extract and before merge.
+     */
+    async renameColumns(tableName, renames) {
+        if (Object.keys(renames).length === 0)
+            return;
+        const existingColumns = await this.columnNames(tableName);
+        const unknownKeys = Object.keys(renames).filter((k) => !existingColumns.includes(k));
+        if (unknownKeys.length > 0) {
+            logger.warn({ tableName, unknownKeys }, 'rename map contains columns not found in table');
+        }
+        const selectList = existingColumns
+            .map((col) => {
+            const newName = renames[col];
+            return newName !== undefined
+                ? `${quoteIdent(col)} AS ${quoteIdent(newName)}`
+                : quoteIdent(col);
+        })
+            .join(', ');
+        await this.exec(`CREATE OR REPLACE TABLE ${quoteIdent(tableName)} AS SELECT ${selectList} FROM ${quoteIdent(tableName)}`);
+    }
+    // ── Phase 4a stubs (implemented by @caracal-lynx/sluice-enrich) ────────
+    //
+    // The private `@caracal-lynx/sluice-enrich` package overwrites these three
+    // methods on `StagingStore.prototype` at import time via a `patchStagingStore()`
+    // helper. Until that package is installed and imported, the methods throw.
+    /**
+     * Returns distinct non-null, non-empty values for a column in stg_raw.
+     * Used by the Phase 4a enrich runner to deduplicate API calls.
+     */
+    async selectDistinct(_field) {
+        throw new StagingError('StagingStore.selectDistinct: not yet implemented — install @caracal-lynx/sluice-enrich');
+    }
+    /** Adds a column to stg_raw if it does not already exist. */
+    async addColumnIfNotExists(_column, _type) {
+        throw new StagingError('StagingStore.addColumnIfNotExists: not yet implemented — install @caracal-lynx/sluice-enrich');
+    }
+    /**
+     * Batch-updates stg_raw. `updates` is keyed by rowid and maps each row to a
+     * partial column-value record to UPDATE.
+     */
+    async batchUpdateColumns(_updates) {
+        throw new StagingError('StagingStore.batchUpdateColumns: not yet implemented — install @caracal-lynx/sluice-enrich');
+    }
+    // ── internal ───────────────────────────────────────────────────────────
+    /**
+     * Run any SQL statement. Returns native-JS row objects (Date for TIMESTAMP,
+     * bigint for BIGINT, null for NULL) for SELECTs; for DDL/DML the returned
+     * array is empty or a `[{ Count: <n>n }]` summary that callers ignore.
+     */
+    async exec(sql, params = []) {
+        if (!this.conn) {
+            throw new StagingError('StagingStore is not open');
+        }
+        if (params.length === 0) {
+            try {
+                const reader = await this.conn.runAndReadAll(sql);
+                return reader.getRowObjectsJS();
+            }
+            catch (err) {
+                throw new StagingError(`DuckDB query failed: ${err instanceof Error ? err.message : String(err)}`, err);
+            }
+        }
+        return this.execPrepared(sql, params);
+    }
+    /**
+     * Run a parameterised statement via a prepared statement. Translates `?`
+     * placeholders to `$N` and binds positional params with type sniffing.
+     */
+    async execPrepared(sql, params) {
+        if (!this.conn) {
+            throw new StagingError('StagingStore is not open');
+        }
+        const translated = translatePlaceholders(sql);
+        let prep = null;
+        try {
+            prep = await this.conn.prepare(translated);
+            params.forEach((value, i) => {
+                bindParam(prep, i + 1, value);
+            });
+            const reader = await prep.runAndReadAll();
+            return reader.getRowObjectsJS();
+        }
+        catch (err) {
+            throw new StagingError(`DuckDB query failed: ${err instanceof Error ? err.message : String(err)}`, err);
+        }
+        finally {
+            try {
+                prep?.destroySync();
+            }
+            catch {
+                /* noop — destroy errors swallowed; the connection is already done with it */
+            }
+        }
+    }
+}
+//# sourceMappingURL=store.js.map

package/dist/staging/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/staging/store.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;;;;;;GASG;AAEH,OAAO,EACL,cAAc,GAGf,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAmB,MAAM,aAAa,CAAC;AAW/E;;;;;;GAMG;AACH,SAAS,qBAAqB,CAAC,GAAW;IACxC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,SAAS,CAAC,IAA6B,EAAE,GAAW,EAAE,KAAc;IAC3E,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACnB,OAAO;IACT,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7B,OAAO;IACT,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC5B,OAAO;IACT,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,IAAI,KAAK,IAAI,aAAa,EAAE,CAAC;YACjF,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO;IACT,CAAC;IACD,4EAA4E;IAC5E,qEAAqE;IACrE,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAC3C,OAAO;IACT,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,YAAY;IAKM;IAJrB,QAAQ,GAA0B,IAAI,CAAC;IACvC,IAAI,GAA4B,IAAI,CAAC;IAE7C,2DAA2D;IAC3D,YAA6B,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;IAAG,CAAC;IAE/C,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,MAAM,IAAI,YAAY,CACpB,4BAA4B,IAAI,CAAC,MAAM,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC9F,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC;YACH,IAAI,EAAE,cAAc,EAAE,CAAC;YACvB,QAAQ,CAAC,SAAS,EAAE,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,YAAY,CACpB,2BAA2B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC7E,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAY,EAAE,OAAqB;QACnD,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,IAA+B;QAC9D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACvB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,MAAM,eAAe,GAAG,IAAI;aACzB,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;aACtD,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CACpC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACpD,CAAC;QACF,MAAM,GAAG,GAAG,eAAe,UAAU,CAAC,KAAK,CAAC,KAAK,OAAO,YAAY,eAAe,EAAE,CAAC;QACtF,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,KAAK,CAAI,GAAW,EAAE,SAAoB,EAAE;QAChD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC1C,OAAO,IAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAY;QAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAC3B,0EAA0E,EAC1E,CAAC,IAAI,CAAC,CACP,CAAC;QACF,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAC1B,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QAC1B,MAAM,IAAI,CAAC,IAAI,CAAC,wBAAwB,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa;QAC1B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAC3B,6BAA6B,UAAU,CAAC,KAAK,CAAC,EAAE,CACjD,CAAC;QACF,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,KAAa;QAC7B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAC3B,mGAAmG,EACnG,CAAC,KAAK,CAAC,CACR,CAAC;QACF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,UAAkB,EAAE,OAA4B;QAC/E,MAAM,KAAK,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG,CAAC;QACxC,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,IAAI,CAAC;QACvC,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG;YACd,UAAU,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE;YACrC,cAAc,SAAS,GAAG;SAC3B,CAAC;QACF,IAAI,OAAO,EAAE,SAAS,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,IAAI,CAAC,IAAI,CACb,QAAQ,UAAU,CAAC,KAAK,CAAC,QAAQ,QAAQ,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACrE,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,aAAa,CAAC,SAAiB,EAAE,OAA+B;QACpE,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE9C,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE,gDAAgD,CAAC,CAAC;QAC5F,CAAC;QAED,MAAM,UAAU,GAAG,eAAe;aAC/B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACX,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YAC7B,OAAO,OAAO,KAAK,SAAS;gBAC1B,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,UAAU,CAAC,OAAO,CAAC,EAAE;gBAChD,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,IAAI,CAAC,IAAI,CACb,2BAA2B,UAAU,CAAC,SAAS,CAAC,cAAc,UAAU,SAAS,UAAU,CAAC,SAAS,CAAC,EAAE,CACzG,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,EAAE;IACF,2EAA2E;IAC3E,iFAAiF;IACjF,2EAA2E;IAE3E;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,MAAM,IAAI,YAAY,CACpB,wFAAwF,CACzF,CAAC;IACJ,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,oBAAoB,CACxB,OAAe,EACf,KAA4B;QAE5B,MAAM,IAAI,YAAY,CACpB,8FAA8F,CAC/F,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,kBAAkB,CACtB,QAA8C;QAE9C,MAAM,IAAI,YAAY,CACpB,4FAA4F,CAC7F,CAAC;IACJ,CAAC;IAED,0EAA0E;IAE1E;;;;OAIG;IACK,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,SAAoB,EAAE;QACpD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,YAAY,CAAC,0BAA0B,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBAClD,OAAO,MAAM,CAAC,eAAe,EAA+B,CAAC;YAC/D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,YAAY,CACpB,wBAAwB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC1E,GAAG,CACJ,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACxC,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CACxB,GAAW,EACX,MAAiB;QAEjB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,YAAY,CAAC,0BAA0B,CAAC,CAAC;QACrD,CAAC;QACD,MAAM,UAAU,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAC9C,IAAI,IAAI,GAAmC,IAAI,CAAC;QAChD,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAC3C,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBAC1B,SAAS,CAAC,IAAK,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,OAAO,MAAM,CAAC,eAAe,EAA+B,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,YAAY,CACpB,wBAAwB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC1E,GAAG,CACJ,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC;gBACH,IAAI,EAAE,WAAW,EAAE,CAAC;YACtB,CAAC;YAAC,MAAM,CAAC;gBACP,6EAA6E;YAC/E,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/transform/cleanse.d.ts

@@ -0,0 +1,2 @@
+export declare function applyCleanse(value: unknown, spec: string): unknown;
+//# sourceMappingURL=cleanse.d.ts.map

package/dist/transform/cleanse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanse.d.ts","sourceRoot":"","sources":["../../src/transform/cleanse.ts"],"names":[],"mappings":"AA+BA,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAkClE"}

package/dist/transform/cleanse.js

@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+/**
+ * Cleanse operations — pure string functions applied left-to-right through
+ * the pipe-separated `cleanse:` value on a field mapping.
+ *
+ * Each op is referenced by name in CLAUDE.md's cleanse operations table. Do
+ * NOT add new ops without updating that table.
+ */
+import { TransformError } from '../utils/errors.js';
+const OPS = {
+    trim: (v) => v.trim(),
+    uppercase: (v) => v.toUpperCase(),
+    lowercase: (v) => v.toLowerCase(),
+    titleCase: (v) => v.replace(/\w\S*/g, (w) => w.charAt(0).toUpperCase() + w.slice(1).toLowerCase()),
+    stripNonAlpha: (v) => v.replace(/[^a-zA-Z]/g, ''),
+    stripNonNumeric: (v) => v.replace(/[^0-9]/g, ''),
+    stripWhitespace: (v) => v.replace(/\s+/g, ''),
+    nullIfEmpty: (v) => (v === '' ? null : v),
+    normaliseQuotes: (v) => v.replace(/[\u2018\u2019]/g, "'").replace(/[\u201C\u201D]/g, '"'),
+    // NFD → strip combining marks → leave ASCII text
+    normaliseUnicode: (v) => v.normalize('NFD').replace(/[\u0300-\u036f]/g, ''),
+};
+export function applyCleanse(value, spec) {
+    if (value === null || value === undefined)
+        return value;
+    let current = String(value);
+    for (const step of spec.split('|')) {
+        if (current === null)
+            return null;
+        const [name, ...args] = step.split(':');
+        if (!name)
+            continue;
+        if (name === 'padStart') {
+            const width = Number.parseInt(args[0] ?? '0', 10);
+            const pad = args[1] ?? ' ';
+            if (!Number.isFinite(width) || width < 0) {
+                throw new TransformError(`padStart requires a non-negative width, got "${args[0]}"`);
+            }
+            current = current.padStart(width, pad);
+            continue;
+        }
+        if (name === 'truncate') {
+            const len = Number.parseInt(args[0] ?? '0', 10);
+            if (!Number.isFinite(len) || len < 0) {
+                throw new TransformError(`truncate requires a non-negative length, got "${args[0]}"`);
+            }
+            current = current.slice(0, len);
+            continue;
+        }
+        const op = OPS[name];
+        if (!op) {
+            throw new TransformError(`Unknown cleanse operation: "${name}"`);
+        }
+        current = op(current);
+    }
+    return current;
+}
+//# sourceMappingURL=cleanse.js.map

package/dist/transform/cleanse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanse.js","sourceRoot":"","sources":["../../src/transform/cleanse.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;;;GAMG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAIpD,MAAM,GAAG,GAA8B;IACrC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;IACrB,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE;IACjC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE;IACjC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CACf,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAClF,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;IACjD,eAAe,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;IAChD,eAAe,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7C,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACzC,eAAe,EAAE,CAAC,CAAC,EAAE,EAAE,CACrB,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;IACnE,iDAAiD;IACjD,gBAAgB,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;CAC5E,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,KAAc,EAAE,IAAY;IACvD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACxD,IAAI,OAAO,GAAkB,MAAM,CAAC,KAAK,CAAC,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAClC,MAAM,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;YAClD,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;YAC3B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACzC,MAAM,IAAI,cAAc,CAAC,gDAAgD,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACvF,CAAC;YACD,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACvC,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;YAChD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,IAAI,cAAc,CAAC,iDAAiD,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACxF,CAAC;YACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAChC,SAAS;QACX,CAAC;QAED,MAAM,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,MAAM,IAAI,cAAc,CAAC,+BAA+B,IAAI,GAAG,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/transform/engine.d.ts

@@ -0,0 +1,10 @@
+import { TransformRegistry } from '../plugins/registry.js';
+import type { Pipeline } from '../config/types.js';
+import type { StagingStore } from '../staging/index.js';
+import type { TransformResult } from './types.js';
+export declare class TransformEngine {
+    private readonly transforms;
+    constructor(transforms?: TransformRegistry);
+    run(config: Pipeline, store: StagingStore, sourceTable?: string, targetTable?: string, onProgress?: (rows: number) => void): Promise<TransformResult>;
+}
+//# sourceMappingURL=engine.d.ts.map

package/dist/transform/engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/transform/engine.ts"],"names":[],"mappings":"AAkBA,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,OAAO,KAAK,EAAgB,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,KAAK,EAAc,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAOpE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAIlD,qBAAa,eAAe;IACd,OAAO,CAAC,QAAQ,CAAC,UAAU;gBAAV,UAAU,GAAE,iBAA2C;IAE9E,GAAG,CACP,MAAM,EAAE,QAAQ,EAChB,KAAK,EAAE,YAAY,EACnB,WAAW,SAAY,EACvB,WAAW,SAAoB,EAC/B,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAClC,OAAO,CAAC,eAAe,CAAC;CAgE5B"}