@caracal-lynx/sluice 0.1.0

package/dist/adapters/source/rest.js

@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+/**
+ * REST source adapter.
+ *
+ * - axios + axios-retry (3 retries, exponential backoff, 429 + 5xx)
+ * - Flattens nested JSON objects using `__` separator (`a.b` → `a__b`)
+ * - Three pagination styles supported: offset, page, cursor
+ * - `targetTable` defaults to 'stg_raw'.
+ */
+import axios from 'axios';
+import axiosRetry from 'axios-retry';
+import { SourceError } from '../../utils/errors.js';
+import { logger } from '../../utils/logger.js';
+function getPath(obj, dotPath) {
+    let cur = obj;
+    for (const key of dotPath.split('.')) {
+        if (cur === null || cur === undefined || typeof cur !== 'object')
+            return undefined;
+        cur = cur[key];
+    }
+    return cur;
+}
+function flatten(obj, prefix = '', out = {}) {
+    for (const [k, v] of Object.entries(obj)) {
+        const key = prefix ? `${prefix}__${k}` : k;
+        if (v !== null && typeof v === 'object' && !Array.isArray(v)) {
+            flatten(v, key, out);
+        }
+        else {
+            out[key] = Array.isArray(v) ? JSON.stringify(v) : v;
+        }
+    }
+    return out;
+}
+export class RestSourceAdapter {
+    id = 'rest';
+    client = null;
+    async connect(config) {
+        if (!config.endpoint) {
+            throw new SourceError('rest source requires `endpoint`');
+        }
+        const axiosCfg = { timeout: 30000 };
+        if (config.headers)
+            axiosCfg.headers = config.headers;
+        const client = axios.create(axiosCfg);
+        axiosRetry(client, {
+            retries: 3,
+            retryDelay: axiosRetry.exponentialDelay,
+            retryCondition: (err) => {
+                const status = err.response?.status;
+                return (axiosRetry.isNetworkOrIdempotentRequestError(err) ||
+                    status === 429 ||
+                    (status !== undefined && status >= 500));
+            },
+        });
+        this.client = client;
+    }
+    async disconnect() {
+        this.client = null;
+    }
+    async extract(config, store, runConfig, onProgress, targetTable = 'stg_raw') {
+        if (!config.endpoint)
+            throw new SourceError('rest source requires `endpoint`');
+        if (!this.client)
+            throw new SourceError('rest: not connected');
+        const pagination = config.pagination;
+        let allRecords = [];
+        if (!pagination) {
+            allRecords = await this.fetchOne(config.endpoint, config, undefined);
+        }
+        else {
+            allRecords = await this.fetchPaged(config.endpoint, config, pagination);
+        }
+        if (allRecords.length === 0) {
+            throw new SourceError(`rest endpoint "${config.endpoint}" returned zero records`);
+        }
+        const flatRecords = allRecords.map((r) => flatten(r));
+        const columns = collectColumns(flatRecords);
+        await store.createTable(targetTable, columns);
+        for (let i = 0; i < flatRecords.length; i += runConfig.batchSize) {
+            const batch = flatRecords.slice(i, i + runConfig.batchSize);
+            await store.insertBatch(targetTable, batch);
+            onProgress(Math.min(i + batch.length, flatRecords.length));
+        }
+        return { rowsExtracted: flatRecords.length, tableName: targetTable, columns };
+    }
+    async fetchOne(endpoint, config, params) {
+        const body = await this.get(endpoint, params);
+        const dataField = config.pagination?.dataField;
+        const data = dataField ? getPath(body, dataField) : body;
+        if (!Array.isArray(data)) {
+            throw new SourceError(`rest: expected an array at dataField "${dataField ?? '(root)'}", got ${typeof data}`);
+        }
+        return data;
+    }
+    async fetchPaged(endpoint, config, pagination) {
+        const pageSize = pagination.pageSize;
+        const dataField = pagination.dataField ?? 'data';
+        const pageParam = pagination.pageParam ?? 'page';
+        const totalField = pagination.totalField;
+        const all = [];
+        let page = 0;
+        let offset = 0;
+        let cursor;
+        let total;
+        while (true) {
+            const params = {};
+            if (pagination.type === 'offset') {
+                params[pageParam] = offset;
+                params['limit'] = pageSize;
+            }
+            else if (pagination.type === 'page') {
+                params[pageParam] = page + 1;
+                params['pageSize'] = pageSize;
+            }
+            else {
+                if (cursor !== undefined) {
+                    const cursorParam = pagination.cursorParam ?? 'cursor';
+                    params[cursorParam] = cursor;
+                }
+            }
+            const body = await this.get(endpoint, params);
+            const chunk = getPath(body, dataField);
+            if (!Array.isArray(chunk)) {
+                throw new SourceError(`rest: expected array at dataField "${dataField}" on page ${page}`);
+            }
+            all.push(...chunk);
+            if (total === undefined && totalField) {
+                const t = getPath(body, totalField);
+                if (typeof t === 'number')
+                    total = t;
+            }
+            if (chunk.length === 0)
+                break;
+            if (pagination.type === 'offset') {
+                offset += chunk.length;
+                if (total !== undefined && offset >= total)
+                    break;
+                if (chunk.length < pageSize)
+                    break;
+            }
+            else if (pagination.type === 'page') {
+                page += 1;
+                if (total !== undefined && all.length >= total)
+                    break;
+                if (chunk.length < pageSize)
+                    break;
+            }
+            else {
+                const cursorField = pagination.cursorField;
+                if (!cursorField) {
+                    throw new SourceError('rest: cursor pagination requires `pagination.cursorField`');
+                }
+                const next = getPath(body, cursorField);
+                if (next === null || next === undefined || next === '')
+                    break;
+                cursor = String(next);
+            }
+        }
+        logger.debug({ total: all.length, pagination: pagination.type }, 'rest: paged fetch complete');
+        return all;
+    }
+    async get(url, params) {
+        try {
+            const res = await this.client.get(url, { params });
+            return res.data;
+        }
+        catch (err) {
+            throw new SourceError(`rest: GET ${url} failed: ${err instanceof Error ? err.message : String(err)}`, err);
+        }
+    }
+}
+function collectColumns(rows) {
+    const names = new Set();
+    for (const r of rows) {
+        for (const k of Object.keys(r))
+            names.add(k);
+    }
+    return [...names].map((n) => ({ name: n, duckDbType: 'VARCHAR' }));
+}
+//# sourceMappingURL=rest.js.map

package/dist/adapters/source/rest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rest.js","sourceRoot":"","sources":["../../../src/adapters/source/rest.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;;;;GAOG;AAEH,OAAO,KAAsD,MAAM,OAAO,CAAC;AAC3E,OAAO,UAAU,MAAM,aAAa,CAAC;AAIrC,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAM/C,SAAS,OAAO,CAAC,GAAY,EAAE,OAAe;IAC5C,IAAI,GAAG,GAAY,GAAG,CAAC;IACvB,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QACnF,GAAG,GAAI,GAA+B,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,OAAO,CACd,GAA4B,EAC5B,MAAM,GAAG,EAAE,EACX,MAA+B,EAAE;IAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,OAAO,CAAC,CAA4B,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,OAAO,iBAAiB;IACnB,EAAE,GAAG,MAAM,CAAC;IACb,MAAM,GAAyB,IAAI,CAAC;IAE5C,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrB,MAAM,IAAI,WAAW,CAAC,iCAAiC,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,QAAQ,GAAuB,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACxD,IAAI,MAAM,CAAC,OAAO;YAAE,QAAQ,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QACtD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACtC,UAAU,CAAC,MAAM,EAAE;YACjB,OAAO,EAAE,CAAC;YACV,UAAU,EAAE,UAAU,CAAC,gBAAgB;YACvC,cAAc,EAAE,CAAC,GAAG,EAAE,EAAE;gBACtB,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC;gBACpC,OAAO,CACL,UAAU,CAAC,iCAAiC,CAAC,GAAG,CAAC;oBACjD,MAAM,KAAK,GAAG;oBACd,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,IAAI,GAAG,CAAC,CACxC,CAAC;YACJ,CAAC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,KAAmB,EACnB,SAAoB,EACpB,UAAkC,EAClC,WAAW,GAAG,SAAS;QAEvB,IAAI,CAAC,MAAM,CAAC,QAAQ;YAAE,MAAM,IAAI,WAAW,CAAC,iCAAiC,CAAC,CAAC;QAC/E,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,WAAW,CAAC,qBAAqB,CAAC,CAAC;QAE/D,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACrC,IAAI,UAAU,GAA8B,EAAE,CAAC;QAE/C,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,WAAW,CAAC,kBAAkB,MAAM,CAAC,QAAQ,yBAAyB,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,OAAO,GAAiB,cAAc,CAAC,WAAW,CAAC,CAAC;QAC1D,MAAM,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAE9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACjE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;YAC5D,MAAM,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7D,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;IAChF,CAAC;IAEO,KAAK,CAAC,QAAQ,CACpB,QAAgB,EAChB,MAAc,EACd,MAA2C;QAE3C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC;QAC/C,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,WAAW,CACnB,yCAAyC,SAAS,IAAI,QAAQ,UAAU,OAAO,IAAI,EAAE,CACtF,CAAC;QACJ,CAAC;QACD,OAAO,IAAiC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,QAAgB,EAChB,MAAc,EACd,UAA4B;QAE5B,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;QACrC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,IAAI,MAAM,CAAC;QACjD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,IAAI,MAAM,CAAC;QACjD,MAAM,UAAU,GAAG,UAAU,CAAC,UAAU,CAAC;QAEzC,MAAM,GAAG,GAA8B,EAAE,CAAC;QAC1C,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAA0B,CAAC;QAC/B,IAAI,KAAyB,CAAC;QAE9B,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAA4B,EAAE,CAAC;YAC3C,IAAI,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACjC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC;gBAC3B,MAAM,CAAC,OAAO,CAAC,GAAG,QAAQ,CAAC;YAC7B,CAAC;iBAAM,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACtC,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC;gBAC7B,MAAM,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBACzB,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,IAAI,QAAQ,CAAC;oBACvD,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC;gBAC/B,CAAC;YACH,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,WAAW,CAAC,sCAAsC,SAAS,aAAa,IAAI,EAAE,CAAC,CAAC;YAC5F,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,GAAI,KAAmC,CAAC,CAAC;YAElD,IAAI,KAAK,KAAK,SAAS,IAAI,UAAU,EAAE,CAAC;gBACtC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACpC,IAAI,OAAO,CAAC,KAAK,QAAQ;oBAAE,KAAK,GAAG,CAAC,CAAC;YACvC,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC9B,IAAI,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;gBACvB,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,IAAI,KAAK;oBAAE,MAAM;gBAClD,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;oBAAE,MAAM;YACrC,CAAC;iBAAM,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACtC,IAAI,IAAI,CAAC,CAAC;gBACV,IAAI,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,IAAI,KAAK;oBAAE,MAAM;gBACtD,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;oBAAE,MAAM;YACrC,CAAC;iBAAM,CAAC;gBACN,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;gBAC3C,IAAI,CAAC,WAAW,EAAE,CAAC;oBACjB,MAAM,IAAI,WAAW,CAAC,2DAA2D,CAAC,CAAC;gBACrF,CAAC;gBACD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;gBACxC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE;oBAAE,MAAM;gBAC9D,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,EAAE,4BAA4B,CAAC,CAAC;QAC/F,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,GAAG,CACf,GAAW,EACX,MAA2C;QAE3C,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YACpD,OAAO,GAAG,CAAC,IAAI,CAAC;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,WAAW,CACnB,aAAa,GAAG,YAAY,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC9E,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED,SAAS,cAAc,CAAC,IAA+B;IACrD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;AACrE,CAAC"}

package/dist/adapters/source/rest.types.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Local re-exports of the config types most used in rest.ts. Keeps rest.ts
+ * imports readable without pulling broadly from config.
+ */
+export type { RunConfig, SourceConfig } from '../../config/types.js';
+export interface PaginationConfig {
+    type: 'offset' | 'cursor' | 'page';
+    pageSize: number;
+    pageParam?: string;
+    totalField?: string;
+    dataField?: string;
+    cursorField?: string;
+    cursorParam?: string;
+}
+//# sourceMappingURL=rest.types.d.ts.map

package/dist/adapters/source/rest.types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rest.types.d.ts","sourceRoot":"","sources":["../../../src/adapters/source/rest.types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErE,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}

package/dist/adapters/source/rest.types.js

@@ -0,0 +1,6 @@
+/**
+ * Local re-exports of the config types most used in rest.ts. Keeps rest.ts
+ * imports readable without pulling broadly from config.
+ */
+export {};
+//# sourceMappingURL=rest.types.js.map

package/dist/adapters/source/rest.types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rest.types.js","sourceRoot":"","sources":["../../../src/adapters/source/rest.types.ts"],"names":[],"mappings":"AAAA;;;GAGG"}

package/dist/adapters/source/types.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Source adapter interface.
+ *
+ * `ExtractResult.tableName` is caller-supplied; single-source pipelines use
+ * `'stg_raw'`, and `MultiSourcePipelineRunner` passes `'stg_raw_{sourceId}'`
+ * via the optional `targetTable` parameter on `extract()`.
+ */
+import type { RunConfig, SourceConfig } from '../../config/types.js';
+import type { ColumnMeta, StagingStore } from '../../staging/index.js';
+export interface SourceAdapter {
+    readonly id: string;
+    connect(config: SourceConfig): Promise<void>;
+    extract(config: SourceConfig, store: StagingStore, runConfig: RunConfig, onProgress: (rows: number) => void, targetTable?: string): Promise<ExtractResult>;
+    disconnect(): Promise<void>;
+}
+export interface ExtractResult {
+    rowsExtracted: number;
+    /** Caller-supplied (or adapter default). Single-source pipelines use 'stg_raw'. */
+    tableName: string;
+    columns: ColumnMeta[];
+}
+export type { ColumnMeta } from '../../staging/index.js';
+//# sourceMappingURL=types.d.ts.map

package/dist/adapters/source/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/adapters/source/types.ts"],"names":[],"mappings":"AAGA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEvE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7C,OAAO,CACL,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,YAAY,EACnB,SAAS,EAAE,SAAS,EACpB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAClC,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,aAAa,CAAC,CAAC;IAE1B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,mFAAmF;IACnF,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,YAAY,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/adapters/source/types.js

@@ -0,0 +1,4 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+export {};
+//# sourceMappingURL=types.js.map

package/dist/adapters/source/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/adapters/source/types.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC"}

package/dist/adapters/source/xlsx.d.ts

@@ -0,0 +1,10 @@
+import type { RunConfig, SourceConfig } from '../../config/types.js';
+import type { StagingStore } from '../../staging/index.js';
+import type { ExtractResult, SourceAdapter } from './types.js';
+export declare class XlsxSourceAdapter implements SourceAdapter {
+    readonly id = "xlsx";
+    connect(_config: SourceConfig): Promise<void>;
+    disconnect(): Promise<void>;
+    extract(config: SourceConfig, store: StagingStore, runConfig: RunConfig, onProgress: (rows: number) => void, targetTable?: string): Promise<ExtractResult>;
+}
+//# sourceMappingURL=xlsx.d.ts.map

package/dist/adapters/source/xlsx.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"xlsx.d.ts","sourceRoot":"","sources":["../../../src/adapters/source/xlsx.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,KAAK,EAAc,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGvE,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE/D,qBAAa,iBAAkB,YAAW,aAAa;IACrD,QAAQ,CAAC,EAAE,UAAU;IAEf,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7C,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,OAAO,CACX,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,YAAY,EACnB,SAAS,EAAE,SAAS,EACpB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAClC,WAAW,SAAY,GACtB,OAAO,CAAC,aAAa,CAAC;CAiE1B"}

package/dist/adapters/source/xlsx.js

@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+/**
+ * Excel (XLSX) source adapter. Uses SheetJS to convert the selected sheet
+ * to a CSV blob, then re-parses it with csv-parse — all columns VARCHAR.
+ * `targetTable` defaults to 'stg_raw'.
+ */
+import { readFile as readFileAsync } from 'node:fs/promises';
+import { parse } from 'csv-parse/sync';
+import { read as xlsxRead, utils as xlsxUtils } from 'xlsx';
+import { SourceError } from '../../utils/errors.js';
+import { logger } from '../../utils/logger.js';
+export class XlsxSourceAdapter {
+    id = 'xlsx';
+    async connect(_config) {
+        // File-based; nothing to establish.
+    }
+    async disconnect() {
+        // nothing to release.
+    }
+    async extract(config, store, runConfig, onProgress, targetTable = 'stg_raw') {
+        if (!config.file) {
+            throw new SourceError('xlsx source requires `file`');
+        }
+        let buf;
+        try {
+            buf = await readFileAsync(config.file);
+        }
+        catch (err) {
+            throw new SourceError(`xlsx: failed to read "${config.file}": ${err instanceof Error ? err.message : String(err)}`, err);
+        }
+        const workbook = xlsxRead(buf, { type: 'buffer' });
+        const sheetNames = workbook.SheetNames;
+        if (sheetNames.length === 0) {
+            throw new SourceError(`xlsx: workbook "${config.file}" has no sheets`);
+        }
+        if (sheetNames.length > 1 && config.sheet === undefined) {
+            logger.warn({ file: config.file, sheets: sheetNames, using: sheetNames[0] }, 'xlsx: multiple sheets found and `source.sheet` is unset — using first sheet');
+        }
+        const sheetKey = config.sheet === undefined
+            ? sheetNames[0]
+            : typeof config.sheet === 'number'
+                ? sheetNames[config.sheet]
+                : config.sheet;
+        if (!sheetKey || !workbook.Sheets[sheetKey]) {
+            throw new SourceError(`xlsx: sheet "${String(config.sheet)}" not found in ${config.file}`);
+        }
+        const csvText = xlsxUtils.sheet_to_csv(workbook.Sheets[sheetKey], { strip: false });
+        const records = parse(csvText, {
+            columns: true,
+            skip_empty_lines: true,
+            bom: true,
+        });
+        if (records.length === 0) {
+            throw new SourceError(`xlsx: sheet "${sheetKey}" is empty`);
+        }
+        const first = records[0];
+        const columns = Object.keys(first).map((n) => ({
+            name: n,
+            duckDbType: 'VARCHAR',
+        }));
+        await store.createTable(targetTable, columns);
+        for (let i = 0; i < records.length; i += runConfig.batchSize) {
+            const batch = records.slice(i, i + runConfig.batchSize);
+            await store.insertBatch(targetTable, batch);
+            onProgress(Math.min(i + batch.length, records.length));
+        }
+        return { rowsExtracted: records.length, tableName: targetTable, columns };
+    }
+}
+//# sourceMappingURL=xlsx.js.map

package/dist/adapters/source/xlsx.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"xlsx.js","sourceRoot":"","sources":["../../../src/adapters/source/xlsx.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,IAAI,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,IAAI,IAAI,QAAQ,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAI5D,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAG/C,MAAM,OAAO,iBAAiB;IACnB,EAAE,GAAG,MAAM,CAAC;IAErB,KAAK,CAAC,OAAO,CAAC,OAAqB;QACjC,oCAAoC;IACtC,CAAC;IAED,KAAK,CAAC,UAAU;QACd,sBAAsB;IACxB,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAoB,EACpB,KAAmB,EACnB,SAAoB,EACpB,UAAkC,EAClC,WAAW,GAAG,SAAS;QAEvB,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,IAAI,WAAW,CAAC,6BAA6B,CAAC,CAAC;QACvD,CAAC;QAED,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,WAAW,CACnB,yBAAyB,MAAM,CAAC,IAAI,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC5F,GAAG,CACJ,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;QACvC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,WAAW,CAAC,mBAAmB,MAAM,CAAC,IAAI,iBAAiB,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CACT,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,EAC/D,6EAA6E,CAC9E,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GACZ,MAAM,CAAC,KAAK,KAAK,SAAS;YACxB,CAAC,CAAC,UAAU,CAAC,CAAC,CAAE;YAChB,CAAC,CAAC,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ;gBAChC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC1B,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QAErB,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,WAAW,CAAC,gBAAgB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7F,CAAC;QAED,MAAM,OAAO,GAAG,SAAS,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QACpF,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,EAAE;YAC7B,OAAO,EAAE,IAAI;YACb,gBAAgB,EAAE,IAAI;YACtB,GAAG,EAAE,IAAI;SACV,CAA8B,CAAC;QAEhC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,WAAW,CAAC,gBAAgB,QAAQ,YAAY,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;QAC1B,MAAM,OAAO,GAAiB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3D,IAAI,EAAE,CAAC;YACP,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC,CAAC;QACJ,MAAM,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAE9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;IAC5E,CAAC;CACF"}

package/dist/adapters/target/bc.d.ts

@@ -0,0 +1,21 @@
+import { type AxiosInstance } from 'axios';
+import type { RunConfig, TargetConfig } from '../../config/types.js';
+import { type StagingStore } from '../../staging/index.js';
+import type { LoadResult, TargetAdapter } from './types.js';
+export declare class BcTokenManager {
+    private readonly http;
+    private token;
+    private expiresAt;
+    constructor(http?: AxiosInstance);
+    getToken(): Promise<string>;
+}
+export declare class BcTargetAdapter implements TargetAdapter {
+    readonly id = "bc";
+    private tokenManager;
+    private http;
+    connect(config: TargetConfig): Promise<void>;
+    disconnect(): Promise<void>;
+    load(config: TargetConfig, store: StagingStore, runConfig: RunConfig, onProgress: (rows: number) => void): Promise<LoadResult>;
+    private handleRowError;
+}
+//# sourceMappingURL=bc.d.ts.map

package/dist/adapters/target/bc.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bc.d.ts","sourceRoot":"","sources":["../../../src/adapters/target/bc.ts"],"names":[],"mappings":"AAsBA,OAAc,EAAE,KAAK,aAAa,EAAE,MAAM,OAAO,CAAC;AAElD,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,EAAc,KAAK,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAIvE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAM5D,qBAAa,cAAc;IAIb,OAAO,CAAC,QAAQ,CAAC,IAAI;IAHjC,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,SAAS,CAAK;gBAEO,IAAI,GAAE,aAAiD;IAE9E,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;CAiClC;AAED,qBAAa,eAAgB,YAAW,aAAa;IACnD,QAAQ,CAAC,EAAE,QAAQ;IACnB,OAAO,CAAC,YAAY,CAAwC;IAC5D,OAAO,CAAC,IAAI,CAAoD;IAE1D,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAO5C,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,IAAI,CACR,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,YAAY,EACnB,SAAS,EAAE,SAAS,EACpB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GACjC,OAAO,CAAC,UAAU,CAAC;YA+ER,cAAc;CA0B7B"}

package/dist/adapters/target/bc.js

@@ -0,0 +1,188 @@
+// SPDX-License-Identifier: Elastic-2.0
+// Copyright (c) 2026 Caracal Lynx Ltd.
+/**
+ * Business Central REST (OData v4) target adapter.
+ *
+ * Token management: BcTokenManager acquires OAuth2 client-credential tokens
+ * against login.microsoftonline.com, caches in-memory, refreshes 60s before
+ * expiry. Tenant id, client id, and client secret are read from
+ * BC_TENANT_ID / BC_CLIENT_ID / BC_CLIENT_SECRET env vars (per CLAUDE.md).
+ *
+ * Load path: for each batch (max 100 rows per $batch), build a multipart
+ * body and POST to `{baseUrl}/api/{version}/companies({company})/$batch`.
+ * HTTP 409 with onConflict: upsert → individual PATCH. Other 4xx → increment
+ * rowsFailed and continue iff runConfig.onError === 'continue'.
+ *
+ * Implementation is intentionally minimal. Rigorous multipart parsing is
+ * deferred; tests mock the token endpoint and response shape.
+ */
+import { randomUUID } from 'node:crypto';
+import axios from 'axios';
+import { quoteIdent } from '../../staging/index.js';
+import { requireEnv } from '../../utils/env.js';
+import { ConfigError, LoadError } from '../../utils/errors.js';
+import { logger } from '../../utils/logger.js';
+const STAGING_TABLE = 'stg_transformed';
+const MAX_BATCH_OPS = 100;
+const TOKEN_REFRESH_BUFFER_MS = 60_000;
+export class BcTokenManager {
+    http;
+    token = null;
+    expiresAt = 0;
+    constructor(http = axios.create({ timeout: 30_000 })) {
+        this.http = http;
+    }
+    async getToken() {
+        if (this.token && Date.now() < this.expiresAt - TOKEN_REFRESH_BUFFER_MS) {
+            return this.token;
+        }
+        const tenantId = requireEnv('BC_TENANT_ID');
+        const clientId = requireEnv('BC_CLIENT_ID');
+        const clientSecret = requireEnv('BC_CLIENT_SECRET');
+        const url = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
+        try {
+            const res = await this.http.post(url, new URLSearchParams({
+                grant_type: 'client_credentials',
+                client_id: clientId,
+                client_secret: clientSecret,
+                scope: 'https://api.businesscentral.dynamics.com/.default',
+            }));
+            const body = res.data;
+            if (!body.access_token) {
+                throw new LoadError(`bc: token response had no access_token`);
+            }
+            this.token = body.access_token;
+            const expiresIn = typeof body.expires_in === 'number' ? body.expires_in : 3600;
+            this.expiresAt = Date.now() + expiresIn * 1000;
+            return this.token;
+        }
+        catch (err) {
+            throw new LoadError(`bc: token acquisition failed: ${err instanceof Error ? err.message : String(err)}`, err);
+        }
+    }
+}
+export class BcTargetAdapter {
+    id = 'bc';
+    tokenManager = new BcTokenManager();
+    http = axios.create({ timeout: 30_000 });
+    async connect(config) {
+        if (!config.baseUrl)
+            throw new ConfigError('bc target requires `baseUrl`');
+        if (!config.company)
+            throw new ConfigError('bc target requires `company`');
+        if (!config.entity)
+            throw new ConfigError('bc target requires `entity`');
+        // Token is fetched lazily on first load call.
+    }
+    async disconnect() {
+        // nothing to release — axios client is reused per process.
+    }
+    async load(config, store, runConfig, onProgress) {
+        const rows = await store.query(`SELECT * FROM ${quoteIdent(STAGING_TABLE)}`);
+        if (rows.length === 0) {
+            return { rowsLoaded: 0, rowsFailed: 0 };
+        }
+        const token = await this.tokenManager.getToken();
+        const endpoint = `${config.baseUrl}/api/${config.apiVersion}/companies(${encodeURIComponent(config.company)})/${config.entity}`;
+        let rowsLoaded = 0;
+        let rowsFailed = 0;
+        for (let i = 0; i < rows.length; i += MAX_BATCH_OPS) {
+            const batch = rows.slice(i, i + MAX_BATCH_OPS);
+            if (!config.batchEndpoint) {
+                // Fall back to per-row POST when $batch is disabled
+                for (const row of batch) {
+                    try {
+                        await this.http.post(endpoint, row, {
+                            headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+                        });
+                        rowsLoaded++;
+                    }
+                    catch (err) {
+                        const handled = await this.handleRowError(err, endpoint, row, token, config);
+                        if (handled)
+                            rowsLoaded++;
+                        else {
+                            rowsFailed++;
+                            if (runConfig.onError === 'stop') {
+                                throw new LoadError(`bc: row insert failed: ${err instanceof Error ? err.message : String(err)}`, err);
+                            }
+                        }
+                    }
+                    onProgress(rowsLoaded);
+                }
+                continue;
+            }
+            // $batch path
+            const boundary = `batch_${randomUUID()}`;
+            const body = buildBatchBody(batch, endpoint, boundary);
+            const batchUrl = `${config.baseUrl}/api/${config.apiVersion}/companies(${encodeURIComponent(config.company)})/$batch`;
+            try {
+                const res = await this.http.post(batchUrl, body, {
+                    headers: {
+                        Authorization: `Bearer ${token}`,
+                        'Content-Type': `multipart/mixed; boundary=${boundary}`,
+                    },
+                });
+                const counts = countBatchResponses(String(res.data ?? ''));
+                rowsLoaded += counts.succeeded;
+                rowsFailed += counts.failed;
+                if (counts.failed > 0 && runConfig.onError === 'stop') {
+                    throw new LoadError(`bc: ${counts.failed} batch operations failed`);
+                }
+            }
+            catch (err) {
+                rowsFailed += batch.length;
+                if (runConfig.onError === 'stop') {
+                    throw new LoadError(`bc: $batch failed: ${err instanceof Error ? err.message : String(err)}`, err);
+                }
+                logger.warn({ err: err instanceof Error ? err.message : String(err) }, 'bc: $batch failed (onError: continue)');
+            }
+            onProgress(rowsLoaded);
+        }
+        return { rowsLoaded, rowsFailed };
+    }
+    async handleRowError(err, endpoint, row, token, config) {
+        if (!axios.isAxiosError(err))
+            return false;
+        const status = err.response?.status;
+        if (status === 409 && config.onConflict === 'upsert') {
+            try {
+                await this.http.patch(endpoint, row, {
+                    headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+                });
+                return true;
+            }
+            catch (patchErr) {
+                logger.warn({ err: patchErr instanceof Error ? patchErr.message : String(patchErr) }, 'bc: upsert PATCH failed');
+                return false;
+            }
+        }
+        logger.warn({ status }, 'bc: row insert failed');
+        return false;
+    }
+}
+function buildBatchBody(batch, endpoint, boundary) {
+    const parts = [];
+    for (const row of batch) {
+        const payload = JSON.stringify(row);
+        parts.push(`--${boundary}`, 'Content-Type: application/http', 'Content-Transfer-Encoding: binary', '', `POST ${endpoint} HTTP/1.1`, 'Content-Type: application/json', '', payload, '');
+    }
+    parts.push(`--${boundary}--`, '');
+    return parts.join('\r\n');
+}
+function countBatchResponses(body) {
+    // Minimal parser: count HTTP status lines. 2xx → success, else failure.
+    const re = /HTTP\/[0-9.]+ (\d{3})/g;
+    let match;
+    let succeeded = 0;
+    let failed = 0;
+    while ((match = re.exec(body))) {
+        const code = Number.parseInt(match[1], 10);
+        if (code >= 200 && code < 300)
+            succeeded++;
+        else
+            failed++;
+    }
+    return { succeeded, failed };
+}
+//# sourceMappingURL=bc.js.map

package/dist/adapters/target/bc.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bc.js","sourceRoot":"","sources":["../../../src/adapters/target/bc.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,uCAAuC;AAEvC;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,KAA6B,MAAM,OAAO,CAAC;AAGlD,OAAO,EAAE,UAAU,EAAqB,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAG/C,MAAM,aAAa,GAAG,iBAAiB,CAAC;AACxC,MAAM,aAAa,GAAG,GAAG,CAAC;AAC1B,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAEvC,MAAM,OAAO,cAAc;IAII;IAHrB,KAAK,GAAkB,IAAI,CAAC;IAC5B,SAAS,GAAG,CAAC,CAAC;IAEtB,YAA6B,OAAsB,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAAvD,SAAI,GAAJ,IAAI,CAAmD;IAAG,CAAC;IAExF,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,uBAAuB,EAAE,CAAC;YACxE,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QACD,MAAM,QAAQ,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;QAC5C,MAAM,YAAY,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,qCAAqC,QAAQ,oBAAoB,CAAC;QAC9E,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAC9B,GAAG,EACH,IAAI,eAAe,CAAC;gBAClB,UAAU,EAAE,oBAAoB;gBAChC,SAAS,EAAE,QAAQ;gBACnB,aAAa,EAAE,YAAY;gBAC3B,KAAK,EAAE,mDAAmD;aAC3D,CAAC,CACH,CAAC;YACF,MAAM,IAAI,GAAG,GAAG,CAAC,IAAsD,CAAC;YACxE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvB,MAAM,IAAI,SAAS,CAAC,wCAAwC,CAAC,CAAC;YAChE,CAAC;YACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;YAC/B,MAAM,SAAS,GAAG,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/E,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,IAAI,CAAC;YAC/C,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,SAAS,CACjB,iCAAiC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EACnF,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED,MAAM,OAAO,eAAe;IACjB,EAAE,GAAG,IAAI,CAAC;IACX,YAAY,GAAmB,IAAI,cAAc,EAAE,CAAC;IACpD,IAAI,GAAkB,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAEhE,KAAK,CAAC,OAAO,CAAC,MAAoB;QAChC,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,MAAM,IAAI,WAAW,CAAC,8BAA8B,CAAC,CAAC;QAC3E,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,MAAM,IAAI,WAAW,CAAC,8BAA8B,CAAC,CAAC;QAC3E,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,MAAM,IAAI,WAAW,CAAC,6BAA6B,CAAC,CAAC;QACzE,8CAA8C;IAChD,CAAC;IAED,KAAK,CAAC,UAAU;QACd,2DAA2D;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAAoB,EACpB,KAAmB,EACnB,SAAoB,EACpB,UAAkC;QAElC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAC5B,iBAAiB,UAAU,CAAC,aAAa,CAAC,EAAE,CAC7C,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;QAC1C,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,OAAO,QAAQ,MAAM,CAAC,UAAU,cAAc,kBAAkB,CAAC,MAAM,CAAC,OAAQ,CAAC,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QAEjI,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,aAAa,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,CAAC;YAE/C,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC1B,oDAAoD;gBACpD,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACxB,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE;4BAClC,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;yBAClF,CAAC,CAAC;wBACH,UAAU,EAAE,CAAC;oBACf,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;wBAC7E,IAAI,OAAO;4BAAE,UAAU,EAAE,CAAC;6BACrB,CAAC;4BACJ,UAAU,EAAE,CAAC;4BACb,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;gCACjC,MAAM,IAAI,SAAS,CACjB,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC5E,GAAG,CACJ,CAAC;4BACJ,CAAC;wBACH,CAAC;oBACH,CAAC;oBACD,UAAU,CAAC,UAAU,CAAC,CAAC;gBACzB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,cAAc;YACd,MAAM,QAAQ,GAAG,SAAS,UAAU,EAAE,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACvD,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,OAAO,QAAQ,MAAM,CAAC,UAAU,cAAc,kBAAkB,CAAC,MAAM,CAAC,OAAQ,CAAC,UAAU,CAAC;YACvH,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE;oBAC/C,OAAO,EAAE;wBACP,aAAa,EAAE,UAAU,KAAK,EAAE;wBAChC,cAAc,EAAE,6BAA6B,QAAQ,EAAE;qBACxD;iBACF,CAAC,CAAC;gBACH,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC3D,UAAU,IAAI,MAAM,CAAC,SAAS,CAAC;gBAC/B,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC5B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;oBACtD,MAAM,IAAI,SAAS,CAAC,OAAO,MAAM,CAAC,MAAM,0BAA0B,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;gBAC3B,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;oBACjC,MAAM,IAAI,SAAS,CACjB,sBAAsB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EACxE,GAAG,CACJ,CAAC;gBACJ,CAAC;gBACD,MAAM,CAAC,IAAI,CACT,EAAE,GAAG,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EACzD,uCAAuC,CACxC,CAAC;YACJ,CAAC;YACD,UAAU,CAAC,UAAU,CAAC,CAAC;QACzB,CAAC;QAED,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;IACpC,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,GAAY,EACZ,QAAgB,EAChB,GAA4B,EAC5B,KAAa,EACb,MAAoB;QAEpB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAC3C,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC;QACpC,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YACrD,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE;oBACnC,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;iBAClF,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,OAAO,QAAQ,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CACT,EAAE,GAAG,EAAE,QAAQ,YAAY,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EACxE,yBAAyB,CAC1B,CAAC;gBACF,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,uBAAuB,CAAC,CAAC;QACjD,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAED,SAAS,cAAc,CACrB,KAAgC,EAChC,QAAgB,EAChB,QAAgB;IAEhB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CACR,KAAK,QAAQ,EAAE,EACf,gCAAgC,EAChC,mCAAmC,EACnC,EAAE,EACF,QAAQ,QAAQ,WAAW,EAC3B,gCAAgC,EAChC,EAAE,EACF,OAAO,EACP,EAAE,CACH,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC;IAClC,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,wEAAwE;IACxE,MAAM,EAAE,GAAG,wBAAwB,CAAC;IACpC,IAAI,KAA6B,CAAC;IAClC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,GAAG,GAAG;YAAE,SAAS,EAAE,CAAC;;YACtC,MAAM,EAAE,CAAC;IAChB,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAC/B,CAAC"}

package/dist/adapters/target/bluecherry.d.ts

@@ -0,0 +1,10 @@
+import type { RunConfig, TargetConfig } from '../../config/types.js';
+import type { StagingStore } from '../../staging/index.js';
+import type { LoadResult, TargetAdapter } from './types.js';
+export declare class BlueCherryTargetAdapter implements TargetAdapter {
+    readonly id = "bluecherry";
+    connect(config: TargetConfig): Promise<void>;
+    disconnect(): Promise<void>;
+    load(config: TargetConfig, store: StagingStore, _runConfig: RunConfig, onProgress: (rows: number) => void): Promise<LoadResult>;
+}
+//# sourceMappingURL=bluecherry.d.ts.map

package/dist/adapters/target/bluecherry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bluecherry.d.ts","sourceRoot":"","sources":["../../../src/adapters/target/bluecherry.ts"],"names":[],"mappings":"AA4BA,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAG3D,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AA6B5D,qBAAa,uBAAwB,YAAW,aAAa;IAC3D,QAAQ,CAAC,EAAE,gBAAgB;IAErB,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAa5C,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,IAAI,CACR,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,YAAY,EACnB,UAAU,EAAE,SAAS,EACrB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GACjC,OAAO,CAAC,UAAU,CAAC;CA6EvB"}