@channel.io/app-sdk-server 0.13.1 → 0.15.0

package/dist/datasource/arrow.js

@@ -0,0 +1,261 @@
+import { createRequire } from "node:module";
+import { DataSourceErrorCode, DataSourceExecutionError } from "./types.js";
+let cachedArrow;
+export function encodeRowsToArrowIPC(columns, rows) {
+    if (columns.length === 0) {
+        throw new Error("columns are required");
+    }
+    return rowsToArrowIPC(columns, rows.map((row) => columns.map((column) => row[column.name])));
+}
+export function arrowRowsChunk(columns, rows) {
+    return {
+        arrow: {
+            dataBody: encodeRowsToArrowIPC(columns, rows),
+        },
+    };
+}
+export function resultChunk(rowCount, limitExceeded = false, executionMs = 0) {
+    return {
+        result: {
+            rowCount,
+            limitExceeded,
+            executionMs,
+        },
+    };
+}
+export function writeRowsAsArrowIpcBatch(sink, fields, rows, options = {}) {
+    const bytes = rowsToArrowIPC(fields, rows);
+    let schemaSent = false;
+    let frameCount = 0;
+    for (const frame of splitArrowIpcStream(bytes)) {
+        if (frame.kind === "schema") {
+            if (options.sendSchema !== false) {
+                options.byteLimit?.reserve(frame.bytes.byteLength);
+                sink.send({ arrow: { dataHeader: frame.bytes } });
+                schemaSent = true;
+                frameCount++;
+            }
+            continue;
+        }
+        options.byteLimit?.reserve(frame.bytes.byteLength);
+        sink.send({ arrow: { dataBody: frame.bytes } });
+        frameCount++;
+    }
+    return { schemaSent, frameCount };
+}
+function rowsToArrowIPC(fields, rows) {
+    if (fields.length === 0) {
+        throw new Error("columns are required");
+    }
+    const arrow = loadApacheArrow();
+    const usedNames = new Map();
+    const arrays = {};
+    fields.forEach((field, index) => {
+        const name = uniqueColumnName(field.name || `column_${index + 1}`, usedNames);
+        const values = rows.map((row) => normalizeArrowValue(row[index], field));
+        const type = arrowTypeFromField(arrow, field);
+        arrays[name] = arrow.vectorFromArray ? arrow.vectorFromArray(values, type) : values;
+    });
+    const table = arrow.vectorFromArray
+        ? new arrow.Table(arrays)
+        : arrow.tableFromArrays(arrays);
+    return arrow.tableToIPC(table, "stream");
+}
+function splitArrowIpcStream(bytes) {
+    const arrow = loadApacheArrow();
+    const frames = [];
+    let offset = 0;
+    while (offset < bytes.byteLength) {
+        const frameStart = offset;
+        let metadataLength = readInt32LE(bytes, offset);
+        offset += 4;
+        if (metadataLength === 0) {
+            break;
+        }
+        if (metadataLength === -1) {
+            metadataLength = readInt32LE(bytes, offset);
+            offset += 4;
+            if (metadataLength === 0) {
+                break;
+            }
+        }
+        const metadataStart = offset;
+        const metadataEnd = metadataStart + metadataLength;
+        if (metadataEnd > bytes.byteLength) {
+            throw new Error("invalid Arrow IPC metadata length");
+        }
+        const message = arrow.Message.decode(bytes.slice(metadataStart, metadataEnd));
+        const bodyLength = Number(message.bodyLength ?? 0);
+        const frameEnd = metadataEnd + bodyLength;
+        if (frameEnd > bytes.byteLength) {
+            throw new Error("invalid Arrow IPC body length");
+        }
+        if (message.headerType === arrow.MessageHeader.Schema) {
+            frames.push({ kind: "schema", bytes: bytes.slice(frameStart, frameEnd) });
+        }
+        else if (message.headerType === arrow.MessageHeader.RecordBatch ||
+            message.headerType === arrow.MessageHeader.DictionaryBatch) {
+            frames.push({ kind: "body", bytes: bytes.slice(frameStart, frameEnd) });
+        }
+        offset = frameEnd;
+    }
+    return frames;
+}
+function arrowTypeFromField(arrow, field) {
+    switch (field.dataTypeID) {
+        case 16:
+            return new arrow.Bool();
+        case 21:
+            return new arrow.Int16();
+        case 23:
+            return new arrow.Int32();
+        case 20:
+            return new arrow.Int64();
+        case 700:
+        case 701:
+            return new arrow.Float64();
+        case 17:
+            return new arrow.Binary();
+        case 1082:
+            return new arrow.DateDay();
+        case 1114:
+        case 1184:
+            return new arrow.TimestampMillisecond("UTC");
+        default:
+            return arrowTypeFromDatabaseTypeName(arrow, field.databaseType ?? field.type);
+    }
+}
+function arrowTypeFromDatabaseTypeName(arrow, databaseType) {
+    switch (databaseType?.trim().toUpperCase()) {
+        case "BOOL":
+        case "BOOLEAN":
+            return new arrow.Bool();
+        case "INT2":
+        case "SMALLINT":
+            return new arrow.Int16();
+        case "INT4":
+        case "INTEGER":
+        case "SERIAL":
+            return new arrow.Int32();
+        case "INT8":
+        case "BIGINT":
+        case "BIGSERIAL":
+            return new arrow.Int64();
+        case "FLOAT4":
+        case "FLOAT8":
+        case "REAL":
+        case "DOUBLE":
+        case "DOUBLE PRECISION":
+        case "NUMERIC":
+        case "DECIMAL":
+            return new arrow.Float64();
+        case "BYTEA":
+        case "BINARY":
+        case "VARBINARY":
+            return new arrow.Binary();
+        case "DATE":
+            return new arrow.DateDay();
+        case "TIMESTAMP":
+        case "TIMESTAMPTZ":
+        case "TIMESTAMPZ":
+        case "DATETIME":
+            return new arrow.TimestampMillisecond("UTC");
+        default:
+            return new arrow.Utf8();
+    }
+}
+function normalizeArrowValue(value, field) {
+    if (value === null || value === undefined) {
+        return null;
+    }
+    switch (field.dataTypeID) {
+        case 20:
+            return typeof value === "bigint" ? value : BigInt(stringifyArrowValue(value));
+        case 21:
+        case 23:
+        case 700:
+        case 701:
+            return Number(value);
+        case 17:
+            return value instanceof Uint8Array ? value : Buffer.from(stringifyArrowValue(value));
+        case 1082:
+        case 1114:
+        case 1184:
+            return value instanceof Date ? value : new Date(stringifyArrowValue(value));
+        case 114:
+        case 3802:
+        case 1700:
+            return typeof value === "string" ? value : JSON.stringify(value);
+        default:
+            if (value instanceof Date || value instanceof Uint8Array) {
+                return value;
+            }
+            if (typeof value === "bigint") {
+                return value;
+            }
+            if (typeof value === "object") {
+                return JSON.stringify(value);
+            }
+            return value;
+    }
+}
+function stringifyArrowValue(value) {
+    switch (typeof value) {
+        case "string":
+            return value;
+        case "number":
+        case "boolean":
+        case "bigint":
+            return value.toString();
+        case "symbol":
+            return value.description ?? "";
+        case "undefined":
+            return "";
+        case "function":
+            return "";
+        case "object":
+            if (value === null) {
+                return "";
+            }
+            if (value instanceof Date) {
+                return value.toISOString();
+            }
+            if (value instanceof Uint8Array) {
+                return Buffer.from(value).toString();
+            }
+            return JSON.stringify(value);
+    }
+}
+function uniqueColumnName(name, usedNames) {
+    const previous = usedNames.get(name) ?? 0;
+    usedNames.set(name, previous + 1);
+    return previous === 0 ? name : `${name}_${previous + 1}`;
+}
+function readInt32LE(bytes, offset) {
+    if (offset + 4 > bytes.byteLength) {
+        throw new Error("invalid Arrow IPC frame");
+    }
+    return new DataView(bytes.buffer, bytes.byteOffset + offset, 4).getInt32(0, true);
+}
+function loadApacheArrow() {
+    if (cachedArrow) {
+        return cachedArrow;
+    }
+    try {
+        const require = createRequire(import.meta.url);
+        cachedArrow = require("apache-arrow");
+        return cachedArrow;
+    }
+    catch (error) {
+        throw new DataSourceExecutionError({
+            code: DataSourceErrorCode.Unavailable,
+            message: `apache-arrow is required for datasource Arrow conversion: ${errorMessage(error)}`,
+            retryable: false,
+            cause: error,
+        });
+    }
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+//# sourceMappingURL=arrow.js.map

package/dist/datasource/arrow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow.js","sourceRoot":"","sources":["../../src/datasource/arrow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAqD3E,IAAI,WAA0C,CAAC;AAE/C,MAAM,UAAU,oBAAoB,CAAC,OAAsB,EAAE,IAAgB;IAC3E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,cAAc,CACnB,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAC7D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAsB,EAAE,IAAgB;IACrE,OAAO;QACL,KAAK,EAAE;YACL,QAAQ,EAAE,oBAAoB,CAAC,OAAO,EAAE,IAAI,CAAC;SAC9C;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,QAAgB,EAAE,aAAa,GAAG,KAAK,EAAE,WAAW,GAAG,CAAC;IAClF,OAAO;QACL,MAAM,EAAE;YACN,QAAQ;YACR,aAAa;YACb,WAAW;SACZ;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,IAAoB,EACpB,MAA6B,EAC7B,IAAqC,EACrC,UAAkC,EAAE;IAEpC,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,KAAK,IAAI,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5B,IAAI,OAAO,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;gBACjC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACnD,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAClD,UAAU,GAAG,IAAI,CAAC;gBAClB,UAAU,EAAE,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QACD,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACnD,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChD,UAAU,EAAE,CAAC;IACf,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;AACpC,CAAC;AAED,SAAS,cAAc,CACrB,MAA6B,EAC7B,IAAqC;IAErC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,MAAM,MAAM,GAA4B,EAAE,CAAC;IAE3C,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9B,MAAM,IAAI,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,IAAI,UAAU,KAAK,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QACzE,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC9C,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,KAAK,CAAC,eAAe;QACjC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;QACzB,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,MAAmC,CAAC,CAAC;IAC/D,OAAO,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;AAC3C,CAAC;AASD,SAAS,mBAAmB,CAAC,KAAiB;IAC5C,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;IAChC,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,MAAM,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,UAAU,GAAG,MAAM,CAAC;QAC1B,IAAI,cAAc,GAAG,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAChD,MAAM,IAAI,CAAC,CAAC;QACZ,IAAI,cAAc,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM;QACR,CAAC;QACD,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;YAC1B,cAAc,GAAG,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC5C,MAAM,IAAI,CAAC,CAAC;YACZ,IAAI,cAAc,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,CAAC;QAC7B,MAAM,WAAW,GAAG,aAAa,GAAG,cAAc,CAAC;QACnD,IAAI,WAAW,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC,CAAC;QAC9E,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,WAAW,GAAG,UAAU,CAAC;QAC1C,IAAI,QAAQ,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,KAAK,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC5E,CAAC;aAAM,IACL,OAAO,CAAC,UAAU,KAAK,KAAK,CAAC,aAAa,CAAC,WAAW;YACtD,OAAO,CAAC,UAAU,KAAK,KAAK,CAAC,aAAa,CAAC,eAAe,EAC1D,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,GAAG,QAAQ,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB,EAAE,KAAiB;IACrE,QAAQ,KAAK,CAAC,UAAU,EAAE,CAAC;QACzB,KAAK,EAAE;YACL,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC1B,KAAK,EAAE;YACL,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,KAAK,EAAE;YACL,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,KAAK,EAAE;YACL,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,KAAK,GAAG,CAAC;QACT,KAAK,GAAG;YACN,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC7B,KAAK,EAAE;YACL,OAAO,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC5B,KAAK,IAAI;YACP,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC7B,KAAK,IAAI,CAAC;QACV,KAAK,IAAI;YACP,OAAO,IAAI,KAAK,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAC/C;YACE,OAAO,6BAA6B,CAAC,KAAK,EAAE,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IAClF,CAAC;AACH,CAAC;AAED,SAAS,6BAA6B,CACpC,KAAwB,EACxB,YAAgC;IAEhC,QAAQ,YAAY,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QAC3C,KAAK,MAAM,CAAC;QACZ,KAAK,SAAS;YACZ,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC;QACZ,KAAK,UAAU;YACb,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC;QACZ,KAAK,SAAS,CAAC;QACf,KAAK,QAAQ;YACX,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ,CAAC;QACd,KAAK,WAAW;YACd,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,KAAK,QAAQ,CAAC;QACd,KAAK,QAAQ,CAAC;QACd,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ,CAAC;QACd,KAAK,kBAAkB,CAAC;QACxB,KAAK,SAAS,CAAC;QACf,KAAK,SAAS;YACZ,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC7B,KAAK,OAAO,CAAC;QACb,KAAK,QAAQ,CAAC;QACd,KAAK,WAAW;YACd,OAAO,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC5B,KAAK,MAAM;YACT,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC7B,KAAK,WAAW,CAAC;QACjB,KAAK,aAAa,CAAC;QACnB,KAAK,YAAY,CAAC;QAClB,KAAK,UAAU;YACb,OAAO,IAAI,KAAK,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAC/C;YACE,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc,EAAE,KAAiB;IAC5D,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,QAAQ,KAAK,CAAC,UAAU,EAAE,CAAC;QACzB,KAAK,EAAE;YACL,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;QAChF,KAAK,EAAE,CAAC;QACR,KAAK,EAAE,CAAC;QACR,KAAK,GAAG,CAAC;QACT,KAAK,GAAG;YACN,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,KAAK,EAAE;YACL,OAAO,KAAK,YAAY,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;QACvF,KAAK,IAAI,CAAC;QACV,KAAK,IAAI,CAAC;QACV,KAAK,IAAI;YACP,OAAO,KAAK,YAAY,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9E,KAAK,GAAG,CAAC;QACT,KAAK,IAAI,CAAC;QACV,KAAK,IAAI;YACP,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACnE;YACE,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;gBACzD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;YACD,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc;IACzC,QAAQ,OAAO,KAAK,EAAE,CAAC;QACrB,KAAK,QAAQ;YACX,OAAO,KAAK,CAAC;QACf,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS,CAAC;QACf,KAAK,QAAQ;YACX,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC1B,KAAK,QAAQ;YACX,OAAO,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC;QACjC,KAAK,WAAW;YACd,OAAO,EAAE,CAAC;QACZ,KAAK,UAAU;YACb,OAAO,EAAE,CAAC;QACZ,KAAK,QAAQ;YACX,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;gBAC1B,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;YAC7B,CAAC;YACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;gBAChC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;YACvC,CAAC;YACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAA8B;IACpE,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClC,OAAO,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;AAC3D,CAAC;AAED,SAAS,WAAW,CAAC,KAAiB,EAAE,MAAc;IACpD,IAAI,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AACpF,CAAC;AAED,SAAS,eAAe;IACtB,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,WAAW,GAAG,OAAO,CAAC,cAAc,CAAsB,CAAC;QAC3D,OAAO,WAAW,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,wBAAwB,CAAC;YACjC,IAAI,EAAE,mBAAmB,CAAC,WAAW;YACrC,OAAO,EAAE,6DAA6D,YAAY,CAAC,KAAK,CAAC,EAAE;YAC3F,SAAS,EAAE,KAAK;YAChB,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}

package/dist/datasource/bigquery.d.ts

@@ -0,0 +1,97 @@
+import type { DataSourceTableConfig } from "./policy.js";
+import { type DataSourceQueryExecutor, type ExecuteQueryRequest, type QueryChunkSink } from "./types.js";
+export interface BigQueryDataSourceConfig {
+    projectId: string;
+    credentialsJson?: string;
+    credentialsJsonEnv?: string;
+    credentialsEnvVars?: readonly string[];
+    keyFilename?: string;
+    clientOptions?: Record<string, unknown>;
+    bigQueryClient?: BigQueryClientLike;
+    readClient?: BigQueryReadClientLike;
+    sources: readonly BigQueryDataSourceSourceConfig[];
+}
+export interface BigQueryDataSourceSourceConfig {
+    sourceId: string;
+    projectId?: string;
+    datasetId: string;
+    location?: string;
+    tables?: readonly DataSourceTableConfig[];
+    maxStreamCount?: number;
+    maximumBytesBilled?: string | number | bigint;
+}
+export declare function createBigQueryDataSourceExecutor(config: BigQueryDataSourceConfig): BigQueryDataSourceExecutor;
+export declare class BigQueryDataSourceExecutor implements DataSourceQueryExecutor {
+    private readonly config;
+    private readonly sources;
+    private bigQueryClient?;
+    private readClient?;
+    constructor(config: BigQueryDataSourceConfig);
+    executeQuery(request: ExecuteQueryRequest, sink: QueryChunkSink): Promise<void>;
+    close(): Promise<void>;
+    private ensureClients;
+}
+export interface BigQueryClientLike {
+    createQueryJob(options: BigQueryQueryJobOptions): Promise<[BigQueryJobLike]>;
+}
+export interface BigQueryJobLike {
+    getQueryResults(options: Record<string, unknown>): Promise<unknown>;
+    getMetadata(): Promise<[BigQueryJobMetadata]>;
+}
+export interface BigQueryReadClientLike {
+    createReadSession(request: BigQueryCreateReadSessionRequest): Promise<[BigQueryReadSession]>;
+    readRows(request: {
+        readStream: string;
+    }): AsyncIterable<BigQueryReadRowsResponse>;
+    close?(): Promise<void>;
+}
+export interface BigQueryQueryJobOptions {
+    query: string;
+    useLegacySql: boolean;
+    defaultDataset: {
+        projectId: string;
+        datasetId: string;
+    };
+    location?: string;
+    jobTimeoutMs?: number;
+    maximumBytesBilled?: string;
+    labels?: Record<string, string>;
+}
+export interface BigQueryJobMetadata {
+    configuration?: {
+        query?: {
+            destinationTable?: Partial<BigQueryTableRef>;
+        };
+    };
+}
+export interface BigQueryTableRef {
+    projectId: string;
+    datasetId: string;
+    tableId: string;
+}
+export interface BigQueryCreateReadSessionRequest {
+    parent: string;
+    readSession: {
+        table: string;
+        dataFormat: "ARROW";
+    };
+    maxStreamCount: number;
+}
+export interface BigQueryReadSession {
+    arrowSchema?: {
+        serializedSchema?: unknown;
+    } | null;
+    streams?: {
+        name?: string | null;
+    }[] | null;
+}
+export interface BigQueryReadRowsResponse {
+    arrowSchema?: {
+        serializedSchema?: unknown;
+    } | null;
+    arrowRecordBatch?: {
+        serializedRecordBatch?: unknown;
+    } | null;
+    rowCount?: unknown;
+}
+//# sourceMappingURL=bigquery.d.ts.map

package/dist/datasource/bigquery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bigquery.d.ts","sourceRoot":"","sources":["../../src/datasource/bigquery.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAGL,KAAK,uBAAuB,EAC5B,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACpB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,OAAO,EAAE,SAAS,8BAA8B,EAAE,CAAC;CACpD;AAED,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,SAAS,qBAAqB,EAAE,CAAC;IAC1C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;CAC/C;AAQD,wBAAgB,gCAAgC,CAC9C,MAAM,EAAE,wBAAwB,GAC/B,0BAA0B,CAE5B;AAED,qBAAa,0BAA2B,YAAW,uBAAuB;IAK5D,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqD;IAC7E,OAAO,CAAC,cAAc,CAAC,CAAqB;IAC5C,OAAO,CAAC,UAAU,CAAC,CAAyB;gBAEf,MAAM,EAAE,wBAAwB;IAiBvD,YAAY,CAAC,OAAO,EAAE,mBAAmB,EAAE,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IA+B/E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAMd,aAAa;CAY5B;AA0RD,MAAM,WAAW,kBAAkB;IACjC,cAAc,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;CAC9E;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACpE,WAAW,IAAI,OAAO,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,sBAAsB;IACrC,iBAAiB,CAAC,OAAO,EAAE,gCAAgC,GAAG,OAAO,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAC7F,QAAQ,CAAC,OAAO,EAAE;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,aAAa,CAAC,wBAAwB,CAAC,CAAC;IACnF,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,mBAAmB;IAClC,aAAa,CAAC,EAAE;QACd,KAAK,CAAC,EAAE;YACN,gBAAgB,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;SAC9C,CAAC;KACH,CAAC;CACH;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gCAAgC;IAC/C,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC;IACF,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,WAAW,CAAC,EAAE;QACZ,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC5B,GAAG,IAAI,CAAC;IACT,OAAO,CAAC,EACJ;QACE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KACtB,EAAE,GACH,IAAI,CAAC;CACV;AAED,MAAM,WAAW,wBAAwB;IACvC,WAAW,CAAC,EAAE;QACZ,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC5B,GAAG,IAAI,CAAC;IACT,gBAAgB,CAAC,EAAE;QACjB,qBAAqB,CAAC,EAAE,OAAO,CAAC;KACjC,GAAG,IAAI,CAAC;IACT,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB"}

package/dist/datasource/bigquery.js

@@ -0,0 +1,297 @@
+import { DataSourceByteLimitTracker, queryWithRowLimit, validateReadOnlyQuery } from "./policy.js";
+import { DataSourceErrorCode, DataSourceExecutionError, } from "./types.js";
+export function createBigQueryDataSourceExecutor(config) {
+    return new BigQueryDataSourceExecutor(config);
+}
+export class BigQueryDataSourceExecutor {
+    config;
+    sources = new Map();
+    bigQueryClient;
+    readClient;
+    constructor(config) {
+        this.config = config;
+        const projectId = config.projectId.trim();
+        if (!projectId) {
+            throw new Error("BigQuery project_id is required");
+        }
+        if (config.sources.length === 0) {
+            throw new Error("at least one BigQuery datasource source is required");
+        }
+        for (const source of config.sources) {
+            const normalized = normalizeSource(source, projectId);
+            if (this.sources.has(normalized.sourceId)) {
+                throw new Error(`duplicate datasource source_id: ${normalized.sourceId}`);
+            }
+            this.sources.set(normalized.sourceId, normalized);
+        }
+    }
+    async executeQuery(request, sink) {
+        const req = normalizeRequest(request);
+        const source = this.sources.get(req.sourceId);
+        if (!source) {
+            throw new DataSourceExecutionError({
+                code: DataSourceErrorCode.SourceNotFound,
+                message: `unknown datasource source_id: ${req.sourceId}`,
+            });
+        }
+        validateReadOnlyQuery(req.query, [], source.tables ?? []);
+        const startedAt = Date.now();
+        const clients = await this.ensureClients();
+        const byteLimit = new DataSourceByteLimitTracker(req.byteLimit);
+        try {
+            const destination = await runQueryJob(clients.bigQueryClient, source, req);
+            const rowCount = await streamBigQueryResult(clients.readClient, source, destination, sink, byteLimit);
+            sink.send({
+                result: { rowCount, limitExceeded: false, executionMs: Date.now() - startedAt },
+            });
+        }
+        catch (error) {
+            throw toBigQueryDataSourceError(error);
+        }
+    }
+    async close() {
+        if (!this.config.readClient && this.readClient?.close) {
+            await this.readClient.close();
+        }
+    }
+    async ensureClients() {
+        if (this.bigQueryClient && this.readClient) {
+            return { bigQueryClient: this.bigQueryClient, readClient: this.readClient };
+        }
+        const options = clientOptions(this.config);
+        this.bigQueryClient = this.config.bigQueryClient ?? (await createBigQueryClient(options));
+        this.readClient = this.config.readClient ?? (await createBigQueryReadClient(options));
+        return { bigQueryClient: this.bigQueryClient, readClient: this.readClient };
+    }
+}
+async function runQueryJob(client, source, req) {
+    const options = {
+        query: queryWithRowLimit(req.query, req.rowLimit),
+        useLegacySql: false,
+        defaultDataset: {
+            projectId: source.projectId,
+            datasetId: source.datasetId,
+        },
+        labels: { component: "datasource" },
+    };
+    if (source.location) {
+        options.location = source.location;
+    }
+    if (req.timeoutMs && req.timeoutMs > 0) {
+        options.jobTimeoutMs = Math.trunc(req.timeoutMs);
+    }
+    if (source.maximumBytesBilled !== undefined) {
+        options.maximumBytesBilled = String(source.maximumBytesBilled);
+    }
+    const [job] = await client.createQueryJob(options);
+    const queryResultsOptions = {
+        autoPaginate: false,
+        maxResults: 0,
+    };
+    if (source.location) {
+        queryResultsOptions["location"] = source.location;
+    }
+    await job.getQueryResults(queryResultsOptions);
+    const [metadata] = await job.getMetadata();
+    const table = metadata.configuration?.query?.destinationTable;
+    if (!table?.projectId || !table.datasetId || !table.tableId) {
+        throw new Error("BigQuery query job destination table is missing");
+    }
+    return {
+        projectId: table.projectId,
+        datasetId: table.datasetId,
+        tableId: table.tableId,
+    };
+}
+async function streamBigQueryResult(client, source, table, sink, byteLimit) {
+    const [session] = await client.createReadSession({
+        parent: `projects/${source.projectId}`,
+        readSession: {
+            table: `projects/${table.projectId}/datasets/${table.datasetId}/tables/${table.tableId}`,
+            dataFormat: "ARROW",
+        },
+        maxStreamCount: source.maxStreamCount,
+    });
+    let schemaSent = false;
+    const sessionSchema = bytesFromUnknown(session.arrowSchema?.serializedSchema);
+    if (sessionSchema) {
+        byteLimit.reserve(sessionSchema.byteLength);
+        sink.send({ arrow: { dataHeader: sessionSchema } });
+        schemaSent = true;
+    }
+    let rowCount = 0;
+    for (const stream of session.streams ?? []) {
+        if (!stream.name) {
+            continue;
+        }
+        const rows = client.readRows({ readStream: stream.name });
+        for await (const response of rows) {
+            const responseSchema = bytesFromUnknown(response.arrowSchema?.serializedSchema);
+            if (responseSchema && !schemaSent) {
+                byteLimit.reserve(responseSchema.byteLength);
+                sink.send({ arrow: { dataHeader: responseSchema } });
+                schemaSent = true;
+            }
+            const batch = bytesFromUnknown(response.arrowRecordBatch?.serializedRecordBatch);
+            if (batch) {
+                byteLimit.reserve(batch.byteLength);
+                sink.send({ arrow: { dataBody: batch } });
+            }
+            rowCount += numberFromLong(response.rowCount);
+        }
+    }
+    return rowCount;
+}
+function normalizeSource(source, defaultProjectId) {
+    const sourceId = source.sourceId.trim();
+    if (!sourceId) {
+        throw new Error("source_id is required");
+    }
+    if (!source.datasetId.trim()) {
+        throw new Error(`BigQuery dataset_id is required for source_id ${sourceId}`);
+    }
+    return {
+        ...source,
+        sourceId,
+        projectId: normalizedProjectId(source.projectId, defaultProjectId),
+        maxStreamCount: source.maxStreamCount && source.maxStreamCount > 0 ? Math.trunc(source.maxStreamCount) : 1,
+    };
+}
+function normalizedProjectId(projectId, defaultProjectId) {
+    const trimmed = projectId?.trim();
+    return trimmed && trimmed.length > 0 ? trimmed : defaultProjectId;
+}
+function normalizeRequest(request) {
+    return {
+        sourceId: request.sourceId?.trim() ?? "",
+        query: request.query?.trim() ?? "",
+        rowLimit: request.rowLimit,
+        byteLimit: request.byteLimit,
+        timeoutMs: request.timeoutMs,
+    };
+}
+function clientOptions(config) {
+    const options = {
+        ...config.clientOptions,
+        projectId: config.projectId,
+    };
+    if (config.keyFilename) {
+        options["keyFilename"] = config.keyFilename;
+    }
+    const credentialsJson = credentialJson(config);
+    if (credentialsJson) {
+        options["credentials"] = JSON.parse(credentialsJson);
+    }
+    return options;
+}
+function credentialJson(config) {
+    if (config.credentialsJson?.trim()) {
+        return config.credentialsJson;
+    }
+    if (config.credentialsJsonEnv?.trim()) {
+        const value = process.env[config.credentialsJsonEnv];
+        if (value?.trim()) {
+            return value;
+        }
+    }
+    for (const envName of config.credentialsEnvVars ?? []) {
+        const value = process.env[envName];
+        if (value?.trim()) {
+            return value;
+        }
+    }
+    return undefined;
+}
+async function createBigQueryClient(options) {
+    try {
+        const mod = (await importOptionalModule("@google-cloud/bigquery"));
+        return new mod.BigQuery(options);
+    }
+    catch (error) {
+        throw new DataSourceExecutionError({
+            code: DataSourceErrorCode.Unavailable,
+            message: `@google-cloud/bigquery is required for BigQuery datasource execution: ${errorMessage(error)}`,
+            cause: error,
+        });
+    }
+}
+async function createBigQueryReadClient(options) {
+    try {
+        const mod = (await importOptionalModule("@google-cloud/bigquery-storage"));
+        return new mod.BigQueryReadClient(options);
+    }
+    catch (error) {
+        throw new DataSourceExecutionError({
+            code: DataSourceErrorCode.Unavailable,
+            message: `@google-cloud/bigquery-storage is required for BigQuery Storage streaming: ${errorMessage(error)}`,
+            cause: error,
+        });
+    }
+}
+function bytesFromUnknown(value) {
+    if (value === null || value === undefined) {
+        return undefined;
+    }
+    if (value instanceof Uint8Array) {
+        return value;
+    }
+    if (typeof value === "string") {
+        return Buffer.from(value, "base64");
+    }
+    return undefined;
+}
+function numberFromLong(value) {
+    if (typeof value === "number") {
+        return value;
+    }
+    if (typeof value === "bigint") {
+        return Number(value);
+    }
+    if (typeof value === "string") {
+        const parsed = Number(value);
+        return Number.isFinite(parsed) ? parsed : 0;
+    }
+    if (value && typeof value === "object" && "toString" in value) {
+        const objectValue = value;
+        if (objectValue.toString === Object.prototype.toString) {
+            return 0;
+        }
+        const parsed = Number(objectValue.toString());
+        return Number.isFinite(parsed) ? parsed : 0;
+    }
+    return 0;
+}
+function toBigQueryDataSourceError(error) {
+    if (error instanceof DataSourceExecutionError) {
+        return error;
+    }
+    if (error instanceof Error) {
+        const upstream = {
+            engine: "bigquery",
+            message: error.message,
+        };
+        const code = errorCode(error);
+        if (code) {
+            upstream.code = code;
+        }
+        return new DataSourceExecutionError({
+            code: DataSourceErrorCode.ExternalError,
+            message: "BigQuery datasource query failed",
+            upstream,
+            cause: error,
+        });
+    }
+    return error;
+}
+function errorCode(error) {
+    const code = error.code;
+    return typeof code === "string" ? code : undefined;
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+async function importOptionalModule(specifier) {
+    const mod = await import(specifier);
+    return mod;
+}
+//# sourceMappingURL=bigquery.js.map