@terascope/data-mate 2.24.0 → 2.25.1

package/dist/src/teal-frame/output.js

@@ -0,0 +1,63 @@
+import { DataType } from '@terascope/data-types';
+import { quoteIdentifier, quoteLiteral } from '@terascope/sql-builder';
+import { buildJsonExpression, buildJsonListElements, buildJsonRowFilter, resolveRules } from './export-json.js';
+import { int64Column } from './parquet-columns.js';
+/**
+ * Implementations of `TealFrame`'s output methods: `writeParquet`, the JSON-lines query behind
+ * `ndjson`, `writeNDJSON` and `rows(options)`, and `listValues`. Their contracts are documented
+ * on `TealFrame`.
+*/
+/** Writes the frame to a zstd Parquet file, narrowing 128-bit columns to their declared type. */
+export async function writeParquet(ctx, from, config, path) {
+    const columns = await ctx.columnTypes(`SELECT * FROM ${from}`);
+    const declared = new DataType(config).toDuckDB();
+    const list = columns
+        .map(({ name, type }) => int64Column(name, type, declared[name]))
+        .join(', ');
+    // GEOPARQUET_VERSION 'BOTH' writes Parquet's native GEOMETRY logical type, which, unlike
+    // the GeoParquet 1.0 default, also works for nested values. A GeoJSON array or an object's
+    // GeoJSON member then reads back as GEOMETRY instead of BLOB (BLOB has no cast to
+    // GEOMETRY). The 1.0 metadata is still written for readers that only support it.
+    await ctx.run(`COPY (SELECT ${list} FROM ${from}) TO ${quoteLiteral(path)}`
+        + ' (FORMAT parquet, COMPRESSION zstd, GEOPARQUET_VERSION \'BOTH\')');
+}
+/**
+ * SQL that renders one JSON line per row, as `DataFrame.toJSON` does with the same options.
+ * Rows that `skipEmptyObjects` drops are filtered out. See `export-json.ts`.
+*/
+export function jsonLinesSQL(from, config, options) {
+    const expression = buildJsonExpression(config, options);
+    const filter = buildJsonRowFilter(config, options);
+    return `SELECT ${expression} AS json FROM ${from}` + (filter ? ` WHERE ${filter}` : '');
+}
+/** Yields one row's LIST column element by element, like `ListVector.getRowIterator`. */
+export async function* listValues(ctx, from, config, column, options) {
+    const rules = resolveRules(options);
+    if (rules.skipDuplicateObjects) {
+        throw new Error('ListVector->getIterator is incompatible with skipDuplicateObjects');
+    }
+    const { sql, isObject } = buildJsonListElements(config, column, options);
+    const nilValue = rules.useNullForUndefined ? null : undefined;
+    const skipNil = rules.skipNilListValues || (isObject && rules.skipNilObjectValues);
+    if (rules.skipEmptyArrays) {
+        const values = await ctx.scalar(`SELECT count(${quoteIdentifier(column)}) FROM ${from}`);
+        if (Number(values) === 0)
+            return;
+    }
+    const [row] = await ctx.rows(`SELECT ${sql} FROM ${from} LIMIT 1`);
+    if (row == null)
+        return;
+    const elements = row[0];
+    if (elements == null) {
+        if (!skipNil)
+            yield nilValue;
+        return;
+    }
+    for (const element of elements) {
+        if (element != null)
+            yield JSON.parse(element);
+        else if (!skipNil)
+            yield nilValue;
+    }
+}
+//# sourceMappingURL=output.js.map

package/dist/src/teal-frame/output.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"output.js","sourceRoot":"","sources":["../../../src/teal-frame/output.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,EACH,mBAAmB,EAAE,qBAAqB,EAAE,kBAAkB,EAC9D,YAAY,EACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAGnD;;;;EAIE;AAEF,iGAAiG;AACjG,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,GAAgB,EAChB,IAAY,EACZ,MAAmB,EACnB,IAAY;IAEZ,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,WAAW,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,MAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC;IACnE,MAAM,IAAI,GAAG,OAAO;SACf,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;SAChE,IAAI,CAAC,IAAI,CAAC,CAAC;IAEhB,yFAAyF;IACzF,2FAA2F;IAC3F,kFAAkF;IAClF,iFAAiF;IACjF,MAAM,GAAG,CAAC,GAAG,CACT,gBAAgB,IAAI,SAAS,IAAI,QAAQ,YAAY,CAAC,IAAI,CAAC,EAAE;UAC3D,kEAAkE,CACvE,CAAC;AACN,CAAC;AAED;;;EAGE;AACF,MAAM,UAAU,YAAY,CACxB,IAAY,EACZ,MAAmB,EACnB,OAA0B;IAE1B,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnD,OAAO,UAAU,UAAU,iBAAiB,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAC5F,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,UAAU,CAC7B,GAAgB,EAChB,IAAY,EACZ,MAAmB,EACnB,MAAc,EACd,OAA0B;IAE1B,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,KAAK,CAAC,oBAAoB,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9D,MAAM,OAAO,GAAG,KAAK,CAAC,iBAAiB,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAEnF,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,gBAAgB,eAAe,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QACzF,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YAAE,OAAO;IACrC,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,SAAS,IAAI,UAAU,CAAC,CAAC;IACnE,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO;IAExB,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,CAA6B,CAAC;IACpD,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACnB,IAAI,CAAC,OAAO;YAAE,MAAM,QAAQ,CAAC;QAC7B,OAAO;IACX,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,OAAO,IAAI,IAAI;YAAE,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;aAC1C,IAAI,CAAC,OAAO;YAAE,MAAM,QAAQ,CAAC;IACtC,CAAC;AACL,CAAC"}

package/dist/src/teal-frame/parquet-columns.d.ts

@@ -0,0 +1,15 @@
+import { DuckDBType } from '@duckdb/node-api';
+/**
+ * Returns one column of the Parquet projection, with any 128-bit integer narrowed.
+ *
+ * **Parquet has no 128-bit integer, and DuckDB silently writes one as a DOUBLE**, which changes
+ * every value past 2^53. A stored Long is already a BIGINT; this handles computed values
+ * (`n::HUGEINT + 1`, a UDF result, `ipToInt` on IPv6).
+ *
+ * The column is cast to the type declared in the config. Editing DuckDB's rendered type string
+ * instead would also rewrite a struct member named `HUGEINT`. The cast is exact for values that
+ * fit and fails the write with DuckDB's out-of-range error for values that do not. A wide column
+ * the config does not declare is rejected, because there is no correct type to choose.
+*/
+export declare function int64Column(name: string, type: DuckDBType, declared: string | undefined): string;
+//# sourceMappingURL=parquet-columns.d.ts.map

package/dist/src/teal-frame/parquet-columns.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-columns.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/parquet-columns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAgB,MAAM,kBAAkB,CAAC;AA8B5D;;;;;;;;;;;EAWE;AACF,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAUhG"}

package/dist/src/teal-frame/parquet-columns.js

@@ -0,0 +1,45 @@
+import { DuckDBTypeId } from '@duckdb/node-api';
+import { quoteIdentifier } from '@terascope/sql-builder';
+/**
+ * Builds the column list `writeParquet` selects, so every column is written as a type Parquet
+ * can hold exactly. See `int64Column`.
+*/
+/** DuckDB's 128-bit integers, which Parquet has no type for. */
+const WIDE_INTEGER_IDS = new Set([
+    DuckDBTypeId.HUGEINT, DuckDBTypeId.UHUGEINT,
+]);
+/** True when a 128-bit integer appears anywhere in a type, including nested members. */
+function hasWideInteger(type) {
+    if (WIDE_INTEGER_IDS.has(type.typeId))
+        return true;
+    const nested = type;
+    return [
+        ...(nested.entryTypes ?? []),
+        ...(nested.memberTypes ?? []),
+        ...(nested.valueType ? [nested.valueType] : []),
+        ...(nested.keyType ? [nested.keyType] : []),
+    ].some(hasWideInteger);
+}
+/**
+ * Returns one column of the Parquet projection, with any 128-bit integer narrowed.
+ *
+ * **Parquet has no 128-bit integer, and DuckDB silently writes one as a DOUBLE**, which changes
+ * every value past 2^53. A stored Long is already a BIGINT; this handles computed values
+ * (`n::HUGEINT + 1`, a UDF result, `ipToInt` on IPv6).
+ *
+ * The column is cast to the type declared in the config. Editing DuckDB's rendered type string
+ * instead would also rewrite a struct member named `HUGEINT`. The cast is exact for values that
+ * fit and fails the write with DuckDB's out-of-range error for values that do not. A wide column
+ * the config does not declare is rejected, because there is no correct type to choose.
+*/
+export function int64Column(name, type, declared) {
+    const column = quoteIdentifier(name);
+    if (!hasWideInteger(type))
+        return column;
+    if (declared == null) {
+        throw new TypeError(`Cannot write column "${name}" to Parquet: it holds a 128-bit integer (${type}), which`
+            + ' Parquet cannot carry, and the frame config does not declare it');
+    }
+    return `CAST(${column} AS ${declared}) AS ${column}`;
+}
+//# sourceMappingURL=parquet-columns.js.map

package/dist/src/teal-frame/parquet-columns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-columns.js","sourceRoot":"","sources":["../../../src/teal-frame/parquet-columns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD;;;EAGE;AAEF,gEAAgE;AAChE,MAAM,gBAAgB,GAA8B,IAAI,GAAG,CAAC;IACxD,YAAY,CAAC,OAAO,EAAE,YAAY,CAAC,QAAQ;CAC9C,CAAC,CAAC;AAEH,wFAAwF;AACxF,SAAS,cAAc,CAAC,IAAgB;IACpC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACnD,MAAM,MAAM,GAAG,IAKd,CAAC;IACF,OAAO;QACH,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;QAC5B,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC;QAC7B,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;;;;;;;EAWE;AACF,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,IAAgB,EAAE,QAA4B;IACpF,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IACzC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,SAAS,CACf,wBAAwB,IAAI,6CAA6C,IAAI,UAAU;cACrF,iEAAiE,CACtE,CAAC;IACN,CAAC;IACD,OAAO,QAAQ,MAAM,OAAO,QAAQ,QAAQ,MAAM,EAAE,CAAC;AACzD,CAAC"}

package/dist/src/teal-frame/plain-values.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Converts values read from DuckDB into the plain JS values `rows()` returns.
+ *
+ * `rows()` results are sent straight to callers, so no DuckDB value class may leak through:
+ *
+ * - LIST arrives as a `DuckDBListValue`, which would serialize as `{"items":[...]}`. It becomes
+ *   an array.
+ * - TIMESTAMP arrives as a `DuckDBTimestampValue`. It is rendered with `toISO8601`, as
+ *   `DateVector.toJSONCompatibleValue` does.
+ * - GEOMETRY arrives as WKB bytes. It is decoded back to the GeoShape a `GeoJSON` field holds.
+ * - BIGINT and HUGEINT arrive as a JS `bigint`, which `JSON.stringify` cannot serialize.
+ *   `bigIntToPlain` returns a number when the value fits exactly and a decimal string otherwise.
+ *
+ * `core-utils`' `bigIntToJSON` is not used, because it is wrong for values read from DuckDB
+ * (#4555). It subtracts 1 from every positive value above the safe limit, to cancel a `+1` that
+ * core-utils' parser adds, so a value that was never parsed comes back one too low. Its bound
+ * is also a signed comparison, so large negative values lose precision instead of becoming
+ * strings.
+ *
+ * The rule here is the same as in `export-json.ts` (`abs(value) > MAX_SAFE` becomes a decimal
+ * string, anything smaller a number), so `rows()` and `ndjson()` always agree.
+*/
+export declare const MAX_SAFE_BIGINT = 9007199254740991n;
+export declare function toPlainValue(value: unknown): unknown;
+/**
+ * Parses a JSON column's text into the value it encodes. `Any` and untyped `Object` fields are
+ * stored as DuckDB JSON columns, which DuckDB returns as text.
+*/
+export declare function parseJSONText(value: unknown): unknown;
+//# sourceMappingURL=plain-values.d.ts.map

package/dist/src/teal-frame/plain-values.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"plain-values.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/plain-values.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;;;;;;;EAqBE;AACF,eACA,MAAM,eAAe,oBAAoB,CAAC;AAc1C,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CA0BpD;AAED;;;EAGE;AACF,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAErD"}

package/dist/src/teal-frame/plain-values.js

@@ -0,0 +1,69 @@
+import { DuckDBTimestampValue, DuckDBGeometryValue } from '@duckdb/node-api';
+import { toISO8601 } from '@terascope/core-utils';
+import { wkbToGeoShape } from './wkb.js';
+/**
+ * Converts values read from DuckDB into the plain JS values `rows()` returns.
+ *
+ * `rows()` results are sent straight to callers, so no DuckDB value class may leak through:
+ *
+ * - LIST arrives as a `DuckDBListValue`, which would serialize as `{"items":[...]}`. It becomes
+ *   an array.
+ * - TIMESTAMP arrives as a `DuckDBTimestampValue`. It is rendered with `toISO8601`, as
+ *   `DateVector.toJSONCompatibleValue` does.
+ * - GEOMETRY arrives as WKB bytes. It is decoded back to the GeoShape a `GeoJSON` field holds.
+ * - BIGINT and HUGEINT arrive as a JS `bigint`, which `JSON.stringify` cannot serialize.
+ *   `bigIntToPlain` returns a number when the value fits exactly and a decimal string otherwise.
+ *
+ * `core-utils`' `bigIntToJSON` is not used, because it is wrong for values read from DuckDB
+ * (#4555). It subtracts 1 from every positive value above the safe limit, to cancel a `+1` that
+ * core-utils' parser adds, so a value that was never parsed comes back one too low. Its bound
+ * is also a signed comparison, so large negative values lose precision instead of becoming
+ * strings.
+ *
+ * The rule here is the same as in `export-json.ts` (`abs(value) > MAX_SAFE` becomes a decimal
+ * string, anything smaller a number), so `rows()` and `ndjson()` always agree.
+*/
+export /** The largest whole number JSON can carry without `JSON.parse` rounding it. */ const MAX_SAFE_BIGINT = 9007199254740991n;
+/**
+ * Converts a bigint to a number when it fits exactly, or to an exact decimal string when it
+ * does not.
+ *
+ * The bound applies to the magnitude, matching `abs(x) > 9007199254740991` in `export-json.ts`.
+ * No arithmetic is applied to the value.
+*/
+function bigIntToPlain(value) {
+    if (value > MAX_SAFE_BIGINT || value < -MAX_SAFE_BIGINT)
+        return value.toString(10);
+    return Number(value);
+}
+export function toPlainValue(value) {
+    if (value == null)
+        return value;
+    if (typeof value === 'bigint')
+        return bigIntToPlain(value);
+    if (typeof value !== 'object')
+        return value;
+    if (value instanceof DuckDBTimestampValue) {
+        return toISO8601(Number(value.micros / 1000n));
+    }
+    // a GeoJSON column is a GEOMETRY, which the binding hands over as WKB
+    if (value instanceof DuckDBGeometryValue)
+        return wkbToGeoShape(value.bytes);
+    const items = value.items;
+    if (Array.isArray(items))
+        return items.map(toPlainValue);
+    const entries = value.entries;
+    if (entries != null && typeof entries === 'object') {
+        return Object.fromEntries(Object.entries(entries)
+            .map(([key, val]) => [key, toPlainValue(val)]));
+    }
+    return value;
+}
+/**
+ * Parses a JSON column's text into the value it encodes. `Any` and untyped `Object` fields are
+ * stored as DuckDB JSON columns, which DuckDB returns as text.
+*/
+export function parseJSONText(value) {
+    return typeof value === 'string' ? JSON.parse(value) : value;
+}
+//# sourceMappingURL=plain-values.js.map

package/dist/src/teal-frame/plain-values.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"plain-values.js","sourceRoot":"","sources":["../../../src/teal-frame/plain-values.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;EAqBE;AACF,MAAM,CAAC,gFAAgF,CACvF,MAAM,eAAe,GAAG,iBAAiB,CAAC;AAE1C;;;;;;EAME;AACF,SAAS,aAAa,CAAC,KAAa;IAChC,IAAI,KAAK,GAAG,eAAe,IAAI,KAAK,GAAG,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACnF,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAc;IACvC,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,KAAK,CAAC;IAEhC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC;IAE3D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAE5C,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;QACxC,OAAO,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,sEAAsE;IACtE,IAAI,KAAK,YAAY,mBAAmB;QAAE,OAAO,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAE5E,MAAM,KAAK,GAAI,KAA6B,CAAC,KAAK,CAAC;IACnD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAEzD,MAAM,OAAO,GAAI,KAA+B,CAAC,OAAO,CAAC;IACzD,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QACjD,OAAO,MAAM,CAAC,WAAW,CACrB,MAAM,CAAC,OAAO,CAAC,OAAkC,CAAC;aAC7C,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CACrD,CAAC;IACN,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;EAGE;AACF,MAAM,UAAU,aAAa,CAAC,KAAc;IACxC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AACjE,CAAC"}

package/dist/src/teal-frame/row-shape.d.ts

@@ -0,0 +1,21 @@
+import { DataTypeConfig, DataTypeFields, ReadonlyDataTypeConfig, ReadonlyDataTypeFields, FieldType } from '@terascope/types';
+/**
+ * Shapes `rows()` output where the right shape depends on the field config, not the value.
+ *
+ * - `toPlainValue` turns every STRUCT into an object. A `Tuple` is stored as
+ *   `STRUCT("0" ..., "1" ...)` (a DuckDB LIST cannot mix element types), but `DataFrame`
+ *   returns it as an array, so it is converted back.
+ * - A `Date` is stored as `{ ms, tz_offset }` and rendered as `toISO8601` renders it. See
+ *   `date-value.ts`.
+ * - A JSON column (`Any`, or an `Object`/`Tuple` with no declared children) arrives as text and is
+ *   parsed, so `{x: 1}` is returned as an object rather than the string `'{"x":1}'`.
+ *
+ * Shapers are built once per `rows()` call. A column that needs no shaping gets none and costs
+ * nothing per row.
+*/
+export type ValueShaper = (value: unknown) => unknown;
+/** Returns a shaper for each top-level column that needs one, keyed by column name. */
+export declare function makeRowShapers(config: DataTypeConfig | ReadonlyDataTypeConfig): ReadonlyMap<string, ValueShaper>;
+/** Whether the column is stored as JSON (`toDuckDB` maps these types to `JSON`). */
+export declare function isJSONColumn(type: FieldType, childConfig?: DataTypeFields | ReadonlyDataTypeFields): boolean;
+//# sourceMappingURL=row-shape.d.ts.map

package/dist/src/teal-frame/row-shape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"row-shape.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/row-shape.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,cAAc,EAAuB,cAAc,EAAE,sBAAsB,EAC3E,sBAAsB,EAAE,SAAS,EACpC,MAAM,kBAAkB,CAAC;AAK1B;;;;;;;;;;;;;EAaE;AAEF,MAAM,MAAM,WAAW,GAAG,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;AAEtD,uFAAuF;AACvF,wBAAgB,cAAc,CAC1B,MAAM,EAAE,cAAc,GAAG,sBAAsB,GAChD,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAalC;AAaD,oFAAoF;AACpF,wBAAgB,YAAY,CACxB,IAAI,EAAE,SAAS,EACf,WAAW,CAAC,EAAE,cAAc,GAAG,sBAAsB,GACtD,OAAO,CAKT"}

package/dist/src/teal-frame/row-shape.js

@@ -0,0 +1,94 @@
+import { FieldType } from '@terascope/types';
+import { getChildDataTypeConfig } from '../core/utils.js';
+import { parseJSONText } from './plain-values.js';
+import { renderPlainDate } from './date-value.js';
+/** Returns a shaper for each top-level column that needs one, keyed by column name. */
+export function makeRowShapers(config) {
+    const fields = (config.fields ?? {});
+    const shapers = new Map();
+    for (const [name, fieldConfig] of Object.entries(fields)) {
+        if (name.includes('.'))
+            continue;
+        const shaper = fieldShaper(fieldConfig, getChildDataTypeConfig(fields, name, fieldConfig.type));
+        if (shaper)
+            shapers.set(name, shaper);
+    }
+    return shapers;
+}
+function fieldShaper(fieldConfig, childConfig) {
+    const element = elementShaper(fieldConfig.type, childConfig);
+    if (element == null)
+        return undefined;
+    if (!fieldConfig.array)
+        return element;
+    return (value) => (Array.isArray(value) ? value.map(element) : value);
+}
+/** Whether the column is stored as JSON (`toDuckDB` maps these types to `JSON`). */
+export function isJSONColumn(type, childConfig) {
+    if (type === FieldType.Any)
+        return true;
+    if (type !== FieldType.Object && type !== FieldType.Tuple)
+        return false;
+    return childConfig == null
+        || Object.keys(childConfig).every((name) => name.includes('.'));
+}
+function elementShaper(type, childConfig) {
+    if (isJSONColumn(type, childConfig))
+        return parseJSONText;
+    if (type === FieldType.Date)
+        return renderPlainDate;
+    if (type === FieldType.Tuple)
+        return tupleShaper(childConfig);
+    return objectShaper(type, childConfig);
+}
+/** Returns `[name, shaper]` for each direct child; the shaper is undefined when not needed. */
+function childShapers(childConfig) {
+    if (childConfig == null)
+        return [];
+    return Object.entries(childConfig)
+        .filter(([name]) => !name.includes('.'))
+        .map(([name, config]) => [
+        name,
+        fieldShaper(config, getChildDataTypeConfig(childConfig, name, config.type)),
+    ]);
+}
+/**
+ * Converts `{"0": a, "1": b}` to `[a, b]` in position order, shaping each position. A NULL
+ * position stays `null` in the array, as in `DataFrame`.
+*/
+function tupleShaper(childConfig) {
+    const positions = childShapers(childConfig)
+        .sort(([a], [b]) => Number(a) - Number(b));
+    return (value) => {
+        if (value == null || typeof value !== 'object' || Array.isArray(value))
+            return value;
+        const struct = value;
+        return positions.map(([name, shape]) => {
+            const member = struct[name] ?? null;
+            return shape ? shape(member) : member;
+        });
+    };
+}
+/**
+ * Shapes an `Object` whose children need shaping (for example a Tuple or JSON child). Returns a
+ * new object and never mutates the input.
+*/
+function objectShaper(type, childConfig) {
+    if (type !== FieldType.Object)
+        return undefined;
+    const shaped = childShapers(childConfig)
+        .filter((entry) => entry[1] != null);
+    if (shaped.length === 0)
+        return undefined;
+    return (value) => {
+        if (value == null || typeof value !== 'object' || Array.isArray(value))
+            return value;
+        const copy = { ...value };
+        for (const [name, shape] of shaped) {
+            if (name in copy)
+                copy[name] = shape(copy[name]);
+        }
+        return copy;
+    };
+}
+//# sourceMappingURL=row-shape.js.map

package/dist/src/teal-frame/row-shape.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"row-shape.js","sourceRoot":"","sources":["../../../src/teal-frame/row-shape.ts"],"names":[],"mappings":"AAAA,OAAO,EAEqB,SAAS,EACpC,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAmBlD,uFAAuF;AACvF,MAAM,UAAU,cAAc,CAC1B,MAA+C;IAE/C,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAmB,CAAC;IACvD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE/C,KAAK,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACvD,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QACjC,MAAM,MAAM,GAAG,WAAW,CACtB,WAAW,EACX,sBAAsB,CAAC,MAAM,EAAE,IAAI,EAAE,WAAW,CAAC,IAAiB,CAAC,CACtE,CAAC;QACF,IAAI,MAAM;YAAE,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAChB,WAAgC,EAChC,WAAqD;IAErD,MAAM,OAAO,GAAG,aAAa,CAAC,WAAW,CAAC,IAAiB,EAAE,WAAW,CAAC,CAAC;IAE1E,IAAI,OAAO,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACtC,IAAI,CAAC,WAAW,CAAC,KAAK;QAAE,OAAO,OAAO,CAAC;IACvC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAC1E,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,YAAY,CACxB,IAAe,EACf,WAAqD;IAErD,IAAI,IAAI,KAAK,SAAS,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACxC,IAAI,IAAI,KAAK,SAAS,CAAC,MAAM,IAAI,IAAI,KAAK,SAAS,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACxE,OAAO,WAAW,IAAI,IAAI;WACnB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,aAAa,CAClB,IAAe,EACf,WAAqD;IAErD,IAAI,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC;QAAE,OAAO,aAAa,CAAC;IAC1D,IAAI,IAAI,KAAK,SAAS,CAAC,IAAI;QAAE,OAAO,eAAe,CAAC;IACpD,IAAI,IAAI,KAAK,SAAS,CAAC,KAAK;QAAE,OAAO,WAAW,CAAC,WAAW,CAAC,CAAC;IAC9D,OAAO,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;AAC3C,CAAC;AAED,+FAA+F;AAC/F,SAAS,YAAY,CACjB,WAAqD;IAErD,IAAI,WAAW,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IACnC,OAAO,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;SAC7B,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;SACvC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC;QACrB,IAAI;QACJ,WAAW,CACP,MAAM,EACN,sBAAsB,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,IAAiB,CAAC,CACtE;KACJ,CAAC,CAAC;AACX,CAAC;AAED;;;EAGE;AACF,SAAS,WAAW,CAAC,WAAqD;IACtE,MAAM,SAAS,GAAG,YAAY,CAAC,WAAW,CAAC;SACtC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/C,OAAO,CAAC,KAAK,EAAE,EAAE;QACb,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACrF,MAAM,MAAM,GAAG,KAAgC,CAAC;QAChD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;YACpC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1C,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;AACN,CAAC;AAED;;;EAGE;AACF,SAAS,YAAY,CACjB,IAAe,EACf,WAAqD;IAErD,IAAI,IAAI,KAAK,SAAS,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAChD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC;SACnC,MAAM,CAAC,CAAC,KAAK,EAAkC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IACzE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE1C,OAAO,CAAC,KAAK,EAAE,EAAE;QACb,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACrF,MAAM,IAAI,GAAG,EAAE,GAAG,KAAgC,EAAE,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YACjC,IAAI,IAAI,IAAI,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;AACN,CAAC"}

package/dist/src/teal-frame/scalar-function.d.ts

@@ -0,0 +1,66 @@
+import { DuckDBScalarFunction } from '@duckdb/node-api';
+import { FieldType, DataTypeFieldConfig, DataTypeFields, ReadonlyDataTypeFields } from '@terascope/types';
+/**
+ * A per-value implementation, matching what `FieldTransformConfig.create` /
+ * `FieldValidateConfig.create` return: `(value, index) => result`.
+ */
+export type ScalarFunctionImpl = (value: unknown, index: number) => unknown;
+export interface ScalarFunctionSpec {
+    /** The name it is called by in SQL. */
+    name: string;
+    /**
+     * The single value argument: a bare field type, or a full field config for a list, struct or
+     * JSON value, which a bare type cannot express. See `udfTypeFor`.
+    */
+    parameter: FieldType | DataTypeFieldConfig;
+    /** Children of the parameter, for an Object/Tuple parameter. */
+    parameterChildren?: DataTypeFields | ReadonlyDataTypeFields;
+    /**
+     * The result's field config, not just its type.
+     *
+     * The returned JS value must be converted to DuckDB's representation (a BIGINT vector
+     * rejects a `number`, for example). `makeValueConverter`, which `fromRecords` also uses,
+     * does that from the config.
+     */
+    returns: DataTypeFieldConfig;
+    /** Children of the result type, for Object/Tuple results. */
+    returnsChildren?: DataTypeFields | ReadonlyDataTypeFields;
+    /**
+     * The implementation, called once per row.
+     *
+     * This should come from a function config's `create()` via `tealFrameAdapter`, not be
+     * written by hand. The function configs already define the name, arguments, accepted types,
+     * `output_type`, and null and array handling, so restating them here would duplicate them.
+     *
+     * Exactly one of `fn` and `fnForArgs` is set.
+     */
+    fn?: ScalarFunctionImpl;
+    /**
+     * Builds the implementation for one argument set from its JSON text, which the SQL passes as
+     * a second, constant parameter: `dm_lookup_x(code, '{"in":{...}}')`.
+     *
+     * This keeps the number of registered UDFs bounded. DuckDB cannot unregister a scalar
+     * function, so registering one UDF per argument set (per `lookup` table, regex or format)
+     * would grow without limit in a long-running process. Passing the arguments as a parameter
+     * means one UDF per function and type signature, and each argument set's implementation is
+     * cached in a bounded LRU (`ARGUMENT_SETS_KEPT`).
+    */
+    fnForArgs?: (argsText: string) => ScalarFunctionImpl;
+    /**
+     * Call `fn` for rows where the argument is null. Defaults to false, which matches
+     * `DataFrame`: `transformVectorToBuilder` iterates `vector.values()`, which skips nil
+     * entries, so a transform never receives a null. DuckDB passes nulls to a UDF, so the skip
+     * is done here.
+     */
+    callWithNull?: boolean;
+}
+/** How many argument sets' implementations one UDF keeps; the least recently used is dropped. */
+export declare const ARGUMENT_SETS_KEPT = 256;
+/**
+ * Wraps a per-row JS function as a vectorized DuckDB scalar function.
+ *
+ * The binding is chunk-based (`mainFunction(info, chunk, outputVector)`), so the JS boundary is
+ * crossed once per chunk of about 2048 rows, not once per row.
+ */
+export declare function createScalarFunction(spec: ScalarFunctionSpec): DuckDBScalarFunction;
+//# sourceMappingURL=scalar-function.d.ts.map

package/dist/src/teal-frame/scalar-function.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scalar-function.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/scalar-function.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAW,MAAM,kBAAkB,CAAC;AACjE,OAAO,EACH,SAAS,EAAE,mBAAmB,EAAE,cAAc,EAAE,sBAAsB,EACzE,MAAM,kBAAkB,CAAC;AAI1B;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;AAE5E,MAAM,WAAW,kBAAkB;IAC/B,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb;;;MAGE;IACF,SAAS,EAAE,SAAS,GAAG,mBAAmB,CAAC;IAC3C,gEAAgE;IAChE,iBAAiB,CAAC,EAAE,cAAc,GAAG,sBAAsB,CAAC;IAC5D;;;;;;OAMG;IACH,OAAO,EAAE,mBAAmB,CAAC;IAC7B,6DAA6D;IAC7D,eAAe,CAAC,EAAE,cAAc,GAAG,sBAAsB,CAAC;IAC1D;;;;;;;;OAQG;IACH,EAAE,CAAC,EAAE,kBAAkB,CAAC;IACxB;;;;;;;;;MASE;IACF,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,kBAAkB,CAAC;IACrD;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,iGAAiG;AACjG,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAqBtC;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,kBAAkB,GAAG,oBAAoB,CAkDnF"}

package/dist/src/teal-frame/scalar-function.js

@@ -0,0 +1,75 @@
+import { DuckDBScalarFunction, VARCHAR } from '@duckdb/node-api';
+import { makeValueConverter } from './duck-values.js';
+import { makeInputConverter, udfTypeFor } from './udf-types.js';
+/** How many argument sets' implementations one UDF keeps; the least recently used is dropped. */
+export const ARGUMENT_SETS_KEPT = 256;
+/** Memoizes `build` per argument text in a bounded LRU. */
+function perArgumentSet(build) {
+    const kept = new Map();
+    return (argsText) => {
+        const found = kept.get(argsText);
+        if (found) {
+            kept.delete(argsText);
+            kept.set(argsText, found);
+            return found;
+        }
+        const built = build(argsText);
+        kept.set(argsText, built);
+        if (kept.size > ARGUMENT_SETS_KEPT)
+            kept.delete(kept.keys().next().value);
+        return built;
+    };
+}
+/**
+ * Wraps a per-row JS function as a vectorized DuckDB scalar function.
+ *
+ * The binding is chunk-based (`mainFunction(info, chunk, outputVector)`), so the JS boundary is
+ * crossed once per chunk of about 2048 rows, not once per row.
+ */
+export function createScalarFunction(spec) {
+    const callWithNull = spec.callWithNull ?? false;
+    // a computed result may be wider than a stored Long; see `wideIntegers`
+    const convert = makeValueConverter(spec.returns, spec.returnsChildren, { wideIntegers: true });
+    const parameter = typeof spec.parameter === 'string'
+        ? { type: spec.parameter }
+        : spec.parameter;
+    /**
+     * Input values need converting too. Without it, a date arrives as a DuckDB value class
+     * that the date functions would misread (for example, as host-local time). See
+     * `makeInputConverter`.
+    */
+    const toInput = makeInputConverter(parameter, spec.parameterChildren);
+    const forArgs = spec.fnForArgs ? perArgumentSet(spec.fnForArgs) : undefined;
+    if (!forArgs && !spec.fn) {
+        throw new TypeError(`Scalar function "${spec.name}" needs \`fn\` or \`fnForArgs\``);
+    }
+    return DuckDBScalarFunction.create({
+        name: spec.name,
+        parameterTypes: forArgs
+            ? [udfTypeFor(parameter, spec.parameterChildren), VARCHAR]
+            : [udfTypeFor(parameter, spec.parameterChildren)],
+        returnType: udfTypeFor(spec.returns, spec.returnsChildren, { wide: true }),
+        mainFunction: (_info, chunk, output) => {
+            const values = chunk.getColumnValues(0);
+            /**
+             * The arguments are one constant literal per call (`udfCall`), so only the first row
+             * is read. Reading the whole column would copy the JSON into a JS string for every
+             * row, which is very slow for a large `lookup` table.
+            */
+            const impl = forArgs
+                ? forArgs(chunk.getColumnVector(1).getItem(0))
+                : spec.fn;
+            for (let row = 0; row < chunk.rowCount; row++) {
+                const value = values[row];
+                if (value == null && !callWithNull) {
+                    output.setItem(row, null);
+                    continue;
+                }
+                const result = impl(toInput(value), row);
+                output.setItem(row, (result == null ? null : convert(result)));
+            }
+            output.flush();
+        },
+    });
+}
+//# sourceMappingURL=scalar-function.js.map

package/dist/src/teal-frame/scalar-function.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scalar-function.js","sourceRoot":"","sources":["../../../src/teal-frame/scalar-function.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAIjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AA0DhE,iGAAiG;AACjG,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC,2DAA2D;AAC3D,SAAS,cAAc,CACnB,KAA+C;IAE/C,MAAM,IAAI,GAAG,IAAI,GAAG,EAA8B,CAAC;IACnD,OAAO,CAAC,QAAQ,EAAE,EAAE;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjC,IAAI,KAAK,EAAE,CAAC;YACR,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC1B,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC1B,IAAI,IAAI,CAAC,IAAI,GAAG,kBAAkB;YAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAe,CAAC,CAAC;QACpF,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;AACN,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAwB;IACzD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,KAAK,CAAC;IAChD,wEAAwE;IACxE,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/F,MAAM,SAAS,GAAwB,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ;QACrE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE;QAC1B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;IACrB;;;;MAIE;IACF,MAAM,OAAO,GAAG,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5E,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACvB,MAAM,IAAI,SAAS,CAAC,oBAAoB,IAAI,CAAC,IAAI,iCAAiC,CAAC,CAAC;IACxF,CAAC;IAED,OAAO,oBAAoB,CAAC,MAAM,CAAC;QAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,cAAc,EAAE,OAAO;YACnB,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;YAC1D,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACrD,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC1E,YAAY,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC,CAAc,CAAC;YACrD;;;;cAIE;YACF,MAAM,IAAI,GAAG,OAAO;gBAChB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAW,CAAC;gBACxD,CAAC,CAAC,IAAI,CAAC,EAAG,CAAC;YAEf,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC;gBAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;gBAE1B,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBACjC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;oBAC1B,SAAS;gBACb,CAAC;gBAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC;gBACzC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAQ,CAAC,CAAC;YAC1E,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}

package/dist/src/teal-frame/schema-check.d.ts

@@ -0,0 +1,43 @@
+import { FieldType } from '@terascope/types';
+import { TealFrame } from './TealFrame.js';
+/**
+ * One disagreement between a frame's declared `DataTypeConfig` and the column types DuckDB
+ * actually produces.
+ */
+export interface SchemaMismatch {
+    column: string;
+    /**
+     * The `FieldType` the config declares. Reports lead with this, because DuckDB types are a
+     * storage detail. Absent when `kind` is `unexpected` (the column was never declared).
+    */
+    declared?: FieldType;
+    kind: 'type' | 'missing' | 'unexpected';
+    /**
+     * For diagnosis: the DuckDB type `declared` maps to, and the type DuckDB actually produced.
+     *
+     * There is no DuckDB-to-`FieldType` mapping, because it would be ambiguous (`VARCHAR` backs
+     * `Keyword`, `Text`, `IP`, `Binary` and more). So types are compared as DuckDB types.
+    */
+    storage?: {
+        declared?: string;
+        actual?: string;
+    };
+}
+/**
+ * Returns the DuckDB column type of each column in a frame, for diagnosis and for checking a
+ * config against what DuckDB produced.
+*/
+export declare function describeColumns(frame: TealFrame): Promise<Record<string, string>>;
+/**
+ * Compares a frame's declared config with the column types DuckDB actually produced. An empty
+ * array means they agree.
+ *
+ * A column's type comes from its SQL expression, not from the config, so a projection can
+ * declare `Integer` while its SQL yields VARCHAR. Aggregations are a common case: DuckDB
+ * promotes `sum(BIGINT)` to HUGEINT, while data-mate treats the result as a `Long`.
+ *
+ * Returns the differences rather than throwing, so a caller can assert on them, log them, or
+ * ignore kinds it does not care about.
+ */
+export declare function diffSchema(frame: TealFrame): Promise<SchemaMismatch[]>;
+//# sourceMappingURL=schema-check.d.ts.map

package/dist/src/teal-frame/schema-check.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-check.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/schema-check.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuC,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG3C;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf;;;MAGE;IACF,QAAQ,CAAC,EAAE,SAAS,CAAC;IACrB,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,YAAY,CAAC;IACxC;;;;;MAKE;IACF,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACpD;AAED;;;EAGE;AACF,wBAAsB,eAAe,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAKvF;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAgC5E"}