@terascope/data-mate 2.24.0 → 2.25.0

package/dist/src/teal-frame/deep-select.js

@@ -0,0 +1,125 @@
+import { FieldType } from '@terascope/types';
+import { quoteIdentifier } from '@terascope/sql-builder';
+import { getChildDataTypeConfig } from '../core/utils.js';
+function allowsEmpty(config) {
+    return config?._allow_empty === true;
+}
+function isObject(config) {
+    return config?.type === FieldType.Object;
+}
+/** Maps each matched top-level column to the child paths kept under it, in config order. */
+function match(fields, selectors) {
+    const matched = new Map();
+    for (const field of Object.keys(fields)) {
+        const selected = selectors.some((selector) => field === selector
+            || (field.startsWith(`${selector}.`) && !allowsEmpty(fields[selector])));
+        if (!selected)
+            continue;
+        const [base] = field.split('.', 1);
+        if (!matched.has(base))
+            matched.set(base, new Set());
+        if (field !== base)
+            matched.get(base).add(field.slice(base.length + 1));
+    }
+    return matched;
+}
+/** Like `selectSubFields`: the selected child paths, plus every Object above one of them. */
+function keptChildren(children, selected) {
+    const kept = new Set();
+    for (const path of selected) {
+        if (children[path])
+            kept.add(path);
+        let parent = '';
+        for (const part of path.split('.')) {
+            parent = parent ? `${parent}.${part}` : part;
+            if (isObject(children[parent]))
+                kept.add(parent);
+        }
+    }
+    return kept;
+}
+/** The paths directly or indirectly under `member`, relative to it. */
+function under(paths, member) {
+    const prefix = `${member}.`;
+    return new Set([...paths]
+        .filter((path) => path.startsWith(prefix))
+        .map((path) => path.slice(prefix.length)));
+}
+/** Applies `one` to the value, or to each element when it is a list; null stays null. */
+function each(expression, array, one) {
+    if (!array)
+        return one(expression);
+    return `CASE WHEN ${expression} IS NULL THEN NULL`
+        + ` ELSE list_transform(${expression}, lambda x : ${one('x')}) END`;
+}
+/**
+ * SQL that narrows an Object value to the `kept` paths (relative to it). Returns the value
+ * unchanged when nothing is dropped.
+*/
+function narrow(expression, config, children, kept) {
+    const all = Object.keys(children);
+    if (all.length === 0 || all.every((path) => kept.has(path)))
+        return expression;
+    const array = config.array === true;
+    if (kept.size === 0) {
+        return each(expression, array, (value) => `CASE WHEN ${value} IS NULL THEN NULL ELSE ${allowsEmpty(config) ? 'CAST(\'{}\' AS JSON)' : `CAST(to_json(${value}) AS JSON)`} END`);
+    }
+    const members = [...kept].filter((path) => !path.includes('.'));
+    return each(expression, array, (value) => {
+        const packed = members.map((member) => {
+            const ref = `${value}.${quoteIdentifier(member)}`;
+            const memberConfig = children[member];
+            const memberChildren = isObject(memberConfig)
+                ? getChildDataTypeConfig(children, member, FieldType.Object) ?? {}
+                : {};
+            return `${quoteIdentifier(member)} := ${narrow(ref, memberConfig, memberChildren, under(kept, member))}`;
+        });
+        return `CASE WHEN ${value} IS NULL THEN NULL ELSE struct_pack(${packed.join(', ')}) END`;
+    });
+}
+export function deepSelectSQL(from, columns, config, selectors) {
+    const fields = (config.fields ?? {});
+    const matched = [...match(fields, selectors)].filter(([base]) => columns.includes(base));
+    if (matched.length === 0) {
+        return { sql: `SELECT * FROM ${from} LIMIT 0`, config: { ...config, fields: {} }, columns: [] };
+    }
+    const list = [];
+    const fieldsOut = {};
+    for (const [base, selected] of matched) {
+        const baseConfig = fields[base];
+        const children = isObject(baseConfig)
+            ? getChildDataTypeConfig(fields, base, FieldType.Object) ?? {}
+            : {};
+        const kept = keptChildren(children, selected);
+        const all = Object.keys(children);
+        const whole = all.length === 0 || all.every((path) => kept.has(path));
+        const ref = quoteIdentifier(base);
+        list.push(whole ? ref : `${narrow(ref, baseConfig, children, kept)} AS ${ref}`);
+        fieldsOut[base] = baseConfig;
+        for (const path of whole ? all : kept)
+            fieldsOut[`${base}.${path}`] = children[path];
+    }
+    return {
+        sql: `SELECT ${list.join(', ')} FROM ${from}`,
+        config: { ...config, fields: fieldsOut },
+        columns: matched.map(([base]) => base),
+    };
+}
+/**
+ * Implements the `@skipIfEmpty` GraphQL directive: an Object column marked `_skip_if_empty`
+ * is dropped once `deepSelect` has removed all of its declared children.
+*/
+function isEmptyObject(fields, name) {
+    const children = getChildDataTypeConfig(fields, name, FieldType.Object);
+    return Object.keys(children ?? {}).length === 0;
+}
+export function emptySkippedObjects(config) {
+    const fields = (config.fields ?? {});
+    return Object.entries(fields)
+        .filter(([name, field]) => !name.includes('.')
+        && field._skip_if_empty === true
+        && isObject(field)
+        && isEmptyObject(fields, name))
+        .map(([name]) => name);
+}
+//# sourceMappingURL=deep-select.js.map

package/dist/src/teal-frame/deep-select.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deep-select.js","sourceRoot":"","sources":["../../../src/teal-frame/deep-select.ts"],"names":[],"mappings":"AAAA,OAAO,EACkC,SAAS,EACjD,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AA8B1D,SAAS,WAAW,CAAC,MAAuC;IACxD,OAAQ,MAAkC,EAAE,YAAY,KAAK,IAAI,CAAC;AACtE,CAAC;AAED,SAAS,QAAQ,CAAC,MAAuC;IACrD,OAAO,MAAM,EAAE,IAAI,KAAK,SAAS,CAAC,MAAM,CAAC;AAC7C,CAAC;AAED,4FAA4F;AAC5F,SAAS,KAAK,CAAC,MAAc,EAAE,SAA4B;IACvD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE/C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,KAAK,QAAQ;eACzD,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,QAAQ,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,IAAI,CAAC,QAAQ;YAAE,SAAS;QAExB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACrD,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,6FAA6F;AAC7F,SAAS,YAAY,CAAC,QAAgB,EAAE,QAA6B;IACjE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,QAAQ,CAAC,IAAI,CAAC;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAC7C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,uEAAuE;AACvE,SAAS,KAAK,CAAC,KAAuB,EAAE,MAAc;IAClD,MAAM,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC;IAC5B,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;SACpB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;SACzC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,yFAAyF;AACzF,SAAS,IAAI,CAAC,UAAkB,EAAE,KAAc,EAAE,GAA8B;IAC5E,IAAI,CAAC,KAAK;QAAE,OAAO,GAAG,CAAC,UAAU,CAAC,CAAC;IACnC,OAAO,aAAa,UAAU,oBAAoB;UAC5C,wBAAwB,UAAU,gBAAgB,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC;AAC5E,CAAC;AAED;;;EAGE;AACF,SAAS,MAAM,CACX,UAAkB,EAClB,MAA2B,EAC3B,QAAgB,EAChB,IAAyB;IAEzB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,UAAU,CAAC;IAE/E,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC;IACpC,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,KAAK,2BACxD,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,gBAAgB,KAAK,YACxE,MAAM,CAAC,CAAC;IACZ,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAChE,OAAO,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAClC,MAAM,GAAG,GAAG,GAAG,KAAK,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;YAClD,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC;gBACzC,CAAC,CAAC,sBAAsB,CAAC,QAA0B,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE;gBACpF,CAAC,CAAC,EAAE,CAAC;YACT,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,OAC7B,MAAM,CAAC,GAAG,EAAE,YAAY,EAAE,cAAc,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CACjE,EAAE,CAAC;QACP,CAAC,CAAC,CAAC;QACH,OAAO,aAAa,KAAK,uCAAuC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;IAC7F,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,aAAa,CACzB,IAAY,EACZ,OAA0B,EAC1B,MAAmB,EACnB,SAA4B;IAE5B,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAW,CAAC;IAC/C,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAEzF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,GAAG,EAAE,iBAAiB,IAAI,UAAU,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACpG,CAAC;IAED,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,OAAO,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjC,CAAC,CAAC,sBAAsB,CAAC,MAAwB,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE;YAChF,CAAC,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAEtE,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QAEhF,SAAS,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;QAC7B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI;YAAE,SAAS,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IACzF,CAAC;IAED,OAAO;QACH,GAAG,EAAE,UAAU,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE;QAC7C,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE;QACxC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC;KACzC,CAAC;AACN,CAAC;AAED;;;EAGE;AACF,SAAS,aAAa,CAAC,MAAc,EAAE,IAAY;IAC/C,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAwB,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAC1F,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAmB;IACnD,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAW,CAAC;IAC/C,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SACxB,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;WACtC,KAAsC,CAAC,cAAc,KAAK,IAAI;WAC/D,QAAQ,CAAC,KAAK,CAAC;WACf,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;SAClC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;AAC/B,CAAC"}

package/dist/src/teal-frame/duck-values.d.ts

@@ -0,0 +1,37 @@
+import { DataTypeFieldConfig, DataTypeFields, ReadonlyDataTypeFields } from '@terascope/types';
+/**
+ * Converts values already produced by `coerceToType` into the form a DuckDB vector accepts.
+ *
+ * Coercion itself is done by `coerceToType`, so field semantics are not reimplemented here.
+ * The conversions follow the field config and must match `DataType.toDuckDB()`; a mismatch makes
+ * DuckDB reject the append rather than store the wrong value.
+*/
+export type ValueConverter = (value: unknown) => unknown;
+/** Rewrites the raw input for one field into a form that ingest coerces exactly. */
+export type InputNormalizer = (value: unknown) => unknown;
+/**
+ * Returns the bigint normalizer for one field, or `undefined` when the field contains no `Long`,
+ * so most fields pay no per-row cost.
+ *
+ * It follows the field's config, not the value, so only `Long` positions change: a bigint in an
+ * `Any` field stays a bigint. It returns new values and never modifies the caller's record.
+*/
+export declare function makeBigIntInputNormalizer(fieldConfig: DataTypeFieldConfig, childConfig?: DataTypeFields | ReadonlyDataTypeFields): InputNormalizer | undefined;
+export interface ValueConverterOptions {
+    /**
+     * Allows integers beyond the signed 64-bit range. Leave it off for a stored column: a `Long`
+     * is a BIGINT, and a value it cannot hold is a coercion failure. Turn it on for a scalar
+     * function's result, which goes into a wider vector (`ipToInt` on an IPv6 address returns a
+     * 128-bit `Long`).
+    */
+    wideIntegers?: boolean;
+}
+/**
+ * Returns the converter for one field.
+ *
+ * Null and undefined become null. `coerceToType` skips nils via `callIfNotNil`, so the
+ * converter must not turn a nil into an empty value either (such as a `[NULL]` list or a
+ * struct of nulls).
+*/
+export declare function makeValueConverter(fieldConfig: DataTypeFieldConfig, childConfig?: DataTypeFields | ReadonlyDataTypeFields, options?: ValueConverterOptions): ValueConverter;
+//# sourceMappingURL=duck-values.d.ts.map

package/dist/src/teal-frame/duck-values.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"duck-values.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/duck-values.ts"],"names":[],"mappings":"AAGA,OAAO,EACH,mBAAmB,EAAE,cAAc,EAAE,sBAAsB,EAE9D,MAAM,kBAAkB,CAAC;AAW1B;;;;;;EAME;AAEF,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;AA4DzD,oFAAoF;AACpF,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;AAE1D;;;;;;EAME;AACF,wBAAgB,yBAAyB,CACrC,WAAW,EAAE,mBAAmB,EAChC,WAAW,CAAC,EAAE,cAAc,GAAG,sBAAsB,GACtD,eAAe,GAAG,SAAS,CAS7B;AA2CD,MAAM,WAAW,qBAAqB;IAClC;;;;;MAKE;IACF,YAAY,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;;;;;EAME;AACF,wBAAgB,kBAAkB,CAC9B,WAAW,EAAE,mBAAmB,EAChC,WAAW,CAAC,EAAE,cAAc,GAAG,sBAAsB,EACrD,OAAO,GAAE,qBAA0B,GACpC,cAAc,CAgEhB"}

package/dist/src/teal-frame/duck-values.js

@@ -0,0 +1,173 @@
+import { structValue, listValue, geometryValue } from '@duckdb/node-api';
+import { FieldType } from '@terascope/types';
+import { getChildDataTypeConfig } from '../core/utils.js';
+import { geoShapeToWKB } from './wkb.js';
+import { toDateValue } from './date-value.js';
+/** Field types whose DuckDB column is a BIGINT, so they need a real BigInt. */
+const BIG_INT_TYPES = new Set([
+    FieldType.Integer, FieldType.Long,
+]);
+/** Field types held as a DOUBLE. */
+const DOUBLE_TYPES = new Set([
+    FieldType.Float, FieldType.Double, FieldType.Number, FieldType.Vector,
+]);
+/** Field types held as a narrow signed int, which want a Number not a BigInt. */
+const SMALL_INT_TYPES = new Set([
+    FieldType.Byte, FieldType.Short,
+]);
+const GEO_POINT_TYPES = new Set([
+    FieldType.GeoPoint, FieldType.Geo,
+]);
+/**
+ * `core-utils`' `_maxBigInt`, i.e. `BigInt(Number.MAX_SAFE_INTEGER)`.
+ *
+ * `toBigIntOrThrow` adds 1 to every value it parses above this bound, and `bigIntToJSON`
+ * subtracts it again, so the two cancel out for `DataFrame` (#4555). The same `-1` is applied
+ * here so DuckDB stores the original value.
+ *
+ * That only works if every value above the bound received the `+1`. `toBigIntOrThrow` returns a
+ * bigint input unchanged, so {@link makeBigIntInputNormalizer} passes such a bigint to coercion
+ * as a decimal string, which is parsed the same way as any string input.
+*/
+const MAX_SAFE_BIGINT = 9007199254740991n;
+/** The signed 64-bit range of a BIGINT column, which matches an Elasticsearch `long`. */
+const MIN_INT64 = -9223372036854775808n;
+const MAX_INT64 = 9223372036854775807n;
+function toBigInt(value, int64) {
+    if (typeof value === 'bigint') {
+        const exact = value > MAX_SAFE_BIGINT ? value - 1n : value;
+        // throw so ingest records a coercion failure instead of passing a BIGINT vector a
+        // value it cannot hold
+        if (int64 && (exact < MIN_INT64 || exact > MAX_INT64)) {
+            throw new RangeError(`${exact} is outside the signed 64-bit range of a Long`);
+        }
+        return exact;
+    }
+    const num = Number(value);
+    return Number.isFinite(num) ? BigInt(Math.trunc(num)) : null;
+}
+/**
+ * Turns a bigint above the safe range into a decimal string, so core-utils parses it and
+ * applies the `+1` that the converter later removes.
+*/
+function unparsedBigInt(value) {
+    return typeof value === 'bigint' && value > MAX_SAFE_BIGINT ? value.toString(10) : value;
+}
+/**
+ * Returns the bigint normalizer for one field, or `undefined` when the field contains no `Long`,
+ * so most fields pay no per-row cost.
+ *
+ * It follows the field's config, not the value, so only `Long` positions change: a bigint in an
+ * `Any` field stays a bigint. It returns new values and never modifies the caller's record.
+*/
+export function makeBigIntInputNormalizer(fieldConfig, childConfig) {
+    const type = fieldConfig.type;
+    const element = type === FieldType.Long
+        ? unparsedBigInt
+        : childNormalizer(type, childConfig);
+    if (element == null)
+        return undefined;
+    if (!fieldConfig.array)
+        return element;
+    return (value) => (Array.isArray(value) ? value.map(element) : element(value));
+}
+/** Normalizes the `Long` children of an Object or Tuple, by name or by position. */
+function childNormalizer(type, childConfig) {
+    if ((type !== FieldType.Object && type !== FieldType.Tuple) || childConfig == null) {
+        return undefined;
+    }
+    const children = Object.entries(childConfig)
+        .filter(([name]) => !name.includes('.'))
+        .map(([name, config]) => ({
+        name,
+        normalize: makeBigIntInputNormalizer(config, getChildDataTypeConfig(childConfig, name, config.type)),
+    }))
+        .filter((child) => (child.normalize != null));
+    if (children.length === 0)
+        return undefined;
+    return (value) => {
+        if (value == null || typeof value !== 'object')
+            return value;
+        const copy = Array.isArray(value)
+            ? [...value]
+            : { ...value };
+        for (const { name, normalize } of children) {
+            if (name in copy)
+                copy[name] = normalize(copy[name]);
+        }
+        return copy;
+    };
+}
+function toGeoPoint(value) {
+    if (typeof value !== 'object' || value == null)
+        return null;
+    const point = value;
+    return structValue({ lat: Number(point.lat), lon: Number(point.lon) });
+}
+/**
+ * Returns the converter for one field.
+ *
+ * Null and undefined become null. `coerceToType` skips nils via `callIfNotNil`, so the
+ * converter must not turn a nil into an empty value either (such as a `[NULL]` list or a
+ * struct of nulls).
+*/
+export function makeValueConverter(fieldConfig, childConfig, options = {}) {
+    const type = fieldConfig.type;
+    const int64 = options.wideIntegers !== true;
+    if (fieldConfig.array) {
+        const element = makeValueConverter({ ...fieldConfig, array: false }, childConfig, options);
+        return (value) => {
+            if (value == null)
+                return null;
+            const items = Array.isArray(value) ? value : [value];
+            return listValue(items.map(element));
+        };
+    }
+    if (type === FieldType.Object || type === FieldType.Tuple) {
+        if (childConfig == null || Object.keys(childConfig).length === 0) {
+            // no declared shape, so the column is JSON and the value goes as text
+            return (value) => (value == null ? null : JSON.stringify(value));
+        }
+        const children = Object.entries(childConfig)
+            .filter(([name]) => !name.includes('.'))
+            .map(([name, config]) => ({
+            name,
+            convert: makeValueConverter(config, getChildDataTypeConfig(childConfig, name, config.type), options),
+        }));
+        return (value) => {
+            if (value == null)
+                return null;
+            const record = value;
+            return structValue(Object.fromEntries(children.map(({ name, convert }) => [name, convert(record[name])])));
+        };
+    }
+    if (GEO_POINT_TYPES.has(type))
+        return (value) => (value == null ? null : toGeoPoint(value));
+    if (type === FieldType.Boundary) {
+        return (value) => {
+            if (value == null)
+                return null;
+            const points = Array.isArray(value) ? value : [value];
+            return listValue(points.map(toGeoPoint));
+        };
+    }
+    if (BIG_INT_TYPES.has(type))
+        return (value) => (value == null ? null : toBigInt(value, int64));
+    if (SMALL_INT_TYPES.has(type))
+        return (value) => (value == null ? null : Number(value));
+    if (DOUBLE_TYPES.has(type))
+        return (value) => (value == null ? null : Number(value));
+    // epoch millis, or a `DateTuple` whose offset is kept; see `date-value.ts`
+    if (type === FieldType.Date)
+        return toDateValue;
+    // GeoJSON is a GEOMETRY column, and the binding writes one only as WKB through a vector
+    if (type === FieldType.GeoJSON) {
+        return (value) => (value == null ? null : geometryValue(geoShapeToWKB(value)));
+    }
+    // Any is a JSON column; everything else is VARCHAR/BOOLEAN and goes as is
+    if (type === FieldType.Any) {
+        return (value) => (value == null ? null : JSON.stringify(value));
+    }
+    return (value) => (value == null ? null : value);
+}
+//# sourceMappingURL=duck-values.js.map

package/dist/src/teal-frame/duck-values.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"duck-values.js","sourceRoot":"","sources":["../../../src/teal-frame/duck-values.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,WAAW,EAAE,SAAS,EAAE,aAAa,EACxC,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAEH,SAAS,EACZ,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAkB9C,+EAA+E;AAC/E,MAAM,aAAa,GAAwB,IAAI,GAAG,CAAS;IACvD,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI;CACpC,CAAC,CAAC;AAEH,oCAAoC;AACpC,MAAM,YAAY,GAAwB,IAAI,GAAG,CAAS;IACtD,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM;CACxE,CAAC,CAAC;AAEH,iFAAiF;AACjF,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAS;IACzD,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,KAAK;CAClC,CAAC,CAAC;AAEH,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAS;IACzD,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG;CACpC,CAAC,CAAC;AAEH;;;;;;;;;;EAUE;AACF,MAAM,eAAe,GAAG,iBAAiB,CAAC;AAE1C,yFAAyF;AACzF,MAAM,SAAS,GAAG,CAAC,oBAAoB,CAAC;AACxC,MAAM,SAAS,GAAG,oBAAoB,CAAC;AAEvC,SAAS,QAAQ,CAAC,KAAc,EAAE,KAAc;IAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,KAAK,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3D,kFAAkF;QAClF,uBAAuB;QACvB,IAAI,KAAK,IAAI,CAAC,KAAK,GAAG,SAAS,IAAI,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,UAAU,CAAC,GAAG,KAAK,+CAA+C,CAAC,CAAC;QAClF,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjE,CAAC;AAED;;;EAGE;AACF,SAAS,cAAc,CAAC,KAAc;IAClC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAC7F,CAAC;AAKD;;;;;;EAME;AACF,MAAM,UAAU,yBAAyB,CACrC,WAAgC,EAChC,WAAqD;IAErD,MAAM,IAAI,GAAG,WAAW,CAAC,IAAqB,CAAC;IAC/C,MAAM,OAAO,GAAG,IAAI,KAAK,SAAS,CAAC,IAAI;QACnC,CAAC,CAAC,cAAc;QAChB,CAAC,CAAC,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAEzC,IAAI,OAAO,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACtC,IAAI,CAAC,WAAW,CAAC,KAAK;QAAE,OAAO,OAAO,CAAC;IACvC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,oFAAoF;AACpF,SAAS,eAAe,CACpB,IAAmB,EACnB,WAAqD;IAErD,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM,IAAI,IAAI,KAAK,SAAS,CAAC,KAAK,CAAC,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;QACjF,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;SACvC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;SACvC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QACtB,IAAI;QACJ,SAAS,EAAE,yBAAyB,CAChC,MAAM,EACN,sBAAsB,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,IAAiB,CAAC,CACtE;KACJ,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,KAAK,EAAyD,EAAE,CAAC,CACtE,KAAK,CAAC,SAAS,IAAI,IAAI,CAC1B,CAAC,CAAC;IAEP,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE5C,OAAO,CAAC,KAAK,EAAE,EAAE;QACb,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC7D,MAAM,IAAI,GAA4B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACtD,CAAC,CAAC,CAAC,GAAG,KAAK,CAAuC;YAClD,CAAC,CAAC,EAAE,GAAG,KAAgC,EAAE,CAAC;QAC9C,KAAK,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,QAAQ,EAAE,CAAC;YACzC,IAAI,IAAI,IAAI,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;AACN,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC;IAC5D,MAAM,KAAK,GAAG,KAAyC,CAAC;IACxD,OAAO,WAAW,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAC3E,CAAC;AAYD;;;;;;EAME;AACF,MAAM,UAAU,kBAAkB,CAC9B,WAAgC,EAChC,WAAqD,EACrD,UAAiC,EAAE;IAEnC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAqB,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,KAAK,IAAI,CAAC;IAE5C,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,kBAAkB,CAAC,EAAE,GAAG,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;QAC3F,OAAO,CAAC,KAAK,EAAE,EAAE;YACb,IAAI,KAAK,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC;YAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACrD,OAAO,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAY,CAAC,CAAC;QACpD,CAAC,CAAC;IACN,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,CAAC,MAAM,IAAI,IAAI,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC;QACxD,IAAI,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/D,sEAAsE;YACtE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;aACvC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;aACvC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;YACtB,IAAI;YACJ,OAAO,EAAE,kBAAkB,CACvB,MAAM,EACN,sBAAsB,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,IAAiB,CAAC,EACnE,OAAO,CACV;SACJ,CAAC,CAAC,CAAC;QACR,OAAO,CAAC,KAAK,EAAE,EAAE;YACb,IAAI,KAAK,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC;YAC/B,MAAM,MAAM,GAAG,KAAgC,CAAC;YAChD,OAAO,WAAW,CAAC,MAAM,CAAC,WAAW,CACjC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAC5C,CAAC,CAAC;QAChC,CAAC,CAAC;IACN,CAAC;IAED,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IAE5F,IAAI,IAAI,KAAK,SAAS,CAAC,QAAQ,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,EAAE,EAAE;YACb,IAAI,KAAK,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC;YAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACtD,OAAO,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAY,CAAC,CAAC;QACxD,CAAC,CAAC;IACN,CAAC;IAED,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IAC/F,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACxF,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACrF,2EAA2E;IAC3E,IAAI,IAAI,KAAK,SAAS,CAAC,IAAI;QAAE,OAAO,WAAW,CAAC;IAEhD,wFAAwF;IACxF,IAAI,IAAI,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,aAAa,CAAC,KAAiB,CAAC,CAAC,CAAC,CAAC;IAC/F,CAAC;IAED,0EAA0E;IAC1E,IAAI,IAAI,KAAK,SAAS,CAAC,GAAG,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AACrD,CAAC"}

package/dist/src/teal-frame/errors.d.ts

@@ -0,0 +1,38 @@
+import { AppendFailure, CoercionFailure, ExtensionFailure } from './interfaces.js';
+/**
+ * Errors thrown by the DuckDB frame.
+ *
+ * They are classes rather than plain `Error`s so they can carry context: a coercion failure
+ * lists each failing field with an example value, and an append failure reports what is left
+ * in the table after the rollback.
+*/
+/** Raised by strict mode when a value does not fit its declared field type. */
+export declare class CoercionFailureError extends Error {
+    readonly failures: readonly CoercionFailure[];
+    constructor(message: string, failures: readonly CoercionFailure[]);
+}
+/**
+ * Thrown when an `append` fails and is rolled back.
+ *
+ * An append runs in a transaction, so a failure leaves the table as it was. The message says
+ * how many rows remain, so a caller building one table from many payloads knows its earlier
+ * payloads are intact. The underlying error is kept as `cause`.
+ *
+ * `fromRecords` does not use this error: it has no earlier table to report on, and it throws
+ * what `DataFrame` throws.
+*/
+export declare class AppendError extends Error {
+    readonly failure: AppendFailure;
+    constructor(failure: AppendFailure, cause: unknown);
+}
+/**
+ * Thrown when required DuckDB extensions cannot be loaded as the database opens.
+ *
+ * Failing at startup gives a clear message. Otherwise the first query that needs the
+ * extension fails with a DuckDB autoload error that does not explain how to fix it.
+*/
+export declare class DuckExtensionError extends Error {
+    readonly failure: ExtensionFailure;
+    constructor(failure: ExtensionFailure);
+}
+//# sourceMappingURL=errors.d.ts.map

package/dist/src/teal-frame/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEnF;;;;;;EAME;AAEF,+EAA+E;AAC/E,qBAAa,oBAAqB,SAAQ,KAAK;IACd,QAAQ,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE;gBAA9D,OAAO,EAAE,MAAM,EAAW,QAAQ,EAAE,SAAS,eAAe,EAAE;CAI7E;AAED;;;;;;;;;EASE;AACF,qBAAa,WAAY,SAAQ,KAAK;IACtB,QAAQ,CAAC,OAAO,EAAE,aAAa;gBAAtB,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO;CAa9D;AAED;;;;;EAKE;AACF,qBAAa,kBAAmB,SAAQ,KAAK;IAC7B,QAAQ,CAAC,OAAO,EAAE,gBAAgB;gBAAzB,OAAO,EAAE,gBAAgB;CAgBjD"}

package/dist/src/teal-frame/errors.js

@@ -0,0 +1,63 @@
+/**
+ * Errors thrown by the DuckDB frame.
+ *
+ * They are classes rather than plain `Error`s so they can carry context: a coercion failure
+ * lists each failing field with an example value, and an append failure reports what is left
+ * in the table after the rollback.
+*/
+/** Raised by strict mode when a value does not fit its declared field type. */
+export class CoercionFailureError extends Error {
+    failures;
+    constructor(message, failures) {
+        super(message);
+        this.failures = failures;
+        this.name = 'CoercionFailureError';
+    }
+}
+/**
+ * Thrown when an `append` fails and is rolled back.
+ *
+ * An append runs in a transaction, so a failure leaves the table as it was. The message says
+ * how many rows remain, so a caller building one table from many payloads knows its earlier
+ * payloads are intact. The underlying error is kept as `cause`.
+ *
+ * `fromRecords` does not use this error: it has no earlier table to report on, and it throws
+ * what `DataFrame` throws.
+*/
+export class AppendError extends Error {
+    failure;
+    constructor(failure, cause) {
+        const reason = cause instanceof Error ? cause.message : String(cause);
+        const survived = failure.rowsRemaining == null
+            ? 'the table could not be counted afterwards, so its contents are in doubt'
+            : `the table is unchanged and still has ${failure.rowsRemaining} row(s)`;
+        super(`appending ${failure.describedSource} to table "${failure.table}" failed`
+            + ` - it was rolled back, so ${survived}. Cause: ${reason}`, { cause });
+        this.failure = failure;
+        this.name = 'AppendError';
+    }
+}
+/**
+ * Thrown when required DuckDB extensions cannot be loaded as the database opens.
+ *
+ * Failing at startup gives a clear message. Otherwise the first query that needs the
+ * extension fails with a DuckDB autoload error that does not explain how to fix it.
+*/
+export class DuckExtensionError extends Error {
+    failure;
+    constructor(failure) {
+        const names = failure.missing.map(({ name }) => name).join(', ');
+        const details = failure.missing
+            .map(({ name, reason, error }) => `  ${name} (${reason}): ${error}`)
+            .join('\n');
+        const fix = failure.autoinstall
+            ? 'the download failed, so this machine cannot reach the extension repository'
+            : 'autoinstall is off, so they must be installed when the image is built:'
+                + ` \`duckdb-extensions install --dir ${failure.directory}\``;
+        super(`DuckDB extension(s) ${names} could not be loaded (duckdb ${failure.version},`
+            + ` ${failure.platform}, directory ${failure.directory}) - ${fix}\n${details}`);
+        this.failure = failure;
+        this.name = 'DuckExtensionError';
+    }
+}
+//# sourceMappingURL=errors.js.map

package/dist/src/teal-frame/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../../src/teal-frame/errors.ts"],"names":[],"mappings":"AAEA;;;;;;EAME;AAEF,+EAA+E;AAC/E,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACL;IAAtC,YAAY,OAAe,EAAW,QAAoC;QACtE,KAAK,CAAC,OAAO,CAAC,CAAC;QADmB,aAAQ,GAAR,QAAQ,CAA4B;QAEtE,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACvC,CAAC;CACJ;AAED;;;;;;;;;EASE;AACF,MAAM,OAAO,WAAY,SAAQ,KAAK;IACb;IAArB,YAAqB,OAAsB,EAAE,KAAc;QACvD,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI;YAC1C,CAAC,CAAC,yEAAyE;YAC3E,CAAC,CAAC,wCAAwC,OAAO,CAAC,aAAa,SAAS,CAAC;QAE7E,KAAK,CACD,aAAa,OAAO,CAAC,eAAe,cAAc,OAAO,CAAC,KAAK,UAAU;cACvE,6BAA6B,QAAQ,YAAY,MAAM,EAAE,EAC3D,EAAE,KAAK,EAAE,CACZ,CAAC;QAVe,YAAO,GAAP,OAAO,CAAe;QAWvC,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC9B,CAAC;CACJ;AAED;;;;;EAKE;AACF,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IACpB;IAArB,YAAqB,OAAyB;QAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO;aAC1B,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,MAAM,MAAM,KAAK,EAAE,CAAC;aACnE,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW;YAC3B,CAAC,CAAC,4EAA4E;YAC9E,CAAC,CAAC,wEAAwE;kBACpE,sCAAsC,OAAO,CAAC,SAAS,IAAI,CAAC;QAEtE,KAAK,CACD,uBAAuB,KAAK,gCAAgC,OAAO,CAAC,OAAO,GAAG;cAC5E,IAAI,OAAO,CAAC,QAAQ,eAAe,OAAO,CAAC,SAAS,OAAO,GAAG,KAAK,OAAO,EAAE,CACjF,CAAC;QAbe,YAAO,GAAP,OAAO,CAAkB;QAc1C,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACrC,CAAC;CACJ"}

package/dist/src/teal-frame/expand-values.d.ts

@@ -0,0 +1,29 @@
+import { FrameConfig } from './interfaces.js';
+/**
+ * SQL for the `@expandValues` directive, which turns each array element into its own output
+ * row. There are two modes:
+ *
+ * - `correlate: true` zips the lists by position: a row produces as many rows as its longest
+ *   list, and shorter lists are padded with null.
+ * - `correlate: false` stacks the lists: a row produces as many rows as its list lengths add
+ *   up to. Each field gets its own block of rows, with the other expanded fields null there.
+ *
+ * In both modes, a row whose expanded lists are all null or empty is kept as one row with those
+ * fields null (a plain `unnest` would drop it). Fields that are not expanded are copied onto
+ * every output row. Blocks follow the frame's column order, not the order of `fields`. The
+ * expanded fields come out with `array: false`.
+ *
+ * It is a single SELECT with no UNION ALL, so row order is kept. Each row unnests a list of
+ * `{ k, i }` pairs (which field, which position), and each expanded column is `field[i]` where
+ * `k` selects it. The pairs have the same type whatever the fields hold, so no element type
+ * has to be spelled out.
+*/
+export interface ExpandValuesOptions {
+    /** `true` zips the lists by position; `false` stacks one block of rows per field. */
+    readonly correlate: boolean;
+}
+export declare function expandValuesSQL(from: string, columns: readonly string[], config: FrameConfig, fields: readonly string[], options: ExpandValuesOptions): {
+    sql: string;
+    config: FrameConfig;
+};
+//# sourceMappingURL=expand-values.d.ts.map

package/dist/src/teal-frame/expand-values.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"expand-values.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/expand-values.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C;;;;;;;;;;;;;;;;;;EAkBE;AAEF,MAAM,WAAW,mBAAmB;IAChC,qFAAqF;IACrF,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC/B;AAgCD,wBAAgB,eAAe,CAC3B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,OAAO,EAAE,mBAAmB,GAC7B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAA;CAAE,CA6BtC"}

package/dist/src/teal-frame/expand-values.js

@@ -0,0 +1,53 @@
+import { TSError } from '@terascope/core-utils';
+import { quoteIdentifier } from '@terascope/sql-builder';
+const PAIR = '__teal_expand';
+function invalid(field) {
+    return new TSError(`Invalid use of @expandValues on field "${field}", expected array value`, {
+        statusCode: 400,
+        context: { safe: true },
+    });
+}
+/** The list of `{ k, i }` pairs one input row expands into. */
+function pairs(expanded, correlate) {
+    const length = (field) => `coalesce(len(${quoteIdentifier(field)}), 0)`;
+    // a row with nothing to expand still produces one row: field -1, position 0, null everywhere
+    const placeholder = `[{'k': -1, 'i': 0}]`;
+    if (correlate) {
+        const longest = expanded.length === 1
+            ? length(expanded[0])
+            : `greatest(${expanded.map(length).join(', ')})`;
+        return `CASE WHEN ${longest} = 0 THEN ${placeholder}`
+            + ` ELSE list_transform(range(1, ${longest} + 1), lambda i : {'k': 0, 'i': i}) END`;
+    }
+    const blocks = expanded.map((field, k) => (`list_transform(range(1, ${length(field)} + 1), lambda i : {'k': ${k}, 'i': i})`));
+    const all = blocks.length === 1 ? blocks[0] : `list_concat(${blocks.join(', ')})`;
+    return `CASE WHEN len(${all}) = 0 THEN ${placeholder} ELSE ${all} END`;
+}
+export function expandValuesSQL(from, columns, config, fields, options) {
+    const declared = (config.fields ?? {});
+    for (const field of fields) {
+        if (!columns.includes(field)) {
+            throw new TSError(`Unknown column "${field}" in @expandValues`, {
+                statusCode: 400,
+                context: { safe: true },
+            });
+        }
+        if (declared[field]?.array !== true)
+            throw invalid(field);
+    }
+    const wanted = new Set(fields);
+    const expanded = columns.filter((column) => wanted.has(column));
+    const pick = (column) => {
+        const k = expanded.indexOf(column);
+        const ref = quoteIdentifier(column);
+        if (k === -1)
+            return ref;
+        const which = options.correlate ? `${PAIR}.k = 0` : `${PAIR}.k = ${k}`;
+        return `CASE WHEN ${which} THEN ${ref}[${PAIR}.i] END AS ${ref}`;
+    };
+    const inner = `SELECT *, unnest(${pairs(expanded, options.correlate)}) AS ${PAIR} FROM ${from}`;
+    const sql = `SELECT ${columns.map(pick).join(', ')} FROM (${inner})`;
+    const fieldsOut = Object.fromEntries(Object.entries(declared).map(([name, field]) => [name, wanted.has(name) ? { ...field, array: false } : field]));
+    return { sql, config: { ...config, fields: fieldsOut } };
+}
+//# sourceMappingURL=expand-values.js.map

package/dist/src/teal-frame/expand-values.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"expand-values.js","sourceRoot":"","sources":["../../../src/teal-frame/expand-values.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AA4BzD,MAAM,IAAI,GAAG,eAAe,CAAC;AAE7B,SAAS,OAAO,CAAC,KAAa;IAC1B,OAAO,IAAI,OAAO,CAAC,0CAA0C,KAAK,yBAAyB,EAAE;QACzF,UAAU,EAAE,GAAG;QACf,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;KAC1B,CAAC,CAAC;AACP,CAAC;AAED,+DAA+D;AAC/D,SAAS,KAAK,CAAC,QAA2B,EAAE,SAAkB;IAC1D,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,gBAAgB,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC;IAChF,6FAA6F;IAC7F,MAAM,WAAW,GAAG,qBAAqB,CAAC;IAE1C,IAAI,SAAS,EAAE,CAAC;QACZ,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC;YACjC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACrB,CAAC,CAAC,YAAY,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QACrD,OAAO,aAAa,OAAO,aAAa,WAAW,EAAE;cAC/C,iCAAiC,OAAO,yCAAyC,CAAC;IAC5F,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CACtC,2BAA2B,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,YAAY,CACnF,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAClF,OAAO,iBAAiB,GAAG,cAAc,WAAW,SAAS,GAAG,MAAM,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,eAAe,CAC3B,IAAY,EACZ,OAA0B,EAC1B,MAAmB,EACnB,MAAyB,EACzB,OAA4B;IAE5B,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAmB,CAAC;IACzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,OAAO,CAAC,mBAAmB,KAAK,oBAAoB,EAAE;gBAC5D,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;aAC1B,CAAC,CAAC;QACP,CAAC;QACD,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI;YAAE,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,MAAM,IAAI,GAAG,CAAC,MAAc,EAAE,EAAE;QAC5B,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC;QACzB,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC;QACvE,OAAO,aAAa,KAAK,SAAS,GAAG,IAAI,IAAI,cAAc,GAAG,EAAE,CAAC;IACrE,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,oBAAoB,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,QAAQ,IAAI,SAAS,IAAI,EAAE,CAAC;IAChG,MAAM,GAAG,GAAG,UAAU,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,KAAK,GAAG,CAAC;IAErE,MAAM,SAAS,GAAmB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAC7E,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACnF,CAAC,CAAC;IACH,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;AAC7D,CAAC"}

package/dist/src/teal-frame/export-json.d.ts

@@ -0,0 +1,58 @@
+import { DataTypeConfig, ReadonlyDataTypeConfig } from '@terascope/types';
+import { SerializeOptions } from '../vector/interfaces.js';
+/**
+ * How an export renders fields and nulls: `DataFrame`'s `SerializeOptions`, with the same
+ * meaning, plus `removeNullFields` as a shorthand for two common presets. See
+ * {@link resolveRules}.
+*/
+export interface JsonExportOptions extends SerializeOptions {
+    /**
+     * Selects a preset: `true` omits null fields and empty objects, `false` writes null fields
+     * as `null`. Explicit `SerializeOptions` flags override the preset.
+    */
+    removeNullFields?: boolean;
+}
+/**
+ * The seven `SerializeOptions` flags, resolved. See {@link resolveRules}.
+*/
+interface Rules {
+    useNullForUndefined: boolean;
+    skipNilValues: boolean;
+    skipEmptyArrays: boolean;
+    skipNilListValues: boolean;
+    skipNilObjectValues: boolean;
+    skipEmptyObjects: boolean;
+    skipDuplicateObjects: boolean;
+}
+/**
+ * Resolves the rules an export follows.
+ *
+ * If any `SerializeOptions` flag is given, the flags mean exactly what they mean to
+ * `DataFrame.toJSON`, and an unset flag is false. `removeNullFields` selects a preset that
+ * explicit flags override. With neither, the default is the `removeNullFields: true` preset.
+*/
+export declare function resolveRules(options?: JsonExportOptions): Rules;
+/**
+ * SQL rendering one LIST column's elements as JSON text (see `TealFrame.listValues`), and
+ * whether the elements are objects, which decides whether `skipNilObjectValues` applies.
+*/
+export declare function buildJsonListElements(config: DataTypeConfig | ReadonlyDataTypeConfig, column: string, options?: JsonExportOptions): {
+    sql: string;
+    isObject: boolean;
+};
+/**
+ * The predicate for the rows an export writes, or undefined when it writes all of them. It is
+ * separate from {@link buildJsonExpression} so the filter never evaluates the rendered row.
+*/
+export declare function buildJsonRowFilter(config: DataTypeConfig | ReadonlyDataTypeConfig, options?: JsonExportOptions): string | undefined;
+/**
+ * A SQL expression producing one JSON string per row, byte-identical to `DataFrame.toJSON` with
+ * the same options, or NULL for a row that `skipEmptyObjects` drops.
+ *
+ * Two things are left to the caller: removing duplicate rows under `skipDuplicateObjects`,
+ * which compares whole rendered rows in order and is done on the stream (`TealFrame.ndjson`),
+ * and filtering rows, which uses {@link buildJsonRowFilter}.
+*/
+export declare function buildJsonExpression(config: DataTypeConfig | ReadonlyDataTypeConfig, options?: JsonExportOptions): string;
+export {};
+//# sourceMappingURL=export-json.d.ts.map

package/dist/src/teal-frame/export-json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"export-json.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/export-json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,sBAAsB,EAA6B,MAAM,kBAAkB,CAAC;AAErG,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AA0D3D;;;;EAIE;AACF,MAAM,WAAW,iBAAkB,SAAQ,gBAAgB;IACvD;;;MAGE;IACF,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAoDD;;EAEE;AACF,UAAU,KAAK;IACX,mBAAmB,EAAE,OAAO,CAAC;IAC7B,aAAa,EAAE,OAAO,CAAC;IACvB,eAAe,EAAE,OAAO,CAAC;IACzB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,mBAAmB,EAAE,OAAO,CAAC;IAC7B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,oBAAoB,EAAE,OAAO,CAAC;CACjC;AA4BD;;;;;;EAME;AACF,wBAAgB,YAAY,CAAC,OAAO,GAAE,iBAAsB,GAAG,KAAK,CAOnE;AAmQD;;;EAGE;AACF,wBAAgB,qBAAqB,CACjC,MAAM,EAAE,cAAc,GAAG,sBAAsB,EAC/C,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,iBAAsB,GAChC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,CASpC;AAED;;;EAGE;AACF,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,cAAc,GAAG,sBAAsB,EAC/C,OAAO,GAAE,iBAAsB,GAChC,MAAM,GAAG,SAAS,CAGpB;AAED;;;;;;;EAOE;AACF,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,cAAc,GAAG,sBAAsB,EAC/C,OAAO,GAAE,iBAAsB,GAChC,MAAM,CAMR"}