@terascope/data-mate 2.23.1 → 2.25.0

package/dist/src/teal-frame/schema-check.js

@@ -0,0 +1,66 @@
+import { DataType } from '@terascope/data-types';
+/**
+ * Returns the DuckDB column type of each column in a frame, for diagnosis and for checking a
+ * config against what DuckDB produced.
+*/
+export async function describeColumns(frame) {
+    const rows = await frame.rawRows(`DESCRIBE SELECT * FROM ${frame.from}`);
+    return Object.fromEntries(rows.map((row) => [String(row[0]), String(row[1])]));
+}
+/**
+ * Compares a frame's declared config with the column types DuckDB actually produced. An empty
+ * array means they agree.
+ *
+ * A column's type comes from its SQL expression, not from the config, so a projection can
+ * declare `Integer` while its SQL yields VARCHAR. Aggregations are a common case: DuckDB
+ * promotes `sum(BIGINT)` to HUGEINT, while data-mate treats the result as a `Long`.
+ *
+ * Returns the differences rather than throwing, so a caller can assert on them, log them, or
+ * ignore kinds it does not care about.
+ */
+export async function diffSchema(frame) {
+    const declared = new DataType(frame.config).toDuckDB();
+    const actual = await describeColumns(frame);
+    const mismatches = [];
+    const fields = frame.config.fields ?? {};
+    for (const [column, declaredType] of Object.entries(declared)) {
+        const actualType = actual[column];
+        const fieldType = fields[column]?.type;
+        if (actualType == null) {
+            mismatches.push({
+                column, declared: fieldType, kind: 'missing', storage: { declared: declaredType }
+            });
+        }
+        else if (normalize(declaredType) !== normalize(actualType)) {
+            mismatches.push({
+                column,
+                declared: fieldType,
+                kind: 'type',
+                storage: { declared: declaredType, actual: actualType },
+            });
+        }
+    }
+    for (const [column, actualType] of Object.entries(actual)) {
+        if (!(column in declared)) {
+            mismatches.push({ column, kind: 'unexpected', storage: { actual: actualType } });
+        }
+    }
+    return mismatches;
+}
+/**
+ * Normalizes a DuckDB type string for comparison.
+ *
+ * `toDuckDB()` and `DESCRIBE` use the same canonical spellings, so only case, whitespace and
+ * quotes need normalizing. If a spelling-only mismatch appears, add an alias here.
+ *
+ * Quotes are removed because `toDuckDB()` always quotes STRUCT member names (names such as
+ * `order` are reserved words), while `DESCRIBE` reports them unquoted.
+ */
+function normalize(type) {
+    return type.trim().toUpperCase()
+        .replace(/"/g, '')
+        // a GEOMETRY read from GeoParquet carries its CRS, as in `GEOMETRY('OGC:CRS84')`;
+        // that is metadata, not a different type
+        .replace(/GEOMETRY\('[^']*'\)/g, 'GEOMETRY');
+}
+//# sourceMappingURL=schema-check.js.map

package/dist/src/teal-frame/schema-check.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-check.js","sourceRoot":"","sources":["../../../src/teal-frame/schema-check.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAuBjD;;;EAGE;AACF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAgB;IAClD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,0BAA0B,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACzE,OAAO,MAAM,CAAC,WAAW,CACrB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACtD,CAAC;AACN,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAgB;IAC7C,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC;IACzE,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAqB,EAAE,CAAC;IAExC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;IAEzC,KAAK,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,SAAS,GAAI,MAAM,CAAC,MAAM,CAAqC,EAAE,IAAiB,CAAC;QAEzF,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACrB,UAAU,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE;aACpF,CAAC,CAAC;QACP,CAAC;aAAM,IAAI,SAAS,CAAC,YAAY,CAAC,KAAK,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3D,UAAU,CAAC,IAAI,CAAC;gBACZ,MAAM;gBACN,QAAQ,EAAE,SAAS;gBACnB,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE;aAC1D,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,KAAK,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACxD,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,EAAE,CAAC;YACxB,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;QACrF,CAAC;IACL,CAAC;IAED,OAAO,UAAU,CAAC;AACtB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,SAAS,CAAC,IAAY;IAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE;SAC3B,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QAClB,kFAAkF;QAClF,yCAAyC;SACxC,OAAO,CAAC,sBAAsB,EAAE,UAAU,CAAC,CAAC;AACrD,CAAC"}

package/dist/src/teal-frame/sources.d.ts

@@ -0,0 +1,14 @@
+import { AppendSource } from './interfaces.js';
+/**
+ * Helpers that describe where rows come from: a Parquet relation, or an append's source.
+*/
+/**
+ * Returns `read_parquet(...)` over one path or many.
+ *
+ * A list of files reads as a single relation, so several Parquet payloads can be loaded as
+ * one table.
+*/
+export declare function readParquetSource(paths: string | readonly string[]): string;
+/** Names what was being appended, for an error message. */
+export declare function describeSource(source: AppendSource): string;
+//# sourceMappingURL=sources.d.ts.map

package/dist/src/teal-frame/sources.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/sources.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C;;EAEE;AAEF;;;;;EAKE;AACF,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,GAAG,MAAM,CAM3E;AAED,2DAA2D;AAC3D,wBAAgB,cAAc,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAK3D"}

package/dist/src/teal-frame/sources.js

@@ -0,0 +1,28 @@
+import { quoteLiteral } from '@terascope/sql-builder';
+/**
+ * Helpers that describe where rows come from: a Parquet relation, or an append's source.
+*/
+/**
+ * Returns `read_parquet(...)` over one path or many.
+ *
+ * A list of files reads as a single relation, so several Parquet payloads can be loaded as
+ * one table.
+*/
+export function readParquetSource(paths) {
+    if (typeof paths === 'string')
+        return `read_parquet(${quoteLiteral(paths)})`;
+    if (paths.length === 0) {
+        throw new TypeError('at least one Parquet path is required');
+    }
+    return `read_parquet([${paths.map(quoteLiteral).join(', ')}])`;
+}
+/** Names what was being appended, for an error message. */
+export function describeSource(source) {
+    if (source.records !== undefined)
+        return `${source.records.length} record(s)`;
+    const { parquet } = source;
+    if (typeof parquet === 'string')
+        return `Parquet "${parquet}"`;
+    return `${parquet.length} Parquet path(s)`;
+}
+//# sourceMappingURL=sources.js.map

package/dist/src/teal-frame/sources.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sources.js","sourceRoot":"","sources":["../../../src/teal-frame/sources.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGtD;;EAEE;AAEF;;;;;EAKE;AACF,MAAM,UAAU,iBAAiB,CAAC,KAAiC;IAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,gBAAgB,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC;IAC7E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,iBAAiB,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACnE,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,cAAc,CAAC,MAAoB;IAC/C,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,YAAY,CAAC;IAC9E,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAC3B,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,YAAY,OAAO,GAAG,CAAC;IAC/D,OAAO,GAAG,OAAO,CAAC,MAAM,kBAAkB,CAAC;AAC/C,CAAC"}

package/dist/src/teal-frame/table.d.ts

@@ -0,0 +1,19 @@
+import { DuckContext } from './DuckContext.js';
+import { CreatedTable, FieldPlan, FrameConfig } from './interfaces.js';
+/** Returns a unique table name. Names never come from user data, so a counter is enough. */
+export declare function nextTableName(name?: string): string;
+/**
+ * Resolves each top-level field to its coercion and its DuckDB value conversion.
+ *
+ * Dot-notation children are folded into their parent, so the plan matches the table's
+ * real column set.
+*/
+export declare function buildPlan(config: FrameConfig): FieldPlan[];
+/**
+ * Creates the table for a config and returns its name and columns.
+ *
+ * Used by both `fromRecords` (which then appends records) and `empty` (which is then filled by
+ * `appendParquet`).
+*/
+export declare function createTable(context: DuckContext, config: FrameConfig, name?: string): Promise<CreatedTable>;
+//# sourceMappingURL=table.d.ts.map

package/dist/src/teal-frame/table.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"table.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/table.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAQvE,4FAA4F;AAC5F,wBAAgB,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAInD;AAED;;;;;EAKE;AACF,wBAAgB,SAAS,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,EAAE,CAe1D;AAeD;;;;;EAKE;AACF,wBAAsB,WAAW,CAC7B,OAAO,EAAE,WAAW,EACpB,MAAM,EAAE,WAAW,EACnB,IAAI,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,YAAY,CAAC,CAevB"}

package/dist/src/teal-frame/table.js

@@ -0,0 +1,64 @@
+import { DataType } from '@terascope/data-types';
+import { quoteIdentifier } from '@terascope/sql-builder';
+import { getChildDataTypeConfig } from '../core/utils.js';
+import { coerceToType } from '../builder/type-coercion.js';
+import { makeValueConverter, makeBigIntInputNormalizer } from './duck-values.js';
+/**
+ * Creates the table that holds a frame's rows, and plans how each field is coerced into it.
+*/
+let tableCounter = 0;
+/** Returns a unique table name. Names never come from user data, so a counter is enough. */
+export function nextTableName(name) {
+    tableCounter += 1;
+    const base = (name ?? 'teal_frame').replace(/[^A-Za-z0-9_]/g, '_');
+    return `${base}_${tableCounter}`;
+}
+/**
+ * Resolves each top-level field to its coercion and its DuckDB value conversion.
+ *
+ * Dot-notation children are folded into their parent, so the plan matches the table's
+ * real column set.
+*/
+export function buildPlan(config) {
+    const fields = config.fields ?? {};
+    return Object.entries(fields)
+        .filter(([name]) => !name.includes('.'))
+        .map(([name, fieldConfig]) => {
+        const children = getChildDataTypeConfig(fields, name, fieldConfig.type);
+        return {
+            name,
+            fieldType: String(fieldConfig.type),
+            coerce: exactCoerce(fieldConfig, children),
+            convert: makeValueConverter(fieldConfig, children),
+        };
+    });
+}
+/**
+ * Returns `coerceToType` for a field, with large bigint inputs first rewritten as strings so
+ * they are coerced exactly. See `makeBigIntInputNormalizer`.
+*/
+function exactCoerce(fieldConfig, children) {
+    const coerce = coerceToType(fieldConfig, children);
+    const normalize = makeBigIntInputNormalizer(fieldConfig, children);
+    return normalize ? (value) => coerce(normalize(value)) : coerce;
+}
+/**
+ * Creates the table for a config and returns its name and columns.
+ *
+ * Used by both `fromRecords` (which then appends records) and `empty` (which is then filled by
+ * `appendParquet`).
+*/
+export async function createTable(context, config, name) {
+    const plan = buildPlan(config);
+    if (plan.length === 0) {
+        throw new TypeError('A DataType config must declare at least one field');
+    }
+    const table = nextTableName(name);
+    const columnTypes = new DataType(config).toDuckDB();
+    const ddl = plan
+        .map(({ name: column }) => `${quoteIdentifier(column)} ${columnTypes[column]}`)
+        .join(', ');
+    await context.run(`CREATE OR REPLACE TABLE ${quoteIdentifier(table)} (${ddl})`);
+    return { table, columns: plan.map(({ name: column }) => column) };
+}
+//# sourceMappingURL=table.js.map

package/dist/src/teal-frame/table.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"table.js","sourceRoot":"","sources":["../../../src/teal-frame/table.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,kBAAkB,CAAC;AAIjF;;EAEE;AAEF,IAAI,YAAY,GAAG,CAAC,CAAC;AAErB,4FAA4F;AAC5F,MAAM,UAAU,aAAa,CAAC,IAAa;IACvC,YAAY,IAAI,CAAC,CAAC;IAClB,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;IACnE,OAAO,GAAG,IAAI,IAAI,YAAY,EAAE,CAAC;AACrC,CAAC;AAED;;;;;EAKE;AACF,MAAM,UAAU,SAAS,CAAC,MAAmB;IACzC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;IACnC,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SACxB,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;SACvC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,EAAE;QACzB,MAAM,QAAQ,GAAG,sBAAsB,CACnC,MAAM,EAAE,IAAI,EAAE,WAAW,CAAC,IAAiB,CAChB,CAAC;QAChC,OAAO;YACH,IAAI;YACJ,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC;YACnC,MAAM,EAAE,WAAW,CAAC,WAAkC,EAAE,QAAQ,CAAC;YACjE,OAAO,EAAE,kBAAkB,CAAC,WAAkC,EAAE,QAAQ,CAAC;SAC5E,CAAC;IACN,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;;EAGE;AACF,SAAS,WAAW,CAChB,WAAgC,EAChC,QAAyB;IAEzB,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,yBAAyB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IACnE,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;AACpE,CAAC;AAED;;;;;EAKE;AACF,MAAM,CAAC,KAAK,UAAU,WAAW,CAC7B,OAAoB,EACpB,MAAmB,EACnB,IAAa;IAEb,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAC/B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,SAAS,CAAC,mDAAmD,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,WAAW,GAAG,IAAI,QAAQ,CAAC,MAAwB,CAAC,CAAC,QAAQ,EAAE,CAAC;IACtE,MAAM,GAAG,GAAG,IAAI;SACX,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;SAC9E,IAAI,CAAC,IAAI,CAAC,CAAC;IAEhB,MAAM,OAAO,CAAC,GAAG,CAAC,2BAA2B,eAAe,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;IAEhF,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;AACtE,CAAC"}

package/dist/src/teal-frame/udf-types.d.ts

@@ -0,0 +1,45 @@
+import { DuckDBType } from '@duckdb/node-api';
+import { DataTypeFieldConfig, DataTypeFields, ReadonlyDataTypeFields, FieldType, DeprecatedFieldType } from '@terascope/types';
+/**
+ * Maps field configs to the DuckDB type objects a scalar function is registered with, and
+ * converts the values DuckDB passes to it.
+ *
+ * Both work from the full field config rather than a bare `FieldType`, because a bare type
+ * cannot describe a list, a struct with members, or JSON.
+ *
+ * **This must agree with `DataType.toDuckDB()`**, which produces the same mapping as type
+ * strings for DDL. Registration needs the binding's type objects, and `@terascope/data-types`
+ * does not depend on `@duckdb/node-api`. A mismatch shows up as a bind error, or as a column
+ * whose type contradicts its config (which `diffSchema` reports).
+*/
+/**
+ * `DataTypeFieldConfig.type` is either the enum or the deprecated string union, and configs still
+ * use both. The enum's values are the same strings.
+*/
+type FieldTypeLike = FieldType | DeprecatedFieldType;
+type Children = DataTypeFields | ReadonlyDataTypeFields;
+export interface UdfTypeOptions {
+    /**
+     * Widens `Long` and `Integer` to HUGEINT. Use it for results only: a computed result may be
+     * wider than a stored Long (`ipToInt` on an IPv6 address is 128 bits), while a parameter is
+     * read from a stored BIGINT column.
+    */
+    wide?: boolean;
+}
+/** Returns the DuckDB type object for a field config, or throws if the type has none. */
+export declare function udfTypeFor(fieldConfig: DataTypeFieldConfig, childConfig?: Children, options?: UdfTypeOptions): DuckDBType;
+export type InputConverter = (value: unknown) => unknown;
+/**
+ * The inverse of `makeValueConverter`: converts the value DuckDB passes to a scalar function into
+ * the value `DataFrame` would pass to the same function.
+ *
+ * - A `Date` becomes what `DateVector` holds: epoch millis, or an `[ms, offset]` tuple. No string
+ *   parsing is involved, so the result does not depend on the host time zone.
+ * - A LIST becomes a JS array, and a STRUCT becomes an object (or, for a Tuple, an array).
+ * - A GEOMETRY becomes the GeoShape its WKB encodes.
+ * - A JSON column (`Any` or an untyped `Object`) is stored as JSON text, so it is parsed.
+ * - A `Long` stays a `bigint`, which the functions already expect.
+*/
+export declare function makeInputConverter(field: FieldTypeLike | DataTypeFieldConfig, childConfig?: Children): InputConverter;
+export {};
+//# sourceMappingURL=udf-types.d.ts.map

package/dist/src/teal-frame/udf-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"udf-types.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/udf-types.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,UAAU,EAGb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACH,mBAAmB,EAAE,cAAc,EAAE,sBAAsB,EAAE,SAAS,EAAE,mBAAmB,EAC9F,MAAM,kBAAkB,CAAC;AAO1B;;;;;;;;;;;EAWE;AAEF;;;EAGE;AACF,KAAK,aAAa,GAAG,SAAS,GAAG,mBAAmB,CAAC;AAErD,KAAK,QAAQ,GAAG,cAAc,GAAG,sBAAsB,CAAC;AA0CxD,MAAM,WAAW,cAAc;IAC3B;;;;MAIE;IACF,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB;AAOD,yFAAyF;AACzF,wBAAgB,UAAU,CACtB,WAAW,EAAE,mBAAmB,EAChC,WAAW,CAAC,EAAE,QAAQ,EACtB,OAAO,GAAE,cAAmB,GAC7B,UAAU,CA+BZ;AAED,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;AAkDzD;;;;;;;;;;EAUE;AACF,wBAAgB,kBAAkB,CAC9B,KAAK,EAAE,aAAa,GAAG,mBAAmB,EAC1C,WAAW,CAAC,EAAE,QAAQ,GACvB,cAAc,CA0ChB"}

package/dist/src/teal-frame/udf-types.js

@@ -0,0 +1,157 @@
+import { DuckDBGeometryValue, DuckDBVarCharType, BOOLEAN, TINYINT, SMALLINT, BIGINT, HUGEINT, DOUBLE, VARCHAR, GEOMETRY, LIST, STRUCT, } from '@duckdb/node-api';
+import { FieldType } from '@terascope/types';
+import { getChildDataTypeConfig } from '../core/utils.js';
+import { wkbToGeoShape } from './wkb.js';
+import { parseJSONText } from './plain-values.js';
+import { isJSONColumn } from './row-shape.js';
+import { DATE_TYPE, fromDateValue } from './date-value.js';
+/**
+ * DuckDB's `JSON` type is a VARCHAR with an alias, registered by the json extension. The binding
+ * exports no constant for it, so it is created here.
+*/
+const JSON_TYPE = DuckDBVarCharType.create('JSON');
+/** A GeoPoint is `STRUCT(lat DOUBLE, lon DOUBLE)`, as `geo-point.ts` declares it. */
+const GEO_POINT_TYPE = STRUCT({ lat: DOUBLE, lon: DOUBLE });
+const SCALAR_TYPES = {
+    [FieldType.Boolean]: BOOLEAN,
+    [FieldType.Byte]: TINYINT,
+    [FieldType.Short]: SMALLINT,
+    [FieldType.Integer]: BIGINT,
+    [FieldType.Long]: BIGINT,
+    [FieldType.Float]: DOUBLE,
+    [FieldType.Double]: DOUBLE,
+    [FieldType.Number]: DOUBLE,
+    [FieldType.Vector]: DOUBLE,
+    [FieldType.Keyword]: VARCHAR,
+    [FieldType.KeywordCaseInsensitive]: VARCHAR,
+    [FieldType.KeywordTokens]: VARCHAR,
+    [FieldType.KeywordTokensCaseInsensitive]: VARCHAR,
+    [FieldType.KeywordPathAnalyzer]: VARCHAR,
+    [FieldType.NgramTokens]: VARCHAR,
+    [FieldType.Text]: VARCHAR,
+    [FieldType.String]: VARCHAR,
+    [FieldType.Domain]: VARCHAR,
+    [FieldType.Hostname]: VARCHAR,
+    [FieldType.IP]: VARCHAR,
+    [FieldType.IPRange]: VARCHAR,
+    [FieldType.Binary]: VARCHAR,
+    [FieldType.Date]: DATE_TYPE,
+    [FieldType.GeoPoint]: GEO_POINT_TYPE,
+    [FieldType.Geo]: GEO_POINT_TYPE,
+    [FieldType.Boundary]: LIST(GEO_POINT_TYPE),
+    [FieldType.GeoJSON]: GEOMETRY,
+    [FieldType.Any]: JSON_TYPE,
+};
+/** The direct members of an Object or Tuple; dotted descendants are skipped. */
+function members(childConfig) {
+    return Object.entries(childConfig).filter(([name]) => !name.includes('.'));
+}
+/** Returns the DuckDB type object for a field config, or throws if the type has none. */
+export function udfTypeFor(fieldConfig, childConfig, options = {}) {
+    if (fieldConfig.array) {
+        return LIST(udfTypeFor({ ...fieldConfig, array: false }, childConfig, options));
+    }
+    const type = fieldConfig.type;
+    if (type === FieldType.Object || type === FieldType.Tuple) {
+        // no declared shape, so the column is JSON (as in `object.ts` and `tuple-type.ts`)
+        if (isJSONColumn(type, childConfig))
+            return JSON_TYPE;
+        return STRUCT(Object.fromEntries(members(childConfig).map(([name, config]) => [
+            name,
+            udfTypeFor(config, getChildDataTypeConfig(childConfig, name, config.type), options),
+        ])));
+    }
+    if (options.wide && (type === FieldType.Long || type === FieldType.Integer))
+        return HUGEINT;
+    const mapped = SCALAR_TYPES[type];
+    if (mapped == null) {
+        throw new TypeError(`Field type "${type}" cannot be a scalar function parameter or result`
+            + ' - it has no DuckDB type object mapping');
+    }
+    return mapped;
+}
+const identity = (value) => value;
+/** The plain items of a LIST value; `DuckDBListValue` carries them as `items`. */
+function listItems(value) {
+    const items = value?.items;
+    return Array.isArray(items) ? items : undefined;
+}
+/** The members of a STRUCT value; `DuckDBStructValue` carries them as `entries`. */
+function structEntries(value) {
+    const entries = value?.entries;
+    return entries != null && typeof entries === 'object'
+        ? entries
+        : undefined;
+}
+/**
+ * Builds the converter for a STRUCT value. An Object becomes an object; a Tuple becomes an array
+ * in position order, which is how `DataFrame` holds it.
+*/
+function structConverter(type, childConfig, declared) {
+    const converters = declared.map(([name, config]) => ({
+        name,
+        convert: makeInputConverter(config, getChildDataTypeConfig(childConfig, name, config.type)),
+    }));
+    if (type === FieldType.Tuple) {
+        converters.sort((a, b) => Number(a.name) - Number(b.name));
+    }
+    return (value) => {
+        const entries = structEntries(value);
+        if (entries == null)
+            return value;
+        if (type === FieldType.Tuple) {
+            return converters.map(({ name, convert }) => convert(entries[name] ?? null));
+        }
+        return Object.fromEntries(converters.map(({ name, convert }) => [name, convert(entries[name] ?? null)]));
+    };
+}
+/**
+ * The inverse of `makeValueConverter`: converts the value DuckDB passes to a scalar function into
+ * the value `DataFrame` would pass to the same function.
+ *
+ * - A `Date` becomes what `DateVector` holds: epoch millis, or an `[ms, offset]` tuple. No string
+ *   parsing is involved, so the result does not depend on the host time zone.
+ * - A LIST becomes a JS array, and a STRUCT becomes an object (or, for a Tuple, an array).
+ * - A GEOMETRY becomes the GeoShape its WKB encodes.
+ * - A JSON column (`Any` or an untyped `Object`) is stored as JSON text, so it is parsed.
+ * - A `Long` stays a `bigint`, which the functions already expect.
+*/
+export function makeInputConverter(field, childConfig) {
+    const fieldConfig = typeof field === 'string'
+        ? { type: field }
+        : field;
+    const type = fieldConfig.type;
+    if (fieldConfig.array) {
+        const element = makeInputConverter({ ...fieldConfig, array: false }, childConfig);
+        return (value) => {
+            const items = listItems(value);
+            if (items == null)
+                return value;
+            return items.map((item) => (item == null ? item : element(item)));
+        };
+    }
+    // epoch millis, or a `DateTuple`, as `DataFrame` passes to the same function
+    if (type === FieldType.Date)
+        return fromDateValue;
+    if (type === FieldType.GeoJSON) {
+        return (value) => (value instanceof DuckDBGeometryValue
+            ? wkbToGeoShape(value.bytes)
+            : value);
+    }
+    if (isJSONColumn(type, childConfig))
+        return parseJSONText;
+    if (type === FieldType.Object || type === FieldType.Tuple) {
+        return structConverter(type, childConfig, members(childConfig));
+    }
+    if (type === FieldType.GeoPoint || type === FieldType.Geo) {
+        return (value) => structEntries(value) ?? value;
+    }
+    if (type === FieldType.Boundary) {
+        return (value) => {
+            const items = listItems(value);
+            return items == null ? value : items.map((item) => structEntries(item) ?? item);
+        };
+    }
+    return identity;
+}
+//# sourceMappingURL=udf-types.js.map

package/dist/src/teal-frame/udf-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"udf-types.js","sourceRoot":"","sources":["../../../src/teal-frame/udf-types.ts"],"names":[],"mappings":"AAAA,OAAO,EACS,mBAAmB,EAAE,iBAAiB,EAClD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EACtE,IAAI,EAAE,MAAM,GACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAC0D,SAAS,EACzE,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAuB3D;;;EAGE;AACF,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAEnD,qFAAqF;AACrF,MAAM,cAAc,GAAG,MAAM,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;AAE5D,MAAM,YAAY,GAAqD;IACnE,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO;IAC5B,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO;IACzB,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,QAAQ;IAC3B,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM;IAC3B,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM;IACxB,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM;IACzB,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM;IAC1B,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM;IAC1B,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM;IAC1B,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO;IAC5B,CAAC,SAAS,CAAC,sBAAsB,CAAC,EAAE,OAAO;IAC3C,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,OAAO;IAClC,CAAC,SAAS,CAAC,4BAA4B,CAAC,EAAE,OAAO;IACjD,CAAC,SAAS,CAAC,mBAAmB,CAAC,EAAE,OAAO;IACxC,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,OAAO;IAChC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO;IACzB,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,OAAO;IAC3B,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,OAAO;IAC3B,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,OAAO;IAC7B,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,OAAO;IACvB,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO;IAC5B,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,OAAO;IAC3B,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS;IAC3B,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,cAAc;IACpC,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,cAAc;IAC/B,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC;IAC1C,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,QAAQ;IAC7B,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,SAAS;CAC7B,CAAC;AAWF,gFAAgF;AAChF,SAAS,OAAO,CAAC,WAAqB;IAClC,OAAO,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/E,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,UAAU,CACtB,WAAgC,EAChC,WAAsB,EACtB,UAA0B,EAAE;IAE5B,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;QACpB,OAAO,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,IAAqB,CAAC;IAE/C,IAAI,IAAI,KAAK,SAAS,CAAC,MAAM,IAAI,IAAI,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC;QACxD,mFAAmF;QACnF,IAAI,YAAY,CAAC,IAAiB,EAAE,WAAW,CAAC;YAAE,OAAO,SAAS,CAAC;QAEnE,OAAO,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,WAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC;YAC3E,IAAI;YACJ,UAAU,CACN,MAAM,EACN,sBAAsB,CAAC,WAAY,EAAE,IAAI,EAAE,MAAM,CAAC,IAAiB,CAAC,EACpE,OAAO,CACV;SACJ,CAAC,CAAC,CAAC,CAAC;IACT,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,IAAI,IAAI,KAAK,SAAS,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAC;IAE5F,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACjB,MAAM,IAAI,SAAS,CACf,eAAe,IAAI,mDAAmD;cACpE,yCAAyC,CAC9C,CAAC;IACN,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAID,MAAM,QAAQ,GAAmB,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;AAElD,kFAAkF;AAClF,SAAS,SAAS,CAAC,KAAc;IAC7B,MAAM,KAAK,GAAI,KAAoC,EAAE,KAAK,CAAC;IAC3D,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACpD,CAAC;AAED,oFAAoF;AACpF,SAAS,aAAa,CAAC,KAAc;IACjC,MAAM,OAAO,GAAI,KAAsC,EAAE,OAAO,CAAC;IACjE,OAAO,OAAO,IAAI,IAAI,IAAI,OAAO,OAAO,KAAK,QAAQ;QACjD,CAAC,CAAC,OAAkC;QACpC,CAAC,CAAC,SAAS,CAAC;AACpB,CAAC;AAED;;;EAGE;AACF,SAAS,eAAe,CACpB,IAAmB,EACnB,WAAqB,EACrB,QAAuD;IAEvD,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QACjD,IAAI;QACJ,OAAO,EAAE,kBAAkB,CACvB,MAAM,EACN,sBAAsB,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,IAAiB,CAAC,CACtE;KACJ,CAAC,CAAC,CAAC;IACJ,IAAI,IAAI,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC;QAC3B,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,CAAC,KAAK,EAAE,EAAE;QACb,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,OAAO,IAAI,IAAI;YAAE,OAAO,KAAK,CAAC;QAClC,IAAI,IAAI,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC;YAC3B,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;QACjF,CAAC;QACD,OAAO,MAAM,CAAC,WAAW,CACrB,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAChF,CAAC;IACN,CAAC,CAAC;AACN,CAAC;AAED;;;;;;;;;;EAUE;AACF,MAAM,UAAU,kBAAkB,CAC9B,KAA0C,EAC1C,WAAsB;IAEtB,MAAM,WAAW,GAAwB,OAAO,KAAK,KAAK,QAAQ;QAC9D,CAAC,CAAC,EAAE,IAAI,EAAE,KAAkB,EAAE;QAC9B,CAAC,CAAC,KAAK,CAAC;IACZ,MAAM,IAAI,GAAG,WAAW,CAAC,IAAqB,CAAC;IAE/C,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,kBAAkB,CAAC,EAAE,GAAG,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,WAAW,CAAC,CAAC;QAClF,OAAO,CAAC,KAAK,EAAE,EAAE;YACb,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;YAC/B,IAAI,KAAK,IAAI,IAAI;gBAAE,OAAO,KAAK,CAAC;YAChC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtE,CAAC,CAAC;IACN,CAAC;IAED,6EAA6E;IAC7E,IAAI,IAAI,KAAK,SAAS,CAAC,IAAI;QAAE,OAAO,aAAa,CAAC;IAElD,IAAI,IAAI,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,YAAY,mBAAmB;YACnD,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC;YAC5B,CAAC,CAAC,KAAK,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,YAAY,CAAC,IAAiB,EAAE,WAAW,CAAC;QAAE,OAAO,aAAa,CAAC;IAEvE,IAAI,IAAI,KAAK,SAAS,CAAC,MAAM,IAAI,IAAI,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC;QACxD,OAAO,eAAe,CAAC,IAAI,EAAE,WAAY,EAAE,OAAO,CAAC,WAAY,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,CAAC,QAAQ,IAAI,IAAI,KAAK,SAAS,CAAC,GAAG,EAAE,CAAC;QACxD,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;IACpD,CAAC;IAED,IAAI,IAAI,KAAK,SAAS,CAAC,QAAQ,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,EAAE,EAAE;YACb,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;YAC/B,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QACpF,CAAC,CAAC;IACN,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC"}

package/dist/src/teal-frame/wkb.d.ts

@@ -0,0 +1,6 @@
+import { GeoShape } from '@terascope/types';
+/** Encodes a GeoShape as little-endian ISO WKB, the same bytes DuckDB's `ST_AsWKB` writes. */
+export declare function geoShapeToWKB(shape: GeoShape): Uint8Array;
+/** Decodes WKB of a Point, Polygon or MultiPolygon, in either byte order, to a GeoShape. */
+export declare function wkbToGeoShape(bytes: Uint8Array): GeoShape;
+//# sourceMappingURL=wkb.d.ts.map

package/dist/src/teal-frame/wkb.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wkb.d.ts","sourceRoot":"","sources":["../../../src/teal-frame/wkb.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EACX,MAAM,kBAAkB,CAAC;AA8B1B,8FAA8F;AAC9F,wBAAgB,aAAa,CAAC,KAAK,EAAE,QAAQ,GAAG,UAAU,CAUzD;AAED,4FAA4F;AAC5F,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,QAAQ,CAEzD"}

package/dist/src/teal-frame/wkb.js

@@ -0,0 +1,167 @@
+import { GeoShapeType } from '@terascope/types';
+/**
+ * Converts between GeoJSON and WKB for `GeoJSON` fields, which are stored as DuckDB `GEOMETRY`.
+ *
+ * The node binding can only write a `GEOMETRY` value as WKB bytes through a data-chunk vector
+ * (DuckDB's C API has no geometry constructor). So ingest encodes here and `rows()` decodes here.
+ *
+ * Output is ISO WKB, 2D, little-endian; input may use either byte order. Only Point, Polygon and
+ * MultiPolygon are supported, the shapes `toGeoJSONOrThrow` produces. Any other type throws a
+ * `TypeError`, which ingest records as a coercion failure.
+*/
+const WKB_POINT = 1;
+const WKB_POLYGON = 3;
+const WKB_MULTI_POLYGON = 6;
+/** WKB type codes keyed by lowercase `GeoShapeType`, so Elasticsearch's `'polygon'` matches. */
+const SHAPE_TYPES = {
+    point: WKB_POINT,
+    polygon: WKB_POLYGON,
+    multipolygon: WKB_MULTI_POLYGON,
+};
+const HEADER_BYTES = 5; // byte order + uint32 type
+const POINT_BYTES = 16; // two float64
+/** Encodes a GeoShape as little-endian ISO WKB, the same bytes DuckDB's `ST_AsWKB` writes. */
+export function geoShapeToWKB(shape) {
+    const code = SHAPE_TYPES[String(shape?.type).toLowerCase()];
+    if (code == null) {
+        throw new TypeError(`Cannot store a geometry of type ${JSON.stringify(shape?.type)}`);
+    }
+    const { coordinates } = shape;
+    if (code === WKB_POINT)
+        return writePoint(toPosition(coordinates));
+    if (code === WKB_POLYGON)
+        return writePolygon(toPolygon(coordinates));
+    return writeMultiPolygon(toList(coordinates, 'a MultiPolygon').map(toPolygon));
+}
+/** Decodes WKB of a Point, Polygon or MultiPolygon, in either byte order, to a GeoShape. */
+export function wkbToGeoShape(bytes) {
+    return new WKBReader(bytes).readGeometry();
+}
+function writePoint(position) {
+    const out = new DataView(new ArrayBuffer(HEADER_BYTES + POINT_BYTES));
+    writeHeader(out, 0, WKB_POINT);
+    writePosition(out, HEADER_BYTES, position);
+    return new Uint8Array(out.buffer);
+}
+function polygonBytes(polygon) {
+    return HEADER_BYTES + 4
+        + polygon.reduce((sum, ring) => sum + 4 + ring.length * POINT_BYTES, 0);
+}
+function writePolygon(polygon) {
+    const out = new DataView(new ArrayBuffer(polygonBytes(polygon)));
+    writePolygonAt(out, 0, polygon);
+    return new Uint8Array(out.buffer);
+}
+function writeMultiPolygon(polygons) {
+    const size = HEADER_BYTES + 4 + polygons.reduce((sum, p) => sum + polygonBytes(p), 0);
+    const out = new DataView(new ArrayBuffer(size));
+    writeHeader(out, 0, WKB_MULTI_POLYGON);
+    out.setUint32(HEADER_BYTES, polygons.length, true);
+    let offset = HEADER_BYTES + 4;
+    for (const polygon of polygons)
+        offset = writePolygonAt(out, offset, polygon);
+    return new Uint8Array(out.buffer);
+}
+/** Writes one polygon at `offset`, returning the offset after it. */
+function writePolygonAt(out, start, polygon) {
+    writeHeader(out, start, WKB_POLYGON);
+    out.setUint32(start + HEADER_BYTES, polygon.length, true);
+    let offset = start + HEADER_BYTES + 4;
+    for (const ring of polygon) {
+        out.setUint32(offset, ring.length, true);
+        offset += 4;
+        for (const position of ring) {
+            writePosition(out, offset, position);
+            offset += POINT_BYTES;
+        }
+    }
+    return offset;
+}
+function writeHeader(out, offset, code) {
+    out.setUint8(offset, 1); // little-endian
+    out.setUint32(offset + 1, code, true);
+}
+function writePosition(out, offset, [x, y]) {
+    out.setFloat64(offset, x, true);
+    out.setFloat64(offset + 8, y, true);
+}
+function toList(value, what) {
+    if (!Array.isArray(value))
+        throw new TypeError(`Expected ${what} to be an array`);
+    return value;
+}
+function toPolygon(value) {
+    return toList(value, 'a Polygon').map((ring) => toList(ring, 'a ring').map(toPosition));
+}
+/**
+ * Validates a 2D position. A third ordinate is rejected rather than dropped, so a different
+ * shape is never stored silently.
+*/
+function toPosition(value) {
+    const position = toList(value, 'a position');
+    if (position.length !== 2) {
+        throw new TypeError(`Expected a 2D position, got ${position.length} ordinates`);
+    }
+    const [x, y] = position;
+    if (typeof x !== 'number' || typeof y !== 'number') {
+        throw new TypeError(`Expected numeric ordinates, got ${JSON.stringify(position)}`);
+    }
+    return [x, y];
+}
+/** A cursor over WKB bytes; each geometry carries its own byte order. */
+class WKBReader {
+    view;
+    offset = 0;
+    constructor(bytes) {
+        this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    }
+    readGeometry() {
+        const little = this.view.getUint8(this.offset) === 1;
+        const code = this.view.getUint32(this.offset + 1, little);
+        this.offset += HEADER_BYTES;
+        if (code === WKB_POINT) {
+            const point = {
+                type: GeoShapeType.Point, coordinates: this.readPosition(little),
+            };
+            return point;
+        }
+        if (code === WKB_POLYGON) {
+            const polygon = {
+                type: GeoShapeType.Polygon, coordinates: this.readPolygonBody(little),
+            };
+            return polygon;
+        }
+        if (code === WKB_MULTI_POLYGON) {
+            const count = this.readCount(little);
+            const coordinates = Array.from({ length: count }, () => {
+                const inner = this.readGeometry();
+                if (inner.type !== GeoShapeType.Polygon) {
+                    throw new TypeError('A MultiPolygon member must be a Polygon');
+                }
+                return inner.coordinates;
+            });
+            const multi = { type: GeoShapeType.MultiPolygon, coordinates };
+            return multi;
+        }
+        throw new TypeError(`WKB geometry type ${code} is not a Point, Polygon or MultiPolygon`);
+    }
+    readPolygonBody(little) {
+        const rings = this.readCount(little);
+        return Array.from({ length: rings }, () => {
+            const points = this.readCount(little);
+            return Array.from({ length: points }, () => this.readPosition(little));
+        });
+    }
+    readCount(little) {
+        const count = this.view.getUint32(this.offset, little);
+        this.offset += 4;
+        return count;
+    }
+    readPosition(little) {
+        const x = this.view.getFloat64(this.offset, little);
+        const y = this.view.getFloat64(this.offset + 8, little);
+        this.offset += POINT_BYTES;
+        return [x, y];
+    }
+}
+//# sourceMappingURL=wkb.js.map

package/dist/src/teal-frame/wkb.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wkb.js","sourceRoot":"","sources":["../../../src/teal-frame/wkb.ts"],"names":[],"mappings":"AAAA,OAAO,EACO,YAAY,EACzB,MAAM,kBAAkB,CAAC;AAE1B;;;;;;;;;EASE;AAEF,MAAM,SAAS,GAAG,CAAC,CAAC;AACpB,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,gGAAgG;AAChG,MAAM,WAAW,GAAqC;IAClD,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,WAAW;IACpB,YAAY,EAAE,iBAAiB;CAClC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,2BAA2B;AACnD,MAAM,WAAW,GAAG,EAAE,CAAC,CAAC,cAAc;AAKtC,8FAA8F;AAC9F,MAAM,UAAU,aAAa,CAAC,KAAe;IACzC,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAC5D,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;QACf,MAAM,IAAI,SAAS,CAAC,mCAAmC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,MAAM,EAAE,WAAW,EAAE,GAAG,KAAiC,CAAC;IAC1D,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,UAAU,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACnE,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,YAAY,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC;IACtE,OAAO,iBAAiB,CAAC,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,aAAa,CAAC,KAAiB;IAC3C,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,YAAY,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,UAAU,CAAC,QAAyB;IACzC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,IAAI,WAAW,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC;IACtE,WAAW,CAAC,GAAG,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;IAC/B,aAAa,CAAC,GAAG,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;IAC3C,OAAO,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,YAAY,CAAC,OAAgB;IAClC,OAAO,YAAY,GAAG,CAAC;UACjB,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,YAAY,CAAC,OAAgB;IAClC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,IAAI,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACjE,cAAc,CAAC,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IAChC,OAAO,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,iBAAiB,CAAC,QAA4B;IACnD,MAAM,IAAI,GAAG,YAAY,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACtF,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;IAChD,WAAW,CAAC,GAAG,EAAE,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACvC,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAEnD,IAAI,MAAM,GAAG,YAAY,GAAG,CAAC,CAAC;IAC9B,KAAK,MAAM,OAAO,IAAI,QAAQ;QAAE,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9E,OAAO,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,qEAAqE;AACrE,SAAS,cAAc,CAAC,GAAa,EAAE,KAAa,EAAE,OAAgB;IAClE,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;IACrC,GAAG,CAAC,SAAS,CAAC,KAAK,GAAG,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAE1D,IAAI,MAAM,GAAG,KAAK,GAAG,YAAY,GAAG,CAAC,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QACzB,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,CAAC;QACZ,KAAK,MAAM,QAAQ,IAAI,IAAI,EAAE,CAAC;YAC1B,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YACrC,MAAM,IAAI,WAAW,CAAC;QAC1B,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,GAAa,EAAE,MAAc,EAAE,IAAY;IAC5D,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,gBAAgB;IACzC,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,aAAa,CAAC,GAAa,EAAE,MAAc,EAAE,CAAC,CAAC,EAAE,CAAC,CAAkB;IACzE,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAChC,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,MAAM,CAAC,KAAc,EAAE,IAAY;IACxC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,YAAY,IAAI,iBAAiB,CAAC,CAAC;IAClF,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC7B,OAAO,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED;;;EAGE;AACF,SAAS,UAAU,CAAC,KAAc;IAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;IAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,SAAS,CAAC,+BAA+B,QAAQ,CAAC,MAAM,YAAY,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC;IACxB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QACjD,MAAM,IAAI,SAAS,CAAC,mCAAmC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,yEAAyE;AACzE,MAAM,SAAS;IACM,IAAI,CAAW;IAExB,MAAM,GAAG,CAAC,CAAC;IAEnB,YAAY,KAAiB;QACzB,IAAI,CAAC,IAAI,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAC/E,CAAC;IAED,YAAY;QACR,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC;QAE5B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,MAAM,KAAK,GAAkB;gBACzB,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;aACnE,CAAC;YACF,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YACvB,MAAM,OAAO,GAAoB;gBAC7B,IAAI,EAAE,YAAY,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;aACxE,CAAC;YACF,OAAO,OAAO,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE;gBACnD,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC,OAAO,EAAE,CAAC;oBACtC,MAAM,IAAI,SAAS,CAAC,yCAAyC,CAAC,CAAC;gBACnE,CAAC;gBACD,OAAO,KAAK,CAAC,WAAW,CAAC;YAC7B,CAAC,CAAC,CAAC;YACH,MAAM,KAAK,GAAyB,EAAE,IAAI,EAAE,YAAY,CAAC,YAAY,EAAE,WAAW,EAAE,CAAC;YACrF,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,IAAI,SAAS,CAAC,qBAAqB,IAAI,0CAA0C,CAAC,CAAC;IAC7F,CAAC;IAEO,eAAe,CAAC,MAAe;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE;YACtC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACtC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,SAAS,CAAC,MAAe;QAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QACjB,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,YAAY,CAAC,MAAe;QAChC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACpD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC;QAC3B,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAClB,CAAC;CACJ"}