@terascope/data-mate 2.23.1 → 2.25.0

package/dist/src/function-configs/sql-helpers.d.ts

@@ -0,0 +1,115 @@
+import { FieldType } from '@terascope/types';
+/**
+ * Helpers shared by the `sql` expressions on the function configs.
+ *
+ * Each one handles a place where DuckDB and JavaScript behave differently, so the fix is written
+ * once.
+*/
+/**
+ * A SQL string literal. Re-exported from `@terascope/sql-builder` so that one function decides how
+ * a value becomes a literal.
+*/
+export { quoteLiteral as sqlLiteral } from '@terascope/sql-builder';
+/**
+ * Every code point `String.prototype.trim` strips: ECMAScript `WhiteSpace` and `LineTerminator`.
+ *
+ * In order: TAB, LF, VT, FF, CR, SPACE, NBSP, the `Zs` category (OGHAM SPACE MARK, EN QUAD through
+ * HAIR SPACE, NARROW NO-BREAK SPACE, MEDIUM MATHEMATICAL SPACE, IDEOGRAPHIC SPACE), the line and
+ * paragraph separators, and ZERO WIDTH NO-BREAK SPACE.
+ *
+ * DuckDB's one-argument `trim(x)` strips only spaces, so `TAB + " x " + LF` is `"x"` in
+ * JavaScript and unchanged in SQL. Passing this set as the second argument makes them agree.
+ *
+ * U+0085 (NEL) is left out on purpose: it is category Cc, so JavaScript does not trim it.
+*/
+export declare const JS_WHITESPACE: string;
+/**
+ * True when every code point of `value` is below 128.
+ *
+ * `length()` counts characters and `strlen()` counts bytes, so they are equal exactly when the
+ * string is ASCII. (`octet_length` has no VARCHAR overload.)
+ *
+ * Case conversion depends on this guard. JavaScript uses full Unicode case mapping and DuckDB uses
+ * simple mapping, so they disagree outside ASCII: `'ß'.toUpperCase()` is `'SS'` in JavaScript and
+ * `'ẞ'` in SQL, and `lower('İstanbul')` keeps a combining dot in JavaScript but drops it in SQL.
+ * Within ASCII they agree.
+*/
+export declare function isAsciiSql(value: string): string;
+/**
+ * `runMathFn`'s rule in SQL: a non-finite result becomes null.
+ *
+ * Every numeric transform goes through `runMathFn`, which returns null when the result is `NaN` or
+ * infinite, so `log(0)` and an overflow give null. A bare SQL call could return `nan` or `-inf`
+ * instead.
+*/
+export declare function finiteOrNull(expression: string): string;
+/**
+ * Starts a `CASE` that uses `native` when `value` fits in an `Integer` output. The caller adds
+ * the `ELSE` branch.
+ *
+ * Used by `ceil`, `floor` and `round`, whose `output_type` is `Integer`. Outside that range the
+ * UDF path returns a BIGINT rendered as a string while SQL returns the true value, so those values
+ * go to the UDF to keep the results identical.
+ *
+ * `+ 0` turns `-0` into `0`, as the UDF path's conversion does.
+*/
+export declare function withinIntegerRange(value: string, native: string): string;
+/**
+ * Values whose reversal needs grapheme segmentation, as an RE2 class: an astral code point, a
+ * zero-width joiner, or a combining mark. The same set as `NEEDS_SEGMENTATION` in `reverse.ts`.
+*/
+export declare const NEEDS_GRAPHEME_REVERSE = "[\\p{M}\\x{200D}\\x{10000}-\\x{10FFFF}]";
+/**
+ * Returns null instead of calling `native` when `condition` is false.
+ *
+ * DuckDB throws where JavaScript returns NaN: `sqrt(-1)` raises `Out of Range Error`, and `ln(0)`
+ * and `ln(-1)` raise errors too, while `Math.sqrt(-1)` is `NaN` and `runMathFn` turns it into
+ * null. An unguarded call would abort the query, so the check must come before the call.
+ * `finiteOrNull` checks the result afterwards, which is too late.
+*/
+export declare function inDomain(condition: string, native: string): string;
+/**
+ * Values that contain a code point outside the BMP, as an RE2 class.
+ *
+ * A guard for functions whose JavaScript implementation counts UTF-16 code units while SQL counts
+ * characters. `truncate` with size 3 over four thumbs-up emoji returns `'\u{1F44D}\ud83d'` (one
+ * emoji and a lone surrogate) in JavaScript, where `substring` returns three whole emoji.
+ * `isLength` with size 5 counts five emoji as length 10. SQL cannot reproduce either, so astral
+ * input goes to the UDF.
+*/
+export declare const HAS_ASTRAL = "[\\x{10000}-\\x{10FFFF}]";
+/**
+ * An `applies` for an expression that inserts all of these arguments as numbers.
+ *
+ * Use it when the arguments are required, as for `add`, `pow` and `truncate`. `subtract` uses it
+ * too: it does not declare its argument required, but its expression cannot handle a missing one.
+*/
+export declare function needsNumericArgs<T extends Record<string, any>>(...names: readonly string[]): (args: T) => boolean;
+/**
+ * An `applies` for optional arguments that must be numbers when present.
+ *
+ * `inNumberRange` and `isLength` build their expression from whichever bounds were given, so only
+ * a present non-number is a problem.
+*/
+export declare function allowsNumericArgs<T extends Record<string, any>>(...names: readonly string[]): (args: T) => boolean;
+/** The string-shaped field types, as `validateAccepts` treats them. */
+export declare const STRING_FIELD_TYPES: readonly FieldType[];
+/** The numeric field types. */
+export declare const NUMERIC_FIELD_TYPES: readonly FieldType[];
+/**
+ * A SQL literal for an argument declared `FieldType.Any`, or null when it has none.
+ *
+ * `setDefault` and `equals` take any value, but only primitives have a literal form. An object or
+ * array would need the structural comparison that `isDeepEqual` does, so it keeps the UDF; the
+ * caller's `applies` checks for null.
+*/
+export declare function primitiveLiteral(value: unknown): string | null;
+/**
+ * Whether a `FieldType.Any` argument is the same kind of value as the column it is compared or
+ * coalesced with.
+ *
+ * SQL allows mixing them (`coalesce(varchar_col, 5)` casts), but DuckDB's cast and JavaScript's
+ * may not turn `5` into the same string. Only same-kind pairs run as SQL; the rest keep the UDF.
+*/
+export declare function literalMatchesColumn(value: unknown, type: FieldType): boolean;
+//# sourceMappingURL=sql-helpers.d.ts.map

package/dist/src/function-configs/sql-helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql-helpers.d.ts","sourceRoot":"","sources":["../../../src/function-configs/sql-helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7C;;;;;EAKE;AAEF;;;EAGE;AACF,OAAO,EAAE,YAAY,IAAI,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpE;;;;;;;;;;;EAWE;AACF,eAAO,MAAM,aAAa,QAGA,CAAC;AAE3B;;;;;;;;;;EAUE;AACF,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED;;;;;;EAME;AACF,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;;;;;;EASE;AACF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAExE;AAED;;;EAGE;AACF,eAAO,MAAM,sBAAsB,4CAA4C,CAAC;AAEhF;;;;;;;EAOE;AACF,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAElE;AAED;;;;;;;;EAQE;AACF,eAAO,MAAM,UAAU,6BAA6B,CAAC;AAcrD;;;;;EAKE;AACF,wBAAgB,gBAAgB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC1D,GAAG,KAAK,EAAE,SAAS,MAAM,EAAE,GAC5B,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAEtB;AAED;;;;;EAKE;AACF,wBAAgB,iBAAiB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3D,GAAG,KAAK,EAAE,SAAS,MAAM,EAAE,GAC5B,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAEtB;AAED,uEAAuE;AACvE,eAAO,MAAM,kBAAkB,EAAE,SAAS,SAAS,EAElD,CAAC;AAEF,+BAA+B;AAC/B,eAAO,MAAM,mBAAmB,EAAE,SAAS,SAAS,EAQnD,CAAC;AAEF;;;;;;EAME;AACF,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAK9D;AAED;;;;;;EAME;AACF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,CAK7E"}

package/dist/src/function-configs/sql-helpers.js

@@ -0,0 +1,169 @@
+import { FieldType } from '@terascope/types';
+// imported as well as re-exported: `primitiveLiteral` below needs it as a local binding
+import { quoteLiteral } from '@terascope/sql-builder';
+/**
+ * Helpers shared by the `sql` expressions on the function configs.
+ *
+ * Each one handles a place where DuckDB and JavaScript behave differently, so the fix is written
+ * once.
+*/
+/**
+ * A SQL string literal. Re-exported from `@terascope/sql-builder` so that one function decides how
+ * a value becomes a literal.
+*/
+export { quoteLiteral as sqlLiteral } from '@terascope/sql-builder';
+/**
+ * Every code point `String.prototype.trim` strips: ECMAScript `WhiteSpace` and `LineTerminator`.
+ *
+ * In order: TAB, LF, VT, FF, CR, SPACE, NBSP, the `Zs` category (OGHAM SPACE MARK, EN QUAD through
+ * HAIR SPACE, NARROW NO-BREAK SPACE, MEDIUM MATHEMATICAL SPACE, IDEOGRAPHIC SPACE), the line and
+ * paragraph separators, and ZERO WIDTH NO-BREAK SPACE.
+ *
+ * DuckDB's one-argument `trim(x)` strips only spaces, so `TAB + " x " + LF` is `"x"` in
+ * JavaScript and unchanged in SQL. Passing this set as the second argument makes them agree.
+ *
+ * U+0085 (NEL) is left out on purpose: it is category Cc, so JavaScript does not trim it.
+*/
+export const JS_WHITESPACE = ''
+    + '\u0009\u000a\u000b\u000c\u000d\u0020\u00a0\u1680\u2000\u2001\u2002'
+    + '\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f'
+    + '\u205f\u3000\ufeff';
+/**
+ * True when every code point of `value` is below 128.
+ *
+ * `length()` counts characters and `strlen()` counts bytes, so they are equal exactly when the
+ * string is ASCII. (`octet_length` has no VARCHAR overload.)
+ *
+ * Case conversion depends on this guard. JavaScript uses full Unicode case mapping and DuckDB uses
+ * simple mapping, so they disagree outside ASCII: `'ß'.toUpperCase()` is `'SS'` in JavaScript and
+ * `'ẞ'` in SQL, and `lower('İstanbul')` keeps a combining dot in JavaScript but drops it in SQL.
+ * Within ASCII they agree.
+*/
+export function isAsciiSql(value) {
+    return `strlen(${value}) = length(${value})`;
+}
+/**
+ * `runMathFn`'s rule in SQL: a non-finite result becomes null.
+ *
+ * Every numeric transform goes through `runMathFn`, which returns null when the result is `NaN` or
+ * infinite, so `log(0)` and an overflow give null. A bare SQL call could return `nan` or `-inf`
+ * instead.
+*/
+export function finiteOrNull(expression) {
+    return `CASE WHEN isfinite(${expression}) THEN ${expression} ELSE NULL END`;
+}
+/**
+ * Starts a `CASE` that uses `native` when `value` fits in an `Integer` output. The caller adds
+ * the `ELSE` branch.
+ *
+ * Used by `ceil`, `floor` and `round`, whose `output_type` is `Integer`. Outside that range the
+ * UDF path returns a BIGINT rendered as a string while SQL returns the true value, so those values
+ * go to the UDF to keep the results identical.
+ *
+ * `+ 0` turns `-0` into `0`, as the UDF path's conversion does.
+*/
+export function withinIntegerRange(value, native) {
+    return `CASE WHEN abs(${value}) <= 2147483647 THEN ${native} + 0`;
+}
+/**
+ * Values whose reversal needs grapheme segmentation, as an RE2 class: an astral code point, a
+ * zero-width joiner, or a combining mark. The same set as `NEEDS_SEGMENTATION` in `reverse.ts`.
+*/
+export const NEEDS_GRAPHEME_REVERSE = '[\\p{M}\\x{200D}\\x{10000}-\\x{10FFFF}]';
+/**
+ * Returns null instead of calling `native` when `condition` is false.
+ *
+ * DuckDB throws where JavaScript returns NaN: `sqrt(-1)` raises `Out of Range Error`, and `ln(0)`
+ * and `ln(-1)` raise errors too, while `Math.sqrt(-1)` is `NaN` and `runMathFn` turns it into
+ * null. An unguarded call would abort the query, so the check must come before the call.
+ * `finiteOrNull` checks the result afterwards, which is too late.
+*/
+export function inDomain(condition, native) {
+    return `CASE WHEN ${condition} THEN ${native} ELSE NULL END`;
+}
+/**
+ * Values that contain a code point outside the BMP, as an RE2 class.
+ *
+ * A guard for functions whose JavaScript implementation counts UTF-16 code units while SQL counts
+ * characters. `truncate` with size 3 over four thumbs-up emoji returns `'\u{1F44D}\ud83d'` (one
+ * emoji and a lone surrogate) in JavaScript, where `substring` returns three whole emoji.
+ * `isLength` with size 5 counts five emoji as length 10. SQL cannot reproduce either, so astral
+ * input goes to the UDF.
+*/
+export const HAS_ASTRAL = '[\\x{10000}-\\x{10FFFF}]';
+/**
+ * True for an argument that can be inserted into SQL as a numeric literal.
+ *
+ * `subtract` declares no `required_arguments`, so a call without `value` reaches the expression as
+ * `undefined`. `Number(undefined)` is `NaN`, which would be inserted as a bare identifier and fail
+ * with `Binder Error: Referenced column "NaN" not found`, while the UDF path returns `NaN` for
+ * every row. `Infinity` fails the same way, so a non-finite argument keeps the UDF.
+*/
+function isNumericArg(value) {
+    return typeof value === 'number' && Number.isFinite(value);
+}
+/**
+ * An `applies` for an expression that inserts all of these arguments as numbers.
+ *
+ * Use it when the arguments are required, as for `add`, `pow` and `truncate`. `subtract` uses it
+ * too: it does not declare its argument required, but its expression cannot handle a missing one.
+*/
+export function needsNumericArgs(...names) {
+    return (args) => names.every((name) => isNumericArg(args[name]));
+}
+/**
+ * An `applies` for optional arguments that must be numbers when present.
+ *
+ * `inNumberRange` and `isLength` build their expression from whichever bounds were given, so only
+ * a present non-number is a problem.
+*/
+export function allowsNumericArgs(...names) {
+    return (args) => names.every((name) => args[name] == null || isNumericArg(args[name]));
+}
+/** The string-shaped field types, as `validateAccepts` treats them. */
+export const STRING_FIELD_TYPES = [
+    FieldType.String, FieldType.Keyword, FieldType.Text, FieldType.NgramTokens,
+];
+/** The numeric field types. */
+export const NUMERIC_FIELD_TYPES = [
+    FieldType.Number,
+    FieldType.Byte,
+    FieldType.Short,
+    FieldType.Integer,
+    FieldType.Long,
+    FieldType.Float,
+    FieldType.Double,
+];
+/**
+ * A SQL literal for an argument declared `FieldType.Any`, or null when it has none.
+ *
+ * `setDefault` and `equals` take any value, but only primitives have a literal form. An object or
+ * array would need the structural comparison that `isDeepEqual` does, so it keeps the UDF; the
+ * caller's `applies` checks for null.
+*/
+export function primitiveLiteral(value) {
+    if (typeof value === 'string')
+        return quoteLiteral(value);
+    if (typeof value === 'boolean')
+        return value ? 'TRUE' : 'FALSE';
+    if (typeof value === 'number' && Number.isFinite(value))
+        return String(value);
+    return null;
+}
+/**
+ * Whether a `FieldType.Any` argument is the same kind of value as the column it is compared or
+ * coalesced with.
+ *
+ * SQL allows mixing them (`coalesce(varchar_col, 5)` casts), but DuckDB's cast and JavaScript's
+ * may not turn `5` into the same string. Only same-kind pairs run as SQL; the rest keep the UDF.
+*/
+export function literalMatchesColumn(value, type) {
+    if (typeof value === 'string')
+        return STRING_FIELD_TYPES.includes(type);
+    if (typeof value === 'boolean')
+        return type === FieldType.Boolean;
+    if (typeof value === 'number')
+        return NUMERIC_FIELD_TYPES.includes(type);
+    return false;
+}
+//# sourceMappingURL=sql-helpers.js.map

package/dist/src/function-configs/sql-helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql-helpers.js","sourceRoot":"","sources":["../../../src/function-configs/sql-helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,wFAAwF;AACxF,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEtD;;;;;EAKE;AAEF;;;EAGE;AACF,OAAO,EAAE,YAAY,IAAI,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpE;;;;;;;;;;;EAWE;AACF,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE;MACzB,oEAAoE;MACpE,oEAAoE;MACpE,oBAAoB,CAAC;AAE3B;;;;;;;;;;EAUE;AACF,MAAM,UAAU,UAAU,CAAC,KAAa;IACpC,OAAO,UAAU,KAAK,cAAc,KAAK,GAAG,CAAC;AACjD,CAAC;AAED;;;;;;EAME;AACF,MAAM,UAAU,YAAY,CAAC,UAAkB;IAC3C,OAAO,sBAAsB,UAAU,UAAU,UAAU,gBAAgB,CAAC;AAChF,CAAC;AAED;;;;;;;;;EASE;AACF,MAAM,UAAU,kBAAkB,CAAC,KAAa,EAAE,MAAc;IAC5D,OAAO,iBAAiB,KAAK,wBAAwB,MAAM,MAAM,CAAC;AACtE,CAAC;AAED;;;EAGE;AACF,MAAM,CAAC,MAAM,sBAAsB,GAAG,yCAAyC,CAAC;AAEhF;;;;;;;EAOE;AACF,MAAM,UAAU,QAAQ,CAAC,SAAiB,EAAE,MAAc;IACtD,OAAO,aAAa,SAAS,SAAS,MAAM,gBAAgB,CAAC;AACjE,CAAC;AAED;;;;;;;;EAQE;AACF,MAAM,CAAC,MAAM,UAAU,GAAG,0BAA0B,CAAC;AAErD;;;;;;;EAOE;AACF,SAAS,YAAY,CAAC,KAAc;IAChC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;EAKE;AACF,MAAM,UAAU,gBAAgB,CAC5B,GAAG,KAAwB;IAE3B,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;;;EAKE;AACF,MAAM,UAAU,iBAAiB,CAC7B,GAAG,KAAwB;IAE3B,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,uEAAuE;AACvE,MAAM,CAAC,MAAM,kBAAkB,GAAyB;IACpD,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,WAAW;CAC7E,CAAC;AAEF,+BAA+B;AAC/B,MAAM,CAAC,MAAM,mBAAmB,GAAyB;IACrD,SAAS,CAAC,MAAM;IAChB,SAAS,CAAC,IAAI;IACd,SAAS,CAAC,KAAK;IACf,SAAS,CAAC,OAAO;IACjB,SAAS,CAAC,IAAI;IACd,SAAS,CAAC,KAAK;IACf,SAAS,CAAC,MAAM;CACnB,CAAC;AAEF;;;;;;EAME;AACF,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC3C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;IAC1D,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;IAChE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9E,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;;EAME;AACF,MAAM,UAAU,oBAAoB,CAAC,KAAc,EAAE,IAAe;IAChE,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxE,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,KAAK,SAAS,CAAC,OAAO,CAAC;IAClE,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,mBAAmB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACzE,OAAO,KAAK,CAAC;AACjB,CAAC"}

package/dist/src/function-configs/string/contains.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"contains.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/contains.ts"],"names":[],"mappings":"AAEA,OAAO,EACH,mBAAmB,EAEtB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,cAAc,EAAE,mBAAmB,CAAC,YAAY,CAyC5D,CAAC"}
+{"version":3,"file":"contains.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/contains.ts"],"names":[],"mappings":"AAEA,OAAO,EACH,mBAAmB,EAEtB,MAAM,kBAAkB,CAAC;AAG1B,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,cAAc,EAAE,mBAAmB,CAAC,YAAY,CA6C5D,CAAC"}

package/dist/src/function-configs/string/contains.js

@@ -1,6 +1,7 @@
 import { containsFP } from '@terascope/core-utils';
 import { FieldType } from '@terascope/types';
 import { ProcessMode, FunctionDefinitionType, FunctionDefinitionCategory } from '../interfaces.js';
+import { sqlLiteral } from '../sql-helpers.js';
 export const containsConfig = {
     name: 'contains',
     type: FunctionDefinitionType.FIELD_VALIDATION,
@@ -33,6 +34,10 @@ export const containsConfig = {
     create({ args: { value } }) {
         return containsFP(value);
     },
+    /** `contains` is native and byte-oriented, the same as `String.prototype.includes`. */
+    sql: {
+        expression: ({ value, args }) => `contains(${value}, ${sqlLiteral(args.value)})`,
+    },
     accepts: [FieldType.String],
     argument_schema: {
         value: {

package/dist/src/function-configs/string/contains.js.map

@@ -1 +1 @@
-{"version":3,"file":"contains.js","sourceRoot":"","sources":["../../../../src/function-configs/string/contains.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EACkB,WAAW,EAAE,sBAAsB,EACxD,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAM1B,MAAM,CAAC,MAAM,cAAc,GAAsC;IAC7D,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,sBAAsB,CAAC,gBAAgB;IAC7C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,6HAA6H;IAC1I,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE;YACxB,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE;YACzE,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,SAAS;YAChB,MAAM,EAAE,SAAS;SACpB;QACD;YACI,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE;YAC1B,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE;YACzE,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,SAAS;YAChB,MAAM,EAAE,SAAS;SACpB;QACD;YACI,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE;YACvB,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE;YACzE,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,SAAS;YAChB,MAAM,EAAE,IAAI;SACf;KACJ;IACD,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE;QACtB,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,eAAe,EAAE;QACb,KAAK,EAAE;YACH,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,kDAAkD;SAClE;KACJ;IACD,kBAAkB,EAAE,CAAC,OAAO,CAAC;CAChC,CAAC"}
+{"version":3,"file":"contains.js","sourceRoot":"","sources":["../../../../src/function-configs/string/contains.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EACkB,WAAW,EAAE,sBAAsB,EACxD,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAM/C,MAAM,CAAC,MAAM,cAAc,GAAsC;IAC7D,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,sBAAsB,CAAC,gBAAgB;IAC7C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,6HAA6H;IAC1I,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE;YACxB,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE;YACzE,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,SAAS;YAChB,MAAM,EAAE,SAAS;SACpB;QACD;YACI,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE;YAC1B,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE;YACzE,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,SAAS;YAChB,MAAM,EAAE,SAAS;SACpB;QACD;YACI,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE;YACvB,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE;YACzE,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,SAAS;YAChB,MAAM,EAAE,IAAI;SACf;KACJ;IACD,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE;QACtB,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IACD,uFAAuF;IACvF,GAAG,EAAE;QACD,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,YAAY,KAAK,KAAK,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG;KACnF;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,eAAe,EAAE;QACb,KAAK,EAAE;YACH,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,kDAAkD;SAClE;KACJ;IACD,kBAAkB,EAAE,CAAC,OAAO,CAAC;CAChC,CAAC"}

package/dist/src/function-configs/string/createID.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"createID.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/createID.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAG1B,MAAM,WAAW,YAAY;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,oBAAoB,CAAC;CACjC;AAKD,eAAO,MAAM,cAAc,EAAE,oBAAoB,CAAC,YAAY,CA6H7D,CAAC"}
+{"version":3,"file":"createID.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/createID.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAK1B,MAAM,WAAW,YAAY;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,oBAAoB,CAAC;CACjC;AAKD,eAAO,MAAM,cAAc,EAAE,oBAAoB,CAAC,YAAY,CA6I7D,CAAC"}

package/dist/src/function-configs/string/createID.js

@@ -2,6 +2,8 @@ import { FieldType } from '@terascope/types';
 import { toString } from '@terascope/core-utils';
 import { ProcessMode, FunctionDefinitionType, FunctionDefinitionCategory } from '../interfaces.js';
 import { encodeAny } from './encode-utils.js';
+import { hasHashSql, hashSql } from './sql-utils.js';
+import { STRING_FIELD_TYPES } from '../sql-helpers.js';
 const hashDefault = 'md5';
 const digestDefault = 'hex';
 export const createIDConfig = {
@@ -106,6 +108,20 @@ export const createIDConfig = {
             return encode(toString(value));
         };
     },
+    sql: {
+        /**
+         * A scalar string column only.
+         *
+         * `createID` hashes `toString(value)`, or for an array
+         * `value.flat().map(toString).join('')`. On a string column `toString` is the identity, so
+         * the hash is exact. Other column types would need SQL that reproduces how `toString`
+         * renders numbers, booleans, objects and nested arrays, so they keep the UDF.
+        */
+        types: STRING_FIELD_TYPES,
+        applies: ({ hash, digest }, inputConfig) => !inputConfig.field_config.array
+            && hasHashSql(hash ?? hashDefault, digest ?? digestDefault),
+        expression: ({ value, args: { hash = hashDefault, digest = digestDefault } }) => hashSql(hash, digest, value),
+    },
     accepts: [],
     argument_schema: {
         hash: {

package/dist/src/function-configs/string/createID.js.map

@@ -1 +1 @@
-{"version":3,"file":"createID.js","sourceRoot":"","sources":["../../../../src/function-configs/string/createID.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAO9C,MAAM,WAAW,GAAG,KAAK,CAAC;AAC1B,MAAM,aAAa,GAAyB,KAAK,CAAC;AAElD,MAAM,CAAC,MAAM,cAAc,GAAuC;IAC9D,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,WAAW;IACrC,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,uHAAuH;IACpI,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,kCAAkC;YAC1C,WAAW,EAAE,mCAAmC;SACnD;QACD;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;wBACtB,KAAK,EAAE,IAAI;qBACd;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;YACvB,MAAM,EAAE,kCAAkC;SAC7C;QACD;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;wBACtB,KAAK,EAAE,IAAI;qBACd;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;YACvB,MAAM,EAAE,kEAAkE;SAC7E;QACD;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;wBACtB,KAAK,EAAE,IAAI;qBACd;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;YACvB,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,kCAAkC;SAC7C;QACD;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,KAAK;qBACxB;oBACD,aAAa,EAAE;wBACX,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;oBACD,aAAa,EAAE;wBACX,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;YAClB,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,kCAAkC;SAC7C;KACJ;IACD,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,GAAG,WAAW,EAAE,MAAM,GAAG,aAAa,EAAE,EAAE;QAC3D,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACvC,OAAO,SAAS,SAAS,CAAC,KAAK;YAC3B,IAAI,KAAK,IAAI,IAAI;gBAAE,OAAO;YAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;qBACnC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACnB,CAAC;YACD,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QACnC,CAAC,CAAC;IACN,CAAC;IACD,OAAO,EAAE,EAAE;IACX,eAAe,EAAE;QACb,IAAI,EAAE;YACF,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,oDAAoD;SACpE;QACD,MAAM,EAAE;YACJ,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,qFAAqF;SACrG;KACJ;IACD,kBAAkB,CAAC,EAAE,IAAI,EAAE;QACvB,IAAI,IAAI,IAAI,IAAI;YAAE,OAAO;IAC7B,CAAC;IACD,WAAW;QACP,OAAO;YACH,YAAY,EAAE;gBACV,IAAI,EAAE,SAAS,CAAC,OAAO;aAC1B;YACD,YAAY,EAAE,SAAS;SAC1B,CAAC;IACN,CAAC;CACJ,CAAC"}
+{"version":3,"file":"createID.js","sourceRoot":"","sources":["../../../../src/function-configs/string/createID.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAOvD,MAAM,WAAW,GAAG,KAAK,CAAC;AAC1B,MAAM,aAAa,GAAyB,KAAK,CAAC;AAElD,MAAM,CAAC,MAAM,cAAc,GAAuC;IAC9D,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,WAAW;IACrC,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,uHAAuH;IACpI,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,kCAAkC;YAC1C,WAAW,EAAE,mCAAmC;SACnD;QACD;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;wBACtB,KAAK,EAAE,IAAI;qBACd;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;YACvB,MAAM,EAAE,kCAAkC;SAC7C;QACD;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;wBACtB,KAAK,EAAE,IAAI;qBACd;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;YACvB,MAAM,EAAE,kEAAkE;SAC7E;QACD;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;wBACtB,KAAK,EAAE,IAAI;qBACd;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;YACvB,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,kCAAkC;SAC7C;QACD;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,KAAK;qBACxB;oBACD,aAAa,EAAE;wBACX,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;oBACD,aAAa,EAAE;wBACX,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;YAClB,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,kCAAkC;SAC7C;KACJ;IACD,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,GAAG,WAAW,EAAE,MAAM,GAAG,aAAa,EAAE,EAAE;QAC3D,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACvC,OAAO,SAAS,SAAS,CAAC,KAAK;YAC3B,IAAI,KAAK,IAAI,IAAI;gBAAE,OAAO;YAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;qBACnC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACnB,CAAC;YACD,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QACnC,CAAC,CAAC;IACN,CAAC;IACD,GAAG,EAAE;QACD;;;;;;;UAOE;QACF,KAAK,EAAE,kBAAkB;QACzB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,KAAK;eACpE,UAAU,CAAC,IAAI,IAAI,WAAW,EAAE,MAAM,IAAI,aAAa,CAAC;QAC/D,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,IAAI,GAAG,WAAW,EAAE,MAAM,GAAG,aAAa,EAAE,EAAE,EAAE,EAAE,CAC5E,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAC9B;KACJ;IACD,OAAO,EAAE,EAAE;IACX,eAAe,EAAE;QACb,IAAI,EAAE;YACF,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,oDAAoD;SACpE;QACD,MAAM,EAAE;YACJ,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,qFAAqF;SACrG;KACJ;IACD,kBAAkB,CAAC,EAAE,IAAI,EAAE;QACvB,IAAI,IAAI,IAAI,IAAI;YAAE,OAAO;IAC7B,CAAC;IACD,WAAW;QACP,OAAO;YACH,YAAY,EAAE;gBACV,IAAI,EAAE,SAAS,CAAC,OAAO;aAC1B;YACD,YAAY,EAAE,SAAS;SAC1B,CAAC;IACN,CAAC;CACJ,CAAC"}

package/dist/src/function-configs/string/decodeBase64.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"decodeBase64.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeBase64.ts"],"names":[],"mappings":"AACA,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAE1B,eAAO,MAAM,kBAAkB,EAAE,oBAsChC,CAAC"}
+{"version":3,"file":"decodeBase64.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeBase64.ts"],"names":[],"mappings":"AACA,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAE1B,eAAO,MAAM,kBAAkB,EAAE,oBAoDhC,CAAC"}

package/dist/src/function-configs/string/decodeBase64.js

@@ -25,6 +25,20 @@ export const decodeBase64Config = {
     create() {
         return (input) => Buffer.from(input, 'base64').toString('utf8');
     },
+    /**
+     * `coalesce(try(decode(from_base64(x))), udf(x))`.
+     *
+     * DuckDB throws on malformed base64, which would abort the whole query, while `Buffer.from(x,
+     * 'base64')` is lenient and returns garbled text (`'hello'` becomes `'��e'`). `try` turns the
+     * error into NULL, and the UDF then returns the same garbled text as JavaScript.
+     *
+     * `try` is used rather than a validity regex because it also catches base64 that decodes to
+     * invalid UTF-8, which a regex on the input cannot detect.
+    */
+    sql: {
+        needs_udf_fallback: true,
+        expression: ({ value, udf }) => `coalesce(try(decode(from_base64(${value}))), ${udf(value)})`,
+    },
     accepts: [FieldType.String],
     output_type(inputConfig) {
         const { field_config, child_config } = inputConfig;

package/dist/src/function-configs/string/decodeBase64.js.map

@@ -1 +1 @@
-{"version":3,"file":"decodeBase64.js","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeBase64.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAE1B,MAAM,CAAC,MAAM,kBAAkB,GAAyB;IACpD,IAAI,EAAE,cAAc;IACpB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,wDAAwD;IACrE,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,kBAAkB;YACzB,MAAM,EAAE,aAAa;SAExB;KACJ;IACD,MAAM;QACF,OAAO,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAe,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QAEnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}
+{"version":3,"file":"decodeBase64.js","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeBase64.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAE1B,MAAM,CAAC,MAAM,kBAAkB,GAAyB;IACpD,IAAI,EAAE,cAAc;IACpB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,wDAAwD;IACrE,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,kBAAkB;YACzB,MAAM,EAAE,aAAa;SAExB;KACJ;IACD,MAAM;QACF,OAAO,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAe,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACvF,CAAC;IACD;;;;;;;;;MASE;IACF,GAAG,EAAE;QACD,kBAAkB,EAAE,IAAI;QACxB,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,mCAAmC,KAAK,QAAQ,GAAG,CAAC,KAAK,CAAC,GAAG;KAChG;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QAEnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}

package/dist/src/function-configs/string/decodeHex.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"decodeHex.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeHex.ts"],"names":[],"mappings":"AACA,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAE1B,eAAO,MAAM,eAAe,EAAE,oBAqC7B,CAAC"}
+{"version":3,"file":"decodeHex.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeHex.ts"],"names":[],"mappings":"AACA,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAE1B,eAAO,MAAM,eAAe,EAAE,oBAkD7B,CAAC"}

package/dist/src/function-configs/string/decodeHex.js

@@ -25,6 +25,19 @@ export const decodeHexConfig = {
     create() {
         return (input) => Buffer.from(input, 'hex').toString('utf8');
     },
+    /**
+     * `coalesce(try(decode(unhex(x))), udf(x))`.
+     *
+     * DuckDB throws on a non-hex digit, on an odd length, and on hex whose bytes are not valid
+     * UTF-8 (`'0123456789'` decodes to a byte `0x89`). JavaScript does not throw: `Buffer.from(x,
+     * 'hex')` stops at the first bad pair (`'hello'` gives `''`, `'686'` gives `'h'`) and replaces
+     * invalid bytes with U+FFFD. `try` turns each DuckDB error into NULL, and the UDF supplies the
+     * JavaScript result.
+    */
+    sql: {
+        needs_udf_fallback: true,
+        expression: ({ value, udf }) => `coalesce(try(decode(unhex(${value}))), ${udf(value)})`,
+    },
     accepts: [FieldType.String],
     output_type(inputConfig) {
         const { field_config, child_config } = inputConfig;

package/dist/src/function-configs/string/decodeHex.js.map

@@ -1 +1 @@
-{"version":3,"file":"decodeHex.js","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeHex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAE1B,MAAM,CAAC,MAAM,eAAe,GAAyB;IACjD,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,6DAA6D;IAC1E,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,wDAAwD;YAC/D,MAAM,EAAE,6BAA6B;SACxC;KACJ;IACD,MAAM;QACF,OAAO,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAe,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QAEnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}
+{"version":3,"file":"decodeHex.js","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeHex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAE1B,MAAM,CAAC,MAAM,eAAe,GAAyB;IACjD,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,6DAA6D;IAC1E,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,wDAAwD;YAC/D,MAAM,EAAE,6BAA6B;SACxC;KACJ;IACD,MAAM;QACF,OAAO,CAAC,KAAc,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAe,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpF,CAAC;IACD;;;;;;;;MAQE;IACF,GAAG,EAAE;QACD,kBAAkB,EAAE,IAAI;QACxB,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,6BAA6B,KAAK,QAAQ,GAAG,CAAC,KAAK,CAAC,GAAG;KAC1F;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QAEnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}

package/dist/src/function-configs/string/decodeURL.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"decodeURL.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeURL.ts"],"names":[],"mappings":"AACA,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAE1B,eAAO,MAAM,eAAe,EAAE,oBAqC7B,CAAC"}
+{"version":3,"file":"decodeURL.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeURL.ts"],"names":[],"mappings":"AACA,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAE1B,eAAO,MAAM,eAAe,EAAE,oBA4C7B,CAAC"}

package/dist/src/function-configs/string/decodeURL.js

@@ -25,6 +25,13 @@ export const decodeURLConfig = {
     create() {
         return (input) => decodeURIComponent(input);
     },
+    /**
+     * `url_decode`, which matches `decodeURIComponent` for spaces, `+`, `%`, non-ASCII and astral
+     * input.
+    */
+    sql: {
+        expression: ({ value }) => `url_decode(${value})`,
+    },
     accepts: [FieldType.String],
     output_type(inputConfig) {
         const { field_config, child_config } = inputConfig;

package/dist/src/function-configs/string/decodeURL.js.map

@@ -1 +1 @@
-{"version":3,"file":"decodeURL.js","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeURL.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAE1B,MAAM,CAAC,MAAM,eAAe,GAAyB;IACjD,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,qDAAqD;IAClE,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,wCAAwC;YAC/C,MAAM,EAAE,gCAAgC;SAC3C;KACJ;IACD,MAAM;QACF,OAAO,CAAC,KAAc,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAe,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QAEnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}
+{"version":3,"file":"decodeURL.js","sourceRoot":"","sources":["../../../../src/function-configs/string/decodeURL.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAE1B,MAAM,CAAC,MAAM,eAAe,GAAyB;IACjD,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,qDAAqD;IAClE,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,wCAAwC;YAC/C,MAAM,EAAE,gCAAgC;SAC3C;KACJ;IACD,MAAM;QACF,OAAO,CAAC,KAAc,EAAE,EAAE,CAAC,kBAAkB,CAAC,KAAe,CAAC,CAAC;IACnE,CAAC;IACD;;;MAGE;IACF,GAAG,EAAE;QACD,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,cAAc,KAAK,GAAG;KACpD;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QAEnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}

package/dist/src/function-configs/string/encode.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"encode.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/encode.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAG1B,MAAM,WAAW,UAAU;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,oBAAoB,CAAC;CACjC;AAED,eAAO,MAAM,YAAY,EAAE,oBAAoB,CAAC,UAAU,CA4GzD,CAAC"}
+{"version":3,"file":"encode.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/encode.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAI1B,MAAM,WAAW,UAAU;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,oBAAoB,CAAC;CACjC;AAED,eAAO,MAAM,YAAY,EAAE,oBAAoB,CAAC,UAAU,CAyHzD,CAAC"}

package/dist/src/function-configs/string/encode.js

@@ -1,6 +1,7 @@
 import { FieldType } from '@terascope/types';
 import { ProcessMode, FunctionDefinitionType, FunctionDefinitionCategory, } from '../interfaces.js';
 import { encodeAny } from './encode-utils.js';
+import { encodeURIComponentSql, hasHashSql, hashSql } from './sql-utils.js';
 export const encodeConfig = {
     name: 'encode',
     type: FunctionDefinitionType.FIELD_TRANSFORM,
@@ -83,6 +84,19 @@ export const encodeConfig = {
     create({ args: { algo, digest } }) {
         return encodeAny(algo, digest);
     },
+    sql: {
+        /**
+         * Every algorithm and digest pair DuckDB can express.
+         *
+         * `encodeAny` handles `url`, the two `Buffer` encodings and `createHash`, and each has a
+         * native form. Anything else, such as `sha512` or a `latin1` digest, keeps the UDF. See
+         * `hashSql`.
+        */
+        applies: ({ algo, digest }) => algo === 'url' || hasHashSql(algo, digest),
+        expression: ({ value, args: { algo, digest } }) => (algo === 'url'
+            ? encodeURIComponentSql(value)
+            : hashSql(algo, digest ?? 'hex', value)),
+    },
     accepts: [FieldType.String],
     argument_schema: {
         algo: {

package/dist/src/function-configs/string/encode.js.map

@@ -1 +1 @@
-{"version":3,"file":"encode.js","sourceRoot":"","sources":["../../../../src/function-configs/string/encode.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,GAC7B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAO9C,MAAM,CAAC,MAAM,YAAY,GAAqC;IAC1D,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,gHAAgH;IAC7H,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,oBAAoB;YAC3B,MAAM,EAAE,kEAAkE;YAC1E,WAAW,EAAE,+DAA+D;SAC/E;QACD;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;YACrB,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,oBAAoB;YAC3B,MAAM,EAAE,kCAAkC;SAC7C;QACD;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;YACrB,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,gCAAgC;YACvC,MAAM,EAAE,wCAAwC;SACnD;QACD;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,mBAAmB;YAC1B,MAAM,EAAE,0BAA0B;SACrC;QACD;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE;YACxC,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,oBAAoB;YAC3B,MAAM,EAAE,8BAA8B;SACzC;KACJ;IACD,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;QAC7B,OAAO,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,eAAe,EAAE;QACb,IAAI,EAAE;YACF,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,6CAA6C;SAC7D;QACD,MAAM,EAAE;YACJ,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,mGAAmG;kBAC1G,qDAAqD;SAC9D;KACJ;IACD,kBAAkB,EAAE,CAAC,MAAM,CAAC;IAC5B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QAEnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}
+{"version":3,"file":"encode.js","sourceRoot":"","sources":["../../../../src/function-configs/string/encode.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,GAC7B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAO5E,MAAM,CAAC,MAAM,YAAY,GAAqC;IAC1D,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,gHAAgH;IAC7H,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,oBAAoB;YAC3B,MAAM,EAAE,kEAAkE;YAC1E,WAAW,EAAE,+DAA+D;SAC/E;QACD;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;YACrB,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,oBAAoB;YAC3B,MAAM,EAAE,kCAAkC;SAC7C;QACD;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;YACrB,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,gCAAgC;YACvC,MAAM,EAAE,wCAAwC;SACnD;QACD;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,mBAAmB;YAC1B,MAAM,EAAE,0BAA0B;SACrC;QACD;YACI,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE;YACxC,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,oBAAoB;YAC3B,MAAM,EAAE,8BAA8B;SACzC;KACJ;IACD,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;QAC7B,OAAO,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,GAAG,EAAE;QACD;;;;;;UAME;QACF,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,IAAI,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC;QACzE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,KAAK;YAC9D,CAAC,CAAC,qBAAqB,CAAC,KAAK,CAAC;YAC9B,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,IAAI,KAAK,EAAE,KAAK,CAAW,CAAC;KACzD;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,eAAe,EAAE;QACb,IAAI,EAAE;YACF,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,6CAA6C;SAC7D;QACD,MAAM,EAAE;YACJ,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,mGAAmG;kBAC1G,qDAAqD;SAC9D;KACJ;IACD,kBAAkB,EAAE,CAAC,MAAM,CAAC;IAC5B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QAEnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}

package/dist/src/function-configs/string/encodeBase64.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"encodeBase64.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/encodeBase64.ts"],"names":[],"mappings":"AACA,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAG1B,eAAO,MAAM,kBAAkB,EAAE,oBAqChC,CAAC"}
+{"version":3,"file":"encodeBase64.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/encodeBase64.ts"],"names":[],"mappings":"AACA,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAG1B,eAAO,MAAM,kBAAkB,EAAE,oBA+ChC,CAAC"}

package/dist/src/function-configs/string/encodeBase64.js

@@ -26,6 +26,16 @@ export const encodeBase64Config = {
     create() {
         return bufferEncode('base64');
     },
+    /**
+     * `to_base64(encode(x))`. This uses `encode`, not `x::BLOB`.
+     *
+     * A `VARCHAR` to `BLOB` cast rejects non-ASCII input ("All non-ascii characters must be escaped
+     * with hex codes"). `encode()` converts to UTF-8, so the result matches `Buffer.from(input,
+     * 'utf8').toString('base64')`, including for astral characters.
+    */
+    sql: {
+        expression: ({ value }) => `to_base64(encode(${value}))`,
+    },
     accepts: [FieldType.String],
     output_type(inputConfig) {
         const { field_config, child_config } = inputConfig;

package/dist/src/function-configs/string/encodeBase64.js.map

@@ -1 +1 @@
-{"version":3,"file":"encodeBase64.js","sourceRoot":"","sources":["../../../../src/function-configs/string/encodeBase64.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,CAAC,MAAM,kBAAkB,GAAyB;IACpD,IAAI,EAAE,cAAc;IACpB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,qDAAqD;IAClE,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,aAAa;YACpB,MAAM,EAAE,kBAAkB;SAC7B;KACJ;IACD,MAAM;QACF,OAAO,YAAY,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QACnD,mDAAmD;QACnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}
+{"version":3,"file":"encodeBase64.js","sourceRoot":"","sources":["../../../../src/function-configs/string/encodeBase64.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,CAAC,MAAM,kBAAkB,GAAyB;IACpD,IAAI,EAAE,cAAc;IACpB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,qDAAqD;IAClE,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,aAAa;YACpB,MAAM,EAAE,kBAAkB;SAC7B;KACJ;IACD,MAAM;QACF,OAAO,YAAY,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IACD;;;;;;MAME;IACF,GAAG,EAAE;QACD,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,oBAAoB,KAAK,IAAI;KAC3D;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QACnD,mDAAmD;QACnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}

package/dist/src/function-configs/string/encodeHex.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"encodeHex.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/encodeHex.ts"],"names":[],"mappings":"AACA,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAG1B,eAAO,MAAM,eAAe,EAAE,oBAqC7B,CAAC"}
+{"version":3,"file":"encodeHex.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/encodeHex.ts"],"names":[],"mappings":"AACA,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAG1B,eAAO,MAAM,eAAe,EAAE,oBA8C7B,CAAC"}

package/dist/src/function-configs/string/encodeHex.js

@@ -26,6 +26,15 @@ export const encodeHexConfig = {
     create() {
         return bufferEncode('hex');
     },
+    /**
+     * `lower(hex(encode(x)))`. This uses `encode`, not `x::BLOB`.
+     *
+     * A `VARCHAR` to `BLOB` cast rejects non-ASCII input, while `encode()` converts to UTF-8. The
+     * result is lowercased to match `Buffer.toString('hex')`.
+    */
+    sql: {
+        expression: ({ value }) => `lower(hex(encode(${value})))`,
+    },
     accepts: [FieldType.String],
     output_type(inputConfig) {
         const { field_config, child_config } = inputConfig;

package/dist/src/function-configs/string/encodeHex.js.map

@@ -1 +1 @@
-{"version":3,"file":"encodeHex.js","sourceRoot":"","sources":["../../../../src/function-configs/string/encodeHex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,CAAC,MAAM,eAAe,GAAyB;IACjD,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,2DAA2D;IACxE,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,6BAA6B;YACpC,MAAM,EAAE,wDAAwD;SACnE;KACJ;IACD,MAAM;QACF,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QAEnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}
+{"version":3,"file":"encodeHex.js","sourceRoot":"","sources":["../../../../src/function-configs/string/encodeHex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAEH,WAAW,EACX,sBAAsB,EAEtB,0BAA0B,EAC7B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,MAAM,CAAC,MAAM,eAAe,GAAyB;IACjD,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,sBAAsB,CAAC,eAAe;IAC5C,YAAY,EAAE,WAAW,CAAC,iBAAiB;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,MAAM;IAC3C,WAAW,EAAE,2DAA2D;IACxE,QAAQ,EAAE;QACN;YACI,IAAI,EAAE,EAAE;YACR,MAAM,EAAE;gBACJ,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE;oBACJ,SAAS,EAAE;wBACP,IAAI,EAAE,SAAS,CAAC,MAAM;qBACzB;iBACJ;aACJ;YACD,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE,6BAA6B;YACpC,MAAM,EAAE,wDAAwD;SACnE;KACJ;IACD,MAAM;QACF,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD;;;;;MAKE;IACF,GAAG,EAAE;QACD,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,oBAAoB,KAAK,KAAK;KAC5D;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC;IAC3B,WAAW,CAAC,WAAqC;QAC7C,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;QAEnD,OAAO;YACH,YAAY,EAAE;gBACV,GAAG,YAAY;gBACf,IAAI,EAAE,SAAS,CAAC,MAAM;aACzB;YACD,YAAY;SACf,CAAC;IACN,CAAC;CACJ,CAAC"}

package/dist/src/function-configs/string/encodeSHA.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"encodeSHA.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/encodeSHA.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAG1B,MAAM,WAAW,aAAa;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,oBAAoB,CAAC;CACjC;AAKD,eAAO,MAAM,eAAe,EAAE,oBAAoB,CAAC,aAAa,CAsE/D,CAAC"}
+{"version":3,"file":"encodeSHA.d.ts","sourceRoot":"","sources":["../../../../src/function-configs/string/encodeSHA.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EACH,oBAAoB,EAKvB,MAAM,kBAAkB,CAAC;AAG1B,MAAM,WAAW,aAAa;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,oBAAoB,CAAC;CACjC;AAKD,eAAO,MAAM,eAAe,EAAE,oBAAoB,CAAC,aAAa,CAiF/D,CAAC"}

package/dist/src/function-configs/string/encodeSHA.js

@@ -43,6 +43,17 @@ export const encodeSHAConfig = {
     create({ args: { hash = hashDefault, digest = digestDefault } }) {
         return cryptoEncode(hash, digest);
     },
+    /**
+     * `sha256` with a hex digest, which are the defaults.
+     *
+     * DuckDB has `md5`, `sha1` and `sha256` but no `sha512`, and its digests are hex. Any other
+     * hash or digest keeps the UDF.
+    */
+    sql: {
+        // `validate_arguments` rejects any hash not starting with `sha`, and DuckDB has no `sha512`
+        applies: (args) => (args.digest ?? 'hex') === 'hex' && (args.hash ?? 'sha256') === 'sha256',
+        expression: ({ value }) => `sha256(${value})`,
+    },
     accepts: [FieldType.String],
     argument_schema: {
         hash: {