@kensio/yulin 1.21.22 → 1.21.23

package/dist/service/athena/engine/sim-athena-ast-nodes.d.ts

@@ -18,6 +18,17 @@ export interface SimAthenaFromItem extends SimAthenaAstNode {
     table: string;
     as?: string | null;
 }
+/** One value as a node, or nothing where it is not an object. */
+export declare function asAstNode(value: unknown): SimAthenaAstNode | undefined;
+/**
+ * The name a call carries, where the statement wrote it as one part.
+ *
+ * The parser holds a name as a list, one entry per piece of a qualified name,
+ * and it holds an `UNNEST` alias as a call of its own. A name written in more
+ * than one part answers with nothing, since neither the engine's functions nor
+ * an alias is reached that way.
+ */
+export declare function simAthenaCalledName(node: SimAthenaAstNode | undefined): string | undefined;
 /** Every node in the tree, the outermost first. */
 export declare function simAthenaAstNodes(root: unknown): Generator<SimAthenaAstNode>;
 /** Whether this node references a column. */

package/dist/service/athena/engine/sim-athena-ast-nodes.js

@@ -1,3 +1,25 @@
+/** One value as a node, or nothing where it is not an object. */
+export function asAstNode(value) {
+    return typeof value === "object" && value !== null
+        ? value
+        : undefined;
+}
+/**
+ * The name a call carries, where the statement wrote it as one part.
+ *
+ * The parser holds a name as a list, one entry per piece of a qualified name,
+ * and it holds an `UNNEST` alias as a call of its own. A name written in more
+ * than one part answers with nothing, since neither the engine's functions nor
+ * an alias is reached that way.
+ */
+export function simAthenaCalledName(node) {
+    const parts = asAstNode(node?.["name"])?.["name"];
+    const first = Array.isArray(parts) && parts.length === 1
+        ? asAstNode(parts[0])
+        : undefined;
+    const name = first?.["value"];
+    return typeof name === "string" ? name : undefined;
+}
 /** Every node in the tree, the outermost first. */
 export function* simAthenaAstNodes(root) {
     if (Array.isArray(root)) {

package/dist/service/athena/engine/sim-athena-string-shims.js

@@ -1,4 +1,5 @@
-import { shimNumber, shimText, simAthenaScalarShim, } from "./sim-athena-shim-registry.js";
+import { SimAthenaSetUpError } from "../error/sim-athena.error.js";
+import { isExplicitNull, shimNumber, shimText, simAthenaScalarShim, } from "./sim-athena-shim-registry.js";
 /**
  * Trino's string functions.
  *
@@ -8,9 +9,36 @@ import { shimNumber, shimText, simAthenaScalarShim, } from "./sim-athena-shim-re
  * with something that works less well.
  */
 export function simAthenaInstallStringShims(database) {
+    simAthenaScalarShim(database, "split", (...values) => isExplicitNull(values, 2)
+        ? null
+        : split(shimText(values.at(0)), shimText(values.at(1)), shimNumber(values.at(2))));
     simAthenaScalarShim(database, "split_part", (value, delimiter, index) => splitPart(shimText(value), shimText(delimiter), shimNumber(index)));
     simAthenaScalarShim(database, "strpos", (value, search) => position(shimText(value), shimText(search)));
 }
+/**
+ * One value cut into an array, written as the JSON text an array is held as.
+ *
+ * Trino answers with one empty element over an empty value, and refuses an
+ * empty delimiter rather than answering a character at a time. A limit caps
+ * how many elements come back and leaves the rest of the value in the last of
+ * them.
+ */
+function split(value, delimiter, limit) {
+    if (value === undefined || delimiter === undefined) {
+        return null;
+    }
+    if (delimiter === "" || (limit !== undefined && limit < 1)) {
+        throw new SimAthenaSetUpError("split takes a delimiter of at least one character and a limit from one");
+    }
+    const parts = value.split(delimiter);
+    if (limit === undefined || parts.length <= limit) {
+        return JSON.stringify(parts);
+    }
+    return JSON.stringify([
+        ...parts.slice(0, limit - 1),
+        parts.slice(limit - 1).join(delimiter),
+    ]);
+}
 /** Trino counts the fields from one, and has no field below that. */
 function splitPart(value, delimiter, index) {
     if (value === undefined || delimiter === undefined || index === undefined) {

package/dist/service/athena/engine/sim-athena-unnest-item.js

@@ -1,4 +1,4 @@
-import { simAthenaAstNodes, } from "./sim-athena-ast-nodes.js";
+import { asAstNode, simAthenaAstNodes, simAthenaCalledName, } from "./sim-athena-ast-nodes.js";
 /**
  * The `UNNEST` a statement flattens an array or a map with.
  *
@@ -19,8 +19,8 @@ export function simAthenaUnnestItem(ast) {
     }
     const alias = aliasOf(item);
     const columns = aliasColumns(item);
-    const source = item["expr"];
-    if (alias === undefined || columns === undefined || !isNode(source)) {
+    const source = asAstNode(item["expr"]);
+    if (alias === undefined || columns === undefined || source === undefined) {
         return { kind: "unreadable" };
     }
     return { kind: "found", item, alias, columns, source };
@@ -38,25 +38,16 @@ function isCrossJoined(item) {
 }
 /** The alias name, which the parser holds as a function's name. */
 function aliasOf(item) {
-    const parts = asNode(asNode(item["as"])?.["name"])?.["name"];
-    const first = Array.isArray(parts) ? asNode(parts[0]) : undefined;
-    const value = first?.["value"];
-    return typeof value === "string" ? value : undefined;
+    return simAthenaCalledName(asAstNode(item["as"]));
 }
 /** The names inside the alias, which the parser holds as a function's arguments. */
 function aliasColumns(item) {
-    const values = asNode(asNode(item["as"])?.["args"])?.["value"];
+    const values = asAstNode(asAstNode(item["as"])?.["args"])?.["value"];
     if (!Array.isArray(values) || values.length === 0) {
         return undefined;
     }
-    const columns = values.map((value) => asNode(value)?.["column"]);
+    const columns = values.map((value) => asAstNode(value)?.["column"]);
     return columns.every((column) => typeof column === "string")
         ? columns
         : undefined;
 }
-function isNode(value) {
-    return typeof value === "object" && value !== null;
-}
-function asNode(value) {
-    return isNode(value) ? value : undefined;
-}

package/dist/service/athena/engine/sim-athena-unnest-rewrite.d.ts

@@ -7,10 +7,10 @@ interface SimAthenaUnnestRewriteRequest {
 /**
  * Turn an `UNNEST` into the `json_each` SQLite flattens JSON with.
  *
- * An array or a map column is held as its JSON text, and `json_each` reads that
- * as one row per element with a `key` and a `value`. The `FROM` entry becomes a
- * call to it and every reference to the alias is pointed at the column it
- * answers with.
+ * An array or a map column is held as its JSON text, as is what a function like
+ * `split` answers, and `json_each` reads that as one row per element with a
+ * `key` and a `value`. The `FROM` entry becomes a call to it and every
+ * reference to the alias is pointed at the column it answers with.
  *
  * Answers with whether the statement can run. A statement carrying no `UNNEST`
  * runs untouched, and one carrying an `UNNEST` this cannot turn into

package/dist/service/athena/engine/sim-athena-unnest-rewrite.js

@@ -6,10 +6,10 @@ import { simAthenaUnnestTargets } from "./sim-athena-unnest-targets.js";
 /**
  * Turn an `UNNEST` into the `json_each` SQLite flattens JSON with.
  *
- * An array or a map column is held as its JSON text, and `json_each` reads that
- * as one row per element with a `key` and a `value`. The `FROM` entry becomes a
- * call to it and every reference to the alias is pointed at the column it
- * answers with.
+ * An array or a map column is held as its JSON text, as is what a function like
+ * `split` answers, and `json_each` reads that as one row per element with a
+ * `key` and a `value`. The `FROM` entry becomes a call to it and every
+ * reference to the alias is pointed at the column it answers with.
  *
  * Answers with whether the statement can run. A statement carrying no `UNNEST`
  * runs untouched, and one carrying an `UNNEST` this cannot turn into

package/dist/service/athena/engine/sim-athena-unnest-source.d.ts

@@ -3,17 +3,18 @@ import { type SimAthenaAstNode } from "./sim-athena-ast-nodes.js";
 /** The two things Athena flattens with `UNNEST`. */
 export type SimAthenaUnnestKind = "array" | "map";
 /**
- * What the statement is flattening, read off the Glue schema.
+ * What the statement is flattening.
  *
- * The catalog is the only thing that says whether a column holds an array, a
- * map or a scalar, and `json_each` answers with different columns for the first
- * two. A column the schema calls anything else answers with nothing here, and
- * the query falls back rather than reading a value as a collection it never
- * was.
+ * A column is read off the Glue schema. The catalog is the only thing that says
+ * whether a column holds an array, a map or a scalar, and `json_each` answers
+ * with different columns for the first two. A column the schema calls anything
+ * else answers with nothing here, and the query falls back rather than reading
+ * a value as a collection it never was.
  *
- * An expression that is not a plain column reference answers with nothing too.
- * `UNNEST(split(x, ','))` is a real Athena query and the schema says nothing
- * about what it comes to.
+ * An expression is read off the function it calls. `UNNEST(split(x, ','))` is
+ * how a statement flattens a delimited string, and nothing but the call says
+ * what it comes to. A call to anything else answers with nothing, since the
+ * flattening would otherwise run over whatever SQLite made of a scalar.
  */
 export declare function simAthenaUnnestKind(source: SimAthenaAstNode, ast: unknown, tables: readonly SimAthenaCatalogTable[]): SimAthenaUnnestKind | undefined;
 /** Every catalog table the statement reads, by the name it reaches each one by. */

package/dist/service/athena/engine/sim-athena-unnest-source.js

@@ -1,20 +1,29 @@
-import { isColumnRef, isFromItem, simAthenaAstNodes, } from "./sim-athena-ast-nodes.js";
+import { isColumnRef, isFromItem, simAthenaAstNodes, simAthenaCalledName, } from "./sim-athena-ast-nodes.js";
 /**
- * What the statement is flattening, read off the Glue schema.
+ * The engine's own functions that answer an array.
  *
- * The catalog is the only thing that says whether a column holds an array, a
- * map or a scalar, and `json_each` answers with different columns for the first
- * two. A column the schema calls anything else answers with nothing here, and
- * the query falls back rather than reading a value as a collection it never
- * was.
+ * An expression has no catalog entry to read, so the call itself is what says
+ * the value is a collection. Each of these answers with the JSON text an array
+ * column is held as, which is what `json_each` reads.
+ */
+const arrayFunctions = new Set(["split", "slice"]);
+/**
+ * What the statement is flattening.
+ *
+ * A column is read off the Glue schema. The catalog is the only thing that says
+ * whether a column holds an array, a map or a scalar, and `json_each` answers
+ * with different columns for the first two. A column the schema calls anything
+ * else answers with nothing here, and the query falls back rather than reading
+ * a value as a collection it never was.
  *
- * An expression that is not a plain column reference answers with nothing too.
- * `UNNEST(split(x, ','))` is a real Athena query and the schema says nothing
- * about what it comes to.
+ * An expression is read off the function it calls. `UNNEST(split(x, ','))` is
+ * how a statement flattens a delimited string, and nothing but the call says
+ * what it comes to. A call to anything else answers with nothing, since the
+ * flattening would otherwise run over whatever SQLite made of a scalar.
  */
 export function simAthenaUnnestKind(source, ast, tables) {
     if (!isColumnRef(source)) {
-        return undefined;
+        return answersArray(source) ? "array" : undefined;
     }
     const declared = declaredType(source, ast, tables);
     if (declared === undefined) {
@@ -26,6 +35,11 @@ export function simAthenaUnnestKind(source, ast, tables) {
     }
     return type.startsWith("map") ? "map" : undefined;
 }
+/** Whether this expression is a call to a function answering an array. */
+function answersArray(source) {
+    const name = source["type"] === "function" ? simAthenaCalledName(source) : undefined;
+    return name !== undefined && arrayFunctions.has(name.toLowerCase());
+}
 /** Every catalog table the statement reads, by the name it reaches each one by. */
 export function simAthenaQueryTables(ast, tables) {
     const byName = new Map();

package/dist/service/athena/sim-athena-flattened-events.fixture.d.ts

@@ -0,0 +1,3 @@
+import { type SimAthenaEngineSimulation } from "./sim-athena-engine.fixture.js";
+/** A simulation holding the events, with the engine on. */
+export declare function anEventSimulation(): Promise<SimAthenaEngineSimulation>;

package/dist/service/athena/sim-athena-flattened-events.fixture.js

@@ -0,0 +1,46 @@
+import { aCatalogTable, anEngineSimulation, aSeededJson, } from "./sim-athena-engine.fixture.js";
+/**
+ * The events a flattening query reads.
+ *
+ * Each holds a collection of every shape an `UNNEST` reaches. An array and a
+ * map that are empty, a delimited string that is empty, and a row carrying no
+ * delimited string at all, which is what a flattening answering no rows runs
+ * over.
+ */
+const events = [
+    {
+        id: 1,
+        tags: ["red", "blue"],
+        attrs: { size: "large" },
+        name: "one",
+        packed: "red;blue",
+    },
+    { id: 2, tags: [], attrs: {}, name: "two", packed: "" },
+    {
+        id: 3,
+        tags: ["green"],
+        attrs: { size: "small", colour: "green" },
+        name: "three",
+    },
+];
+/** A simulation holding the events, with the engine on. */
+export async function anEventSimulation() {
+    const simulation = await anEngineSimulation();
+    aCatalogTable(simulation.simAws, {
+        name: "events",
+        columns: [
+            { Name: "id", Type: "int" },
+            { Name: "tags", Type: "array<string>" },
+            { Name: "attrs", Type: "map<string,string>" },
+            { Name: "name", Type: "string" },
+            { Name: "packed", Type: "string" },
+        ],
+    });
+    await aSeededJson(simulation.simAws, "events/part-0.json", events);
+    await simulation.simAws.athena().engine().enable();
+    simulation.simAws
+        .athena()
+        .results()
+        .byDefault({ columns: ["fallback"], rows: [["declared"]] });
+    return simulation;
+}

package/docs/services/athena/README.md

@@ -486,7 +486,7 @@ Every ascending sort is emitted carrying `NULLS LAST`, because Trino orders null
 direction it sorts and SQLite orders them first ascending. Both were cases where a query answered
 differently while still succeeding, which is the failure that costs the most to find.
 
-### Flattening an array or a map
+### Flattening an array, a map or a split value
 
 `UNNEST` runs. An array or a map column is held as its JSON text, and SQLite reads that with
 `json_each`, so a statement flattening one returns a row per element the way Athena does.
@@ -505,6 +505,22 @@ falls back rather than reading a scalar as a collection.
 One flattening per statement is what this covers, joined with `CROSS JOIN`. A second `UNNEST`, a
 `LEFT JOIN UNNEST`, a `SELECT *` beside one, and a position taken from a map all fall back.
 
+A table with no array column in it can still be flattened. `split` answers an array, and an `UNNEST`
+over the call cuts one stored string into a row per part. This is how a packed access log row is
+read, since Glue builds a log table's columns from what the delivery was configured with.
+
+```sql
+SELECT t.part
+FROM rainlytics.logs
+CROSS JOIN UNNEST(split(url_extract_parameter(cs_uri_stem || '?' || cs_uri_query, 'b'), ';'))
+  AS t(part)
+```
+
+Splitting an empty value answers one empty part, as it does on Trino. A row whose value is null
+answers no parts at all and drops out, the way an empty array column does. The call is what says the
+value is a collection (an expression has no schema entry to read). `split` and `slice` are the two
+calls the flattening reaches, and any other expression under an `UNNEST` falls back.
+
 ### The functions a statement can call
 
 SQLite carries a much smaller function library than Trino, and the engine fills the gap for the ones
@@ -516,7 +532,7 @@ its declared result.
 | Date and time | `current_date`, `current_timestamp`, `date_add`, `date_diff`, `date_trunc`, `date_format`, `at_timezone`, `from_unixtime`, `to_unixtime`, `from_iso8601_timestamp`, `from_iso8601_date`, `to_iso8601` |
 | JSON          | `json_extract`, `json_extract_scalar`, `json_parse`, `json_size`                                                                                                                                      |
 | Array and map | `array_agg`, `cardinality`, `contains`, `element_at`, `array_join`, `slice`                                                                                                                           |
-| String        | `regexp_like`, `regexp_extract`, `regexp_replace`, `split_part`, `strpos`                                                                                                                             |
+| String        | `regexp_like`, `regexp_extract`, `regexp_replace`, `split`, `split_part`, `strpos`                                                                                                                    |
 | Binary        | `md5`, `sha1`, `sha256`, `sha512`, `xxhash64`, `murmur3`, `crc32`, `to_hex`, `from_hex`, `to_base64`, `from_base64`, `to_utf8`, `from_utf8`                                                           |
 | URL           | `url_extract_host`, `url_extract_path`, `url_extract_protocol`, `url_extract_port`, `url_extract_query`, `url_extract_fragment`, `url_extract_parameter`, `url_decode`, `url_encode`                  |
 | Approximate   | `approx_distinct`, `approx_percentile`                                                                                                                                                                |
@@ -992,6 +1008,7 @@ own and authorizes work on one against the workgroup it belongs to. This asks th
 - `StartQueryExecution`, `GetQueryExecution`, `GetQueryResults` and `StopQueryExecution`
 - A `SELECT` run for real over JSON lines and CSV objects in simulated S3, answered by SQLite
 - `UNNEST` over an array or a map column, with `WITH ORDINALITY` where a query wants the position
+- `UNNEST` over a `split` or a `slice` call, flattening one stored string into a row per part
 - Trino's date, JSON, array, string and URL functions, with `current_timestamp` reading the
   simulated clock
 - Declared results, matched on the query text, ahead of the engine for one statement and behind it
@@ -1027,6 +1044,8 @@ Current documented limitations:
   gives a map's keys rather than its positions.
 - `UNNEST` over a `ROW` or a struct array falls back. The element needs field access and the
   flattened column is JSON text here.
+- `UNNEST` over an expression reaches `split` and `slice`. The engine reads the call to decide that
+  a value is a collection, and a call to anything else falls back.
 - The Trino function library reaches as far as the table under
   [the functions a statement can call](#the-functions-a-statement-can-call). A query reaching for
   anything else Trino has and SQLite lacks falls back.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kensio/yulin",
-  "version": "1.21.22",
+  "version": "1.21.23",
   "description": "AWS system behaviour simulation for isolated unit testing",
   "repository": "https://github.com/KensioSoftware/yulin",
   "homepage": "https://yulinsim.dev/",