@kensio/yulin 1.20.10 → 1.20.12

package/dist/service/athena/engine/sim-athena-json-records.d.ts

@@ -0,0 +1,23 @@
+import type { SimAthenaEngineRow } from "./sim-athena-engine-row.js";
+/**
+ * How one table's JSON records reach its columns.
+ *
+ * `mappings` holds the column names the OpenX SerDe's `mapping.<column>`
+ * parameters declare, each against the record key it reads. A table declaring
+ * none leaves it empty, and every column is read by its own name.
+ *
+ * `caseInsensitive` is the SerDe's own `case.insensitive`, on unless a table
+ * turns it off. The SerDe folds a record's keys before looking one up. A
+ * mapping matches a key of any case until a table declares `FALSE`.
+ */
+export interface SimAthenaJsonFormat {
+    readonly mappings: ReadonlyMap<string, string>;
+    readonly caseInsensitive: boolean;
+}
+/**
+ * One object of JSON lines, read into rows.
+ *
+ * A nested object or array is kept as its JSON text, which is what makes
+ * `json_extract_scalar` and `cardinality` reach into it.
+ */
+export declare function simAthenaJsonRows(text: string, format: SimAthenaJsonFormat): readonly SimAthenaEngineRow[];

package/dist/service/athena/engine/sim-athena-json-records.js

@@ -0,0 +1,46 @@
+/**
+ * One object of JSON lines, read into rows.
+ *
+ * A nested object or array is kept as its JSON text, which is what makes
+ * `json_extract_scalar` and `cardinality` reach into it.
+ */
+export function simAthenaJsonRows(text, format) {
+    return text
+        .split("\n")
+        .filter((line) => line.trim().length > 0)
+        .map((line) => JSON.parse(line))
+        .map((record) => mappedRow(record, format));
+}
+/**
+ * One record with its mapped columns laid over it.
+ *
+ * The record is kept whole underneath. A column no mapping names still reads
+ * by its own name. A mapped column is written whether or not the record
+ * carries the key, since the mapping is where that column reads from.
+ */
+function mappedRow(record, format) {
+    if (format.mappings.size === 0) {
+        return record;
+    }
+    const row = { ...record };
+    for (const [column, key] of format.mappings) {
+        row[column] = mappedValue(record, key, format.caseInsensitive);
+    }
+    return row;
+}
+function mappedValue(record, key, caseInsensitive) {
+    const exact = record[key];
+    if (exact !== undefined) {
+        return exact;
+    }
+    if (!caseInsensitive) {
+        return null;
+    }
+    const wanted = key.toLowerCase();
+    for (const [held, value] of Object.entries(record)) {
+        if (held.toLowerCase() === wanted) {
+            return value;
+        }
+    }
+    return null;
+}

package/dist/service/athena/engine/sim-athena-object-codec.d.ts

@@ -0,0 +1,13 @@
+/**
+ * One object's bytes, decompressed by the codec its key names.
+ *
+ * Athena reads the file extension, and so does this. A key ending `.gz` is
+ * gzip whatever its bytes hold, and a key ending anything else is text. The
+ * magic number would be the other way to tell, and it would read an object
+ * real Athena skips.
+ *
+ * A key naming a codec this simulation has no decompressor for raises, and the
+ * engine turns the query down. The declaration a test wrote answers it, which
+ * is what an object the engine cannot open already does.
+ */
+export declare function simAthenaDecompressedBytes(key: string, bytes: Buffer): Buffer;

package/dist/service/athena/engine/sim-athena-object-codec.js

@@ -0,0 +1,40 @@
+import { gunzipSync, inflateSync, zstdDecompressSync } from "node:zlib";
+import { SimAthenaSetUpError } from "../error/sim-athena.error.js";
+/** The file extensions naming a codec, against what decompresses each one. */
+const codecs = new Map([
+    ["gz", gunzipSync],
+    ["zst", zstdDecompressSync],
+    ["deflate", inflateSync],
+]);
+/**
+ * The file extensions naming a codec nothing here decompresses.
+ *
+ * Node's standard library has gzip, zstd and deflate. The rest would need a
+ * dependency, and this package takes none for a test to install.
+ */
+const unreadableCodecs = new Set(["bz2", "bzip2", "lz4", "lzo", "snappy"]);
+/**
+ * One object's bytes, decompressed by the codec its key names.
+ *
+ * Athena reads the file extension, and so does this. A key ending `.gz` is
+ * gzip whatever its bytes hold, and a key ending anything else is text. The
+ * magic number would be the other way to tell, and it would read an object
+ * real Athena skips.
+ *
+ * A key naming a codec this simulation has no decompressor for raises, and the
+ * engine turns the query down. The declaration a test wrote answers it, which
+ * is what an object the engine cannot open already does.
+ */
+export function simAthenaDecompressedBytes(key, bytes) {
+    const name = key.slice(key.lastIndexOf("/") + 1);
+    const dot = name.lastIndexOf(".");
+    if (dot === -1) {
+        return bytes;
+    }
+    const extension = name.slice(dot + 1).toLowerCase();
+    if (unreadableCodecs.has(extension)) {
+        throw new SimAthenaSetUpError(`Unsupported sim Athena compression: ${extension}`);
+    }
+    const decompress = codecs.get(extension);
+    return decompress === undefined ? bytes : decompress(bytes);
+}

package/dist/service/athena/engine/sim-athena-record-reader.js

@@ -1,10 +1,15 @@
 import { simAthenaDelimitedRows, } from "./sim-athena-delimited-records.js";
+import { simAthenaJsonRows, } from "./sim-athena-json-records.js";
+/** The SerDe class name that takes `mapping.<column>` parameters. */
+const openXSerDe = "org.openx.data.jsonserde.jsonserde";
 /** The SerDe class names that mean JSON lines. */
 const jsonSerDes = new Set([
-    "org.openx.data.jsonserde.jsonserde",
+    openXSerDe,
     "org.apache.hive.hcatalog.data.jsonserde",
     "org.apache.hadoop.hive.serde2.jsonserde",
 ]);
+/** What a `mapping.<column>` parameter is named with. */
+const mappingPrefix = "mapping.";
 /**
  * The SerDe class names that mean delimited text, and what each one reads
  * before the table's own parameters are applied.
@@ -39,7 +44,8 @@ export function simAthenaRecordReader(table) {
         return undefined;
     }
     if (jsonSerDes.has(library)) {
-        return jsonRows;
+        const format = jsonFormat(table, library);
+        return (text) => simAthenaJsonRows(text, format);
     }
     const defaults = delimitedSerDes.get(library);
     if (defaults === undefined) {
@@ -50,16 +56,26 @@ export function simAthenaRecordReader(table) {
     return (text) => simAthenaDelimitedRows(text, format, columns);
 }
 /**
- * JSON lines, one record per line.
+ * How one table's JSON records reach its columns.
  *
- * A nested object or array is kept as its JSON text, which is what makes
- * `json_extract_scalar` and `cardinality` reach into it.
+ * Only the OpenX SerDe takes the mappings. The Hive JSON SerDes have no
+ * `mapping` property, and a table declaring one against them reads by its
+ * column names on real Athena the same as here.
  */
-function jsonRows(text) {
-    return text
-        .split("\n")
-        .filter((line) => line.trim().length > 0)
-        .map((line) => JSON.parse(line));
+function jsonFormat(table, library) {
+    const parameters = library === openXSerDe
+        ? (table.storageDescriptor?.SerdeInfo?.Parameters ?? {})
+        : {};
+    const mappings = new Map();
+    for (const [name, key] of Object.entries(parameters)) {
+        if (name.toLowerCase().startsWith(mappingPrefix)) {
+            mappings.set(name.slice(mappingPrefix.length), key);
+        }
+    }
+    return {
+        mappings,
+        caseInsensitive: parameters["case.insensitive"]?.toLowerCase() !== "false",
+    };
 }
 function delimitedFormat(table, defaults) {
     const parameters = table.storageDescriptor?.SerdeInfo?.Parameters ?? {};

package/dist/service/athena/engine/sim-athena-regexp-flags.d.ts

@@ -0,0 +1,22 @@
+/** One pattern and the flags to compile it with. */
+export interface SimAthenaLiftedPattern {
+    readonly pattern: string;
+    readonly flags: string;
+}
+/**
+ * One pattern with the flags Java writes inside it lifted out beside it.
+ *
+ * Athena's regular expressions are Joni, which reads `(?i)` at the head of a
+ * pattern as a flag over the whole of it. JavaScript reads the same text as a
+ * group and refuses it. `(?i)`, `(?m)` and `(?s)` map onto flags of the same
+ * letter, and lifting them is all it takes.
+ *
+ * A flag Joni has and JavaScript has not, `(?x)` among them, stays in the
+ * pattern for `RegExp` to turn down. So does a group written anywhere but the
+ * head, since JavaScript has no way to turn a flag on part way through. The
+ * scoped form `(?i:...)` needs nothing done to it and already runs.
+ *
+ * A flag group captures nothing in Java, so taking one off the front leaves
+ * the capture groups numbered as the statement wrote them.
+ */
+export declare function simAthenaLiftedPattern(pattern: string, flags: string): SimAthenaLiftedPattern;

package/dist/service/athena/engine/sim-athena-regexp-flags.js

@@ -0,0 +1,34 @@
+/** One of Java's inline flag groups at the head of a pattern, as `(?i)`. */
+const leadingFlag = /^\(\?[ims]+\)/u;
+/**
+ * One pattern with the flags Java writes inside it lifted out beside it.
+ *
+ * Athena's regular expressions are Joni, which reads `(?i)` at the head of a
+ * pattern as a flag over the whole of it. JavaScript reads the same text as a
+ * group and refuses it. `(?i)`, `(?m)` and `(?s)` map onto flags of the same
+ * letter, and lifting them is all it takes.
+ *
+ * A flag Joni has and JavaScript has not, `(?x)` among them, stays in the
+ * pattern for `RegExp` to turn down. So does a group written anywhere but the
+ * head, since JavaScript has no way to turn a flag on part way through. The
+ * scoped form `(?i:...)` needs nothing done to it and already runs.
+ *
+ * A flag group captures nothing in Java, so taking one off the front leaves
+ * the capture groups numbered as the statement wrote them.
+ */
+export function simAthenaLiftedPattern(pattern, flags) {
+    const letters = new Set();
+    let rest = pattern;
+    let head = leadingFlag.exec(rest);
+    while (head !== null) {
+        // Everything the group holds between its `(?` and its `)`.
+        const written = head[0].slice(2, -1);
+        for (const letter of written) {
+            letters.add(letter);
+        }
+        rest = rest.slice(head[0].length);
+        head = leadingFlag.exec(rest);
+    }
+    const added = [...letters].filter((letter) => !flags.includes(letter));
+    return { pattern: rest, flags: flags + added.join("") };
+}

package/dist/service/athena/engine/sim-athena-regexp-shims.js

@@ -1,4 +1,5 @@
 import { SimAthenaSetUpError } from "../error/sim-athena.error.js";
+import { simAthenaLiftedPattern } from "./sim-athena-regexp-flags.js";
 import { isExplicitNull, shimNumber, shimText, simAthenaScalarShim, } from "./sim-athena-shim-registry.js";
 /**
  * Trino's regular expression functions.
@@ -79,10 +80,11 @@ function expressionFor(pattern, flags = "u") {
     if (pattern === undefined) {
         return undefined;
     }
+    const lifted = simAthenaLiftedPattern(pattern, flags);
     try {
         // The pattern is the query's own, which is the whole point of the function.
         // oxlint-disable-next-line security/detect-non-literal-regexp
-        return new RegExp(pattern, flags);
+        return new RegExp(lifted.pattern, lifted.flags);
     }
     catch {
         return undefined;

package/dist/service/athena/engine/sim-athena-table-objects.d.ts

@@ -26,5 +26,5 @@ export interface SimAthenaTableObjects extends SimAthenaScannedObjects {
  * there because nothing holds the data it would answer from.
  */
 export declare function simAthenaTableObjects(s3: Partial<SimAthenaTableObjects> | undefined): SimAthenaTableObjects | undefined;
-/** One object's bytes, read as UTF-8 text. */
+/** One object's bytes, decompressed by its key and read as UTF-8 text. */
 export declare function simAthenaObjectText(objects: SimAthenaTableObjects, bucket: string, key: string, caller: SimAwsCaller | undefined): Promise<string>;

package/dist/service/athena/engine/sim-athena-table-objects.js

@@ -1,3 +1,4 @@
+import { simAthenaDecompressedBytes } from "./sim-athena-object-codec.js";
 /**
  * Whether this simulated S3 can open an object as well as list one.
  *
@@ -9,9 +10,10 @@ export function simAthenaTableObjects(s3) {
         ? undefined
         : s3;
 }
-/** One object's bytes, read as UTF-8 text. */
+/** One object's bytes, decompressed by its key and read as UTF-8 text. */
 export async function simAthenaObjectText(objects, bucket, key, caller) {
     const got = await objects.getObject({ input: { Bucket: bucket, Key: key } }, caller === undefined ? undefined : { caller });
     const chunks = await Array.fromAsync(got.Body ?? []);
-    return new TextDecoder().decode(Buffer.concat(chunks));
+    const bytes = simAthenaDecompressedBytes(key, Buffer.concat(chunks));
+    return new TextDecoder().decode(bytes);
 }

package/dist/service/athena/engine/sim-athena-url-parts.d.ts

@@ -0,0 +1,24 @@
+/** One URI reference split the way Java's `URI` splits it. */
+export interface SimAthenaUrlParts {
+    readonly protocol: string;
+    readonly host: string;
+    readonly port: number | null;
+    readonly path: string;
+    readonly query: string;
+    readonly fragment: string;
+}
+/**
+ * One URI reference split into its parts, or nothing where it is no reference.
+ *
+ * Trino reads a URL with `new URI(text)`, which takes a reference with no
+ * scheme of its own. `/reports/august?tenant=acme` is one, and it is the shape
+ * a CloudFront access log holds, since the log carries the path and the query
+ * in columns of their own and no whole URL anywhere.
+ *
+ * The parts are read off the text rather than resolved against a base. That is
+ * what `URI` does, and it keeps a relative path as the text wrote it.
+ *
+ * A part the reference leaves out comes back as the empty string, which is
+ * what Trino answers for one. The port has no empty form and answers null.
+ */
+export declare function simAthenaUrlParts(text: string): SimAthenaUrlParts | undefined;

package/dist/service/athena/engine/sim-athena-url-parts.js

@@ -0,0 +1,76 @@
+/**
+ * A URI reference, split as RFC 3986 writes the expression for it.
+ *
+ * This is the expression the RFC prints in its own appendix, character for
+ * character. Its groups are optional and never nested, so what a reference
+ * that does not match costs in backtracking is bounded by the text's length.
+ */
+const reference = 
+// oxlint-disable-next-line security/detect-unsafe-regex
+/^(?:([^:/?#]+):)?(?:\/\/([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?$/u;
+/** The characters RFC 2396 allows, which is what Java's `URI` takes. */
+const disallowed = /[^A-Za-z0-9;/?:@&=+$,\-_.!~*'()%[\]#]/u;
+/** A percent naming no byte, which Java's `URI` reads as malformed. */
+const malformedEscape = /%(?![0-9A-Fa-f]{2})/u;
+/** A scheme starts with a letter and carries no other punctuation. */
+const schemeName = /^[A-Za-z][A-Za-z0-9+.-]*$/u;
+const digits = /^\d+$/u;
+/**
+ * One URI reference split into its parts, or nothing where it is no reference.
+ *
+ * Trino reads a URL with `new URI(text)`, which takes a reference with no
+ * scheme of its own. `/reports/august?tenant=acme` is one, and it is the shape
+ * a CloudFront access log holds, since the log carries the path and the query
+ * in columns of their own and no whole URL anywhere.
+ *
+ * The parts are read off the text rather than resolved against a base. That is
+ * what `URI` does, and it keeps a relative path as the text wrote it.
+ *
+ * A part the reference leaves out comes back as the empty string, which is
+ * what Trino answers for one. The port has no empty form and answers null.
+ */
+export function simAthenaUrlParts(text) {
+    if (disallowed.test(text) || malformedEscape.test(text)) {
+        return undefined;
+    }
+    const found = reference.exec(text);
+    if (found === null) {
+        return undefined;
+    }
+    const scheme = found.at(1);
+    if (scheme !== undefined && !schemeName.test(scheme)) {
+        return undefined;
+    }
+    const authority = found.at(2);
+    const path = found.at(3) ?? "";
+    // A scheme followed by anything but a slash is an opaque URI, and Java reads
+    // neither a path nor a query out of one. `mailto:a@b` is the case.
+    const opaque = scheme !== undefined && authority === undefined && !path.startsWith("/");
+    return {
+        protocol: scheme ?? "",
+        ...hostAndPort(authority),
+        path: opaque ? "" : path,
+        query: opaque ? "" : (found.at(4) ?? ""),
+        fragment: found.at(5) ?? "",
+    };
+}
+/**
+ * The host and the port one authority names.
+ *
+ * The user's own credentials sit before an `@` and are no part of the host, so
+ * the digits in `user:1234@rain.example` are no port. A colon with anything
+ * but digits after it belongs to the host, which is what keeps the one inside
+ * an IPv6 literal out of the port.
+ */
+function hostAndPort(authority) {
+    if (authority === undefined) {
+        return { host: "", port: null };
+    }
+    const written = authority.slice(authority.lastIndexOf("@") + 1);
+    const colon = written.lastIndexOf(":");
+    const port = written.slice(colon + 1);
+    if (colon === -1 || !digits.test(port)) {
+        return { host: written, port: null };
+    }
+    return { host: written.slice(0, colon), port: Number(port) };
+}

package/dist/service/athena/engine/sim-athena-url-shims.d.ts

@@ -2,8 +2,9 @@ import type { DatabaseSync } from "node:sqlite";
 /**
  * Trino's URL functions, which a query over access logs reaches for.
  *
- * Trino fails a query over text that is no URL and these answer null, the same
- * forgiving direction the rest of the engine takes.
+ * The extract functions answer null over text that is no URI reference, and so
+ * does Trino. Each one is `neverFails` there and answers null off a `URI` that
+ * would not parse.
  *
  * Trino answers with an empty string for a part the URL leaves out, apart from
  * the port, which has no empty form and answers null.

package/dist/service/athena/engine/sim-athena-url-shims.js

@@ -1,17 +1,20 @@
 import { shimText, simAthenaScalarShim } from "./sim-athena-shim-registry.js";
-/** What each `url_extract` function reads off a parsed URL. */
+import { simAthenaUrlParts, } from "./sim-athena-url-parts.js";
+/** What each `url_extract` function reads off a split reference. */
 const parts = new Map([
-    ["url_extract_host", (url) => url.hostname],
-    ["url_extract_path", (url) => url.pathname],
-    ["url_extract_protocol", (url) => url.protocol.replace(":", "")],
-    ["url_extract_fragment", (url) => url.hash.replace("#", "")],
-    ["url_extract_query", (url) => url.search.replace("?", "")],
+    ["url_extract_host", (url) => url.host],
+    ["url_extract_path", (url) => url.path],
+    ["url_extract_protocol", (url) => url.protocol],
+    ["url_extract_fragment", (url) => url.fragment],
+    ["url_extract_query", (url) => url.query],
+    ["url_extract_port", (url) => url.port],
 ]);
 /**
  * Trino's URL functions, which a query over access logs reaches for.
  *
- * Trino fails a query over text that is no URL and these answer null, the same
- * forgiving direction the rest of the engine takes.
+ * The extract functions answer null over text that is no URI reference, and so
+ * does Trino. Each one is `neverFails` there and answers null off a `URI` that
+ * would not parse.
  *
  * Trino answers with an empty string for a part the URL leaves out, apart from
  * the port, which has no empty form and answers null.
@@ -19,39 +22,68 @@ const parts = new Map([
 export function simAthenaInstallUrlShims(database) {
     for (const [name, read] of parts) {
         simAthenaScalarShim(database, name, (value) => {
-            const url = parsedUrl(shimText(value));
+            const url = readParts(shimText(value));
             return url === undefined ? null : read(url);
         });
     }
-    simAthenaScalarShim(database, "url_extract_port", (value) => writtenPort(shimText(value)));
     simAthenaScalarShim(database, "url_extract_parameter", (value, name) => {
         const wanted = shimText(name);
-        const url = parsedUrl(shimText(value));
+        const url = readParts(shimText(value));
         if (url === undefined || wanted === undefined) {
             return null;
         }
-        return url.searchParams.get(wanted);
+        return new URLSearchParams(url.query).get(wanted);
     });
+    simAthenaScalarShim(database, "url_decode", (value) => decoded(shimText(value)));
+    simAthenaScalarShim(database, "url_encode", (value) => encoded(shimText(value)));
 }
-/** Everything between the scheme and the path, which is where a port is written. */
-const authority = /^[A-Za-z][\w+.-]*:\/\/[^/?#]*/u;
 /**
- * The port one URL names, or nothing where it names none.
+ * One value with its escapes read back, the way `url_decode` reads them.
  *
- * Read off the text rather than off the parsed URL, because the parser drops a
- * port that is the scheme's own default. Trino answers with the port a URL was
- * written with, so `http://rain.example:80/a` is eighty rather than nothing.
+ * Trino runs `URLDecoder.decode` over UTF-8, which reads `+` as a space and
+ * `%2B` as a plus. Replacing the plus before decoding is what keeps the two
+ * apart.
+ *
+ * Trino raises over an escape that names no byte, and writes a replacement
+ * character where the bytes it names are no UTF-8. `decodeURIComponent` throws
+ * over both, and this answers null for both, the same forgiving direction the
+ * rest of the file takes.
  */
-function writtenPort(value) {
-    if (value === undefined || parsedUrl(value) === undefined) {
+function decoded(value) {
+    if (value === undefined) {
+        return null;
+    }
+    try {
+        return decodeURIComponent(value.replaceAll("+", " "));
+    }
+    catch {
         return null;
     }
-    const written = /:(\d+)$/u.exec(authority.exec(value)?.[0] ?? "")?.[1];
-    return written === undefined ? null : Number(written);
 }
-function parsedUrl(value) {
+/**
+ * One value escaped for a query string, the way `url_encode` escapes it.
+ *
+ * Trino runs Guava's form-parameter escaper, which keeps `-`, `_`, `.` and `*`
+ * alone and writes a space as `+`. `encodeURIComponent` keeps five characters
+ * beyond those four, and each of those is escaped here.
+ *
+ * No escape written here carries a character a later pass looks for, and a
+ * space is the only character `encodeURIComponent` writes as `%20`, since a
+ * percent in the value has already become `%25` by then.
+ */
+function encoded(value) {
     if (value === undefined) {
-        return undefined;
+        return null;
     }
-    return URL.parse(value) ?? undefined;
+    return encodeURIComponent(value)
+        .replaceAll("!", "%21")
+        .replaceAll("'", "%27")
+        .replaceAll("(", "%28")
+        .replaceAll(")", "%29")
+        .replaceAll("~", "%7E")
+        .replaceAll("%20", "+");
+}
+/** One value split into its URL parts, or nothing where it is null or no URL. */
+function readParts(value) {
+    return value === undefined ? undefined : simAthenaUrlParts(value);
 }

package/dist/service/athena/sim-athena-engine.fixture.d.ts

@@ -34,5 +34,7 @@ export declare function anEngineSimulation(clock?: SimClock): Promise<SimAthenaE
 export declare function aCatalogTable(simAws: SimAws, table: SimAthenaEngineTableInput): void;
 /** Put one object of literal text under the logs Bucket. */
 export declare function aSeededObject(simAws: SimAws, key: string, body: string): Promise<void>;
+/** Put one object of raw bytes under the logs Bucket. */
+export declare function aSeededBytes(simAws: SimAws, key: string, body: Uint8Array): Promise<void>;
 /** Put one object of JSON lines under the logs Bucket. */
 export declare function aSeededJson(simAws: SimAws, key: string, records: readonly Record<string, unknown>[]): Promise<void>;

package/dist/service/athena/sim-athena-engine.fixture.js

@@ -58,6 +58,12 @@ export async function aSeededObject(simAws, key, body) {
         .s3()
         .putObject({ input: { Bucket: logsBucket, Key: key, Body: body } });
 }
+/** Put one object of raw bytes under the logs Bucket. */
+export async function aSeededBytes(simAws, key, body) {
+    await simAws
+        .s3()
+        .putObject({ input: { Bucket: logsBucket, Key: key, Body: body } });
+}
 /** Put one object of JSON lines under the logs Bucket. */
 export async function aSeededJson(simAws, key, records) {
     const lines = records.map((record) => JSON.stringify(record));

package/dist/service/athena/table/sim-athena-negated-columns.d.ts

@@ -0,0 +1,17 @@
+import type { SimAthenaSqlToken } from "./sim-athena-sql-tokens.js";
+/**
+ * The columns a `NOT` in this query could have negated.
+ *
+ * A negated value names the partition a query does not want, so a column under
+ * a `NOT` is left unconstrained. A query filtering bots out of a day's logs
+ * negates nothing about the day, and reading how far each `NOT` reaches is
+ * what tells the two apart.
+ *
+ * An infix `NOT`, as in `day NOT IN ('a')`, is read as reaching nothing. Its
+ * own column keeps whatever else the statement constrains it to, and that is
+ * always a superset of what Athena reads, since `day = 'a' AND day NOT IN (x)`
+ * can only ever narrow. The query applies the exclusion itself once the rows
+ * are in. Naming the column here would drop its filter and widen the scan to
+ * every partition.
+ */
+export declare function simAthenaNegatedColumns(tokens: readonly SimAthenaSqlToken[]): ReadonlySet<string>;

package/dist/service/athena/table/sim-athena-negated-columns.js

@@ -0,0 +1,80 @@
+/**
+ * The columns a `NOT` in this query could have negated.
+ *
+ * A negated value names the partition a query does not want, so a column under
+ * a `NOT` is left unconstrained. A query filtering bots out of a day's logs
+ * negates nothing about the day, and reading how far each `NOT` reaches is
+ * what tells the two apart.
+ *
+ * An infix `NOT`, as in `day NOT IN ('a')`, is read as reaching nothing. Its
+ * own column keeps whatever else the statement constrains it to, and that is
+ * always a superset of what Athena reads, since `day = 'a' AND day NOT IN (x)`
+ * can only ever narrow. The query applies the exclusion itself once the rows
+ * are in. Naming the column here would drop its filter and widen the scan to
+ * every partition.
+ */
+export function simAthenaNegatedColumns(tokens) {
+    const negated = new Set();
+    for (const [position, token] of tokens.entries()) {
+        if (token.kind === "word" && token.text === "not") {
+            for (const name of namesUnder(tokens, position)) {
+                negated.add(name.toLowerCase());
+            }
+        }
+    }
+    return negated;
+}
+/** The words a `NOT` reaches no further than, `OR` aside. */
+const boundary = new Set([
+    "and",
+    "except",
+    "from",
+    "group",
+    "having",
+    "intersect",
+    "limit",
+    "offset",
+    "order",
+    "union",
+    "where",
+    "window",
+]);
+/**
+ * The identifiers one `NOT` reaches.
+ *
+ * `NOT` binds tighter than `AND`, so its reach ends at the next of the words
+ * above, outside any brackets, and at the bracket the `NOT` itself sits in.
+ * `OR` is absent from that list because a query carrying one is left
+ * unfiltered before this runs.
+ *
+ * Nothing here parses, so a name a `NOT` could not really have reached is
+ * dropped anyway. That direction only ever leaves a column unfiltered, which
+ * is the answer a query with no filter gets.
+ */
+function namesUnder(tokens, position) {
+    const names = [];
+    let depth = 0;
+    let cursor = position + 1;
+    while (cursor < tokens.length) {
+        const token = tokens.at(cursor);
+        if (token?.kind === "symbol" && token.text === "(") {
+            depth += 1;
+        }
+        else if (token?.kind === "symbol" && token.text === ")") {
+            if (depth === 0) {
+                return names;
+            }
+            depth -= 1;
+        }
+        else if (depth === 0 &&
+            token?.kind === "word" &&
+            boundary.has(token.text)) {
+            return names;
+        }
+        else if (token?.kind === "word" || token?.kind === "quoted") {
+            names.push(token.text);
+        }
+        cursor += 1;
+    }
+    return names;
+}

package/dist/service/athena/table/sim-athena-partition-filters.d.ts

@@ -2,10 +2,13 @@
  * The partition values a query's `WHERE` clause pins down.
  *
  * Only two shapes are read, `column = 'value'` and `column IN ('a', 'b')`, and
- * they are read wherever they appear. A query carrying `OR` or `NOT` anywhere
- * is left unfiltered. A value under one arm of an `OR` constrains nothing on
- * its own, and a negated one names the partition the query does not want, so
- * reading either as a constraint would answer from the wrong prefixes.
+ * they are read wherever they appear. A query carrying `OR` anywhere is left
+ * unfiltered, since a value under one arm constrains nothing on its own.
+ *
+ * A `NOT` leaves the columns it reaches unconstrained and the rest alone. A
+ * negated value names the partition the query does not want, and reading it as
+ * a constraint would answer from the wrong prefixes. A negation on any other
+ * column leaves the partition constraints as true as they were.
  *
  * Two constraints on one column are intersected, since a query carrying both
  * wants the rows they agree on.

package/dist/service/athena/table/sim-athena-partition-filters.js

@@ -1,12 +1,16 @@
+import { simAthenaNegatedColumns } from "./sim-athena-negated-columns.js";
 import { simAthenaSqlTokens } from "./sim-athena-sql-tokens.js";
 /**
  * The partition values a query's `WHERE` clause pins down.
  *
  * Only two shapes are read, `column = 'value'` and `column IN ('a', 'b')`, and
- * they are read wherever they appear. A query carrying `OR` or `NOT` anywhere
- * is left unfiltered. A value under one arm of an `OR` constrains nothing on
- * its own, and a negated one names the partition the query does not want, so
- * reading either as a constraint would answer from the wrong prefixes.
+ * they are read wherever they appear. A query carrying `OR` anywhere is left
+ * unfiltered, since a value under one arm constrains nothing on its own.
+ *
+ * A `NOT` leaves the columns it reaches unconstrained and the rest alone. A
+ * negated value names the partition the query does not want, and reading it as
+ * a constraint would answer from the wrong prefixes. A negation on any other
+ * column leaves the partition constraints as true as they were.
  *
  * Two constraints on one column are intersected, since a query carrying both
  * wants the rows they agree on.
@@ -52,6 +56,9 @@ export function simAthenaPartitionFilters(sql) {
             }
         }
     }
+    for (const name of simAthenaNegatedColumns(tokens)) {
+        byColumn.delete(name);
+    }
     return new SimAthenaPartitionFilters(byColumn);
 }
 /**
@@ -68,7 +75,7 @@ function narrow(byColumn, columnName, values) {
     byColumn.set(name, already.filter((value) => wanted.has(value)));
 }
 function isUnreadableTerm(token) {
-    return token.kind === "word" && (token.text === "or" || token.text === "not");
+    return token.kind === "word" && token.text === "or";
 }
 /**
  * Read `('a', 'b')` starting at its opening bracket.

package/dist/service/wafv2/command/ip-set/sim-wafv2-ip-set-commands.js

@@ -1,7 +1,7 @@
 import { SimWafIpSet, requiredSimWafIpAddressVersion, } from "../../ip-set/sim-waf-ip-set.js";
 import { requiredSimWafScope } from "../../scope/sim-waf-scope.js";
 import { SimWafPage } from "../sim-wafv2-page.js";
-import { refuseSimWafTags, requiredSimWafName } from "../sim-wafv2-input.js";
+import { checkedSimWafDescription, refuseSimWafTags, requiredSimWafName, } from "../sim-wafv2-input.js";
 import { requireSimWafResource } from "../sim-wafv2-resource-lookup.js";
 /**
  * The commands that make, read, list and remove IP sets.
@@ -29,7 +29,7 @@ export class SimWafIpSetCommands {
             name: requiredSimWafName(input.Name),
             scope: requiredSimWafScope(input.Scope, this.#accountRegionScope.regionName),
             accountRegionScope: this.#accountRegionScope,
-            description: input.Description,
+            description: checkedSimWafDescription(input.Description),
             ipAddressVersion: requiredSimWafIpAddressVersion(input.IPAddressVersion),
             addresses: input.Addresses ?? [],
         });
@@ -59,10 +59,11 @@ export class SimWafIpSetCommands {
      */
     updateIpSet(command, options) {
         const { input } = command;
+        const description = checkedSimWafDescription(input.Description);
         const ipSet = this.require(input, "wafv2:UpdateIPSet", options);
         ipSet.replaceAddresses({
             addresses: input.Addresses ?? [],
-            description: input.Description,
+            description,
             lockToken: input.LockToken,
         });
         return { $metadata: {}, NextLockToken: ipSet.lockToken };

package/dist/service/wafv2/command/regex-pattern-set/sim-wafv2-regex-pattern-set-commands.js

@@ -1,7 +1,7 @@
 import { SimWafRegexPatternSet } from "../../regex-pattern-set/sim-waf-regex-pattern-set.js";
 import { requiredSimWafScope } from "../../scope/sim-waf-scope.js";
 import { SimWafPage } from "../sim-wafv2-page.js";
-import { refuseSimWafTags, requiredSimWafName } from "../sim-wafv2-input.js";
+import { checkedSimWafDescription, refuseSimWafTags, requiredSimWafName, } from "../sim-wafv2-input.js";
 import { requireSimWafResource } from "../sim-wafv2-resource-lookup.js";
 /**
  * The commands that make, read, list and remove regex pattern sets.
@@ -29,7 +29,7 @@ export class SimWafRegexPatternSetCommands {
             name: requiredSimWafName(input.Name),
             scope: requiredSimWafScope(input.Scope, this.#accountRegionScope.regionName),
             accountRegionScope: this.#accountRegionScope,
-            description: input.Description,
+            description: checkedSimWafDescription(input.Description),
             regularExpressions: input.RegularExpressionList ?? [],
         });
         this.#authorizer.authorizeResource("wafv2:CreateRegexPatternSet", patternSet.arn, options?.caller);
@@ -66,10 +66,11 @@ export class SimWafRegexPatternSetCommands {
      */
     updateRegexPatternSet(command, options) {
         const { input } = command;
+        const description = checkedSimWafDescription(input.Description);
         const patternSet = this.require(input, "wafv2:UpdateRegexPatternSet", options);
         patternSet.replaceExpressions({
             regularExpressions: input.RegularExpressionList ?? [],
-            description: input.Description,
+            description,
             lockToken: input.LockToken,
         });
         return { $metadata: {}, NextLockToken: patternSet.lockToken };

package/dist/service/wafv2/command/sim-wafv2-input.d.ts

@@ -24,3 +24,18 @@ export declare function refuseSimWafTags(tags: readonly unknown[] | undefined, o
  * by ARN, and neither has anything else to fall back on.
  */
 export declare function requiredSimWafArn(arn: string | undefined, parameter: string): string;
+/**
+ * Read the description a write gave, refusing one WAFv2 will not store.
+ *
+ * The empty string is the one worth catching. Code that reads a resource,
+ * changes part of it and writes the rest back hands the description straight
+ * through, and AWS answers `""` for some resources nobody has described. WAFv2
+ * refuses that write. A simulation that took it would leave the failure for
+ * the account to report.
+ *
+ * The two constraints are checked apart because AWS checks them apart. The
+ * pattern matches three characters at the shortest. `ab` is long enough for
+ * the length and still refused, and `""` fails both at once, which is how one
+ * message comes to report two errors.
+ */
+export declare function checkedSimWafDescription(description: string | undefined): string | undefined;

package/dist/service/wafv2/command/sim-wafv2-input.js

@@ -1,4 +1,4 @@
-import { SimWafInvalidParameterException, SimWafUnsimulatedInputException, } from "../error/sim-wafv2.error.js";
+import { SimWafInvalidParameterException, SimWafUnsimulatedInputException, SimWafValidationException, } from "../error/sim-wafv2.error.js";
 /**
  * Read the name a request named, refusing a request with none.
  */
@@ -47,3 +47,68 @@ export function requiredSimWafArn(arn, parameter) {
     }
     return arn;
 }
+/**
+ * The shape WAFv2 documents for the description of a web ACL, an IP set and a
+ * regex pattern set.
+ *
+ * The refusal below quotes this expression back, the way WAFv2 quotes its own.
+ * WAFv2 writes the dots inside the character classes escaped and this does
+ * not, which is the same set of characters written two ways.
+ */
+const descriptionExpression = /^[\w+=:#@/\-,.][\w+=:#@/\-,.\s]+[\w+=:#@/\-,.]$/;
+const descriptionMaxLength = 256;
+/**
+ * Read the description a write gave, refusing one WAFv2 will not store.
+ *
+ * The empty string is the one worth catching. Code that reads a resource,
+ * changes part of it and writes the rest back hands the description straight
+ * through, and AWS answers `""` for some resources nobody has described. WAFv2
+ * refuses that write. A simulation that took it would leave the failure for
+ * the account to report.
+ *
+ * The two constraints are checked apart because AWS checks them apart. The
+ * pattern matches three characters at the shortest. `ab` is long enough for
+ * the length and still refused, and `""` fails both at once, which is how one
+ * message comes to report two errors.
+ */
+export function checkedSimWafDescription(description) {
+    if (description === undefined) {
+        return undefined;
+    }
+    const failures = [
+        ...lengthFailures(description),
+        ...patternFailures(description),
+    ];
+    if (failures.length === 0) {
+        return description;
+    }
+    throw new SimWafValidationException(`${failures.length} validation error${failures.length === 1 ? "" : "s"} detected: ${failures
+        .map((failure) => `Value '${description}' at 'description' failed to satisfy ` +
+        `constraint: ${failure}`)
+        .join("; ")}`);
+}
+/**
+ * What the length of a description falls foul of, if anything.
+ */
+function lengthFailures(description) {
+    if (description.length === 0) {
+        return ["Member must have length greater than or equal to 1"];
+    }
+    if (description.length > descriptionMaxLength) {
+        return [
+            `Member must have length less than or equal to ${descriptionMaxLength}`,
+        ];
+    }
+    return [];
+}
+/**
+ * What the shape of a description falls foul of, if anything.
+ */
+function patternFailures(description) {
+    if (descriptionExpression.test(description)) {
+        return [];
+    }
+    return [
+        `Member must satisfy regular expression pattern: ${descriptionExpression.source}`,
+    ];
+}

package/dist/service/wafv2/command/web-acl/sim-wafv2-web-acl-commands.js

@@ -2,7 +2,7 @@ import { SimWafAssociatedItemException } from "../../error/sim-wafv2.error.js";
 import { requiredSimWafScope } from "../../scope/sim-waf-scope.js";
 import { SimWafWebAcl } from "../../web-acl/sim-waf-web-acl.js";
 import { SimWafPage } from "../sim-wafv2-page.js";
-import { requiredSimWafName } from "../sim-wafv2-input.js";
+import { checkedSimWafDescription, requiredSimWafName, } from "../sim-wafv2-input.js";
 import { requireSimWafResource, } from "../sim-wafv2-resource-lookup.js";
 import { refuseUnsimulatedSimWafWebAclInput } from "./sim-wafv2-unsimulated-web-acl-input.js";
 import { simWafWebAclOutput } from "./sim-waf-web-acl-output.js";
@@ -38,12 +38,13 @@ export class SimWafWebAclCommands {
         const name = requiredSimWafName(input.Name);
         const scope = requiredSimWafScope(input.Scope, this.#accountRegionScope.regionName);
         refuseUnsimulatedSimWafWebAclInput(input, "CreateWebACL");
+        const configuration = configurationOf(input);
         const webAcl = new SimWafWebAcl({
             name,
             scope,
             accountRegionScope: this.#accountRegionScope,
-            description: input.Description,
-            configuration: configurationOf(input),
+            description: configuration.description,
+            configuration,
             regexPatternSets: this.#regexPatternSets,
             managedRules: this.#managedRules,
             clock: this.#clock,
@@ -68,8 +69,9 @@ export class SimWafWebAclCommands {
     updateWebAcl(command, options) {
         const { input } = command;
         refuseUnsimulatedSimWafWebAclInput(input, "UpdateWebACL");
+        const configuration = configurationOf(input);
         const webAcl = this.require(input, "wafv2:UpdateWebACL", options);
-        webAcl.reconfigure(configurationOf(input), input.LockToken);
+        webAcl.reconfigure(configuration, input.LockToken);
         return { $metadata: {}, NextLockToken: webAcl.lockToken };
     }
     /**
@@ -134,6 +136,6 @@ function configurationOf(input) {
         rules: input.Rules,
         customResponseBodies: input.CustomResponseBodies,
         visibilityConfig: input.VisibilityConfig,
-        description: input.Description,
+        description: checkedSimWafDescription(input.Description),
     };
 }

package/dist/service/wafv2/error/sim-wafv2.error.d.ts

@@ -89,6 +89,17 @@ export declare class SimWafAssociatedItemException extends SimWafError {
     readonly name = "WAFAssociatedItemException";
     constructor(message: string);
 }
+/**
+ * Simulated WAFv2 ValidationException error.
+ *
+ * WAFv2 checks the length and shape of a few members before the request
+ * reaches the rest of the API, and reports every failure in one message. A
+ * description of `""` fails two of those checks at once.
+ */
+export declare class SimWafValidationException extends SimWafError {
+    readonly name = "ValidationException";
+    constructor(message: string);
+}
 /**
  * A match declared against this simulation that it cannot answer with.
  *

package/dist/service/wafv2/error/sim-wafv2.error.js

@@ -100,6 +100,19 @@ export class SimWafAssociatedItemException extends SimWafError {
         super(message, { httpStatusCode: 400 });
     }
 }
+/**
+ * Simulated WAFv2 ValidationException error.
+ *
+ * WAFv2 checks the length and shape of a few members before the request
+ * reaches the rest of the API, and reports every failure in one message. A
+ * description of `""` fails two of those checks at once.
+ */
+export class SimWafValidationException extends SimWafError {
+    name = "ValidationException";
+    constructor(message) {
+        super(message, { httpStatusCode: 400 });
+    }
+}
 /**
  * A match declared against this simulation that it cannot answer with.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kensio/yulin",
-  "version": "1.20.10",
+  "version": "1.20.12",
   "description": "AWS system behaviour simulation for isolated unit testing",
   "repository": "https://github.com/KensioSoftware/yulin",
   "homepage": "https://yulinsim.dev/",