@kensio/yulin 1.20.10 → 1.20.11

package/dist/service/athena/engine/sim-athena-json-records.d.ts

@@ -0,0 +1,23 @@
+import type { SimAthenaEngineRow } from "./sim-athena-engine-row.js";
+/**
+ * How one table's JSON records reach its columns.
+ *
+ * `mappings` holds the column names the OpenX SerDe's `mapping.<column>`
+ * parameters declare, each against the record key it reads. A table declaring
+ * none leaves it empty, and every column is read by its own name.
+ *
+ * `caseInsensitive` is the SerDe's own `case.insensitive`, on unless a table
+ * turns it off. The SerDe folds a record's keys before looking one up. A
+ * mapping matches a key of any case until a table declares `FALSE`.
+ */
+export interface SimAthenaJsonFormat {
+    readonly mappings: ReadonlyMap<string, string>;
+    readonly caseInsensitive: boolean;
+}
+/**
+ * One object of JSON lines, read into rows.
+ *
+ * A nested object or array is kept as its JSON text, which is what makes
+ * `json_extract_scalar` and `cardinality` reach into it.
+ */
+export declare function simAthenaJsonRows(text: string, format: SimAthenaJsonFormat): readonly SimAthenaEngineRow[];

package/dist/service/athena/engine/sim-athena-json-records.js

@@ -0,0 +1,46 @@
+/**
+ * One object of JSON lines, read into rows.
+ *
+ * A nested object or array is kept as its JSON text, which is what makes
+ * `json_extract_scalar` and `cardinality` reach into it.
+ */
+export function simAthenaJsonRows(text, format) {
+    return text
+        .split("\n")
+        .filter((line) => line.trim().length > 0)
+        .map((line) => JSON.parse(line))
+        .map((record) => mappedRow(record, format));
+}
+/**
+ * One record with its mapped columns laid over it.
+ *
+ * The record is kept whole underneath. A column no mapping names still reads
+ * by its own name. A mapped column is written whether or not the record
+ * carries the key, since the mapping is where that column reads from.
+ */
+function mappedRow(record, format) {
+    if (format.mappings.size === 0) {
+        return record;
+    }
+    const row = { ...record };
+    for (const [column, key] of format.mappings) {
+        row[column] = mappedValue(record, key, format.caseInsensitive);
+    }
+    return row;
+}
+function mappedValue(record, key, caseInsensitive) {
+    const exact = record[key];
+    if (exact !== undefined) {
+        return exact;
+    }
+    if (!caseInsensitive) {
+        return null;
+    }
+    const wanted = key.toLowerCase();
+    for (const [held, value] of Object.entries(record)) {
+        if (held.toLowerCase() === wanted) {
+            return value;
+        }
+    }
+    return null;
+}

package/dist/service/athena/engine/sim-athena-object-codec.d.ts

@@ -0,0 +1,13 @@
+/**
+ * One object's bytes, decompressed by the codec its key names.
+ *
+ * Athena reads the file extension, and so does this. A key ending `.gz` is
+ * gzip whatever its bytes hold, and a key ending anything else is text. The
+ * magic number would be the other way to tell, and it would read an object
+ * real Athena skips.
+ *
+ * A key naming a codec this simulation has no decompressor for raises, and the
+ * engine turns the query down. The declaration a test wrote answers it, which
+ * is what an object the engine cannot open already does.
+ */
+export declare function simAthenaDecompressedBytes(key: string, bytes: Buffer): Buffer;

package/dist/service/athena/engine/sim-athena-object-codec.js

@@ -0,0 +1,40 @@
+import { gunzipSync, inflateSync, zstdDecompressSync } from "node:zlib";
+import { SimAthenaSetUpError } from "../error/sim-athena.error.js";
+/** The file extensions naming a codec, against what decompresses each one. */
+const codecs = new Map([
+    ["gz", gunzipSync],
+    ["zst", zstdDecompressSync],
+    ["deflate", inflateSync],
+]);
+/**
+ * The file extensions naming a codec nothing here decompresses.
+ *
+ * Node's standard library has gzip, zstd and deflate. The rest would need a
+ * dependency, and this package takes none for a test to install.
+ */
+const unreadableCodecs = new Set(["bz2", "bzip2", "lz4", "lzo", "snappy"]);
+/**
+ * One object's bytes, decompressed by the codec its key names.
+ *
+ * Athena reads the file extension, and so does this. A key ending `.gz` is
+ * gzip whatever its bytes hold, and a key ending anything else is text. The
+ * magic number would be the other way to tell, and it would read an object
+ * real Athena skips.
+ *
+ * A key naming a codec this simulation has no decompressor for raises, and the
+ * engine turns the query down. The declaration a test wrote answers it, which
+ * is what an object the engine cannot open already does.
+ */
+export function simAthenaDecompressedBytes(key, bytes) {
+    const name = key.slice(key.lastIndexOf("/") + 1);
+    const dot = name.lastIndexOf(".");
+    if (dot === -1) {
+        return bytes;
+    }
+    const extension = name.slice(dot + 1).toLowerCase();
+    if (unreadableCodecs.has(extension)) {
+        throw new SimAthenaSetUpError(`Unsupported sim Athena compression: ${extension}`);
+    }
+    const decompress = codecs.get(extension);
+    return decompress === undefined ? bytes : decompress(bytes);
+}

package/dist/service/athena/engine/sim-athena-record-reader.js

@@ -1,10 +1,15 @@
 import { simAthenaDelimitedRows, } from "./sim-athena-delimited-records.js";
+import { simAthenaJsonRows, } from "./sim-athena-json-records.js";
+/** The SerDe class name that takes `mapping.<column>` parameters. */
+const openXSerDe = "org.openx.data.jsonserde.jsonserde";
 /** The SerDe class names that mean JSON lines. */
 const jsonSerDes = new Set([
-    "org.openx.data.jsonserde.jsonserde",
+    openXSerDe,
     "org.apache.hive.hcatalog.data.jsonserde",
     "org.apache.hadoop.hive.serde2.jsonserde",
 ]);
+/** What a `mapping.<column>` parameter is named with. */
+const mappingPrefix = "mapping.";
 /**
  * The SerDe class names that mean delimited text, and what each one reads
  * before the table's own parameters are applied.
@@ -39,7 +44,8 @@ export function simAthenaRecordReader(table) {
         return undefined;
     }
     if (jsonSerDes.has(library)) {
-        return jsonRows;
+        const format = jsonFormat(table, library);
+        return (text) => simAthenaJsonRows(text, format);
     }
     const defaults = delimitedSerDes.get(library);
     if (defaults === undefined) {
@@ -50,16 +56,26 @@ export function simAthenaRecordReader(table) {
     return (text) => simAthenaDelimitedRows(text, format, columns);
 }
 /**
- * JSON lines, one record per line.
+ * How one table's JSON records reach its columns.
  *
- * A nested object or array is kept as its JSON text, which is what makes
- * `json_extract_scalar` and `cardinality` reach into it.
+ * Only the OpenX SerDe takes the mappings. The Hive JSON SerDes have no
+ * `mapping` property, and a table declaring one against them reads by its
+ * column names on real Athena the same as here.
  */
-function jsonRows(text) {
-    return text
-        .split("\n")
-        .filter((line) => line.trim().length > 0)
-        .map((line) => JSON.parse(line));
+function jsonFormat(table, library) {
+    const parameters = library === openXSerDe
+        ? (table.storageDescriptor?.SerdeInfo?.Parameters ?? {})
+        : {};
+    const mappings = new Map();
+    for (const [name, key] of Object.entries(parameters)) {
+        if (name.toLowerCase().startsWith(mappingPrefix)) {
+            mappings.set(name.slice(mappingPrefix.length), key);
+        }
+    }
+    return {
+        mappings,
+        caseInsensitive: parameters["case.insensitive"]?.toLowerCase() !== "false",
+    };
 }
 function delimitedFormat(table, defaults) {
     const parameters = table.storageDescriptor?.SerdeInfo?.Parameters ?? {};

package/dist/service/athena/engine/sim-athena-table-objects.d.ts

@@ -26,5 +26,5 @@ export interface SimAthenaTableObjects extends SimAthenaScannedObjects {
  * there because nothing holds the data it would answer from.
  */
 export declare function simAthenaTableObjects(s3: Partial<SimAthenaTableObjects> | undefined): SimAthenaTableObjects | undefined;
-/** One object's bytes, read as UTF-8 text. */
+/** One object's bytes, decompressed by its key and read as UTF-8 text. */
 export declare function simAthenaObjectText(objects: SimAthenaTableObjects, bucket: string, key: string, caller: SimAwsCaller | undefined): Promise<string>;

package/dist/service/athena/engine/sim-athena-table-objects.js

@@ -1,3 +1,4 @@
+import { simAthenaDecompressedBytes } from "./sim-athena-object-codec.js";
 /**
  * Whether this simulated S3 can open an object as well as list one.
  *
@@ -9,9 +10,10 @@ export function simAthenaTableObjects(s3) {
         ? undefined
         : s3;
 }
-/** One object's bytes, read as UTF-8 text. */
+/** One object's bytes, decompressed by its key and read as UTF-8 text. */
 export async function simAthenaObjectText(objects, bucket, key, caller) {
     const got = await objects.getObject({ input: { Bucket: bucket, Key: key } }, caller === undefined ? undefined : { caller });
     const chunks = await Array.fromAsync(got.Body ?? []);
-    return new TextDecoder().decode(Buffer.concat(chunks));
+    const bytes = simAthenaDecompressedBytes(key, Buffer.concat(chunks));
+    return new TextDecoder().decode(bytes);
 }

package/dist/service/athena/sim-athena-engine.fixture.d.ts

@@ -34,5 +34,7 @@ export declare function anEngineSimulation(clock?: SimClock): Promise<SimAthenaE
 export declare function aCatalogTable(simAws: SimAws, table: SimAthenaEngineTableInput): void;
 /** Put one object of literal text under the logs Bucket. */
 export declare function aSeededObject(simAws: SimAws, key: string, body: string): Promise<void>;
+/** Put one object of raw bytes under the logs Bucket. */
+export declare function aSeededBytes(simAws: SimAws, key: string, body: Uint8Array): Promise<void>;
 /** Put one object of JSON lines under the logs Bucket. */
 export declare function aSeededJson(simAws: SimAws, key: string, records: readonly Record<string, unknown>[]): Promise<void>;

package/dist/service/athena/sim-athena-engine.fixture.js

@@ -58,6 +58,12 @@ export async function aSeededObject(simAws, key, body) {
         .s3()
         .putObject({ input: { Bucket: logsBucket, Key: key, Body: body } });
 }
+/** Put one object of raw bytes under the logs Bucket. */
+export async function aSeededBytes(simAws, key, body) {
+    await simAws
+        .s3()
+        .putObject({ input: { Bucket: logsBucket, Key: key, Body: body } });
+}
 /** Put one object of JSON lines under the logs Bucket. */
 export async function aSeededJson(simAws, key, records) {
     const lines = records.map((record) => JSON.stringify(record));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kensio/yulin",
-  "version": "1.20.10",
+  "version": "1.20.11",
   "description": "AWS system behaviour simulation for isolated unit testing",
   "repository": "https://github.com/KensioSoftware/yulin",
   "homepage": "https://yulinsim.dev/",