@kensio/yulin 1.21.3 → 1.21.4

package/dist/util/background/non-deterministic-background.js

@@ -1,7 +1,7 @@
 import { SimRealClock } from "../clock/sim-clock.js";
+import { BackgroundPendingTasks } from "./background-pending-tasks.js";
 import { BackgroundDueTasks, } from "./background-due-tasks.js";
 import { BackgroundJitter } from "./background-jitter.js";
-import { BackgroundSettledTasks } from "./background-settled-tasks.js";
 /* oxlint-disable unicorn-js/prefer-await  */
 /**
  * Non-deterministic async background tasks scheduler.
@@ -10,7 +10,7 @@ import { BackgroundSettledTasks } from "./background-settled-tasks.js";
  * potentially completing out of sequence.
  */
 export class NonDeterministicBackgroundTasks {
-    pending = new Set();
+    pending = new BackgroundPendingTasks();
     dueTasks = new BackgroundDueTasks();
     jitter;
     clock;
@@ -35,19 +35,22 @@ export class NonDeterministicBackgroundTasks {
      * Schedule a task to happen asynchronously in the background.
      */
     schedule(task) {
-        const promise = this.sequence()
-            .then(task)
-            .then(() => {
-            //
-        }, (error) => {
-            // Keep the promise rejected so complete() can surface failures
-            // deterministically.
-            throw error;
-        })
-            .finally(() => {
-            this.pending.delete(promise);
+        this.pending.hold(async () => {
+            await this.sequence();
+            await task();
         });
-        this.pending.add(promise);
+    }
+    /**
+     * Run a task that is already due, keeping it outstanding until it settles.
+     */
+    runDue(task) {
+        this.pending.hold(task);
+    }
+    /**
+     * Count work a caller started as outstanding until it settles.
+     */
+    async outstanding(work) {
+        return await this.pending.holdCallers(work);
     }
     /**
      * Schedule a task to happen once simulated time reaches an instant.
@@ -61,6 +64,12 @@ export class NonDeterministicBackgroundTasks {
     cancelScheduled(task) {
         this.dueTasks.cancel(task);
     }
+    /**
+     * Say that the scheduled task running now is waiting on the clock.
+     */
+    waitingOnClock() {
+        return this.pending.waitingOnClock();
+    }
     /**
      * Take the next task due at or before an instant, earliest first.
      */
@@ -72,12 +81,7 @@ export class NonDeterministicBackgroundTasks {
      * If tasks schedule more tasks, this will continue draining until idle.
      */
     async complete() {
-        while (this.pending.size > 0) {
-            // oxlint-disable-next-line no-await-in-loop
-            const results = await Promise.allSettled(this.pending);
-            // Surface a task that failed in the background to whoever waited here.
-            new BackgroundSettledTasks(results).throwFirstFailure();
-        }
+        await this.pending.complete();
     }
     /**
      * See how many outstanding background tasks are scheduled.

package/dist/util/clock/sim-clock-control.js

@@ -1,3 +1,4 @@
+import { setTimeout } from "node:timers";
 import { SimDuration } from "./sim-duration.js";
 /**
  * Control over one simulation's sense of time.
@@ -67,6 +68,7 @@ export class SimClockControl {
      */
     async setTo(instant) {
         this.clock.freeze();
+        await reachWhatIsWaitedOn();
         if (instant.getTime() >= this.now().getTime()) {
             await this.runDueTasksUpTo(instant);
         }
@@ -115,8 +117,7 @@ export class SimClockControl {
         let due = this.background.takeNextDueBy(instant);
         while (due !== undefined) {
             this.clock.setTo(this.laterOfNow(due.dueTime));
-            // oxlint-disable-next-line no-await-in-loop
-            await due.task();
+            this.background.runDue(due.task);
             // oxlint-disable-next-line no-await-in-loop
             await this.background.complete();
             due = this.background.takeNextDueBy(instant);
@@ -134,3 +135,23 @@ export class SimClockControl {
         return instant;
     }
 }
+/**
+ * Let work already under way reach whatever it is waiting for.
+ *
+ * A test that starts something and moves the clock without waiting for it has
+ * a real ordering problem to solve. An invocation asked for on the line above
+ * has not run a line of handler code yet, so the timer the handler is about to
+ * ask for is not on the clock when the clock moves, and moving time past an
+ * instant nothing is waiting on releases nothing.
+ *
+ * One turn of the host event loop settles that. It runs everything already
+ * queued, which carries a promise chain started on the previous line as far as
+ * its first real wait, and the clock then moves past instants that are
+ * actually being waited on. The clock is frozen before this, so the time that
+ * work reads while it gets there is the instant the test moved from.
+ */
+async function reachWhatIsWaitedOn() {
+    await new Promise((resolve) => {
+        setTimeout(resolve, 0);
+    });
+}

package/docs/services/apigateway/README.md

@@ -1792,7 +1792,9 @@ of that name by itself and the template carries one Resource fewer.
 A template naming the SAM transform reaches the same resource types.
 `AWS::Serverless::Api` becomes a REST API with its deployment and stage, and the `Api` event of an
 `AWS::Serverless::Function` becomes the path resources, the method and the invoke permission that
-put the function behind it. Events naming no `RestApiId` share one API on a `Prod` stage.
+put the function behind it. Events naming no `RestApiId` share one API on a `Prod` stage. An `Auth`
+block on the API becomes the `AWS::ApiGateway::Authorizer` resources it declares, and `Auth` on the
+event picks which of them guards that method.
 [The SAM section of the CloudFormation docs](https://yulinsim.dev/services/cloudformation/#rest-apis) covers both.
 
 ### CDK

package/docs/services/athena/README.md

@@ -196,7 +196,8 @@ loads the rows into an in-memory SQLite database, and answers the statement from
 nineteen queries in twenty of the shapes a test writes run this way.
 
 The engine is off until a test turns it on, and it needs `node-sql-parser` in the project. The
-parser is an optional peer dependency, so a project that never runs a query never installs it.
+parser is an optional peer dependency, so a project that never runs a query never installs it. A
+Parquet table needs `hyparquet` on the same terms.
 
 ```bash
 pnpm add -D node-sql-parser
@@ -299,6 +300,96 @@ The engine turns a query down where the parser refuses the statement, where SQLi
 it, where a table declares a format it has no reader for, and where an object it needs cannot be
 opened. Every one of those ends the same way, with the declared result answering.
 
+### Failing a query the engine turns down
+
+A turn-down is quiet. The query succeeds, the declared rows come back, and a test meaning to
+exercise the engine passes either way. Strict mode fails the query instead.
+
+```typescript sim-athena-engine-strict
+/**
+ * A query the engine cannot run, failing rather than falling back to a
+ * declaration nobody meant to use.
+ */
+
+import { SimAws } from "@kensio/yulin";
+
+const simAws = new SimAws();
+
+await simAws.s3().createBucket({ input: { Bucket: "rainlytics-logs" } });
+await simAws.s3().createBucket({ input: { Bucket: "rainlytics-results" } });
+await simAws.athena().createWorkGroup({
+  input: {
+    Name: "rainlytics",
+    Configuration: {
+      ResultConfiguration: { OutputLocation: "s3://rainlytics-results/q/" },
+    },
+  },
+});
+
+simAws.glue().createDatabase({
+  input: { DatabaseInput: { Name: "rainlytics" } },
+});
+simAws.glue().createTable({
+  input: {
+    DatabaseName: "rainlytics",
+    TableInput: {
+      Name: "orders",
+      StorageDescriptor: {
+        Columns: [{ Name: "id", Type: "int" }],
+        Location: "s3://rainlytics-logs/orders/",
+        SerdeInfo: {
+          SerializationLibrary: "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+        },
+      },
+    },
+  },
+});
+
+// Without this the query below succeeds, answering with the declared rows.
+simAws
+  .athena()
+  .results()
+  .byDefault({ columns: ["id"], rows: [["1"]] });
+
+await simAws.athena().engine().enable({ strict: true });
+
+const started = await simAws.athena().startQueryExecution({
+  input: {
+    QueryString: "SELECT id FROM rainlytics.orders",
+    WorkGroup: "rainlytics",
+  },
+});
+
+await simAws.backgroundTasksComplete();
+
+const described = await simAws.athena().getQueryExecution({
+  input: { QueryExecutionId: started.QueryExecutionId },
+});
+
+// "FAILED", because nothing here reads ORC.
+console.log(described.QueryExecution?.Status?.State);
+
+// The reason names the SerDe, the table declaring it, and what to do instead.
+console.log(described.QueryExecution?.Status?.StateChangeReason);
+```
+
+The query reaches `FAILED`, and `StateChangeReason` names what the engine could not do. Each
+turn-down says something different, so a reader of the failing test learns which one it hit rather
+than only that something went wrong. The Athena grammar refusing a statement, an `UNNEST` with no
+`json_each` to become, a statement SQLite will not run, a table in a format with no reader, and
+nothing in scope holding the objects all read differently. A statement SQLite refuses carries
+SQLite's own message out with it.
+
+```
+The simulated Athena query engine has no reader for
+org.apache.hadoop.hive.ql.io.orc.OrcSerde, which table rainlytics.orders declares. Declare what this
+query answers with through results(), or leave strict mode off to fall back to a declaration.
+```
+
+Strict mode is off unless a test asks for it, and `disable()` takes it off with the engine. A
+declaration written against one exact query text still answers, with no failure, because that is a
+test's own statement about a query the engine gets wrong.
+
 ### The objects it reads
 
 The SerDe class name in the table's storage descriptor says how its objects are decoded.
@@ -309,6 +400,8 @@ The SerDe class name in the table's storage descriptor says how its objects are
   that needs it.
 - `org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe` reads the delimiter `field.delim` names,
   which defaults to the control character Hive uses.
+- `org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe` reads Parquet. So does the older
+  `parquet.hive.serde.ParquetHiveSerDe` that a Hive table carries.
 
 `separatorChar`, `quoteChar` and `escapeChar` in the SerDe's parameters override those defaults, and
 `skip.header.line.count` on the table drops the first lines of every object. An empty field reads as
@@ -336,6 +429,29 @@ partitions takes it from the projection, and a table laid out Hive style under i
 takes it from the `key=value` segments of the object's key. Either way the column reads on every
 row, though no object holds it.
 
+### Parquet
+
+A Parquet table needs [`hyparquet`](https://github.com/hyparam/hyparquet) in the project, alongside
+the SQL parser. It is an optional peer dependency as well, and a project querying only JSON and CSV
+tables never installs it. The reader loads the first time a query reaches a Parquet table.
+
+```bash
+npm install --save-dev hyparquet
+```
+
+A Parquet file carries its own schema. The table's Glue columns say only what to call each column
+and what type to report it as. A column the table declares and the file has not got reads null, the
+way a JSON record missing a key does. An `int64` column keeps every digit, past what a double holds
+exactly. A `TIMESTAMP`, `DATE` or `INT96` column reads as an instant, and the Glue column type
+decides whether it renders as a day or as a day and a time. A `BYTE_ARRAY` column reads as text.
+
+Parquet compresses per column chunk inside the file, and an object's key says nothing about it. A
+file written with `SNAPPY`, `GZIP`, `ZSTD`, `BROTLI` or no compression at all reads, which covers
+what Athena, Glue and Firehose write. A file written with `LZO`, `LZ4` or `LZ4_RAW` turns the query
+down, and the declaration a test wrote answers it.
+
+ORC has no reader. A table declaring one falls back to its declared result.
+
 ### What it answers with
 
 Column types come from the Glue schema, written the way Athena writes them, so Hive's `string`
@@ -958,8 +1074,15 @@ Current documented limitations:
 - A declaration that fails the query wins from any tier, the engine included. `failsWith` is a
   statement about the query rather than about its rows, so a workgroup rule or a default carrying
   one fails every query it covers whether or not the engine could have answered.
-- Parquet and ORC are absent. A table declaring either falls back to its declared result, and so
-  does a table declaring no SerDe at all.
+- ORC is absent. A table declaring it falls back to its declared result, and so does a table
+  declaring no SerDe at all. The only ORC reader published for Node is AGPL licensed and builds
+  native bindings, which is more than this package asks a test to install.
+- A Parquet table in a project without `hyparquet` falls back the same way, with nothing said. The
+  engine turns down what it cannot do, and a missing install reaches a test through `answeredBy`
+  reading `declaration`.
+- A Parquet query reports every byte of every object it opens as scanned, whatever columns the
+  statement touched. Real Athena bills a columnar read by the columns alone. A cutoff tuned
+  against a real workload therefore refuses a query here that AWS would allow.
 - A null in a result row reads as an empty string. Real Athena leaves the value out of the row.
 - A computed boolean reads as `1` and `0`. The Glue column type is what makes a boolean column read
   as `true` and `false`, and an expression has no column type behind it.