@dudousxd/nestjs-catalog 0.26.0 → 0.28.0

package/dist/catalog.aggregate.d.ts

@@ -0,0 +1,376 @@
+/**
+ * The fold behind the `aggregate` node: one pass, one entry per group.
+ *
+ * ## The property this file exists to have
+ *
+ * A hash aggregate **consumes its input as a stream and holds only the groups.**
+ * It never needs the row it read two rows ago, and it never needs the row it is
+ * about to read. So the heap it occupies is a function of how many distinct
+ * groups there are, and not of how many rows there were.
+ *
+ * That is the entire argument for the node, and it is worth stating with the
+ * measurement that produced it. flip's `wo` derivation groups **44,720 rows into
+ * 16,119 groups** with about fifty aggregates over them. Today it is a
+ * whole-batch transform, and as a whole-batch transform it is one column away
+ * from dying:
+ *
+ * - the child is handed 65.51 MiB on stdin;
+ * - it answers with **one JSON line of 25.01 MiB — 78.2% of the hard 32 MiB
+ *   output cap**, which the parent buffers as a single string before parsing;
+ * - and before any of that, the parent materialises all 44,720 rows through
+ *   `readInputs`. A standalone equivalent peaked at 237 MiB heap, 406 MiB RSS.
+ *
+ * At **1.28×** that file it stops working, and it stops by being killed at the
+ * cap rather than by getting slower. SUBWO is already the largest file in the
+ * drop it comes from. Run through this file instead, the same derivation holds
+ * 16,119 accumulator rows and reads one staged batch at a time.
+ *
+ * ## Which is exactly why the bound is loud
+ *
+ * "Holds only the groups" is a *cheap* promise only while the groups are far
+ * fewer than the rows. Group by something near-unique and a hash aggregate holds
+ * the entire load, which is the thing being fixed rather than a corner of it. So
+ * {@link AggregateTable} carries a group ceiling and **refuses** when it is
+ * crossed, naming the columns it was grouping on. Peaking silently at the
+ * ceiling is the failure mode this node was written about.
+ *
+ * ## Every arithmetic decision in one place
+ *
+ * The node is the model; this is the semantics. Both are here rather than in the
+ * runner so that the canvas, a spec and a durable step get the same answer out
+ * of the same code, and so that the decisions below have one address:
+ *
+ * - **Comparison order for `min`/`max` over strings** — see {@link compareValues}.
+ * - **Summation error** — see {@link addToSum}.
+ * - **The bound on a `join`, and what happens at it** — see {@link appendJoin}.
+ * - **What makes two records the same group** — see {@link groupKeyOf}.
+ *
+ * None of them is inherited from MySQL, and where the answer differs from
+ * MySQL's the docblock says so and says why, because the alternative is a load
+ * that silently disagrees with the query it replaced.
+ */
+import { type WorkflowAggregate, type WorkflowAggregateNode } from './catalog.pipeline';
+/**
+ * A refusal raised while folding, rather than a wrong answer carried forward.
+ *
+ * Its own class so a caller can tell "this data cannot be aggregated the way the
+ * node says" from a bug, and turn the first into a 400 with the sentence intact.
+ * Every message it carries names the column, the group and the values, because a
+ * refusal at row ninety thousand that says only "incomparable" is a refusal
+ * somebody has to reproduce before they can act on it.
+ */
+export declare class WorkflowAggregateError extends Error {
+    constructor(message: string);
+}
+/** What the fold learned on the way through, for the run log. */
+export interface AggregateTableStats {
+    /** Records handed to {@link AggregateTable.push}. */
+    rowsIn: number;
+    /** Distinct groups held. The number the memory bound is about. */
+    groups: number;
+    /**
+     * Columns the node names that were present in **no** record of the whole run.
+     *
+     * Almost always a typo in a header, and the symptom otherwise is a column of
+     * nulls and a green run — the same trap `renameStagePayload` reports and for
+     * the same reason. Not a refusal, because a source legitimately omits a column
+     * for a whole load, and `sum`/`min`/`max` over nothing answers `null` rather
+     * than `0`, which is visibly different from a real zero.
+     */
+    unseenColumns: string[];
+    /**
+     * How many values a `sum` or an `avg` had to read out of a string.
+     *
+     * A CSV delivers every number as text, so this is usually the whole column and
+     * is not a problem. It is reported because the one case that *is* a problem —
+     * a column that is numbers in one source and text in another — looks identical
+     * from the answer and different from here.
+     */
+    coercedFromString: number;
+    /** The longest value any `join` produced, in characters. See {@link appendJoin}. */
+    longestJoin: number;
+}
+/** `sum` and `avg`. See {@link addToSum} for why there are two numbers. */
+interface SumAcc {
+    /** The running total. */
+    sum: number;
+    /** Neumaier's compensation term: the low-order bits `sum` could not hold. */
+    compensation: number;
+    /** Non-null values seen. Zero means the answer is `null`, never `0`. */
+    n: number;
+}
+/** `join`. See {@link appendJoin} for the bound and what happens at it. */
+interface JoinAcc {
+    text: string;
+    n: number;
+}
+/**
+ * The fold, as an object you push records into.
+ *
+ * An object rather than a `reduce` because the caller is a loop over staged
+ * batches with an `await` in it: the table has to survive between batches, and
+ * nothing about it may scale with how many batches there were.
+ *
+ * Built from the node, and it refuses a node the validator would refuse rather
+ * than folding under a configuration nobody could have saved — a graph can reach
+ * a runner from a database written by an older build, and `aggregateRefusals` is
+ * the one place that rule lives.
+ */
+export declare class AggregateTable {
+    private readonly node;
+    private readonly groups;
+    private readonly groupBy;
+    private readonly aggregates;
+    private readonly maxGroups;
+    /** Named columns not yet seen in any record. Emptied as they turn up. */
+    private readonly unseen;
+    private rowsIn;
+    private coercedFromString;
+    private longestJoin;
+    constructor(node: WorkflowAggregateNode);
+    /**
+     * Fold one record in.
+     *
+     * Allocates nothing per record except when a record opens a group, which is
+     * the property that makes the whole thing a stream: the cost of a row is the
+     * group-key encoding plus one accumulator update per aggregate.
+     */
+    push(row: Record<string, unknown>): void;
+    /**
+     * One aggregate, one record. Narrowed off the function list, never asserted.
+     *
+     * A dispatch and four one-function methods rather than four branches inline,
+     * so each function's rule about nulls — which is the part that has to match
+     * SQL and the part that is easy to get subtly wrong — sits next to its own
+     * name. Ends in the exhaustiveness guard, so a seventh function is a compile
+     * error here rather than a value nobody computed.
+     */
+    private accumulate;
+    /**
+     * `COUNT(*)` with no column and `COUNT(col)` with one.
+     *
+     * The two differ exactly where the data is sparse, which is where somebody is
+     * most likely to want to know — so both are offered rather than one being
+     * picked on the reader's behalf.
+     */
+    private count;
+    /** `sum` and `avg`, which share an accumulator and differ only at the finish. */
+    private total;
+    /** `min` and `max`. Nulls are skipped, as SQL skips them. */
+    private extremum;
+    /** `join`. Nulls skipped, empty strings kept — which is what GROUP_CONCAT does. */
+    private join;
+    /**
+     * A value on its way into a `sum` or an `avg`, or nothing.
+     *
+     * Three answers rather than two, and the middle one is the decision:
+     *
+     * - **null, undefined, or a string that is only whitespace** — skipped, and
+     *   `n` does not move. A blank cell in a CSV is an absent number and not a
+     *   zero, and SQL agrees: `SUM` over nothing but nulls is `NULL`, which is
+     *   visibly different from a real total of zero.
+     * - **a string that parses as a finite number** — accepted, and counted in
+     *   {@link AggregateTableStats.coercedFromString}. Refusing would make the
+     *   node useless on the exact data it was written for, since a CSV delivers
+     *   every number as text.
+     * - **anything else** — refused, naming the column, the group and the value.
+     *   MySQL reads `'n/a'` as `0` and raises a warning MikroORM does not surface,
+     *   which is how a total ends up quietly too small. This is that behaviour,
+     *   not reproduced.
+     */
+    private toNumber;
+    /**
+     * Every group, as records, in the order the groups were first seen.
+     *
+     * **Insertion order rather than sorted**, and that is a decision about
+     * determinism rather than about cost. Sorting would need a total order over
+     * heterogeneous key values — the thing {@link compareValues} refuses to invent
+     * — and would buy an ordering nobody asked for. Insertion order is a function
+     * of the input order, and the input is a numbered list of staged batches, so
+     * two runs over the same staged rows emit the same rows in the same order.
+     * What it does **not** promise is stability across a source that returns its
+     * own rows in a different order; a `SELECT` without an `ORDER BY` promises
+     * nothing and this node cannot promise more than it was given.
+     *
+     * Every record carries **every** output column — the group keys, then the
+     * aggregates, in the node's order — including the ones whose answer is `null`.
+     * That is what makes an aggregate's output set *exact* rather than an upper
+     * bound, which is the claim `producedColumns` makes about this kind and about
+     * no other.
+     */
+    emit(): Generator<Record<string, unknown>>;
+    /** What the run log says. Cheap: everything here was counted on the way past. */
+    stats(): AggregateTableStats;
+}
+/** The columns a node's aggregates read, deduplicated, `count(*)` excluded. */
+export declare function aggregateInputColumns(node: WorkflowAggregateNode): string[];
+/**
+ * What makes two records the same group.
+ *
+ * The key is the group-by values, encoded into one string so a `Map` can hold
+ * them. Every part is **length-prefixed and type-tagged**, which closes two
+ * holes that a naive `values.join('|')` leaves open:
+ *
+ * - `["a|b", "c"]` and `["a", "b|c"]` are different groups and must stay
+ *   different. Length prefixes make the encoding unambiguous whatever the values
+ *   contain.
+ * - `1` and `"1"` are different groups, and that is a **decision against SQL**,
+ *   which would coerce them together. Merging them means merging two things the
+ *   source considered distinct, on a rule nobody chose; within one load a column
+ *   comes from one system and is one type, so the coercion buys nothing and
+ *   risks a group that quietly swallowed another.
+ *
+ * **Missing and null are the same group**, and that is a decision *for* SQL.
+ * `GROUP BY` collects all NULLs into one group, and a record that simply lacks
+ * the column is a record whose column is null. The stage encoding makes "absent"
+ * and "null" a difference in physical layout rather than in meaning — see the
+ * shape dictionary in `catalog.stage-encoding.ts` — so grouping them apart would
+ * make the answer depend on which shape a row happened to land in, which is not
+ * a thing to build a load on.
+ *
+ * A value that is an object or an array is **refused**. Two objects are equal
+ * under some rule and unequal under others, every one of those rules is
+ * somebody's convention, and picking one silently decides how many rows a
+ * published type ends up holding.
+ */
+export declare function groupKeyOf(values: readonly unknown[], columns: readonly string[], node: {
+    id: string;
+    name: string;
+}): string;
+/**
+ * The order `min` and `max` compare in, and the one place it is decided.
+ *
+ * ## Strings compare by **code point**, and that is not MySQL's answer
+ *
+ * The column this node most often replaces sits in MySQL under
+ * `utf8mb4_0900_ai_ci` — case-insensitive and accent-insensitive — where
+ * `'apple' > 'Banana'` is false. JavaScript's `<` compares UTF-16 code points,
+ * where `'B'` is 66 and `'a'` is 97, so `'apple' > 'Banana'` is true. On the real
+ * SUBWO data the two answers agree on most columns and **differ on two**: 18 of
+ * 16,119 groups for `lastUpdatedBy` and 23 for `maintenanceLocation`, both of
+ * which hold mixed-case usernames and location codes.
+ *
+ * Code point order is implemented anyway, deliberately:
+ *
+ * - It is **total, stable and machine-independent.** The same rows produce the
+ *   same answer on every Node build, in every locale, with no ICU data loaded.
+ * - The alternative is not "MySQL's answer" — it is *an approximation of one
+ *   deployment's collation*. `Intl.Collator` implements the Unicode collation
+ *   algorithm, MySQL implements its own table per collation, and a graph can
+ *   read from a source that is not MySQL at all. Matching would mean carrying a
+ *   collation name on the node and reimplementing it, which is a database inside
+ *   a pipeline node.
+ * - Being *near* MySQL is worse than being clearly different. A comparison that
+ *   agrees 99.8% of the time is one nobody checks and everybody trusts.
+ *
+ * So it is code point order, it is written down here, and the run log says
+ * nothing about it because there is nothing conditional to report — what a
+ * reader needs is this paragraph, which is why it is this long.
+ *
+ * ## Types are not compared across classes
+ *
+ * Numbers with numbers, strings with strings, dates with dates, booleans with
+ * booleans. A `max` over a column holding both `12` and `"12"` is **refused**,
+ * naming the column, the group and both values. There is no ordering between a
+ * number and a string that is not somebody's coercion rule, and every such rule
+ * produces a maximum that depends on which row arrived first — a wrong answer
+ * that reports success, which is the shape of failure this node was written
+ * about. The fix is one node upstream: normalise the column in a transform.
+ */
+export declare function compareValues(left: unknown, right: unknown, column: string, groupId: string): number;
+/**
+ * Add one value to a running total, with the low-order bits kept.
+ *
+ * ## The measurement this is here for
+ *
+ * A prior comparison of flip's `wo` derivation against a JavaScript
+ * reimplementation found **17 of 16,119 groups differing in the last float64
+ * ulp** — `6442.999999999999` against `6443` — purely from the order the terms
+ * were added in. The grand totals reconciled exactly (212,192,113 on both
+ * sides), so the difference was tolerable. It was also nobody's decision, and
+ * that is the part worth fixing: a load that is off by an ulp because of an
+ * accident is one nobody can reason about the next time it is off by more.
+ *
+ * ## What is implemented, and what it does and does not promise
+ *
+ * **Neumaier summation.** One extra float per accumulator carries the bits the
+ * running total could not hold, and they are added back once at the end. On
+ * decimal money and hours — which is what every column this node sums actually
+ * is — the result is the correctly-rounded sum, so `6443` comes out as `6443`.
+ *
+ * What it promises:
+ *
+ * - The answer is **far** closer to the exact sum than `+=` is, and for values
+ *   with a couple of decimal places it is the exact sum rounded once.
+ * - It costs one number per accumulator and two floating-point operations per
+ *   row. At 44,720 rows that is unmeasurable.
+ *
+ * What it does **not** promise:
+ *
+ * - **Order independence.** Nothing short of exact arithmetic gives that, and
+ *   exact arithmetic means holding a big decimal per accumulator, which trades
+ *   the property the node is built on for a rounding difference nobody can
+ *   observe in a report.
+ * - **Bit-for-bit agreement with MySQL.** MySQL sums in its own order and, for a
+ *   `DECIMAL` column, not in binary floating point at all. Where the source
+ *   column is `DECIMAL` the honest answer is that these are two different
+ *   arithmetics and the difference is bounded by one rounding, not that they
+ *   agree.
+ */
+export declare function addToSum(slot: SumAcc, value: number): void;
+/**
+ * Append one value to a joined string, and refuse rather than truncate.
+ *
+ * ## The behaviour this is designed against, which is live today
+ *
+ * flip's `wo` derivation uses `GROUP_CONCAT(... SEPARATOR '; ')`. In that
+ * deployment `group_concat_max_len` is **1024**. Real values reach **1,700 and
+ * 1,883 characters**, and **5 of 16,119 groups exceed the limit on each of two
+ * columns**. MySQL truncates at the limit and raises a warning, and MikroORM
+ * does not surface the warning. So five rows per column have been silently
+ * missing their tail, in committed data, with a green run every time.
+ *
+ * That is not a MySQL bug and it is not a configuration slip anybody would
+ * notice: the default is 1024, the query does not mention it, and a truncated
+ * string is a perfectly plausible string.
+ *
+ * ## What this does instead
+ *
+ * **Refuses, at the bound, naming the group and the length.** Not truncation
+ * with a log line, and not an unbounded string.
+ *
+ * - Truncating loudly was the other candidate and it loses on the same argument
+ *   as everything else here: the run would go green, the snapshot would commit,
+ *   and the log line would be one of twenty on a node nobody reads unless
+ *   something already went wrong. The value would still be wrong in the
+ *   warehouse.
+ * - Unbounded loses on the node's own thesis. A `join` is the one accumulator
+ *   whose size is not bounded by the group count, so an unbounded one is a hole
+ *   straight through "holds only the groups".
+ *
+ * The default bound is {@link WORKFLOW_AGGREGATE_JOIN_MAX_LENGTH} — 65,535
+ * characters, which is what one MySQL `TEXT` column holds, because a value the
+ * target column cannot store is the same defect one layer further down. flip's
+ * real maximum of 1,883 is 2.9% of it, so the derivation this node was written
+ * for runs untouched and would have refused loudly at 35× the data instead of
+ * quietly at 1.5×.
+ *
+ * An author may lower it (a column that should never exceed 200 characters is
+ * worth saying so about) or raise it up to a hard ceiling, which is on
+ * {@link workflowAggregateJoinMaxLength}.
+ */
+export declare function appendJoin(slot: JoinAcc, text: string, aggregate: WorkflowAggregate, node: {
+    id: string;
+    name: string;
+}, groupId: string): void;
+/**
+ * A group key, back in something a person can look up.
+ *
+ * The key is a length-prefixed encoding, which is unreadable on purpose and
+ * useless in an error message. This walks it back into the values it was built
+ * from, so a refusal names the row somebody has to go and find.
+ */
+export declare function describeGroup(key: string): string;
+/** A value, short enough for an error message and honest about its type. */
+export declare function describeValue(value: unknown): string;
+export {};