@dudousxd/nestjs-catalog 0.26.0 → 0.27.0

package/dist/catalog.pipeline.d.ts

@@ -1199,7 +1199,7 @@ export interface WorkflowNodeOutcome {
  *   outside a run, which is why `call` names one and not a step. If a step is
  *   what you want, the thing to call is a one-step workflow wrapping it.
  */
-export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter", "rename"];
+export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter", "rename", "aggregate"];
 export type WorkflowNodeKind = (typeof WORKFLOW_NODE_KINDS)[number];
 /** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
 export declare function isWorkflowNodeKind(value: unknown): value is WorkflowNodeKind;
@@ -2340,13 +2340,392 @@ export declare function renameColumnRefusals(columns: Record<string, string>): s
  * map, and the values are checked one by one rather than trusted from the type.
  */
 export declare function isWorkflowRenameColumns(value: unknown): value is Record<string, string>;
+/**
+ * The aggregate functions this node computes, and the rule that closes the list.
+ *
+ * A closed list with an exhaustiveness guard, for the reason every other list in
+ * this file is one. What is different here is that the list has a **stated
+ * admission rule**, because "we will keep it narrow" is a promise nobody can
+ * check and a rule is:
+ *
+ * > A function is in if it can be computed from a **fixed-size accumulator**,
+ * > and if its answer does not depend on a decision the config would have to
+ * > carry.
+ *
+ * The first half is the node's whole reason to exist. A hash aggregate is cheap
+ * because it holds one entry per group; an accumulator whose size grows with the
+ * number of *rows* in a group puts the rows back in memory and gives up the
+ * property. The second half is what keeps the config from becoming a small
+ * language: a function that needs an extra field to say what it means is a
+ * function whose meaning was not decided.
+ *
+ * What the rule excludes, so the omissions are on the record rather than
+ * implied:
+ *
+ * - **`countDistinct`** — the sharpest one. It needs a set of the distinct
+ *   values *per group per column*, so its accumulator is O(distinct values) and
+ *   a high-cardinality column inside a group holds the load. It is the exact
+ *   thing this node was built to stop doing, wearing an aggregate's name. A
+ *   sketch (HyperLogLog) is fixed-size and is a different function — an
+ *   estimate — which is not something to ship under the word `distinct`.
+ * - **`median`, percentiles, `stddev` of a stream** — all need the values, or a
+ *   digest that is an approximation with an error bound the config would have to
+ *   carry.
+ * - **`first` / `last`** — fixed-size, and excluded on the other half of the
+ *   rule: they mean "in input order", and this node's input order is a
+ *   `SELECT` without an `ORDER BY`. An aggregate that returns a different value
+ *   on a rerun is a load nobody can diff. `min`/`max` are the order-independent
+ *   version and are what somebody reaching for `first` usually wants.
+ * - **Conditional aggregation — `MAX(CASE WHEN … THEN … END)`** — deliberately
+ *   out of scope, and it is the one omission a reader of flip's `wo` query will
+ *   go looking for, because that query has a three-branch status ladder in it.
+ *   Admitting it means admitting a predicate *inside* an aggregate, which is a
+ *   second expression language nested in the first, evaluated per row per
+ *   aggregate. That is transform territory and the generic
+ *   {@link WorkflowTransformNode} still exists. What the ladder actually is, is
+ *   a priority ordering over a closed set of codes, and it composes: map the
+ *   code to a rank in a transform above this node, `min` the rank, map it back
+ *   below. Two cheap per-record steps instead of a language.
+ * - **`avg` is in**, and it is in *because* of the rule rather than despite it.
+ *   It is `sum` and `count` in one accumulator, both of which are already here,
+ *   and SQL has exactly one answer for it. Excluding it would have made the list
+ *   an arbitrary set that happened to cover one query, which is the thing the
+ *   rule is for.
+ */
+export declare const WORKFLOW_AGGREGATE_FUNCTIONS: readonly ["count", "sum", "avg", "min", "max", "join"];
+export type WorkflowAggregateFunction = (typeof WORKFLOW_AGGREGATE_FUNCTIONS)[number];
+/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
+export declare function isWorkflowAggregateFunction(value: unknown): value is WorkflowAggregateFunction;
+/**
+ * {@link unreachableNodeKind}, one level down, and for the identical reason.
+ *
+ * Every branch over {@link WorkflowAggregateFunction} ends here, so a seventh
+ * function added to the list without an accumulator, a finisher, a canonical
+ * form and a sentence is a type error naming the file rather than a node that
+ * saves, draws and then computes nothing. It throws as well, because a function
+ * name arrives as JSON out of a column and a build older than the data is a
+ * thing that happens.
+ */
+export declare function unreachableAggregateFunction(fn: never, where: string): never;
+/**
+ * How many columns one node may group on.
+ *
+ * The same argument {@link WORKFLOW_RENAME_MAX_COLUMNS} makes, plus one specific
+ * to this node: every extra group-by column can only ever *increase* the number
+ * of groups, so a long list is the shape a high-cardinality grouping arrives in.
+ * flip's real derivation groups on two.
+ */
+export declare const WORKFLOW_AGGREGATE_MAX_GROUP_BY = 16;
+/**
+ * How many aggregates one node may compute.
+ *
+ * flip's `wo` derivation has 49, so the bound has to be comfortably above that
+ * or the node does not do the job it was written for. Past a few hundred the
+ * thing being expressed is a table definition rather than a summary, and the
+ * cost is real: every aggregate is an accumulator held **per group**, so this
+ * number multiplies {@link WORKFLOW_AGGREGATE_MAX_GROUPS} in the heap.
+ */
+export declare const WORKFLOW_AGGREGATE_MAX_AGGREGATES = 256;
+/**
+ * The default ceiling on distinct groups, and the loud refusal that goes with
+ * it.
+ *
+ * A hash aggregate is cheap **only while the groups are far fewer than the
+ * rows**. Group on a near-unique column and it holds one accumulator row per
+ * input row, which is the whole-batch behaviour this node replaces, arrived at
+ * by a different route and with nothing on the canvas to point at. So the
+ * ceiling exists, it is crossed loudly, and the message names the columns being
+ * grouped on — because a bound that is merely reported is a bound that is
+ * discovered by the machine running out of memory.
+ *
+ * A million is chosen against the measurement rather than as a round number:
+ * flip's derivation holds 16,119, so the default is 62× the real case and no
+ * author of a sane grouping ever meets it. What it catches is `groupBy:
+ * ['combinedId']` on a 44,720-row type — a grouping that is *legal*, produces
+ * one group per row, and is somebody having picked the wrong column.
+ *
+ * The number is a proxy and it is worth saying which part it cannot see: what a
+ * group costs in bytes depends on how many aggregates the node has and how long
+ * a `join` grows. The first is bounded by
+ * {@link WORKFLOW_AGGREGATE_MAX_AGGREGATES}; the second has its own bound on the
+ * aggregate, because it is the one accumulator whose size is not fixed by the
+ * group count.
+ */
+export declare const WORKFLOW_AGGREGATE_MAX_GROUPS = 1000000;
+/**
+ * The highest ceiling an author may ask for.
+ *
+ * Configurable because "how many groups is too many" genuinely depends on the
+ * machine and on how wide the node is, and a hard-coded limit would make the
+ * node unusable for the one legitimate large grouping. Bounded because past this
+ * the answer is not a bigger number — it is that the grouping belongs in the
+ * source query, where the database already has spill-to-disk and this process
+ * does not.
+ */
+export declare const WORKFLOW_AGGREGATE_GROUPS_CEILING = 20000000;
+/**
+ * The default bound on one joined value, in characters.
+ *
+ * 65,535 because that is what a MySQL `TEXT` column holds, and a value the
+ * target column cannot store is the same defect one layer further down. See
+ * `appendJoin` for the full argument, including the five groups per column that
+ * are silently truncated in production today under a limit of 1,024.
+ */
+export declare const WORKFLOW_AGGREGATE_JOIN_MAX_LENGTH = 65535;
+/** The highest an author may raise a `join` bound to. One `MEDIUMTEXT`. */
+export declare const WORKFLOW_AGGREGATE_JOIN_LENGTH_CEILING = 16777215;
+/** The longest separator a `join` may use. Long enough for `" | "`, short enough not to be data. */
+export declare const WORKFLOW_AGGREGATE_MAX_SEPARATOR = 16;
+/** The separator a `join` uses when the aggregate does not name one. */
+export declare const WORKFLOW_AGGREGATE_DEFAULT_SEPARATOR = ", ";
+/**
+ * One computed column: a function, what it reads, and what it is called.
+ *
+ * A list of these rather than a `Record<name, spec>`, which is the opposite of
+ * the shape {@link WorkflowRenameNode.columns} takes, and the reason is that the
+ * two nodes are keyed by different things. A rename is keyed by its **source**
+ * column and a source column can only be renamed once, so an object gets that
+ * refusal for free. An aggregate is keyed by its **output** name, and the same
+ * source column being read by several aggregates — `min(closedDate)` and
+ * `max(closedDate)` — is the normal case rather than the mistake. A list also
+ * keeps the author's order, which is the order the output columns come out in.
+ *
+ * Two aggregates sharing one `as` is the mistake, it is representable, and
+ * {@link aggregateRefusals} names both of them.
+ */
+export interface WorkflowAggregate {
+    /**
+     * The output column name. Matches {@link WORKFLOW_FILTER_COLUMN_PATTERN}.
+     *
+     * Constrained for the reason {@link WorkflowRenameNode.columns}' targets are,
+     * and it is the same trap: `property-names.ts` refuses a published property
+     * whose name cannot become a column, a load looks every field up as
+     * `row[name]`, and a name this service cannot carry downstream loads NULL into
+     * every row and reports success.
+     */
+    as: string;
+    /** Which function. See {@link WORKFLOW_AGGREGATE_FUNCTIONS}. */
+    fn: WorkflowAggregateFunction;
+    /**
+     * The column it reads. Absent is allowed **only** for `count`, where it means
+     * `COUNT(*)` — how many records landed in the group.
+     *
+     * Also matches {@link WORKFLOW_FILTER_COLUMN_PATTERN}, and that is a rule
+     * about *input* rather than output, which the rename node deliberately does
+     * not have. It is here because the filter node already draws the line in the
+     * same place and for the same forward-looking reason: an aggregate that reads
+     * a column no `GROUP BY` could name is one that could never be pushed into the
+     * query the source already runs. A source whose own headers are `Work Order
+     * Id` is what the `rename` node is for, one node upstream.
+     */
+    column?: string;
+    /**
+     * `join` only. What goes between the values. Absent means
+     * {@link WORKFLOW_AGGREGATE_DEFAULT_SEPARATOR}.
+     */
+    separator?: string;
+    /**
+     * `join` only. The bound, in characters, that this aggregate refuses at.
+     * Absent means {@link WORKFLOW_AGGREGATE_JOIN_MAX_LENGTH}.
+     */
+    maxLength?: number;
+}
+/**
+ * Groups records and computes a summary of each group. No author code.
+ *
+ * ## Why this is a node kind, and why it stays small
+ *
+ * The argument the `rename` node made, and it holds unchanged: the generic
+ * {@link WorkflowTransformNode} continues to exist, and **that is what lets this
+ * node stay deliberately narrow forever**. "I need more than this" is answered
+ * with *use a transform*, never with *add a field here*. The refusals in this
+ * docblock are the point of it, not an apology.
+ *
+ * What is different from `rename` is that the payoff is not speed, it is a
+ * **memory bound**, and it is the difference between working and being killed.
+ *
+ * ## The measurement
+ *
+ * flip's `wo` table is a `GROUP BY` over 44,720 SUBWO rows producing 16,119
+ * groups with 49 aggregates. Reproduced as a whole-batch transform — which is
+ * the only way this graph can express it today — it works, and only just:
+ *
+ * - the child process is handed **65.51 MiB** on stdin;
+ * - it answers with a single JSON line of **25.01 MiB, which is 78.2% of the
+ *   hard 32 MiB output cap**, buffered by the parent as one string;
+ * - and `readInputs` materialises all 44,720 records before the child starts. A
+ *   standalone equivalent peaked at 237 MiB heap and 406 MiB RSS.
+ *
+ * **It fails at 1.28× that file** — one more summed column, or one more month of
+ * work orders — and it fails by being killed at the cap rather than by
+ * degrading. SUBWO is already the largest file in the drop it comes from. So the
+ * commonest aggregation in the system is one column away from dying at a
+ * ceiling.
+ *
+ * A hash aggregate consumes its input as a stream and **holds only the groups**:
+ * 44,720 rows in, 16,119 accumulator rows held, one staged batch decoded at a
+ * time. For most real aggregations the group count is far smaller than the row
+ * count, and that gap is the entire feature.
+ *
+ * ## Which is why the bound is loud rather than absent
+ *
+ * "Holds only the groups" stops being cheap the moment the groups approach the
+ * rows. So {@link maxGroups} is a ceiling that **refuses**, naming the columns
+ * being grouped on. Peaking silently is the failure being fixed, and a node that
+ * fixed it by peaking somewhere else would not have fixed anything.
+ *
+ * ## What comes out
+ *
+ * The group-by columns, then one column per aggregate, in the node's order —
+ * **on every record, always, including where the answer is null**. That makes an
+ * aggregate the only node kind whose output column set is *exact* rather than an
+ * upper bound; see `producedColumns`.
+ *
+ * ## Every edge case, decided rather than discovered
+ *
+ * - **A group-by column not present in a record** — that record groups with the
+ *   records whose column is null, and the output carries `null`. SQL groups all
+ *   NULLs together, and the stage encoding makes absent-versus-null a difference
+ *   in physical layout rather than in meaning, so splitting them would make the
+ *   answer depend on which shape a row landed in.
+ * - **A null group key** — a group like any other, exactly as `GROUP BY`
+ *   produces.
+ * - **`1` and `"1"` as group keys** — *different* groups. SQL would coerce them
+ *   together; merging two things the source considered distinct on a rule nobody
+ *   chose is the worse error, and within one load a column is one type. See
+ *   `groupKeyOf`.
+ * - **Empty input** — no groups, so no rows staged, which is what `GROUP BY`
+ *   over an empty table produces. The sink already refuses to commit an empty
+ *   full snapshot, so nothing new is needed and nothing is silently green.
+ * - **An empty {@link groupBy}** — *refused*, and it is worth saying why rather
+ *   than leaving it as a bound. With no grouping, SQL returns exactly one row
+ *   whether the input had a billion rows or none, and a run that commits one row
+ *   of nulls looks identical to a run that summarised everything. A grand total
+ *   is genuinely wanted sometimes; the way to ask for it is a constant column
+ *   from a transform, which makes the one row visible in the graph.
+ * - **Two aggregates writing the same output name** — refused, naming both, from
+ *   the config alone with nothing to run.
+ * - **An aggregate whose `as` collides with a group-by column** — refused, same
+ *   sentence, same reason: one name, two values, and every rule for picking a
+ *   winner is arbitrary.
+ * - **Aggregating a column that does not exist upstream** — refused at *save*
+ *   wherever `workflowKnownColumns` can prove it (below a `rename` that drops
+ *   what it does not name, or below a `catalog` source when the caller supplied
+ *   a column lookup). Where the graph cannot prove it, the run says so out loud:
+ *   a column present in **no** record of the whole run is reported, and `sum`,
+ *   `min`, `max` and `join` over nothing answer `null` rather than `0` or `""`,
+ *   which is visibly different from a real answer.
+ * - **Mixed types in a `min`/`max`, or non-numeric text in a `sum`** — refused
+ *   at run time, naming the column, the group and the values. MySQL coerces;
+ *   coercing here would make the answer depend on which record arrived first and
+ *   the run would still be green.
+ *
+ * ## Ordering, and what is actually promised
+ *
+ * `GROUP_CONCAT` without an `ORDER BY` is unordered, which is why a comparison
+ * against flip found identical-length, different-order strings. This node joins
+ * in **input order** and emits groups in **first-seen order**, both of which are
+ * functions of the numbered list of staged batches it reads. So two runs over
+ * the same staged input produce the same bytes. What is *not* promised is
+ * stability across a source that returns its own rows in a different order — a
+ * `SELECT` without an `ORDER BY` promises nothing, and this node cannot promise
+ * more than it was handed.
+ */
+export interface WorkflowAggregateNode extends WorkflowNodeBase {
+    kind: 'aggregate';
+    /**
+     * The columns that define a group. Never empty; at most
+     * {@link WORKFLOW_AGGREGATE_MAX_GROUP_BY}; each matches
+     * {@link WORKFLOW_FILTER_COLUMN_PATTERN}; no duplicates.
+     *
+     * These are output column names as well as input ones — the group key comes
+     * out under the name it went in under — which is why they carry the target
+     * name rule and not only the input one.
+     */
+    groupBy: string[];
+    /**
+     * What to compute per group. Never empty; at most
+     * {@link WORKFLOW_AGGREGATE_MAX_AGGREGATES}; no two share an `as`.
+     *
+     * Empty is refused rather than treated as "just deduplicate", which is what it
+     * would silently be: a node emitting the distinct combinations of its group-by
+     * columns and dropping every other column of every row. That is a real
+     * operation and a completely different one, and reaching it by deleting the
+     * last row of a form is how a published type loses forty columns.
+     */
+    aggregates: WorkflowAggregate[];
+    /**
+     * The ceiling on distinct groups. Absent means
+     * {@link WORKFLOW_AGGREGATE_MAX_GROUPS}; at most
+     * {@link WORKFLOW_AGGREGATE_GROUPS_CEILING}.
+     */
+    maxGroups?: number;
+}
+/** {@link WorkflowAggregateNode.maxGroups}, resolved. One reader of the default. */
+export declare function workflowAggregateMaxGroups(node: WorkflowAggregateNode): number;
+/** {@link WorkflowAggregate.separator}, resolved. One reader of the default. */
+export declare function workflowAggregateSeparator(aggregate: WorkflowAggregate): string;
+/** {@link WorkflowAggregate.maxLength}, resolved. One reader of the default. */
+export declare function workflowAggregateJoinMaxLength(aggregate: WorkflowAggregate): number;
+/** Whether this function reads a column. Only `count` may go without one. */
+export declare function workflowAggregateNeedsColumn(fn: WorkflowAggregateFunction): boolean;
+/**
+ * The columns an aggregate node **reads**: its group keys and its inputs.
+ *
+ * What `checkColumnsProduced` tests against what the graph can prove is there,
+ * and what the run log reports as never-seen. Deduplicated and in a stable
+ * order, because it goes into a sentence.
+ */
+export declare function workflowAggregateColumns(node: WorkflowAggregateNode): string[];
+/**
+ * The columns an aggregate node **produces**, which is all of them and nothing
+ * else.
+ *
+ * Closed by the config, and closed *exactly* rather than as an upper bound —
+ * every emitted record carries every one of these keys, whatever was upstream
+ * and whatever the values turned out to be. That is a stronger claim than the
+ * one `rename` introduced, and it is stronger for a structural reason: a rename
+ * only produces a target where the input actually held the source column,
+ * whereas an aggregate writes a group's answer whether or not anything in the
+ * group had a value for it.
+ *
+ * The one thing it does not claim is that the values are useful. An aggregate
+ * over a column that no record carried produces the column, holding `null`.
+ */
+export declare function workflowAggregateOutputColumns(node: WorkflowAggregateNode): string[];
+/**
+ * Every reason an aggregate cannot be stored, as sentences, or empty.
+ *
+ * One function, called by {@link validateWorkflow}, by the HTTP boundary, by the
+ * canvas and by the fold itself, for the reason {@link renameColumnRefusals} is
+ * shared: a screen with its own copy of the identifier pattern is a screen that
+ * accepts something the server refuses, halfway through a save.
+ *
+ * All of them rather than the first, exactly as
+ * {@link refuseUnpublishablePropertyNames} argues: a node with forty aggregates
+ * typed in one sitting is usually wrong about several in the same way.
+ */
+export declare function aggregateRefusals(node: {
+    groupBy?: unknown;
+    aggregates?: unknown;
+    maxGroups?: unknown;
+}): string[];
+/**
+ * Whether a stored aggregate list is one this build can run.
+ *
+ * Refused rather than repaired, the stance {@link isWorkflowRenameColumns} takes
+ * and for the same reason one step further along: an aggregate list read back
+ * with one entry silently dropped is a graph that commits a column of nulls
+ * under a name somebody put in an object type on purpose.
+ */
+export declare function isWorkflowAggregates(value: unknown): value is WorkflowAggregate[];
 /**
  * A discriminated union, so narrowing a node is `node.kind === "sink"` and
  * never a type assertion. This is why the kind list is not simply a string on
  * one node shape with every field optional: that shape lets a source node carry
  * a `transformId` and nothing catches it.
  */
-export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode | WorkflowRenameNode;
+export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode | WorkflowRenameNode | WorkflowAggregateNode;
 /**
  * The node kinds that can be saved once and used in several graphs.
  *
@@ -2403,6 +2782,13 @@ export declare function isReusableNodeKind(value: unknown): value is ReusableNod
  *   *about* one drop of one file. `Mgmt Cd → mgmtCd` saved under a name and
  *   dropped into a graph reading a different system renames nothing at all, and
  *   the symptom is a column of NULLs rather than a failure.
+ * - `aggregate` — the same again, and it fails in both directions at once. Its
+ *   group-by columns and its inputs name one type's columns, so a shared one
+ *   groups a graph it was not written for on a column that is not there — which
+ *   collapses every record into one null-keyed group rather than erroring. And
+ *   its *output* column set is the thing downstream nodes are validated against,
+ *   so a shared node editable from elsewhere would silently change what another
+ *   graph's sink is allowed to write.
  */
 export declare const NODE_KIND_IS_REUSABLE: {
     readonly source: true;
@@ -2412,6 +2798,7 @@ export declare const NODE_KIND_IS_REUSABLE: {
     readonly if: false;
     readonly filter: false;
     readonly rename: false;
+    readonly aggregate: false;
 };
 /** Whether this kind can be saved as a reusable node. Reads {@link NODE_KIND_IS_REUSABLE}. */
 export declare function nodeKindIsReusable(kind: WorkflowNodeKind): boolean;
@@ -3210,7 +3597,7 @@ export interface CallableWorkflowBlock {
 }
 export declare function callableWorkflowBlock(ref: CallableWorkflowRef): CallableWorkflowBlock | undefined;
 /** Every way a graph can be refused. Exported so a canvas can key off the code. */
-export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "source-type-not-named", "call-not-named", "call-plain-has-output", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing", "rename-invalid", "column-not-produced", "version-pin-invalid"];
+export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "source-type-not-named", "call-not-named", "call-plain-has-output", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing", "rename-invalid", "aggregate-invalid", "column-not-produced", "version-pin-invalid"];
 export type WorkflowIssueCode = (typeof WORKFLOW_ISSUE_CODES)[number];
 export interface WorkflowValidationIssue {
     code: WorkflowIssueCode;