@dudousxd/nestjs-catalog 0.26.0 → 0.28.0

package/dist/catalog.pipeline.d.ts

@@ -1183,12 +1183,25 @@ export interface WorkflowNodeOutcome {
  *   {@link WorkflowIfNode} is the conditional one, and it earns its kind by
  *   doing something no wiring can express — deciding that one of those
  *   successors, and everything only it feeds, does not run at all.
- * - **merge / join** — a node with several inbound edges receives its inputs
- *   concatenated in edge order (see {@link WorkflowEdge}). A keyed join is then
- *   ordinary code inside the transform, which can already see every record.
- *   A `merge` kind would have had to carry a strategy field whose values the
- *   runner would have to implement one by one, and an unimplemented strategy in
- *   a dropdown is the failure this list exists to avoid.
+ * - **merge (unkeyed)** — a node with several inbound edges already receives its
+ *   inputs concatenated in edge order (see {@link WorkflowEdge}), so a `merge`
+ *   kind would be a box that draws what the wires already say. It would also
+ *   have had to carry a strategy field whose values the runner implements one by
+ *   one, and an unimplemented strategy in a dropdown is the failure this list
+ *   exists to avoid.
+ * - **join (keyed)** — *this half used to be refused with the entry above, and
+ *   the reversal is left visible rather than edited out*, the way the `filter`
+ *   entry leaves its own. The old argument was that a keyed join is ordinary
+ *   code inside a transform, which can already see every record. Every word of
+ *   that is true and it is exactly the problem: "can already see every record"
+ *   is the same sentence as "holds the whole load", and it is why a transform
+ *   makes `ConnectorRunnerService` log *"Held all N records in memory"*. A join
+ *   does not need both sides held. It needs **one** side held — as a map, keyed
+ *   — while the other streams past it, and that asymmetry is a property of the
+ *   operation that a function over a batch cannot express and a runner therefore
+ *   cannot exploit. {@link WorkflowLookupNode} is the keyed half, built narrow:
+ *   one key, named enrichment fields, and a reference side that is bounded and
+ *   refused loudly rather than held quietly.
  * - **call a durable *step*** — the sibling of {@link WorkflowCallNode} that
  *   somebody will eventually come looking for, and it cannot be built. A
  *   durable step has no global identity: it is dispatched by a routing name
@@ -1199,7 +1212,7 @@ export interface WorkflowNodeOutcome {
  *   outside a run, which is why `call` names one and not a step. If a step is
  *   what you want, the thing to call is a one-step workflow wrapping it.
  */
-export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter", "rename"];
+export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter", "rename", "aggregate", "lookup"];
 export type WorkflowNodeKind = (typeof WORKFLOW_NODE_KINDS)[number];
 /** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
 export declare function isWorkflowNodeKind(value: unknown): value is WorkflowNodeKind;
@@ -2340,13 +2353,730 @@ export declare function renameColumnRefusals(columns: Record<string, string>): s
  * map, and the values are checked one by one rather than trusted from the type.
  */
 export declare function isWorkflowRenameColumns(value: unknown): value is Record<string, string>;
+/**
+ * The aggregate functions this node computes, and the rule that closes the list.
+ *
+ * A closed list with an exhaustiveness guard, for the reason every other list in
+ * this file is one. What is different here is that the list has a **stated
+ * admission rule**, because "we will keep it narrow" is a promise nobody can
+ * check and a rule is:
+ *
+ * > A function is in if it can be computed from a **fixed-size accumulator**,
+ * > and if its answer does not depend on a decision the config would have to
+ * > carry.
+ *
+ * The first half is the node's whole reason to exist. A hash aggregate is cheap
+ * because it holds one entry per group; an accumulator whose size grows with the
+ * number of *rows* in a group puts the rows back in memory and gives up the
+ * property. The second half is what keeps the config from becoming a small
+ * language: a function that needs an extra field to say what it means is a
+ * function whose meaning was not decided.
+ *
+ * What the rule excludes, so the omissions are on the record rather than
+ * implied:
+ *
+ * - **`countDistinct`** — the sharpest one. It needs a set of the distinct
+ *   values *per group per column*, so its accumulator is O(distinct values) and
+ *   a high-cardinality column inside a group holds the load. It is the exact
+ *   thing this node was built to stop doing, wearing an aggregate's name. A
+ *   sketch (HyperLogLog) is fixed-size and is a different function — an
+ *   estimate — which is not something to ship under the word `distinct`.
+ * - **`median`, percentiles, `stddev` of a stream** — all need the values, or a
+ *   digest that is an approximation with an error bound the config would have to
+ *   carry.
+ * - **`first` / `last`** — fixed-size, and excluded on the other half of the
+ *   rule: they mean "in input order", and this node's input order is a
+ *   `SELECT` without an `ORDER BY`. An aggregate that returns a different value
+ *   on a rerun is a load nobody can diff. `min`/`max` are the order-independent
+ *   version and are what somebody reaching for `first` usually wants.
+ * - **Conditional aggregation — `MAX(CASE WHEN … THEN … END)`** — deliberately
+ *   out of scope, and it is the one omission a reader of flip's `wo` query will
+ *   go looking for, because that query has a three-branch status ladder in it.
+ *   Admitting it means admitting a predicate *inside* an aggregate, which is a
+ *   second expression language nested in the first, evaluated per row per
+ *   aggregate. That is transform territory and the generic
+ *   {@link WorkflowTransformNode} still exists. What the ladder actually is, is
+ *   a priority ordering over a closed set of codes, and it composes: map the
+ *   code to a rank in a transform above this node, `min` the rank, map it back
+ *   below. Two cheap per-record steps instead of a language.
+ * - **`avg` is in**, and it is in *because* of the rule rather than despite it.
+ *   It is `sum` and `count` in one accumulator, both of which are already here,
+ *   and SQL has exactly one answer for it. Excluding it would have made the list
+ *   an arbitrary set that happened to cover one query, which is the thing the
+ *   rule is for.
+ */
+export declare const WORKFLOW_AGGREGATE_FUNCTIONS: readonly ["count", "sum", "avg", "min", "max", "join"];
+export type WorkflowAggregateFunction = (typeof WORKFLOW_AGGREGATE_FUNCTIONS)[number];
+/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
+export declare function isWorkflowAggregateFunction(value: unknown): value is WorkflowAggregateFunction;
+/**
+ * {@link unreachableNodeKind}, one level down, and for the identical reason.
+ *
+ * Every branch over {@link WorkflowAggregateFunction} ends here, so a seventh
+ * function added to the list without an accumulator, a finisher, a canonical
+ * form and a sentence is a type error naming the file rather than a node that
+ * saves, draws and then computes nothing. It throws as well, because a function
+ * name arrives as JSON out of a column and a build older than the data is a
+ * thing that happens.
+ */
+export declare function unreachableAggregateFunction(fn: never, where: string): never;
+/**
+ * What happens to a driving row whose key matches no reference row.
+ *
+ * Three words rather than a boolean, because the three are the three joins SQL
+ * has names for and each is a different node. See
+ * {@link WorkflowLookupNode.unmatched} for which one to reach for.
+ */
+export declare const WORKFLOW_LOOKUP_UNMATCHED: readonly ["null", "drop", "fail"];
+export type WorkflowLookupUnmatched = (typeof WORKFLOW_LOOKUP_UNMATCHED)[number];
+/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
+export declare function isWorkflowLookupUnmatched(value: unknown): value is WorkflowLookupUnmatched;
+/**
+ * The exhaustiveness guard for {@link WORKFLOW_LOOKUP_UNMATCHED}.
+ *
+ * {@link unreachableNodeKind}, one level down, and for the identical reason: the
+ * three words decide whether a row keeps its data, disappears from the load, or
+ * stops the run, and a fourth added without a branch would silently pick
+ * whichever the last `if` was — which here means silently changing which rows
+ * reach a published type.
+ */
+export declare function unreachableLookupUnmatched(value: never, where: string): never;
+/**
+ * How many columns one node may group on.
+ *
+ * The same argument {@link WORKFLOW_RENAME_MAX_COLUMNS} makes, plus one specific
+ * to this node: every extra group-by column can only ever *increase* the number
+ * of groups, so a long list is the shape a high-cardinality grouping arrives in.
+ * flip's real derivation groups on two.
+ */
+export declare const WORKFLOW_AGGREGATE_MAX_GROUP_BY = 16;
+/**
+ * How many aggregates one node may compute.
+ *
+ * flip's `wo` derivation has 49, so the bound has to be comfortably above that
+ * or the node does not do the job it was written for. Past a few hundred the
+ * thing being expressed is a table definition rather than a summary, and the
+ * cost is real: every aggregate is an accumulator held **per group**, so this
+ * number multiplies {@link WORKFLOW_AGGREGATE_MAX_GROUPS} in the heap.
+ */
+export declare const WORKFLOW_AGGREGATE_MAX_AGGREGATES = 256;
+/**
+ * The default ceiling on distinct groups, and the loud refusal that goes with
+ * it.
+ *
+ * A hash aggregate is cheap **only while the groups are far fewer than the
+ * rows**. Group on a near-unique column and it holds one accumulator row per
+ * input row, which is the whole-batch behaviour this node replaces, arrived at
+ * by a different route and with nothing on the canvas to point at. So the
+ * ceiling exists, it is crossed loudly, and the message names the columns being
+ * grouped on — because a bound that is merely reported is a bound that is
+ * discovered by the machine running out of memory.
+ *
+ * A million is chosen against the measurement rather than as a round number:
+ * flip's derivation holds 16,119, so the default is 62× the real case and no
+ * author of a sane grouping ever meets it. What it catches is `groupBy:
+ * ['combinedId']` on a 44,720-row type — a grouping that is *legal*, produces
+ * one group per row, and is somebody having picked the wrong column.
+ *
+ * The number is a proxy and it is worth saying which part it cannot see: what a
+ * group costs in bytes depends on how many aggregates the node has and how long
+ * a `join` grows. The first is bounded by
+ * {@link WORKFLOW_AGGREGATE_MAX_AGGREGATES}; the second has its own bound on the
+ * aggregate, because it is the one accumulator whose size is not fixed by the
+ * group count.
+ */
+export declare const WORKFLOW_AGGREGATE_MAX_GROUPS = 1000000;
+/**
+ * The highest ceiling an author may ask for.
+ *
+ * Configurable because "how many groups is too many" genuinely depends on the
+ * machine and on how wide the node is, and a hard-coded limit would make the
+ * node unusable for the one legitimate large grouping. Bounded because past this
+ * the answer is not a bigger number — it is that the grouping belongs in the
+ * source query, where the database already has spill-to-disk and this process
+ * does not.
+ */
+export declare const WORKFLOW_AGGREGATE_GROUPS_CEILING = 20000000;
+/**
+ * The default bound on one joined value, in characters.
+ *
+ * 65,535 because that is what a MySQL `TEXT` column holds, and a value the
+ * target column cannot store is the same defect one layer further down. See
+ * `appendJoin` for the full argument, including the five groups per column that
+ * are silently truncated in production today under a limit of 1,024.
+ */
+export declare const WORKFLOW_AGGREGATE_JOIN_MAX_LENGTH = 65535;
+/** The highest an author may raise a `join` bound to. One `MEDIUMTEXT`. */
+export declare const WORKFLOW_AGGREGATE_JOIN_LENGTH_CEILING = 16777215;
+/** The longest separator a `join` may use. Long enough for `" | "`, short enough not to be data. */
+export declare const WORKFLOW_AGGREGATE_MAX_SEPARATOR = 16;
+/** The separator a `join` uses when the aggregate does not name one. */
+export declare const WORKFLOW_AGGREGATE_DEFAULT_SEPARATOR = ", ";
+/**
+ * One computed column: a function, what it reads, and what it is called.
+ *
+ * A list of these rather than a `Record<name, spec>`, which is the opposite of
+ * the shape {@link WorkflowRenameNode.columns} takes, and the reason is that the
+ * two nodes are keyed by different things. A rename is keyed by its **source**
+ * column and a source column can only be renamed once, so an object gets that
+ * refusal for free. An aggregate is keyed by its **output** name, and the same
+ * source column being read by several aggregates — `min(closedDate)` and
+ * `max(closedDate)` — is the normal case rather than the mistake. A list also
+ * keeps the author's order, which is the order the output columns come out in.
+ *
+ * Two aggregates sharing one `as` is the mistake, it is representable, and
+ * {@link aggregateRefusals} names both of them.
+ */
+export interface WorkflowAggregate {
+    /**
+     * The output column name. Matches {@link WORKFLOW_FILTER_COLUMN_PATTERN}.
+     *
+     * Constrained for the reason {@link WorkflowRenameNode.columns}' targets are,
+     * and it is the same trap: `property-names.ts` refuses a published property
+     * whose name cannot become a column, a load looks every field up as
+     * `row[name]`, and a name this service cannot carry downstream loads NULL into
+     * every row and reports success.
+     */
+    as: string;
+    /** Which function. See {@link WORKFLOW_AGGREGATE_FUNCTIONS}. */
+    fn: WorkflowAggregateFunction;
+    /**
+     * The column it reads. Absent is allowed **only** for `count`, where it means
+     * `COUNT(*)` — how many records landed in the group.
+     *
+     * Also matches {@link WORKFLOW_FILTER_COLUMN_PATTERN}, and that is a rule
+     * about *input* rather than output, which the rename node deliberately does
+     * not have. It is here because the filter node already draws the line in the
+     * same place and for the same forward-looking reason: an aggregate that reads
+     * a column no `GROUP BY` could name is one that could never be pushed into the
+     * query the source already runs. A source whose own headers are `Work Order
+     * Id` is what the `rename` node is for, one node upstream.
+     */
+    column?: string;
+    /**
+     * `join` only. What goes between the values. Absent means
+     * {@link WORKFLOW_AGGREGATE_DEFAULT_SEPARATOR}.
+     */
+    separator?: string;
+    /**
+     * `join` only. The bound, in characters, that this aggregate refuses at.
+     * Absent means {@link WORKFLOW_AGGREGATE_JOIN_MAX_LENGTH}.
+     */
+    maxLength?: number;
+}
+/**
+ * Groups records and computes a summary of each group. No author code.
+ *
+ * ## Why this is a node kind, and why it stays small
+ *
+ * The argument the `rename` node made, and it holds unchanged: the generic
+ * {@link WorkflowTransformNode} continues to exist, and **that is what lets this
+ * node stay deliberately narrow forever**. "I need more than this" is answered
+ * with *use a transform*, never with *add a field here*. The refusals in this
+ * docblock are the point of it, not an apology.
+ *
+ * What is different from `rename` is that the payoff is not speed, it is a
+ * **memory bound**, and it is the difference between working and being killed.
+ *
+ * ## The measurement
+ *
+ * flip's `wo` table is a `GROUP BY` over 44,720 SUBWO rows producing 16,119
+ * groups with 49 aggregates. Reproduced as a whole-batch transform — which is
+ * the only way this graph can express it today — it works, and only just:
+ *
+ * - the child process is handed **65.51 MiB** on stdin;
+ * - it answers with a single JSON line of **25.01 MiB, which is 78.2% of the
+ *   hard 32 MiB output cap**, buffered by the parent as one string;
+ * - and `readInputs` materialises all 44,720 records before the child starts. A
+ *   standalone equivalent peaked at 237 MiB heap and 406 MiB RSS.
+ *
+ * **It fails at 1.28× that file** — one more summed column, or one more month of
+ * work orders — and it fails by being killed at the cap rather than by
+ * degrading. SUBWO is already the largest file in the drop it comes from. So the
+ * commonest aggregation in the system is one column away from dying at a
+ * ceiling.
+ *
+ * A hash aggregate consumes its input as a stream and **holds only the groups**:
+ * 44,720 rows in, 16,119 accumulator rows held, one staged batch decoded at a
+ * time. For most real aggregations the group count is far smaller than the row
+ * count, and that gap is the entire feature.
+ *
+ * ## Which is why the bound is loud rather than absent
+ *
+ * "Holds only the groups" stops being cheap the moment the groups approach the
+ * rows. So {@link maxGroups} is a ceiling that **refuses**, naming the columns
+ * being grouped on. Peaking silently is the failure being fixed, and a node that
+ * fixed it by peaking somewhere else would not have fixed anything.
+ *
+ * ## What comes out
+ *
+ * The group-by columns, then one column per aggregate, in the node's order —
+ * **on every record, always, including where the answer is null**. That makes an
+ * aggregate the only node kind whose output column set is *exact* rather than an
+ * upper bound; see `producedColumns`.
+ *
+ * ## Every edge case, decided rather than discovered
+ *
+ * - **A group-by column not present in a record** — that record groups with the
+ *   records whose column is null, and the output carries `null`. SQL groups all
+ *   NULLs together, and the stage encoding makes absent-versus-null a difference
+ *   in physical layout rather than in meaning, so splitting them would make the
+ *   answer depend on which shape a row landed in.
+ * - **A null group key** — a group like any other, exactly as `GROUP BY`
+ *   produces.
+ * - **`1` and `"1"` as group keys** — *different* groups. SQL would coerce them
+ *   together; merging two things the source considered distinct on a rule nobody
+ *   chose is the worse error, and within one load a column is one type. See
+ *   `groupKeyOf`.
+ * - **Empty input** — no groups, so no rows staged, which is what `GROUP BY`
+ *   over an empty table produces. The sink already refuses to commit an empty
+ *   full snapshot, so nothing new is needed and nothing is silently green.
+ * - **An empty {@link groupBy}** — *refused*, and it is worth saying why rather
+ *   than leaving it as a bound. With no grouping, SQL returns exactly one row
+ *   whether the input had a billion rows or none, and a run that commits one row
+ *   of nulls looks identical to a run that summarised everything. A grand total
+ *   is genuinely wanted sometimes; the way to ask for it is a constant column
+ *   from a transform, which makes the one row visible in the graph.
+ * - **Two aggregates writing the same output name** — refused, naming both, from
+ *   the config alone with nothing to run.
+ * - **An aggregate whose `as` collides with a group-by column** — refused, same
+ *   sentence, same reason: one name, two values, and every rule for picking a
+ *   winner is arbitrary.
+ * - **Aggregating a column that does not exist upstream** — refused at *save*
+ *   wherever `workflowKnownColumns` can prove it (below a `rename` that drops
+ *   what it does not name, or below a `catalog` source when the caller supplied
+ *   a column lookup). Where the graph cannot prove it, the run says so out loud:
+ *   a column present in **no** record of the whole run is reported, and `sum`,
+ *   `min`, `max` and `join` over nothing answer `null` rather than `0` or `""`,
+ *   which is visibly different from a real answer.
+ * - **Mixed types in a `min`/`max`, or non-numeric text in a `sum`** — refused
+ *   at run time, naming the column, the group and the values. MySQL coerces;
+ *   coercing here would make the answer depend on which record arrived first and
+ *   the run would still be green.
+ *
+ * ## Ordering, and what is actually promised
+ *
+ * `GROUP_CONCAT` without an `ORDER BY` is unordered, which is why a comparison
+ * against flip found identical-length, different-order strings. This node joins
+ * in **input order** and emits groups in **first-seen order**, both of which are
+ * functions of the numbered list of staged batches it reads. So two runs over
+ * the same staged input produce the same bytes. What is *not* promised is
+ * stability across a source that returns its own rows in a different order — a
+ * `SELECT` without an `ORDER BY` promises nothing, and this node cannot promise
+ * more than it was handed.
+ */
+export interface WorkflowAggregateNode extends WorkflowNodeBase {
+    kind: 'aggregate';
+    /**
+     * The columns that define a group. Never empty; at most
+     * {@link WORKFLOW_AGGREGATE_MAX_GROUP_BY}; each matches
+     * {@link WORKFLOW_FILTER_COLUMN_PATTERN}; no duplicates.
+     *
+     * These are output column names as well as input ones — the group key comes
+     * out under the name it went in under — which is why they carry the target
+     * name rule and not only the input one.
+     */
+    groupBy: string[];
+    /**
+     * What to compute per group. Never empty; at most
+     * {@link WORKFLOW_AGGREGATE_MAX_AGGREGATES}; no two share an `as`.
+     *
+     * Empty is refused rather than treated as "just deduplicate", which is what it
+     * would silently be: a node emitting the distinct combinations of its group-by
+     * columns and dropping every other column of every row. That is a real
+     * operation and a completely different one, and reaching it by deleting the
+     * last row of a form is how a published type loses forty columns.
+     */
+    aggregates: WorkflowAggregate[];
+    /**
+     * The ceiling on distinct groups. Absent means
+     * {@link WORKFLOW_AGGREGATE_MAX_GROUPS}; at most
+     * {@link WORKFLOW_AGGREGATE_GROUPS_CEILING}.
+     */
+    maxGroups?: number;
+}
+/** {@link WorkflowAggregateNode.maxGroups}, resolved. One reader of the default. */
+export declare function workflowAggregateMaxGroups(node: WorkflowAggregateNode): number;
+/** {@link WorkflowAggregate.separator}, resolved. One reader of the default. */
+export declare function workflowAggregateSeparator(aggregate: WorkflowAggregate): string;
+/** {@link WorkflowAggregate.maxLength}, resolved. One reader of the default. */
+export declare function workflowAggregateJoinMaxLength(aggregate: WorkflowAggregate): number;
+/** Whether this function reads a column. Only `count` may go without one. */
+export declare function workflowAggregateNeedsColumn(fn: WorkflowAggregateFunction): boolean;
+/**
+ * The columns an aggregate node **reads**: its group keys and its inputs.
+ *
+ * What `checkColumnsProduced` tests against what the graph can prove is there,
+ * and what the run log reports as never-seen. Deduplicated and in a stable
+ * order, because it goes into a sentence.
+ */
+export declare function workflowAggregateColumns(node: WorkflowAggregateNode): string[];
+/**
+ * The columns an aggregate node **produces**, which is all of them and nothing
+ * else.
+ *
+ * Closed by the config, and closed *exactly* rather than as an upper bound —
+ * every emitted record carries every one of these keys, whatever was upstream
+ * and whatever the values turned out to be. That is a stronger claim than the
+ * one `rename` introduced, and it is stronger for a structural reason: a rename
+ * only produces a target where the input actually held the source column,
+ * whereas an aggregate writes a group's answer whether or not anything in the
+ * group had a value for it.
+ *
+ * The one thing it does not claim is that the values are useful. An aggregate
+ * over a column that no record carried produces the column, holding `null`.
+ */
+export declare function workflowAggregateOutputColumns(node: WorkflowAggregateNode): string[];
+/**
+ * Every reason an aggregate cannot be stored, as sentences, or empty.
+ *
+ * One function, called by {@link validateWorkflow}, by the HTTP boundary, by the
+ * canvas and by the fold itself, for the reason {@link renameColumnRefusals} is
+ * shared: a screen with its own copy of the identifier pattern is a screen that
+ * accepts something the server refuses, halfway through a save.
+ *
+ * All of them rather than the first, exactly as
+ * {@link refuseUnpublishablePropertyNames} argues: a node with forty aggregates
+ * typed in one sitting is usually wrong about several in the same way.
+ */
+export declare function aggregateRefusals(node: {
+    groupBy?: unknown;
+    aggregates?: unknown;
+    maxGroups?: unknown;
+}): string[];
+/**
+ * How many fields one lookup may bring across.
+ *
+ * The same argument {@link WORKFLOW_RENAME_MAX_COLUMNS} makes, plus one that is
+ * specific to this node: every named field is held *per distinct key* for the
+ * whole run, so this number multiplies {@link WORKFLOW_LOOKUP_MAX_REFERENCE_ROWS}
+ * into the actual memory bill. Past a few dozen the thing being expressed is
+ * "give me that whole table beside this one", which is a second source and a
+ * union, not an enrichment.
+ */
+export declare const WORKFLOW_LOOKUP_MAX_FIELDS = 64;
+/**
+ * How many rows the reference side may have before the node refuses to run.
+ *
+ * ## Why there is a number here at all
+ *
+ * Because exactly one side of a join can stream, and this node holds the other
+ * one. That is the whole property it exists to have (see
+ * {@link WorkflowLookupNode}) and it is also the whole hazard: a graph whose
+ * reference edge is accidentally wired to the 7.6-million-row side does not fail,
+ * it allocates until the pod is killed — and a pod killed by the kernel produces
+ * no run log, no failed node and no message, which is the silence this file is
+ * arranged against.
+ *
+ * ## Why it is measured in rows, and why it is free
+ *
+ * A staged input announces its `rowCount` before a single row is read back (see
+ * `WorkflowStageRef`), so the refusal happens *before* anything is held. A bound
+ * in bytes would have to be discovered by holding rows until they weighed too
+ * much, which is a bound that has already done the damage by the time it fires.
+ *
+ * Two hundred thousand, and the arithmetic rather than a round number that feels
+ * safe: what is retained per key is the key string plus the values of the named
+ * fields — not the reference row — so a reference of this size with a handful of
+ * short fields is tens of megabytes, and one at {@link WORKFLOW_LOOKUP_MAX_FIELDS}
+ * is the point where it stops being obviously fine. The real reference tables
+ * this was built against are three orders of magnitude below it: a work-plan code
+ * table is hundreds of rows and a unit dictionary is dozens.
+ *
+ * The refusal names the count, the bound and the fix, because the fix is
+ * genuinely available in the graph: put a filter on the reference side, or swap
+ * the two edges if the smaller side is the one being streamed.
+ */
+export declare const WORKFLOW_LOOKUP_MAX_REFERENCE_ROWS = 200000;
+/**
+ * Enriches each row with fields from a reference dataset, matched by key.
+ *
+ * ## The property that justifies a kind rather than a transform
+ *
+ * **One side is held; the other streams.** The reference is read once into a map
+ * keyed by its key column, and then the driving rows go past it one batch at a
+ * time and never accumulate. That asymmetry is the entire content of the node,
+ * and it is not something a {@link WorkflowTransformNode} can express: a
+ * transform is a function over what it is given, so a join written as one has to
+ * be handed *both* sides at once — which is why a transform makes
+ * `ConnectorRunnerService` log *"Held all N records in memory"*, and why the
+ * whole-batch version of this exact join reached 78% of a hard 32 MiB output
+ * bound on 44,720 rows before it did anything interesting.
+ *
+ * A per-record transform ({@link CatalogTransform} in `record` mode) cannot do it
+ * either, and the reason is sharper: a function over one record has nowhere to
+ * put the map. It would rebuild it per record, or reach a database per record,
+ * and 44,720 round trips is not a shape anybody would choose on purpose.
+ *
+ * ## What it stays narrow about, deliberately
+ *
+ * The generic transform still exists, and that is what lets this node refuse
+ * every next field forever — the argument {@link WorkflowRenameNode} makes at
+ * length. No join *type* beyond {@link unmatched}, no composite keys, no
+ * expressions on either side, no aggregation of the matched rows. The answer to
+ * "I need more than this" is always *use a transform*, and never *add a field
+ * here*. Two of those refusals have a second reason on top:
+ *
+ * - **No composite key.** Two columns concatenated is a rule about a separator,
+ *   and a separator that occurs inside a value silently merges two different
+ *   keys into one. A {@link WorkflowRenameNode} cannot build one either, which is
+ *   the honest statement: build the key in a transform, then join on it, and the
+ *   separator is a decision somebody wrote down.
+ * - **No expressions on either side.** `UPPER(key)` looks harmless and is the
+ *   single most dangerous thing that could be added, because normalising a key
+ *   is a rule about which of two values are "the same value" — see the note on
+ *   {@link key} about what happens when only one side is normalised.
+ *
+ * ## Where the reference comes from: an edge, named
+ *
+ * The reference is **another node in this graph**, wired into this one, and
+ * {@link reference} says which of the inbound edges it is. Not a connector on
+ * this node, and not the first inbound edge.
+ *
+ * *Not a connector on this node*, because a source is already a modelled thing
+ * with a kind, an optional named connection, a secret, a mode, a config, schema
+ * discovery and a staging path — and putting a second, smaller copy of all that
+ * inside this node would fork it. Wiring a source in instead means the reference
+ * composes with everything: a `catalog` source reads the **current** snapshot of
+ * a published type, resolved when the run reaches it (see `sourceKind: 'catalog'`),
+ * which is the natural reference and names no physical table; a `sql` source
+ * reads a code table straight out of an operational database; and a filter or a
+ * rename may sit in between, which is how a reference with duplicate keys is
+ * made unambiguous (see below) without this node growing a rule for it.
+ *
+ * *Not the first inbound edge*, and this one is the sharp decision. Edge order
+ * is defined and does decide what a multi-input node receives — but for every
+ * other kind, reordering two wires changes only the order rows are concatenated
+ * in, which is at worst cosmetic. Here it would decide **which side is held
+ * entirely in memory**, and swapping them silently turns a working graph into
+ * one that either holds 44,720 rows to enrich 200, or joins the two datasets the
+ * wrong way round and reports success. Reordering edges is invisible on a canvas.
+ * So the node names its reference by node id, and `validateWorkflow` refuses a
+ * name that is not one of its inbound edges.
+ *
+ * ## The counts, which are the point
+ *
+ * The failure this node was built against is not a crash. flip's SUBWO reader
+ * builds exactly these two maps and reads them per row, and when the reference
+ * table is empty it produces **unenriched rows and a green run** — 44,720 rows
+ * with `planName`, `planDescription` and `unitMel` hard null, no error, no
+ * warning, and a documented seeding prerequisite that nothing checks. A zero-match
+ * join is indistinguishable from a working one by looking at the run.
+ *
+ * So the run log always carries three numbers, whatever {@link unmatched} says:
+ * how many rows matched, how many had a key that matched nothing, and how many
+ * had no key at all. The third is separate from the second on purpose — they
+ * have different causes and different fixes, and flip's reader folds both into
+ * the same NULL. A run where nothing matched gets a line of its own, the way
+ * `filterLogLines` calls out a filter that kept nothing.
+ *
+ * ## The decisions, each made rather than discovered
+ *
+ * - **A key that matches nothing** — {@link unmatched}, defaulting to `null`.
+ * - **Two reference rows for one key** — refused, *when they disagree*. See
+ *   {@link fields}.
+ * - **An enriched name the driving row already carries** — filled when it is
+ *   empty, refused when it holds a value. The rule is about destroying data
+ *   rather than about a name being taken, and the distinction is the whole
+ *   usefulness of the node: a published type *declares* the columns it holds, so
+ *   a graph reading one back to enrich it receives every one of them. The real
+ *   measurement is the argument — `SubwoReplica` hands over 44,720 rows all
+ *   carrying `planName`, `planDescription` and `unitMel` with `null` in them,
+ *   which is exactly the three columns this node was built to fill. A target
+ *   holding an actual value is two columns and one name and fails the node,
+ *   naming the row, which is the sentence {@link renameColumnRefusals} says about
+ *   the same problem arriving from the other direction.
+ * - **A reference row with no key** — not indexed, and counted. A real work-plan
+ *   table has them: flip writes `planId: row.planId ?? ""` when a load has no
+ *   plan code, so the empty-string key is in the table by construction and can
+ *   never be matched by anything. Refusing the whole reference over one of those
+ *   would make the node unusable against the data it was built for; indexing it
+ *   silently would let one keyless row become the answer for every keyless
+ *   driving row.
+ */
+export interface WorkflowLookupNode extends WorkflowNodeBase {
+    kind: 'lookup';
+    /**
+     * The id of the inbound node whose rows are the reference side.
+     *
+     * Held in memory for the whole node; everything else wired in streams past it.
+     * `validateWorkflow` refuses an id that is not one of this node's inbound
+     * edges, and refuses a lookup whose *only* inbound edge is this one — a lookup
+     * with nothing to enrich produces nothing, and would commit an empty snapshot.
+     */
+    reference: string;
+    /**
+     * The column on the **driving** row holding the key.
+     *
+     * Deliberately unconstrained in spelling, the way a rename's *source* names
+     * are: `Reg Number` and `Mgmt Cd` are what real drops are keyed by.
+     *
+     * ## How two keys are compared, stated once because it is the whole join
+     *
+     * A key is read off the row, and a value of `null` or `undefined` — or a
+     * column that is absent from that row — means the row **has no key**. It is
+     * counted separately and never matches, including against a reference row
+     * that also has no key.
+     *
+     * Anything else is compared **as a string**, by `String(value)`, with no
+     * trimming, no case folding and no other normalisation.
+     *
+     * The coercion is a decision and so is its limit. It is there because the two
+     * sides routinely come from different engines: a work-plan code arriving as a
+     * MySQL `VARCHAR` and the same code arriving as a number out of a spreadsheet
+     * parser are the same key to everyone except `===`, and a join that matched
+     * nothing for that reason is the exact silent-zero this node reports counts to
+     * prevent. What is *not* done is normalising the shape of the value, because
+     * every one of those is a rule about which of somebody's values are the same
+     * value: `"21 CES"` and `"21CES"` are not the same unit unless a person says
+     * so, and flip's own reader is the cautionary tale — it normalises the driving
+     * unit and compares it against a reference column normalised at write time by
+     * a different screen, so the two agree only for as long as nobody edits either.
+     * If a key needs normalising, normalise it in a transform, on both sides,
+     * where it is visible.
+     */
+    key: string;
+    /** The column on the **reference** row holding the key. Compared as {@link key} describes. */
+    referenceKey: string;
+    /**
+     * Reference column → the name it lands under on the driving row. Never empty;
+     * at most {@link WORKFLOW_LOOKUP_MAX_FIELDS} entries; every target matches
+     * {@link WORKFLOW_FILTER_COLUMN_PATTERN} and no two share one.
+     *
+     * A `Record` rather than a list of pairs, for the reason
+     * {@link WorkflowRenameNode.columns} is one: a key cannot appear twice, so
+     * bringing one reference column across twice is unrepresentable. Two brought
+     * across *onto* one name is representable and refused.
+     *
+     * The target pattern is the identifier rule and it is here for the reason the
+     * rename's targets are: a load looks every field up as `row[name]`, so a name
+     * no property can carry loads NULL into every row and reports success. That is
+     * the failure this node is supposed to be *fixing*, so producing it would be
+     * the trap re-armed one node further along.
+     *
+     * ## Empty is refused rather than treated as a no-op
+     *
+     * A lookup that brings nothing across is a node that draws as configured,
+     * costs a full pass over both sides, and changes nothing — except under `drop`,
+     * where it silently becomes a semi-join that deletes every row whose key is not
+     * in the reference. Two silent opposites reached by deleting the last row of a
+     * form, which is exactly the argument {@link renameColumnRefusals} makes.
+     *
+     * ## Why the fields decide the duplicate-key rule
+     *
+     * Two reference rows for one key means either the join multiplies rows or
+     * something picks a winner, and picking a winner is a rule about whose data
+     * survives — the reasoning {@link renameColumnRefusals} already refused, for
+     * two columns renamed onto one name. Multiplying rows is worse here than it
+     * looks: this node's contract is that it enriches, so a sink downstream would
+     * commit more rows than were read with nothing on the canvas saying why.
+     *
+     * So: **two reference rows for one key are refused when they disagree about
+     * any named field, and collapsed when they agree.** Agreeing costs nobody
+     * anything — there is no winner, the answer is the same either way — and it is
+     * what a real reference table looks like when it has one row per key *and*
+     * something else, which is the common case. Disagreeing fails the node, naming
+     * the key, the field and both values.
+     *
+     * Note what makes this rule cheap and total: it is decided over the **named
+     * fields only**, so two reference rows that differ in a column this node does
+     * not bring across are not a conflict, because nothing about them reaches the
+     * output. And it is decided while the map is built, before a single driving
+     * row is read, so it fails at the start of the node rather than at row ninety
+     * thousand.
+     *
+     * flip's own reader is the argument for refusing rather than choosing. It
+     * builds the plan map with `plansMap.set(plan.planId, plan)`, which keeps the
+     * **last** row; it resolves the unit dictionary with `Array.prototype.find`,
+     * which keeps the **first**; neither key column has a unique constraint; and
+     * the two rules live forty lines apart in one file. Nobody chose either of
+     * them.
+     */
+    fields: Record<string, string>;
+    /**
+     * What happens to a driving row whose key matches nothing. Absent means `null`.
+     *
+     * `null` is the default because it is the only one of the three that changes
+     * neither which rows exist nor whether the run finishes, so it is the one that
+     * can be the answer for a graph nobody has thought about yet. It is also what
+     * flip does today — and the difference this node makes is not the disposition,
+     * it is that the count is reported rather than left to be discovered by
+     * querying the committed snapshot.
+     *
+     * `drop` is an INNER JOIN and it removes rows, so it is never a default: a
+     * lookup wired in front of a full-mode sink under `drop` shrinks a published
+     * type, which is the accident {@link WorkflowFilterNode.narrows} exists about.
+     *
+     * `fail` is for a reference that is a **documented prerequisite**, which is not
+     * a hypothetical: flip's docs make seeding the unit dictionary a prerequisite
+     * of MEL, MVR and SUBWO, and an unseeded one yields unnormalized rows rather
+     * than an error. A load that cannot be enriched is a load that should not
+     * commit, and this is how somebody says so.
+     */
+    unmatched?: WorkflowLookupUnmatched;
+}
+/** {@link WorkflowLookupNode.unmatched}, resolved. One reader of the default. */
+export declare function workflowLookupUnmatched(node: WorkflowLookupNode): WorkflowLookupUnmatched;
+/**
+ * A key, as this node compares them, or `undefined` for a row that has none.
+ *
+ * One function, exported, and called by the runner for both sides — because the
+ * one way a join goes silently wrong is the two sides being read by two pieces
+ * of code that agree today. See {@link WorkflowLookupNode.key} for what it does
+ * and, more to the point, what it deliberately does not do.
+ */
+export declare function workflowLookupKey(value: unknown): string | undefined;
+/**
+ * Every reason a lookup's configuration cannot be stored, as sentences, or empty.
+ *
+ * One function, called by {@link validateWorkflow}, by the HTTP boundary and by
+ * the canvas, for the reason {@link renameColumnRefusals} is: a screen that
+ * checked a target name against its own copy of the pattern is a screen that
+ * eventually accepts something the server refuses, halfway through a save.
+ *
+ * All of them rather than the first, for the reason `refuseUnpublishablePropertyNames`
+ * gives: a form filled in one sitting is usually wrong about several things in
+ * the same way.
+ *
+ * The *wiring* rules — that {@link WorkflowLookupNode.reference} names an inbound
+ * edge, and that something other than the reference is wired in — are not here,
+ * and that is not an omission. They are facts about the graph rather than about
+ * the node, so they cannot be answered from the node alone; `validateWorkflow`
+ * owns them and the inspector reads them from `validateWorkflow`.
+ */
+export declare function lookupConfigRefusals(node: {
+    key?: unknown;
+    referenceKey?: unknown;
+    fields?: Record<string, string>;
+}): string[];
+/**
+ * Whether a stored aggregate list is one this build can run.
+ *
+ * Refused rather than repaired, the stance {@link isWorkflowRenameColumns} takes
+ * and for the same reason one step further along: an aggregate list read back
+ * with one entry silently dropped is a graph that commits a column of nulls
+ * under a name somebody put in an object type on purpose.
+ */
+export declare function isWorkflowAggregates(value: unknown): value is WorkflowAggregate[];
+/**
+ * Whether a stored field map is one this build can run.
+ *
+ * Refused rather than repaired, the stance {@link isWorkflowRenameColumns} takes
+ * and for the same reason: a map read back with one entry silently dropped is a
+ * graph that commits a column of NULLs under a name nobody can now explain.
+ *
+ * `Object.entries` rather than a `for…in`, so an inherited key cannot enter the
+ * map. The key columns are not checked here because they are not this value;
+ * {@link lookupConfigRefusals} is what sees the whole node.
+ */
+export declare function isWorkflowLookupFields(value: unknown): value is Record<string, string>;
 /**
  * A discriminated union, so narrowing a node is `node.kind === "sink"` and
  * never a type assertion. This is why the kind list is not simply a string on
  * one node shape with every field optional: that shape lets a source node carry
  * a `transformId` and nothing catches it.
  */
-export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode | WorkflowRenameNode;
+export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode | WorkflowRenameNode | WorkflowAggregateNode | WorkflowLookupNode;
 /**
  * The node kinds that can be saved once and used in several graphs.
  *
@@ -2403,6 +3133,18 @@ export declare function isReusableNodeKind(value: unknown): value is ReusableNod
  *   *about* one drop of one file. `Mgmt Cd → mgmtCd` saved under a name and
  *   dropped into a graph reading a different system renames nothing at all, and
  *   the symptom is a column of NULLs rather than a failure.
+ * - `aggregate` — the same again, and it fails in both directions at once. Its
+ *   group-by columns and its inputs name one type's columns, so a shared one
+ *   groups a graph it was not written for on a column that is not there — which
+ *   collapses every record into one null-keyed group rather than erroring. And
+ *   its *output* column set is the thing downstream nodes are validated against,
+ *   so a shared node editable from elsewhere would silently change what another
+ *   graph's sink is allowed to write.
+ * - `lookup` — that argument, and one that is not an argument at all but an
+ *   impossibility: {@link WorkflowLookupNode.reference} is **a node id in this
+ *   graph**. A shared body carrying one would name a node the adopting graph has
+ *   never had, and it is not a cosmetic field — it is the one that decides which
+ *   side of the join is held in memory.
  */
 export declare const NODE_KIND_IS_REUSABLE: {
     readonly source: true;
@@ -2412,6 +3154,8 @@ export declare const NODE_KIND_IS_REUSABLE: {
     readonly if: false;
     readonly filter: false;
     readonly rename: false;
+    readonly aggregate: false;
+    readonly lookup: false;
 };
 /** Whether this kind can be saved as a reusable node. Reads {@link NODE_KIND_IS_REUSABLE}. */
 export declare function nodeKindIsReusable(kind: WorkflowNodeKind): boolean;
@@ -3210,7 +3954,7 @@ export interface CallableWorkflowBlock {
 }
 export declare function callableWorkflowBlock(ref: CallableWorkflowRef): CallableWorkflowBlock | undefined;
 /** Every way a graph can be refused. Exported so a canvas can key off the code. */
-export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "source-type-not-named", "call-not-named", "call-plain-has-output", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing", "rename-invalid", "column-not-produced", "version-pin-invalid"];
+export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "source-type-not-named", "call-not-named", "call-plain-has-output", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing", "rename-invalid", "aggregate-invalid", "lookup-invalid", "lookup-reference-not-wired", "lookup-nothing-to-enrich", "column-not-produced", "version-pin-invalid"];
 export type WorkflowIssueCode = (typeof WORKFLOW_ISSUE_CODES)[number];
 export interface WorkflowValidationIssue {
     code: WorkflowIssueCode;
@@ -3443,7 +4187,39 @@ export declare function workflowFilterColumns(predicate: WorkflowFilterPredicate
  * cyclic graph before it gets here, but the canvas calls this while a graph is
  * being drawn and is entitled to a wrong-but-terminating answer.
  */
-export declare function workflowKnownColumns(graph: WorkflowGraph, nodeId: string, knowledge?: WorkflowColumnKnowledge): ReadonlySet<string> | undefined;
+export declare function workflowKnownColumns(graph: WorkflowGraph, nodeId: string, knowledge?: WorkflowColumnKnowledge, onlyFrom?: WorkflowInputFilter): ReadonlySet<string> | undefined;
+/**
+ * Which of a node's inbound edges a question is about.
+ *
+ * One node kind needs this and it is {@link WorkflowLookupNode}, which is the
+ * only one whose inputs are not interchangeable: the reference side is held as a
+ * map and its columns do **not** flow on, so the union of everything wired in is
+ * the wrong answer to "what does this node pass down". Without the distinction
+ * the walk would report the reference's columns as available downstream, and a
+ * filter naming one of them would be accepted by the validator and then match no
+ * row at run time — the precise silent failure `checkColumnsProduced` exists to
+ * catch, produced by the check itself.
+ *
+ * Optional everywhere it appears, and omitting it means every inbound edge —
+ * which is what every other kind wants and what every existing caller gets.
+ */
+export type WorkflowInputFilter = (fromNodeId: string) => boolean;
+/**
+ * The two column sets a lookup sees, told apart.
+ *
+ * Exported because three callers need the same split and each one getting it
+ * right separately is how they come to disagree: the validator refuses a key
+ * column that is not on the driving side, the walk answers what the node passes
+ * on, and the inspector says both out loud on the screen where the columns are
+ * typed.
+ *
+ * Either side answers `undefined` for the ordinary reason — see
+ * {@link workflowKnownColumns} — and `undefined` must not be read as empty.
+ */
+export declare function workflowLookupColumns(graph: WorkflowGraph, node: WorkflowLookupNode, knowledge?: WorkflowColumnKnowledge): {
+    driving: ReadonlySet<string> | undefined;
+    reference: ReadonlySet<string> | undefined;
+};
 /**
  * What a caller can tell the column walk that the graph does not hold.
  *