@dudousxd/nestjs-catalog 0.27.0 → 0.28.0

package/dist/catalog.pipeline.js

@@ -9,7 +9,7 @@
  * systems each believing they decide when a load runs.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.REDACTED_SECRET = exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_BRANCH_LABELS = exports.NODE_KIND_IS_REUSABLE = exports.REUSABLE_NODE_KINDS = exports.WORKFLOW_AGGREGATE_DEFAULT_SEPARATOR = exports.WORKFLOW_AGGREGATE_MAX_SEPARATOR = exports.WORKFLOW_AGGREGATE_JOIN_LENGTH_CEILING = exports.WORKFLOW_AGGREGATE_JOIN_MAX_LENGTH = exports.WORKFLOW_AGGREGATE_GROUPS_CEILING = exports.WORKFLOW_AGGREGATE_MAX_GROUPS = exports.WORKFLOW_AGGREGATE_MAX_AGGREGATES = exports.WORKFLOW_AGGREGATE_MAX_GROUP_BY = exports.WORKFLOW_AGGREGATE_FUNCTIONS = exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_RENAME_UNNAMED = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_SKIP_REASONS = exports.CODE_CONTEXT_CONTRACT = exports.TRANSFORM_RUNNER = exports.TRANSFORM_MODES = exports.TRANSFORM_LANGUAGES = exports.SOURCE_FORMATS = exports.CATALOG_SOURCE_TYPE_KEY = exports.CONNECTOR_KINDS = void 0;
+exports.REDACTED_SECRET = exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_BRANCH_LABELS = exports.NODE_KIND_IS_REUSABLE = exports.REUSABLE_NODE_KINDS = exports.WORKFLOW_LOOKUP_MAX_REFERENCE_ROWS = exports.WORKFLOW_LOOKUP_MAX_FIELDS = exports.WORKFLOW_AGGREGATE_DEFAULT_SEPARATOR = exports.WORKFLOW_AGGREGATE_MAX_SEPARATOR = exports.WORKFLOW_AGGREGATE_JOIN_LENGTH_CEILING = exports.WORKFLOW_AGGREGATE_JOIN_MAX_LENGTH = exports.WORKFLOW_AGGREGATE_GROUPS_CEILING = exports.WORKFLOW_AGGREGATE_MAX_GROUPS = exports.WORKFLOW_AGGREGATE_MAX_AGGREGATES = exports.WORKFLOW_AGGREGATE_MAX_GROUP_BY = exports.WORKFLOW_LOOKUP_UNMATCHED = exports.WORKFLOW_AGGREGATE_FUNCTIONS = exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_RENAME_UNNAMED = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_SKIP_REASONS = exports.CODE_CONTEXT_CONTRACT = exports.TRANSFORM_RUNNER = exports.TRANSFORM_MODES = exports.TRANSFORM_LANGUAGES = exports.SOURCE_FORMATS = exports.CATALOG_SOURCE_TYPE_KEY = exports.CONNECTOR_KINDS = void 0;
 exports.isConnectorKind = isConnectorKind;
 exports.unreachableConnectorKind = unreachableConnectorKind;
 exports.workflowSourceObjectType = workflowSourceObjectType;
@@ -46,6 +46,8 @@ exports.renameColumnRefusals = renameColumnRefusals;
 exports.isWorkflowRenameColumns = isWorkflowRenameColumns;
 exports.isWorkflowAggregateFunction = isWorkflowAggregateFunction;
 exports.unreachableAggregateFunction = unreachableAggregateFunction;
+exports.isWorkflowLookupUnmatched = isWorkflowLookupUnmatched;
+exports.unreachableLookupUnmatched = unreachableLookupUnmatched;
 exports.workflowAggregateMaxGroups = workflowAggregateMaxGroups;
 exports.workflowAggregateSeparator = workflowAggregateSeparator;
 exports.workflowAggregateJoinMaxLength = workflowAggregateJoinMaxLength;
@@ -53,7 +55,11 @@ exports.workflowAggregateNeedsColumn = workflowAggregateNeedsColumn;
 exports.workflowAggregateColumns = workflowAggregateColumns;
 exports.workflowAggregateOutputColumns = workflowAggregateOutputColumns;
 exports.aggregateRefusals = aggregateRefusals;
+exports.workflowLookupUnmatched = workflowLookupUnmatched;
+exports.workflowLookupKey = workflowLookupKey;
+exports.lookupConfigRefusals = lookupConfigRefusals;
 exports.isWorkflowAggregates = isWorkflowAggregates;
+exports.isWorkflowLookupFields = isWorkflowLookupFields;
 exports.isReusableNodeKind = isReusableNodeKind;
 exports.nodeKindIsReusable = nodeKindIsReusable;
 exports.unreachableReusableNodeKind = unreachableReusableNodeKind;
@@ -74,6 +80,7 @@ exports.workflowNodeRuns = workflowNodeRuns;
 exports.workflowGraphHash = workflowGraphHash;
 exports.workflowFilterColumns = workflowFilterColumns;
 exports.workflowKnownColumns = workflowKnownColumns;
+exports.workflowLookupColumns = workflowLookupColumns;
 exports.isWorkflowNode = isWorkflowNode;
 exports.isWorkflowEdge = isWorkflowEdge;
 exports.supportsWorkflows = supportsWorkflows;
@@ -511,12 +518,25 @@ function isWorkflowSkipReason(value) {
  *   {@link WorkflowIfNode} is the conditional one, and it earns its kind by
  *   doing something no wiring can express — deciding that one of those
  *   successors, and everything only it feeds, does not run at all.
- * - **merge / join** — a node with several inbound edges receives its inputs
- *   concatenated in edge order (see {@link WorkflowEdge}). A keyed join is then
- *   ordinary code inside the transform, which can already see every record.
- *   A `merge` kind would have had to carry a strategy field whose values the
- *   runner would have to implement one by one, and an unimplemented strategy in
- *   a dropdown is the failure this list exists to avoid.
+ * - **merge (unkeyed)** — a node with several inbound edges already receives its
+ *   inputs concatenated in edge order (see {@link WorkflowEdge}), so a `merge`
+ *   kind would be a box that draws what the wires already say. It would also
+ *   have had to carry a strategy field whose values the runner implements one by
+ *   one, and an unimplemented strategy in a dropdown is the failure this list
+ *   exists to avoid.
+ * - **join (keyed)** — *this half used to be refused with the entry above, and
+ *   the reversal is left visible rather than edited out*, the way the `filter`
+ *   entry leaves its own. The old argument was that a keyed join is ordinary
+ *   code inside a transform, which can already see every record. Every word of
+ *   that is true and it is exactly the problem: "can already see every record"
+ *   is the same sentence as "holds the whole load", and it is why a transform
+ *   makes `ConnectorRunnerService` log *"Held all N records in memory"*. A join
+ *   does not need both sides held. It needs **one** side held — as a map, keyed
+ *   — while the other streams past it, and that asymmetry is a property of the
+ *   operation that a function over a batch cannot express and a runner therefore
+ *   cannot exploit. {@link WorkflowLookupNode} is the keyed half, built narrow:
+ *   one key, named enrichment fields, and a reference side that is bounded and
+ *   refused loudly rather than held quietly.
  * - **call a durable *step*** — the sibling of {@link WorkflowCallNode} that
  *   somebody will eventually come looking for, and it cannot be built. A
  *   durable step has no global identity: it is dispatched by a routing name
@@ -544,6 +564,8 @@ exports.WORKFLOW_NODE_KINDS = [
     'rename',
     /** Groups records and summarises each group. See {@link WorkflowAggregateNode}. */
     'aggregate',
+    /** Enriches each row from a reference dataset, by key. See {@link WorkflowLookupNode}. */
+    'lookup',
 ];
 /** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
 function isWorkflowNodeKind(value) {
@@ -1402,6 +1424,38 @@ function isWorkflowAggregateFunction(value) {
 function unreachableAggregateFunction(fn, where) {
     throw new Error(`${where} has no rule for the aggregate function ${JSON.stringify(fn)}. It was added to WORKFLOW_AGGREGATE_FUNCTIONS without teaching this code how to compute it, and guessing would commit a number nobody derived.`);
 }
+/* --- lookup -------------------------------------------------------------- */
+/**
+ * What happens to a driving row whose key matches no reference row.
+ *
+ * Three words rather than a boolean, because the three are the three joins SQL
+ * has names for and each is a different node. See
+ * {@link WorkflowLookupNode.unmatched} for which one to reach for.
+ */
+exports.WORKFLOW_LOOKUP_UNMATCHED = [
+    /** The enriched columns are set to `null` on that row. A LEFT JOIN. The default. */
+    'null',
+    /** The row does not reach the next node at all. An INNER JOIN. */
+    'drop',
+    /** The node fails, naming the key. For a reference that is a prerequisite. */
+    'fail',
+];
+/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
+function isWorkflowLookupUnmatched(value) {
+    return exports.WORKFLOW_LOOKUP_UNMATCHED.some((each) => each === value);
+}
+/**
+ * The exhaustiveness guard for {@link WORKFLOW_LOOKUP_UNMATCHED}.
+ *
+ * {@link unreachableNodeKind}, one level down, and for the identical reason: the
+ * three words decide whether a row keeps its data, disappears from the load, or
+ * stops the run, and a fourth added without a branch would silently pick
+ * whichever the last `if` was — which here means silently changing which rows
+ * reach a published type.
+ */
+function unreachableLookupUnmatched(value, where) {
+    throw new Error(`${where} does not handle the lookup disposition ${JSON.stringify(value)}. The list and every decision made per entry are meant to move together.`);
+}
 /**
  * How many columns one node may group on.
  *
@@ -1570,6 +1624,117 @@ function aggregateRefusals(node) {
     ];
     return refusals;
 }
+/**
+ * How many fields one lookup may bring across.
+ *
+ * The same argument {@link WORKFLOW_RENAME_MAX_COLUMNS} makes, plus one that is
+ * specific to this node: every named field is held *per distinct key* for the
+ * whole run, so this number multiplies {@link WORKFLOW_LOOKUP_MAX_REFERENCE_ROWS}
+ * into the actual memory bill. Past a few dozen the thing being expressed is
+ * "give me that whole table beside this one", which is a second source and a
+ * union, not an enrichment.
+ */
+exports.WORKFLOW_LOOKUP_MAX_FIELDS = 64;
+/**
+ * How many rows the reference side may have before the node refuses to run.
+ *
+ * ## Why there is a number here at all
+ *
+ * Because exactly one side of a join can stream, and this node holds the other
+ * one. That is the whole property it exists to have (see
+ * {@link WorkflowLookupNode}) and it is also the whole hazard: a graph whose
+ * reference edge is accidentally wired to the 7.6-million-row side does not fail,
+ * it allocates until the pod is killed — and a pod killed by the kernel produces
+ * no run log, no failed node and no message, which is the silence this file is
+ * arranged against.
+ *
+ * ## Why it is measured in rows, and why it is free
+ *
+ * A staged input announces its `rowCount` before a single row is read back (see
+ * `WorkflowStageRef`), so the refusal happens *before* anything is held. A bound
+ * in bytes would have to be discovered by holding rows until they weighed too
+ * much, which is a bound that has already done the damage by the time it fires.
+ *
+ * Two hundred thousand, and the arithmetic rather than a round number that feels
+ * safe: what is retained per key is the key string plus the values of the named
+ * fields — not the reference row — so a reference of this size with a handful of
+ * short fields is tens of megabytes, and one at {@link WORKFLOW_LOOKUP_MAX_FIELDS}
+ * is the point where it stops being obviously fine. The real reference tables
+ * this was built against are three orders of magnitude below it: a work-plan code
+ * table is hundreds of rows and a unit dictionary is dozens.
+ *
+ * The refusal names the count, the bound and the fix, because the fix is
+ * genuinely available in the graph: put a filter on the reference side, or swap
+ * the two edges if the smaller side is the one being streamed.
+ */
+exports.WORKFLOW_LOOKUP_MAX_REFERENCE_ROWS = 200_000;
+/** {@link WorkflowLookupNode.unmatched}, resolved. One reader of the default. */
+function workflowLookupUnmatched(node) {
+    return node.unmatched ?? 'null';
+}
+/**
+ * A key, as this node compares them, or `undefined` for a row that has none.
+ *
+ * One function, exported, and called by the runner for both sides — because the
+ * one way a join goes silently wrong is the two sides being read by two pieces
+ * of code that agree today. See {@link WorkflowLookupNode.key} for what it does
+ * and, more to the point, what it deliberately does not do.
+ */
+function workflowLookupKey(value) {
+    if (value === undefined || value === null)
+        return undefined;
+    // Objects and arrays are refused rather than stringified: `String({})` is
+    // `"[object Object]"`, which is a key that every JSON column in a row would
+    // share, so a join on a mis-chosen column would match everything to everything
+    // and report a very large number of matches.
+    if (typeof value === 'object')
+        return undefined;
+    const key = String(value);
+    // The empty string is "no key", not a key. A reference table writes it where a
+    // code was missing (flip's `planId: row.planId ?? ""`), so treating it as a
+    // value would make one keyless reference row the answer for every keyless
+    // driving row.
+    return key.length === 0 ? undefined : key;
+}
+/**
+ * Every reason a lookup's configuration cannot be stored, as sentences, or empty.
+ *
+ * One function, called by {@link validateWorkflow}, by the HTTP boundary and by
+ * the canvas, for the reason {@link renameColumnRefusals} is: a screen that
+ * checked a target name against its own copy of the pattern is a screen that
+ * eventually accepts something the server refuses, halfway through a save.
+ *
+ * All of them rather than the first, for the reason `refuseUnpublishablePropertyNames`
+ * gives: a form filled in one sitting is usually wrong about several things in
+ * the same way.
+ *
+ * The *wiring* rules — that {@link WorkflowLookupNode.reference} names an inbound
+ * edge, and that something other than the reference is wired in — are not here,
+ * and that is not an omission. They are facts about the graph rather than about
+ * the node, so they cannot be answered from the node alone; `validateWorkflow`
+ * owns them and the inspector reads them from `validateWorkflow`.
+ */
+function lookupConfigRefusals(node) {
+    const refusals = [];
+    if (typeof node.key !== 'string' || node.key.length === 0) {
+        refusals.push('It names no key column on the rows being enriched, so there is nothing to match on.');
+    }
+    if (typeof node.referenceKey !== 'string' || node.referenceKey.length === 0) {
+        refusals.push('It names no key column on the reference rows, so there is nothing to match against.');
+    }
+    refusals.push(...lookupFieldRefusals(node.fields ?? {}));
+    // Last, so a blank key has already been reported as a blank key rather than as
+    // a collision with itself, and so this reads after the sentences about the map.
+    if (typeof node.key === 'string' && node.key.length > 0) {
+        const onto = Object.entries(node.fields ?? {})
+            .filter(([, to]) => to === node.key)
+            .map(([from]) => JSON.stringify(from));
+        if (onto.length > 0) {
+            refusals.push(`${onto.join(' and ')} would land as ${JSON.stringify(node.key)}, which is the column this node matches on. The key would be overwritten by the reference's copy of it on every row that matched and left alone on every row that did not, so afterwards the column would no longer say which rows were enriched.`);
+        }
+    }
+    return refusals;
+}
 /**
  * What the grouping half can be wrong about, and the names it accepted.
  *
@@ -1654,6 +1819,44 @@ function oneAggregateRefusals(entry, groupedOn, names) {
     refusals.push(...joinFieldRefusals(entry, fn, label));
     return refusals;
 }
+/**
+ * The half of {@link lookupConfigRefusals} that a field map answers on its own.
+ *
+ * Split out because {@link isWorkflowLookupFields} is handed a map and nothing
+ * else, and calling the whole function with invented key columns would either
+ * report two refusals about keys it was not asked about, or — with plausible
+ * stand-ins — report a collision between a real target and a made-up key. A
+ * guard that fires on the value it was given plus two values it made up is a
+ * guard that rejects a graph for a reason that is not in the graph.
+ */
+function lookupFieldRefusals(fields) {
+    const refusals = [];
+    const entries = Object.entries(fields);
+    if (entries.length === 0) {
+        refusals.push('It brings no fields across. An empty list is refused rather than stored: it is a node that draws as configured, costs a pass over both sides and changes nothing — and with unmatched rows dropped it silently becomes a filter that deletes every row whose key is not in the reference.');
+    }
+    if (entries.length > exports.WORKFLOW_LOOKUP_MAX_FIELDS) {
+        refusals.push(`It brings ${entries.length} fields across, and at most ${exports.WORKFLOW_LOOKUP_MAX_FIELDS} may come from one lookup. Every one of them is held per distinct key for the whole run, and past this the thing being expressed is a second dataset beside this one rather than an enrichment.`);
+    }
+    const targets = new Map();
+    for (const [from, to] of entries) {
+        if (from.length === 0) {
+            refusals.push('One field names no column on the reference rows, so there is nothing for it to bring across.');
+            continue;
+        }
+        if (typeof to !== 'string' || !exports.WORKFLOW_FILTER_COLUMN_PATTERN.test(to)) {
+            refusals.push(`${JSON.stringify(from)} would land as ${JSON.stringify(to)}, which is not a name a column can have: letters, digits and underscore, starting with a letter or an underscore. A load looks every field up as \`row[name]\`, so a column this service cannot name downstream is one that loads NULL into every row and reports success — which is the failure this node exists to end, not to relocate.`);
+            continue;
+        }
+        targets.set(to, [...(targets.get(to) ?? []), from]);
+    }
+    for (const [to, sources] of targets) {
+        if (sources.length < 2)
+            continue;
+        refusals.push(`${sources.map((from) => JSON.stringify(from)).join(' and ')} would both land as ${JSON.stringify(to)}. Two columns cannot share one name, and picking a winner would be a rule about which of somebody's data survives.`);
+    }
+    return refusals;
+}
 /**
  * The two fields only `join` reads.
  *
@@ -1715,6 +1918,31 @@ function isWorkflowAggregates(value) {
         return false;
     return aggregateRefusals({ groupBy: ['x'], aggregates: value }).length === 0;
 }
+/**
+ * Whether a stored field map is one this build can run.
+ *
+ * Refused rather than repaired, the stance {@link isWorkflowRenameColumns} takes
+ * and for the same reason: a map read back with one entry silently dropped is a
+ * graph that commits a column of NULLs under a name nobody can now explain.
+ *
+ * `Object.entries` rather than a `for…in`, so an inherited key cannot enter the
+ * map. The key columns are not checked here because they are not this value;
+ * {@link lookupConfigRefusals} is what sees the whole node.
+ */
+function isWorkflowLookupFields(value) {
+    if (typeof value !== 'object' || value === null || Array.isArray(value))
+        return false;
+    const entries = Object.entries(value);
+    const fields = {};
+    for (const [from, to] of entries) {
+        if (typeof to !== 'string')
+            return false;
+        fields[from] = to;
+    }
+    // Only the field-shaped refusals can be answered from this value alone. See
+    // {@link lookupFieldRefusals} for why the whole function is not called here.
+    return lookupFieldRefusals(fields).length === 0;
+}
 /* --- reusable nodes ------------------------------------------------------ */
 /**
  * The node kinds that can be saved once and used in several graphs.
@@ -1780,6 +2008,11 @@ function isReusableNodeKind(value) {
  *   its *output* column set is the thing downstream nodes are validated against,
  *   so a shared node editable from elsewhere would silently change what another
  *   graph's sink is allowed to write.
+ * - `lookup` — that argument, and one that is not an argument at all but an
+ *   impossibility: {@link WorkflowLookupNode.reference} is **a node id in this
+ *   graph**. A shared body carrying one would name a node the adopting graph has
+ *   never had, and it is not a cosmetic field — it is the one that decides which
+ *   side of the join is held in memory.
  */
 exports.NODE_KIND_IS_REUSABLE = {
     source: true,
@@ -1790,6 +2023,7 @@ exports.NODE_KIND_IS_REUSABLE = {
     filter: false,
     rename: false,
     aggregate: false,
+    lookup: false,
 };
 /** Whether this kind can be saved as a reusable node. Reads {@link NODE_KIND_IS_REUSABLE}. */
 function nodeKindIsReusable(kind) {
@@ -2224,6 +2458,29 @@ exports.WORKFLOW_ISSUE_CODES = [
      * on. See {@link aggregateRefusals} for the sentences.
      */
     'aggregate-invalid',
+    /**
+     * A lookup whose configuration cannot be stored: no key on one side or the
+     * other, no fields, too many, or a target that is not a column name, that two
+     * reference columns share, or that would overwrite the key. All from the node
+     * alone — see {@link lookupConfigRefusals} for the sentences.
+     */
+    'lookup-invalid',
+    /**
+     * A lookup whose `reference` does not name one of its inbound edges.
+     *
+     * A code of its own rather than part of `lookup-invalid`, because it is a fact
+     * about the *graph* and points at two boxes: this node and, when it exists, the
+     * node it wrongly names. It is also the one lookup mistake that produces a
+     * green run — with no reference rows to hold, every row comes out enriched with
+     * nulls and the load commits.
+     */
+    'lookup-reference-not-wired',
+    /**
+     * A lookup with nothing wired in but its reference. The reference is held as a
+     * map and never passed on, so the node produces no rows and a full-mode sink
+     * below it commits an empty snapshot over what is published.
+     */
+    'lookup-nothing-to-enrich',
     /**
      * A node naming a column that nothing upstream can produce.
      *
@@ -2300,6 +2557,10 @@ function validateWorkflow(graph, knowledge) {
     const roots = nodes.filter(runsWithoutInput);
     const sinks = nodes.filter((node) => node.kind === 'sink');
     checkNodeWiring(nodes, incoming, outgoing, issues);
+    // After `checkEdges` has returned, so every id named by an edge is a node that
+    // exists and this cannot report a reference whose real problem is a wire
+    // pointing at something deleted.
+    checkLookupWiring(nodes, incoming, byId, issues);
     checkEndpoints(originators, sinks, issues);
     checkBranches(edges, byId, issues);
     checkPlainCallOutputs(nodes, outgoing, byId, issues);
@@ -2695,6 +2956,8 @@ function nodeIsUnconfigured(node) {
         return renameIsUnconfigured(node);
     if (node.kind === 'aggregate')
         return aggregateIsUnconfigured(node);
+    if (node.kind === 'lookup')
+        return lookupIsUnconfigured(node);
     return undefined;
 }
 /**
@@ -2718,6 +2981,89 @@ function aggregateIsUnconfigured(node) {
         message: `Aggregate "${node.name}" (${node.id}) cannot be stored as it is. ${refusals.join(' ')}`,
     };
 }
+/**
+ * A lookup whose configuration this service will not store.
+ *
+ * The refusals come from {@link lookupConfigRefusals} rather than being restated
+ * here, so the canvas, the HTTP boundary and this validator say the same sentence
+ * about the same node. Every one of them is a *silent* failure if let through: no
+ * key column is a join that matches nothing, no fields is a node that either does
+ * nothing or quietly becomes a filter, and a target this service cannot name
+ * downstream loads NULL into every row and reports success.
+ *
+ * The wiring rules are next door in {@link checkLookupWiring}, because they are
+ * facts about the graph rather than about the node — and because a lookup with
+ * the reference edge missing has to point at *two* boxes, which is a thing an
+ * `unconfigured` issue about one node cannot do.
+ */
+function lookupIsUnconfigured(node) {
+    const refusals = lookupConfigRefusals(node);
+    if (refusals.length === 0)
+        return undefined;
+    return {
+        code: 'lookup-invalid',
+        nodeIds: [node.id],
+        message: `Lookup "${node.name}" (${node.id}) cannot be stored as it is. ${refusals.join(' ')}`,
+    };
+}
+/**
+ * A lookup's two edges: the reference it holds, and something to enrich.
+ *
+ * ## Why this is refused here rather than discovered at run time
+ *
+ * Both failures produce a run that finishes. A lookup whose `reference` names a
+ * node that is not wired into it has no reference rows at all, so under the
+ * default disposition every row is enriched with nulls and the load commits —
+ * which is the exact defect this node was built to end, arrived at through the
+ * wiring instead of through the data. A lookup with *only* the reference wired in
+ * has nothing to enrich, so it produces zero rows and a full-mode sink below it
+ * commits an empty snapshot over whatever was live.
+ *
+ * Named by id rather than taken from edge order, and {@link WorkflowLookupNode}
+ * argues why at length: edge order decides which side is held in memory, and
+ * reordering two wires is invisible on a canvas.
+ */
+function checkLookupWiring(nodes, incoming, byId, issues) {
+    for (const node of nodes) {
+        if (node.kind !== 'lookup')
+            continue;
+        const feeds = incoming.get(node.id) ?? [];
+        const named = typeof node.reference === 'string' ? node.reference : '';
+        if (named.length === 0 || !feeds.includes(named)) {
+            issues.push(lookupReferenceNotWired(node, named, feeds, byId));
+            continue;
+        }
+        if (feeds.length < 2) {
+            issues.push({
+                code: 'lookup-nothing-to-enrich',
+                nodeIds: [node.id, named],
+                message: `Lookup "${node.name}" (${node.id}) has only its reference wired into it. The reference is held as a map and is not passed on, so this node would produce no rows at all and a full-mode sink below it would commit an empty snapshot over whatever is published. Wire in the rows you want enriched as well.`,
+            });
+        }
+    }
+}
+/**
+ * The sentence for a reference that is not one of this node's inbound edges.
+ *
+ * Its own function because it has three cases to name — a reference that is
+ * blank, one that names a node in the graph, and one that names nothing at all —
+ * and each has to say something different for the message to be worth reading.
+ * The last two are genuinely different mistakes: the first is a wire that was
+ * never drawn, the second is an id that was typed or that survived a node being
+ * deleted.
+ */
+function lookupReferenceNotWired(node, named, feeds, byId) {
+    const known = byId.get(named);
+    const names = named.length === 0 ? 'no node at all' : known ? `"${known.name}" (${named})` : `"${named}"`;
+    const wired = feeds.length === 0
+        ? 'Nothing is wired into it.'
+        : `What is wired into it is ${listNodes(feeds, byId)}.`;
+    return {
+        code: 'lookup-reference-not-wired',
+        nodeIds: known ? [node.id, named] : [node.id],
+        message: `Lookup "${node.name}" (${node.id}) takes its reference rows from ${names}, which is not wired into it. ${wired} A lookup holds one of its inputs in memory as a map and streams the rest past it, so it has to be told which one — and with no reference rows to hold, every row would come out enriched with nulls and the load would commit.`,
+    };
+}
 /**
  * A rename whose map this service will not store.
  *
@@ -3250,25 +3596,8 @@ function workflowGraphHash(graph) {
 }
 /** The parts of a node that change what a run produces. */
 function canonicalNode(node) {
-    if (node.kind === 'source') {
-        return JSON.stringify([
-            node.id,
-            node.kind,
-            node.sourceKind,
-            node.connectionId ?? '',
-            node.secretEnvVar ?? '',
-            node.mode ?? 'full',
-            // Sorted keys, so a canvas that rewrites the object in a different order
-            // does not look like an edit.
-            sortedEntries(node.config),
-            // Appended only when there is a reference, exactly as `edge.branch` above
-            // is appended only when there is a label, and for the same reason: adding
-            // reusable nodes to this file must not renumber the version of a single
-            // graph that did not change. Every source drawn before they existed
-            // hashes to the string it always did.
-            ...canonicalReuse(node),
-        ]);
-    }
+    if (node.kind === 'source')
+        return canonicalSource(node);
     if (node.kind === 'transform') {
         // The transform's *version as stored* is deliberately not in here, and that
         // has not changed: editing a transform is recorded as a new transform
@@ -3330,6 +3659,8 @@ function canonicalNode(node) {
         return canonicalRename(node);
     if (node.kind === 'aggregate')
         return canonicalAggregate(node);
+    if (node.kind === 'lookup')
+        return canonicalLookup(node);
     if (node.kind === 'sink') {
         return JSON.stringify([
             node.id,
@@ -3341,6 +3672,34 @@ function canonicalNode(node) {
     }
     return unreachableNodeKind(node, 'workflowGraphHash');
 }
+/**
+ * A source, canonicalised.
+ *
+ * Its own function, beside the ones every other configured kind already has, for
+ * the reason `toTransformNode` is one package over: `canonicalNode` is a
+ * dispatcher, and a dispatcher that also carries the longest of its cases inline
+ * is where the next case gets written inline too. Nothing about what is hashed
+ * changed with the move — this is the string it always produced.
+ */
+function canonicalSource(node) {
+    return JSON.stringify([
+        node.id,
+        node.kind,
+        node.sourceKind,
+        node.connectionId ?? '',
+        node.secretEnvVar ?? '',
+        node.mode ?? 'full',
+        // Sorted keys, so a canvas that rewrites the object in a different order
+        // does not look like an edit.
+        sortedEntries(node.config),
+        // Appended only when there is a reference, exactly as `edge.branch` above
+        // is appended only when there is a label, and for the same reason: adding
+        // reusable nodes to this file must not renumber the version of a single
+        // graph that did not change. Every source drawn before they existed
+        // hashes to the string it always did.
+        ...canonicalReuse(node),
+    ]);
+}
 /**
  * A rename, canonicalised.
  *
@@ -3404,6 +3763,38 @@ function canonicalAggregate(node) {
         ...(node.maxGroups === undefined ? [] : [node.maxGroups]),
     ]);
 }
+/**
+ * A lookup, canonicalised.
+ *
+ * `reference` is in here and it is the field somebody would be tempted to leave
+ * out, because it names a node rather than describing an operation. It has to be
+ * in: repointing the reference edge at a different node is a change to what the
+ * load produces on every row, and it is one of the few edits that leaves the
+ * canvas looking identical.
+ *
+ * The fields are sorted by reference column, safe for the reason a rename's map
+ * is: the whole map is applied to one row at once, so its order changes nothing
+ * about the result and a canvas rewriting the object in another order is not an
+ * edit.
+ *
+ * `unmatched` is appended only when it is not the default, exactly as
+ * `edge.branch` is appended only when there is a label. That rule buys nothing
+ * today — no stored graph has a lookup in it, since the kind is new — and it is
+ * followed anyway, because the version *after* this one is where a default gets
+ * normalised onto the node by some canvas and renumbers every graph that has one.
+ */
+function canonicalLookup(node) {
+    const unmatched = workflowLookupUnmatched(node);
+    return JSON.stringify([
+        node.id,
+        node.kind,
+        node.reference,
+        node.key,
+        node.referenceKey,
+        sortedEntries(node.fields),
+        ...(unmatched === 'null' ? [] : [unmatched]),
+    ]);
+}
 /**
  * The reusable reference, as zero, one or two trailing hash components.
  *
@@ -3588,7 +3979,7 @@ function workflowFilterColumns(predicate) {
  * cyclic graph before it gets here, but the canvas calls this while a graph is
  * being drawn and is entitled to a wrong-but-terminating answer.
  */
-function workflowKnownColumns(graph, nodeId, knowledge) {
+function workflowKnownColumns(graph, nodeId, knowledge, onlyFrom) {
     const nodes = graph.nodes ?? [];
     const byId = new Map(nodes.map((node) => [node.id, node]));
     const { incoming } = buildAdjacency(nodes, graph.edges ?? []);
@@ -3605,13 +3996,16 @@ function workflowKnownColumns(graph, nodeId, knowledge) {
         if (!node)
             return undefined;
         open.add(id);
-        const produced = producedColumns(node, () => intoNode(id), knowledge);
+        const produced = producedColumns(node, (only) => intoNode(id, only), knowledge);
         open.delete(id);
+        // Memoised without the filter, and safe because the filter is only ever
+        // applied at the node being *asked about*: `outOf` answers what a node
+        // passes on, which does not depend on who is asking.
         answered.set(id, produced);
         return produced;
     };
-    const intoNode = (id) => {
-        const feeds = incoming.get(id) ?? [];
+    const intoNode = (id, only) => {
+        const feeds = (incoming.get(id) ?? []).filter((from) => only?.(from) ?? true);
         if (feeds.length === 0)
             return undefined;
         const union = new Set();
@@ -3627,7 +4021,28 @@ function workflowKnownColumns(graph, nodeId, knowledge) {
         }
         return union;
     };
-    return intoNode(nodeId);
+    return intoNode(nodeId, onlyFrom);
+}
+/**
+ * The two column sets a lookup sees, told apart.
+ *
+ * Exported because three callers need the same split and each one getting it
+ * right separately is how they come to disagree: the validator refuses a key
+ * column that is not on the driving side, the walk answers what the node passes
+ * on, and the inspector says both out loud on the screen where the columns are
+ * typed.
+ *
+ * Either side answers `undefined` for the ordinary reason — see
+ * {@link workflowKnownColumns} — and `undefined` must not be read as empty.
+ */
+function workflowLookupColumns(graph, node, knowledge) {
+    const named = typeof node.reference === 'string' ? node.reference : '';
+    return {
+        driving: workflowKnownColumns(graph, node.id, knowledge, (from) => from !== named),
+        reference: named.length === 0
+            ? undefined
+            : workflowKnownColumns(graph, node.id, knowledge, (from) => from === named),
+    };
 }
 /**
  * What one node passes on, given what reaches it.
@@ -3642,17 +4057,10 @@ function workflowKnownColumns(graph, nodeId, knowledge) {
  * *safe* wrong answer and would therefore never be noticed.
  */
 function producedColumns(node, upstream, knowledge) {
-    if (node.kind === 'rename') {
-        if (workflowRenameUnnamed(node) === 'drop')
-            return new Set(Object.values(node.columns ?? {}));
-        const known = upstream();
-        if (known === undefined)
-            return undefined;
-        const renamed = new Set();
-        for (const column of known)
-            renamed.add(node.columns?.[column] ?? column);
-        return renamed;
-    }
+    if (node.kind === 'lookup')
+        return lookupProducedColumns(node, upstream);
+    if (node.kind === 'rename')
+        return renameProducedColumns(node, upstream);
     // The one kind whose output set is *exact* rather than an upper bound, and it
     // is exact without looking upstream at all: an aggregate emits its group-by
     // columns and its named aggregates on every record it produces, whatever it
@@ -3676,6 +4084,50 @@ function producedColumns(node, upstream, knowledge) {
     }
     return unreachableNodeKind(node, 'workflowKnownColumns');
 }
+/**
+ * What a rename passes on, given what reaches it.
+ *
+ * The one that makes the upstream argument a thunk: a rename that drops its
+ * unnamed columns answers from its own config, so the walk stops here rather
+ * than climbing to a source it would learn nothing from.
+ */
+function renameProducedColumns(node, upstream) {
+    if (workflowRenameUnnamed(node) === 'drop')
+        return new Set(Object.values(node.columns ?? {}));
+    const known = upstream();
+    if (known === undefined)
+        return undefined;
+    const renamed = new Set();
+    for (const column of known)
+        renamed.add(node.columns?.[column] ?? column);
+    return renamed;
+}
+/**
+ * What a lookup passes on, which is not what it was given.
+ *
+ * The reference side is asked *not* to contribute, and that is the whole reason
+ * {@link producedColumns} hands its caller a filter. Its rows are held as a map
+ * and never passed on, so a column that only exists over there is not a column
+ * anything below this node can see — and a set that claimed otherwise would make
+ * the validator accept a filter that matches no row, which is the precise failure
+ * `checkColumnsProduced` exists to catch.
+ *
+ * Exact rather than an upper bound, and that is the payoff of the config being
+ * data: what leaves a lookup is what arrived plus the names it was told to add,
+ * and nothing else can appear. Under `unmatched: 'null'` and `'fail'` every row
+ * carries every target; under `'drop'` the rows that would not have are gone, so
+ * it holds there too.
+ */
+function lookupProducedColumns(node, upstream) {
+    const named = typeof node.reference === 'string' ? node.reference : '';
+    const driving = upstream((from) => from !== named);
+    if (driving === undefined)
+        return undefined;
+    const enriched = new Set(driving);
+    for (const to of Object.values(node.fields ?? {}))
+        enriched.add(to);
+    return enriched;
+}
 /**
  * What a source produces, per source kind.
  *
@@ -3746,22 +4198,11 @@ function checkColumnsProduced(graph, issues, knowledge) {
         // Narrowed off the union rather than tested with a property check, so a kind
         // that starts naming columns without being answered for here is a type error
         // at `missingColumnMessage` and not a check that silently passes.
-        if (node.kind !== 'filter' && node.kind !== 'rename' && node.kind !== 'aggregate')
-            continue;
-        const named = columnsNamedBy(node);
-        if (named.length === 0)
-            continue;
-        const known = workflowKnownColumns(graph, node.id, knowledge);
-        if (known === undefined)
-            continue;
-        const missing = named.filter((column) => column.length > 0 && !known.has(column));
-        if (missing.length === 0)
-            continue;
-        issues.push({
-            code: 'column-not-produced',
-            nodeIds: [node.id],
-            message: missingColumnMessage(node, missing, known),
-        });
+        if (node.kind === 'lookup')
+            checkLookupColumns(graph, node, issues, knowledge);
+        else if (node.kind === 'filter' || node.kind === 'rename' || node.kind === 'aggregate') {
+            checkNamedColumns(graph, node, issues, knowledge);
+        }
     }
 }
 /**
@@ -3779,6 +4220,109 @@ function columnsNamedBy(node) {
         return Object.keys(node.columns ?? {});
     return workflowAggregateColumns(node);
 }
+/**
+ * The columns a filter, a rename or an aggregate names, checked against
+ * everything reaching the node.
+ *
+ * The union is the right set for these three, and it is the whole difference
+ * from {@link checkLookupColumns}: the rows a multi-input node receives arrive
+ * concatenated, so a column any one of its inputs carries is a column it can
+ * see. A lookup is the one kind that is not like that — its reference is held as
+ * a map and never passed on — which is why it is dispatched away from here
+ * rather than folded in.
+ */
+function checkNamedColumns(graph, node, issues, knowledge) {
+    const named = columnsNamedBy(node);
+    if (named.length === 0)
+        return;
+    const known = workflowKnownColumns(graph, node.id, knowledge);
+    if (known === undefined)
+        return;
+    const missing = named.filter((column) => column.length > 0 && !known.has(column));
+    if (missing.length === 0)
+        return;
+    issues.push({
+        code: 'column-not-produced',
+        nodeIds: [node.id],
+        message: missingColumnMessage(node, missing, known),
+    });
+}
+/**
+ * A lookup's columns, checked against the side each of them lives on.
+ *
+ * Its own function rather than a branch inside {@link checkColumnsProduced},
+ * because a lookup is the one node whose named columns do not all come from one
+ * place: `key` is on the rows being enriched, and `referenceKey` and every field
+ * source are on the reference. Folding it into the loop above would have meant
+ * checking both against the union — which is *worse than not checking*, because
+ * the union would accept a key column that only exists on the reference. That
+ * join matches nothing on every row and the load commits, which is what this node
+ * is for.
+ *
+ * What is deliberately **not** checked here is a field landing on a name the
+ * driving rows already carry. See {@link checkLookupDriving}.
+ *
+ * Every check is skipped where the set is unknown, for the reason the walk
+ * answers `undefined` rather than empty: silence is correct here, and refusing a
+ * column the graph merely has no opinion about would make every lookup below a
+ * transform unsaveable.
+ */
+function checkLookupColumns(graph, node, issues, knowledge) {
+    const { driving, reference } = workflowLookupColumns(graph, node, knowledge);
+    if (driving !== undefined)
+        checkLookupDriving(node, driving, issues);
+    if (reference !== undefined)
+        checkLookupReference(node, reference, issues);
+}
+/** `"a", "b"`, for a message that has to name a set of columns. */
+function quotedColumns(names) {
+    return [...names].map((column) => JSON.stringify(column)).join(', ');
+}
+/**
+ * The key this lookup matches on, against the rows it enriches.
+ *
+ * The *driving* set specifically, and pooling the inputs would make this wrong in
+ * the dangerous direction: a key column that exists only on the reference would
+ * be accepted here and then match nothing on every row.
+ *
+ * ## Why a name the driving rows already carry is not refused here
+ *
+ * Because *that is the normal case*, and refusing it made the node unable to do
+ * the one thing it was built for. A published object type declares the columns it
+ * holds, so a graph that reads a type back to enrich it receives every one of
+ * those columns — `SubwoReplica` hands over 44,720 rows all carrying `planName`,
+ * `planDescription` and `unitMel` with `null` in them, which is precisely the
+ * three columns the lookup exists to fill. Whether a column holds a *value* is a
+ * fact about the data and not about the graph, so the run decides it, per row:
+ * empty is filled and occupied fails the node, naming the row. See
+ * `withLookupFields`.
+ */
+function checkLookupDriving(node, driving, issues) {
+    if (typeof node.key === 'string' && node.key.length > 0 && !driving.has(node.key)) {
+        issues.push({
+            code: 'column-not-produced',
+            nodeIds: [node.id],
+            message: `Lookup "${node.name}" (${node.id}) matches on ${quotedColumns([node.key])}, and nothing feeding the rows it enriches produces that column. Something above this node closes the column set — a rename that drops what it does not name, or a source reading a published object type — so what reaches here is exactly ${quotedColumns(driving)}. A key column that is not there has no key on any row, so nothing would match and every row would come out with the enriched columns null.`,
+        });
+    }
+}
+/** The reference key and every field source, against the reference side only. */
+function checkLookupReference(node, reference, issues) {
+    const wanted = [
+        ...(typeof node.referenceKey === 'string' && node.referenceKey.length > 0
+            ? [node.referenceKey]
+            : []),
+        ...Object.keys(node.fields ?? {}).filter((from) => from.length > 0),
+    ];
+    const missing = wanted.filter((column) => !reference.has(column));
+    if (missing.length === 0)
+        return;
+    issues.push({
+        code: 'column-not-produced',
+        nodeIds: [node.id, node.reference],
+        message: `Lookup "${node.name}" (${node.id}) reads ${quotedColumns(missing)} off its reference rows, and nothing on that side produces ${missing.length === 1 ? 'that column' : 'those columns'} — what reaches it is exactly ${quotedColumns(reference)}. A reference key that is not there means no reference row has a key, so nothing matches; a field that is not there lands as undefined on every row that did.`,
+    });
+}
 /** The sentence {@link checkColumnsProduced} says, per kind. */
 function missingColumnMessage(node, missing, known) {
     const quoted = (names) => [...names].map((column) => JSON.stringify(column)).join(', ');
@@ -3851,13 +4395,25 @@ function isWorkflowNode(value) {
         return isRenameNodeShape(value);
     if (kind === 'aggregate')
         return isAggregateNodeShape(value);
-    if (kind === 'source') {
-        const sourceKind = Reflect.get(value, 'sourceKind');
-        const config = Reflect.get(value, 'config');
-        return isConnectorKind(sourceKind) && typeof config === 'object' && config !== null;
-    }
+    if (kind === 'lookup')
+        return isLookupNodeShape(value);
+    if (kind === 'source')
+        return isSourceNodeShape(value);
     return isWorkflowNodeKindUnhandled(kind);
 }
+/**
+ * Everything a `source` node carries.
+ *
+ * Its own function beside the four kinds that already have one, and for the
+ * reason those have one: {@link isWorkflowNode} is a narrowing dispatcher, and a
+ * dispatcher that also carries its cases inline stops being readable at exactly
+ * the point a new kind is added to it.
+ */
+function isSourceNodeShape(value) {
+    const sourceKind = Reflect.get(value, 'sourceKind');
+    const config = Reflect.get(value, 'config');
+    return isConnectorKind(sourceKind) && typeof config === 'object' && config !== null;
+}
 /**
  * Everything a `rename` node carries.
  *
@@ -3891,6 +4447,35 @@ function isAggregateNodeShape(value) {
         maxGroups: Reflect.get(value, 'maxGroups'),
     }).length === 0);
 }
+/**
+ * Everything a `lookup` node carries.
+ *
+ * `reference` is checked as a non-empty string and no further, because whether
+ * it names a node that is actually wired in is a fact about the graph and this
+ * guard sees one node. `validateWorkflow` owns that, and refuses it loudly.
+ *
+ * An unrecognised `unmatched` is refused rather than defaulted, and this is the
+ * one where defaulting would be worst of the three kinds that have the same
+ * field: reading an unknown word back as `null` would turn a `fail` — which
+ * somebody chose because the reference is a prerequisite — into a load that
+ * commits nulls and reports success.
+ */
+function isLookupNodeShape(value) {
+    const unmatched = Reflect.get(value, 'unmatched');
+    if (unmatched !== undefined && !isWorkflowLookupUnmatched(unmatched))
+        return false;
+    const reference = Reflect.get(value, 'reference');
+    if (typeof reference !== 'string' || reference.length === 0)
+        return false;
+    // Read into an `unknown` before the guard, so the narrowing that follows is
+    // the guard's and not an assertion dressed as one.
+    const fields = Reflect.get(value, 'fields');
+    if (!isWorkflowLookupFields(fields))
+        return false;
+    const key = Reflect.get(value, 'key');
+    const referenceKey = Reflect.get(value, 'referenceKey');
+    return lookupConfigRefusals({ key, referenceKey, fields }).length === 0;
+}
 /**
  * Everything a `call` node carries, checked as strictly as a source's.
  *