@dudousxd/nestjs-catalog 0.22.0 → 0.23.0

package/dist/catalog.pipeline.d.ts

@@ -918,7 +918,7 @@ export interface WorkflowNodeOutcome {
  *   outside a run, which is why `call` names one and not a step. If a step is
  *   what you want, the thing to call is a one-step workflow wrapping it.
  */
-export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter"];
+export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter", "rename"];
 export type WorkflowNodeKind = (typeof WORKFLOW_NODE_KINDS)[number];
 /** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
 export declare function isWorkflowNodeKind(value: unknown): value is WorkflowNodeKind;
@@ -1876,13 +1876,196 @@ export interface WorkflowFilterNode extends WorkflowNodeBase {
      */
     narrows?: string[];
 }
+/**
+ * What happens to a column the rename does not name.
+ *
+ * Two words rather than a boolean, because the two are genuinely different
+ * nodes and a boolean called `drop` would read as a modifier on one node. See
+ * {@link WorkflowRenameNode.unnamed} for what each costs.
+ */
+export declare const WORKFLOW_RENAME_UNNAMED: readonly ["keep", "drop"];
+export type WorkflowRenameUnnamed = (typeof WORKFLOW_RENAME_UNNAMED)[number];
+/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
+export declare function isWorkflowRenameUnnamed(value: unknown): value is WorkflowRenameUnnamed;
+/**
+ * The exhaustiveness guard for {@link WORKFLOW_RENAME_UNNAMED}.
+ *
+ * {@link unreachableNodeKind}, one level down, and for the identical reason: the
+ * two words decide whether a batch is rewritten or only re-labelled, and a third
+ * one added without a branch would silently pick whichever the last `if` was.
+ */
+export declare function unreachableRenameUnnamed(value: never, where: string): never;
+/**
+ * How many columns one rename may name.
+ *
+ * The same argument {@link WORKFLOW_FILTER_MAX_VALUES} makes: the map travels in
+ * the graph and into the graph fingerprint, and past a few hundred entries the
+ * thing being expressed is a schema mapping that belongs in a stored object
+ * rather than in a node. It is also the bound that keeps
+ * {@link isWorkflowRenameColumns} — which is run on JSON out of a column — from
+ * being a place to hand a service a million-key object.
+ */
+export declare const WORKFLOW_RENAME_MAX_COLUMNS = 500;
+/**
+ * Renames columns, and does nothing else, ever.
+ *
+ * ## Why this is a node kind and not a flag, and why it stays small
+ *
+ * The generic {@link WorkflowTransformNode} continues to exist for everything
+ * else, and **that is what lets this node stay deliberately narrow**. The usual
+ * objection to a declarative shortcut is that it grows — rename, then cast, then
+ * default, then trim, and then it is a small language nobody designed and
+ * everybody has to learn. With a real code node sitting beside it, the answer to
+ * "I need more than renaming" is always *use a transform*, and never *add a
+ * field here*. That is the constraint that keeps this honest, and it is the
+ * reason to refuse the next field rather than a reason to feel bad about
+ * refusing it.
+ *
+ * ## What it buys, measured rather than asserted
+ *
+ * **1. It streams by construction.** A rename is per record. It cannot
+ * aggregate, deduplicate, sort or look at a neighbour, so there is no batch it
+ * has to hold. A transform node cannot make that promise — an author's function
+ * is handed the whole batch and may legitimately reduce over it — which is why
+ * `ConnectorRunnerService` has to log *"Held all N records in memory"* when a
+ * transform is present. This node never contributes that line.
+ *
+ * **2. It needs no child process.** A transform round trip for 103,087 rows
+ * costs ~338 ms, of which the author's `.map` is ~5 ms. The transport — encode,
+ * write, decode, run, encode, read — is the entire bill. A rename does not need
+ * transport, and the same rename in process is ~14 ms.
+ *
+ * **3. On staged data it is metadata-only, and exactly when.** A staged batch is
+ * a shape dictionary (see `catalog.stage-encoding.ts`): `shapes` holds each
+ * distinct key-set once, `shapeOf[i]` indexes it, and `values[i]` is a
+ * positional array parallel to that shape. A positional array does not care what
+ * the key is called, so renaming a key is a rewrite of `shapes` and nothing
+ * else — tens of strings for a hundred thousand rows.
+ *
+ * **That last claim holds for `unnamed: 'keep'` and does not hold for
+ * `unnamed: 'drop'`.** Dropping a column removes a position, so every `values`
+ * row has to be rebuilt and the cost is back to O(rows). Both are worth having
+ * and they are not the same operation, so the node says which one it did:
+ * `renameStagePayload` reports `metadataOnly`, and the run log prints it.
+ *
+ * ## The config
+ *
+ * {@link columns} is a map of **old name → new name**, applied
+ * **simultaneously** rather than in sequence. `{"a": "b", "b": "c"}` maps `a` to
+ * `b` and `b` to `c`; it does not chain `a → b → c`. Sequential application
+ * would make the result depend on the iteration order of a JSON object, which is
+ * not a thing to build a load on.
+ *
+ * A `Record` rather than a list of pairs, and the reason is a refusal it buys
+ * for free: a key cannot appear twice in an object, so *two renames of the same
+ * source column* is unrepresentable. The mirror mistake — two renames **onto**
+ * the same target — is representable and is refused by `validateWorkflow`, which
+ * can see it from the config alone with nothing to run.
+ *
+ * The four remaining edge cases, all decided rather than discovered:
+ *
+ * - **A column the map does not name** — {@link unnamed} decides, and it
+ *   defaults to `keep`.
+ * - **A named column that is not in a given record** — nothing happens to that
+ *   record. This is not an error, because a batch legitimately holds rows with
+ *   different key-sets; that is the entire reason the stage encoding is a shape
+ *   *dictionary* and not one column list. A source column that turns out to be
+ *   in **no** row of the whole run is reported loudly in the run log, because it
+ *   is almost always a typo in a header and the symptom otherwise is a column of
+ *   NULLs and a green run.
+ * - **A rename onto a name the record already holds** — refused, at run time,
+ *   naming both columns. There are two columns and one name, and every rule for
+ *   picking a winner is arbitrary. It is detected per *shape* rather than per
+ *   row, so it fails on the first batch rather than at row ninety thousand.
+ *   Under `unnamed: 'drop'` there is no collision to have: a column the map does
+ *   not name does not exist in the output, so it cannot occupy anything.
+ * - **`a → a`** — allowed. Under `keep` it is a no-op; under `drop` it is how a
+ *   column is *selected*, which is a real use.
+ *
+ * ## What the target names have to be
+ *
+ * {@link WORKFLOW_FILTER_COLUMN_PATTERN}, checked at authoring time. Two
+ * separate reasons, and both are about a failure that reports success:
+ *
+ * - `property-names.ts` refuses a *published property* whose name cannot become
+ *   a column, and its docblock is the record of what happens when a name and the
+ *   key in the record disagree: the load looks every field up as `row[name]`, so
+ *   the column takes NULL in every row and the run is green. Thirteen types went
+ *   in that way. A rename is the tool that makes the record's key match the
+ *   property, so a rename that produces a name no property can carry is the
+ *   trap re-armed one node upstream.
+ * - The filter node's columns follow the same pattern precisely so a predicate
+ *   could one day be pushed into a `WHERE`. A rename whose target cannot be
+ *   named by a filter would author a graph today that could never be pushed down
+ *   tomorrow.
+ *
+ * The *source* names are deliberately unconstrained. `Mgmt Cd`, `VEH Type Name`
+ * and `Reg Number` are exactly what real drops are keyed by, and being able to
+ * name them is the entire point of the node.
+ */
+export interface WorkflowRenameNode extends WorkflowNodeBase {
+    kind: 'rename';
+    /**
+     * Old name → new name, applied simultaneously. Never empty; at most
+     * {@link WORKFLOW_RENAME_MAX_COLUMNS} entries; every target matches
+     * {@link WORKFLOW_FILTER_COLUMN_PATTERN} and no two share one.
+     *
+     * Empty is refused rather than treated as a no-op, for the reason an empty
+     * filter group is: under `keep` it is a node that draws as configured and does
+     * nothing, and under `drop` it deletes every column of every row and commits
+     * the result. Two silent opposites reached by deleting the last row of a form.
+     */
+    columns: Record<string, string>;
+    /**
+     * What happens to the columns this node does not name. Absent means `keep`.
+     *
+     * Absent is `keep` rather than being required, because `keep` is the node this
+     * one is called after: a rename that also deleted everything it did not
+     * mention would be a projection wearing the word "rename", and somebody would
+     * find that out by looking at a committed snapshot.
+     *
+     * `drop` is here because the shape it replaces is real —
+     * `records.map((r) => ({ mgmtCd: r["Mgmt Cd"] }))` is a rename *and* a
+     * projection, and it is what a drop of Air Force fleet data forces today. It
+     * costs the metadata-only property (see the docblock above), and the run says
+     * so rather than leaving the difference to be guessed at.
+     */
+    unnamed?: WorkflowRenameUnnamed;
+}
+/** {@link WorkflowRenameNode.unnamed}, resolved. One reader of the default. */
+export declare function workflowRenameUnnamed(node: WorkflowRenameNode): WorkflowRenameUnnamed;
+/**
+ * Every reason a rename map cannot be stored, as sentences, or empty.
+ *
+ * One function, called by {@link validateWorkflow}, by the HTTP boundary and by
+ * the canvas, for the reason `validateWorkflow` itself is shared: a screen that
+ * checked a target name against its own copy of the pattern is a screen that
+ * eventually accepts something the server refuses, halfway through a save.
+ *
+ * All of them rather than the first, exactly as
+ * {@link refuseUnpublishablePropertyNames} argues: a map of forty columns typed
+ * in one sitting is usually wrong about several in the same way.
+ */
+export declare function renameColumnRefusals(columns: Record<string, string>): string[];
+/**
+ * Whether a stored rename map is one this build can run.
+ *
+ * Refused rather than repaired, the stance {@link isWorkflowFilterPredicate}
+ * takes about a predicate and for the same reason one step further along: a
+ * rename read back with one entry silently dropped is a graph that commits a
+ * column of NULLs under a name nobody can now explain.
+ *
+ * `Object.entries` rather than a `for…in`, so an inherited key cannot enter the
+ * map, and the values are checked one by one rather than trusted from the type.
+ */
+export declare function isWorkflowRenameColumns(value: unknown): value is Record<string, string>;
 /**
  * A discriminated union, so narrowing a node is `node.kind === "sink"` and
  * never a type assertion. This is why the kind list is not simply a string on
  * one node shape with every field optional: that shape lets a source node carry
  * a `transformId` and nothing catches it.
  */
-export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode;
+export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode | WorkflowRenameNode;
 /**
  * The node kinds that can be saved once and used in several graphs.
  *
@@ -1934,6 +2117,11 @@ export declare function isReusableNodeKind(value: unknown): value is ReusableNod
  *   may not have, and a filter is worse: {@link WorkflowFilterNode.narrows} is
  *   an acknowledgement about *this* graph's sinks, so a shared one would carry
  *   somebody else's acknowledgement into a graph they never saw.
+ * - `rename` — the same argument as `if` and `filter`, and the sharpest version
+ *   of it: a rename map names the source's own spelling of its columns, so it is
+ *   *about* one drop of one file. `Mgmt Cd → mgmtCd` saved under a name and
+ *   dropped into a graph reading a different system renames nothing at all, and
+ *   the symptom is a column of NULLs rather than a failure.
  */
 export declare const NODE_KIND_IS_REUSABLE: {
     readonly source: true;
@@ -1942,6 +2130,7 @@ export declare const NODE_KIND_IS_REUSABLE: {
     readonly call: false;
     readonly if: false;
     readonly filter: false;
+    readonly rename: false;
 };
 /** Whether this kind can be saved as a reusable node. Reads {@link NODE_KIND_IS_REUSABLE}. */
 export declare function nodeKindIsReusable(kind: WorkflowNodeKind): boolean;
@@ -2740,7 +2929,7 @@ export interface CallableWorkflowBlock {
 }
 export declare function callableWorkflowBlock(ref: CallableWorkflowRef): CallableWorkflowBlock | undefined;
 /** Every way a graph can be refused. Exported so a canvas can key off the code. */
-export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "call-not-named", "call-plain-has-output", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing", "version-pin-invalid"];
+export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "call-not-named", "call-plain-has-output", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing", "rename-invalid", "column-not-produced", "version-pin-invalid"];
 export type WorkflowIssueCode = (typeof WORKFLOW_ISSUE_CODES)[number];
 export interface WorkflowValidationIssue {
     code: WorkflowIssueCode;
@@ -2896,6 +3085,62 @@ export declare function workflowNodeRuns(entry: {
  * a thing to plan for.
  */
 export declare function workflowGraphHash(graph: WorkflowGraph): string;
+/**
+ * Every column a filter predicate names, once each, in the order they appear.
+ *
+ * Its own function rather than a walk inlined into the validator, because two
+ * things want it — the refusal below and anything on a screen that wants to say
+ * which columns a node depends on — and a second copy of a tree walk is a second
+ * copy that forgets the `oneOf` branch.
+ */
+export declare function workflowFilterColumns(predicate: WorkflowFilterPredicate): string[];
+/**
+ * The columns that can reach this node, when the graph knows — and `undefined`
+ * when it does not.
+ *
+ * ## What this is for
+ *
+ * It is the one thing a declarative rename buys that a transform cannot, and it
+ * is worth being precise about how far it reaches rather than overselling it.
+ *
+ * With a JS transform, the catalog cannot know what columns come out — the
+ * answer is inside a function body — which is why the property-name rule in
+ * `property-names.ts` fires at publish time and why a mismatch between a
+ * property and a record key is discovered as a column of NULLs. A rename is
+ * **data**, so for one arrangement the answer is exact:
+ *
+ * > A rename with `unnamed: 'drop'` produces its targets and **nothing else**,
+ * > whatever it was handed.
+ *
+ * That set is *closed* — an upper bound that holds regardless of what is
+ * upstream — and it survives every node that does not touch columns. So a filter
+ * or a second rename downstream of one can be told, at authoring time, that it
+ * names a column which cannot be there.
+ *
+ * ## What it deliberately does not claim
+ *
+ * - **It is an upper bound, not the output.** A target only appears in a row
+ *   whose input actually held the source column. So a column *inside* the set
+ *   may still be absent, and nothing here says otherwise.
+ * - **A `keep` rename tells you nothing on its own.** Its output is its input
+ *   with some keys re-labelled, and its input is unknown unless something
+ *   upstream closed it. So `undefined` propagates, and that is the honest
+ *   answer rather than an empty set.
+ * - **A source, a transform and a call are always unknown.** A source's shape is
+ *   discovered against the live system rather than declared in the graph; a
+ *   transform is a function body; a call is a workflow this graph does not own.
+ * - **It says nothing about a sink's declared properties.** That is the check
+ *   worth wanting — "this sink writes a property no upstream node produces" —
+ *   and it is *not* available here: a {@link WorkflowSinkNode} carries a
+ *   `targetType` and nothing else, so the property list would have to be
+ *   threaded into a validator that is pure and dependency-free on purpose. What
+ *   is built instead is the run log, which prints the columns a rename produced.
+ *
+ * Cycles answer `undefined` rather than looping. `validateWorkflow` refuses a
+ * cyclic graph before it gets here, but the canvas calls this while a graph is
+ * being drawn and is entitled to a wrong-but-terminating answer.
+ */
+export declare function workflowKnownColumns(graph: WorkflowGraph, nodeId: string): ReadonlySet<string> | undefined;
 /**
  * Narrow a stored node, loudly.
  *
@@ -3244,6 +3489,39 @@ export interface CatalogStageStore {
      * as long as something might still resume onto them.
      */
     dropStages(runId: string): Promise<number>;
+    /**
+     * The batch exactly as it is stored, without decoding it into rows.
+     *
+     * Optional, and the only thing that reads it is the `rename` node. See
+     * {@link renameStagePayload}: a staged batch names its columns once, in
+     * `shapes`, and carries the data in positional arrays — so renaming a column
+     * is a rewrite of tens of strings rather than a rebuild of a hundred thousand
+     * objects. `readStage` cannot express that, because decoding to
+     * `Record<string, unknown>` *is* the rebuild.
+     *
+     * Optional rather than required so that a store written against the shipped
+     * interface keeps working: {@link supportsStagePayloads} is what asks, and a
+     * store that answers no gets the row path, which produces the same rows more
+     * slowly. It is deliberately `unknown` — the encoding is
+     * `catalog.stage-encoding.ts`'s business and a store's job is to hand back
+     * what it was given.
+     */
+    readStagePayload?(ref: {
+        runId: string;
+        nodeId: string;
+        batch: number;
+    }): Promise<unknown>;
+    /** The other half. Idempotent per `(runId, nodeId, batch)`, exactly like {@link writeStage}. */
+    writeStagePayload?(input: {
+        runId: string;
+        nodeId: string;
+        batch: number;
+        payload: unknown;
+        /** How many rows the payload holds. The store does not decode it to count. */
+        rows: number;
+    }): Promise<{
+        written: number;
+    }>;
 }
 /**
  * Whether this store can hold workflows at all.
@@ -3318,6 +3596,21 @@ export type CatalogReusableNodeStore = Required<Pick<CatalogPipelineStore, 'list
  */
 export declare function supportsReusableNodes(store: CatalogPipelineStore): store is CatalogPipelineStore & CatalogReusableNodeStore;
 export declare function supportsWorkflowStages(store: CatalogPipelineStore): store is CatalogPipelineStore & CatalogStageStore;
+/**
+ * Whether this store will hand a staged batch over without decoding it.
+ *
+ * Both methods, never one: a rename that could read the payload and not write
+ * one back would have to decode its own output to store it, which is the rebuild
+ * the pair exists to avoid. The methods rather than a flag, the same argument
+ * {@link supportsWorkflows} makes.
+ *
+ * A store that answers no is not broken and nothing degrades except speed — the
+ * rename node falls back to `readStage`/`writeStage` and produces identical
+ * rows. Which path ran is said in the run log, because "this rename was
+ * metadata-only" is a claim, and a claim that could quietly stop being true is
+ * worse than no claim.
+ */
+export declare function supportsStagePayloads(store: CatalogPipelineStore): store is CatalogPipelineStore & CatalogStageStore & Required<Pick<CatalogStageStore, 'readStagePayload' | 'writeStagePayload'>>;
 /**
  * A store that really does hold operator-set expectations, all four members
  * present.