@dudousxd/nestjs-catalog 0.27.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.pipeline.d.ts +399 -10
- package/dist/catalog.pipeline.js +648 -63
- package/dist/client.d.ts +2 -2
- package/dist/client.js +20 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +17 -4
- package/package.json +1 -1
|
@@ -1183,12 +1183,25 @@ export interface WorkflowNodeOutcome {
|
|
|
1183
1183
|
* {@link WorkflowIfNode} is the conditional one, and it earns its kind by
|
|
1184
1184
|
* doing something no wiring can express — deciding that one of those
|
|
1185
1185
|
* successors, and everything only it feeds, does not run at all.
|
|
1186
|
-
* - **merge
|
|
1187
|
-
* concatenated in edge order (see {@link WorkflowEdge})
|
|
1188
|
-
*
|
|
1189
|
-
*
|
|
1190
|
-
*
|
|
1191
|
-
*
|
|
1186
|
+
* - **merge (unkeyed)** — a node with several inbound edges already receives its
|
|
1187
|
+
* inputs concatenated in edge order (see {@link WorkflowEdge}), so a `merge`
|
|
1188
|
+
* kind would be a box that draws what the wires already say. It would also
|
|
1189
|
+
* have had to carry a strategy field whose values the runner implements one by
|
|
1190
|
+
* one, and an unimplemented strategy in a dropdown is the failure this list
|
|
1191
|
+
* exists to avoid.
|
|
1192
|
+
* - **join (keyed)** — *this half used to be refused with the entry above, and
|
|
1193
|
+
* the reversal is left visible rather than edited out*, the way the `filter`
|
|
1194
|
+
* entry leaves its own. The old argument was that a keyed join is ordinary
|
|
1195
|
+
* code inside a transform, which can already see every record. Every word of
|
|
1196
|
+
* that is true and it is exactly the problem: "can already see every record"
|
|
1197
|
+
* is the same sentence as "holds the whole load", and it is why a transform
|
|
1198
|
+
* makes `ConnectorRunnerService` log *"Held all N records in memory"*. A join
|
|
1199
|
+
* does not need both sides held. It needs **one** side held — as a map, keyed
|
|
1200
|
+
* — while the other streams past it, and that asymmetry is a property of the
|
|
1201
|
+
* operation that a function over a batch cannot express and a runner therefore
|
|
1202
|
+
* cannot exploit. {@link WorkflowLookupNode} is the keyed half, built narrow:
|
|
1203
|
+
* one key, named enrichment fields, and a reference side that is bounded and
|
|
1204
|
+
* refused loudly rather than held quietly.
|
|
1192
1205
|
* - **call a durable *step*** — the sibling of {@link WorkflowCallNode} that
|
|
1193
1206
|
* somebody will eventually come looking for, and it cannot be built. A
|
|
1194
1207
|
* durable step has no global identity: it is dispatched by a routing name
|
|
@@ -1199,7 +1212,7 @@ export interface WorkflowNodeOutcome {
|
|
|
1199
1212
|
* outside a run, which is why `call` names one and not a step. If a step is
|
|
1200
1213
|
* what you want, the thing to call is a one-step workflow wrapping it.
|
|
1201
1214
|
*/
|
|
1202
|
-
export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter", "rename", "aggregate"];
|
|
1215
|
+
export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter", "rename", "aggregate", "lookup"];
|
|
1203
1216
|
export type WorkflowNodeKind = (typeof WORKFLOW_NODE_KINDS)[number];
|
|
1204
1217
|
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
1205
1218
|
export declare function isWorkflowNodeKind(value: unknown): value is WorkflowNodeKind;
|
|
@@ -2407,6 +2420,27 @@ export declare function isWorkflowAggregateFunction(value: unknown): value is Wo
|
|
|
2407
2420
|
* thing that happens.
|
|
2408
2421
|
*/
|
|
2409
2422
|
export declare function unreachableAggregateFunction(fn: never, where: string): never;
|
|
2423
|
+
/**
|
|
2424
|
+
* What happens to a driving row whose key matches no reference row.
|
|
2425
|
+
*
|
|
2426
|
+
* Three words rather than a boolean, because the three are the three joins SQL
|
|
2427
|
+
* has names for and each is a different node. See
|
|
2428
|
+
* {@link WorkflowLookupNode.unmatched} for which one to reach for.
|
|
2429
|
+
*/
|
|
2430
|
+
export declare const WORKFLOW_LOOKUP_UNMATCHED: readonly ["null", "drop", "fail"];
|
|
2431
|
+
export type WorkflowLookupUnmatched = (typeof WORKFLOW_LOOKUP_UNMATCHED)[number];
|
|
2432
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
2433
|
+
export declare function isWorkflowLookupUnmatched(value: unknown): value is WorkflowLookupUnmatched;
|
|
2434
|
+
/**
|
|
2435
|
+
* The exhaustiveness guard for {@link WORKFLOW_LOOKUP_UNMATCHED}.
|
|
2436
|
+
*
|
|
2437
|
+
* {@link unreachableNodeKind}, one level down, and for the identical reason: the
|
|
2438
|
+
* three words decide whether a row keeps its data, disappears from the load, or
|
|
2439
|
+
* stops the run, and a fourth added without a branch would silently pick
|
|
2440
|
+
* whichever the last `if` was — which here means silently changing which rows
|
|
2441
|
+
* reach a published type.
|
|
2442
|
+
*/
|
|
2443
|
+
export declare function unreachableLookupUnmatched(value: never, where: string): never;
|
|
2410
2444
|
/**
|
|
2411
2445
|
* How many columns one node may group on.
|
|
2412
2446
|
*
|
|
@@ -2710,6 +2744,311 @@ export declare function aggregateRefusals(node: {
|
|
|
2710
2744
|
aggregates?: unknown;
|
|
2711
2745
|
maxGroups?: unknown;
|
|
2712
2746
|
}): string[];
|
|
2747
|
+
/**
|
|
2748
|
+
* How many fields one lookup may bring across.
|
|
2749
|
+
*
|
|
2750
|
+
* The same argument {@link WORKFLOW_RENAME_MAX_COLUMNS} makes, plus one that is
|
|
2751
|
+
* specific to this node: every named field is held *per distinct key* for the
|
|
2752
|
+
* whole run, so this number multiplies {@link WORKFLOW_LOOKUP_MAX_REFERENCE_ROWS}
|
|
2753
|
+
* into the actual memory bill. Past a few dozen the thing being expressed is
|
|
2754
|
+
* "give me that whole table beside this one", which is a second source and a
|
|
2755
|
+
* union, not an enrichment.
|
|
2756
|
+
*/
|
|
2757
|
+
export declare const WORKFLOW_LOOKUP_MAX_FIELDS = 64;
|
|
2758
|
+
/**
|
|
2759
|
+
* How many rows the reference side may have before the node refuses to run.
|
|
2760
|
+
*
|
|
2761
|
+
* ## Why there is a number here at all
|
|
2762
|
+
*
|
|
2763
|
+
* Because exactly one side of a join can stream, and this node holds the other
|
|
2764
|
+
* one. That is the whole property it exists to have (see
|
|
2765
|
+
* {@link WorkflowLookupNode}) and it is also the whole hazard: a graph whose
|
|
2766
|
+
* reference edge is accidentally wired to the 7.6-million-row side does not fail,
|
|
2767
|
+
* it allocates until the pod is killed — and a pod killed by the kernel produces
|
|
2768
|
+
* no run log, no failed node and no message, which is the silence this file is
|
|
2769
|
+
* arranged against.
|
|
2770
|
+
*
|
|
2771
|
+
* ## Why it is measured in rows, and why it is free
|
|
2772
|
+
*
|
|
2773
|
+
* A staged input announces its `rowCount` before a single row is read back (see
|
|
2774
|
+
* `WorkflowStageRef`), so the refusal happens *before* anything is held. A bound
|
|
2775
|
+
* in bytes would have to be discovered by holding rows until they weighed too
|
|
2776
|
+
* much, which is a bound that has already done the damage by the time it fires.
|
|
2777
|
+
*
|
|
2778
|
+
* Two hundred thousand, and the arithmetic rather than a round number that feels
|
|
2779
|
+
* safe: what is retained per key is the key string plus the values of the named
|
|
2780
|
+
* fields — not the reference row — so a reference of this size with a handful of
|
|
2781
|
+
* short fields is tens of megabytes, and one at {@link WORKFLOW_LOOKUP_MAX_FIELDS}
|
|
2782
|
+
* is the point where it stops being obviously fine. The real reference tables
|
|
2783
|
+
* this was built against are three orders of magnitude below it: a work-plan code
|
|
2784
|
+
* table is hundreds of rows and a unit dictionary is dozens.
|
|
2785
|
+
*
|
|
2786
|
+
* The refusal names the count, the bound and the fix, because the fix is
|
|
2787
|
+
* genuinely available in the graph: put a filter on the reference side, or swap
|
|
2788
|
+
* the two edges if the smaller side is the one being streamed.
|
|
2789
|
+
*/
|
|
2790
|
+
export declare const WORKFLOW_LOOKUP_MAX_REFERENCE_ROWS = 200000;
|
|
2791
|
+
/**
|
|
2792
|
+
* Enriches each row with fields from a reference dataset, matched by key.
|
|
2793
|
+
*
|
|
2794
|
+
* ## The property that justifies a kind rather than a transform
|
|
2795
|
+
*
|
|
2796
|
+
* **One side is held; the other streams.** The reference is read once into a map
|
|
2797
|
+
* keyed by its key column, and then the driving rows go past it one batch at a
|
|
2798
|
+
* time and never accumulate. That asymmetry is the entire content of the node,
|
|
2799
|
+
* and it is not something a {@link WorkflowTransformNode} can express: a
|
|
2800
|
+
* transform is a function over what it is given, so a join written as one has to
|
|
2801
|
+
* be handed *both* sides at once — which is why a transform makes
|
|
2802
|
+
* `ConnectorRunnerService` log *"Held all N records in memory"*, and why the
|
|
2803
|
+
* whole-batch version of this exact join reached 78% of a hard 32 MiB output
|
|
2804
|
+
* bound on 44,720 rows before it did anything interesting.
|
|
2805
|
+
*
|
|
2806
|
+
* A per-record transform ({@link CatalogTransform} in `record` mode) cannot do it
|
|
2807
|
+
* either, and the reason is sharper: a function over one record has nowhere to
|
|
2808
|
+
* put the map. It would rebuild it per record, or reach a database per record,
|
|
2809
|
+
* and 44,720 round trips is not a shape anybody would choose on purpose.
|
|
2810
|
+
*
|
|
2811
|
+
* ## What it stays narrow about, deliberately
|
|
2812
|
+
*
|
|
2813
|
+
* The generic transform still exists, and that is what lets this node refuse
|
|
2814
|
+
* every next field forever — the argument {@link WorkflowRenameNode} makes at
|
|
2815
|
+
* length. No join *type* beyond {@link unmatched}, no composite keys, no
|
|
2816
|
+
* expressions on either side, no aggregation of the matched rows. The answer to
|
|
2817
|
+
* "I need more than this" is always *use a transform*, and never *add a field
|
|
2818
|
+
* here*. Two of those refusals have a second reason on top:
|
|
2819
|
+
*
|
|
2820
|
+
* - **No composite key.** Two columns concatenated is a rule about a separator,
|
|
2821
|
+
* and a separator that occurs inside a value silently merges two different
|
|
2822
|
+
* keys into one. A {@link WorkflowRenameNode} cannot build one either, which is
|
|
2823
|
+
* the honest statement: build the key in a transform, then join on it, and the
|
|
2824
|
+
* separator is a decision somebody wrote down.
|
|
2825
|
+
* - **No expressions on either side.** `UPPER(key)` looks harmless and is the
|
|
2826
|
+
* single most dangerous thing that could be added, because normalising a key
|
|
2827
|
+
* is a rule about which of two values are "the same value" — see the note on
|
|
2828
|
+
* {@link key} about what happens when only one side is normalised.
|
|
2829
|
+
*
|
|
2830
|
+
* ## Where the reference comes from: an edge, named
|
|
2831
|
+
*
|
|
2832
|
+
* The reference is **another node in this graph**, wired into this one, and
|
|
2833
|
+
* {@link reference} says which of the inbound edges it is. Not a connector on
|
|
2834
|
+
* this node, and not the first inbound edge.
|
|
2835
|
+
*
|
|
2836
|
+
* *Not a connector on this node*, because a source is already a modelled thing
|
|
2837
|
+
* with a kind, an optional named connection, a secret, a mode, a config, schema
|
|
2838
|
+
* discovery and a staging path — and putting a second, smaller copy of all that
|
|
2839
|
+
* inside this node would fork it. Wiring a source in instead means the reference
|
|
2840
|
+
* composes with everything: a `catalog` source reads the **current** snapshot of
|
|
2841
|
+
* a published type, resolved when the run reaches it (see `sourceKind: 'catalog'`),
|
|
2842
|
+
* which is the natural reference and names no physical table; a `sql` source
|
|
2843
|
+
* reads a code table straight out of an operational database; and a filter or a
|
|
2844
|
+
* rename may sit in between, which is how a reference with duplicate keys is
|
|
2845
|
+
* made unambiguous (see below) without this node growing a rule for it.
|
|
2846
|
+
*
|
|
2847
|
+
* *Not the first inbound edge*, and this one is the sharp decision. Edge order
|
|
2848
|
+
* is defined and does decide what a multi-input node receives — but for every
|
|
2849
|
+
* other kind, reordering two wires changes only the order rows are concatenated
|
|
2850
|
+
* in, which is at worst cosmetic. Here it would decide **which side is held
|
|
2851
|
+
* entirely in memory**, and swapping them silently turns a working graph into
|
|
2852
|
+
* one that either holds 44,720 rows to enrich 200, or joins the two datasets the
|
|
2853
|
+
* wrong way round and reports success. Reordering edges is invisible on a canvas.
|
|
2854
|
+
* So the node names its reference by node id, and `validateWorkflow` refuses a
|
|
2855
|
+
* name that is not one of its inbound edges.
|
|
2856
|
+
*
|
|
2857
|
+
* ## The counts, which are the point
|
|
2858
|
+
*
|
|
2859
|
+
* The failure this node was built against is not a crash. flip's SUBWO reader
|
|
2860
|
+
* builds exactly these two maps and reads them per row, and when the reference
|
|
2861
|
+
* table is empty it produces **unenriched rows and a green run** — 44,720 rows
|
|
2862
|
+
* with `planName`, `planDescription` and `unitMel` hard null, no error, no
|
|
2863
|
+
* warning, and a documented seeding prerequisite that nothing checks. A zero-match
|
|
2864
|
+
* join is indistinguishable from a working one by looking at the run.
|
|
2865
|
+
*
|
|
2866
|
+
* So the run log always carries three numbers, whatever {@link unmatched} says:
|
|
2867
|
+
* how many rows matched, how many had a key that matched nothing, and how many
|
|
2868
|
+
* had no key at all. The third is separate from the second on purpose — they
|
|
2869
|
+
* have different causes and different fixes, and flip's reader folds both into
|
|
2870
|
+
* the same NULL. A run where nothing matched gets a line of its own, the way
|
|
2871
|
+
* `filterLogLines` calls out a filter that kept nothing.
|
|
2872
|
+
*
|
|
2873
|
+
* ## The decisions, each made rather than discovered
|
|
2874
|
+
*
|
|
2875
|
+
* - **A key that matches nothing** — {@link unmatched}, defaulting to `null`.
|
|
2876
|
+
* - **Two reference rows for one key** — refused, *when they disagree*. See
|
|
2877
|
+
* {@link fields}.
|
|
2878
|
+
* - **An enriched name the driving row already carries** — filled when it is
|
|
2879
|
+
* empty, refused when it holds a value. The rule is about destroying data
|
|
2880
|
+
* rather than about a name being taken, and the distinction is the whole
|
|
2881
|
+
* usefulness of the node: a published type *declares* the columns it holds, so
|
|
2882
|
+
* a graph reading one back to enrich it receives every one of them. The real
|
|
2883
|
+
* measurement is the argument — `SubwoReplica` hands over 44,720 rows all
|
|
2884
|
+
* carrying `planName`, `planDescription` and `unitMel` with `null` in them,
|
|
2885
|
+
* which is exactly the three columns this node was built to fill. A target
|
|
2886
|
+
* holding an actual value is two columns and one name and fails the node,
|
|
2887
|
+
* naming the row, which is the sentence {@link renameColumnRefusals} says about
|
|
2888
|
+
* the same problem arriving from the other direction.
|
|
2889
|
+
* - **A reference row with no key** — not indexed, and counted. A real work-plan
|
|
2890
|
+
* table has them: flip writes `planId: row.planId ?? ""` when a load has no
|
|
2891
|
+
* plan code, so the empty-string key is in the table by construction and can
|
|
2892
|
+
* never be matched by anything. Refusing the whole reference over one of those
|
|
2893
|
+
* would make the node unusable against the data it was built for; indexing it
|
|
2894
|
+
* silently would let one keyless row become the answer for every keyless
|
|
2895
|
+
* driving row.
|
|
2896
|
+
*/
|
|
2897
|
+
export interface WorkflowLookupNode extends WorkflowNodeBase {
|
|
2898
|
+
kind: 'lookup';
|
|
2899
|
+
/**
|
|
2900
|
+
* The id of the inbound node whose rows are the reference side.
|
|
2901
|
+
*
|
|
2902
|
+
* Held in memory for the whole node; everything else wired in streams past it.
|
|
2903
|
+
* `validateWorkflow` refuses an id that is not one of this node's inbound
|
|
2904
|
+
* edges, and refuses a lookup whose *only* inbound edge is this one — a lookup
|
|
2905
|
+
* with nothing to enrich produces nothing, and would commit an empty snapshot.
|
|
2906
|
+
*/
|
|
2907
|
+
reference: string;
|
|
2908
|
+
/**
|
|
2909
|
+
* The column on the **driving** row holding the key.
|
|
2910
|
+
*
|
|
2911
|
+
* Deliberately unconstrained in spelling, the way a rename's *source* names
|
|
2912
|
+
* are: `Reg Number` and `Mgmt Cd` are what real drops are keyed by.
|
|
2913
|
+
*
|
|
2914
|
+
* ## How two keys are compared, stated once because it is the whole join
|
|
2915
|
+
*
|
|
2916
|
+
* A key is read off the row, and a value of `null` or `undefined` — or a
|
|
2917
|
+
* column that is absent from that row — means the row **has no key**. It is
|
|
2918
|
+
* counted separately and never matches, including against a reference row
|
|
2919
|
+
* that also has no key.
|
|
2920
|
+
*
|
|
2921
|
+
* Anything else is compared **as a string**, by `String(value)`, with no
|
|
2922
|
+
* trimming, no case folding and no other normalisation.
|
|
2923
|
+
*
|
|
2924
|
+
* The coercion is a decision and so is its limit. It is there because the two
|
|
2925
|
+
* sides routinely come from different engines: a work-plan code arriving as a
|
|
2926
|
+
* MySQL `VARCHAR` and the same code arriving as a number out of a spreadsheet
|
|
2927
|
+
* parser are the same key to everyone except `===`, and a join that matched
|
|
2928
|
+
* nothing for that reason is the exact silent-zero this node reports counts to
|
|
2929
|
+
* prevent. What is *not* done is normalising the shape of the value, because
|
|
2930
|
+
* every one of those is a rule about which of somebody's values are the same
|
|
2931
|
+
* value: `"21 CES"` and `"21CES"` are not the same unit unless a person says
|
|
2932
|
+
* so, and flip's own reader is the cautionary tale — it normalises the driving
|
|
2933
|
+
* unit and compares it against a reference column normalised at write time by
|
|
2934
|
+
* a different screen, so the two agree only for as long as nobody edits either.
|
|
2935
|
+
* If a key needs normalising, normalise it in a transform, on both sides,
|
|
2936
|
+
* where it is visible.
|
|
2937
|
+
*/
|
|
2938
|
+
key: string;
|
|
2939
|
+
/** The column on the **reference** row holding the key. Compared as {@link key} describes. */
|
|
2940
|
+
referenceKey: string;
|
|
2941
|
+
/**
|
|
2942
|
+
* Reference column → the name it lands under on the driving row. Never empty;
|
|
2943
|
+
* at most {@link WORKFLOW_LOOKUP_MAX_FIELDS} entries; every target matches
|
|
2944
|
+
* {@link WORKFLOW_FILTER_COLUMN_PATTERN} and no two share one.
|
|
2945
|
+
*
|
|
2946
|
+
* A `Record` rather than a list of pairs, for the reason
|
|
2947
|
+
* {@link WorkflowRenameNode.columns} is one: a key cannot appear twice, so
|
|
2948
|
+
* bringing one reference column across twice is unrepresentable. Two brought
|
|
2949
|
+
* across *onto* one name is representable and refused.
|
|
2950
|
+
*
|
|
2951
|
+
* The target pattern is the identifier rule and it is here for the reason the
|
|
2952
|
+
* rename's targets are: a load looks every field up as `row[name]`, so a name
|
|
2953
|
+
* no property can carry loads NULL into every row and reports success. That is
|
|
2954
|
+
* the failure this node is supposed to be *fixing*, so producing it would be
|
|
2955
|
+
* the trap re-armed one node further along.
|
|
2956
|
+
*
|
|
2957
|
+
* ## Empty is refused rather than treated as a no-op
|
|
2958
|
+
*
|
|
2959
|
+
* A lookup that brings nothing across is a node that draws as configured,
|
|
2960
|
+
* costs a full pass over both sides, and changes nothing — except under `drop`,
|
|
2961
|
+
* where it silently becomes a semi-join that deletes every row whose key is not
|
|
2962
|
+
* in the reference. Two silent opposites reached by deleting the last row of a
|
|
2963
|
+
* form, which is exactly the argument {@link renameColumnRefusals} makes.
|
|
2964
|
+
*
|
|
2965
|
+
* ## Why the fields decide the duplicate-key rule
|
|
2966
|
+
*
|
|
2967
|
+
* Two reference rows for one key means either the join multiplies rows or
|
|
2968
|
+
* something picks a winner, and picking a winner is a rule about whose data
|
|
2969
|
+
* survives — the reasoning {@link renameColumnRefusals} already refused, for
|
|
2970
|
+
* two columns renamed onto one name. Multiplying rows is worse here than it
|
|
2971
|
+
* looks: this node's contract is that it enriches, so a sink downstream would
|
|
2972
|
+
* commit more rows than were read with nothing on the canvas saying why.
|
|
2973
|
+
*
|
|
2974
|
+
* So: **two reference rows for one key are refused when they disagree about
|
|
2975
|
+
* any named field, and collapsed when they agree.** Agreeing costs nobody
|
|
2976
|
+
* anything — there is no winner, the answer is the same either way — and it is
|
|
2977
|
+
* what a real reference table looks like when it has one row per key *and*
|
|
2978
|
+
* something else, which is the common case. Disagreeing fails the node, naming
|
|
2979
|
+
* the key, the field and both values.
|
|
2980
|
+
*
|
|
2981
|
+
* Note what makes this rule cheap and total: it is decided over the **named
|
|
2982
|
+
* fields only**, so two reference rows that differ in a column this node does
|
|
2983
|
+
* not bring across are not a conflict, because nothing about them reaches the
|
|
2984
|
+
* output. And it is decided while the map is built, before a single driving
|
|
2985
|
+
* row is read, so it fails at the start of the node rather than at row ninety
|
|
2986
|
+
* thousand.
|
|
2987
|
+
*
|
|
2988
|
+
* flip's own reader is the argument for refusing rather than choosing. It
|
|
2989
|
+
* builds the plan map with `plansMap.set(plan.planId, plan)`, which keeps the
|
|
2990
|
+
* **last** row; it resolves the unit dictionary with `Array.prototype.find`,
|
|
2991
|
+
* which keeps the **first**; neither key column has a unique constraint; and
|
|
2992
|
+
* the two rules live forty lines apart in one file. Nobody chose either of
|
|
2993
|
+
* them.
|
|
2994
|
+
*/
|
|
2995
|
+
fields: Record<string, string>;
|
|
2996
|
+
/**
|
|
2997
|
+
* What happens to a driving row whose key matches nothing. Absent means `null`.
|
|
2998
|
+
*
|
|
2999
|
+
* `null` is the default because it is the only one of the three that changes
|
|
3000
|
+
* neither which rows exist nor whether the run finishes, so it is the one that
|
|
3001
|
+
* can be the answer for a graph nobody has thought about yet. It is also what
|
|
3002
|
+
* flip does today — and the difference this node makes is not the disposition,
|
|
3003
|
+
* it is that the count is reported rather than left to be discovered by
|
|
3004
|
+
* querying the committed snapshot.
|
|
3005
|
+
*
|
|
3006
|
+
* `drop` is an INNER JOIN and it removes rows, so it is never a default: a
|
|
3007
|
+
* lookup wired in front of a full-mode sink under `drop` shrinks a published
|
|
3008
|
+
* type, which is the accident {@link WorkflowFilterNode.narrows} exists about.
|
|
3009
|
+
*
|
|
3010
|
+
* `fail` is for a reference that is a **documented prerequisite**, which is not
|
|
3011
|
+
* a hypothetical: flip's docs make seeding the unit dictionary a prerequisite
|
|
3012
|
+
* of MEL, MVR and SUBWO, and an unseeded one yields unnormalized rows rather
|
|
3013
|
+
* than an error. A load that cannot be enriched is a load that should not
|
|
3014
|
+
* commit, and this is how somebody says so.
|
|
3015
|
+
*/
|
|
3016
|
+
unmatched?: WorkflowLookupUnmatched;
|
|
3017
|
+
}
|
|
3018
|
+
/** {@link WorkflowLookupNode.unmatched}, resolved. One reader of the default. */
|
|
3019
|
+
export declare function workflowLookupUnmatched(node: WorkflowLookupNode): WorkflowLookupUnmatched;
|
|
3020
|
+
/**
|
|
3021
|
+
* A key, as this node compares them, or `undefined` for a row that has none.
|
|
3022
|
+
*
|
|
3023
|
+
* One function, exported, and called by the runner for both sides — because the
|
|
3024
|
+
* one way a join goes silently wrong is the two sides being read by two pieces
|
|
3025
|
+
* of code that agree today. See {@link WorkflowLookupNode.key} for what it does
|
|
3026
|
+
* and, more to the point, what it deliberately does not do.
|
|
3027
|
+
*/
|
|
3028
|
+
export declare function workflowLookupKey(value: unknown): string | undefined;
|
|
3029
|
+
/**
|
|
3030
|
+
* Every reason a lookup's configuration cannot be stored, as sentences, or empty.
|
|
3031
|
+
*
|
|
3032
|
+
* One function, called by {@link validateWorkflow}, by the HTTP boundary and by
|
|
3033
|
+
* the canvas, for the reason {@link renameColumnRefusals} is: a screen that
|
|
3034
|
+
* checked a target name against its own copy of the pattern is a screen that
|
|
3035
|
+
* eventually accepts something the server refuses, halfway through a save.
|
|
3036
|
+
*
|
|
3037
|
+
* All of them rather than the first, for the reason `refuseUnpublishablePropertyNames`
|
|
3038
|
+
* gives: a form filled in one sitting is usually wrong about several things in
|
|
3039
|
+
* the same way.
|
|
3040
|
+
*
|
|
3041
|
+
* The *wiring* rules — that {@link WorkflowLookupNode.reference} names an inbound
|
|
3042
|
+
* edge, and that something other than the reference is wired in — are not here,
|
|
3043
|
+
* and that is not an omission. They are facts about the graph rather than about
|
|
3044
|
+
* the node, so they cannot be answered from the node alone; `validateWorkflow`
|
|
3045
|
+
* owns them and the inspector reads them from `validateWorkflow`.
|
|
3046
|
+
*/
|
|
3047
|
+
export declare function lookupConfigRefusals(node: {
|
|
3048
|
+
key?: unknown;
|
|
3049
|
+
referenceKey?: unknown;
|
|
3050
|
+
fields?: Record<string, string>;
|
|
3051
|
+
}): string[];
|
|
2713
3052
|
/**
|
|
2714
3053
|
* Whether a stored aggregate list is one this build can run.
|
|
2715
3054
|
*
|
|
@@ -2719,13 +3058,25 @@ export declare function aggregateRefusals(node: {
|
|
|
2719
3058
|
* under a name somebody put in an object type on purpose.
|
|
2720
3059
|
*/
|
|
2721
3060
|
export declare function isWorkflowAggregates(value: unknown): value is WorkflowAggregate[];
|
|
3061
|
+
/**
|
|
3062
|
+
* Whether a stored field map is one this build can run.
|
|
3063
|
+
*
|
|
3064
|
+
* Refused rather than repaired, the stance {@link isWorkflowRenameColumns} takes
|
|
3065
|
+
* and for the same reason: a map read back with one entry silently dropped is a
|
|
3066
|
+
* graph that commits a column of NULLs under a name nobody can now explain.
|
|
3067
|
+
*
|
|
3068
|
+
* `Object.entries` rather than a `for…in`, so an inherited key cannot enter the
|
|
3069
|
+
* map. The key columns are not checked here because they are not this value;
|
|
3070
|
+
* {@link lookupConfigRefusals} is what sees the whole node.
|
|
3071
|
+
*/
|
|
3072
|
+
export declare function isWorkflowLookupFields(value: unknown): value is Record<string, string>;
|
|
2722
3073
|
/**
|
|
2723
3074
|
* A discriminated union, so narrowing a node is `node.kind === "sink"` and
|
|
2724
3075
|
* never a type assertion. This is why the kind list is not simply a string on
|
|
2725
3076
|
* one node shape with every field optional: that shape lets a source node carry
|
|
2726
3077
|
* a `transformId` and nothing catches it.
|
|
2727
3078
|
*/
|
|
2728
|
-
export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode | WorkflowRenameNode | WorkflowAggregateNode;
|
|
3079
|
+
export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode | WorkflowRenameNode | WorkflowAggregateNode | WorkflowLookupNode;
|
|
2729
3080
|
/**
|
|
2730
3081
|
* The node kinds that can be saved once and used in several graphs.
|
|
2731
3082
|
*
|
|
@@ -2789,6 +3140,11 @@ export declare function isReusableNodeKind(value: unknown): value is ReusableNod
|
|
|
2789
3140
|
* its *output* column set is the thing downstream nodes are validated against,
|
|
2790
3141
|
* so a shared node editable from elsewhere would silently change what another
|
|
2791
3142
|
* graph's sink is allowed to write.
|
|
3143
|
+
* - `lookup` — that argument, and one that is not an argument at all but an
|
|
3144
|
+
* impossibility: {@link WorkflowLookupNode.reference} is **a node id in this
|
|
3145
|
+
* graph**. A shared body carrying one would name a node the adopting graph has
|
|
3146
|
+
* never had, and it is not a cosmetic field — it is the one that decides which
|
|
3147
|
+
* side of the join is held in memory.
|
|
2792
3148
|
*/
|
|
2793
3149
|
export declare const NODE_KIND_IS_REUSABLE: {
|
|
2794
3150
|
readonly source: true;
|
|
@@ -2799,6 +3155,7 @@ export declare const NODE_KIND_IS_REUSABLE: {
|
|
|
2799
3155
|
readonly filter: false;
|
|
2800
3156
|
readonly rename: false;
|
|
2801
3157
|
readonly aggregate: false;
|
|
3158
|
+
readonly lookup: false;
|
|
2802
3159
|
};
|
|
2803
3160
|
/** Whether this kind can be saved as a reusable node. Reads {@link NODE_KIND_IS_REUSABLE}. */
|
|
2804
3161
|
export declare function nodeKindIsReusable(kind: WorkflowNodeKind): boolean;
|
|
@@ -3597,7 +3954,7 @@ export interface CallableWorkflowBlock {
|
|
|
3597
3954
|
}
|
|
3598
3955
|
export declare function callableWorkflowBlock(ref: CallableWorkflowRef): CallableWorkflowBlock | undefined;
|
|
3599
3956
|
/** Every way a graph can be refused. Exported so a canvas can key off the code. */
|
|
3600
|
-
export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "source-type-not-named", "call-not-named", "call-plain-has-output", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing", "rename-invalid", "aggregate-invalid", "column-not-produced", "version-pin-invalid"];
|
|
3957
|
+
export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "source-type-not-named", "call-not-named", "call-plain-has-output", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing", "rename-invalid", "aggregate-invalid", "lookup-invalid", "lookup-reference-not-wired", "lookup-nothing-to-enrich", "column-not-produced", "version-pin-invalid"];
|
|
3601
3958
|
export type WorkflowIssueCode = (typeof WORKFLOW_ISSUE_CODES)[number];
|
|
3602
3959
|
export interface WorkflowValidationIssue {
|
|
3603
3960
|
code: WorkflowIssueCode;
|
|
@@ -3830,7 +4187,39 @@ export declare function workflowFilterColumns(predicate: WorkflowFilterPredicate
|
|
|
3830
4187
|
* cyclic graph before it gets here, but the canvas calls this while a graph is
|
|
3831
4188
|
* being drawn and is entitled to a wrong-but-terminating answer.
|
|
3832
4189
|
*/
|
|
3833
|
-
export declare function workflowKnownColumns(graph: WorkflowGraph, nodeId: string, knowledge?: WorkflowColumnKnowledge): ReadonlySet<string> | undefined;
|
|
4190
|
+
export declare function workflowKnownColumns(graph: WorkflowGraph, nodeId: string, knowledge?: WorkflowColumnKnowledge, onlyFrom?: WorkflowInputFilter): ReadonlySet<string> | undefined;
|
|
4191
|
+
/**
|
|
4192
|
+
* Which of a node's inbound edges a question is about.
|
|
4193
|
+
*
|
|
4194
|
+
* One node kind needs this and it is {@link WorkflowLookupNode}, which is the
|
|
4195
|
+
* only one whose inputs are not interchangeable: the reference side is held as a
|
|
4196
|
+
* map and its columns do **not** flow on, so the union of everything wired in is
|
|
4197
|
+
* the wrong answer to "what does this node pass down". Without the distinction
|
|
4198
|
+
* the walk would report the reference's columns as available downstream, and a
|
|
4199
|
+
* filter naming one of them would be accepted by the validator and then match no
|
|
4200
|
+
* row at run time — the precise silent failure `checkColumnsProduced` exists to
|
|
4201
|
+
* catch, produced by the check itself.
|
|
4202
|
+
*
|
|
4203
|
+
* Optional everywhere it appears, and omitting it means every inbound edge —
|
|
4204
|
+
* which is what every other kind wants and what every existing caller gets.
|
|
4205
|
+
*/
|
|
4206
|
+
export type WorkflowInputFilter = (fromNodeId: string) => boolean;
|
|
4207
|
+
/**
|
|
4208
|
+
* The two column sets a lookup sees, told apart.
|
|
4209
|
+
*
|
|
4210
|
+
* Exported because three callers need the same split and each one getting it
|
|
4211
|
+
* right separately is how they come to disagree: the validator refuses a key
|
|
4212
|
+
* column that is not on the driving side, the walk answers what the node passes
|
|
4213
|
+
* on, and the inspector says both out loud on the screen where the columns are
|
|
4214
|
+
* typed.
|
|
4215
|
+
*
|
|
4216
|
+
* Either side answers `undefined` for the ordinary reason — see
|
|
4217
|
+
* {@link workflowKnownColumns} — and `undefined` must not be read as empty.
|
|
4218
|
+
*/
|
|
4219
|
+
export declare function workflowLookupColumns(graph: WorkflowGraph, node: WorkflowLookupNode, knowledge?: WorkflowColumnKnowledge): {
|
|
4220
|
+
driving: ReadonlySet<string> | undefined;
|
|
4221
|
+
reference: ReadonlySet<string> | undefined;
|
|
4222
|
+
};
|
|
3834
4223
|
/**
|
|
3835
4224
|
* What a caller can tell the column walk that the graph does not hold.
|
|
3836
4225
|
*
|