@dudousxd/nestjs-catalog 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.pipeline.d.ts +393 -8
- package/dist/catalog.pipeline.js +438 -1
- package/dist/catalog.stage-encoding.d.ts +72 -0
- package/dist/catalog.stage-encoding.js +100 -0
- package/dist/client.d.ts +3 -2
- package/dist/client.js +28 -2
- package/dist/index.d.ts +3 -2
- package/dist/index.js +21 -3
- package/dist/transform-runner.d.ts +7 -1
- package/dist/transform-runner.js +184 -26
- package/dist/transform-shape.d.ts +106 -0
- package/dist/transform-shape.js +419 -0
- package/package.json +1 -1
|
@@ -446,18 +446,39 @@ export interface CatalogTransform {
|
|
|
446
446
|
description?: string;
|
|
447
447
|
language: TransformLanguage;
|
|
448
448
|
/**
|
|
449
|
-
*
|
|
450
|
-
* `context`, and returns the rows to store.
|
|
449
|
+
* A function over one batch, in either of two shapes.
|
|
451
450
|
*
|
|
452
451
|
* A batch rather than a record at a time: a transform that needs to look up,
|
|
453
452
|
* deduplicate or aggregate cannot do it one row at a time, and paying one
|
|
454
453
|
* process spawn per record would make any real load unusable.
|
|
455
454
|
*
|
|
455
|
+
* **The supported shape** is a module exporting a function that takes one
|
|
456
|
+
* object — a {@link CatalogTransformInput} — and returns the rows to store:
|
|
457
|
+
*
|
|
458
|
+
* ```js
|
|
459
|
+
* export default function transform({ records, context }) {
|
|
460
|
+
* return records.map((r) => ({ mgmtCd: r["Mgmt Cd"] }));
|
|
461
|
+
* }
|
|
462
|
+
* ```
|
|
463
|
+
*
|
|
464
|
+
* One object rather than positional parameters, because the object is the
|
|
465
|
+
* only shape that can gain a field later. `context` arrived as a second
|
|
466
|
+
* positional parameter and got away with it; a third would have redefined
|
|
467
|
+
* what every signature already written means.
|
|
468
|
+
*
|
|
469
|
+
* **The bare-body shape** — the text between a function's braces, with
|
|
470
|
+
* `records` and `context` simply in scope — is what every transform stored
|
|
471
|
+
* before that shape existed is written in, and it keeps running byte for
|
|
472
|
+
* byte: same wrapper, same interpreter flags, same everything. See
|
|
473
|
+
* `transform-shape.ts` for how the two are told apart and why the rule cannot
|
|
474
|
+
* misread one for the other.
|
|
475
|
+
*
|
|
456
476
|
* `context` is a {@link CatalogCodeContext} — the run, the node, the counts
|
|
457
477
|
* of what fed it, and the environment variables this deployment admits.
|
|
458
|
-
*
|
|
459
|
-
*
|
|
460
|
-
*
|
|
478
|
+
*
|
|
479
|
+
* Python has neither shape and needs neither: its harness writes the `def`
|
|
480
|
+
* itself, so a Python transform is a body that never states a signature, and
|
|
481
|
+
* a new field costs one generated line rather than an edit to stored code.
|
|
461
482
|
*/
|
|
462
483
|
code: string;
|
|
463
484
|
version: number;
|
|
@@ -465,6 +486,77 @@ export interface CatalogTransform {
|
|
|
465
486
|
createdAt: string;
|
|
466
487
|
updatedAt: string;
|
|
467
488
|
}
|
|
489
|
+
/**
|
|
490
|
+
* The single argument a module-shaped transform is called with.
|
|
491
|
+
*
|
|
492
|
+
* ## Why one object
|
|
493
|
+
*
|
|
494
|
+
* So that the next thing a transform needs can be added without changing what
|
|
495
|
+
* any existing transform's signature means. Positional parameters spend that
|
|
496
|
+
* option the first time they are used: `(records, context)` fixed the list at
|
|
497
|
+
* two, and a third would silently redefine every signature ever written —
|
|
498
|
+
* including the ones in a database somewhere that nobody will re-read. A field
|
|
499
|
+
* on an object is additive by construction, and a transform that never names it
|
|
500
|
+
* is untouched by it.
|
|
501
|
+
*
|
|
502
|
+
* ## What is on it, and what is not
|
|
503
|
+
*
|
|
504
|
+
* {@link records} and {@link context}, and deliberately nothing else yet.
|
|
505
|
+
*
|
|
506
|
+
* - **No `log`.** Python's harness has one, because Python's `print` used to go
|
|
507
|
+
* nowhere; JavaScript's `console.log` — and `info`, `warn`, `error`, `debug`,
|
|
508
|
+
* `trace` — is already captured in call order. A second spelling that worked
|
|
509
|
+
* only in the new shape would split the idiom for no gain.
|
|
510
|
+
* - **No `env` shortcut.** It is already `context.env`, filtered by the same
|
|
511
|
+
* credential allow-list that governs connectors. Two paths to one value is
|
|
512
|
+
* how the two come to disagree.
|
|
513
|
+
* - **No `signal`.** The timeout is a `SIGKILL` to the whole process group;
|
|
514
|
+
* there is nothing for user code to cooperate with, and an `AbortSignal` that
|
|
515
|
+
* never fires would be a promise the runner cannot keep.
|
|
516
|
+
*
|
|
517
|
+
* The point of the object is that each of those can be reconsidered later
|
|
518
|
+
* without a migration. That is the argument, not the current field list.
|
|
519
|
+
*
|
|
520
|
+
* @typeParam TRecord - what one inbound record looks like. Editor help only:
|
|
521
|
+
* types are erased before the code runs, so a wrong one is a squiggle, never a
|
|
522
|
+
* failed run. See {@link CatalogTransformFunction}.
|
|
523
|
+
*/
|
|
524
|
+
export interface CatalogTransformInput<TRecord = Record<string, unknown>> {
|
|
525
|
+
/** The batch, exactly as the source produced it. */
|
|
526
|
+
records: TRecord[];
|
|
527
|
+
/** The run, the node, the counts, and the admitted environment variables. */
|
|
528
|
+
context: CatalogCodeContext;
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* The function a module-shaped transform exports, as `export default` or as a
|
|
532
|
+
* named export called `transform`.
|
|
533
|
+
*
|
|
534
|
+
* **This type is for the editor and for nothing else.** TypeScript transforms
|
|
535
|
+
* run through Node's own type *stripping* — the annotations are erased on the
|
|
536
|
+
* way in and never checked, by this runner or by anything else — so a transform
|
|
537
|
+
* whose types are wrong runs anyway, and produces exactly the rows its code
|
|
538
|
+
* produces. What the type buys is completion on `records` and `context` while
|
|
539
|
+
* writing, and a red underline in an editor that happens to be type-aware. What
|
|
540
|
+
* it does not buy is a single guarantee at run time; the try pane is what
|
|
541
|
+
* catches a mistake.
|
|
542
|
+
*
|
|
543
|
+
* Referencing it costs nothing at run time either, and that is a property of
|
|
544
|
+
* `import type` specifically: the stripper erases the whole statement, so
|
|
545
|
+
* nothing tries to resolve `@dudousxd/nestjs-catalog/client` inside a child
|
|
546
|
+
* process that has no `node_modules` to resolve it in. A *value* import of the
|
|
547
|
+
* same module would fail — there is no package to find from the temporary
|
|
548
|
+
* directory a transform runs in.
|
|
549
|
+
*
|
|
550
|
+
* ```ts
|
|
551
|
+
* import type { CatalogTransformFunction } from '@dudousxd/nestjs-catalog/client';
|
|
552
|
+
*
|
|
553
|
+
* const transform: CatalogTransformFunction<{ 'Mgmt Cd': string }> = ({ records }) =>
|
|
554
|
+
* records.map((r) => ({ mgmtCd: r['Mgmt Cd'] }));
|
|
555
|
+
*
|
|
556
|
+
* export default transform;
|
|
557
|
+
* ```
|
|
558
|
+
*/
|
|
559
|
+
export type CatalogTransformFunction<TRecord = Record<string, unknown>> = (input: CatalogTransformInput<TRecord>) => Array<Record<string, unknown>> | Promise<Array<Record<string, unknown>>>;
|
|
468
560
|
export interface TransformResult {
|
|
469
561
|
rows: Array<Record<string, unknown>>;
|
|
470
562
|
/**
|
|
@@ -826,7 +918,7 @@ export interface WorkflowNodeOutcome {
|
|
|
826
918
|
* outside a run, which is why `call` names one and not a step. If a step is
|
|
827
919
|
* what you want, the thing to call is a one-step workflow wrapping it.
|
|
828
920
|
*/
|
|
829
|
-
export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter"];
|
|
921
|
+
export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter", "rename"];
|
|
830
922
|
export type WorkflowNodeKind = (typeof WORKFLOW_NODE_KINDS)[number];
|
|
831
923
|
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
832
924
|
export declare function isWorkflowNodeKind(value: unknown): value is WorkflowNodeKind;
|
|
@@ -1784,13 +1876,196 @@ export interface WorkflowFilterNode extends WorkflowNodeBase {
|
|
|
1784
1876
|
*/
|
|
1785
1877
|
narrows?: string[];
|
|
1786
1878
|
}
|
|
1879
|
+
/**
|
|
1880
|
+
* What happens to a column the rename does not name.
|
|
1881
|
+
*
|
|
1882
|
+
* Two words rather than a boolean, because the two are genuinely different
|
|
1883
|
+
* nodes and a boolean called `drop` would read as a modifier on one node. See
|
|
1884
|
+
* {@link WorkflowRenameNode.unnamed} for what each costs.
|
|
1885
|
+
*/
|
|
1886
|
+
export declare const WORKFLOW_RENAME_UNNAMED: readonly ["keep", "drop"];
|
|
1887
|
+
export type WorkflowRenameUnnamed = (typeof WORKFLOW_RENAME_UNNAMED)[number];
|
|
1888
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
1889
|
+
export declare function isWorkflowRenameUnnamed(value: unknown): value is WorkflowRenameUnnamed;
|
|
1890
|
+
/**
|
|
1891
|
+
* The exhaustiveness guard for {@link WORKFLOW_RENAME_UNNAMED}.
|
|
1892
|
+
*
|
|
1893
|
+
* {@link unreachableNodeKind}, one level down, and for the identical reason: the
|
|
1894
|
+
* two words decide whether a batch is rewritten or only re-labelled, and a third
|
|
1895
|
+
* one added without a branch would silently pick whichever the last `if` was.
|
|
1896
|
+
*/
|
|
1897
|
+
export declare function unreachableRenameUnnamed(value: never, where: string): never;
|
|
1898
|
+
/**
|
|
1899
|
+
* How many columns one rename may name.
|
|
1900
|
+
*
|
|
1901
|
+
* The same argument {@link WORKFLOW_FILTER_MAX_VALUES} makes: the map travels in
|
|
1902
|
+
* the graph and into the graph fingerprint, and past a few hundred entries the
|
|
1903
|
+
* thing being expressed is a schema mapping that belongs in a stored object
|
|
1904
|
+
* rather than in a node. It is also the bound that keeps
|
|
1905
|
+
* {@link isWorkflowRenameColumns} — which is run on JSON out of a column — from
|
|
1906
|
+
* being a place to hand a service a million-key object.
|
|
1907
|
+
*/
|
|
1908
|
+
export declare const WORKFLOW_RENAME_MAX_COLUMNS = 500;
|
|
1909
|
+
/**
|
|
1910
|
+
* Renames columns, and does nothing else, ever.
|
|
1911
|
+
*
|
|
1912
|
+
* ## Why this is a node kind and not a flag, and why it stays small
|
|
1913
|
+
*
|
|
1914
|
+
* The generic {@link WorkflowTransformNode} continues to exist for everything
|
|
1915
|
+
* else, and **that is what lets this node stay deliberately narrow**. The usual
|
|
1916
|
+
* objection to a declarative shortcut is that it grows — rename, then cast, then
|
|
1917
|
+
* default, then trim, and then it is a small language nobody designed and
|
|
1918
|
+
* everybody has to learn. With a real code node sitting beside it, the answer to
|
|
1919
|
+
* "I need more than renaming" is always *use a transform*, and never *add a
|
|
1920
|
+
* field here*. That is the constraint that keeps this honest, and it is the
|
|
1921
|
+
* reason to refuse the next field rather than a reason to feel bad about
|
|
1922
|
+
* refusing it.
|
|
1923
|
+
*
|
|
1924
|
+
* ## What it buys, measured rather than asserted
|
|
1925
|
+
*
|
|
1926
|
+
* **1. It streams by construction.** A rename is per record. It cannot
|
|
1927
|
+
* aggregate, deduplicate, sort or look at a neighbour, so there is no batch it
|
|
1928
|
+
* has to hold. A transform node cannot make that promise — an author's function
|
|
1929
|
+
* is handed the whole batch and may legitimately reduce over it — which is why
|
|
1930
|
+
* `ConnectorRunnerService` has to log *"Held all N records in memory"* when a
|
|
1931
|
+
* transform is present. This node never contributes that line.
|
|
1932
|
+
*
|
|
1933
|
+
* **2. It needs no child process.** A transform round trip for 103,087 rows
|
|
1934
|
+
* costs ~338 ms, of which the author's `.map` is ~5 ms. The transport — encode,
|
|
1935
|
+
* write, decode, run, encode, read — is the entire bill. A rename does not need
|
|
1936
|
+
* transport, and the same rename in process is ~14 ms.
|
|
1937
|
+
*
|
|
1938
|
+
* **3. On staged data it is metadata-only, and exactly when.** A staged batch is
|
|
1939
|
+
* a shape dictionary (see `catalog.stage-encoding.ts`): `shapes` holds each
|
|
1940
|
+
* distinct key-set once, `shapeOf[i]` indexes it, and `values[i]` is a
|
|
1941
|
+
* positional array parallel to that shape. A positional array does not care what
|
|
1942
|
+
* the key is called, so renaming a key is a rewrite of `shapes` and nothing
|
|
1943
|
+
* else — tens of strings for a hundred thousand rows.
|
|
1944
|
+
*
|
|
1945
|
+
* **That last claim holds for `unnamed: 'keep'` and does not hold for
|
|
1946
|
+
* `unnamed: 'drop'`.** Dropping a column removes a position, so every `values`
|
|
1947
|
+
* row has to be rebuilt and the cost is back to O(rows). Both are worth having
|
|
1948
|
+
* and they are not the same operation, so the node says which one it did:
|
|
1949
|
+
* `renameStagePayload` reports `metadataOnly`, and the run log prints it.
|
|
1950
|
+
*
|
|
1951
|
+
* ## The config
|
|
1952
|
+
*
|
|
1953
|
+
* {@link columns} is a map of **old name → new name**, applied
|
|
1954
|
+
* **simultaneously** rather than in sequence. `{"a": "b", "b": "c"}` maps `a` to
|
|
1955
|
+
* `b` and `b` to `c`; it does not chain `a → b → c`. Sequential application
|
|
1956
|
+
* would make the result depend on the iteration order of a JSON object, which is
|
|
1957
|
+
* not a thing to build a load on.
|
|
1958
|
+
*
|
|
1959
|
+
* A `Record` rather than a list of pairs, and the reason is a refusal it buys
|
|
1960
|
+
* for free: a key cannot appear twice in an object, so *two renames of the same
|
|
1961
|
+
* source column* is unrepresentable. The mirror mistake — two renames **onto**
|
|
1962
|
+
* the same target — is representable and is refused by `validateWorkflow`, which
|
|
1963
|
+
* can see it from the config alone with nothing to run.
|
|
1964
|
+
*
|
|
1965
|
+
* The four remaining edge cases, all decided rather than discovered:
|
|
1966
|
+
*
|
|
1967
|
+
* - **A column the map does not name** — {@link unnamed} decides, and it
|
|
1968
|
+
* defaults to `keep`.
|
|
1969
|
+
* - **A named column that is not in a given record** — nothing happens to that
|
|
1970
|
+
* record. This is not an error, because a batch legitimately holds rows with
|
|
1971
|
+
* different key-sets; that is the entire reason the stage encoding is a shape
|
|
1972
|
+
* *dictionary* and not one column list. A source column that turns out to be
|
|
1973
|
+
* in **no** row of the whole run is reported loudly in the run log, because it
|
|
1974
|
+
* is almost always a typo in a header and the symptom otherwise is a column of
|
|
1975
|
+
* NULLs and a green run.
|
|
1976
|
+
* - **A rename onto a name the record already holds** — refused, at run time,
|
|
1977
|
+
* naming both columns. There are two columns and one name, and every rule for
|
|
1978
|
+
* picking a winner is arbitrary. It is detected per *shape* rather than per
|
|
1979
|
+
* row, so it fails on the first batch rather than at row ninety thousand.
|
|
1980
|
+
* Under `unnamed: 'drop'` there is no collision to have: a column the map does
|
|
1981
|
+
* not name does not exist in the output, so it cannot occupy anything.
|
|
1982
|
+
* - **`a → a`** — allowed. Under `keep` it is a no-op; under `drop` it is how a
|
|
1983
|
+
* column is *selected*, which is a real use.
|
|
1984
|
+
*
|
|
1985
|
+
* ## What the target names have to be
|
|
1986
|
+
*
|
|
1987
|
+
* {@link WORKFLOW_FILTER_COLUMN_PATTERN}, checked at authoring time. Two
|
|
1988
|
+
* separate reasons, and both are about a failure that reports success:
|
|
1989
|
+
*
|
|
1990
|
+
* - `property-names.ts` refuses a *published property* whose name cannot become
|
|
1991
|
+
* a column, and its docblock is the record of what happens when a name and the
|
|
1992
|
+
* key in the record disagree: the load looks every field up as `row[name]`, so
|
|
1993
|
+
* the column takes NULL in every row and the run is green. Thirteen types went
|
|
1994
|
+
* in that way. A rename is the tool that makes the record's key match the
|
|
1995
|
+
* property, so a rename that produces a name no property can carry is the
|
|
1996
|
+
* trap re-armed one node upstream.
|
|
1997
|
+
* - The filter node's columns follow the same pattern precisely so a predicate
|
|
1998
|
+
* could one day be pushed into a `WHERE`. A rename whose target cannot be
|
|
1999
|
+
* named by a filter would author a graph today that could never be pushed down
|
|
2000
|
+
* tomorrow.
|
|
2001
|
+
*
|
|
2002
|
+
* The *source* names are deliberately unconstrained. `Mgmt Cd`, `VEH Type Name`
|
|
2003
|
+
* and `Reg Number` are exactly what real drops are keyed by, and being able to
|
|
2004
|
+
* name them is the entire point of the node.
|
|
2005
|
+
*/
|
|
2006
|
+
export interface WorkflowRenameNode extends WorkflowNodeBase {
|
|
2007
|
+
kind: 'rename';
|
|
2008
|
+
/**
|
|
2009
|
+
* Old name → new name, applied simultaneously. Never empty; at most
|
|
2010
|
+
* {@link WORKFLOW_RENAME_MAX_COLUMNS} entries; every target matches
|
|
2011
|
+
* {@link WORKFLOW_FILTER_COLUMN_PATTERN} and no two share one.
|
|
2012
|
+
*
|
|
2013
|
+
* Empty is refused rather than treated as a no-op, for the reason an empty
|
|
2014
|
+
* filter group is: under `keep` it is a node that draws as configured and does
|
|
2015
|
+
* nothing, and under `drop` it deletes every column of every row and commits
|
|
2016
|
+
* the result. Two silent opposites reached by deleting the last row of a form.
|
|
2017
|
+
*/
|
|
2018
|
+
columns: Record<string, string>;
|
|
2019
|
+
/**
|
|
2020
|
+
* What happens to the columns this node does not name. Absent means `keep`.
|
|
2021
|
+
*
|
|
2022
|
+
* Absent is `keep` rather than being required, because `keep` is the node this
|
|
2023
|
+
* one is called after: a rename that also deleted everything it did not
|
|
2024
|
+
* mention would be a projection wearing the word "rename", and somebody would
|
|
2025
|
+
* find that out by looking at a committed snapshot.
|
|
2026
|
+
*
|
|
2027
|
+
* `drop` is here because the shape it replaces is real —
|
|
2028
|
+
* `records.map((r) => ({ mgmtCd: r["Mgmt Cd"] }))` is a rename *and* a
|
|
2029
|
+
* projection, and it is what a drop of Air Force fleet data forces today. It
|
|
2030
|
+
* costs the metadata-only property (see the docblock above), and the run says
|
|
2031
|
+
* so rather than leaving the difference to be guessed at.
|
|
2032
|
+
*/
|
|
2033
|
+
unnamed?: WorkflowRenameUnnamed;
|
|
2034
|
+
}
|
|
2035
|
+
/** {@link WorkflowRenameNode.unnamed}, resolved. One reader of the default. */
|
|
2036
|
+
export declare function workflowRenameUnnamed(node: WorkflowRenameNode): WorkflowRenameUnnamed;
|
|
2037
|
+
/**
|
|
2038
|
+
* Every reason a rename map cannot be stored, as sentences, or empty.
|
|
2039
|
+
*
|
|
2040
|
+
* One function, called by {@link validateWorkflow}, by the HTTP boundary and by
|
|
2041
|
+
* the canvas, for the reason `validateWorkflow` itself is shared: a screen that
|
|
2042
|
+
* checked a target name against its own copy of the pattern is a screen that
|
|
2043
|
+
* eventually accepts something the server refuses, halfway through a save.
|
|
2044
|
+
*
|
|
2045
|
+
* All of them rather than the first, exactly as
|
|
2046
|
+
* {@link refuseUnpublishablePropertyNames} argues: a map of forty columns typed
|
|
2047
|
+
* in one sitting is usually wrong about several in the same way.
|
|
2048
|
+
*/
|
|
2049
|
+
export declare function renameColumnRefusals(columns: Record<string, string>): string[];
|
|
2050
|
+
/**
|
|
2051
|
+
* Whether a stored rename map is one this build can run.
|
|
2052
|
+
*
|
|
2053
|
+
* Refused rather than repaired, the stance {@link isWorkflowFilterPredicate}
|
|
2054
|
+
* takes about a predicate and for the same reason one step further along: a
|
|
2055
|
+
* rename read back with one entry silently dropped is a graph that commits a
|
|
2056
|
+
* column of NULLs under a name nobody can now explain.
|
|
2057
|
+
*
|
|
2058
|
+
* `Object.entries` rather than a `for…in`, so an inherited key cannot enter the
|
|
2059
|
+
* map, and the values are checked one by one rather than trusted from the type.
|
|
2060
|
+
*/
|
|
2061
|
+
export declare function isWorkflowRenameColumns(value: unknown): value is Record<string, string>;
|
|
1787
2062
|
/**
|
|
1788
2063
|
* A discriminated union, so narrowing a node is `node.kind === "sink"` and
|
|
1789
2064
|
* never a type assertion. This is why the kind list is not simply a string on
|
|
1790
2065
|
* one node shape with every field optional: that shape lets a source node carry
|
|
1791
2066
|
* a `transformId` and nothing catches it.
|
|
1792
2067
|
*/
|
|
1793
|
-
export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode;
|
|
2068
|
+
export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode | WorkflowRenameNode;
|
|
1794
2069
|
/**
|
|
1795
2070
|
* The node kinds that can be saved once and used in several graphs.
|
|
1796
2071
|
*
|
|
@@ -1842,6 +2117,11 @@ export declare function isReusableNodeKind(value: unknown): value is ReusableNod
|
|
|
1842
2117
|
* may not have, and a filter is worse: {@link WorkflowFilterNode.narrows} is
|
|
1843
2118
|
* an acknowledgement about *this* graph's sinks, so a shared one would carry
|
|
1844
2119
|
* somebody else's acknowledgement into a graph they never saw.
|
|
2120
|
+
* - `rename` — the same argument as `if` and `filter`, and the sharpest version
|
|
2121
|
+
* of it: a rename map names the source's own spelling of its columns, so it is
|
|
2122
|
+
* *about* one drop of one file. `Mgmt Cd → mgmtCd` saved under a name and
|
|
2123
|
+
* dropped into a graph reading a different system renames nothing at all, and
|
|
2124
|
+
* the symptom is a column of NULLs rather than a failure.
|
|
1845
2125
|
*/
|
|
1846
2126
|
export declare const NODE_KIND_IS_REUSABLE: {
|
|
1847
2127
|
readonly source: true;
|
|
@@ -1850,6 +2130,7 @@ export declare const NODE_KIND_IS_REUSABLE: {
|
|
|
1850
2130
|
readonly call: false;
|
|
1851
2131
|
readonly if: false;
|
|
1852
2132
|
readonly filter: false;
|
|
2133
|
+
readonly rename: false;
|
|
1853
2134
|
};
|
|
1854
2135
|
/** Whether this kind can be saved as a reusable node. Reads {@link NODE_KIND_IS_REUSABLE}. */
|
|
1855
2136
|
export declare function nodeKindIsReusable(kind: WorkflowNodeKind): boolean;
|
|
@@ -2648,7 +2929,7 @@ export interface CallableWorkflowBlock {
|
|
|
2648
2929
|
}
|
|
2649
2930
|
export declare function callableWorkflowBlock(ref: CallableWorkflowRef): CallableWorkflowBlock | undefined;
|
|
2650
2931
|
/** Every way a graph can be refused. Exported so a canvas can key off the code. */
|
|
2651
|
-
export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "call-not-named", "call-plain-has-output", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing", "version-pin-invalid"];
|
|
2932
|
+
export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "call-not-named", "call-plain-has-output", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing", "rename-invalid", "column-not-produced", "version-pin-invalid"];
|
|
2652
2933
|
export type WorkflowIssueCode = (typeof WORKFLOW_ISSUE_CODES)[number];
|
|
2653
2934
|
export interface WorkflowValidationIssue {
|
|
2654
2935
|
code: WorkflowIssueCode;
|
|
@@ -2804,6 +3085,62 @@ export declare function workflowNodeRuns(entry: {
|
|
|
2804
3085
|
* a thing to plan for.
|
|
2805
3086
|
*/
|
|
2806
3087
|
export declare function workflowGraphHash(graph: WorkflowGraph): string;
|
|
3088
|
+
/**
|
|
3089
|
+
* Every column a filter predicate names, once each, in the order they appear.
|
|
3090
|
+
*
|
|
3091
|
+
* Its own function rather than a walk inlined into the validator, because two
|
|
3092
|
+
* things want it — the refusal below and anything on a screen that wants to say
|
|
3093
|
+
* which columns a node depends on — and a second copy of a tree walk is a second
|
|
3094
|
+
* copy that forgets the `oneOf` branch.
|
|
3095
|
+
*/
|
|
3096
|
+
export declare function workflowFilterColumns(predicate: WorkflowFilterPredicate): string[];
|
|
3097
|
+
/**
|
|
3098
|
+
* The columns that can reach this node, when the graph knows — and `undefined`
|
|
3099
|
+
* when it does not.
|
|
3100
|
+
*
|
|
3101
|
+
* ## What this is for
|
|
3102
|
+
*
|
|
3103
|
+
* It is the one thing a declarative rename buys that a transform cannot, and it
|
|
3104
|
+
* is worth being precise about how far it reaches rather than overselling it.
|
|
3105
|
+
*
|
|
3106
|
+
* With a JS transform, the catalog cannot know what columns come out — the
|
|
3107
|
+
* answer is inside a function body — which is why the property-name rule in
|
|
3108
|
+
* `property-names.ts` fires at publish time and why a mismatch between a
|
|
3109
|
+
* property and a record key is discovered as a column of NULLs. A rename is
|
|
3110
|
+
* **data**, so for one arrangement the answer is exact:
|
|
3111
|
+
*
|
|
3112
|
+
* > A rename with `unnamed: 'drop'` produces its targets and **nothing else**,
|
|
3113
|
+
* > whatever it was handed.
|
|
3114
|
+
*
|
|
3115
|
+
* That set is *closed* — an upper bound that holds regardless of what is
|
|
3116
|
+
* upstream — and it survives every node that does not touch columns. So a filter
|
|
3117
|
+
* or a second rename downstream of one can be told, at authoring time, that it
|
|
3118
|
+
* names a column which cannot be there.
|
|
3119
|
+
*
|
|
3120
|
+
* ## What it deliberately does not claim
|
|
3121
|
+
*
|
|
3122
|
+
* - **It is an upper bound, not the output.** A target only appears in a row
|
|
3123
|
+
* whose input actually held the source column. So a column *inside* the set
|
|
3124
|
+
* may still be absent, and nothing here says otherwise.
|
|
3125
|
+
* - **A `keep` rename tells you nothing on its own.** Its output is its input
|
|
3126
|
+
* with some keys re-labelled, and its input is unknown unless something
|
|
3127
|
+
* upstream closed it. So `undefined` propagates, and that is the honest
|
|
3128
|
+
* answer rather than an empty set.
|
|
3129
|
+
* - **A source, a transform and a call are always unknown.** A source's shape is
|
|
3130
|
+
* discovered against the live system rather than declared in the graph; a
|
|
3131
|
+
* transform is a function body; a call is a workflow this graph does not own.
|
|
3132
|
+
* - **It says nothing about a sink's declared properties.** That is the check
|
|
3133
|
+
* worth wanting — "this sink writes a property no upstream node produces" —
|
|
3134
|
+
* and it is *not* available here: a {@link WorkflowSinkNode} carries a
|
|
3135
|
+
* `targetType` and nothing else, so the property list would have to be
|
|
3136
|
+
* threaded into a validator that is pure and dependency-free on purpose. What
|
|
3137
|
+
* is built instead is the run log, which prints the columns a rename produced.
|
|
3138
|
+
*
|
|
3139
|
+
* Cycles answer `undefined` rather than looping. `validateWorkflow` refuses a
|
|
3140
|
+
* cyclic graph before it gets here, but the canvas calls this while a graph is
|
|
3141
|
+
* being drawn and is entitled to a wrong-but-terminating answer.
|
|
3142
|
+
*/
|
|
3143
|
+
export declare function workflowKnownColumns(graph: WorkflowGraph, nodeId: string): ReadonlySet<string> | undefined;
|
|
2807
3144
|
/**
|
|
2808
3145
|
* Narrow a stored node, loudly.
|
|
2809
3146
|
*
|
|
@@ -3152,6 +3489,39 @@ export interface CatalogStageStore {
|
|
|
3152
3489
|
* as long as something might still resume onto them.
|
|
3153
3490
|
*/
|
|
3154
3491
|
dropStages(runId: string): Promise<number>;
|
|
3492
|
+
/**
|
|
3493
|
+
* The batch exactly as it is stored, without decoding it into rows.
|
|
3494
|
+
*
|
|
3495
|
+
* Optional, and the only thing that reads it is the `rename` node. See
|
|
3496
|
+
* {@link renameStagePayload}: a staged batch names its columns once, in
|
|
3497
|
+
* `shapes`, and carries the data in positional arrays — so renaming a column
|
|
3498
|
+
* is a rewrite of tens of strings rather than a rebuild of a hundred thousand
|
|
3499
|
+
* objects. `readStage` cannot express that, because decoding to
|
|
3500
|
+
* `Record<string, unknown>` *is* the rebuild.
|
|
3501
|
+
*
|
|
3502
|
+
* Optional rather than required so that a store written against the shipped
|
|
3503
|
+
* interface keeps working: {@link supportsStagePayloads} is what asks, and a
|
|
3504
|
+
* store that answers no gets the row path, which produces the same rows more
|
|
3505
|
+
* slowly. It is deliberately `unknown` — the encoding is
|
|
3506
|
+
* `catalog.stage-encoding.ts`'s business and a store's job is to hand back
|
|
3507
|
+
* what it was given.
|
|
3508
|
+
*/
|
|
3509
|
+
readStagePayload?(ref: {
|
|
3510
|
+
runId: string;
|
|
3511
|
+
nodeId: string;
|
|
3512
|
+
batch: number;
|
|
3513
|
+
}): Promise<unknown>;
|
|
3514
|
+
/** The other half. Idempotent per `(runId, nodeId, batch)`, exactly like {@link writeStage}. */
|
|
3515
|
+
writeStagePayload?(input: {
|
|
3516
|
+
runId: string;
|
|
3517
|
+
nodeId: string;
|
|
3518
|
+
batch: number;
|
|
3519
|
+
payload: unknown;
|
|
3520
|
+
/** How many rows the payload holds. The store does not decode it to count. */
|
|
3521
|
+
rows: number;
|
|
3522
|
+
}): Promise<{
|
|
3523
|
+
written: number;
|
|
3524
|
+
}>;
|
|
3155
3525
|
}
|
|
3156
3526
|
/**
|
|
3157
3527
|
* Whether this store can hold workflows at all.
|
|
@@ -3226,6 +3596,21 @@ export type CatalogReusableNodeStore = Required<Pick<CatalogPipelineStore, 'list
|
|
|
3226
3596
|
*/
|
|
3227
3597
|
export declare function supportsReusableNodes(store: CatalogPipelineStore): store is CatalogPipelineStore & CatalogReusableNodeStore;
|
|
3228
3598
|
export declare function supportsWorkflowStages(store: CatalogPipelineStore): store is CatalogPipelineStore & CatalogStageStore;
|
|
3599
|
+
/**
|
|
3600
|
+
* Whether this store will hand a staged batch over without decoding it.
|
|
3601
|
+
*
|
|
3602
|
+
* Both methods, never one: a rename that could read the payload and not write
|
|
3603
|
+
* one back would have to decode its own output to store it, which is the rebuild
|
|
3604
|
+
* the pair exists to avoid. The methods rather than a flag, the same argument
|
|
3605
|
+
* {@link supportsWorkflows} makes.
|
|
3606
|
+
*
|
|
3607
|
+
* A store that answers no is not broken and nothing degrades except speed — the
|
|
3608
|
+
* rename node falls back to `readStage`/`writeStage` and produces identical
|
|
3609
|
+
* rows. Which path ran is said in the run log, because "this rename was
|
|
3610
|
+
* metadata-only" is a claim, and a claim that could quietly stop being true is
|
|
3611
|
+
* worse than no claim.
|
|
3612
|
+
*/
|
|
3613
|
+
export declare function supportsStagePayloads(store: CatalogPipelineStore): store is CatalogPipelineStore & CatalogStageStore & Required<Pick<CatalogStageStore, 'readStagePayload' | 'writeStagePayload'>>;
|
|
3229
3614
|
/**
|
|
3230
3615
|
* A store that really does hold operator-set expectations, all four members
|
|
3231
3616
|
* present.
|