@dudousxd/nestjs-catalog-react 0.17.0 → 0.19.0-preview.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -13
- package/dist/CatalogManager.d.ts +23 -1
- package/dist/CatalogManager.js +76 -13
- package/dist/ConnectionPanel.d.ts +11 -7
- package/dist/ConnectionPanel.js +78 -149
- package/dist/LoadedBySection.d.ts +39 -0
- package/dist/LoadedBySection.js +143 -0
- package/dist/PipelineConsole.d.ts +23 -126
- package/dist/PipelineConsole.js +39 -551
- package/dist/WorkflowCanvas.d.ts +16 -0
- package/dist/WorkflowCanvas.js +1064 -64
- package/dist/connection-form.d.ts +241 -0
- package/dist/connection-form.js +235 -0
- package/dist/context.d.ts +236 -38
- package/dist/context.js +46 -11
- package/dist/index.d.ts +10 -3
- package/dist/index.js +63 -14
- package/dist/routes.d.ts +84 -13
- package/dist/routes.js +7 -4
- package/dist/schema-discovery.d.ts +232 -0
- package/dist/schema-discovery.js +225 -0
- package/dist/source-connection.d.ts +82 -0
- package/dist/source-connection.js +176 -0
- package/dist/source-fields.d.ts +16 -14
- package/dist/source-fields.js +28 -18
- package/dist/ui/code-editor.d.ts +44 -2
- package/dist/ui/code-editor.js +10 -2
- package/dist/ui/combobox.d.ts +97 -0
- package/dist/ui/combobox.js +80 -0
- package/dist/ui/dialog.d.ts +13 -1
- package/dist/ui/dialog.js +13 -2
- package/dist/workflow/edges.d.ts +110 -0
- package/dist/workflow/edges.js +112 -0
- package/dist/workflow/graph.d.ts +39 -1
- package/dist/workflow/graph.js +212 -52
- package/dist/workflow/index.d.ts +8 -0
- package/dist/workflow/index.js +12 -0
- package/dist/workflow/lifecycle.d.ts +135 -0
- package/dist/workflow/lifecycle.js +216 -0
- package/dist/workflow/model.d.ts +17 -3
- package/dist/workflow/model.js +7 -2
- package/dist/workflow/nodes.d.ts +4 -1
- package/dist/workflow/nodes.js +198 -10
- package/dist/workflow/runs.d.ts +38 -0
- package/dist/workflow/runs.js +101 -0
- package/dist/workflow/shape.d.ts +169 -0
- package/dist/workflow/shape.js +249 -0
- package/dist/workflow/templates.d.ts +623 -0
- package/dist/workflow/templates.js +1102 -0
- package/dist/workflow/validate.d.ts +20 -1
- package/dist/workflow/validate.js +52 -3
- package/dist/workflow/wiring.d.ts +84 -0
- package/dist/workflow/wiring.js +254 -0
- package/package.json +9 -9
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
|
|
2
|
+
import { useQuery } from '@tanstack/react-query';
|
|
3
|
+
import { CircleCheck, CircleX, Loader2, RefreshCw } from 'lucide-react';
|
|
4
|
+
import { useMemo, useState } from 'react';
|
|
5
|
+
import { cn } from '../cn';
|
|
6
|
+
import { catalogQueryKeys, useCatalogClient } from '../context';
|
|
7
|
+
import { RevisionHistorySheet } from '../diff/RevisionDiff';
|
|
8
|
+
import { Tooltip } from '../ui/tooltip';
|
|
9
|
+
import { WORKFLOW_NAME } from './name';
|
|
10
|
+
const MUTED = 'text-zinc-400 dark:text-zinc-500';
|
|
11
|
+
const RULE = 'border-zinc-200 dark:border-zinc-800';
|
|
12
|
+
const PANEL = 'bg-white dark:bg-zinc-900';
|
|
13
|
+
/**
|
|
14
|
+
* The connector this graph runs as, and its recent loads.
|
|
15
|
+
*
|
|
16
|
+
* Rendered for a stored graph only. A draft has nothing behind it — that is
|
|
17
|
+
* what a draft is — and an empty "Runs as" panel over a canvas somebody is
|
|
18
|
+
* still drawing would be a permanent heading nobody reads.
|
|
19
|
+
*/
|
|
20
|
+
export function RunsAsPanel({ workflowId, status, transforms, }) {
|
|
21
|
+
const client = useCatalogClient();
|
|
22
|
+
const { data: connectors = [], isPending } = useQuery({
|
|
23
|
+
queryKey: catalogQueryKeys.connectors,
|
|
24
|
+
queryFn: () => client.listConnectors(),
|
|
25
|
+
});
|
|
26
|
+
const connector = useMemo(() => connectors.find((candidate) => candidate.workflowId === workflowId), [connectors, workflowId]);
|
|
27
|
+
return (_jsxs("section", { className: cn('rounded-lg border p-3', RULE, PANEL), children: [_jsx("h2", { className: cn('font-mono text-[10px] uppercase tracking-[0.14em]', MUTED), children: "Runs as" }), isPending && _jsx("p", { className: cn('mt-2 text-[11px]', MUTED), children: "Reading\u2026" }), !isPending && !connector && (_jsx("p", { className: cn('mt-2 text-[11px] leading-relaxed', MUTED), children: status === 'ready'
|
|
28
|
+
? `This ${WORKFLOW_NAME.singular} is ready but nothing is running it yet. That is unusual — publishing is what mints the connector a run history is keyed on — and it is worth a look before trusting a schedule on it.`
|
|
29
|
+
: `Nothing yet. Publishing is what mints the connector this ${WORKFLOW_NAME.singular} runs as; until then a run is something you press, and there is no history to key.` })), connector && _jsx(ConnectorIdentity, { connector: connector }), connector && _jsx(RecentRuns, { connectorId: connector.id, transforms: transforms })] }));
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* The connector's id and state, and nothing else.
|
|
33
|
+
*
|
|
34
|
+
* The id is the load-bearing part: it is what `GET pipeline/runs?connector=` is
|
|
35
|
+
* keyed on and what an operator greps a log for. Its name and target type are
|
|
36
|
+
* the graph's, restated, so they are not repeated here.
|
|
37
|
+
*/
|
|
38
|
+
function ConnectorIdentity({ connector }) {
|
|
39
|
+
return (_jsxs("div", { className: "mt-2 space-y-1", children: [_jsx(Tooltip, { content: "The id the run history, the incremental watermark and the singleton lock are all keyed on. It survives an unpublish and a re-publish, which is what makes those three carry over.", children: _jsx("div", { className: cn('flex cursor-help items-center gap-1.5 font-mono text-[10px]', MUTED), children: _jsx("span", { className: "truncate", children: connector.id }) }) }), _jsxs("div", { className: "flex flex-wrap items-center gap-1.5", children: [connector.mode === 'incremental' && (_jsx(Tooltip, { content: "Reads only what changed since the last run, and carries the rest forward. The position is kept per source node.", children: _jsxs("span", { className: cn('flex cursor-help items-center gap-1 font-mono text-[10px]', MUTED), children: [_jsx(RefreshCw, { size: 10 }), "incremental"] }) })), !connector.enabled && (_jsx(Tooltip, { content: "Disabled. Unpublishing does this, and it is why the id and the history survive one.", children: _jsx("span", { className: "cursor-help rounded-sm bg-zinc-100 px-1.5 py-0.5 font-mono text-[10px] uppercase dark:bg-zinc-800", children: "disabled" }) }))] }), _jsxs("p", { className: cn('text-[10px] leading-relaxed', MUTED), children: ["Read-only. Running, scheduling and deleting are all done on the ", WORKFLOW_NAME.singular, ' ', "above \u2014 this row is minted by publishing it and removed with it."] })] }));
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* What this workflow has done lately, and the way from a load to the code that
|
|
43
|
+
* produced it.
|
|
44
|
+
*
|
|
45
|
+
* The `code v3` on a row is a control rather than a fact: it is where the
|
|
46
|
+
* question "why did this load come out different" actually occurs to somebody,
|
|
47
|
+
* and making the number the way in means nobody has to carry it to another
|
|
48
|
+
* screen and pick it out of a dropdown — which is the step where you pick the
|
|
49
|
+
* wrong one and conclude the code was fine.
|
|
50
|
+
*
|
|
51
|
+
* It is only a control when the answer is unambiguous. A run records ONE
|
|
52
|
+
* `transformVersion`, which meant something exact when a connector had at most
|
|
53
|
+
* one transform; a graph may run three in a row, and offering a comparison
|
|
54
|
+
* against whichever one this screen happened to pick first would be worse than
|
|
55
|
+
* offering none. So the compare is available for a graph with exactly one
|
|
56
|
+
* transform, and the version is plain text otherwise.
|
|
57
|
+
*/
|
|
58
|
+
function RecentRuns({ connectorId, transforms, }) {
|
|
59
|
+
const client = useCatalogClient();
|
|
60
|
+
const [comparing, setComparing] = useState(null);
|
|
61
|
+
const { data: runs = [] } = useQuery({
|
|
62
|
+
queryKey: catalogQueryKeys.runs(connectorId),
|
|
63
|
+
queryFn: () => client.listRuns(connectorId),
|
|
64
|
+
});
|
|
65
|
+
const only = transforms.length === 1 ? transforms[0] : undefined;
|
|
66
|
+
if (runs.length === 0) {
|
|
67
|
+
return _jsx("p", { className: cn('mt-2 text-[11px]', MUTED), children: "No runs recorded against it yet." });
|
|
68
|
+
}
|
|
69
|
+
return (_jsxs(_Fragment, { children: [_jsxs("div", { className: cn('mt-3 border-t pt-2', RULE), children: [_jsx("div", { className: cn('mb-1.5 font-mono text-[10px] uppercase tracking-[0.14em]', MUTED), children: "Recent runs" }), _jsx("div", { className: "space-y-1", children: runs.slice(0, 4).map((entry) => (_jsx(Tooltip, { content: entry.error ? (_jsx("span", { className: "font-mono text-[11px]", children: entry.error })) : (_jsx("span", { className: "font-mono text-[11px]", children: entry.logs.slice(-3).join('\n') || 'No logs.' })), children: _jsxs("div", { className: "flex cursor-help flex-wrap items-center gap-1.5 text-[11px]", children: [_jsxs("span", { className: "sr-only", children: [entry.status, ": "] }), _jsx(RunStatusIcon, { status: entry.status }), _jsx("span", { className: "truncate font-mono text-[10px]", children: entry.snapshotId }), _jsxs("span", { className: cn('font-mono text-[10px]', MUTED), children: [entry.fetched, " fetched \u00B7 ", entry.written, " written"] }), _jsx(RanCodeVersion, { version: entry.transformVersion, comparable: only !== undefined, onCompare: setComparing })] }) }, entry.id))) })] }), only && (_jsx(RevisionHistorySheet, { open: comparing !== null, onOpenChange: (open) => {
|
|
70
|
+
if (!open)
|
|
71
|
+
setComparing(null);
|
|
72
|
+
}, subject: { kind: 'transform', id: only.id, name: only.name }, current: { version: only.version, body: only.code }, ranVersion: comparing ?? undefined }))] }));
|
|
73
|
+
}
|
|
74
|
+
function RunStatusIcon({ status }) {
|
|
75
|
+
if (status === 'succeeded')
|
|
76
|
+
return _jsx(CircleCheck, { size: 11, "aria-hidden": true, className: "text-emerald-600" });
|
|
77
|
+
if (status === 'failed')
|
|
78
|
+
return _jsx(CircleX, { size: 11, "aria-hidden": true, className: "text-red-600" });
|
|
79
|
+
return _jsx(Loader2, { size: 11, "aria-hidden": true, className: "animate-spin text-sky-600" });
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* `code v3` on a run, as a way into the comparison rather than as a fact.
|
|
83
|
+
*
|
|
84
|
+
* Still only text when there is nothing on the other side: a pipeline with no
|
|
85
|
+
* transform never records a version, a version whose transform has since been
|
|
86
|
+
* deleted has no current code to compare against, and a graph with several
|
|
87
|
+
* transforms has no single answer to which one this number names. A control that
|
|
88
|
+
* opens an empty panel — or the wrong one — is a control that teaches people to
|
|
89
|
+
* stop pressing it.
|
|
90
|
+
*/
|
|
91
|
+
function RanCodeVersion({ version, comparable, onCompare, }) {
|
|
92
|
+
if (version === undefined)
|
|
93
|
+
return null;
|
|
94
|
+
if (!comparable)
|
|
95
|
+
return _jsxs("span", { className: cn('font-mono text-[10px]', MUTED), children: ["code v", version] });
|
|
96
|
+
return (
|
|
97
|
+
// Inside the tooltip's trigger rather than beside it, because this row IS
|
|
98
|
+
// the trigger: pulling the button out would put a gap in the hover target
|
|
99
|
+
// exactly where the pointer is heading. A click inside it bubbles normally.
|
|
100
|
+
_jsxs("button", { type: "button", onClick: () => onCompare(version), "aria-label": `Compare the code that ran (v${version}) with the current code`, className: cn('rounded-sm px-1 font-mono text-[10px] underline decoration-dotted underline-offset-2', MUTED, 'hover:text-sky-600'), children: ["code v", version] }));
|
|
101
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import type { ScalarType, WorkflowEdge, WorkflowNode } from '@dudousxd/nestjs-catalog/client';
|
|
2
|
+
import type { WorkflowProblem } from './validate';
|
|
3
|
+
/**
|
|
4
|
+
* Whether what a source produces fits the type a sink writes, asked while the
|
|
5
|
+
* graph is being drawn rather than after a load has committed.
|
|
6
|
+
*
|
|
7
|
+
* WHAT THIS ANSWERS, AND WHY IT IS WORTH ANSWERING
|
|
8
|
+
* ------------------------------------------------
|
|
9
|
+
* Every other check on this canvas is topological: a sink with no type, two
|
|
10
|
+
* sinks on one type, a node nothing reaches. All of them pass a graph whose
|
|
11
|
+
* source does not supply a single one of the columns its sink writes, because
|
|
12
|
+
* nothing in the graph says what the columns *are*. The load then succeeds,
|
|
13
|
+
* commits, and reports its row count — and the rows are null.
|
|
14
|
+
*
|
|
15
|
+
* That is not hypothetical. `subwo` has 84 columns, 73 of them spelled in ways
|
|
16
|
+
* SQL cannot use as an identifier, and the mismatch was found after a run
|
|
17
|
+
* reported `fetched=6905, written=6905`. Thirteen types were published the same
|
|
18
|
+
* way and six came out with most of their columns null, 313,833 rows of the
|
|
19
|
+
* largest — see the `property-names.ts` header in the pipeline package, which
|
|
20
|
+
* moved *its* half of the problem to publish time for the same reason this moves
|
|
21
|
+
* the other half to design time: everything needed to answer the question is
|
|
22
|
+
* already on screen, and answering it costs nothing.
|
|
23
|
+
*
|
|
24
|
+
* THE COMPARISON, WHICH IS THE WHOLE THING
|
|
25
|
+
* ----------------------------------------
|
|
26
|
+
* A published property has two names and they are not interchangeable:
|
|
27
|
+
*
|
|
28
|
+
* - `name` is what the load looks the field up by — `row[property.name]`, see
|
|
29
|
+
* the warehouse store — and nothing on the write path consults anything else.
|
|
30
|
+
* - `columnName` is lineage: how the source spells the field, recorded for
|
|
31
|
+
* tracing when the property ended up called something else.
|
|
32
|
+
*
|
|
33
|
+
* A source with no transform between it and the sink hands its records over
|
|
34
|
+
* exactly as they arrive, keyed by the source's own spelling. So a record has
|
|
35
|
+
* `Asset Id` and the store asks for `Asset_Id`, and the answer is `undefined`,
|
|
36
|
+
* and `undefined` is written as null in every row of every run, forever, and
|
|
37
|
+
* the load reports success.
|
|
38
|
+
*
|
|
39
|
+
* Which means the comparison that matters is `column.name` against
|
|
40
|
+
* `property.name`. Matching against `columnName` instead would report "fits" on
|
|
41
|
+
* precisely the graph that produced the 6,905 rows — the two names agree about
|
|
42
|
+
* the source and disagree about what the store will ask for. `columnName` is
|
|
43
|
+
* still read, but only to *explain* a miss: a source column that matches the
|
|
44
|
+
* property's `columnName` and not its `name` is the split-name case, and saying
|
|
45
|
+
* so is the difference between "this column is missing" and "these two are the
|
|
46
|
+
* same field under two names".
|
|
47
|
+
*
|
|
48
|
+
* Where that split came from, and why the repair is now the cheap one: publish
|
|
49
|
+
* used to refuse any property name that was not a SQL identifier outright,
|
|
50
|
+
* because the name was written verbatim as the view's output alias. Publishers
|
|
51
|
+
* did what the refusal told them — renamed the property to `Asset_Id`, put
|
|
52
|
+
* `Asset Id` in `columnName` — and that is the type that loads nulls. Both
|
|
53
|
+
* aliases now go through `outputAlias`, so `Asset Id` is a perfectly good
|
|
54
|
+
* property name and the type can simply be renamed to what the source calls the
|
|
55
|
+
* field. A transform is still the answer where the name genuinely cannot become
|
|
56
|
+
* a column, which is why the message names both.
|
|
57
|
+
*
|
|
58
|
+
* THREE OUTCOMES, NOT TWO
|
|
59
|
+
* -----------------------
|
|
60
|
+
* Discovery reports how it knows what it knows, and against a real deployment it
|
|
61
|
+
* came back `basis: "driver"` with `sampled: 0` — the driver described the
|
|
62
|
+
* result set and not one row was read. So there are questions this genuinely
|
|
63
|
+
* cannot settle, and pretending otherwise in either direction is the failure:
|
|
64
|
+
*
|
|
65
|
+
* - **fits** — nothing is reported.
|
|
66
|
+
* - **does not fit** — `level: "error"`. Reserved for what is decided by the
|
|
67
|
+
* two schemas alone and cannot come out any other way: a column the source
|
|
68
|
+
* does not produce under the name the store will ask for.
|
|
69
|
+
* - **not known well enough to say** — `level: "warning"`. A type difference
|
|
70
|
+
* `coerce` may or may not survive depending on the values, a column discovery
|
|
71
|
+
* reached no conclusion about, a nullability the two sides disagree about, or
|
|
72
|
+
* a transform on the path.
|
|
73
|
+
*
|
|
74
|
+
* That third outcome uses the level distinction {@link WorkflowProblem} already
|
|
75
|
+
* has rather than a new one. It is not decoration: `hasBlockingProblem` reads
|
|
76
|
+
* `level`, so an unproven column colours nothing and blocks nothing, and the
|
|
77
|
+
* rail already sorts warnings under errors behind an amber glyph. Rendering "I
|
|
78
|
+
* could not tell" as an error is how a panel stops being read, which is the
|
|
79
|
+
* failure `validate.ts` opens by describing.
|
|
80
|
+
*
|
|
81
|
+
* WHEN SOMETHING THAT COMPUTES ROWS IS ON THE PATH
|
|
82
|
+
* ------------------------------------------------
|
|
83
|
+
* Said out loud, never silently passed. What a transform emits is whatever its
|
|
84
|
+
* TypeScript returns, and knowing that means compiling and running it — a
|
|
85
|
+
* different project. A `call` node is further out of reach still: what it emits
|
|
86
|
+
* is decided by a durable workflow this graph does not own, possibly written in
|
|
87
|
+
* another language, and the graph holds nothing but its name and a version. So a
|
|
88
|
+
* sink fed through either gets a warning naming the node, and no error and no
|
|
89
|
+
* silence. Guessing that a node is shape-preserving would be a validator
|
|
90
|
+
* inventing a fact, and a validator that invents facts is worse than none.
|
|
91
|
+
*
|
|
92
|
+
* The branch is on "not a source" rather than on the kind, deliberately: a kind
|
|
93
|
+
* added to the vocabulary tomorrow lands in the honest answer by default instead
|
|
94
|
+
* of falling through the comparison as though a source had produced its rows.
|
|
95
|
+
*
|
|
96
|
+
* WHEN NOTHING IS KNOWN AT ALL
|
|
97
|
+
* ----------------------------
|
|
98
|
+
* If no source feeding a sink has a discovered shape, this reports nothing —
|
|
99
|
+
* not even "could not check". A deployment that has never run discovery would
|
|
100
|
+
* otherwise carry a permanent amber warning on every graph, which is the same
|
|
101
|
+
* noise by another route. The warnings above are for a question that was asked
|
|
102
|
+
* and came back inconclusive, not for one nobody asked.
|
|
103
|
+
*
|
|
104
|
+
* That is not a hypothetical state on the canvas: discovery is a read of a live
|
|
105
|
+
* source behind a `POST`, run when somebody presses the button on a source
|
|
106
|
+
* node's inspector, and never on anybody's behalf. So a graph nobody has asked
|
|
107
|
+
* about is the ordinary case, and silence is what it gets.
|
|
108
|
+
*/
|
|
109
|
+
/**
|
|
110
|
+
* One column, exactly as `POST pipeline/workflows/:id/nodes/:nodeId/discover`
|
|
111
|
+
* reports it.
|
|
112
|
+
*
|
|
113
|
+
* A structural subset of that route's answer rather than an import of it: this
|
|
114
|
+
* module has no business knowing about a screen, and `DiscoveredColumn` in
|
|
115
|
+
* `schema-discovery.tsx` carries fields — a `note`, the source's own type
|
|
116
|
+
* string — that are for a reader and not for a comparison. Written this way, a
|
|
117
|
+
* discovery response IS a {@link SourceShape} with no conversion, which is what
|
|
118
|
+
* the canvas relies on, and a host with some other way to learn a source's
|
|
119
|
+
* columns can still supply one.
|
|
120
|
+
*/
|
|
121
|
+
export interface SourceColumn {
|
|
122
|
+
/**
|
|
123
|
+
* The column as the SOURCE spells it — which is the key its records carry,
|
|
124
|
+
* and therefore the string the store's `row[property.name]` has to match.
|
|
125
|
+
*/
|
|
126
|
+
name: string;
|
|
127
|
+
/** `null` where discovery reached no conclusion, which is not a type. */
|
|
128
|
+
type: ScalarType | null;
|
|
129
|
+
/** `null` where the source did not say. Postgres never does over a query. */
|
|
130
|
+
nullable: boolean | null;
|
|
131
|
+
confidence: 'reported' | 'inferred' | 'unknown';
|
|
132
|
+
/** The source's own vocabulary — a driver type id. Quoted back, never parsed. */
|
|
133
|
+
sourceType?: string;
|
|
134
|
+
}
|
|
135
|
+
export interface SourceShape {
|
|
136
|
+
columns: readonly SourceColumn[];
|
|
137
|
+
/** `driver` means the result set was described; `sample` means rows were read. */
|
|
138
|
+
basis?: 'driver' | 'sample';
|
|
139
|
+
/** How many records the inference looked at. Always 0 for a driver description. */
|
|
140
|
+
sampled?: number;
|
|
141
|
+
}
|
|
142
|
+
/** As much of a published property as this file has an opinion about. */
|
|
143
|
+
export interface TargetProperty {
|
|
144
|
+
/** What the load looks the field up by, as `row[name]`. */
|
|
145
|
+
name: string;
|
|
146
|
+
/** The source's spelling, when the two differ. Read only to explain a miss. */
|
|
147
|
+
columnName?: string;
|
|
148
|
+
type: ScalarType;
|
|
149
|
+
nullable: boolean;
|
|
150
|
+
}
|
|
151
|
+
export interface TargetShape {
|
|
152
|
+
name: string;
|
|
153
|
+
properties: readonly TargetProperty[];
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Where the two halves come from, as lookups rather than as data.
|
|
157
|
+
*
|
|
158
|
+
* Lookups because neither half belongs to this module and both are optional: a
|
|
159
|
+
* console may be able to see the catalog's types and know nothing about any
|
|
160
|
+
* source, and `undefined` from either is a supported answer with its own
|
|
161
|
+
* meaning — see the header.
|
|
162
|
+
*/
|
|
163
|
+
export interface ShapeKnowledge {
|
|
164
|
+
/** What discovery said this source node reads, or nothing if nobody asked. */
|
|
165
|
+
sourceShape(nodeId: string): SourceShape | undefined;
|
|
166
|
+
/** The published type, or nothing if this console cannot see one by that name. */
|
|
167
|
+
targetShape(typeName: string): TargetShape | undefined;
|
|
168
|
+
}
|
|
169
|
+
export declare function checkShapes(nodes: readonly WorkflowNode[], edges: readonly WorkflowEdge[], knowledge: ShapeKnowledge): WorkflowProblem[];
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
import { nodeName } from './model';
|
|
2
|
+
/** How many names a message spells out before it starts counting instead. */
|
|
3
|
+
const NAMED_LIMIT = 6;
|
|
4
|
+
export function checkShapes(nodes, edges, knowledge) {
|
|
5
|
+
const byId = new Map(nodes.map((node) => [node.id, node]));
|
|
6
|
+
const incoming = new Map();
|
|
7
|
+
for (const edge of edges) {
|
|
8
|
+
incoming.set(edge.to, [...(incoming.get(edge.to) ?? []), edge.from]);
|
|
9
|
+
}
|
|
10
|
+
const problems = [];
|
|
11
|
+
for (const node of nodes) {
|
|
12
|
+
if (node.kind !== 'sink')
|
|
13
|
+
continue;
|
|
14
|
+
if (typeof node.targetType !== 'string' || node.targetType.trim().length === 0)
|
|
15
|
+
continue;
|
|
16
|
+
problems.push(...checkSink(node, byId, incoming, knowledge));
|
|
17
|
+
}
|
|
18
|
+
return problems;
|
|
19
|
+
}
|
|
20
|
+
/** Everything reported about one sink, over each of the nodes wired into it. */
|
|
21
|
+
function checkSink(sink, byId, incoming, knowledge) {
|
|
22
|
+
const problems = [];
|
|
23
|
+
const indirect = [];
|
|
24
|
+
for (const fromId of incoming.get(sink.id) ?? []) {
|
|
25
|
+
const from = byId.get(fromId);
|
|
26
|
+
if (!from)
|
|
27
|
+
continue;
|
|
28
|
+
if (from.kind !== 'source') {
|
|
29
|
+
// Anything that is not a source hands the sink rows it computed. Held
|
|
30
|
+
// back until it is known that a shape was discovered somewhere behind it
|
|
31
|
+
// — see the header on staying quiet where nobody asked.
|
|
32
|
+
if (anyShapeBehind(from.id, byId, incoming, knowledge))
|
|
33
|
+
indirect.push(from);
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
const shape = knowledge.sourceShape(from.id);
|
|
37
|
+
if (shape)
|
|
38
|
+
problems.push(...comparePair(from, sink, shape, knowledge));
|
|
39
|
+
}
|
|
40
|
+
if (indirect.length > 0)
|
|
41
|
+
problems.push(notCheckable(indirect, sink));
|
|
42
|
+
return problems;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Does any source that can reach this node have a discovered shape?
|
|
46
|
+
*
|
|
47
|
+
* Backwards from the node, with a seen set. The seen set is not defensive
|
|
48
|
+
* tidiness: a cyclic graph is reported by core and drawn on the canvas anyway,
|
|
49
|
+
* and a traversal that hung would take the browser with it while somebody is
|
|
50
|
+
* still dragging the wire that closed the loop.
|
|
51
|
+
*/
|
|
52
|
+
function anyShapeBehind(startId, byId, incoming, knowledge) {
|
|
53
|
+
const seen = new Set([startId]);
|
|
54
|
+
const pending = [startId];
|
|
55
|
+
while (pending.length > 0) {
|
|
56
|
+
const current = pending.pop();
|
|
57
|
+
if (current === undefined)
|
|
58
|
+
break;
|
|
59
|
+
const node = byId.get(current);
|
|
60
|
+
if (node?.kind === 'source' && knowledge.sourceShape(node.id))
|
|
61
|
+
return true;
|
|
62
|
+
for (const previous of incoming.get(current) ?? []) {
|
|
63
|
+
if (seen.has(previous))
|
|
64
|
+
continue;
|
|
65
|
+
seen.add(previous);
|
|
66
|
+
pending.push(previous);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
/** The computed-rows case: said out loud, as a warning, never as silence. */
|
|
72
|
+
function notCheckable(indirect, sink) {
|
|
73
|
+
const named = indirect.map((node) => `"${nodeName(node)}"`).join(', ');
|
|
74
|
+
return {
|
|
75
|
+
level: 'warning',
|
|
76
|
+
code: 'shape-not-checked',
|
|
77
|
+
message: `Whether what reaches "${nodeName(sink)}" fits ${sink.targetType.trim()} could not be checked: ${named} ${indirect.length === 1 ? 'sits' : 'sit'} in between, and ${indirect.length === 1 ? 'it hands' : 'they hand'} the sink rows ${indirect.length === 1 ? 'it' : 'they'} computed. What a transform emits is whatever its code returns, and what a call node emits is decided by a workflow this graph does not own — neither is knowable without running it, which this canvas does not do. The columns are compared only where a source feeds a sink directly. This is not a problem with the graph; it is the check declining to guess.`,
|
|
78
|
+
nodeIds: [...indirect.map((node) => node.id), sink.id],
|
|
79
|
+
// Deliberately no edge ids. The rail paints a wire red from these, and a
|
|
80
|
+
// red wire for "not known" is the error styling this whole file exists to
|
|
81
|
+
// keep off the uncertain answer.
|
|
82
|
+
edgeIds: [],
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
/** One source feeding one sink with nothing in between: the checkable case. */
|
|
86
|
+
function comparePair(source, sink, shape, knowledge) {
|
|
87
|
+
const typeName = sink.targetType.trim();
|
|
88
|
+
const target = knowledge.targetShape(typeName);
|
|
89
|
+
if (!target) {
|
|
90
|
+
return [
|
|
91
|
+
{
|
|
92
|
+
level: 'warning',
|
|
93
|
+
code: 'shape-not-checked',
|
|
94
|
+
message: `"${nodeName(source)}" reads ${shape.columns.length} column${shape.columns.length === 1 ? '' : 's'}, but there is nothing to compare them against: this console cannot see a type called ${typeName}. It may not be published yet, or it may be one this viewer is not allowed to read. The server checks it either way on save.`,
|
|
95
|
+
nodeIds: [source.id, sink.id],
|
|
96
|
+
edgeIds: [],
|
|
97
|
+
},
|
|
98
|
+
];
|
|
99
|
+
}
|
|
100
|
+
const verdict = compareColumns(shape, target);
|
|
101
|
+
const problems = [];
|
|
102
|
+
if (verdict.respelled.length > 0) {
|
|
103
|
+
problems.push(respelledProblem(source, sink, target, verdict.respelled));
|
|
104
|
+
}
|
|
105
|
+
if (verdict.missing.length > 0) {
|
|
106
|
+
problems.push(missingProblem(source, sink, target, verdict.missing));
|
|
107
|
+
}
|
|
108
|
+
if (verdict.unproven.length > 0) {
|
|
109
|
+
problems.push(unprovenProblem(source, sink, target, verdict.unproven, shape));
|
|
110
|
+
}
|
|
111
|
+
return problems;
|
|
112
|
+
}
|
|
113
|
+
function compareColumns(shape, target) {
|
|
114
|
+
const supplied = new Map(shape.columns.map((column) => [column.name, column]));
|
|
115
|
+
const verdict = { respelled: [], missing: [], unproven: [] };
|
|
116
|
+
for (const property of target.properties) {
|
|
117
|
+
const column = supplied.get(property.name);
|
|
118
|
+
if (column) {
|
|
119
|
+
const doubt = doubtAbout(property, column);
|
|
120
|
+
if (doubt)
|
|
121
|
+
verdict.unproven.push(doubt);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const spelling = property.columnName;
|
|
125
|
+
if (spelling && spelling !== property.name && supplied.has(spelling)) {
|
|
126
|
+
verdict.respelled.push({ property, spelling });
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
verdict.missing.push(property);
|
|
130
|
+
}
|
|
131
|
+
return verdict;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Whether a property the source does supply raises a question, in one sentence.
|
|
135
|
+
*
|
|
136
|
+
* Everything here is a *question*, never a refusal, and the reason is `coerce`
|
|
137
|
+
* in the warehouse store: it is total. It stringifies for a `string`, JSON
|
|
138
|
+
* encodes for a `json`, parses a `date` and gives up as null, and returns null
|
|
139
|
+
* for a number that is not finite. So a `string` column arriving at a `number`
|
|
140
|
+
* property loads perfectly when every value happens to be numeric and writes
|
|
141
|
+
* nulls when one is not — which is a fact about the rows, and against a driver
|
|
142
|
+
* description with `sampled: 0` no rows were read. Calling that "does not fit"
|
|
143
|
+
* would refuse graphs that load correctly every night.
|
|
144
|
+
*/
|
|
145
|
+
function doubtAbout(property, column) {
|
|
146
|
+
const label = `\`${property.name}\``;
|
|
147
|
+
if (column.type === null || column.confidence === 'unknown') {
|
|
148
|
+
return `${label} — discovery reached no conclusion about it${column.sourceType ? ` (the source calls it ${column.sourceType})` : ''}, and ${property.type} is what the type expects`;
|
|
149
|
+
}
|
|
150
|
+
if (!absorbs(property.type, column.type)) {
|
|
151
|
+
return `${label} — the source reports ${column.type} and the type says ${property.type}, so each value is converted on the way in and the ones that will not convert are written as null`;
|
|
152
|
+
}
|
|
153
|
+
if (property.nullable === false && column.nullable === true) {
|
|
154
|
+
return `${label} — the type says it is never null and the source says its column may be`;
|
|
155
|
+
}
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Whether a property's type takes a column's without a question being raised.
|
|
160
|
+
*
|
|
161
|
+
* `string`, `json` and `unknown` take anything, because `coerce` reaches them
|
|
162
|
+
* through `String(...)` and `JSON.stringify(...)`, both of which are total.
|
|
163
|
+
* `unknown` on the column side is not a type at all — it is the answer somebody
|
|
164
|
+
* gave when nothing better fit — so it settles nothing and is left to the
|
|
165
|
+
* caller, which reports it as a question rather than as a fit.
|
|
166
|
+
*/
|
|
167
|
+
function absorbs(property, column) {
|
|
168
|
+
if (property === column)
|
|
169
|
+
return true;
|
|
170
|
+
return property === 'string' || property === 'json' || property === 'unknown';
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* The 6,905-row case, named as itself.
|
|
174
|
+
*
|
|
175
|
+
* Two ways out, and the cheaper one first. Renaming the PROPERTY to the source's
|
|
176
|
+
* spelling is now allowed — `outputAlias` cleans a name SQL cannot take on its
|
|
177
|
+
* way to a column, so `Asset Id` is a legal property name and the split these
|
|
178
|
+
* types carry was a repair for a refusal that no longer exists. A transform is
|
|
179
|
+
* the answer where the name genuinely cannot become a column even cleaned, which
|
|
180
|
+
* is a much smaller set than the one this reports. See the file header.
|
|
181
|
+
*
|
|
182
|
+
* `edgeIds` is left empty here and on every problem below, and that is not an
|
|
183
|
+
* omission: `edgesFor` in `validate.ts` derives the wire from the node ids for
|
|
184
|
+
* exactly the codes that are about a wire, which is where every other edge id on
|
|
185
|
+
* this canvas comes from. Deriving a second copy here is the drift that module
|
|
186
|
+
* is written to prevent. The warnings are absent from that list on purpose — a
|
|
187
|
+
* red wire is error styling, and "not known" must never wear it.
|
|
188
|
+
*/
|
|
189
|
+
function respelledProblem(source, sink, target, respelled) {
|
|
190
|
+
const examples = respelled
|
|
191
|
+
.slice(0, NAMED_LIMIT)
|
|
192
|
+
.map(({ property, spelling }) => `\`${spelling}\` → \`${property.name}\``)
|
|
193
|
+
.join(', ');
|
|
194
|
+
return {
|
|
195
|
+
level: 'error',
|
|
196
|
+
code: 'shape-source-spelling',
|
|
197
|
+
message: `"${nodeName(source)}" produces ${countOf(respelled.length, 'column')} that ${target.name} knows under a different name: ${examples}${andMore(respelled.length)}. The load looks every field up by the property's own name, and a source wired straight into a sink hands its records over exactly as they arrive — so each of these would be written as null in every row, and the load would still report the rows it read. Rename the ${respelled.length === 1 ? 'property' : 'properties'} on ${target.name} to the spelling the source uses, which publishing now accepts; where a name cannot become a column even cleaned, put a transform between them to rename the field instead.`,
|
|
198
|
+
nodeIds: [source.id, sink.id],
|
|
199
|
+
edgeIds: [],
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
function missingProblem(source, sink, target, missing) {
|
|
203
|
+
const examples = missing
|
|
204
|
+
.slice(0, NAMED_LIMIT)
|
|
205
|
+
.map((property) => `\`${property.name}\``)
|
|
206
|
+
.join(', ');
|
|
207
|
+
return {
|
|
208
|
+
level: 'error',
|
|
209
|
+
code: 'shape-missing-column',
|
|
210
|
+
message: `"${nodeName(source)}" does not produce ${countOf(missing.length, 'column')} that ${target.name} is made of: ${examples}${andMore(missing.length)}. Nothing between a source and a sink adds a field, so ${missing.length === 1 ? 'it' : 'each of them'} would be null in every row "${nodeName(sink)}" commits, on every run, and the run would report success.`,
|
|
211
|
+
nodeIds: [source.id, sink.id],
|
|
212
|
+
edgeIds: [],
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* The third outcome, with the basis stated rather than implied.
|
|
217
|
+
*
|
|
218
|
+
* `basis` and `sampled` are quoted because they are what makes this a warning
|
|
219
|
+
* instead of an error: a driver description that read no rows can be trusted
|
|
220
|
+
* completely about names and not at all about values, and a reader deciding
|
|
221
|
+
* whether to go and look needs to know which of the two this is.
|
|
222
|
+
*/
|
|
223
|
+
function unprovenProblem(source, sink, target, unproven, shape) {
|
|
224
|
+
const listed = unproven.slice(0, NAMED_LIMIT).join('; ');
|
|
225
|
+
return {
|
|
226
|
+
level: 'warning',
|
|
227
|
+
code: 'shape-unproven',
|
|
228
|
+
message: `${countOf(unproven.length, 'column')} between "${nodeName(source)}" and ${target.name} could not be decided from the schemas alone: ${listed}${andMore(unproven.length)}. ${describeBasis(shape)} Nothing here says the graph is wrong — it says this much of it was not proven right.`,
|
|
229
|
+
nodeIds: [source.id, sink.id],
|
|
230
|
+
edgeIds: [],
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
/** What the discovery behind this can and cannot promise, in one sentence. */
|
|
234
|
+
function describeBasis(shape) {
|
|
235
|
+
if (shape.basis === 'sample') {
|
|
236
|
+
const sampled = shape.sampled ?? 0;
|
|
237
|
+
return `The types came from ${countOf(sampled, 'record')} read out of the source, so they are an inference over what was seen and not a statement about the rest.`;
|
|
238
|
+
}
|
|
239
|
+
if (shape.basis === 'driver') {
|
|
240
|
+
return `The types came from the driver's description of the result set and no rows were read (sampled ${shape.sampled ?? 0}), so the column names are the source's own and nothing at all is known about the values.`;
|
|
241
|
+
}
|
|
242
|
+
return 'The discovery behind this did not say how it knows, so treat the types as unproven.';
|
|
243
|
+
}
|
|
244
|
+
function countOf(count, noun) {
|
|
245
|
+
return `${count} ${noun}${count === 1 ? '' : 's'}`;
|
|
246
|
+
}
|
|
247
|
+
function andMore(total) {
|
|
248
|
+
return total > NAMED_LIMIT ? `, and ${total - NAMED_LIMIT} more` : '';
|
|
249
|
+
}
|