@dudousxd/nestjs-catalog-react 0.17.0 → 0.19.0-preview.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -13
- package/dist/CatalogManager.d.ts +23 -1
- package/dist/CatalogManager.js +76 -13
- package/dist/ConnectionPanel.d.ts +11 -7
- package/dist/ConnectionPanel.js +78 -149
- package/dist/LoadedBySection.d.ts +39 -0
- package/dist/LoadedBySection.js +143 -0
- package/dist/PipelineConsole.d.ts +23 -126
- package/dist/PipelineConsole.js +39 -551
- package/dist/WorkflowCanvas.d.ts +16 -0
- package/dist/WorkflowCanvas.js +1064 -64
- package/dist/connection-form.d.ts +241 -0
- package/dist/connection-form.js +235 -0
- package/dist/context.d.ts +236 -38
- package/dist/context.js +46 -11
- package/dist/index.d.ts +10 -3
- package/dist/index.js +63 -14
- package/dist/routes.d.ts +84 -13
- package/dist/routes.js +7 -4
- package/dist/schema-discovery.d.ts +232 -0
- package/dist/schema-discovery.js +225 -0
- package/dist/source-connection.d.ts +82 -0
- package/dist/source-connection.js +176 -0
- package/dist/source-fields.d.ts +16 -14
- package/dist/source-fields.js +28 -18
- package/dist/ui/code-editor.d.ts +44 -2
- package/dist/ui/code-editor.js +10 -2
- package/dist/ui/combobox.d.ts +97 -0
- package/dist/ui/combobox.js +80 -0
- package/dist/ui/dialog.d.ts +13 -1
- package/dist/ui/dialog.js +13 -2
- package/dist/workflow/edges.d.ts +110 -0
- package/dist/workflow/edges.js +112 -0
- package/dist/workflow/graph.d.ts +39 -1
- package/dist/workflow/graph.js +212 -52
- package/dist/workflow/index.d.ts +8 -0
- package/dist/workflow/index.js +12 -0
- package/dist/workflow/lifecycle.d.ts +135 -0
- package/dist/workflow/lifecycle.js +216 -0
- package/dist/workflow/model.d.ts +17 -3
- package/dist/workflow/model.js +7 -2
- package/dist/workflow/nodes.d.ts +4 -1
- package/dist/workflow/nodes.js +198 -10
- package/dist/workflow/runs.d.ts +38 -0
- package/dist/workflow/runs.js +101 -0
- package/dist/workflow/shape.d.ts +169 -0
- package/dist/workflow/shape.js +249 -0
- package/dist/workflow/templates.d.ts +623 -0
- package/dist/workflow/templates.js +1102 -0
- package/dist/workflow/validate.d.ts +20 -1
- package/dist/workflow/validate.js +52 -3
- package/dist/workflow/wiring.d.ts +84 -0
- package/dist/workflow/wiring.js +254 -0
- package/package.json +9 -9
package/dist/routes.d.ts
CHANGED
|
@@ -96,25 +96,62 @@ export declare const DEFAULT_PUBLISH_BASE_PATH = "/publish";
|
|
|
96
96
|
export declare const DEFAULT_ACCESS_BASE_PATH = "/access";
|
|
97
97
|
export interface PipelineRoutes {
|
|
98
98
|
capabilities(): string;
|
|
99
|
+
/**
|
|
100
|
+
* What a call node could be pointed at: every workflow the live fleet
|
|
101
|
+
* announces it can execute.
|
|
102
|
+
*
|
|
103
|
+
* Beside `capabilities` and NOT part of it, because the two have opposite
|
|
104
|
+
* lifetimes. Capabilities report what resolved in the server process and
|
|
105
|
+
* cannot change without a redeploy; this is a snapshot of live workers with a
|
|
106
|
+
* resolution of about one heartbeat. A screen caching the second the way it
|
|
107
|
+
* caches the first would show the fleet as it was when the tab opened.
|
|
108
|
+
*/
|
|
109
|
+
callableWorkflows(): string;
|
|
99
110
|
connections(): string;
|
|
100
111
|
connection(id: string): string;
|
|
101
112
|
checkConnection(id: string): string;
|
|
102
|
-
/** Which connectors read through a connection. Named, so a refusal can say. */
|
|
103
|
-
connectionConnectors(id: string): string;
|
|
104
|
-
connectors(): string;
|
|
105
|
-
connector(id: string): string;
|
|
106
|
-
runConnector(id: string): string;
|
|
107
113
|
/**
|
|
108
|
-
*
|
|
114
|
+
* Reach a connection that has not been saved yet.
|
|
115
|
+
*
|
|
116
|
+
* A sibling of {@link PipelineRoutes.checkConnection} and not the same route
|
|
117
|
+
* with an absent id, because the two ask for different scopes and the
|
|
118
|
+
* difference is the point: checking a SAVED connection reaches an address
|
|
119
|
+
* somebody with `catalog:write` already chose and wrote down, so it asks only
|
|
120
|
+
* `catalog:read`; this one reaches an address supplied in the request, which
|
|
121
|
+
* under `catalog:read` would be a port scanner for anybody who may look at the
|
|
122
|
+
* catalog. It therefore asks `catalog:write`.
|
|
123
|
+
*
|
|
124
|
+
* The server declares its handler BEFORE `connections/:id/check`, because Nest
|
|
125
|
+
* matches in declaration order and `:id` captures the literal "check" happily.
|
|
126
|
+
* Nothing on this side can enforce that, and it is written down here because a
|
|
127
|
+
* console that gets a 404 from this path is most likely talking to a server
|
|
128
|
+
* where the two were reordered — which reads like a missing feature rather
|
|
129
|
+
* than like a routing bug.
|
|
130
|
+
*/
|
|
131
|
+
checkUnsavedConnection(): string;
|
|
132
|
+
/**
|
|
133
|
+
* Which pipelines read through a connection. Named, so a refusal can say.
|
|
134
|
+
*
|
|
135
|
+
* `.../connectors` before, and the rename is the question actually being
|
|
136
|
+
* asked rather than a tidy-up: an operator presses this before deleting a
|
|
137
|
+
* connection, and what they need is the list of things that would break —
|
|
138
|
+
* which is a list of workflows, because a workflow is the only thing anybody
|
|
139
|
+
* authors and a connector's name is an implementation detail of one.
|
|
140
|
+
*/
|
|
141
|
+
connectionWorkflows(id: string): string;
|
|
142
|
+
/**
|
|
143
|
+
* The internal records, read-only.
|
|
144
|
+
*
|
|
145
|
+
* **The one connector path left, and it is a `GET`.** There is no `POST`, no
|
|
146
|
+
* `DELETE` and no `.../run`: a connector is what a published workflow runs
|
|
147
|
+
* as, minted by `workflows/:id/publish` and removed with the graph. A builder
|
|
148
|
+
* for a route that creates one would be this package advertising a second way
|
|
149
|
+
* to author a pipeline, which is the thing that was removed.
|
|
109
150
|
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
* generic "describe this database" route because the connector is what
|
|
113
|
-
* already holds the address, the credential reference and the statement —
|
|
114
|
-
* a route taking those as arguments would be a way to point the server at
|
|
115
|
-
* any database at all.
|
|
151
|
+
* It stays because it answers what a graph cannot answer about itself — which
|
|
152
|
+
* id the run history and the incremental watermark are keyed on.
|
|
116
153
|
*/
|
|
117
|
-
|
|
154
|
+
connectors(): string;
|
|
118
155
|
runs(): string;
|
|
119
156
|
transforms(): string;
|
|
120
157
|
transform(id: string): string;
|
|
@@ -146,7 +183,41 @@ export interface PipelineRoutes {
|
|
|
146
183
|
*/
|
|
147
184
|
workflows(): string;
|
|
148
185
|
workflow(id: string): string;
|
|
186
|
+
/**
|
|
187
|
+
* Declare a graph finished, which is also what mints the connector it runs
|
|
188
|
+
* as. Its own path rather than a field on the save, because publishing is a
|
|
189
|
+
* claim that gets checked and a check that fails owes an explanation.
|
|
190
|
+
*/
|
|
191
|
+
publishWorkflow(id: string): string;
|
|
192
|
+
/** Back to draft. The connector keeps its id and its history, and stops. */
|
|
193
|
+
unpublishWorkflow(id: string): string;
|
|
194
|
+
/**
|
|
195
|
+
* Run it now. **The only run path there is** — `connectors/:id/run` is gone,
|
|
196
|
+
* and what it carried that this did not is `expectShrink`, which moved onto
|
|
197
|
+
* this one's body rather than being lost with it.
|
|
198
|
+
*/
|
|
149
199
|
runWorkflow(id: string): string;
|
|
200
|
+
/**
|
|
201
|
+
* When this graph runs, and whether it runs at all.
|
|
202
|
+
*
|
|
203
|
+
* A `PUT` of its own rather than fields on the save: a canvas autosaves, and
|
|
204
|
+
* a cron folded into every save a drag passes through is one a stale tab can
|
|
205
|
+
* silently revert.
|
|
206
|
+
*/
|
|
207
|
+
workflowSchedule(id: string): string;
|
|
208
|
+
/**
|
|
209
|
+
* What the system behind one source node looks like right now. Writes nothing.
|
|
210
|
+
*
|
|
211
|
+
* `connectors/:id/discover` before. It had to move rather than be dropped —
|
|
212
|
+
* discovery is how a type gets its shape, and a sink cannot commit into a
|
|
213
|
+
* type that does not exist — and it deliberately answers on a **draft**:
|
|
214
|
+
* requiring a published graph would require publishing a graph whose target
|
|
215
|
+
* type cannot be created until it is published.
|
|
216
|
+
*
|
|
217
|
+
* Per node, not per graph, because a graph may have several sources and only
|
|
218
|
+
* a source has a shape to report.
|
|
219
|
+
*/
|
|
220
|
+
discoverSourceSchema(id: string, nodeId: string): string;
|
|
150
221
|
/**
|
|
151
222
|
* Every per-type load expectation an operator has stored.
|
|
152
223
|
*
|
package/dist/routes.js
CHANGED
|
@@ -132,14 +132,13 @@ export function pipelineRoutes(basePath = DEFAULT_PIPELINE_BASE_PATH) {
|
|
|
132
132
|
const expectations = pipelineExpectationRoutes(base);
|
|
133
133
|
return {
|
|
134
134
|
capabilities: () => `${base}/capabilities`,
|
|
135
|
+
callableWorkflows: () => `${base}/callable-workflows`,
|
|
135
136
|
connections: () => `${base}/connections`,
|
|
136
137
|
connection: (id) => `${base}/connections/${encodeURIComponent(id)}`,
|
|
137
138
|
checkConnection: (id) => `${base}/connections/${encodeURIComponent(id)}/check`,
|
|
138
|
-
|
|
139
|
+
checkUnsavedConnection: () => `${base}/connections/check`,
|
|
140
|
+
connectionWorkflows: (id) => `${base}/connections/${encodeURIComponent(id)}/workflows`,
|
|
139
141
|
connectors: () => `${base}/connectors`,
|
|
140
|
-
connector: (id) => `${base}/connectors/${encodeURIComponent(id)}`,
|
|
141
|
-
runConnector: (id) => `${base}/connectors/${encodeURIComponent(id)}/run`,
|
|
142
|
-
discoverConnectorSchema: (id) => `${base}/connectors/${encodeURIComponent(id)}/discover`,
|
|
143
142
|
runs: () => `${base}/runs`,
|
|
144
143
|
transforms: () => `${base}/transforms`,
|
|
145
144
|
transform: (id) => `${base}/transforms/${encodeURIComponent(id)}`,
|
|
@@ -147,7 +146,11 @@ export function pipelineRoutes(basePath = DEFAULT_PIPELINE_BASE_PATH) {
|
|
|
147
146
|
tryTransform: () => `${base}/transforms/try`,
|
|
148
147
|
workflows: () => `${base}/workflows`,
|
|
149
148
|
workflow: (id) => `${base}/workflows/${encodeURIComponent(id)}`,
|
|
149
|
+
publishWorkflow: (id) => `${base}/workflows/${encodeURIComponent(id)}/publish`,
|
|
150
|
+
unpublishWorkflow: (id) => `${base}/workflows/${encodeURIComponent(id)}/unpublish`,
|
|
150
151
|
runWorkflow: (id) => `${base}/workflows/${encodeURIComponent(id)}/run`,
|
|
152
|
+
workflowSchedule: (id) => `${base}/workflows/${encodeURIComponent(id)}/schedule`,
|
|
153
|
+
discoverSourceSchema: (id, nodeId) => `${base}/workflows/${encodeURIComponent(id)}/nodes/${encodeURIComponent(nodeId)}/discover`,
|
|
151
154
|
loadExpectations: () => expectations.expectations(),
|
|
152
155
|
loadExpectation: (typeName) => expectations.expectation(typeName),
|
|
153
156
|
};
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Asking a source what its columns are, and turning the answer into a type.
|
|
3
|
+
*
|
|
4
|
+
* ITS OWN MODULE, AND WHY
|
|
5
|
+
* -----------------------
|
|
6
|
+
* All of this used to live inside `PipelineConsole`, reached from a connector
|
|
7
|
+
* editor, because a connector was the thing that held an address and a query.
|
|
8
|
+
* It is not any more: a source node on a graph is, and the route moved with it —
|
|
9
|
+
* `POST connectors/:id/discover` became
|
|
10
|
+
* `POST workflows/:id/nodes/:nodeId/discover`. The panel therefore has to be
|
|
11
|
+
* mountable from the canvas, and the canvas lives behind its own entry point
|
|
12
|
+
* because it imports `@xyflow/react`. Left where it was, either the canvas would
|
|
13
|
+
* import a screen it has nothing else to do with, or there would be two copies
|
|
14
|
+
* of the rules that decide whether a schema somebody ticked is publishable.
|
|
15
|
+
*
|
|
16
|
+
* Nothing here imports React Flow, so both entry points can reach it.
|
|
17
|
+
*
|
|
18
|
+
* WHAT THIS SCREEN IS FOR
|
|
19
|
+
* -----------------------
|
|
20
|
+
* One thing: never letting a guess become a type. Discovery reports columns it
|
|
21
|
+
* could not type, and the job here is to keep those visibly out of the proposal
|
|
22
|
+
* until a person says what they are. A wrong type becomes a wrong column in a
|
|
23
|
+
* lake nobody re-checks, and everything downstream reads it as fact rather than
|
|
24
|
+
* as the guess it was.
|
|
25
|
+
*/
|
|
26
|
+
import type { ScalarType } from '@dudousxd/nestjs-catalog/client';
|
|
27
|
+
/**
|
|
28
|
+
* How this console reaches discovery and publication.
|
|
29
|
+
*
|
|
30
|
+
* A bridge rather than two more calls made inline, because a host embedding
|
|
31
|
+
* these screens may route either through its own gateway — and because
|
|
32
|
+
* `createType` is genuinely optional: the publish route needs `catalog:write`
|
|
33
|
+
* and per-type ownership, and a host that has deliberately not given its console
|
|
34
|
+
* that has to be able to say so without the screen pretending the button is
|
|
35
|
+
* merely broken.
|
|
36
|
+
*
|
|
37
|
+
* `discover` takes a workflow and a node, not a connector. See
|
|
38
|
+
* {@link CatalogClient.discoverSourceSchema} for why the pair replaced the
|
|
39
|
+
* single id, and why it answers on a draft.
|
|
40
|
+
*/
|
|
41
|
+
export interface SchemaDiscoveryBridge {
|
|
42
|
+
discover(workflowId: string, nodeId: string): Promise<ConnectorSchemaDiscovery>;
|
|
43
|
+
/**
|
|
44
|
+
* Create the type from what a person confirmed. Absent is a supported state —
|
|
45
|
+
* the panel does everything up to the confirmation and then prints the exact
|
|
46
|
+
* request it would have sent.
|
|
47
|
+
*/
|
|
48
|
+
createType?(draft: DiscoveredTypeDraft): Promise<unknown>;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* What `POST pipeline/workflows/:id/nodes/:nodeId/discover` answers, mirrored
|
|
52
|
+
* for the browser.
|
|
53
|
+
*
|
|
54
|
+
* Mirrored rather than imported, exactly as `PipelineCapabilities` is in
|
|
55
|
+
* `context.tsx`: the authority is `schema-discovery.ts` in
|
|
56
|
+
* `@dudousxd/nestjs-catalog-pipeline`, which is a NestJS package with a
|
|
57
|
+
* `node:fs` import in its dependency graph and has no business in a browser
|
|
58
|
+
* bundle. The two are kept honest by being the documented response of one route,
|
|
59
|
+
* and by this screen treating every field it renders as something the server may
|
|
60
|
+
* not have sent.
|
|
61
|
+
*
|
|
62
|
+
* Named after the connector still, matching the server's own name for it, and
|
|
63
|
+
* the FIELDS are what changed: `connectorId`/`connectorName` became
|
|
64
|
+
* `workflowId`/`nodeId`/`nodeName`/`kind`. A connector's own kind and config
|
|
65
|
+
* stopped being read the moment it gained a `workflowId`, and every connector
|
|
66
|
+
* has one now — so a connector id here would have named a row whose
|
|
67
|
+
* configuration has nothing to do with the columns underneath it. A graph can
|
|
68
|
+
* also have several sources, which the old pair could not express at all.
|
|
69
|
+
*/
|
|
70
|
+
export interface DiscoveredColumn {
|
|
71
|
+
/**
|
|
72
|
+
* The column as the source spells it, which is also the property name. The
|
|
73
|
+
* server explains at length why it is not tidied: the store matches records to
|
|
74
|
+
* properties by property NAME, so a renamed property is a column that loads
|
|
75
|
+
* null on every run and reports success.
|
|
76
|
+
*/
|
|
77
|
+
name: string;
|
|
78
|
+
/** `null` means discovery reached no conclusion — which is not a type. */
|
|
79
|
+
type: ScalarType | null;
|
|
80
|
+
confidence: 'reported' | 'inferred' | 'unknown';
|
|
81
|
+
sourceType: string;
|
|
82
|
+
/** `null` where the source did not say. Postgres never does. */
|
|
83
|
+
nullable: boolean | null;
|
|
84
|
+
note?: string;
|
|
85
|
+
}
|
|
86
|
+
export interface SchemaDrift {
|
|
87
|
+
added: string[];
|
|
88
|
+
removed: string[];
|
|
89
|
+
retyped: Array<{
|
|
90
|
+
property: string;
|
|
91
|
+
was: ScalarType;
|
|
92
|
+
now: ScalarType;
|
|
93
|
+
}>;
|
|
94
|
+
}
|
|
95
|
+
export interface ConnectorSchemaDiscovery {
|
|
96
|
+
/** The graph this node belongs to, so a report can be traced back to it. */
|
|
97
|
+
workflowId: string;
|
|
98
|
+
nodeId: string;
|
|
99
|
+
nodeName: string;
|
|
100
|
+
/** What kind of system the node reads from, as the server resolved it. */
|
|
101
|
+
kind: string;
|
|
102
|
+
/**
|
|
103
|
+
* The type the graph's sink commits — the graph's, not the node's, because a
|
|
104
|
+
* source node has none. Empty on a draft with no sink drawn yet, which reads
|
|
105
|
+
* back as `typeExists: false` and is the honest answer: there is nothing for
|
|
106
|
+
* the columns to have drifted from.
|
|
107
|
+
*/
|
|
108
|
+
targetType: string;
|
|
109
|
+
typeExists: boolean;
|
|
110
|
+
basis: 'driver' | 'sample';
|
|
111
|
+
sampled: number;
|
|
112
|
+
caveat: string;
|
|
113
|
+
columns: DiscoveredColumn[];
|
|
114
|
+
drift: SchemaDrift | null;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* The discovery response, checked rather than assumed.
|
|
118
|
+
*
|
|
119
|
+
* The client returns `unknown` because this package must not import
|
|
120
|
+
* `@dudousxd/nestjs-catalog-pipeline` — it would drag database drivers behind
|
|
121
|
+
* optional imports into a browser bundle — so the shape above is a mirror of a
|
|
122
|
+
* contract held somewhere else. A mirror can fall behind: a console talking to
|
|
123
|
+
* an older server gets a body with no `columns`, and rendering it would throw
|
|
124
|
+
* somewhere deep in the table with a message about `undefined.map`.
|
|
125
|
+
*
|
|
126
|
+
* Checked at the seam, once, so the failure names itself. Only the fields this
|
|
127
|
+
* screen would crash without — validating every column here would be
|
|
128
|
+
* re-implementing the server's own shape in a place that cannot be kept honest.
|
|
129
|
+
*/
|
|
130
|
+
export declare function narrowDiscovery(answer: unknown): ConnectorSchemaDiscovery;
|
|
131
|
+
/** The subset of a published type this screen is able to state. */
|
|
132
|
+
export interface DiscoveredTypeDraft {
|
|
133
|
+
name: string;
|
|
134
|
+
properties: Array<{
|
|
135
|
+
name: string;
|
|
136
|
+
columnName: string;
|
|
137
|
+
type: ScalarType;
|
|
138
|
+
nullable: boolean;
|
|
139
|
+
}>;
|
|
140
|
+
}
|
|
141
|
+
/** What a person decided about one discovered column. */
|
|
142
|
+
export interface ColumnChoice {
|
|
143
|
+
include: boolean;
|
|
144
|
+
/** Empty means nothing has been chosen yet — see {@link initialChoices}. */
|
|
145
|
+
type: ScalarType | '';
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Where the checkboxes start, and why the untyped ones start off.
|
|
149
|
+
*
|
|
150
|
+
* A column discovery could type is proposed; a column it could not is present,
|
|
151
|
+
* visible, and **excluded** until somebody says what it is. Pre-selecting it
|
|
152
|
+
* with a plausible default is the failure that matters — a wrong type becomes a
|
|
153
|
+
* wrong column in a lake nobody re-checks, and everything downstream reads it
|
|
154
|
+
* as fact rather than as the guess it was.
|
|
155
|
+
*
|
|
156
|
+
* Keyed by position rather than by name on purpose: `SELECT a.id, b.id` gives
|
|
157
|
+
* two columns called `id`, and a map would silently merge their choices.
|
|
158
|
+
*/
|
|
159
|
+
export declare function initialChoices(columns: DiscoveredColumn[]): ColumnChoice[];
|
|
160
|
+
/**
|
|
161
|
+
* The type this would create, and everything standing in the way of creating it.
|
|
162
|
+
*
|
|
163
|
+
* Pure, and separate from the panel, because these are the rules that decide
|
|
164
|
+
* whether a schema somebody approved is one the publish route will accept — and
|
|
165
|
+
* rules that live inside a component are rules only a rendered DOM can check.
|
|
166
|
+
*
|
|
167
|
+
* `columnName` is sent as well as `name`, and they are the same string. The
|
|
168
|
+
* publish service maps `columnName` to `sourceColumn` and `name` to the property
|
|
169
|
+
* itself, and the store reads records by property name; sending them equal is
|
|
170
|
+
* what makes a load match. See the server's `DiscoveredColumn.name`.
|
|
171
|
+
*/
|
|
172
|
+
export declare function proposalFrom(discovery: ConnectorSchemaDiscovery, choices: ColumnChoice[]): {
|
|
173
|
+
draft: DiscoveredTypeDraft;
|
|
174
|
+
problems: string[];
|
|
175
|
+
};
|
|
176
|
+
/**
|
|
177
|
+
* What the source looks like, offered to the person who has to name it.
|
|
178
|
+
*
|
|
179
|
+
* Only for a source node on a SAVED graph, and deliberately not only on a
|
|
180
|
+
* published one. Discovery reads the stored node, so there is nothing to read
|
|
181
|
+
* from a canvas nobody has saved — but requiring a published graph would be
|
|
182
|
+
* circular: a sink cannot commit into a type that does not exist, so the type
|
|
183
|
+
* has to be creatable before the graph is declared finished. `disabledReason`
|
|
184
|
+
* is how the caller states which of those two it is; a panel that simply hid
|
|
185
|
+
* itself would leave somebody hunting for a button that used to be there.
|
|
186
|
+
*
|
|
187
|
+
* The screen's job is to make the difference between what was *reported* and
|
|
188
|
+
* what was *inferred* impossible to miss, because that difference is the whole
|
|
189
|
+
* value of the report. A driver describing a `NUMERIC` column is the database
|
|
190
|
+
* speaking; forty records that all happened to hold a string is a sample, and
|
|
191
|
+
* the forty-first is where somebody finds out.
|
|
192
|
+
*/
|
|
193
|
+
export declare function SchemaDiscoveryPanel({ workflowId, nodeId, bridge, disabledReason, onSave, saving, onDiscovered, }: {
|
|
194
|
+
workflowId: string;
|
|
195
|
+
nodeId: string;
|
|
196
|
+
bridge: SchemaDiscoveryBridge;
|
|
197
|
+
/** Why the button cannot be pressed, said out loud. Absent means it can. */
|
|
198
|
+
disabledReason?: string;
|
|
199
|
+
/**
|
|
200
|
+
* Do the thing `disabledReason` asks for, from here.
|
|
201
|
+
*
|
|
202
|
+
* Both reasons this panel can be disabled for are "save it first", and until
|
|
203
|
+
* this existed the panel said so while the save control lived in a header the
|
|
204
|
+
* panel is not even in — the inspector is a side sheet over the canvas. A
|
|
205
|
+
* reader was being told to do something with no way to do it where they were
|
|
206
|
+
* standing, and the report that produced was "it's disabled and I can't see
|
|
207
|
+
* anything" rather than "ah, I should save".
|
|
208
|
+
*
|
|
209
|
+
* Optional because a host may mount this panel somewhere with no draft behind
|
|
210
|
+
* it; the sentence alone is still correct, just less useful.
|
|
211
|
+
*/
|
|
212
|
+
onSave?: () => void;
|
|
213
|
+
/** Whether that save is in flight, so the button says so rather than lying. */
|
|
214
|
+
saving?: boolean;
|
|
215
|
+
/**
|
|
216
|
+
* Told to whoever mounted this, because the answer outlives the panel.
|
|
217
|
+
*
|
|
218
|
+
* This component holds the report in state, and the canvas unmounts it the
|
|
219
|
+
* moment the inspector closes — so the columns a person just read would be
|
|
220
|
+
* gone by the time they looked at the problems rail, which is the one place
|
|
221
|
+
* the same answer has anything else to say. The canvas keeps them per node
|
|
222
|
+
* and hands them to `validateWorkflow`; see `workflow/shape.ts`.
|
|
223
|
+
*
|
|
224
|
+
* A callback rather than a query the canvas fires itself, because discovery
|
|
225
|
+
* is a **read of a live source** behind a `POST`: opening a graph with four
|
|
226
|
+
* source nodes must not open four database connections nobody asked for. The
|
|
227
|
+
* shape check is therefore answered exactly where somebody asked the
|
|
228
|
+
* question, and stays silent everywhere else — which is what that module is
|
|
229
|
+
* written around.
|
|
230
|
+
*/
|
|
231
|
+
onDiscovered?: (nodeId: string, discovery: ConnectorSchemaDiscovery) => void;
|
|
232
|
+
}): import("react/jsx-runtime").JSX.Element;
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
|
|
2
|
+
import { useMutation } from '@tanstack/react-query';
|
|
3
|
+
import { CircleCheck, Loader2, Save, ScanSearch, TriangleAlert } from 'lucide-react';
|
|
4
|
+
import { useState } from 'react';
|
|
5
|
+
import { cn } from './cn';
|
|
6
|
+
import { Button } from './ui/button';
|
|
7
|
+
const MUTED = 'text-zinc-400 dark:text-zinc-500';
|
|
8
|
+
const RULE = 'border-zinc-200 dark:border-zinc-800';
|
|
9
|
+
/**
|
|
10
|
+
* The discovery response, checked rather than assumed.
|
|
11
|
+
*
|
|
12
|
+
* The client returns `unknown` because this package must not import
|
|
13
|
+
* `@dudousxd/nestjs-catalog-pipeline` — it would drag database drivers behind
|
|
14
|
+
* optional imports into a browser bundle — so the shape above is a mirror of a
|
|
15
|
+
* contract held somewhere else. A mirror can fall behind: a console talking to
|
|
16
|
+
* an older server gets a body with no `columns`, and rendering it would throw
|
|
17
|
+
* somewhere deep in the table with a message about `undefined.map`.
|
|
18
|
+
*
|
|
19
|
+
* Checked at the seam, once, so the failure names itself. Only the fields this
|
|
20
|
+
* screen would crash without — validating every column here would be
|
|
21
|
+
* re-implementing the server's own shape in a place that cannot be kept honest.
|
|
22
|
+
*/
|
|
23
|
+
export function narrowDiscovery(answer) {
|
|
24
|
+
if (!answer || typeof answer !== 'object' || !Array.isArray(Reflect.get(answer, 'columns'))) {
|
|
25
|
+
throw new Error('The server answered the discovery request with something this console does not recognise. ' +
|
|
26
|
+
'It is most likely running a version of @dudousxd/nestjs-catalog-pipeline older than this ' +
|
|
27
|
+
'console expects.');
|
|
28
|
+
}
|
|
29
|
+
return Object.assign(Object.create(null), answer);
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Every type the catalog can hold, in the order a person scans for one.
|
|
33
|
+
*
|
|
34
|
+
* `unknown` is last and named plainly, because choosing it is a decision: the
|
|
35
|
+
* store gives it a TEXT column and stringifies whatever arrives. It is the right
|
|
36
|
+
* answer for a column nobody can classify and the wrong answer for one nobody
|
|
37
|
+
* looked at, which is exactly why discovery never picks it on anyone's behalf.
|
|
38
|
+
*/
|
|
39
|
+
const SCALAR_TYPES = [
|
|
40
|
+
'string',
|
|
41
|
+
'number',
|
|
42
|
+
'boolean',
|
|
43
|
+
'date',
|
|
44
|
+
'uuid',
|
|
45
|
+
'json',
|
|
46
|
+
'unknown',
|
|
47
|
+
];
|
|
48
|
+
/**
|
|
49
|
+
* Where the checkboxes start, and why the untyped ones start off.
|
|
50
|
+
*
|
|
51
|
+
* A column discovery could type is proposed; a column it could not is present,
|
|
52
|
+
* visible, and **excluded** until somebody says what it is. Pre-selecting it
|
|
53
|
+
* with a plausible default is the failure that matters — a wrong type becomes a
|
|
54
|
+
* wrong column in a lake nobody re-checks, and everything downstream reads it
|
|
55
|
+
* as fact rather than as the guess it was.
|
|
56
|
+
*
|
|
57
|
+
* Keyed by position rather than by name on purpose: `SELECT a.id, b.id` gives
|
|
58
|
+
* two columns called `id`, and a map would silently merge their choices.
|
|
59
|
+
*/
|
|
60
|
+
export function initialChoices(columns) {
|
|
61
|
+
return columns.map((column) => ({ include: column.type !== null, type: column.type ?? '' }));
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* The type this would create, and everything standing in the way of creating it.
|
|
65
|
+
*
|
|
66
|
+
* Pure, and separate from the panel, because these are the rules that decide
|
|
67
|
+
* whether a schema somebody approved is one the publish route will accept — and
|
|
68
|
+
* rules that live inside a component are rules only a rendered DOM can check.
|
|
69
|
+
*
|
|
70
|
+
* `columnName` is sent as well as `name`, and they are the same string. The
|
|
71
|
+
* publish service maps `columnName` to `sourceColumn` and `name` to the property
|
|
72
|
+
* itself, and the store reads records by property name; sending them equal is
|
|
73
|
+
* what makes a load match. See the server's `DiscoveredColumn.name`.
|
|
74
|
+
*/
|
|
75
|
+
export function proposalFrom(discovery, choices) {
|
|
76
|
+
const chosen = discovery.columns
|
|
77
|
+
.map((column, index) => ({ column, choice: choices[index] ?? { include: false, type: '' } }))
|
|
78
|
+
.filter((entry) => entry.choice.include);
|
|
79
|
+
const problems = [];
|
|
80
|
+
const untyped = chosen.filter((entry) => entry.choice.type === '');
|
|
81
|
+
if (untyped.length > 0) {
|
|
82
|
+
problems.push(`Choose a type for ${untyped.map((entry) => `"${entry.column.name}"`).join(', ')}, or leave ${untyped.length === 1 ? 'it' : 'them'} out.`);
|
|
83
|
+
}
|
|
84
|
+
const counts = new Map();
|
|
85
|
+
for (const entry of chosen) {
|
|
86
|
+
counts.set(entry.column.name, (counts.get(entry.column.name) ?? 0) + 1);
|
|
87
|
+
}
|
|
88
|
+
const duplicated = [...counts].filter(([, count]) => count > 1).map(([name]) => `"${name}"`);
|
|
89
|
+
if (duplicated.length > 0) {
|
|
90
|
+
problems.push(`${duplicated.join(', ')} appears more than once. A record is an object, so only one of them can survive — leave the others out.`);
|
|
91
|
+
}
|
|
92
|
+
const properties = [];
|
|
93
|
+
for (const entry of chosen) {
|
|
94
|
+
if (entry.choice.type === '')
|
|
95
|
+
continue;
|
|
96
|
+
properties.push({
|
|
97
|
+
name: entry.column.name,
|
|
98
|
+
columnName: entry.column.name,
|
|
99
|
+
type: entry.choice.type,
|
|
100
|
+
// Not stated is published nullable. A column marked NOT NULL that the
|
|
101
|
+
// source sometimes leaves empty is a load that fails at 3am; a nullable
|
|
102
|
+
// column that never holds a null costs nothing at all.
|
|
103
|
+
nullable: entry.column.nullable ?? true,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
if (properties.length === 0) {
|
|
107
|
+
problems.push('Nothing is selected, and a type with no properties is refused.');
|
|
108
|
+
}
|
|
109
|
+
return { draft: { name: discovery.targetType, properties }, problems };
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* What the source looks like, offered to the person who has to name it.
|
|
113
|
+
*
|
|
114
|
+
* Only for a source node on a SAVED graph, and deliberately not only on a
|
|
115
|
+
* published one. Discovery reads the stored node, so there is nothing to read
|
|
116
|
+
* from a canvas nobody has saved — but requiring a published graph would be
|
|
117
|
+
* circular: a sink cannot commit into a type that does not exist, so the type
|
|
118
|
+
* has to be creatable before the graph is declared finished. `disabledReason`
|
|
119
|
+
* is how the caller states which of those two it is; a panel that simply hid
|
|
120
|
+
* itself would leave somebody hunting for a button that used to be there.
|
|
121
|
+
*
|
|
122
|
+
* The screen's job is to make the difference between what was *reported* and
|
|
123
|
+
* what was *inferred* impossible to miss, because that difference is the whole
|
|
124
|
+
* value of the report. A driver describing a `NUMERIC` column is the database
|
|
125
|
+
* speaking; forty records that all happened to hold a string is a sample, and
|
|
126
|
+
* the forty-first is where somebody finds out.
|
|
127
|
+
*/
|
|
128
|
+
export function SchemaDiscoveryPanel({ workflowId, nodeId, bridge, disabledReason, onSave, saving = false, onDiscovered, }) {
|
|
129
|
+
// Counted rather than timestamped so a second discovery always remounts the
|
|
130
|
+
// table below, dropping the choices somebody made against a column list that
|
|
131
|
+
// no longer exists.
|
|
132
|
+
const [discovered, setDiscovered] = useState();
|
|
133
|
+
const discover = useMutation({
|
|
134
|
+
mutationFn: () => bridge.discover(workflowId, nodeId),
|
|
135
|
+
onSuccess: (value) => {
|
|
136
|
+
setDiscovered((previous) => ({ run: (previous?.run ?? 0) + 1, value }));
|
|
137
|
+
onDiscovered?.(nodeId, value);
|
|
138
|
+
},
|
|
139
|
+
});
|
|
140
|
+
return (_jsxs("div", { className: cn('rounded-md border border-dashed p-3', RULE), children: [_jsxs("div", { className: "flex flex-wrap items-center gap-2", children: [_jsxs(Button, { variant: "outline", size: "sm", onClick: () => discover.mutate(), disabled: discover.isPending || disabledReason !== undefined, children: [discover.isPending ? (_jsx(Loader2, { size: 12, className: "animate-spin" })) : (_jsx(ScanSearch, { size: 12 })), discover.isPending ? 'Reading the source…' : 'Discover schema'] }), _jsx("span", { className: cn('text-[11px]', MUTED), children: disabledReason ??
|
|
141
|
+
'Reads the source and reports its columns. Creates nothing on its own, and works on a draft — the type has to exist before a sink can commit into it.' }), disabledReason !== undefined && onSave && (_jsxs(Button, { variant: "secondary", size: "sm", onClick: onSave, disabled: saving, children: [saving ? _jsx(Loader2, { size: 12, className: "animate-spin" }) : _jsx(Save, { size: 12 }), saving ? 'Saving…' : 'Save now'] }))] }), disabledReason !== undefined && onSave && (_jsx("p", { className: cn('mt-1.5 text-[11px]', MUTED), children: "Saving stores the graph as a draft. It does not publish it, and nothing runs because of it." })), discover.error ? (_jsx("p", { className: "mt-2 text-[11px] text-red-600", children: discover.error instanceof Error ? discover.error.message : 'Could not read the source.' })) : null, discovered && (_jsx(DiscoveredSchema, { discovery: discovered.value, createType: bridge.createType }, discovered.run))] }));
|
|
142
|
+
}
|
|
143
|
+
function DiscoveredSchema({ discovery, createType, }) {
|
|
144
|
+
const [choices, setChoices] = useState(() => initialChoices(discovery.columns));
|
|
145
|
+
const { draft, problems } = proposalFrom(discovery, choices);
|
|
146
|
+
const setChoice = (index, next) => setChoices((current) => current.map((choice, position) => (position === index ? { ...choice, ...next } : choice)));
|
|
147
|
+
return (_jsxs("div", { className: "mt-3 space-y-3", children: [_jsx("p", { className: cn('text-[11px]', MUTED), children: discovery.caveat }), discovery.targetType ? (_jsx(DriftReport, { drift: discovery.drift, targetType: discovery.targetType })) : (_jsx("p", { className: cn('text-[11px]', MUTED), children: "This graph has no sink naming an object type yet, so there is nothing to compare these columns against. Draw one and choose a type, and this becomes the way to create it." })), discovery.columns.length === 0 ? (_jsx("p", { className: cn('text-[11px]', MUTED), children: "No columns came back." })) : (_jsxs("table", { className: "w-full text-left text-[11px]", children: [_jsx("thead", { className: cn('font-mono uppercase tracking-[0.14em]', MUTED), children: _jsxs("tr", { children: [_jsx("th", { className: "w-6 pb-1 font-normal", children: _jsx("span", { className: "sr-only", children: "Include" }) }), _jsx("th", { className: "pb-1 font-normal", children: "Column" }), _jsx("th", { className: "pb-1 font-normal", children: "Type" }), _jsx("th", { className: "pb-1 font-normal", children: "Null" }), _jsx("th", { className: "pb-1 font-normal", children: "Source" })] }) }), _jsx("tbody", { children: discovery.columns.map((column, index) => (_jsx(ColumnRow
|
|
148
|
+
// The source may legally return two columns of one name, so the
|
|
149
|
+
// position has to be part of the key: on its own the name would
|
|
150
|
+
// collide, and React would reuse one row's choice for the other.
|
|
151
|
+
, { column: column, choice: choices[index] ?? { include: false, type: '' }, onChange: (next) => setChoice(index, next) }, `${column.name}-${index}`))) })] })), problems.map((problem) => (_jsxs("p", { className: "flex items-start gap-1.5 text-[11px] text-amber-600", children: [_jsx(TriangleAlert, { size: 11, className: "mt-0.5 shrink-0", "aria-hidden": true }), problem] }, problem))), discovery.targetType ? (_jsx(Confirmation, { discovery: discovery, draft: draft, problems: problems, createType: createType })) : null] }));
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* The act that creates the type — the only thing on this screen that writes.
|
|
155
|
+
*
|
|
156
|
+
* Its own component because it has three states and they are not variations on
|
|
157
|
+
* a button: nothing has been created yet, something has, or this console has no
|
|
158
|
+
* way to create anything and has to hand over the request instead.
|
|
159
|
+
*/
|
|
160
|
+
function Confirmation({ discovery, draft, problems, createType, }) {
|
|
161
|
+
const [created, setCreated] = useState(false);
|
|
162
|
+
const create = useMutation({
|
|
163
|
+
mutationFn: () => {
|
|
164
|
+
if (!createType)
|
|
165
|
+
return Promise.reject(new Error('This console cannot create types.'));
|
|
166
|
+
return createType(draft);
|
|
167
|
+
},
|
|
168
|
+
onSuccess: () => setCreated(true),
|
|
169
|
+
});
|
|
170
|
+
if (!createType)
|
|
171
|
+
return _jsx(UnwiredConfirmation, { draft: draft });
|
|
172
|
+
if (created) {
|
|
173
|
+
return (_jsxs("p", { className: "flex items-center gap-1.5 text-[11px] text-emerald-600", children: [_jsx(CircleCheck, { size: 11, "aria-hidden": true }), discovery.targetType, " now exists with ", draft.properties.length, " properties. Running this workflow will load into it."] }));
|
|
174
|
+
}
|
|
175
|
+
return (_jsxs("div", { className: "space-y-1", children: [_jsx(Button, { onClick: () => create.mutate(), disabled: problems.length > 0 || create.isPending, size: "sm", children: create.isPending
|
|
176
|
+
? 'Creating…'
|
|
177
|
+
: discovery.typeExists
|
|
178
|
+
? `Update ${discovery.targetType}`
|
|
179
|
+
: `Create ${discovery.targetType}` }), _jsx("p", { className: cn('text-[11px]', MUTED), children: discovery.typeExists
|
|
180
|
+
? 'Adds and retypes properties. Display names, descriptions and classifications somebody wrote are kept.'
|
|
181
|
+
: 'This is the act that creates the type. Nothing has been created up to here.' }), create.error ? (_jsx("p", { className: "text-[11px] text-red-600", children: create.error instanceof Error ? create.error.message : 'Could not create it.' })) : null] }));
|
|
182
|
+
}
|
|
183
|
+
/** One column, with the two decisions a person can make about it. */
|
|
184
|
+
function ColumnRow({ column, choice, onChange, }) {
|
|
185
|
+
return (_jsxs(_Fragment, { children: [_jsxs("tr", { className: cn('border-t', RULE), children: [_jsx("td", { className: "py-1 align-top", children: _jsx("input", { type: "checkbox", checked: choice.include, "aria-label": `Include ${column.name}`, onChange: (event) => onChange({ include: event.target.checked }) }) }), _jsx("td", { className: "py-1 align-top font-mono", children: column.name }), _jsxs("td", { className: "py-1 align-top", children: [_jsxs("select", { value: choice.type, "aria-label": `Type for ${column.name}`, onChange: (event) => {
|
|
186
|
+
const next = SCALAR_TYPES.find((scalar) => scalar === event.target.value);
|
|
187
|
+
onChange({ type: next ?? '' });
|
|
188
|
+
}, className: cn('rounded-sm border bg-transparent px-1 py-0.5 font-mono text-[11px]', RULE,
|
|
189
|
+
// The popup is the browser's, so its background is not this
|
|
190
|
+
// component's to style — but the options inherit the control's
|
|
191
|
+
// colour, and without an explicit dark background a dark theme
|
|
192
|
+
// renders light-on-light in some engines.
|
|
193
|
+
'text-zinc-900 dark:bg-zinc-900 dark:text-zinc-100', choice.type === '' && 'text-amber-600 dark:text-amber-500'), children: [_jsx("option", { value: "", children: "not typed" }), SCALAR_TYPES.map((scalar) => (_jsx("option", { value: scalar, children: scalar }, scalar)))] }), column.confidence === 'inferred' && (_jsx("span", { className: cn('ml-1.5 font-mono text-[10px]', MUTED), children: "inferred" }))] }), _jsx("td", { className: cn('py-1 align-top font-mono', MUTED), children: column.nullable === null ? 'not stated' : column.nullable ? 'nullable' : 'not null' }), _jsx("td", { className: cn('py-1 align-top font-mono', MUTED), children: column.sourceType })] }), column.note && (_jsxs("tr", { children: [_jsx("td", {}), _jsx("td", { colSpan: 4, className: "pb-1.5 text-[11px] text-amber-600", children: column.note })] }))] }));
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* What the source says now against what the type says now.
|
|
197
|
+
*
|
|
198
|
+
* The part worth having. A first discovery happens once per source; drift
|
|
199
|
+
* happens for as long as the pipeline exists, and every one of these is silent
|
|
200
|
+
* today — an added column is dropped by the store, a removed one loads as null,
|
|
201
|
+
* and a retyped one is coerced into whatever the catalog still believes.
|
|
202
|
+
*/
|
|
203
|
+
function DriftReport({ drift, targetType }) {
|
|
204
|
+
if (!drift) {
|
|
205
|
+
return (_jsxs("p", { className: cn('text-[11px]', MUTED), children: [targetType, " does not exist yet, so there is nothing to compare against."] }));
|
|
206
|
+
}
|
|
207
|
+
const quiet = drift.added.length === 0 && drift.removed.length === 0 && drift.retyped.length === 0;
|
|
208
|
+
if (quiet) {
|
|
209
|
+
return (_jsxs("p", { className: "flex items-center gap-1.5 text-[11px] text-emerald-600", children: [_jsx(CircleCheck, { size: 11, "aria-hidden": true }), "The source still matches ", targetType, "."] }));
|
|
210
|
+
}
|
|
211
|
+
return (_jsxs("div", { className: "space-y-1 rounded-md border border-amber-300 bg-amber-50 p-2 dark:border-amber-900 dark:bg-amber-950/30", children: [_jsxs("p", { className: "font-mono text-[10px] uppercase tracking-[0.14em] text-amber-700 dark:text-amber-400", children: ["Drift since ", targetType, " was written down"] }), drift.added.length > 0 && (_jsxs("p", { className: "text-[11px] text-amber-700 dark:text-amber-300", children: ["New in the source, and dropped by every load until the type has them:", ' ', _jsx("span", { className: "font-mono", children: drift.added.join(', ') })] })), drift.removed.length > 0 && (_jsxs("p", { className: "text-[11px] text-amber-700 dark:text-amber-300", children: ["On the type, absent from the source, so they load as null:", ' ', _jsx("span", { className: "font-mono", children: drift.removed.join(', ') })] })), drift.retyped.map((change) => (_jsxs("p", { className: "text-[11px] text-amber-700 dark:text-amber-300", children: [_jsx("span", { className: "font-mono", children: change.property }), " is ", change.now, " in the source and", ' ', change.was, " here \u2014 every load coerces it, and a value that will not convert becomes null."] }, change.property)))] }));
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* The confirmation, for a console whose host did not wire one.
|
|
215
|
+
*
|
|
216
|
+
* The request is printed rather than the button being greyed out with a
|
|
217
|
+
* shrug. Creating a type is a `PUT` a person can make with a terminal, and a
|
|
218
|
+
* host that deliberately kept type creation out of its console — because the
|
|
219
|
+
* publish route is owned by an application principal, or because schema changes
|
|
220
|
+
* go through review — has a reader here who still needs the body somebody
|
|
221
|
+
* approved on this screen.
|
|
222
|
+
*/
|
|
223
|
+
function UnwiredConfirmation({ draft }) {
|
|
224
|
+
return (_jsxs("div", { className: "space-y-1", children: [_jsx("p", { className: cn('text-[11px]', MUTED), children: "This console cannot create types \u2014 nothing is wired to the publish route. What was confirmed above is this request:" }), _jsx("pre", { className: cn('overflow-x-auto rounded-md border p-2 font-mono text-[10px]', RULE, 'bg-zinc-50 dark:bg-zinc-950'), children: `PUT /publish/${draft.name}/schema\n${JSON.stringify(draft, null, 2)}` })] }));
|
|
225
|
+
}
|