@dudousxd/nestjs-catalog-react 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -13
- package/dist/ConnectionPanel.d.ts +11 -7
- package/dist/ConnectionPanel.js +65 -28
- package/dist/PipelineConsole.d.ts +23 -126
- package/dist/PipelineConsole.js +39 -551
- package/dist/WorkflowCanvas.js +401 -29
- package/dist/context.d.ts +161 -37
- package/dist/context.js +35 -11
- package/dist/index.d.ts +6 -3
- package/dist/index.js +40 -14
- package/dist/routes.d.ts +54 -13
- package/dist/routes.js +5 -4
- package/dist/schema-discovery.d.ts +216 -0
- package/dist/schema-discovery.js +225 -0
- package/dist/source-fields.d.ts +1 -12
- package/dist/source-fields.js +13 -16
- package/dist/ui/code-editor.d.ts +44 -2
- package/dist/ui/code-editor.js +10 -2
- package/dist/ui/dialog.d.ts +13 -1
- package/dist/ui/dialog.js +13 -2
- package/dist/workflow/graph.js +31 -9
- package/dist/workflow/index.d.ts +2 -0
- package/dist/workflow/index.js +7 -0
- package/dist/workflow/lifecycle.d.ts +164 -0
- package/dist/workflow/lifecycle.js +249 -0
- package/dist/workflow/model.d.ts +1 -1
- package/dist/workflow/nodes.js +10 -1
- package/dist/workflow/runs.d.ts +38 -0
- package/dist/workflow/runs.js +101 -0
- package/dist/workflow/shape.d.ts +169 -0
- package/dist/workflow/shape.js +249 -0
- package/dist/workflow/templates.d.ts +623 -0
- package/dist/workflow/templates.js +1092 -0
- package/dist/workflow/validate.d.ts +20 -1
- package/dist/workflow/validate.js +41 -3
- package/package.json +2 -2
package/dist/context.d.ts
CHANGED
|
@@ -116,21 +116,6 @@ export interface ConnectionInput {
|
|
|
116
116
|
config: Record<string, unknown>;
|
|
117
117
|
secretEnvVar?: string;
|
|
118
118
|
}
|
|
119
|
-
/** Same rule as {@link ConnectionInput}: authored fields only, never state. */
|
|
120
|
-
export interface ConnectorInput {
|
|
121
|
-
id?: string;
|
|
122
|
-
name: string;
|
|
123
|
-
description?: string;
|
|
124
|
-
kind: CatalogConnector['kind'];
|
|
125
|
-
targetType: string;
|
|
126
|
-
config: Record<string, unknown>;
|
|
127
|
-
connectionId?: string;
|
|
128
|
-
secretEnvVar?: string;
|
|
129
|
-
transformId?: string;
|
|
130
|
-
schedule?: string;
|
|
131
|
-
mode?: 'full' | 'incremental';
|
|
132
|
-
enabled: boolean;
|
|
133
|
-
}
|
|
134
119
|
export interface TransformInput {
|
|
135
120
|
id?: string;
|
|
136
121
|
name: string;
|
|
@@ -138,6 +123,58 @@ export interface TransformInput {
|
|
|
138
123
|
language: TransformLanguage;
|
|
139
124
|
code: string;
|
|
140
125
|
}
|
|
126
|
+
/**
|
|
127
|
+
* One pipeline that would break if a connection were deleted.
|
|
128
|
+
*
|
|
129
|
+
* A workflow rather than a connector, because that is the object somebody
|
|
130
|
+
* recognises. `through` is the server's sentence about WHICH source reaches it
|
|
131
|
+
* — `"Warehouse"`, or `"Warehouse" via connector abc` — so a refusal can point
|
|
132
|
+
* at a box on a canvas rather than at a row in a table nobody can see.
|
|
133
|
+
*/
|
|
134
|
+
export interface ConnectionUse {
|
|
135
|
+
id: string;
|
|
136
|
+
name: string;
|
|
137
|
+
status: CatalogWorkflow['status'];
|
|
138
|
+
through: string;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* What a manual run may say about itself.
|
|
142
|
+
*
|
|
143
|
+
* Both fields are absent on the run the Run button sends, and both exist for
|
|
144
|
+
* the case where somebody is re-driving a load they already know something
|
|
145
|
+
* about. See {@link CatalogClient.runWorkflow} for why `expectShrink` had to
|
|
146
|
+
* survive the removal of the connector run route.
|
|
147
|
+
*/
|
|
148
|
+
export interface WorkflowRunOptions {
|
|
149
|
+
/** Re-drive under an identity that already exists, rather than minting one. */
|
|
150
|
+
snapshotId?: string;
|
|
151
|
+
/**
|
|
152
|
+
* Why this load is expected to lose rows, in a sentence.
|
|
153
|
+
*
|
|
154
|
+
* Sent only when somebody wrote one. The distinction between absent and
|
|
155
|
+
* present-but-empty is real and is the server's: absent means nobody said
|
|
156
|
+
* anything and the bound decides; empty means somebody claimed an
|
|
157
|
+
* acknowledgement and gave no reason, which is refused.
|
|
158
|
+
*/
|
|
159
|
+
expectShrink?: string;
|
|
160
|
+
}
|
|
161
|
+
export interface WorkflowScheduleInput {
|
|
162
|
+
/** Cron-ish, interpreted by whatever schedules it. Empty means manual only. */
|
|
163
|
+
schedule?: string;
|
|
164
|
+
enabled?: boolean;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* The stored graph, plus what the server thinks of the schedule on it.
|
|
168
|
+
*
|
|
169
|
+
* `warning` is `null` when the schedule will fire. It is a string when it will
|
|
170
|
+
* not — and that is the entire reason this call answers with more than the
|
|
171
|
+
* workflow: a cron stored on a draft, on a disabled graph, or in a syntax
|
|
172
|
+
* nothing can parse is a pipeline that looks scheduled and never runs, which is
|
|
173
|
+
* a failure with an incident behind it rather than a hypothetical.
|
|
174
|
+
*/
|
|
175
|
+
export interface ScheduledWorkflow extends CatalogWorkflow {
|
|
176
|
+
warning?: string | null;
|
|
177
|
+
}
|
|
141
178
|
/**
|
|
142
179
|
* One saved version of the two things in this catalog whose text a person edits.
|
|
143
180
|
*
|
|
@@ -407,27 +444,40 @@ export interface CatalogClient {
|
|
|
407
444
|
* system it names.
|
|
408
445
|
*/
|
|
409
446
|
checkConnection(id: string): Promise<ConnectionCheck>;
|
|
410
|
-
/**
|
|
411
|
-
|
|
447
|
+
/**
|
|
448
|
+
* Which pipelines read through it. Named, so a refusal can say which.
|
|
449
|
+
*
|
|
450
|
+
* Workflows and not connectors, because that is the question somebody is
|
|
451
|
+
* actually asking before they delete a connection: what breaks. A connector
|
|
452
|
+
* is an implementation detail of a published graph and naming one in a
|
|
453
|
+
* refusal would send the reader looking for a screen that no longer exists.
|
|
454
|
+
*
|
|
455
|
+
* The server's answer, and it outranks anything a screen derives from the
|
|
456
|
+
* workflow list: a source node may reach a connection through a connector it
|
|
457
|
+
* names in `config.connectorId` rather than through its own `connectionId`,
|
|
458
|
+
* and only the server folds both in.
|
|
459
|
+
*/
|
|
460
|
+
connectionWorkflows(id: string): Promise<ConnectionUse[]>;
|
|
412
461
|
deleteConnection(id: string): Promise<{
|
|
413
462
|
deleted: boolean;
|
|
414
463
|
}>;
|
|
415
|
-
listConnectors(): Promise<CatalogConnector[]>;
|
|
416
|
-
saveConnector(input: ConnectorInput): Promise<CatalogConnector>;
|
|
417
|
-
deleteConnector(id: string): Promise<{
|
|
418
|
-
deleted: boolean;
|
|
419
|
-
}>;
|
|
420
|
-
runConnector(id: string): Promise<ConnectorRun>;
|
|
421
464
|
/**
|
|
422
|
-
*
|
|
423
|
-
*
|
|
424
|
-
*
|
|
425
|
-
*
|
|
426
|
-
*
|
|
427
|
-
*
|
|
428
|
-
*
|
|
465
|
+
* The connectors, read-only.
|
|
466
|
+
*
|
|
467
|
+
* **There is deliberately no `saveConnector`, `deleteConnector` or
|
|
468
|
+
* `runConnector` on this client, because there are no such routes.** A
|
|
469
|
+
* connector is what a published workflow runs as: `publishWorkflow` mints
|
|
470
|
+
* one, `deleteWorkflow` takes it away, and `runWorkflow` is the only way to
|
|
471
|
+
* start one. A method here that posted to `pipeline/connectors` would 404,
|
|
472
|
+
* and one that quietly did nothing would be worse.
|
|
473
|
+
*
|
|
474
|
+
* This read stays because it answers what a graph cannot answer about itself:
|
|
475
|
+
* which id the run history and the incremental watermark are keyed on, and —
|
|
476
|
+
* on a deployment upgraded rather than built fresh — which rows have not been
|
|
477
|
+
* adopted into a workflow yet. `CatalogConnector.workflowId` is what joins one
|
|
478
|
+
* to the graph that runs it.
|
|
429
479
|
*/
|
|
430
|
-
|
|
480
|
+
listConnectors(): Promise<CatalogConnector[]>;
|
|
431
481
|
/**
|
|
432
482
|
* Publish an object type's schema — create it, or update the shape of one
|
|
433
483
|
* that exists.
|
|
@@ -505,6 +555,29 @@ export interface CatalogClient {
|
|
|
505
555
|
*/
|
|
506
556
|
listWorkflows(): Promise<CatalogWorkflow[]>;
|
|
507
557
|
saveWorkflow(input: WorkflowInput): Promise<CatalogWorkflow>;
|
|
558
|
+
/**
|
|
559
|
+
* Declare it finished, and mint the connector it runs as.
|
|
560
|
+
*
|
|
561
|
+
* Its own call rather than a field on the save, because publishing is a claim
|
|
562
|
+
* the server checks and a failed check owes an explanation naming the nodes —
|
|
563
|
+
* which an autosave has nowhere to put. A draft saves without validating;
|
|
564
|
+
* this is where the rules are answered for real.
|
|
565
|
+
*/
|
|
566
|
+
publishWorkflow(id: string): Promise<CatalogWorkflow>;
|
|
567
|
+
/**
|
|
568
|
+
* Back to draft. Disables the connector, keeping its id, its run history and
|
|
569
|
+
* its watermark, so re-publishing resumes the same pipeline rather than
|
|
570
|
+
* starting a second one beside it.
|
|
571
|
+
*/
|
|
572
|
+
unpublishWorkflow(id: string): Promise<CatalogWorkflow>;
|
|
573
|
+
/**
|
|
574
|
+
* Delete it, and the connector with it.
|
|
575
|
+
*
|
|
576
|
+
* The run history goes too, which is why the console asks first. It used to
|
|
577
|
+
* be refused while a connector still ran the graph; that check would now
|
|
578
|
+
* refuse every delete there is, since a published graph runs as exactly one
|
|
579
|
+
* connector — its own.
|
|
580
|
+
*/
|
|
508
581
|
deleteWorkflow(id: string): Promise<{
|
|
509
582
|
deleted: boolean;
|
|
510
583
|
}>;
|
|
@@ -512,8 +585,54 @@ export interface CatalogClient {
|
|
|
512
585
|
* Execute it. When a durable engine is available each node runs as its own
|
|
513
586
|
* step, so a failure part-way resumes rather than restarts — see
|
|
514
587
|
* `PipelineCapabilities.durable`, which is what the screen states.
|
|
588
|
+
*
|
|
589
|
+
* `expectShrink` is the escape hatch for a load the row-count bound refused,
|
|
590
|
+
* and it reaches the server **only through here** — `POST connectors/:id/run`
|
|
591
|
+
* carried it and that route is gone. Without it an operator's only recourse
|
|
592
|
+
* would be raising `rowCount.maxShrink` in the type's policy, which stands the
|
|
593
|
+
* guard down for every future load of that type instead of for one snapshot.
|
|
594
|
+
*
|
|
595
|
+
* It is a reason, not a flag: the string is written into the snapshot's
|
|
596
|
+
* `_expectShrink` label and is the only answer anybody will have in six
|
|
597
|
+
* months to "why was this load allowed to lose most of the data?". Sending it
|
|
598
|
+
* empty is refused with a 400 asking for one, which is why the console's
|
|
599
|
+
* dialog will not submit a blank box rather than sending it and hoping.
|
|
600
|
+
*/
|
|
601
|
+
runWorkflow(id: string, options?: WorkflowRunOptions): Promise<WorkflowRun>;
|
|
602
|
+
/**
|
|
603
|
+
* Set when this graph runs, and whether it runs at all.
|
|
604
|
+
*
|
|
605
|
+
* Both fields are optional and an absent one means "leave it alone", so a
|
|
606
|
+
* screen rendering only the cron cannot silently re-enable a pipeline
|
|
607
|
+
* somebody turned off. The answer is the stored workflow plus `warning`,
|
|
608
|
+
* which is the server naming a schedule that will never fire — a draft, a
|
|
609
|
+
* disabled graph, an unparseable cron. A screen that dropped it would repeat
|
|
610
|
+
* the incident this route was written after: a scheduler announcing it was
|
|
611
|
+
* watching schedules while parsing nothing.
|
|
612
|
+
*/
|
|
613
|
+
scheduleWorkflow(id: string, input: WorkflowScheduleInput): Promise<ScheduledWorkflow>;
|
|
614
|
+
/**
|
|
615
|
+
* What the system behind one source node looks like right now. Writes nothing.
|
|
616
|
+
*
|
|
617
|
+
* Takes a node and not a connector, and the swap is the whole of the move:
|
|
618
|
+
* this was `POST connectors/:id/discover`, which refused outright for any
|
|
619
|
+
* connector carrying a `workflowId` — and every connector carries one now, so
|
|
620
|
+
* the old shape would have refused every connector there is.
|
|
621
|
+
*
|
|
622
|
+
* **It answers on a draft, deliberately.** A sink cannot commit into a type
|
|
623
|
+
* that does not exist, so requiring a published graph would require publishing
|
|
624
|
+
* a graph whose target type cannot be created until it is published. What it
|
|
625
|
+
* does need is a graph that has been SAVED: the server reads the stored node,
|
|
626
|
+
* not the one on screen.
|
|
627
|
+
*
|
|
628
|
+
* Typed loosely on purpose. The shape comes from
|
|
629
|
+
* `@dudousxd/nestjs-catalog-pipeline`, and this package must not import that
|
|
630
|
+
* one — it would drag a package built for a Node process, with database
|
|
631
|
+
* drivers behind optional imports, into a browser bundle. `PipelineCapabilities`
|
|
632
|
+
* above is mirrored for exactly the same reason; `narrowDiscovery` in
|
|
633
|
+
* `schema-discovery.tsx` is what checks the answer at the seam.
|
|
515
634
|
*/
|
|
516
|
-
|
|
635
|
+
discoverSourceSchema(workflowId: string, nodeId: string): Promise<unknown>;
|
|
517
636
|
/**
|
|
518
637
|
* Who may do what.
|
|
519
638
|
*
|
|
@@ -579,11 +698,16 @@ export declare const catalogQueryKeys: {
|
|
|
579
698
|
search: (term: string, limit?: number) => readonly ["nestjs-catalog", "search", string, number | null];
|
|
580
699
|
/**
|
|
581
700
|
* The pipeline keys are nested under one prefix so a host — or a screen that
|
|
582
|
-
* just ran a
|
|
583
|
-
*
|
|
584
|
-
*
|
|
585
|
-
*
|
|
586
|
-
*
|
|
701
|
+
* just ran a pipeline — can invalidate the whole of it at once. Running one
|
|
702
|
+
* changes the runs, the connector's own last-run fields, and the catalog
|
|
703
|
+
* snapshot the load wrote into, and invalidating them one by one is how a
|
|
704
|
+
* screen ends up showing a run that finished next to a pipeline that still
|
|
705
|
+
* says it never has.
|
|
706
|
+
*
|
|
707
|
+
* `connectors` is still here and is still worth invalidating, even though
|
|
708
|
+
* nothing authors one any more: publishing a graph MINTS one, unpublishing
|
|
709
|
+
* disables it, and deleting takes it away — so the read that says which id a
|
|
710
|
+
* run history is keyed on goes stale on all three.
|
|
587
711
|
*/
|
|
588
712
|
pipeline: readonly ["nestjs-catalog", "pipeline"];
|
|
589
713
|
capabilities: readonly ["nestjs-catalog", "pipeline", "capabilities"];
|
package/dist/context.js
CHANGED
|
@@ -55,13 +55,9 @@ export function CatalogProvider({ transport, pipelineBasePath = DEFAULT_PIPELINE
|
|
|
55
55
|
listConnections: () => transport.get(pipeline.connections()),
|
|
56
56
|
saveConnection: (input) => transport.post(pipeline.connections(), input),
|
|
57
57
|
checkConnection: (id) => transport.post(pipeline.checkConnection(id), {}),
|
|
58
|
-
|
|
58
|
+
connectionWorkflows: (id) => transport.get(pipeline.connectionWorkflows(id)),
|
|
59
59
|
deleteConnection: (id) => transport.delete(pipeline.connection(id)),
|
|
60
60
|
listConnectors: () => transport.get(pipeline.connectors()),
|
|
61
|
-
saveConnector: (input) => transport.post(pipeline.connectors(), input),
|
|
62
|
-
deleteConnector: (id) => transport.delete(pipeline.connector(id)),
|
|
63
|
-
runConnector: (id) => transport.post(pipeline.runConnector(id), {}),
|
|
64
|
-
discoverConnectorSchema: (id) => transport.post(pipeline.discoverConnectorSchema(id), {}),
|
|
65
61
|
publishType: (name, schema) => {
|
|
66
62
|
if (!transport.put) {
|
|
67
63
|
throw new Error('This transport cannot PUT, so it cannot publish a type. Implement `put` on the ' +
|
|
@@ -92,8 +88,31 @@ export function CatalogProvider({ transport, pipelineBasePath = DEFAULT_PIPELINE
|
|
|
92
88
|
tryTransform: (input) => transport.post(pipeline.tryTransform(), input),
|
|
93
89
|
listWorkflows: () => transport.get(pipeline.workflows()),
|
|
94
90
|
saveWorkflow: (input) => transport.post(pipeline.workflows(), input),
|
|
91
|
+
publishWorkflow: (id) => transport.post(pipeline.publishWorkflow(id), {}),
|
|
92
|
+
unpublishWorkflow: (id) => transport.post(pipeline.unpublishWorkflow(id), {}),
|
|
95
93
|
deleteWorkflow: (id) => transport.delete(pipeline.workflow(id)),
|
|
96
|
-
|
|
94
|
+
// The two fields are spread conditionally rather than sent as
|
|
95
|
+
// `undefined`, and for `expectShrink` that is the whole contract: the
|
|
96
|
+
// server distinguishes a body with no such key — nobody said anything,
|
|
97
|
+
// let the bound decide — from one carrying an empty string, which is an
|
|
98
|
+
// acknowledgement with no reason behind it and is refused with a 400
|
|
99
|
+
// asking for one. A transport that serialises `undefined` as `null`, or
|
|
100
|
+
// one that drops it, would flatten those two into each other.
|
|
101
|
+
runWorkflow: (id, options) => transport.post(pipeline.runWorkflow(id), {
|
|
102
|
+
...(options?.snapshotId === undefined ? {} : { snapshotId: options.snapshotId }),
|
|
103
|
+
...(options && 'expectShrink' in options && options.expectShrink !== undefined
|
|
104
|
+
? { expectShrink: options.expectShrink }
|
|
105
|
+
: {}),
|
|
106
|
+
}),
|
|
107
|
+
scheduleWorkflow: (id, input) => {
|
|
108
|
+
if (!transport.put) {
|
|
109
|
+
throw new Error('This transport cannot PUT, so it cannot set a schedule. Implement `put` on the ' +
|
|
110
|
+
'transport you passed to CatalogProvider — a schedule is an idempotent upsert of ' +
|
|
111
|
+
'the fields it names, which is why the route is a PUT and not a POST.');
|
|
112
|
+
}
|
|
113
|
+
return transport.put(pipeline.workflowSchedule(id), input);
|
|
114
|
+
},
|
|
115
|
+
discoverSourceSchema: (workflowId, nodeId) => transport.post(pipeline.discoverSourceSchema(workflowId, nodeId), {}),
|
|
97
116
|
listPrincipals: () => transport.get(access.principals()),
|
|
98
117
|
listPeople: (query) => transport.get(access.people(query)),
|
|
99
118
|
upsertPerson: (input) => transport.post(access.people(), input),
|
|
@@ -135,11 +154,16 @@ export const catalogQueryKeys = {
|
|
|
135
154
|
search: (term, limit) => ['nestjs-catalog', 'search', term, limit ?? null],
|
|
136
155
|
/**
|
|
137
156
|
* The pipeline keys are nested under one prefix so a host — or a screen that
|
|
138
|
-
* just ran a
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
157
|
+
* just ran a pipeline — can invalidate the whole of it at once. Running one
|
|
158
|
+
* changes the runs, the connector's own last-run fields, and the catalog
|
|
159
|
+
* snapshot the load wrote into, and invalidating them one by one is how a
|
|
160
|
+
* screen ends up showing a run that finished next to a pipeline that still
|
|
161
|
+
* says it never has.
|
|
162
|
+
*
|
|
163
|
+
* `connectors` is still here and is still worth invalidating, even though
|
|
164
|
+
* nothing authors one any more: publishing a graph MINTS one, unpublishing
|
|
165
|
+
* disables it, and deleting takes it away — so the read that says which id a
|
|
166
|
+
* run history is keyed on goes stale on all three.
|
|
143
167
|
*/
|
|
144
168
|
pipeline: ['nestjs-catalog', 'pipeline'],
|
|
145
169
|
capabilities: ['nestjs-catalog', 'pipeline', 'capabilities'],
|
package/dist/index.d.ts
CHANGED
|
@@ -16,14 +16,17 @@ export { CatalogSearch, type CatalogSearchProps } from './search-console';
|
|
|
16
16
|
export { SavedQueryPanel } from './SavedQueryPanel';
|
|
17
17
|
export { CATALOG_EVENT_META, type CatalogTraceSource, eventMeta, TraceExplorer, useTraceSource, } from './TraceExplorer';
|
|
18
18
|
export { ConnectionPanel, type ConnectionPanelProps, connectionOptionsFor, describeConnection, } from './ConnectionPanel';
|
|
19
|
-
export {
|
|
19
|
+
export { PipelineConsole, type PipelineConsoleProps, } from './PipelineConsole';
|
|
20
|
+
export { type ColumnChoice, type ConnectorSchemaDiscovery, type DiscoveredColumn, type DiscoveredTypeDraft, initialChoices, narrowDiscovery, proposalFrom, type SchemaDiscoveryBridge, SchemaDiscoveryPanel, type SchemaDrift, } from './schema-discovery';
|
|
20
21
|
export { TransformEditor, type TransformEditorProps } from './TransformEditor';
|
|
21
22
|
export { DiffBody, RevisionHistory, type RevisionHistoryProps, RevisionHistoryButton, RevisionHistorySheet, type RevisionHistorySheetProps, type RevisionSubject, type RevisionSubjectKind, } from './diff/RevisionDiff';
|
|
22
23
|
export { AccessConsole, type AccessConsoleProps } from './AccessConsole';
|
|
23
|
-
export { type CatalogWorkflow, describeDurability, type DurabilityCopy, isWorkflowNodeKind, newLocalId, nodeName, producedTypes, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, type WorkflowDurability, type WorkflowEdge, type WorkflowGraph, type WorkflowInput, type WorkflowNode, type WorkflowNodeKind, type WorkflowRun, type WorkflowRunNode, type WorkflowSinkNode, type WorkflowSourceNode, type WorkflowTransformNode, } from './workflow/model';
|
|
24
|
+
export { type CallableWorkflowRef, type CatalogWorkflow, describeDurability, type DurabilityCopy, isWorkflowNodeKind, newLocalId, nodeName, producedTypes, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, type WorkflowCallNode, type WorkflowDurability, type WorkflowEdge, type WorkflowGraph, type WorkflowInput, type WorkflowNode, type WorkflowNodeKind, type WorkflowRun, type WorkflowRunNode, type WorkflowSinkNode, type WorkflowSourceNode, type WorkflowTransformNode, } from './workflow/model';
|
|
24
25
|
export { WORKFLOW_NAME } from './workflow/name';
|
|
26
|
+
export { checkShapes, type ShapeKnowledge, type SourceColumn, type SourceShape, type TargetProperty, type TargetShape, } from './workflow/shape';
|
|
25
27
|
export { canConnect, type ConnectionVerdict, edgeId, hasBlockingProblem, problemsByNode, type ValidateOptions, validateWorkflow, type WorkflowDraft, type WorkflowProblem, type WorkflowProblemCode, type WorkflowProblemLevel, wouldCycle, } from './workflow/validate';
|
|
26
|
-
export {
|
|
28
|
+
export { attachTransformIds, buildWorkflowTemplate, type EnrichOptions, enrichWithLookup, fanOutTypes, type FanOutOptions, type FanOutTarget, FILE_DROP_KINDS, type FileDropOptions, joinSources, type JoinOptions, loadFileDrop, type PeriodicFullReloadOptions, periodicFullReload, planIsRunnable, planToWorkflowInput, refuseUnusableColumnNames, RELOAD_CADENCE_IDS, RELOAD_CADENCES, type ReloadCadence, replicateTable, type ReplicateTableOptions, SOURCE_KINDS, type TemplateDeclaration, type TemplateExpectation, type TemplateOutcome, type TemplatePlan, type TemplateRefusal, type TemplateRefusalCode, type TemplateRequest, type TemplateSchedule, type TemplateTransformRequest, withinMsFor, WORKFLOW_TEMPLATE_IDS, WORKFLOW_TEMPLATES, type WorkflowTemplateId, type WorkflowTemplateMeta, } from './workflow/templates';
|
|
29
|
+
export { type CatalogClient, type CatalogIdentity, type CatalogRevision, type CatalogPeoplePage, type CatalogPersonRole, type CatalogPersonSummary, type CatalogPrincipalSummary, CatalogProvider, type CatalogProviderProps, type CatalogTransport, type ConnectionInput, type ConnectionUse, catalogQueryKeys, type DeleteReconciliation, type LoadExpectation, type LoadExpectationInput, type ResolvedLoadExpectation, type RowCountBound, type StoredLoadExpectation, type PersonInput, type PersonUpsertResult, type PipelineCapabilities, type ScheduledWorkflow, type TransformInput, type WorkflowRunOptions, type WorkflowScheduleInput, useCatalogClient, } from './context';
|
|
27
30
|
export { type AccessRoutes, accessRoutes, DEFAULT_ACCESS_BASE_PATH, DEFAULT_PIPELINE_BASE_PATH, DEFAULT_PUBLISH_BASE_PATH, embedRoutes, type PeopleQuery, type PipelineRoutes, pipelineRoutes, } from './routes';
|
|
28
31
|
export { ConfirmDialog } from './ui/dialog';
|
|
29
32
|
export { FieldGroup, TextAreaField, TextField } from './ui/field';
|
package/dist/index.js
CHANGED
|
@@ -26,24 +26,34 @@ export { QueryConsole } from './QueryConsole';
|
|
|
26
26
|
export { CatalogSearch } from './search-console';
|
|
27
27
|
export { SavedQueryPanel } from './SavedQueryPanel';
|
|
28
28
|
export { CATALOG_EVENT_META, eventMeta, TraceExplorer, useTraceSource, } from './TraceExplorer';
|
|
29
|
-
// Getting data in:
|
|
30
|
-
//
|
|
31
|
-
// installing this package got a console with no way to manage a
|
|
29
|
+
// Getting data in: the addresses a pipeline reads through and the code that
|
|
30
|
+
// reshapes what it fetches. These used to live in the consuming app, which meant
|
|
31
|
+
// any other host installing this package got a console with no way to manage a
|
|
32
|
+
// load.
|
|
33
|
+
//
|
|
34
|
+
// There is deliberately no connector screen here any more. A connector is what a
|
|
35
|
+
// published workflow runs as — nothing authors one — so authoring a pipeline is
|
|
36
|
+
// `WorkflowCanvas`, on the `/workflow` subpath, and this is the two shared
|
|
37
|
+
// objects it borrows.
|
|
32
38
|
export { ConnectionPanel, connectionOptionsFor, describeConnection, } from './ConnectionPanel';
|
|
39
|
+
export { PipelineConsole, } from './PipelineConsole';
|
|
33
40
|
// The schema-discovery seam, whole.
|
|
34
41
|
//
|
|
35
|
-
// `
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
// a file inside `dist/`. Both are the caller paying for a list that fell behind.
|
|
42
|
+
// `SchemaDiscoveryBridge` is a bridge a HOST may implement — this package cannot
|
|
43
|
+
// ask a source for its columns, only render the answer — and every one of these
|
|
44
|
+
// was once reachable only through an indexed access on a component's props, or
|
|
45
|
+
// by deep-importing a file inside `dist/`. Both are the caller paying for a list
|
|
46
|
+
// that fell behind.
|
|
41
47
|
//
|
|
42
48
|
// `initialChoices` and `proposalFrom` come with them because they are the rules
|
|
43
49
|
// that decide whether a schema somebody ticked is one the publish route will
|
|
44
50
|
// accept — pure and deliberately outside the panel, per their own docblocks, and
|
|
45
51
|
// worth nothing there if the only way to run them is to render it.
|
|
46
|
-
|
|
52
|
+
//
|
|
53
|
+
// `SchemaDiscoveryPanel` is exported for the same reason it stopped living
|
|
54
|
+
// inside a screen: the canvas mounts it from behind the `/workflow` subpath, and
|
|
55
|
+
// a host assembling its own source inspector should be able to do the same.
|
|
56
|
+
export { initialChoices, narrowDiscovery, proposalFrom, SchemaDiscoveryPanel, } from './schema-discovery';
|
|
47
57
|
export { TransformEditor } from './TransformEditor';
|
|
48
58
|
// Why one load came out different from the last one.
|
|
49
59
|
//
|
|
@@ -84,11 +94,27 @@ export { AccessConsole } from './AccessConsole';
|
|
|
84
94
|
// and it belongs to the package that executes them. See `workflow/model.ts`.
|
|
85
95
|
export { describeDurability, isWorkflowNodeKind, newLocalId, nodeName, producedTypes, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, } from './workflow/model';
|
|
86
96
|
export { WORKFLOW_NAME } from './workflow/name';
|
|
87
|
-
//
|
|
88
|
-
//
|
|
89
|
-
//
|
|
90
|
-
//
|
|
97
|
+
// Whether what a source reads fits the type its sink writes.
|
|
98
|
+
//
|
|
99
|
+
// The canvas answers this for itself now — it holds what `discoverSourceSchema`
|
|
100
|
+
// returned for each source node and compares it against the snapshot it already
|
|
101
|
+
// reads, so there is no prop and nothing for a host to wire. These are exported
|
|
102
|
+
// because the comparison is pure and a host may well want it somewhere else: a
|
|
103
|
+
// pre-flight before a scheduled run, say. `ConnectorSchemaDiscovery` above is
|
|
104
|
+
// already a `SourceShape`, so that caller has one for free.
|
|
105
|
+
export { checkShapes, } from './workflow/shape';
|
|
106
|
+
// Core's rules, run in the browser, plus the questions core cannot answer:
|
|
107
|
+
// whether a connection is legal mid-drag, whether a node names a transform that
|
|
108
|
+
// still exists, and whether a source supplies the columns its sink writes.
|
|
109
|
+
// Exported so a host can run the same checks before it offers a save — never so
|
|
110
|
+
// it can skip the server's, which is the authority.
|
|
91
111
|
export { canConnect, edgeId, hasBlockingProblem, problemsByNode, validateWorkflow, wouldCycle, } from './workflow/validate';
|
|
112
|
+
// The prefilled graphs, on this entry point rather than on `/workflow` because
|
|
113
|
+
// they import no React Flow: a template produces nodes, edges, transform bodies
|
|
114
|
+
// and expectation payloads, and a host should be able to build one — or check
|
|
115
|
+
// what it would decide — without installing a canvas to do it. Whether a screen
|
|
116
|
+
// then draws the result is the canvas's business, not the template's.
|
|
117
|
+
export { attachTransformIds, buildWorkflowTemplate, enrichWithLookup, fanOutTypes, FILE_DROP_KINDS, joinSources, loadFileDrop, periodicFullReload, planIsRunnable, planToWorkflowInput, refuseUnusableColumnNames, RELOAD_CADENCE_IDS, RELOAD_CADENCES, replicateTable, SOURCE_KINDS, withinMsFor, WORKFLOW_TEMPLATE_IDS, WORKFLOW_TEMPLATES, } from './workflow/templates';
|
|
92
118
|
export { CatalogProvider, catalogQueryKeys, useCatalogClient, } from './context';
|
|
93
119
|
// The paths behind the pipeline and access screens, exported so a host can
|
|
94
120
|
// mount those endpoints wherever it likes and tell the provider where they are.
|
package/dist/routes.d.ts
CHANGED
|
@@ -99,22 +99,29 @@ export interface PipelineRoutes {
|
|
|
99
99
|
connections(): string;
|
|
100
100
|
connection(id: string): string;
|
|
101
101
|
checkConnection(id: string): string;
|
|
102
|
-
/** Which connectors read through a connection. Named, so a refusal can say. */
|
|
103
|
-
connectionConnectors(id: string): string;
|
|
104
|
-
connectors(): string;
|
|
105
|
-
connector(id: string): string;
|
|
106
|
-
runConnector(id: string): string;
|
|
107
102
|
/**
|
|
108
|
-
*
|
|
103
|
+
* Which pipelines read through a connection. Named, so a refusal can say.
|
|
104
|
+
*
|
|
105
|
+
* `.../connectors` before, and the rename is the question actually being
|
|
106
|
+
* asked rather than a tidy-up: an operator presses this before deleting a
|
|
107
|
+
* connection, and what they need is the list of things that would break —
|
|
108
|
+
* which is a list of workflows, because a workflow is the only thing anybody
|
|
109
|
+
* authors and a connector's name is an implementation detail of one.
|
|
110
|
+
*/
|
|
111
|
+
connectionWorkflows(id: string): string;
|
|
112
|
+
/**
|
|
113
|
+
* The internal records, read-only.
|
|
114
|
+
*
|
|
115
|
+
* **The one connector path left, and it is a `GET`.** There is no `POST`, no
|
|
116
|
+
* `DELETE` and no `.../run`: a connector is what a published workflow runs
|
|
117
|
+
* as, minted by `workflows/:id/publish` and removed with the graph. A builder
|
|
118
|
+
* for a route that creates one would be this package advertising a second way
|
|
119
|
+
* to author a pipeline, which is the thing that was removed.
|
|
109
120
|
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
* generic "describe this database" route because the connector is what
|
|
113
|
-
* already holds the address, the credential reference and the statement —
|
|
114
|
-
* a route taking those as arguments would be a way to point the server at
|
|
115
|
-
* any database at all.
|
|
121
|
+
* It stays because it answers what a graph cannot answer about itself — which
|
|
122
|
+
* id the run history and the incremental watermark are keyed on.
|
|
116
123
|
*/
|
|
117
|
-
|
|
124
|
+
connectors(): string;
|
|
118
125
|
runs(): string;
|
|
119
126
|
transforms(): string;
|
|
120
127
|
transform(id: string): string;
|
|
@@ -146,7 +153,41 @@ export interface PipelineRoutes {
|
|
|
146
153
|
*/
|
|
147
154
|
workflows(): string;
|
|
148
155
|
workflow(id: string): string;
|
|
156
|
+
/**
|
|
157
|
+
* Declare a graph finished, which is also what mints the connector it runs
|
|
158
|
+
* as. Its own path rather than a field on the save, because publishing is a
|
|
159
|
+
* claim that gets checked and a check that fails owes an explanation.
|
|
160
|
+
*/
|
|
161
|
+
publishWorkflow(id: string): string;
|
|
162
|
+
/** Back to draft. The connector keeps its id and its history, and stops. */
|
|
163
|
+
unpublishWorkflow(id: string): string;
|
|
164
|
+
/**
|
|
165
|
+
* Run it now. **The only run path there is** — `connectors/:id/run` is gone,
|
|
166
|
+
* and what it carried that this did not is `expectShrink`, which moved onto
|
|
167
|
+
* this one's body rather than being lost with it.
|
|
168
|
+
*/
|
|
149
169
|
runWorkflow(id: string): string;
|
|
170
|
+
/**
|
|
171
|
+
* When this graph runs, and whether it runs at all.
|
|
172
|
+
*
|
|
173
|
+
* A `PUT` of its own rather than fields on the save: a canvas autosaves, and
|
|
174
|
+
* a cron folded into every save a drag passes through is one a stale tab can
|
|
175
|
+
* silently revert.
|
|
176
|
+
*/
|
|
177
|
+
workflowSchedule(id: string): string;
|
|
178
|
+
/**
|
|
179
|
+
* What the system behind one source node looks like right now. Writes nothing.
|
|
180
|
+
*
|
|
181
|
+
* `connectors/:id/discover` before. It had to move rather than be dropped —
|
|
182
|
+
* discovery is how a type gets its shape, and a sink cannot commit into a
|
|
183
|
+
* type that does not exist — and it deliberately answers on a **draft**:
|
|
184
|
+
* requiring a published graph would require publishing a graph whose target
|
|
185
|
+
* type cannot be created until it is published.
|
|
186
|
+
*
|
|
187
|
+
* Per node, not per graph, because a graph may have several sources and only
|
|
188
|
+
* a source has a shape to report.
|
|
189
|
+
*/
|
|
190
|
+
discoverSourceSchema(id: string, nodeId: string): string;
|
|
150
191
|
/**
|
|
151
192
|
* Every per-type load expectation an operator has stored.
|
|
152
193
|
*
|
package/dist/routes.js
CHANGED
|
@@ -135,11 +135,8 @@ export function pipelineRoutes(basePath = DEFAULT_PIPELINE_BASE_PATH) {
|
|
|
135
135
|
connections: () => `${base}/connections`,
|
|
136
136
|
connection: (id) => `${base}/connections/${encodeURIComponent(id)}`,
|
|
137
137
|
checkConnection: (id) => `${base}/connections/${encodeURIComponent(id)}/check`,
|
|
138
|
-
|
|
138
|
+
connectionWorkflows: (id) => `${base}/connections/${encodeURIComponent(id)}/workflows`,
|
|
139
139
|
connectors: () => `${base}/connectors`,
|
|
140
|
-
connector: (id) => `${base}/connectors/${encodeURIComponent(id)}`,
|
|
141
|
-
runConnector: (id) => `${base}/connectors/${encodeURIComponent(id)}/run`,
|
|
142
|
-
discoverConnectorSchema: (id) => `${base}/connectors/${encodeURIComponent(id)}/discover`,
|
|
143
140
|
runs: () => `${base}/runs`,
|
|
144
141
|
transforms: () => `${base}/transforms`,
|
|
145
142
|
transform: (id) => `${base}/transforms/${encodeURIComponent(id)}`,
|
|
@@ -147,7 +144,11 @@ export function pipelineRoutes(basePath = DEFAULT_PIPELINE_BASE_PATH) {
|
|
|
147
144
|
tryTransform: () => `${base}/transforms/try`,
|
|
148
145
|
workflows: () => `${base}/workflows`,
|
|
149
146
|
workflow: (id) => `${base}/workflows/${encodeURIComponent(id)}`,
|
|
147
|
+
publishWorkflow: (id) => `${base}/workflows/${encodeURIComponent(id)}/publish`,
|
|
148
|
+
unpublishWorkflow: (id) => `${base}/workflows/${encodeURIComponent(id)}/unpublish`,
|
|
150
149
|
runWorkflow: (id) => `${base}/workflows/${encodeURIComponent(id)}/run`,
|
|
150
|
+
workflowSchedule: (id) => `${base}/workflows/${encodeURIComponent(id)}/schedule`,
|
|
151
|
+
discoverSourceSchema: (id, nodeId) => `${base}/workflows/${encodeURIComponent(id)}/nodes/${encodeURIComponent(nodeId)}/discover`,
|
|
151
152
|
loadExpectations: () => expectations.expectations(),
|
|
152
153
|
loadExpectation: (typeName) => expectations.expectation(typeName),
|
|
153
154
|
};
|