@dudousxd/nestjs-catalog-react 0.17.0 → 0.19.0-preview.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -13
- package/dist/CatalogManager.d.ts +23 -1
- package/dist/CatalogManager.js +76 -13
- package/dist/ConnectionPanel.d.ts +11 -7
- package/dist/ConnectionPanel.js +78 -149
- package/dist/LoadedBySection.d.ts +39 -0
- package/dist/LoadedBySection.js +143 -0
- package/dist/PipelineConsole.d.ts +23 -126
- package/dist/PipelineConsole.js +39 -551
- package/dist/WorkflowCanvas.d.ts +16 -0
- package/dist/WorkflowCanvas.js +1064 -64
- package/dist/connection-form.d.ts +241 -0
- package/dist/connection-form.js +235 -0
- package/dist/context.d.ts +236 -38
- package/dist/context.js +46 -11
- package/dist/index.d.ts +10 -3
- package/dist/index.js +63 -14
- package/dist/routes.d.ts +84 -13
- package/dist/routes.js +7 -4
- package/dist/schema-discovery.d.ts +232 -0
- package/dist/schema-discovery.js +225 -0
- package/dist/source-connection.d.ts +82 -0
- package/dist/source-connection.js +176 -0
- package/dist/source-fields.d.ts +16 -14
- package/dist/source-fields.js +28 -18
- package/dist/ui/code-editor.d.ts +44 -2
- package/dist/ui/code-editor.js +10 -2
- package/dist/ui/combobox.d.ts +97 -0
- package/dist/ui/combobox.js +80 -0
- package/dist/ui/dialog.d.ts +13 -1
- package/dist/ui/dialog.js +13 -2
- package/dist/workflow/edges.d.ts +110 -0
- package/dist/workflow/edges.js +112 -0
- package/dist/workflow/graph.d.ts +39 -1
- package/dist/workflow/graph.js +212 -52
- package/dist/workflow/index.d.ts +8 -0
- package/dist/workflow/index.js +12 -0
- package/dist/workflow/lifecycle.d.ts +135 -0
- package/dist/workflow/lifecycle.js +216 -0
- package/dist/workflow/model.d.ts +17 -3
- package/dist/workflow/model.js +7 -2
- package/dist/workflow/nodes.d.ts +4 -1
- package/dist/workflow/nodes.js +198 -10
- package/dist/workflow/runs.d.ts +38 -0
- package/dist/workflow/runs.js +101 -0
- package/dist/workflow/shape.d.ts +169 -0
- package/dist/workflow/shape.js +249 -0
- package/dist/workflow/templates.d.ts +623 -0
- package/dist/workflow/templates.js +1102 -0
- package/dist/workflow/validate.d.ts +20 -1
- package/dist/workflow/validate.js +52 -3
- package/dist/workflow/wiring.d.ts +84 -0
- package/dist/workflow/wiring.js +254 -0
- package/package.json +9 -9
package/dist/context.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CatalogAuditEvent, CatalogConnection, CatalogConnector, CatalogObjectPage, CatalogQueryRelation, CatalogQueryResult, CatalogRevision, CatalogSearchResult, CatalogSnapshot, CatalogTraceList, CatalogTransform, ConnectionCheck, ConnectorRun, Dashboard, DashboardCard, DeleteReconciliation, LoadExpectation, LoadExpectationInput, ObjectQueryParams, PropertyPatch, ResolvedLoadExpectation, RowCountBound, SaveQueryInput, SavedQuery, SnapshotRef, StoredLoadExpectation, TraceQuery, TransformLanguage, TransformResult, TypePatch } from '@dudousxd/nestjs-catalog/client';
|
|
1
|
+
import type { CallableWorkflowRef, CatalogAuditEvent, CatalogConnection, CatalogConnector, CatalogObjectPage, CatalogQueryRelation, CatalogQueryResult, CatalogRevision, CatalogSearchResult, CatalogSnapshot, CatalogTraceList, CatalogTransform, ConnectionCheck, ConnectorRun, Dashboard, DashboardCard, DeleteReconciliation, LoadExpectation, LoadExpectationInput, ObjectQueryParams, PropertyPatch, ResolvedLoadExpectation, RowCountBound, SaveQueryInput, SavedQuery, SnapshotRef, StoredLoadExpectation, TraceQuery, TransformLanguage, TransformResult, TypePatch } from '@dudousxd/nestjs-catalog/client';
|
|
2
2
|
import { type ReactNode } from 'react';
|
|
3
3
|
import type { EmbeddedChartPayload, EmbeddedDashboardPayload } from './embed/payload';
|
|
4
4
|
import { type PeopleQuery } from './routes';
|
|
@@ -77,6 +77,37 @@ export interface PipelineCapabilities {
|
|
|
77
77
|
*/
|
|
78
78
|
durable?: WorkflowDurability;
|
|
79
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* What a call node could be pointed at, and whether the question was answerable.
|
|
82
|
+
*
|
|
83
|
+
* Mirrored here for the reason {@link PipelineCapabilities} is: naming the
|
|
84
|
+
* server's own declaration would mean importing
|
|
85
|
+
* `@dudousxd/nestjs-catalog-pipeline`, a package built for a Node process with
|
|
86
|
+
* database drivers behind optional imports. `CallableWorkflowRef` itself is
|
|
87
|
+
* re-exported from `@dudousxd/nestjs-catalog/client` and is NOT copied — the
|
|
88
|
+
* entries are the contract, and a second copy of those would drift.
|
|
89
|
+
*
|
|
90
|
+
* `supported` is the field that stops the list being misread, and it exists
|
|
91
|
+
* because an empty list means two opposite things. With `supported: true` it
|
|
92
|
+
* says the fleet announces nothing — every callable workflow is unregistered or
|
|
93
|
+
* its workers are down. With `supported: false` it says nobody could be asked:
|
|
94
|
+
* this deployment has no durable engine, or the engine could not read the
|
|
95
|
+
* announcements. A screen that rendered "no workflows found" over both would be
|
|
96
|
+
* asserting the first when it was told the second, which is the shape of claim
|
|
97
|
+
* this codebase keeps having to take back out.
|
|
98
|
+
*/
|
|
99
|
+
export interface CallableWorkflowList {
|
|
100
|
+
supported: boolean;
|
|
101
|
+
workflows: CallableWorkflowRef[];
|
|
102
|
+
/**
|
|
103
|
+
* When the fleet was asked, ISO-8601. Rendered, because this is a snapshot
|
|
104
|
+
* with a resolution of about one worker heartbeat and showing it without a
|
|
105
|
+
* time would present a moment as a standing fact.
|
|
106
|
+
*/
|
|
107
|
+
observedAt: string;
|
|
108
|
+
/** Why, in a full sentence. Rendered verbatim under the picker. */
|
|
109
|
+
detail: string;
|
|
110
|
+
}
|
|
80
111
|
/**
|
|
81
112
|
* What a load has to be true of before it becomes the data everybody reads.
|
|
82
113
|
*
|
|
@@ -116,21 +147,6 @@ export interface ConnectionInput {
|
|
|
116
147
|
config: Record<string, unknown>;
|
|
117
148
|
secretEnvVar?: string;
|
|
118
149
|
}
|
|
119
|
-
/** Same rule as {@link ConnectionInput}: authored fields only, never state. */
|
|
120
|
-
export interface ConnectorInput {
|
|
121
|
-
id?: string;
|
|
122
|
-
name: string;
|
|
123
|
-
description?: string;
|
|
124
|
-
kind: CatalogConnector['kind'];
|
|
125
|
-
targetType: string;
|
|
126
|
-
config: Record<string, unknown>;
|
|
127
|
-
connectionId?: string;
|
|
128
|
-
secretEnvVar?: string;
|
|
129
|
-
transformId?: string;
|
|
130
|
-
schedule?: string;
|
|
131
|
-
mode?: 'full' | 'incremental';
|
|
132
|
-
enabled: boolean;
|
|
133
|
-
}
|
|
134
150
|
export interface TransformInput {
|
|
135
151
|
id?: string;
|
|
136
152
|
name: string;
|
|
@@ -138,6 +154,58 @@ export interface TransformInput {
|
|
|
138
154
|
language: TransformLanguage;
|
|
139
155
|
code: string;
|
|
140
156
|
}
|
|
157
|
+
/**
|
|
158
|
+
* One pipeline that would break if a connection were deleted.
|
|
159
|
+
*
|
|
160
|
+
* A workflow rather than a connector, because that is the object somebody
|
|
161
|
+
* recognises. `through` is the server's sentence about WHICH source reaches it
|
|
162
|
+
* — `"Warehouse"`, or `"Warehouse" via connector abc` — so a refusal can point
|
|
163
|
+
* at a box on a canvas rather than at a row in a table nobody can see.
|
|
164
|
+
*/
|
|
165
|
+
export interface ConnectionUse {
|
|
166
|
+
id: string;
|
|
167
|
+
name: string;
|
|
168
|
+
status: CatalogWorkflow['status'];
|
|
169
|
+
through: string;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* What a manual run may say about itself.
|
|
173
|
+
*
|
|
174
|
+
* Both fields are absent on the run the Run button sends, and both exist for
|
|
175
|
+
* the case where somebody is re-driving a load they already know something
|
|
176
|
+
* about. See {@link CatalogClient.runWorkflow} for why `expectShrink` had to
|
|
177
|
+
* survive the removal of the connector run route.
|
|
178
|
+
*/
|
|
179
|
+
export interface WorkflowRunOptions {
|
|
180
|
+
/** Re-drive under an identity that already exists, rather than minting one. */
|
|
181
|
+
snapshotId?: string;
|
|
182
|
+
/**
|
|
183
|
+
* Why this load is expected to lose rows, in a sentence.
|
|
184
|
+
*
|
|
185
|
+
* Sent only when somebody wrote one. The distinction between absent and
|
|
186
|
+
* present-but-empty is real and is the server's: absent means nobody said
|
|
187
|
+
* anything and the bound decides; empty means somebody claimed an
|
|
188
|
+
* acknowledgement and gave no reason, which is refused.
|
|
189
|
+
*/
|
|
190
|
+
expectShrink?: string;
|
|
191
|
+
}
|
|
192
|
+
export interface WorkflowScheduleInput {
|
|
193
|
+
/** Cron-ish, interpreted by whatever schedules it. Empty means manual only. */
|
|
194
|
+
schedule?: string;
|
|
195
|
+
enabled?: boolean;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* The stored graph, plus what the server thinks of the schedule on it.
|
|
199
|
+
*
|
|
200
|
+
* `warning` is `null` when the schedule will fire. It is a string when it will
|
|
201
|
+
* not — and that is the entire reason this call answers with more than the
|
|
202
|
+
* workflow: a cron stored on a draft, on a disabled graph, or in a syntax
|
|
203
|
+
* nothing can parse is a pipeline that looks scheduled and never runs, which is
|
|
204
|
+
* a failure with an incident behind it rather than a hypothetical.
|
|
205
|
+
*/
|
|
206
|
+
export interface ScheduledWorkflow extends CatalogWorkflow {
|
|
207
|
+
warning?: string | null;
|
|
208
|
+
}
|
|
141
209
|
/**
|
|
142
210
|
* One saved version of the two things in this catalog whose text a person edits.
|
|
143
211
|
*
|
|
@@ -397,6 +465,16 @@ export interface CatalogClient {
|
|
|
397
465
|
* not the other.
|
|
398
466
|
*/
|
|
399
467
|
pipelineCapabilities(): Promise<PipelineCapabilities>;
|
|
468
|
+
/**
|
|
469
|
+
* Every workflow the live fleet announces it can execute — what a call node's
|
|
470
|
+
* picker offers.
|
|
471
|
+
*
|
|
472
|
+
* A separate call from {@link pipelineCapabilities} rather than a field on it,
|
|
473
|
+
* because the two have opposite lifetimes. Capabilities cannot change without
|
|
474
|
+
* a redeploy, so the canvas caches them forever. This is a snapshot of live
|
|
475
|
+
* workers with a resolution of about one heartbeat, so it must not be.
|
|
476
|
+
*/
|
|
477
|
+
listCallableWorkflows(): Promise<CallableWorkflowList>;
|
|
400
478
|
listConnections(): Promise<CatalogConnection[]>;
|
|
401
479
|
saveConnection(input: ConnectionInput): Promise<CatalogConnection>;
|
|
402
480
|
/**
|
|
@@ -407,27 +485,64 @@ export interface CatalogClient {
|
|
|
407
485
|
* system it names.
|
|
408
486
|
*/
|
|
409
487
|
checkConnection(id: string): Promise<ConnectionCheck>;
|
|
410
|
-
/**
|
|
411
|
-
|
|
488
|
+
/**
|
|
489
|
+
* Reach an address that has not been stored, and record nothing.
|
|
490
|
+
*
|
|
491
|
+
* The reason this exists rather than a form telling somebody to save first:
|
|
492
|
+
* the field most likely to be wrong is the one that cannot be verified by
|
|
493
|
+
* reading it, and the way you would otherwise find out is a load failing on a
|
|
494
|
+
* schedule hours later. A connection saved to discover a typo is a row
|
|
495
|
+
* somebody then has to remember to delete.
|
|
496
|
+
*
|
|
497
|
+
* Asks for `catalog:write` where {@link CatalogClient.checkConnection} asks
|
|
498
|
+
* only for `catalog:read`, and a host wiring its own transport should know
|
|
499
|
+
* why: this reaches an address supplied in the request, so under the softer
|
|
500
|
+
* scope it is the server connecting wherever a reader of the catalog points
|
|
501
|
+
* it. It grants no reach a `catalog:write` holder did not already have — they
|
|
502
|
+
* could save, check and delete — but it leaves no rows behind, so the server
|
|
503
|
+
* writes what it did to its log instead.
|
|
504
|
+
*
|
|
505
|
+
* `id` on the input is what makes an EDIT testable: the server puts the stored
|
|
506
|
+
* credential back where the caller was only ever shown a redaction of it. A
|
|
507
|
+
* create sends no id, and nothing is restored — the address reached is exactly
|
|
508
|
+
* the one that was typed.
|
|
509
|
+
*/
|
|
510
|
+
checkUnsavedConnection(input: ConnectionInput): Promise<ConnectionCheck>;
|
|
511
|
+
/**
|
|
512
|
+
* Which pipelines read through it. Named, so a refusal can say which.
|
|
513
|
+
*
|
|
514
|
+
* Workflows and not connectors, because that is the question somebody is
|
|
515
|
+
* actually asking before they delete a connection: what breaks. A connector
|
|
516
|
+
* is an implementation detail of a published graph and naming one in a
|
|
517
|
+
* refusal would send the reader looking for a screen that no longer exists.
|
|
518
|
+
*
|
|
519
|
+
* The server's answer, and it outranks anything a screen derives from the
|
|
520
|
+
* workflow list: a source node may reach a connection through a connector it
|
|
521
|
+
* names in `config.connectorId` rather than through its own `connectionId`,
|
|
522
|
+
* and only the server folds both in.
|
|
523
|
+
*/
|
|
524
|
+
connectionWorkflows(id: string): Promise<ConnectionUse[]>;
|
|
412
525
|
deleteConnection(id: string): Promise<{
|
|
413
526
|
deleted: boolean;
|
|
414
527
|
}>;
|
|
415
|
-
listConnectors(): Promise<CatalogConnector[]>;
|
|
416
|
-
saveConnector(input: ConnectorInput): Promise<CatalogConnector>;
|
|
417
|
-
deleteConnector(id: string): Promise<{
|
|
418
|
-
deleted: boolean;
|
|
419
|
-
}>;
|
|
420
|
-
runConnector(id: string): Promise<ConnectorRun>;
|
|
421
528
|
/**
|
|
422
|
-
*
|
|
423
|
-
*
|
|
424
|
-
*
|
|
425
|
-
*
|
|
426
|
-
*
|
|
427
|
-
*
|
|
428
|
-
*
|
|
529
|
+
* The connectors, read-only.
|
|
530
|
+
*
|
|
531
|
+
* **There is deliberately no `saveConnector`, `deleteConnector` or
|
|
532
|
+
* `runConnector` on this client, because there are no such routes.** A
|
|
533
|
+
* connector is what a published workflow runs as: `publishWorkflow` mints
|
|
534
|
+
* one, `deleteWorkflow` takes it away, and `runWorkflow` is the only way to
|
|
535
|
+
* start one. A method here that posted to `pipeline/connectors` would 404,
|
|
536
|
+
* and one that quietly did nothing would be worse.
|
|
537
|
+
*
|
|
538
|
+
* This read stays because it answers what a graph cannot answer about itself:
|
|
539
|
+
* which id the run history and the incremental watermark are keyed on, and —
|
|
540
|
+
* on a deployment upgraded rather than built fresh — which rows belong to no
|
|
541
|
+
* graph at all. Nothing turns those into workflows; they keep loading on the
|
|
542
|
+
* path they were already on, and this read is the only place they are visible.
|
|
543
|
+
* `CatalogConnector.workflowId` is what joins one to the graph that runs it.
|
|
429
544
|
*/
|
|
430
|
-
|
|
545
|
+
listConnectors(): Promise<CatalogConnector[]>;
|
|
431
546
|
/**
|
|
432
547
|
* Publish an object type's schema — create it, or update the shape of one
|
|
433
548
|
* that exists.
|
|
@@ -505,6 +620,29 @@ export interface CatalogClient {
|
|
|
505
620
|
*/
|
|
506
621
|
listWorkflows(): Promise<CatalogWorkflow[]>;
|
|
507
622
|
saveWorkflow(input: WorkflowInput): Promise<CatalogWorkflow>;
|
|
623
|
+
/**
|
|
624
|
+
* Declare it finished, and mint the connector it runs as.
|
|
625
|
+
*
|
|
626
|
+
* Its own call rather than a field on the save, because publishing is a claim
|
|
627
|
+
* the server checks and a failed check owes an explanation naming the nodes —
|
|
628
|
+
* which an autosave has nowhere to put. A draft saves without validating;
|
|
629
|
+
* this is where the rules are answered for real.
|
|
630
|
+
*/
|
|
631
|
+
publishWorkflow(id: string): Promise<CatalogWorkflow>;
|
|
632
|
+
/**
|
|
633
|
+
* Back to draft. Disables the connector, keeping its id, its run history and
|
|
634
|
+
* its watermark, so re-publishing resumes the same pipeline rather than
|
|
635
|
+
* starting a second one beside it.
|
|
636
|
+
*/
|
|
637
|
+
unpublishWorkflow(id: string): Promise<CatalogWorkflow>;
|
|
638
|
+
/**
|
|
639
|
+
* Delete it, and the connector with it.
|
|
640
|
+
*
|
|
641
|
+
* The run history goes too, which is why the console asks first. It used to
|
|
642
|
+
* be refused while a connector still ran the graph; that check would now
|
|
643
|
+
* refuse every delete there is, since a published graph runs as exactly one
|
|
644
|
+
* connector — its own.
|
|
645
|
+
*/
|
|
508
646
|
deleteWorkflow(id: string): Promise<{
|
|
509
647
|
deleted: boolean;
|
|
510
648
|
}>;
|
|
@@ -512,8 +650,54 @@ export interface CatalogClient {
|
|
|
512
650
|
* Execute it. When a durable engine is available each node runs as its own
|
|
513
651
|
* step, so a failure part-way resumes rather than restarts — see
|
|
514
652
|
* `PipelineCapabilities.durable`, which is what the screen states.
|
|
653
|
+
*
|
|
654
|
+
* `expectShrink` is the escape hatch for a load the row-count bound refused,
|
|
655
|
+
* and it reaches the server **only through here** — `POST connectors/:id/run`
|
|
656
|
+
* carried it and that route is gone. Without it an operator's only recourse
|
|
657
|
+
* would be raising `rowCount.maxShrink` in the type's policy, which stands the
|
|
658
|
+
* guard down for every future load of that type instead of for one snapshot.
|
|
659
|
+
*
|
|
660
|
+
* It is a reason, not a flag: the string is written into the snapshot's
|
|
661
|
+
* `_expectShrink` label and is the only answer anybody will have in six
|
|
662
|
+
* months to "why was this load allowed to lose most of the data?". Sending it
|
|
663
|
+
* empty is refused with a 400 asking for one, which is why the console's
|
|
664
|
+
* dialog will not submit a blank box rather than sending it and hoping.
|
|
665
|
+
*/
|
|
666
|
+
runWorkflow(id: string, options?: WorkflowRunOptions): Promise<WorkflowRun>;
|
|
667
|
+
/**
|
|
668
|
+
* Set when this graph runs, and whether it runs at all.
|
|
669
|
+
*
|
|
670
|
+
* Both fields are optional and an absent one means "leave it alone", so a
|
|
671
|
+
* screen rendering only the cron cannot silently re-enable a pipeline
|
|
672
|
+
* somebody turned off. The answer is the stored workflow plus `warning`,
|
|
673
|
+
* which is the server naming a schedule that will never fire — a draft, a
|
|
674
|
+
* disabled graph, an unparseable cron. A screen that dropped it would repeat
|
|
675
|
+
* the incident this route was written after: a scheduler announcing it was
|
|
676
|
+
* watching schedules while parsing nothing.
|
|
677
|
+
*/
|
|
678
|
+
scheduleWorkflow(id: string, input: WorkflowScheduleInput): Promise<ScheduledWorkflow>;
|
|
679
|
+
/**
|
|
680
|
+
* What the system behind one source node looks like right now. Writes nothing.
|
|
681
|
+
*
|
|
682
|
+
* Takes a node and not a connector, and the swap is the whole of the move:
|
|
683
|
+
* this was `POST connectors/:id/discover`, which refused outright for any
|
|
684
|
+
* connector carrying a `workflowId` — and every connector carries one now, so
|
|
685
|
+
* the old shape would have refused every connector there is.
|
|
686
|
+
*
|
|
687
|
+
* **It answers on a draft, deliberately.** A sink cannot commit into a type
|
|
688
|
+
* that does not exist, so requiring a published graph would require publishing
|
|
689
|
+
* a graph whose target type cannot be created until it is published. What it
|
|
690
|
+
* does need is a graph that has been SAVED: the server reads the stored node,
|
|
691
|
+
* not the one on screen.
|
|
692
|
+
*
|
|
693
|
+
* Typed loosely on purpose. The shape comes from
|
|
694
|
+
* `@dudousxd/nestjs-catalog-pipeline`, and this package must not import that
|
|
695
|
+
* one — it would drag a package built for a Node process, with database
|
|
696
|
+
* drivers behind optional imports, into a browser bundle. `PipelineCapabilities`
|
|
697
|
+
* above is mirrored for exactly the same reason; `narrowDiscovery` in
|
|
698
|
+
* `schema-discovery.tsx` is what checks the answer at the seam.
|
|
515
699
|
*/
|
|
516
|
-
|
|
700
|
+
discoverSourceSchema(workflowId: string, nodeId: string): Promise<unknown>;
|
|
517
701
|
/**
|
|
518
702
|
* Who may do what.
|
|
519
703
|
*
|
|
@@ -579,14 +763,28 @@ export declare const catalogQueryKeys: {
|
|
|
579
763
|
search: (term: string, limit?: number) => readonly ["nestjs-catalog", "search", string, number | null];
|
|
580
764
|
/**
|
|
581
765
|
* The pipeline keys are nested under one prefix so a host — or a screen that
|
|
582
|
-
* just ran a
|
|
583
|
-
*
|
|
584
|
-
*
|
|
585
|
-
*
|
|
586
|
-
*
|
|
766
|
+
* just ran a pipeline — can invalidate the whole of it at once. Running one
|
|
767
|
+
* changes the runs, the connector's own last-run fields, and the catalog
|
|
768
|
+
* snapshot the load wrote into, and invalidating them one by one is how a
|
|
769
|
+
* screen ends up showing a run that finished next to a pipeline that still
|
|
770
|
+
* says it never has.
|
|
771
|
+
*
|
|
772
|
+
* `connectors` is still here and is still worth invalidating, even though
|
|
773
|
+
* nothing authors one any more: publishing a graph MINTS one, unpublishing
|
|
774
|
+
* disables it, and deleting takes it away — so the read that says which id a
|
|
775
|
+
* run history is keyed on goes stale on all three.
|
|
587
776
|
*/
|
|
588
777
|
pipeline: readonly ["nestjs-catalog", "pipeline"];
|
|
589
778
|
capabilities: readonly ["nestjs-catalog", "pipeline", "capabilities"];
|
|
779
|
+
/**
|
|
780
|
+
* What the live fleet announces it can execute.
|
|
781
|
+
*
|
|
782
|
+
* Its own key rather than a slice of `capabilities`, because the two are
|
|
783
|
+
* cached on opposite terms — `capabilities` is stale-forever and this is stale
|
|
784
|
+
* in about a heartbeat — and one key cannot hold two staleTimes. A host
|
|
785
|
+
* invalidating the whole `pipeline` prefix still reaches both.
|
|
786
|
+
*/
|
|
787
|
+
callableWorkflows: readonly ["nestjs-catalog", "pipeline", "callable-workflows"];
|
|
590
788
|
connections: readonly ["nestjs-catalog", "pipeline", "connections"];
|
|
591
789
|
connectors: readonly ["nestjs-catalog", "pipeline", "connectors"];
|
|
592
790
|
transforms: readonly ["nestjs-catalog", "pipeline", "transforms"];
|
package/dist/context.js
CHANGED
|
@@ -52,16 +52,14 @@ export function CatalogProvider({ transport, pipelineBasePath = DEFAULT_PIPELINE
|
|
|
52
52
|
listEvents: (query) => transport.get(catalogRoutes.events(), { ...query }),
|
|
53
53
|
listTraces: (query) => transport.get(catalogRoutes.traces(), { ...query }),
|
|
54
54
|
pipelineCapabilities: () => transport.get(pipeline.capabilities()),
|
|
55
|
+
listCallableWorkflows: () => transport.get(pipeline.callableWorkflows()),
|
|
55
56
|
listConnections: () => transport.get(pipeline.connections()),
|
|
56
57
|
saveConnection: (input) => transport.post(pipeline.connections(), input),
|
|
57
58
|
checkConnection: (id) => transport.post(pipeline.checkConnection(id), {}),
|
|
58
|
-
|
|
59
|
+
checkUnsavedConnection: (input) => transport.post(pipeline.checkUnsavedConnection(), input),
|
|
60
|
+
connectionWorkflows: (id) => transport.get(pipeline.connectionWorkflows(id)),
|
|
59
61
|
deleteConnection: (id) => transport.delete(pipeline.connection(id)),
|
|
60
62
|
listConnectors: () => transport.get(pipeline.connectors()),
|
|
61
|
-
saveConnector: (input) => transport.post(pipeline.connectors(), input),
|
|
62
|
-
deleteConnector: (id) => transport.delete(pipeline.connector(id)),
|
|
63
|
-
runConnector: (id) => transport.post(pipeline.runConnector(id), {}),
|
|
64
|
-
discoverConnectorSchema: (id) => transport.post(pipeline.discoverConnectorSchema(id), {}),
|
|
65
63
|
publishType: (name, schema) => {
|
|
66
64
|
if (!transport.put) {
|
|
67
65
|
throw new Error('This transport cannot PUT, so it cannot publish a type. Implement `put` on the ' +
|
|
@@ -92,8 +90,31 @@ export function CatalogProvider({ transport, pipelineBasePath = DEFAULT_PIPELINE
|
|
|
92
90
|
tryTransform: (input) => transport.post(pipeline.tryTransform(), input),
|
|
93
91
|
listWorkflows: () => transport.get(pipeline.workflows()),
|
|
94
92
|
saveWorkflow: (input) => transport.post(pipeline.workflows(), input),
|
|
93
|
+
publishWorkflow: (id) => transport.post(pipeline.publishWorkflow(id), {}),
|
|
94
|
+
unpublishWorkflow: (id) => transport.post(pipeline.unpublishWorkflow(id), {}),
|
|
95
95
|
deleteWorkflow: (id) => transport.delete(pipeline.workflow(id)),
|
|
96
|
-
|
|
96
|
+
// The two fields are spread conditionally rather than sent as
|
|
97
|
+
// `undefined`, and for `expectShrink` that is the whole contract: the
|
|
98
|
+
// server distinguishes a body with no such key — nobody said anything,
|
|
99
|
+
// let the bound decide — from one carrying an empty string, which is an
|
|
100
|
+
// acknowledgement with no reason behind it and is refused with a 400
|
|
101
|
+
// asking for one. A transport that serialises `undefined` as `null`, or
|
|
102
|
+
// one that drops it, would flatten those two into each other.
|
|
103
|
+
runWorkflow: (id, options) => transport.post(pipeline.runWorkflow(id), {
|
|
104
|
+
...(options?.snapshotId === undefined ? {} : { snapshotId: options.snapshotId }),
|
|
105
|
+
...(options && 'expectShrink' in options && options.expectShrink !== undefined
|
|
106
|
+
? { expectShrink: options.expectShrink }
|
|
107
|
+
: {}),
|
|
108
|
+
}),
|
|
109
|
+
scheduleWorkflow: (id, input) => {
|
|
110
|
+
if (!transport.put) {
|
|
111
|
+
throw new Error('This transport cannot PUT, so it cannot set a schedule. Implement `put` on the ' +
|
|
112
|
+
'transport you passed to CatalogProvider — a schedule is an idempotent upsert of ' +
|
|
113
|
+
'the fields it names, which is why the route is a PUT and not a POST.');
|
|
114
|
+
}
|
|
115
|
+
return transport.put(pipeline.workflowSchedule(id), input);
|
|
116
|
+
},
|
|
117
|
+
discoverSourceSchema: (workflowId, nodeId) => transport.post(pipeline.discoverSourceSchema(workflowId, nodeId), {}),
|
|
97
118
|
listPrincipals: () => transport.get(access.principals()),
|
|
98
119
|
listPeople: (query) => transport.get(access.people(query)),
|
|
99
120
|
upsertPerson: (input) => transport.post(access.people(), input),
|
|
@@ -135,14 +156,28 @@ export const catalogQueryKeys = {
|
|
|
135
156
|
search: (term, limit) => ['nestjs-catalog', 'search', term, limit ?? null],
|
|
136
157
|
/**
|
|
137
158
|
* The pipeline keys are nested under one prefix so a host — or a screen that
|
|
138
|
-
* just ran a
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
159
|
+
* just ran a pipeline — can invalidate the whole of it at once. Running one
|
|
160
|
+
* changes the runs, the connector's own last-run fields, and the catalog
|
|
161
|
+
* snapshot the load wrote into, and invalidating them one by one is how a
|
|
162
|
+
* screen ends up showing a run that finished next to a pipeline that still
|
|
163
|
+
* says it never has.
|
|
164
|
+
*
|
|
165
|
+
* `connectors` is still here and is still worth invalidating, even though
|
|
166
|
+
* nothing authors one any more: publishing a graph MINTS one, unpublishing
|
|
167
|
+
* disables it, and deleting takes it away — so the read that says which id a
|
|
168
|
+
* run history is keyed on goes stale on all three.
|
|
143
169
|
*/
|
|
144
170
|
pipeline: ['nestjs-catalog', 'pipeline'],
|
|
145
171
|
capabilities: ['nestjs-catalog', 'pipeline', 'capabilities'],
|
|
172
|
+
/**
|
|
173
|
+
* What the live fleet announces it can execute.
|
|
174
|
+
*
|
|
175
|
+
* Its own key rather than a slice of `capabilities`, because the two are
|
|
176
|
+
* cached on opposite terms — `capabilities` is stale-forever and this is stale
|
|
177
|
+
* in about a heartbeat — and one key cannot hold two staleTimes. A host
|
|
178
|
+
* invalidating the whole `pipeline` prefix still reaches both.
|
|
179
|
+
*/
|
|
180
|
+
callableWorkflows: ['nestjs-catalog', 'pipeline', 'callable-workflows'],
|
|
146
181
|
connections: ['nestjs-catalog', 'pipeline', 'connections'],
|
|
147
182
|
connectors: ['nestjs-catalog', 'pipeline', 'connectors'],
|
|
148
183
|
transforms: ['nestjs-catalog', 'pipeline', 'transforms'],
|
package/dist/index.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ export { CatalogManager, type CatalogManagerProps } from './CatalogManager';
|
|
|
2
2
|
export { CoverageLedger } from './CoverageLedger';
|
|
3
3
|
export { EditableField } from './EditableField';
|
|
4
4
|
export { DELETE_STRATEGIES, LoadExpectationSection, type LoadExpectationSectionProps, } from './LoadExpectationSection';
|
|
5
|
+
export { LoadedBySection, type LoadedBySectionProps, loadersOf, } from './LoadedBySection';
|
|
5
6
|
export { cn } from './cn';
|
|
6
7
|
export { ObjectExplorer, type ObjectExplorerProps } from './ObjectExplorer';
|
|
7
8
|
export { DashboardBoard } from './DashboardBoard';
|
|
@@ -16,18 +17,24 @@ export { CatalogSearch, type CatalogSearchProps } from './search-console';
|
|
|
16
17
|
export { SavedQueryPanel } from './SavedQueryPanel';
|
|
17
18
|
export { CATALOG_EVENT_META, type CatalogTraceSource, eventMeta, TraceExplorer, useTraceSource, } from './TraceExplorer';
|
|
18
19
|
export { ConnectionPanel, type ConnectionPanelProps, connectionOptionsFor, describeConnection, } from './ConnectionPanel';
|
|
19
|
-
export {
|
|
20
|
+
export { CONNECTABLE_KINDS, CONNECTION_KIND_OPTIONS, CONNECTION_KINDS, type ConnectableKind, type ConnectableKindSpec, ConnectionCheckResult, type ConnectionDraft, ConnectionKindFields, type ConnectionKindSpec, type UnconnectableKindSpec, type UpdateDraft, connectableSpec, connectionConfigFor, connectionDraftFrom, connectionIsIncomplete, redactedCredentialIn, toConnectableKind, } from './connection-form';
|
|
21
|
+
export { SourceConnectionCreator } from './source-connection';
|
|
22
|
+
export { PipelineConsole, type PipelineConsoleProps, } from './PipelineConsole';
|
|
23
|
+
export { type ColumnChoice, type ConnectorSchemaDiscovery, type DiscoveredColumn, type DiscoveredTypeDraft, initialChoices, narrowDiscovery, proposalFrom, type SchemaDiscoveryBridge, SchemaDiscoveryPanel, type SchemaDrift, } from './schema-discovery';
|
|
20
24
|
export { TransformEditor, type TransformEditorProps } from './TransformEditor';
|
|
21
25
|
export { DiffBody, RevisionHistory, type RevisionHistoryProps, RevisionHistoryButton, RevisionHistorySheet, type RevisionHistorySheetProps, type RevisionSubject, type RevisionSubjectKind, } from './diff/RevisionDiff';
|
|
22
26
|
export { AccessConsole, type AccessConsoleProps } from './AccessConsole';
|
|
23
|
-
export { type CatalogWorkflow, describeDurability, type DurabilityCopy, isWorkflowNodeKind, newLocalId, nodeName, producedTypes, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, type WorkflowDurability, type WorkflowEdge, type WorkflowGraph, type WorkflowInput, type WorkflowNode, type WorkflowNodeKind, type WorkflowRun, type WorkflowRunNode, type WorkflowSinkNode, type WorkflowSourceNode, type WorkflowTransformNode, } from './workflow/model';
|
|
27
|
+
export { type CallableWorkflowRef, type CatalogWorkflow, describeDurability, type DurabilityCopy, isWorkflowNodeKind, newLocalId, nodeName, producedTypes, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, type WorkflowCallNode, type WorkflowDurability, type WorkflowEdge, type WorkflowGraph, type WorkflowInput, type WorkflowNode, type WorkflowNodeKind, type WorkflowRun, type WorkflowRunNode, type WorkflowSinkNode, type WorkflowSourceNode, type WorkflowTransformNode, } from './workflow/model';
|
|
24
28
|
export { WORKFLOW_NAME } from './workflow/name';
|
|
29
|
+
export { checkShapes, type ShapeKnowledge, type SourceColumn, type SourceShape, type TargetProperty, type TargetShape, } from './workflow/shape';
|
|
25
30
|
export { canConnect, type ConnectionVerdict, edgeId, hasBlockingProblem, problemsByNode, type ValidateOptions, validateWorkflow, type WorkflowDraft, type WorkflowProblem, type WorkflowProblemCode, type WorkflowProblemLevel, wouldCycle, } from './workflow/validate';
|
|
26
|
-
export {
|
|
31
|
+
export { attachTransformIds, buildWorkflowTemplate, type EnrichOptions, enrichWithLookup, fanOutTypes, type FanOutOptions, type FanOutTarget, FILE_DROP_KINDS, type FileDropOptions, joinSources, type JoinOptions, loadFileDrop, type PeriodicFullReloadOptions, periodicFullReload, planIsRunnable, planToWorkflowInput, refuseUnusableColumnNames, RELOAD_CADENCE_IDS, RELOAD_CADENCES, type ReloadCadence, replicateTable, type ReplicateTableOptions, SOURCE_KINDS, type TemplateDeclaration, type TemplateExpectation, type TemplateOutcome, type TemplatePlan, type TemplateRefusal, type TemplateRefusalCode, type TemplateRequest, type TemplateSchedule, type TemplateTransformRequest, withinMsFor, WORKFLOW_TEMPLATE_IDS, WORKFLOW_TEMPLATES, type WorkflowTemplateId, type WorkflowTemplateMeta, } from './workflow/templates';
|
|
32
|
+
export { type CatalogClient, type CatalogIdentity, type CatalogRevision, type CatalogPeoplePage, type CatalogPersonRole, type CatalogPersonSummary, type CatalogPrincipalSummary, CatalogProvider, type CatalogProviderProps, type CatalogTransport, type ConnectionInput, type ConnectionUse, catalogQueryKeys, type DeleteReconciliation, type LoadExpectation, type LoadExpectationInput, type ResolvedLoadExpectation, type RowCountBound, type StoredLoadExpectation, type PersonInput, type PersonUpsertResult, type PipelineCapabilities, type CallableWorkflowList, type ScheduledWorkflow, type TransformInput, type WorkflowRunOptions, type WorkflowScheduleInput, useCatalogClient, } from './context';
|
|
27
33
|
export { type AccessRoutes, accessRoutes, DEFAULT_ACCESS_BASE_PATH, DEFAULT_PIPELINE_BASE_PATH, DEFAULT_PUBLISH_BASE_PATH, embedRoutes, type PeopleQuery, type PipelineRoutes, pipelineRoutes, } from './routes';
|
|
28
34
|
export { ConfirmDialog } from './ui/dialog';
|
|
29
35
|
export { FieldGroup, TextAreaField, TextField } from './ui/field';
|
|
30
36
|
export { Select, SelectField, type SelectOption } from './ui/select';
|
|
37
|
+
export { Combobox, ComboboxField, type ComboOption, type ComboboxProps } from './ui/combobox';
|
|
31
38
|
export { Switch } from './ui/switch';
|
|
32
39
|
export { Sheet } from './ui/sheet';
|
|
33
40
|
export { CodeEditor, type CodeEditorHandle, type CodeEditorProps, codeEditorRoot, codeEditorText, } from './ui/code-editor';
|
package/dist/index.js
CHANGED
|
@@ -5,6 +5,12 @@ export { EditableField } from './EditableField';
|
|
|
5
5
|
// own type panel out of this package's parts should not have to rebuild the one
|
|
6
6
|
// section that can say an incremental load of a type is currently refused.
|
|
7
7
|
export { DELETE_STRATEGIES, LoadExpectationSection, } from './LoadExpectationSection';
|
|
8
|
+
// Mounted on the Model screen too, directly above the expectation section — the
|
|
9
|
+
// two are one question in two halves: who loads this type, and what is checked
|
|
10
|
+
// when they do. `loadersOf` is exported beside the component because it is the
|
|
11
|
+
// whole rule (a graph's SINKS decide, not its stored `targetType`), and a host
|
|
12
|
+
// that renders its own row wants the rule rather than a second reading of it.
|
|
13
|
+
export { LoadedBySection, loadersOf, } from './LoadedBySection';
|
|
8
14
|
export { cn } from './cn';
|
|
9
15
|
export { ObjectExplorer } from './ObjectExplorer';
|
|
10
16
|
export { DashboardBoard } from './DashboardBoard';
|
|
@@ -26,24 +32,50 @@ export { QueryConsole } from './QueryConsole';
|
|
|
26
32
|
export { CatalogSearch } from './search-console';
|
|
27
33
|
export { SavedQueryPanel } from './SavedQueryPanel';
|
|
28
34
|
export { CATALOG_EVENT_META, eventMeta, TraceExplorer, useTraceSource, } from './TraceExplorer';
|
|
29
|
-
// Getting data in:
|
|
30
|
-
//
|
|
31
|
-
// installing this package got a console with no way to manage a
|
|
35
|
+
// Getting data in: the addresses a pipeline reads through and the code that
|
|
36
|
+
// reshapes what it fetches. These used to live in the consuming app, which meant
|
|
37
|
+
// any other host installing this package got a console with no way to manage a
|
|
38
|
+
// load.
|
|
39
|
+
//
|
|
40
|
+
// There is deliberately no connector screen here any more. A connector is what a
|
|
41
|
+
// published workflow runs as — nothing authors one — so authoring a pipeline is
|
|
42
|
+
// `WorkflowCanvas`, on the `/workflow` subpath, and this is the two shared
|
|
43
|
+
// objects it borrows.
|
|
32
44
|
export { ConnectionPanel, connectionOptionsFor, describeConnection, } from './ConnectionPanel';
|
|
45
|
+
// What a connection IS, as data: which kinds can be one, what each needs, and
|
|
46
|
+
// the two rules — completeness, and the redaction placeholder — that decide
|
|
47
|
+
// whether a form may send one.
|
|
48
|
+
//
|
|
49
|
+
// Exported because there are two forms now. The console's is here; the source
|
|
50
|
+
// node's inspector on the canvas has its own, behind the `/workflow` subpath,
|
|
51
|
+
// and a host assembling a third should be reading these rather than restating
|
|
52
|
+
// what an S3 connection needs.
|
|
53
|
+
export { CONNECTABLE_KINDS, CONNECTION_KIND_OPTIONS, CONNECTION_KINDS, ConnectionCheckResult, ConnectionKindFields, connectableSpec, connectionConfigFor, connectionDraftFrom, connectionIsIncomplete, redactedCredentialIn, toConnectableKind, } from './connection-form';
|
|
54
|
+
// Creating a connection from wherever a source is being configured.
|
|
55
|
+
//
|
|
56
|
+
// On the root entry and not behind `/workflow`, for the reason
|
|
57
|
+
// `SchemaDiscoveryPanel` is: it imports no React Flow, and it is the other half
|
|
58
|
+
// of the same offer — the sink's panel makes the type, this makes the address. A
|
|
59
|
+
// host assembling its own source inspector should be able to mount either.
|
|
60
|
+
export { SourceConnectionCreator } from './source-connection';
|
|
61
|
+
export { PipelineConsole, } from './PipelineConsole';
|
|
33
62
|
// The schema-discovery seam, whole.
|
|
34
63
|
//
|
|
35
|
-
// `
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
// a file inside `dist/`. Both are the caller paying for a list that fell behind.
|
|
64
|
+
// `SchemaDiscoveryBridge` is a bridge a HOST may implement — this package cannot
|
|
65
|
+
// ask a source for its columns, only render the answer — and every one of these
|
|
66
|
+
// was once reachable only through an indexed access on a component's props, or
|
|
67
|
+
// by deep-importing a file inside `dist/`. Both are the caller paying for a list
|
|
68
|
+
// that fell behind.
|
|
41
69
|
//
|
|
42
70
|
// `initialChoices` and `proposalFrom` come with them because they are the rules
|
|
43
71
|
// that decide whether a schema somebody ticked is one the publish route will
|
|
44
72
|
// accept — pure and deliberately outside the panel, per their own docblocks, and
|
|
45
73
|
// worth nothing there if the only way to run them is to render it.
|
|
46
|
-
|
|
74
|
+
//
|
|
75
|
+
// `SchemaDiscoveryPanel` is exported for the same reason it stopped living
|
|
76
|
+
// inside a screen: the canvas mounts it from behind the `/workflow` subpath, and
|
|
77
|
+
// a host assembling its own source inspector should be able to do the same.
|
|
78
|
+
export { initialChoices, narrowDiscovery, proposalFrom, SchemaDiscoveryPanel, } from './schema-discovery';
|
|
47
79
|
export { TransformEditor } from './TransformEditor';
|
|
48
80
|
// Why one load came out different from the last one.
|
|
49
81
|
//
|
|
@@ -84,11 +116,27 @@ export { AccessConsole } from './AccessConsole';
|
|
|
84
116
|
// and it belongs to the package that executes them. See `workflow/model.ts`.
|
|
85
117
|
export { describeDurability, isWorkflowNodeKind, newLocalId, nodeName, producedTypes, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, } from './workflow/model';
|
|
86
118
|
export { WORKFLOW_NAME } from './workflow/name';
|
|
87
|
-
//
|
|
88
|
-
//
|
|
89
|
-
//
|
|
90
|
-
//
|
|
119
|
+
// Whether what a source reads fits the type its sink writes.
|
|
120
|
+
//
|
|
121
|
+
// The canvas answers this for itself now — it holds what `discoverSourceSchema`
|
|
122
|
+
// returned for each source node and compares it against the snapshot it already
|
|
123
|
+
// reads, so there is no prop and nothing for a host to wire. These are exported
|
|
124
|
+
// because the comparison is pure and a host may well want it somewhere else: a
|
|
125
|
+
// pre-flight before a scheduled run, say. `ConnectorSchemaDiscovery` above is
|
|
126
|
+
// already a `SourceShape`, so that caller has one for free.
|
|
127
|
+
export { checkShapes, } from './workflow/shape';
|
|
128
|
+
// Core's rules, run in the browser, plus the questions core cannot answer:
|
|
129
|
+
// whether a connection is legal mid-drag, whether a node names a transform that
|
|
130
|
+
// still exists, and whether a source supplies the columns its sink writes.
|
|
131
|
+
// Exported so a host can run the same checks before it offers a save — never so
|
|
132
|
+
// it can skip the server's, which is the authority.
|
|
91
133
|
export { canConnect, edgeId, hasBlockingProblem, problemsByNode, validateWorkflow, wouldCycle, } from './workflow/validate';
|
|
134
|
+
// The prefilled graphs, on this entry point rather than on `/workflow` because
|
|
135
|
+
// they import no React Flow: a template produces nodes, edges, transform bodies
|
|
136
|
+
// and expectation payloads, and a host should be able to build one — or check
|
|
137
|
+
// what it would decide — without installing a canvas to do it. Whether a screen
|
|
138
|
+
// then draws the result is the canvas's business, not the template's.
|
|
139
|
+
export { attachTransformIds, buildWorkflowTemplate, enrichWithLookup, fanOutTypes, FILE_DROP_KINDS, joinSources, loadFileDrop, periodicFullReload, planIsRunnable, planToWorkflowInput, refuseUnusableColumnNames, RELOAD_CADENCE_IDS, RELOAD_CADENCES, replicateTable, SOURCE_KINDS, withinMsFor, WORKFLOW_TEMPLATE_IDS, WORKFLOW_TEMPLATES, } from './workflow/templates';
|
|
92
140
|
export { CatalogProvider, catalogQueryKeys, useCatalogClient, } from './context';
|
|
93
141
|
// The paths behind the pipeline and access screens, exported so a host can
|
|
94
142
|
// mount those endpoints wherever it likes and tell the provider where they are.
|
|
@@ -107,6 +155,7 @@ embedRoutes, pipelineRoutes, } from './routes';
|
|
|
107
155
|
export { ConfirmDialog } from './ui/dialog';
|
|
108
156
|
export { FieldGroup, TextAreaField, TextField } from './ui/field';
|
|
109
157
|
export { Select, SelectField } from './ui/select';
|
|
158
|
+
export { Combobox, ComboboxField } from './ui/combobox';
|
|
110
159
|
export { Switch } from './ui/switch';
|
|
111
160
|
export { Sheet } from './ui/sheet';
|
|
112
161
|
// `codeEditorRoot` and `codeEditorText` come with it because the editor renders
|