@dudousxd/nestjs-catalog-react 0.17.0 → 0.19.0-preview.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +39 -13
  2. package/dist/CatalogManager.d.ts +23 -1
  3. package/dist/CatalogManager.js +76 -13
  4. package/dist/ConnectionPanel.d.ts +11 -7
  5. package/dist/ConnectionPanel.js +78 -149
  6. package/dist/LoadedBySection.d.ts +39 -0
  7. package/dist/LoadedBySection.js +143 -0
  8. package/dist/PipelineConsole.d.ts +23 -126
  9. package/dist/PipelineConsole.js +39 -551
  10. package/dist/WorkflowCanvas.d.ts +16 -0
  11. package/dist/WorkflowCanvas.js +1064 -64
  12. package/dist/connection-form.d.ts +241 -0
  13. package/dist/connection-form.js +235 -0
  14. package/dist/context.d.ts +236 -38
  15. package/dist/context.js +46 -11
  16. package/dist/index.d.ts +10 -3
  17. package/dist/index.js +63 -14
  18. package/dist/routes.d.ts +84 -13
  19. package/dist/routes.js +7 -4
  20. package/dist/schema-discovery.d.ts +232 -0
  21. package/dist/schema-discovery.js +225 -0
  22. package/dist/source-connection.d.ts +82 -0
  23. package/dist/source-connection.js +176 -0
  24. package/dist/source-fields.d.ts +16 -14
  25. package/dist/source-fields.js +28 -18
  26. package/dist/ui/code-editor.d.ts +44 -2
  27. package/dist/ui/code-editor.js +10 -2
  28. package/dist/ui/combobox.d.ts +97 -0
  29. package/dist/ui/combobox.js +80 -0
  30. package/dist/ui/dialog.d.ts +13 -1
  31. package/dist/ui/dialog.js +13 -2
  32. package/dist/workflow/edges.d.ts +110 -0
  33. package/dist/workflow/edges.js +112 -0
  34. package/dist/workflow/graph.d.ts +39 -1
  35. package/dist/workflow/graph.js +212 -52
  36. package/dist/workflow/index.d.ts +8 -0
  37. package/dist/workflow/index.js +12 -0
  38. package/dist/workflow/lifecycle.d.ts +135 -0
  39. package/dist/workflow/lifecycle.js +216 -0
  40. package/dist/workflow/model.d.ts +17 -3
  41. package/dist/workflow/model.js +7 -2
  42. package/dist/workflow/nodes.d.ts +4 -1
  43. package/dist/workflow/nodes.js +198 -10
  44. package/dist/workflow/runs.d.ts +38 -0
  45. package/dist/workflow/runs.js +101 -0
  46. package/dist/workflow/shape.d.ts +169 -0
  47. package/dist/workflow/shape.js +249 -0
  48. package/dist/workflow/templates.d.ts +623 -0
  49. package/dist/workflow/templates.js +1102 -0
  50. package/dist/workflow/validate.d.ts +20 -1
  51. package/dist/workflow/validate.js +52 -3
  52. package/dist/workflow/wiring.d.ts +84 -0
  53. package/dist/workflow/wiring.js +254 -0
  54. package/package.json +9 -9
package/dist/context.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { CatalogAuditEvent, CatalogConnection, CatalogConnector, CatalogObjectPage, CatalogQueryRelation, CatalogQueryResult, CatalogRevision, CatalogSearchResult, CatalogSnapshot, CatalogTraceList, CatalogTransform, ConnectionCheck, ConnectorRun, Dashboard, DashboardCard, DeleteReconciliation, LoadExpectation, LoadExpectationInput, ObjectQueryParams, PropertyPatch, ResolvedLoadExpectation, RowCountBound, SaveQueryInput, SavedQuery, SnapshotRef, StoredLoadExpectation, TraceQuery, TransformLanguage, TransformResult, TypePatch } from '@dudousxd/nestjs-catalog/client';
1
+ import type { CallableWorkflowRef, CatalogAuditEvent, CatalogConnection, CatalogConnector, CatalogObjectPage, CatalogQueryRelation, CatalogQueryResult, CatalogRevision, CatalogSearchResult, CatalogSnapshot, CatalogTraceList, CatalogTransform, ConnectionCheck, ConnectorRun, Dashboard, DashboardCard, DeleteReconciliation, LoadExpectation, LoadExpectationInput, ObjectQueryParams, PropertyPatch, ResolvedLoadExpectation, RowCountBound, SaveQueryInput, SavedQuery, SnapshotRef, StoredLoadExpectation, TraceQuery, TransformLanguage, TransformResult, TypePatch } from '@dudousxd/nestjs-catalog/client';
2
2
  import { type ReactNode } from 'react';
3
3
  import type { EmbeddedChartPayload, EmbeddedDashboardPayload } from './embed/payload';
4
4
  import { type PeopleQuery } from './routes';
@@ -77,6 +77,37 @@ export interface PipelineCapabilities {
77
77
  */
78
78
  durable?: WorkflowDurability;
79
79
  }
80
+ /**
81
+ * What a call node could be pointed at, and whether the question was answerable.
82
+ *
83
+ * Mirrored here for the reason {@link PipelineCapabilities} is: naming the
84
+ * server's own declaration would mean importing
85
+ * `@dudousxd/nestjs-catalog-pipeline`, a package built for a Node process with
86
+ * database drivers behind optional imports. `CallableWorkflowRef` itself is
87
+ * re-exported from `@dudousxd/nestjs-catalog/client` and is NOT copied — the
88
+ * entries are the contract, and a second copy of those would drift.
89
+ *
90
+ * `supported` is the field that stops the list being misread, and it exists
91
+ * because an empty list means two opposite things. With `supported: true` it
92
+ * says the fleet announces nothing — every callable workflow is unregistered or
93
+ * its workers are down. With `supported: false` it says nobody could be asked:
94
+ * this deployment has no durable engine, or the engine could not read the
95
+ * announcements. A screen that rendered "no workflows found" over both would be
96
+ * asserting the first when it was told the second, which is the shape of claim
97
+ * this codebase keeps having to take back out.
98
+ */
99
+ export interface CallableWorkflowList {
100
+ supported: boolean;
101
+ workflows: CallableWorkflowRef[];
102
+ /**
103
+ * When the fleet was asked, ISO-8601. Rendered, because this is a snapshot
104
+ * with a resolution of about one worker heartbeat and showing it without a
105
+ * time would present a moment as a standing fact.
106
+ */
107
+ observedAt: string;
108
+ /** Why, in a full sentence. Rendered verbatim under the picker. */
109
+ detail: string;
110
+ }
80
111
  /**
81
112
  * What a load has to be true of before it becomes the data everybody reads.
82
113
  *
@@ -116,21 +147,6 @@ export interface ConnectionInput {
116
147
  config: Record<string, unknown>;
117
148
  secretEnvVar?: string;
118
149
  }
119
- /** Same rule as {@link ConnectionInput}: authored fields only, never state. */
120
- export interface ConnectorInput {
121
- id?: string;
122
- name: string;
123
- description?: string;
124
- kind: CatalogConnector['kind'];
125
- targetType: string;
126
- config: Record<string, unknown>;
127
- connectionId?: string;
128
- secretEnvVar?: string;
129
- transformId?: string;
130
- schedule?: string;
131
- mode?: 'full' | 'incremental';
132
- enabled: boolean;
133
- }
134
150
  export interface TransformInput {
135
151
  id?: string;
136
152
  name: string;
@@ -138,6 +154,58 @@ export interface TransformInput {
138
154
  language: TransformLanguage;
139
155
  code: string;
140
156
  }
157
+ /**
158
+ * One pipeline that would break if a connection were deleted.
159
+ *
160
+ * A workflow rather than a connector, because that is the object somebody
161
+ * recognises. `through` is the server's sentence about WHICH source reaches it
162
+ * — `"Warehouse"`, or `"Warehouse" via connector abc` — so a refusal can point
163
+ * at a box on a canvas rather than at a row in a table nobody can see.
164
+ */
165
+ export interface ConnectionUse {
166
+ id: string;
167
+ name: string;
168
+ status: CatalogWorkflow['status'];
169
+ through: string;
170
+ }
171
+ /**
172
+ * What a manual run may say about itself.
173
+ *
174
+ * Both fields are absent on the run the Run button sends, and both exist for
175
+ * the case where somebody is re-driving a load they already know something
176
+ * about. See {@link CatalogClient.runWorkflow} for why `expectShrink` had to
177
+ * survive the removal of the connector run route.
178
+ */
179
+ export interface WorkflowRunOptions {
180
+ /** Re-drive under an identity that already exists, rather than minting one. */
181
+ snapshotId?: string;
182
+ /**
183
+ * Why this load is expected to lose rows, in a sentence.
184
+ *
185
+ * Sent only when somebody wrote one. The distinction between absent and
186
+ * present-but-empty is real and is the server's: absent means nobody said
187
+ * anything and the bound decides; empty means somebody claimed an
188
+ * acknowledgement and gave no reason, which is refused.
189
+ */
190
+ expectShrink?: string;
191
+ }
192
+ export interface WorkflowScheduleInput {
193
+ /** Cron-ish, interpreted by whatever schedules it. Empty means manual only. */
194
+ schedule?: string;
195
+ enabled?: boolean;
196
+ }
197
+ /**
198
+ * The stored graph, plus what the server thinks of the schedule on it.
199
+ *
200
+ * `warning` is `null` when the schedule will fire. It is a string when it will
201
+ * not — and that is the entire reason this call answers with more than the
202
+ * workflow: a cron stored on a draft, on a disabled graph, or in a syntax
203
+ * nothing can parse is a pipeline that looks scheduled and never runs, which is
204
+ * a failure with an incident behind it rather than a hypothetical.
205
+ */
206
+ export interface ScheduledWorkflow extends CatalogWorkflow {
207
+ warning?: string | null;
208
+ }
141
209
  /**
142
210
  * One saved version of the two things in this catalog whose text a person edits.
143
211
  *
@@ -397,6 +465,16 @@ export interface CatalogClient {
397
465
  * not the other.
398
466
  */
399
467
  pipelineCapabilities(): Promise<PipelineCapabilities>;
468
+ /**
469
+ * Every workflow the live fleet announces it can execute — what a call node's
470
+ * picker offers.
471
+ *
472
+ * A separate call from {@link pipelineCapabilities} rather than a field on it,
473
+ * because the two have opposite lifetimes. Capabilities cannot change without
474
+ * a redeploy, so the canvas caches them forever. This is a snapshot of live
475
+ * workers with a resolution of about one heartbeat, so it must not be.
476
+ */
477
+ listCallableWorkflows(): Promise<CallableWorkflowList>;
400
478
  listConnections(): Promise<CatalogConnection[]>;
401
479
  saveConnection(input: ConnectionInput): Promise<CatalogConnection>;
402
480
  /**
@@ -407,27 +485,64 @@ export interface CatalogClient {
407
485
  * system it names.
408
486
  */
409
487
  checkConnection(id: string): Promise<ConnectionCheck>;
410
- /** Which connectors read through it. Named, so a refusal can say which. */
411
- connectionConnectors(id: string): Promise<CatalogConnector[]>;
488
+ /**
489
+ * Reach an address that has not been stored, and record nothing.
490
+ *
491
+ * The reason this exists rather than a form telling somebody to save first:
492
+ * the field most likely to be wrong is the one that cannot be verified by
493
+ * reading it, and the way you would otherwise find out is a load failing on a
494
+ * schedule hours later. A connection saved to discover a typo is a row
495
+ * somebody then has to remember to delete.
496
+ *
497
+ * Asks for `catalog:write` where {@link CatalogClient.checkConnection} asks
498
+ * only for `catalog:read`, and a host wiring its own transport should know
499
+ * why: this reaches an address supplied in the request, so under the softer
500
+ * scope it is the server connecting wherever a reader of the catalog points
501
+ * it. It grants no reach a `catalog:write` holder did not already have — they
502
+ * could save, check and delete — but it leaves no rows behind, so the server
503
+ * writes what it did to its log instead.
504
+ *
505
+ * `id` on the input is what makes an EDIT testable: the server puts the stored
506
+ * credential back where the caller was only ever shown a redaction of it. A
507
+ * create sends no id, and nothing is restored — the address reached is exactly
508
+ * the one that was typed.
509
+ */
510
+ checkUnsavedConnection(input: ConnectionInput): Promise<ConnectionCheck>;
511
+ /**
512
+ * Which pipelines read through it. Named, so a refusal can say which.
513
+ *
514
+ * Workflows and not connectors, because that is the question somebody is
515
+ * actually asking before they delete a connection: what breaks. A connector
516
+ * is an implementation detail of a published graph and naming one in a
517
+ * refusal would send the reader looking for a screen that no longer exists.
518
+ *
519
+ * The server's answer, and it outranks anything a screen derives from the
520
+ * workflow list: a source node may reach a connection through a connector it
521
+ * names in `config.connectorId` rather than through its own `connectionId`,
522
+ * and only the server folds both in.
523
+ */
524
+ connectionWorkflows(id: string): Promise<ConnectionUse[]>;
412
525
  deleteConnection(id: string): Promise<{
413
526
  deleted: boolean;
414
527
  }>;
415
- listConnectors(): Promise<CatalogConnector[]>;
416
- saveConnector(input: ConnectorInput): Promise<CatalogConnector>;
417
- deleteConnector(id: string): Promise<{
418
- deleted: boolean;
419
- }>;
420
- runConnector(id: string): Promise<ConnectorRun>;
421
528
  /**
422
- * What the source behind a connector looks like right now. Writes nothing.
423
- *
424
- * Typed loosely on purpose. The shape comes from
425
- * `@dudousxd/nestjs-catalog-pipeline`, and this package must not import that
426
- * one — it would drag a package built for a Node process, with database
427
- * drivers behind optional imports, into a browser bundle. `PipelineCapabilities`
428
- * above is mirrored for exactly the same reason.
529
+ * The connectors, read-only.
530
+ *
531
+ * **There is deliberately no `saveConnector`, `deleteConnector` or
532
+ * `runConnector` on this client, because there are no such routes.** A
533
+ * connector is what a published workflow runs as: `publishWorkflow` mints
534
+ * one, `deleteWorkflow` takes it away, and `runWorkflow` is the only way to
535
+ * start one. A method here that posted to `pipeline/connectors` would 404,
536
+ * and one that quietly did nothing would be worse.
537
+ *
538
+ * This read stays because it answers what a graph cannot answer about itself:
539
+ * which id the run history and the incremental watermark are keyed on, and —
540
+ * on a deployment upgraded rather than built fresh — which rows belong to no
541
+ * graph at all. Nothing turns those into workflows; they keep loading on the
542
+ * path they were already on, and this read is the only place they are visible.
543
+ * `CatalogConnector.workflowId` is what joins one to the graph that runs it.
429
544
  */
430
- discoverConnectorSchema(id: string): Promise<unknown>;
545
+ listConnectors(): Promise<CatalogConnector[]>;
431
546
  /**
432
547
  * Publish an object type's schema — create it, or update the shape of one
433
548
  * that exists.
@@ -505,6 +620,29 @@ export interface CatalogClient {
505
620
  */
506
621
  listWorkflows(): Promise<CatalogWorkflow[]>;
507
622
  saveWorkflow(input: WorkflowInput): Promise<CatalogWorkflow>;
623
+ /**
624
+ * Declare it finished, and mint the connector it runs as.
625
+ *
626
+ * Its own call rather than a field on the save, because publishing is a claim
627
+ * the server checks and a failed check owes an explanation naming the nodes —
628
+ * which an autosave has nowhere to put. A draft saves without validating;
629
+ * this is where the rules are answered for real.
630
+ */
631
+ publishWorkflow(id: string): Promise<CatalogWorkflow>;
632
+ /**
633
+ * Back to draft. Disables the connector, keeping its id, its run history and
634
+ * its watermark, so re-publishing resumes the same pipeline rather than
635
+ * starting a second one beside it.
636
+ */
637
+ unpublishWorkflow(id: string): Promise<CatalogWorkflow>;
638
+ /**
639
+ * Delete it, and the connector with it.
640
+ *
641
+ * The run history goes too, which is why the console asks first. It used to
642
+ * be refused while a connector still ran the graph; that check would now
643
+ * refuse every delete there is, since a published graph runs as exactly one
644
+ * connector — its own.
645
+ */
508
646
  deleteWorkflow(id: string): Promise<{
509
647
  deleted: boolean;
510
648
  }>;
@@ -512,8 +650,54 @@ export interface CatalogClient {
512
650
  * Execute it. When a durable engine is available each node runs as its own
513
651
  * step, so a failure part-way resumes rather than restarts — see
514
652
  * `PipelineCapabilities.durable`, which is what the screen states.
653
+ *
654
+ * `expectShrink` is the escape hatch for a load the row-count bound refused,
655
+ * and it reaches the server **only through here** — `POST connectors/:id/run`
656
+ * carried it and that route is gone. Without it an operator's only recourse
657
+ * would be raising `rowCount.maxShrink` in the type's policy, which stands the
658
+ * guard down for every future load of that type instead of for one snapshot.
659
+ *
660
+ * It is a reason, not a flag: the string is written into the snapshot's
661
+ * `_expectShrink` label and is the only answer anybody will have in six
662
+ * months to "why was this load allowed to lose most of the data?". Sending it
663
+ * empty is refused with a 400 asking for one, which is why the console's
664
+ * dialog will not submit a blank box rather than sending it and hoping.
665
+ */
666
+ runWorkflow(id: string, options?: WorkflowRunOptions): Promise<WorkflowRun>;
667
+ /**
668
+ * Set when this graph runs, and whether it runs at all.
669
+ *
670
+ * Both fields are optional and an absent one means "leave it alone", so a
671
+ * screen rendering only the cron cannot silently re-enable a pipeline
672
+ * somebody turned off. The answer is the stored workflow plus `warning`,
673
+ * which is the server naming a schedule that will never fire — a draft, a
674
+ * disabled graph, an unparseable cron. A screen that dropped it would repeat
675
+ * the incident this route was written after: a scheduler announcing it was
676
+ * watching schedules while parsing nothing.
677
+ */
678
+ scheduleWorkflow(id: string, input: WorkflowScheduleInput): Promise<ScheduledWorkflow>;
679
+ /**
680
+ * What the system behind one source node looks like right now. Writes nothing.
681
+ *
682
+ * Takes a node and not a connector, and the swap is the whole of the move:
683
+ * this was `POST connectors/:id/discover`, which refused outright for any
684
+ * connector carrying a `workflowId` — and every connector carries one now, so
685
+ * the old shape would have refused every connector there is.
686
+ *
687
+ * **It answers on a draft, deliberately.** A sink cannot commit into a type
688
+ * that does not exist, so requiring a published graph would require publishing
689
+ * a graph whose target type cannot be created until it is published. What it
690
+ * does need is a graph that has been SAVED: the server reads the stored node,
691
+ * not the one on screen.
692
+ *
693
+ * Typed loosely on purpose. The shape comes from
694
+ * `@dudousxd/nestjs-catalog-pipeline`, and this package must not import that
695
+ * one — it would drag a package built for a Node process, with database
696
+ * drivers behind optional imports, into a browser bundle. `PipelineCapabilities`
697
+ * above is mirrored for exactly the same reason; `narrowDiscovery` in
698
+ * `schema-discovery.tsx` is what checks the answer at the seam.
515
699
  */
516
- runWorkflow(id: string): Promise<WorkflowRun>;
700
+ discoverSourceSchema(workflowId: string, nodeId: string): Promise<unknown>;
517
701
  /**
518
702
  * Who may do what.
519
703
  *
@@ -579,14 +763,28 @@ export declare const catalogQueryKeys: {
579
763
  search: (term: string, limit?: number) => readonly ["nestjs-catalog", "search", string, number | null];
580
764
  /**
581
765
  * The pipeline keys are nested under one prefix so a host — or a screen that
582
- * just ran a connector — can invalidate the whole of it at once. Running a
583
- * connector changes the runs, the connector's own last-run fields, and the
584
- * catalog snapshot the load wrote into, and invalidating them one by one is
585
- * how a list ends up showing a run that finished next to a connector that
586
- * still says it never has.
766
+ * just ran a pipeline — can invalidate the whole of it at once. Running one
767
+ * changes the runs, the connector's own last-run fields, and the catalog
768
+ * snapshot the load wrote into, and invalidating them one by one is how a
769
+ * screen ends up showing a run that finished next to a pipeline that still
770
+ * says it never has.
771
+ *
772
+ * `connectors` is still here and is still worth invalidating, even though
773
+ * nothing authors one any more: publishing a graph MINTS one, unpublishing
774
+ * disables it, and deleting takes it away — so the read that says which id a
775
+ * run history is keyed on goes stale on all three.
587
776
  */
588
777
  pipeline: readonly ["nestjs-catalog", "pipeline"];
589
778
  capabilities: readonly ["nestjs-catalog", "pipeline", "capabilities"];
779
+ /**
780
+ * What the live fleet announces it can execute.
781
+ *
782
+ * Its own key rather than a slice of `capabilities`, because the two are
783
+ * cached on opposite terms — `capabilities` is stale-forever and this is stale
784
+ * in about a heartbeat — and one key cannot hold two staleTimes. A host
785
+ * invalidating the whole `pipeline` prefix still reaches both.
786
+ */
787
+ callableWorkflows: readonly ["nestjs-catalog", "pipeline", "callable-workflows"];
590
788
  connections: readonly ["nestjs-catalog", "pipeline", "connections"];
591
789
  connectors: readonly ["nestjs-catalog", "pipeline", "connectors"];
592
790
  transforms: readonly ["nestjs-catalog", "pipeline", "transforms"];
package/dist/context.js CHANGED
@@ -52,16 +52,14 @@ export function CatalogProvider({ transport, pipelineBasePath = DEFAULT_PIPELINE
52
52
  listEvents: (query) => transport.get(catalogRoutes.events(), { ...query }),
53
53
  listTraces: (query) => transport.get(catalogRoutes.traces(), { ...query }),
54
54
  pipelineCapabilities: () => transport.get(pipeline.capabilities()),
55
+ listCallableWorkflows: () => transport.get(pipeline.callableWorkflows()),
55
56
  listConnections: () => transport.get(pipeline.connections()),
56
57
  saveConnection: (input) => transport.post(pipeline.connections(), input),
57
58
  checkConnection: (id) => transport.post(pipeline.checkConnection(id), {}),
58
- connectionConnectors: (id) => transport.get(pipeline.connectionConnectors(id)),
59
+ checkUnsavedConnection: (input) => transport.post(pipeline.checkUnsavedConnection(), input),
60
+ connectionWorkflows: (id) => transport.get(pipeline.connectionWorkflows(id)),
59
61
  deleteConnection: (id) => transport.delete(pipeline.connection(id)),
60
62
  listConnectors: () => transport.get(pipeline.connectors()),
61
- saveConnector: (input) => transport.post(pipeline.connectors(), input),
62
- deleteConnector: (id) => transport.delete(pipeline.connector(id)),
63
- runConnector: (id) => transport.post(pipeline.runConnector(id), {}),
64
- discoverConnectorSchema: (id) => transport.post(pipeline.discoverConnectorSchema(id), {}),
65
63
  publishType: (name, schema) => {
66
64
  if (!transport.put) {
67
65
  throw new Error('This transport cannot PUT, so it cannot publish a type. Implement `put` on the ' +
@@ -92,8 +90,31 @@ export function CatalogProvider({ transport, pipelineBasePath = DEFAULT_PIPELINE
92
90
  tryTransform: (input) => transport.post(pipeline.tryTransform(), input),
93
91
  listWorkflows: () => transport.get(pipeline.workflows()),
94
92
  saveWorkflow: (input) => transport.post(pipeline.workflows(), input),
93
+ publishWorkflow: (id) => transport.post(pipeline.publishWorkflow(id), {}),
94
+ unpublishWorkflow: (id) => transport.post(pipeline.unpublishWorkflow(id), {}),
95
95
  deleteWorkflow: (id) => transport.delete(pipeline.workflow(id)),
96
- runWorkflow: (id) => transport.post(pipeline.runWorkflow(id), {}),
96
+ // The two fields are spread conditionally rather than sent as
97
+ // `undefined`, and for `expectShrink` that is the whole contract: the
98
+ // server distinguishes a body with no such key — nobody said anything,
99
+ // let the bound decide — from one carrying an empty string, which is an
100
+ // acknowledgement with no reason behind it and is refused with a 400
101
+ // asking for one. A transport that serialises `undefined` as `null`, or
102
+ // one that drops it, would flatten those two into each other.
103
+ runWorkflow: (id, options) => transport.post(pipeline.runWorkflow(id), {
104
+ ...(options?.snapshotId === undefined ? {} : { snapshotId: options.snapshotId }),
105
+ ...(options && 'expectShrink' in options && options.expectShrink !== undefined
106
+ ? { expectShrink: options.expectShrink }
107
+ : {}),
108
+ }),
109
+ scheduleWorkflow: (id, input) => {
110
+ if (!transport.put) {
111
+ throw new Error('This transport cannot PUT, so it cannot set a schedule. Implement `put` on the ' +
112
+ 'transport you passed to CatalogProvider — a schedule is an idempotent upsert of ' +
113
+ 'the fields it names, which is why the route is a PUT and not a POST.');
114
+ }
115
+ return transport.put(pipeline.workflowSchedule(id), input);
116
+ },
117
+ discoverSourceSchema: (workflowId, nodeId) => transport.post(pipeline.discoverSourceSchema(workflowId, nodeId), {}),
97
118
  listPrincipals: () => transport.get(access.principals()),
98
119
  listPeople: (query) => transport.get(access.people(query)),
99
120
  upsertPerson: (input) => transport.post(access.people(), input),
@@ -135,14 +156,28 @@ export const catalogQueryKeys = {
135
156
  search: (term, limit) => ['nestjs-catalog', 'search', term, limit ?? null],
136
157
  /**
137
158
  * The pipeline keys are nested under one prefix so a host — or a screen that
138
- * just ran a connector — can invalidate the whole of it at once. Running a
139
- * connector changes the runs, the connector's own last-run fields, and the
140
- * catalog snapshot the load wrote into, and invalidating them one by one is
141
- * how a list ends up showing a run that finished next to a connector that
142
- * still says it never has.
159
+ * just ran a pipeline — can invalidate the whole of it at once. Running one
160
+ * changes the runs, the connector's own last-run fields, and the catalog
161
+ * snapshot the load wrote into, and invalidating them one by one is how a
162
+ * screen ends up showing a run that finished next to a pipeline that still
163
+ * says it never has.
164
+ *
165
+ * `connectors` is still here and is still worth invalidating, even though
166
+ * nothing authors one any more: publishing a graph MINTS one, unpublishing
167
+ * disables it, and deleting takes it away — so the read that says which id a
168
+ * run history is keyed on goes stale on all three.
143
169
  */
144
170
  pipeline: ['nestjs-catalog', 'pipeline'],
145
171
  capabilities: ['nestjs-catalog', 'pipeline', 'capabilities'],
172
+ /**
173
+ * What the live fleet announces it can execute.
174
+ *
175
+ * Its own key rather than a slice of `capabilities`, because the two are
176
+ * cached on opposite terms — `capabilities` is stale-forever and this is stale
177
+ * in about a heartbeat — and one key cannot hold two staleTimes. A host
178
+ * invalidating the whole `pipeline` prefix still reaches both.
179
+ */
180
+ callableWorkflows: ['nestjs-catalog', 'pipeline', 'callable-workflows'],
146
181
  connections: ['nestjs-catalog', 'pipeline', 'connections'],
147
182
  connectors: ['nestjs-catalog', 'pipeline', 'connectors'],
148
183
  transforms: ['nestjs-catalog', 'pipeline', 'transforms'],
package/dist/index.d.ts CHANGED
@@ -2,6 +2,7 @@ export { CatalogManager, type CatalogManagerProps } from './CatalogManager';
2
2
  export { CoverageLedger } from './CoverageLedger';
3
3
  export { EditableField } from './EditableField';
4
4
  export { DELETE_STRATEGIES, LoadExpectationSection, type LoadExpectationSectionProps, } from './LoadExpectationSection';
5
+ export { LoadedBySection, type LoadedBySectionProps, loadersOf, } from './LoadedBySection';
5
6
  export { cn } from './cn';
6
7
  export { ObjectExplorer, type ObjectExplorerProps } from './ObjectExplorer';
7
8
  export { DashboardBoard } from './DashboardBoard';
@@ -16,18 +17,24 @@ export { CatalogSearch, type CatalogSearchProps } from './search-console';
16
17
  export { SavedQueryPanel } from './SavedQueryPanel';
17
18
  export { CATALOG_EVENT_META, type CatalogTraceSource, eventMeta, TraceExplorer, useTraceSource, } from './TraceExplorer';
18
19
  export { ConnectionPanel, type ConnectionPanelProps, connectionOptionsFor, describeConnection, } from './ConnectionPanel';
19
- export { type ColumnChoice, type ConnectorSchemaDiscovery, type DiscoveredColumn, type DiscoveredTypeDraft, initialChoices, PipelineConsole, type PipelineConsoleProps, proposalFrom, type SchemaDiscoveryBridge, type SchemaDrift, } from './PipelineConsole';
20
+ export { CONNECTABLE_KINDS, CONNECTION_KIND_OPTIONS, CONNECTION_KINDS, type ConnectableKind, type ConnectableKindSpec, ConnectionCheckResult, type ConnectionDraft, ConnectionKindFields, type ConnectionKindSpec, type UnconnectableKindSpec, type UpdateDraft, connectableSpec, connectionConfigFor, connectionDraftFrom, connectionIsIncomplete, redactedCredentialIn, toConnectableKind, } from './connection-form';
21
+ export { SourceConnectionCreator } from './source-connection';
22
+ export { PipelineConsole, type PipelineConsoleProps, } from './PipelineConsole';
23
+ export { type ColumnChoice, type ConnectorSchemaDiscovery, type DiscoveredColumn, type DiscoveredTypeDraft, initialChoices, narrowDiscovery, proposalFrom, type SchemaDiscoveryBridge, SchemaDiscoveryPanel, type SchemaDrift, } from './schema-discovery';
20
24
  export { TransformEditor, type TransformEditorProps } from './TransformEditor';
21
25
  export { DiffBody, RevisionHistory, type RevisionHistoryProps, RevisionHistoryButton, RevisionHistorySheet, type RevisionHistorySheetProps, type RevisionSubject, type RevisionSubjectKind, } from './diff/RevisionDiff';
22
26
  export { AccessConsole, type AccessConsoleProps } from './AccessConsole';
23
- export { type CatalogWorkflow, describeDurability, type DurabilityCopy, isWorkflowNodeKind, newLocalId, nodeName, producedTypes, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, type WorkflowDurability, type WorkflowEdge, type WorkflowGraph, type WorkflowInput, type WorkflowNode, type WorkflowNodeKind, type WorkflowRun, type WorkflowRunNode, type WorkflowSinkNode, type WorkflowSourceNode, type WorkflowTransformNode, } from './workflow/model';
27
+ export { type CallableWorkflowRef, type CatalogWorkflow, describeDurability, type DurabilityCopy, isWorkflowNodeKind, newLocalId, nodeName, producedTypes, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, type WorkflowCallNode, type WorkflowDurability, type WorkflowEdge, type WorkflowGraph, type WorkflowInput, type WorkflowNode, type WorkflowNodeKind, type WorkflowRun, type WorkflowRunNode, type WorkflowSinkNode, type WorkflowSourceNode, type WorkflowTransformNode, } from './workflow/model';
24
28
  export { WORKFLOW_NAME } from './workflow/name';
29
+ export { checkShapes, type ShapeKnowledge, type SourceColumn, type SourceShape, type TargetProperty, type TargetShape, } from './workflow/shape';
25
30
  export { canConnect, type ConnectionVerdict, edgeId, hasBlockingProblem, problemsByNode, type ValidateOptions, validateWorkflow, type WorkflowDraft, type WorkflowProblem, type WorkflowProblemCode, type WorkflowProblemLevel, wouldCycle, } from './workflow/validate';
26
- export { type CatalogClient, type CatalogIdentity, type CatalogRevision, type CatalogPeoplePage, type CatalogPersonRole, type CatalogPersonSummary, type CatalogPrincipalSummary, CatalogProvider, type CatalogProviderProps, type CatalogTransport, type ConnectionInput, type ConnectorInput, catalogQueryKeys, type DeleteReconciliation, type LoadExpectation, type LoadExpectationInput, type ResolvedLoadExpectation, type RowCountBound, type StoredLoadExpectation, type PersonInput, type PersonUpsertResult, type PipelineCapabilities, type TransformInput, useCatalogClient, } from './context';
31
+ export { attachTransformIds, buildWorkflowTemplate, type EnrichOptions, enrichWithLookup, fanOutTypes, type FanOutOptions, type FanOutTarget, FILE_DROP_KINDS, type FileDropOptions, joinSources, type JoinOptions, loadFileDrop, type PeriodicFullReloadOptions, periodicFullReload, planIsRunnable, planToWorkflowInput, refuseUnusableColumnNames, RELOAD_CADENCE_IDS, RELOAD_CADENCES, type ReloadCadence, replicateTable, type ReplicateTableOptions, SOURCE_KINDS, type TemplateDeclaration, type TemplateExpectation, type TemplateOutcome, type TemplatePlan, type TemplateRefusal, type TemplateRefusalCode, type TemplateRequest, type TemplateSchedule, type TemplateTransformRequest, withinMsFor, WORKFLOW_TEMPLATE_IDS, WORKFLOW_TEMPLATES, type WorkflowTemplateId, type WorkflowTemplateMeta, } from './workflow/templates';
32
+ export { type CatalogClient, type CatalogIdentity, type CatalogRevision, type CatalogPeoplePage, type CatalogPersonRole, type CatalogPersonSummary, type CatalogPrincipalSummary, CatalogProvider, type CatalogProviderProps, type CatalogTransport, type ConnectionInput, type ConnectionUse, catalogQueryKeys, type DeleteReconciliation, type LoadExpectation, type LoadExpectationInput, type ResolvedLoadExpectation, type RowCountBound, type StoredLoadExpectation, type PersonInput, type PersonUpsertResult, type PipelineCapabilities, type CallableWorkflowList, type ScheduledWorkflow, type TransformInput, type WorkflowRunOptions, type WorkflowScheduleInput, useCatalogClient, } from './context';
27
33
  export { type AccessRoutes, accessRoutes, DEFAULT_ACCESS_BASE_PATH, DEFAULT_PIPELINE_BASE_PATH, DEFAULT_PUBLISH_BASE_PATH, embedRoutes, type PeopleQuery, type PipelineRoutes, pipelineRoutes, } from './routes';
28
34
  export { ConfirmDialog } from './ui/dialog';
29
35
  export { FieldGroup, TextAreaField, TextField } from './ui/field';
30
36
  export { Select, SelectField, type SelectOption } from './ui/select';
37
+ export { Combobox, ComboboxField, type ComboOption, type ComboboxProps } from './ui/combobox';
31
38
  export { Switch } from './ui/switch';
32
39
  export { Sheet } from './ui/sheet';
33
40
  export { CodeEditor, type CodeEditorHandle, type CodeEditorProps, codeEditorRoot, codeEditorText, } from './ui/code-editor';
package/dist/index.js CHANGED
@@ -5,6 +5,12 @@ export { EditableField } from './EditableField';
5
5
  // own type panel out of this package's parts should not have to rebuild the one
6
6
  // section that can say an incremental load of a type is currently refused.
7
7
  export { DELETE_STRATEGIES, LoadExpectationSection, } from './LoadExpectationSection';
8
+ // Mounted on the Model screen too, directly above the expectation section — the
9
+ // two are one question in two halves: who loads this type, and what is checked
10
+ // when they do. `loadersOf` is exported beside the component because it is the
11
+ // whole rule (a graph's SINKS decide, not its stored `targetType`), and a host
12
+ // that renders its own row wants the rule rather than a second reading of it.
13
+ export { LoadedBySection, loadersOf, } from './LoadedBySection';
8
14
  export { cn } from './cn';
9
15
  export { ObjectExplorer } from './ObjectExplorer';
10
16
  export { DashboardBoard } from './DashboardBoard';
@@ -26,24 +32,50 @@ export { QueryConsole } from './QueryConsole';
26
32
  export { CatalogSearch } from './search-console';
27
33
  export { SavedQueryPanel } from './SavedQueryPanel';
28
34
  export { CATALOG_EVENT_META, eventMeta, TraceExplorer, useTraceSource, } from './TraceExplorer';
29
- // Getting data in: connections, connectors and the code that reshapes what they
30
- // fetch. These used to live in the consuming app, which meant any other host
31
- // installing this package got a console with no way to manage a load.
35
+ // Getting data in: the addresses a pipeline reads through and the code that
36
+ // reshapes what it fetches. These used to live in the consuming app, which meant
37
+ // any other host installing this package got a console with no way to manage a
38
+ // load.
39
+ //
40
+ // There is deliberately no connector screen here any more. A connector is what a
41
+ // published workflow runs as — nothing authors one — so authoring a pipeline is
42
+ // `WorkflowCanvas`, on the `/workflow` subpath, and this is the two shared
43
+ // objects it borrows.
32
44
  export { ConnectionPanel, connectionOptionsFor, describeConnection, } from './ConnectionPanel';
45
+ // What a connection IS, as data: which kinds can be one, what each needs, and
46
+ // the two rules — completeness, and the redaction placeholder — that decide
47
+ // whether a form may send one.
48
+ //
49
+ // Exported because there are two forms now. The console's is here; the source
50
+ // node's inspector on the canvas has its own, behind the `/workflow` subpath,
51
+ // and a host assembling a third should be reading these rather than restating
52
+ // what an S3 connection needs.
53
+ export { CONNECTABLE_KINDS, CONNECTION_KIND_OPTIONS, CONNECTION_KINDS, ConnectionCheckResult, ConnectionKindFields, connectableSpec, connectionConfigFor, connectionDraftFrom, connectionIsIncomplete, redactedCredentialIn, toConnectableKind, } from './connection-form';
54
+ // Creating a connection from wherever a source is being configured.
55
+ //
56
+ // On the root entry and not behind `/workflow`, for the reason
57
+ // `SchemaDiscoveryPanel` is: it imports no React Flow, and it is the other half
58
+ // of the same offer — the sink's panel makes the type, this makes the address. A
59
+ // host assembling its own source inspector should be able to mount either.
60
+ export { SourceConnectionCreator } from './source-connection';
61
+ export { PipelineConsole, } from './PipelineConsole';
33
62
  // The schema-discovery seam, whole.
34
63
  //
35
- // `PipelineConsoleProps['schemaDiscovery']` is a bridge the HOST implements —
36
- // this package cannot ask a connector for its columns, only render the answer —
37
- // and every one of these was reachable from that prop's type and from nowhere
38
- // else. So the only way to write the bridge with its signature spelled out was
39
- // to name it through an indexed access on a component's props, or to deep-import
40
- // a file inside `dist/`. Both are the caller paying for a list that fell behind.
64
+ // `SchemaDiscoveryBridge` is a bridge a HOST may implement — this package cannot
65
+ // ask a source for its columns, only render the answer — and every one of these
66
+ // was once reachable only through an indexed access on a component's props, or
67
+ // by deep-importing a file inside `dist/`. Both are the caller paying for a list
68
+ // that fell behind.
41
69
  //
42
70
  // `initialChoices` and `proposalFrom` come with them because they are the rules
43
71
  // that decide whether a schema somebody ticked is one the publish route will
44
72
  // accept — pure and deliberately outside the panel, per their own docblocks, and
45
73
  // worth nothing there if the only way to run them is to render it.
46
- export { initialChoices, PipelineConsole, proposalFrom, } from './PipelineConsole';
74
+ //
75
+ // `SchemaDiscoveryPanel` is exported for the same reason it stopped living
76
+ // inside a screen: the canvas mounts it from behind the `/workflow` subpath, and
77
+ // a host assembling its own source inspector should be able to do the same.
78
+ export { initialChoices, narrowDiscovery, proposalFrom, SchemaDiscoveryPanel, } from './schema-discovery';
47
79
  export { TransformEditor } from './TransformEditor';
48
80
  // Why one load came out different from the last one.
49
81
  //
@@ -84,11 +116,27 @@ export { AccessConsole } from './AccessConsole';
84
116
  // and it belongs to the package that executes them. See `workflow/model.ts`.
85
117
  export { describeDurability, isWorkflowNodeKind, newLocalId, nodeName, producedTypes, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, } from './workflow/model';
86
118
  export { WORKFLOW_NAME } from './workflow/name';
87
- // Core's rules, run in the browser, plus the two questions core cannot answer:
88
- // whether a connection is legal mid-drag, and whether a node names a transform
89
- // that still exists. Exported so a host can run the same checks before it offers
90
- // a save — never so it can skip the server's, which is the authority.
119
+ // Whether what a source reads fits the type its sink writes.
120
+ //
121
+ // The canvas answers this for itself now — it holds what `discoverSourceSchema`
122
+ // returned for each source node and compares it against the snapshot it already
123
+ // reads, so there is no prop and nothing for a host to wire. These are exported
124
+ // because the comparison is pure and a host may well want it somewhere else: a
125
+ // pre-flight before a scheduled run, say. `ConnectorSchemaDiscovery` above is
126
+ // already a `SourceShape`, so that caller has one for free.
127
+ export { checkShapes, } from './workflow/shape';
128
+ // Core's rules, run in the browser, plus the questions core cannot answer:
129
+ // whether a connection is legal mid-drag, whether a node names a transform that
130
+ // still exists, and whether a source supplies the columns its sink writes.
131
+ // Exported so a host can run the same checks before it offers a save — never so
132
+ // it can skip the server's, which is the authority.
91
133
  export { canConnect, edgeId, hasBlockingProblem, problemsByNode, validateWorkflow, wouldCycle, } from './workflow/validate';
134
+ // The prefilled graphs, on this entry point rather than on `/workflow` because
135
+ // they import no React Flow: a template produces nodes, edges, transform bodies
136
+ // and expectation payloads, and a host should be able to build one — or check
137
+ // what it would decide — without installing a canvas to do it. Whether a screen
138
+ // then draws the result is the canvas's business, not the template's.
139
+ export { attachTransformIds, buildWorkflowTemplate, enrichWithLookup, fanOutTypes, FILE_DROP_KINDS, joinSources, loadFileDrop, periodicFullReload, planIsRunnable, planToWorkflowInput, refuseUnusableColumnNames, RELOAD_CADENCE_IDS, RELOAD_CADENCES, replicateTable, SOURCE_KINDS, withinMsFor, WORKFLOW_TEMPLATE_IDS, WORKFLOW_TEMPLATES, } from './workflow/templates';
92
140
  export { CatalogProvider, catalogQueryKeys, useCatalogClient, } from './context';
93
141
  // The paths behind the pipeline and access screens, exported so a host can
94
142
  // mount those endpoints wherever it likes and tell the provider where they are.
@@ -107,6 +155,7 @@ embedRoutes, pipelineRoutes, } from './routes';
107
155
  export { ConfirmDialog } from './ui/dialog';
108
156
  export { FieldGroup, TextAreaField, TextField } from './ui/field';
109
157
  export { Select, SelectField } from './ui/select';
158
+ export { Combobox, ComboboxField } from './ui/combobox';
110
159
  export { Switch } from './ui/switch';
111
160
  export { Sheet } from './ui/sheet';
112
161
  // `codeEditorRoot` and `codeEditorText` come with it because the editor renders