@dudousxd/nestjs-catalog 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.controller.js +66 -6
- package/dist/catalog.csv.d.ts +89 -0
- package/dist/catalog.csv.js +163 -0
- package/dist/catalog.filters.d.ts +174 -0
- package/dist/catalog.filters.js +272 -0
- package/dist/catalog.identifiers.d.ts +161 -0
- package/dist/catalog.identifiers.js +195 -0
- package/dist/catalog.pipeline.d.ts +392 -35
- package/dist/catalog.pipeline.js +175 -25
- package/dist/catalog.query-cache.d.ts +0 -2
- package/dist/catalog.query-cache.js +0 -18
- package/dist/catalog.query.d.ts +43 -0
- package/dist/catalog.query.js +5 -0
- package/dist/catalog.service.d.ts +51 -0
- package/dist/catalog.service.js +126 -3
- package/dist/catalog.store.d.ts +84 -22
- package/dist/catalog.store.js +36 -68
- package/dist/catalog.types.d.ts +64 -0
- package/dist/client.d.ts +70 -2
- package/dist/client.js +66 -1
- package/dist/index.d.ts +6 -4
- package/dist/index.js +24 -3
- package/dist/stores/mikro-orm-read.store.d.ts +8 -2
- package/dist/stores/mikro-orm-read.store.js +77 -11
- package/package.json +1 -1
|
@@ -27,6 +27,37 @@ export type ConnectorKind = (typeof CONNECTOR_KINDS)[number];
|
|
|
27
27
|
* entirely. Anything narrowing a stored value narrows against *this*.
|
|
28
28
|
*/
|
|
29
29
|
export declare function isConnectorKind(value: unknown): value is ConnectorKind;
|
|
30
|
+
/**
|
|
31
|
+
* What a published workflow runs as. **Not an authored object.**
|
|
32
|
+
*
|
|
33
|
+
* This used to be the thing somebody created: a screen, a `POST connectors`, a
|
|
34
|
+
* kind and a config and a schedule typed into a form. It is not that any more,
|
|
35
|
+
* and every route that let anybody make one directly has been removed. A
|
|
36
|
+
* connector is now minted by {@link CatalogWorkflowStore.publishWorkflow} and
|
|
37
|
+
* exists for the four jobs a graph cannot do for itself:
|
|
38
|
+
*
|
|
39
|
+
* - **It is the mutex key.** `singleton: { key: connectorMutexKey }` on the
|
|
40
|
+
* durable workflow serialises runs per connector id, which is what stops two
|
|
41
|
+
* workers loading one type at once.
|
|
42
|
+
* - **It is the run's owner.** `ConnectorRun.connectorId` is how `listRuns`
|
|
43
|
+
* groups a history, so a stable id across edits of the graph is what makes
|
|
44
|
+
* "what has this pipeline done" answerable at all.
|
|
45
|
+
* - **It holds the watermark.** `state`, keyed by source-node id — see
|
|
46
|
+
* {@link CatalogConnector.state}.
|
|
47
|
+
* - **It answers "which connectors write this type".** `targetType` is kept
|
|
48
|
+
* equal to the workflow's sink type, so that query keeps working.
|
|
49
|
+
*
|
|
50
|
+
* Everything else on it is derived. `kind`, `config`, `connectionId` and
|
|
51
|
+
* `secretEnvVar` are **not read** on a connector that has a `workflowId`, and
|
|
52
|
+
* every connector has one now; they survive only because a row adopted from
|
|
53
|
+
* before this change still carries what it was configured with, and throwing
|
|
54
|
+
* that away would destroy the evidence of what a load used to do.
|
|
55
|
+
*
|
|
56
|
+
* The reason for keeping a record at all rather than running graphs directly is
|
|
57
|
+
* that all four jobs above need an identity that outlives an edit. A graph's
|
|
58
|
+
* version changes when anybody drags a node; the thing a run belongs to must
|
|
59
|
+
* not.
|
|
60
|
+
*/
|
|
30
61
|
export interface CatalogConnector {
|
|
31
62
|
id: string;
|
|
32
63
|
name: string;
|
|
@@ -54,28 +85,39 @@ export interface CatalogConnector {
|
|
|
54
85
|
/** The transform that turns source records into rows of `targetType`. */
|
|
55
86
|
transformId?: string;
|
|
56
87
|
/**
|
|
57
|
-
* The
|
|
58
|
-
*
|
|
88
|
+
* The graph this connector runs. **The only thing that says what it does.**
|
|
89
|
+
*
|
|
90
|
+
* Optional in the type and mandatory in practice: `publishWorkflow` sets it on
|
|
91
|
+
* every connector it mints, and adoption sets it on every connector that
|
|
92
|
+
* predates this change. It stays optional here because a store read must be
|
|
93
|
+
* able to represent a row written before adoption ran without asserting
|
|
94
|
+
* something about it that is not yet true — narrowing this to `string` would
|
|
95
|
+
* turn "we have not migrated yet" into a type error at the read.
|
|
59
96
|
*
|
|
60
97
|
* Mutually exclusive with {@link transformId}, and the store refuses a
|
|
61
98
|
* connector that sets both: two answers to "what shapes this data" means the
|
|
62
99
|
* runner picks one, and which one it picked is invisible until the load comes
|
|
63
|
-
* out wrong.
|
|
64
|
-
* exactly as it did before workflows existed.
|
|
100
|
+
* out wrong.
|
|
65
101
|
*
|
|
66
|
-
* When this is set
|
|
102
|
+
* When this is set the connector's own `kind`, `config`, `connectionId` and
|
|
67
103
|
* `secretEnvVar` are **not read**: the workflow's source nodes say where the
|
|
68
|
-
* data comes from, and letting the connector also say would be two
|
|
69
|
-
* for one question. `targetType`
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
* `state` keeps its meaning too, but is keyed by node id when a workflow runs:
|
|
74
|
-
* a graph with two sources has two watermarks, and one flat blob would let
|
|
75
|
-
* them overwrite each other.
|
|
104
|
+
* data comes from, and letting the connector also say would be two
|
|
105
|
+
* authorities for one question. `targetType` is kept equal to the workflow's
|
|
106
|
+
* sink type, so every existing "which connectors write this type" answer
|
|
107
|
+
* keeps working.
|
|
76
108
|
*/
|
|
77
109
|
workflowId?: string;
|
|
78
|
-
/**
|
|
110
|
+
/**
|
|
111
|
+
* A **copy** of {@link CatalogWorkflow.schedule}, and nothing reads it.
|
|
112
|
+
*
|
|
113
|
+
* Kept, and kept honest by the store, for exactly one reason: it is the
|
|
114
|
+
* evidence of what a connector was doing before its schedule moved onto the
|
|
115
|
+
* graph. The authority is the workflow, {@link CatalogWorkflowStore} writes
|
|
116
|
+
* this from there on every publish, and the scheduler deliberately does not
|
|
117
|
+
* consult it — see the note on `ConnectorScheduler`, which used to read this
|
|
118
|
+
* field and now reads workflows, because a second copy of a column is how the
|
|
119
|
+
* two come to disagree the first time somebody edits one.
|
|
120
|
+
*/
|
|
79
121
|
schedule?: string;
|
|
80
122
|
/**
|
|
81
123
|
* Whether a run replaces the dataset or adds to it.
|
|
@@ -113,6 +155,13 @@ export interface CatalogConnector {
|
|
|
113
155
|
* is a consequence. Mixing them means a person editing a connector can
|
|
114
156
|
* silently rewind or skip data, and a diff of the config stops meaning what
|
|
115
157
|
* somebody decided.
|
|
158
|
+
*
|
|
159
|
+
* **Keyed by source-node id**, because a graph with two sources has two
|
|
160
|
+
* watermarks and one flat blob would let them overwrite each other. A
|
|
161
|
+
* connector adopted from before workflows existed had a flat blob, and
|
|
162
|
+
* adoption re-keys it under the id of the single source node it was wrapped
|
|
163
|
+
* into — losing that would make the first run after an upgrade re-read an
|
|
164
|
+
* incremental source from the beginning.
|
|
116
165
|
*/
|
|
117
166
|
state?: Record<string, unknown>;
|
|
118
167
|
enabled: boolean;
|
|
@@ -508,12 +557,13 @@ export interface WorkflowNodeOutcome {
|
|
|
508
557
|
/**
|
|
509
558
|
* What a node can be.
|
|
510
559
|
*
|
|
511
|
-
*
|
|
512
|
-
*
|
|
513
|
-
*
|
|
514
|
-
*
|
|
515
|
-
*
|
|
516
|
-
*
|
|
560
|
+
* The first three are exactly the three verbs the existing connector runner
|
|
561
|
+
* already performs in sequence: fetch, transform, publish. The fourth hands a
|
|
562
|
+
* position in the graph to a durable workflow that already exists in the
|
|
563
|
+
* deployment. Nothing here is a kind this service cannot execute, which is the
|
|
564
|
+
* same rule {@link CONNECTOR_KINDS} follows — a kind that exists in the type
|
|
565
|
+
* and throws at run time is worse than one that is absent, because the first
|
|
566
|
+
* looks supported in a palette.
|
|
517
567
|
*
|
|
518
568
|
* The kinds that were considered and rejected, since a small vocabulary is only
|
|
519
569
|
* defensible if the omissions are:
|
|
@@ -531,8 +581,17 @@ export interface WorkflowNodeOutcome {
|
|
|
531
581
|
* A `merge` kind would have had to carry a strategy field whose values the
|
|
532
582
|
* runner would have to implement one by one, and an unimplemented strategy in
|
|
533
583
|
* a dropdown is the failure this list exists to avoid.
|
|
584
|
+
* - **call a durable *step*** — the sibling of {@link WorkflowCallNode} that
|
|
585
|
+
* somebody will eventually come looking for, and it cannot be built. A
|
|
586
|
+
* durable step has no global identity: it is dispatched by a routing name
|
|
587
|
+
* that a worker subscribes to, and within a run it is addressed by its `seq`
|
|
588
|
+
* — a position in one workflow's history. There is no "run step X" entry
|
|
589
|
+
* point on the engine to call, no lifecycle of its own to await, and nothing
|
|
590
|
+
* to cancel. A workflow is the smallest thing that is addressable from
|
|
591
|
+
* outside a run, which is why `call` names one and not a step. If a step is
|
|
592
|
+
* what you want, the thing to call is a one-step workflow wrapping it.
|
|
534
593
|
*/
|
|
535
|
-
export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink"];
|
|
594
|
+
export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call"];
|
|
536
595
|
export type WorkflowNodeKind = (typeof WORKFLOW_NODE_KINDS)[number];
|
|
537
596
|
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
538
597
|
export declare function isWorkflowNodeKind(value: unknown): value is WorkflowNodeKind;
|
|
@@ -620,13 +679,77 @@ export interface WorkflowSinkNode extends WorkflowNodeBase {
|
|
|
620
679
|
*/
|
|
621
680
|
mode?: 'full' | 'incremental';
|
|
622
681
|
}
|
|
682
|
+
/**
|
|
683
|
+
* Hands this position in the graph to a durable workflow that already exists.
|
|
684
|
+
*
|
|
685
|
+
* The node the canvas cannot write the body of: the work happens in a workflow
|
|
686
|
+
* somebody else registered, possibly in another SDK, and this node is the wire
|
|
687
|
+
* from a graph to it. It runs as a **tracked child run** of the catalog's own
|
|
688
|
+
* durable run — `ctx.startChild` then `ctx.child` — so the child has its own
|
|
689
|
+
* lifetime, its own retries and its own history, and the catalog's run is
|
|
690
|
+
* suspended at zero compute while it goes.
|
|
691
|
+
*
|
|
692
|
+
* ## Why the version is stored and not resolved
|
|
693
|
+
*
|
|
694
|
+
* A call names {@link callName} **and** {@link callVersion}, and the pair is
|
|
695
|
+
* part of the graph fingerprint. Storing only the name would mean the person
|
|
696
|
+
* who owns that workflow can change what your load does by registering a new
|
|
697
|
+
* version — a behaviour change in your pipeline with nothing in your diff to
|
|
698
|
+
* point at. So the version is authored, and a run that would have used a
|
|
699
|
+
* different one is refused rather than silently run: see
|
|
700
|
+
* `WorkflowRunSteps.checkCall`, which is where the pin is actually enforced,
|
|
701
|
+
* and which documents exactly how strong the enforcement is.
|
|
702
|
+
*
|
|
703
|
+
* ## What crosses the boundary
|
|
704
|
+
*
|
|
705
|
+
* The same thing that crosses every other boundary here: **handles, never
|
|
706
|
+
* rows.** The child receives a {@link WorkflowCallEnvelope} — the run id, this
|
|
707
|
+
* node's id, and the {@link WorkflowStageRef}s of its inputs — and reads the
|
|
708
|
+
* rows out of the stage store itself if it wants them. A child that produces
|
|
709
|
+
* rows for the graph writes them into the stage store under *this node's* id
|
|
710
|
+
* and returns their shape, which {@link readWorkflowCallOutput} narrows.
|
|
711
|
+
*
|
|
712
|
+
* There is no shared type between a catalog node and an arbitrary durable
|
|
713
|
+
* workflow, and this model does not pretend there is. What it does instead is
|
|
714
|
+
* make the mismatch loud: the envelope is one documented shape, the accepted
|
|
715
|
+
* answers are two documented shapes, and anything else fails the node naming
|
|
716
|
+
* the workflow, the version and the child run id.
|
|
717
|
+
*
|
|
718
|
+
* ## `config` is not a credential store
|
|
719
|
+
*
|
|
720
|
+
* Named `config` rather than `input` so it travels the same path a source
|
|
721
|
+
* node's config does — sealed under `encryptCredentials`, refused in plaintext
|
|
722
|
+
* without it — because a parameter bag that reaches a worker over a queue is
|
|
723
|
+
* exactly the shape a password ends up in. Prefer naming an env var the callee
|
|
724
|
+
* already reads.
|
|
725
|
+
*/
|
|
726
|
+
export interface WorkflowCallNode extends WorkflowNodeBase {
|
|
727
|
+
kind: 'call';
|
|
728
|
+
/**
|
|
729
|
+
* The workflow's registered name, on the wire — the string `engine.start`
|
|
730
|
+
* takes. A class reference is unavailable by construction: the point of this
|
|
731
|
+
* node is calling something the catalog does not compile against, including a
|
|
732
|
+
* body that lives in Python.
|
|
733
|
+
*/
|
|
734
|
+
callName: string;
|
|
735
|
+
/**
|
|
736
|
+
* The registered **version**, as the durable engine spells it: a string, and
|
|
737
|
+
* `'1'` for anything registered without one.
|
|
738
|
+
*
|
|
739
|
+
* Not to be confused with {@link CatalogWorkflow.version}, which counts edits
|
|
740
|
+
* to *this* graph and is a number. This one identifies somebody else's code.
|
|
741
|
+
*/
|
|
742
|
+
callVersion: string;
|
|
743
|
+
/** Parameters the author typed, handed to the child under `input`. */
|
|
744
|
+
config: Record<string, unknown>;
|
|
745
|
+
}
|
|
623
746
|
/**
|
|
624
747
|
* A discriminated union, so narrowing a node is `node.kind === "sink"` and
|
|
625
748
|
* never a type assertion. This is why the kind list is not simply a string on
|
|
626
749
|
* one node shape with every field optional: that shape lets a source node carry
|
|
627
750
|
* a `transformId` and nothing catches it.
|
|
628
751
|
*/
|
|
629
|
-
export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode;
|
|
752
|
+
export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode;
|
|
630
753
|
/**
|
|
631
754
|
* One wire.
|
|
632
755
|
*
|
|
@@ -742,6 +865,31 @@ export interface CatalogWorkflow {
|
|
|
742
865
|
* a connector claims to write something else.
|
|
743
866
|
*/
|
|
744
867
|
targetType: string;
|
|
868
|
+
/**
|
|
869
|
+
* Cron-ish, interpreted by whatever schedules it. Empty means manual only.
|
|
870
|
+
*
|
|
871
|
+
* **Authored here, on the graph, and nowhere else.** It used to live on the
|
|
872
|
+
* connector, which was defensible while a connector was a thing somebody
|
|
873
|
+
* created and is not now that one is minted: a schedule is a statement about
|
|
874
|
+
* a pipeline, and the pipeline is the graph. The connector keeps a copy for
|
|
875
|
+
* evidence — see {@link CatalogConnector.schedule} — and `ConnectorScheduler`
|
|
876
|
+
* reads this field rather than that one, so there is one authority.
|
|
877
|
+
*
|
|
878
|
+
* Only a `ready` graph is scheduled. A draft carrying a cron is a load nobody
|
|
879
|
+
* declared finished, and the scheduler says so out loud rather than skipping
|
|
880
|
+
* it quietly; that silence is the exact shape of the incident this field's
|
|
881
|
+
* predecessor caused.
|
|
882
|
+
*/
|
|
883
|
+
schedule?: string;
|
|
884
|
+
/**
|
|
885
|
+
* Whether this graph runs at all.
|
|
886
|
+
*
|
|
887
|
+
* Both halves of the old `isScheduled` test now live on the workflow, because
|
|
888
|
+
* splitting them across two rows is what made "my connector is enabled but
|
|
889
|
+
* nothing runs" a question with two places to look. Defaults true: a graph
|
|
890
|
+
* somebody went to the trouble of publishing is one they meant to run.
|
|
891
|
+
*/
|
|
892
|
+
enabled: boolean;
|
|
745
893
|
createdBy: string;
|
|
746
894
|
createdAt: string;
|
|
747
895
|
updatedAt: string;
|
|
@@ -814,6 +962,15 @@ export interface WorkflowNodeStepInput {
|
|
|
814
962
|
* graph. Empty for a source node, which reads from a system instead.
|
|
815
963
|
*/
|
|
816
964
|
inputs: WorkflowStageRef[];
|
|
965
|
+
/**
|
|
966
|
+
* The operator's reason for expecting this load to lose rows. Read only by
|
|
967
|
+
* the sink, and absent on every scheduled run by construction.
|
|
968
|
+
*
|
|
969
|
+
* On the step input rather than fetched at the sink because a durable step
|
|
970
|
+
* has to be a pure function of what was checkpointed — see the note where
|
|
971
|
+
* this is passed in `CatalogWorkflowRunWorkflow`.
|
|
972
|
+
*/
|
|
973
|
+
expectShrink?: string;
|
|
817
974
|
}
|
|
818
975
|
/** What a node step returns. Also ids and counters. */
|
|
819
976
|
export interface WorkflowNodeStepOutput {
|
|
@@ -842,8 +999,112 @@ export interface WorkflowNodeStepOutput {
|
|
|
842
999
|
*/
|
|
843
1000
|
logs: string[];
|
|
844
1001
|
}
|
|
1002
|
+
/**
|
|
1003
|
+
* The number in {@link WorkflowCallEnvelope.contract}.
|
|
1004
|
+
*
|
|
1005
|
+
* A version on the *shape the catalog sends*, separate from the version of the
|
|
1006
|
+
* workflow being called, because the two change for different reasons and a
|
|
1007
|
+
* callee written against one has to be able to say which. A callee that reads
|
|
1008
|
+
* this and does not recognise it should refuse rather than guess — a failed
|
|
1009
|
+
* child is a failed node with a name attached, and a guess is a load nobody can
|
|
1010
|
+
* account for.
|
|
1011
|
+
*/
|
|
1012
|
+
export declare const WORKFLOW_CALL_CONTRACT = 1;
|
|
1013
|
+
/**
|
|
1014
|
+
* What a {@link WorkflowCallNode} hands its child.
|
|
1015
|
+
*
|
|
1016
|
+
* One documented shape, and the whole of what the catalog promises a callee.
|
|
1017
|
+
* Everything the catalog knows sits under `catalog`, and everything the author
|
|
1018
|
+
* typed sits under `input`, so a parameter called `runId` cannot ever shadow
|
|
1019
|
+
* the run id — which is the kind of collision that only shows up in production
|
|
1020
|
+
* on somebody else's graph.
|
|
1021
|
+
*
|
|
1022
|
+
* Handles, never rows: `inputs` names the stages this node's inbound edges
|
|
1023
|
+
* produced, in edge order, and the rows themselves stay in the stage store
|
|
1024
|
+
* addressed by `(runId, nodeId, batch)` — one row per batch, keyed
|
|
1025
|
+
* `runId#nodeId#batch`. A callee in another SDK reads them from there. Passing
|
|
1026
|
+
* the rows would write the whole intermediate dataset into
|
|
1027
|
+
* `durable_step_checkpoints` as the child's input, once per call and again on
|
|
1028
|
+
* every replay, which is the measurement {@link WorkflowStageRef} exists for.
|
|
1029
|
+
*/
|
|
1030
|
+
export interface WorkflowCallEnvelope {
|
|
1031
|
+
catalog: {
|
|
1032
|
+
/** {@link WORKFLOW_CALL_CONTRACT} at the time the call was made. */
|
|
1033
|
+
contract: number;
|
|
1034
|
+
/** The catalog run, which is also the snapshot id and the stage key. */
|
|
1035
|
+
runId: string;
|
|
1036
|
+
/** The calling node. Also the id any rows for the graph must be staged under. */
|
|
1037
|
+
nodeId: string;
|
|
1038
|
+
workflowId: string;
|
|
1039
|
+
/** The **graph's** version, not the callee's. See {@link WorkflowCallNode.callVersion}. */
|
|
1040
|
+
workflowVersion: number;
|
|
1041
|
+
principalId: string;
|
|
1042
|
+
/** The stages this node reads, in inbound-edge order. Empty for a call with no input. */
|
|
1043
|
+
inputs: WorkflowStageRef[];
|
|
1044
|
+
};
|
|
1045
|
+
/** {@link WorkflowCallNode.config}, verbatim. Opaque to the catalog. */
|
|
1046
|
+
input: Record<string, unknown>;
|
|
1047
|
+
}
|
|
1048
|
+
/** What a called workflow may say it staged. Both counts, or neither. */
|
|
1049
|
+
export interface WorkflowCallOutput {
|
|
1050
|
+
/** Batches written under `(runId, nodeId, 1..batches)`. */
|
|
1051
|
+
batches: number;
|
|
1052
|
+
rowCount: number;
|
|
1053
|
+
}
|
|
1054
|
+
/**
|
|
1055
|
+
* Read a child's return value as staged rows — or as nothing, or refuse it.
|
|
1056
|
+
*
|
|
1057
|
+
* Three answers rather than two, because a call has two legitimate purposes and
|
|
1058
|
+
* they must not be confused with a bug:
|
|
1059
|
+
*
|
|
1060
|
+
* - `undefined` — the child returned nothing this graph can read rows from. A
|
|
1061
|
+
* perfectly ordinary outcome for a workflow called for its effect, and the
|
|
1062
|
+
* node reports zero rows, out loud, in its logs. It is not silently treated
|
|
1063
|
+
* as success-with-data: a full sink that then receives nothing refuses to
|
|
1064
|
+
* commit an empty snapshot, which is the loud end of this path.
|
|
1065
|
+
* - a {@link WorkflowCallOutput} — the child staged rows for this node.
|
|
1066
|
+
* - a **throw** — the child answered with `batches`/`rowCount` that are not
|
|
1067
|
+
* usable counts. Half a contract is a bug in the callee, and reading it as
|
|
1068
|
+
* "no rows" would turn that bug into a load that quietly came out short.
|
|
1069
|
+
*
|
|
1070
|
+
* The catalog cannot check any of this before the graph runs; there is no
|
|
1071
|
+
* schema for a workflow's output anywhere in the durable contract, and no way
|
|
1072
|
+
* to reach one if there were. So the check is here, at the one moment the
|
|
1073
|
+
* answer exists, and it names what it saw.
|
|
1074
|
+
*/
|
|
1075
|
+
export declare function readWorkflowCallOutput(value: unknown): WorkflowCallOutput | undefined;
|
|
1076
|
+
/**
|
|
1077
|
+
* One workflow a call node could name.
|
|
1078
|
+
*
|
|
1079
|
+
* Declared here with nothing producing it yet, and that is deliberate rather
|
|
1080
|
+
* than premature: a picker needs a list, and **no such list exists**. The
|
|
1081
|
+
* durable engine can answer `workflowBody(name, version)` for the process
|
|
1082
|
+
* asking and nothing more — and a missing body is ambiguous, since it equally
|
|
1083
|
+
* means a body registered in another SDK through `registerRemote` or a group
|
|
1084
|
+
* resolved by convention against a live worker. Inferring the list from what
|
|
1085
|
+
* one pod happens to know would produce a picker that omits exactly the
|
|
1086
|
+
* cross-SDK workflows this node exists to call.
|
|
1087
|
+
*
|
|
1088
|
+
* So the node takes a name and a version as authored text today, and this is
|
|
1089
|
+
* the shape a canvas should be handed the day a deployment can announce its
|
|
1090
|
+
* registrations. One entry per **version**, never per name: a picker that
|
|
1091
|
+
* listed names and resolved the version for you would undo the pin.
|
|
1092
|
+
*/
|
|
1093
|
+
export interface CallableWorkflowRef {
|
|
1094
|
+
name: string;
|
|
1095
|
+
version: string;
|
|
1096
|
+
/** What it does, if the deployment publishes one. Shown beside the name. */
|
|
1097
|
+
description?: string;
|
|
1098
|
+
/**
|
|
1099
|
+
* The worker group its turns are dispatched to, when it has one. The signal
|
|
1100
|
+
* that says "this one's body is not in this process" — a Python workflow, or
|
|
1101
|
+
* a separate TS worker — which is precisely what a caller cannot otherwise
|
|
1102
|
+
* tell from a missing body.
|
|
1103
|
+
*/
|
|
1104
|
+
group?: string;
|
|
1105
|
+
}
|
|
845
1106
|
/** Every way a graph can be refused. Exported so a canvas can key off the code. */
|
|
846
|
-
export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named"];
|
|
1107
|
+
export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "call-not-named"];
|
|
847
1108
|
export type WorkflowIssueCode = (typeof WORKFLOW_ISSUE_CODES)[number];
|
|
848
1109
|
export interface WorkflowValidationIssue {
|
|
849
1110
|
code: WorkflowIssueCode;
|
|
@@ -956,8 +1217,14 @@ export interface CatalogWorkflowStore {
|
|
|
956
1217
|
* point at a ready graph, so a save that dropped the status would disable a
|
|
957
1218
|
* scheduled load with nothing said to anybody. Refusing puts the error in
|
|
958
1219
|
* front of the person who is editing, at the moment they edit. To park a
|
|
959
|
-
* broken idea on a live graph, {@link unpublishWorkflow} it first
|
|
960
|
-
*
|
|
1220
|
+
* broken idea on a live graph, {@link unpublishWorkflow} it first — which
|
|
1221
|
+
* disables the connector it runs as, so the parking is visible rather than a
|
|
1222
|
+
* graph that is quietly still on a cron.
|
|
1223
|
+
*
|
|
1224
|
+
* `schedule` and `enabled` are not inputs either, and for a different reason
|
|
1225
|
+
* from the derived three: they are authored, but not *here*. See
|
|
1226
|
+
* {@link saveWorkflowSchedule} for why a cron must not ride along on an
|
|
1227
|
+
* autosave.
|
|
961
1228
|
*
|
|
962
1229
|
* `version`, `graphHash` and `targetType` are not inputs: the first two are
|
|
963
1230
|
* derived from the graph and the third from the sink, and accepting them from
|
|
@@ -981,24 +1248,114 @@ export interface CatalogWorkflowStore {
|
|
|
981
1248
|
*
|
|
982
1249
|
* Idempotent on an already-ready graph, because the honest answer to "publish
|
|
983
1250
|
* this thing that is published" is the graph, not an error.
|
|
1251
|
+
*
|
|
1252
|
+
* **Publishing is also what mints the connector.** That is the load-bearing
|
|
1253
|
+
* half of "the workflow is the only thing anybody authors": there is no
|
|
1254
|
+
* `POST connectors` any more, so this is the only way one comes into
|
|
1255
|
+
* existence. The minted row carries this workflow's id, its sink's type, its
|
|
1256
|
+
* schedule and its enabled flag, and it is the identity every later run,
|
|
1257
|
+
* watermark and mutex is keyed on — so publishing an already-published graph
|
|
1258
|
+
* must **update** that row rather than mint a second one, or a re-publish
|
|
1259
|
+
* would orphan a pipeline's entire history.
|
|
984
1260
|
*/
|
|
985
1261
|
publishWorkflow(id: string, publishedBy: string): Promise<CatalogWorkflow>;
|
|
986
1262
|
/**
|
|
987
|
-
* Take a graph back to `draft
|
|
1263
|
+
* Take a graph back to `draft`, and stop what it was running.
|
|
988
1264
|
*
|
|
989
|
-
*
|
|
990
|
-
*
|
|
991
|
-
*
|
|
992
|
-
* a
|
|
993
|
-
*
|
|
994
|
-
*
|
|
995
|
-
*
|
|
1265
|
+
* This used to refuse while any connector still ran the graph, on the reasoning
|
|
1266
|
+
* that unpublishing one out from under a schedule breaks a working load and the
|
|
1267
|
+
* operator should point those connectors elsewhere first. That reasoning
|
|
1268
|
+
* assumed a connector was an independently authored object that could be
|
|
1269
|
+
* pointed somewhere; it no longer is. A published graph now has exactly one
|
|
1270
|
+
* connector and it is this graph's own, so the old refusal would refuse
|
|
1271
|
+
* every unpublish there is — a rule that always fires is not a rule.
|
|
1272
|
+
*
|
|
1273
|
+
* So the cascade the old docblock argued against is now the correct behaviour,
|
|
1274
|
+
* and the thing it was protecting is protected differently: the connector is
|
|
1275
|
+
* **disabled, not deleted**. Its id, its run history and its watermark all
|
|
1276
|
+
* survive, so re-publishing resumes the same pipeline rather than starting a
|
|
1277
|
+
* new one, and nothing silently keeps running in the meantime.
|
|
996
1278
|
*/
|
|
997
1279
|
unpublishWorkflow(id: string, unpublishedBy: string): Promise<CatalogWorkflow>;
|
|
998
|
-
/**
|
|
1280
|
+
/**
|
|
1281
|
+
* Delete the graph and the connector it ran as.
|
|
1282
|
+
*
|
|
1283
|
+
* Cascading for the reason {@link unpublishWorkflow} gives — the connector is
|
|
1284
|
+
* this graph's own and there is nowhere else to point it — and destructive
|
|
1285
|
+
* here rather than disabling, because the graph is going too and a connector
|
|
1286
|
+
* whose workflow does not exist is precisely the dangling row this cascade
|
|
1287
|
+
* exists to avoid leaving behind.
|
|
1288
|
+
*/
|
|
999
1289
|
deleteWorkflow(id: string): Promise<boolean>;
|
|
1000
|
-
/**
|
|
1290
|
+
/**
|
|
1291
|
+
* The connector this graph runs as, as a list.
|
|
1292
|
+
*
|
|
1293
|
+
* A list rather than an optional single value, because that is what it
|
|
1294
|
+
* honestly is: a store predating this change may hold several connectors
|
|
1295
|
+
* pointing at one graph, and answering with the first would hide the rest from
|
|
1296
|
+
* the operator who has to reconcile them. A graph published under the current
|
|
1297
|
+
* rules has exactly one.
|
|
1298
|
+
*/
|
|
1001
1299
|
connectorsUsingWorkflow(id: string): Promise<CatalogConnector[]>;
|
|
1300
|
+
/**
|
|
1301
|
+
* Set when this graph runs, and whether it runs.
|
|
1302
|
+
*
|
|
1303
|
+
* Its own method rather than fields on {@link saveWorkflow}, and for the
|
|
1304
|
+
* reason `saveConnectorState` is separate from `saveConnector`: a canvas
|
|
1305
|
+
* autosaves. A cron folded into the save every drag of a node passes through
|
|
1306
|
+
* is a cron that a stale editor tab can silently revert, and the symptom would
|
|
1307
|
+
* be a load that stopped running with a diff nobody made.
|
|
1308
|
+
*
|
|
1309
|
+
* Writes through to the minted connector's copy in the same call, so the two
|
|
1310
|
+
* cannot be observed disagreeing.
|
|
1311
|
+
*/
|
|
1312
|
+
saveWorkflowSchedule(id: string, input: {
|
|
1313
|
+
schedule?: string;
|
|
1314
|
+
enabled?: boolean;
|
|
1315
|
+
}, changedBy: string): Promise<CatalogWorkflow>;
|
|
1316
|
+
/**
|
|
1317
|
+
* Wrap a connector that predates workflows into the graph it always was.
|
|
1318
|
+
*
|
|
1319
|
+
* **The upgrade path, and the reason removing `POST connectors` does not
|
|
1320
|
+
* orphan anything.** A deployment upgrading into this change has connectors
|
|
1321
|
+
* that were authored directly — a kind, a config, optionally one transform,
|
|
1322
|
+
* and a target type — and no route can edit them any more. Three outcomes
|
|
1323
|
+
* were possible: migrate them, leave them readable but frozen, or make
|
|
1324
|
+
* somebody rebuild each by hand. This is the first, and it is chosen because
|
|
1325
|
+
* the shape was already exactly representable: a connector *is* a
|
|
1326
|
+
* single-source, single-sink graph with at most one transform in the middle,
|
|
1327
|
+
* and it has been describable as one since `workflowId` existed.
|
|
1328
|
+
*
|
|
1329
|
+
* ## What is preserved, and why each matters
|
|
1330
|
+
*
|
|
1331
|
+
* - **The connector id.** It is the mutex key, the owner of every
|
|
1332
|
+
* `ConnectorRun` and the holder of the watermark. Minting a fresh connector
|
|
1333
|
+
* and leaving the old one would split one pipeline's history in two and
|
|
1334
|
+
* leave nothing serialising the halves against each other.
|
|
1335
|
+
* - **The watermark, re-keyed.** A plain connector kept a flat `state`; a
|
|
1336
|
+
* graph keys it by source-node id. Carrying the blob over unchanged would
|
|
1337
|
+
* leave the new source node with no watermark at all, and the first run
|
|
1338
|
+
* after the upgrade would re-read an incremental source from the beginning —
|
|
1339
|
+
* which is not data loss, but is a surprise measured in hours on a large
|
|
1340
|
+
* source.
|
|
1341
|
+
* - **The schedule**, moved to the graph, which is where it is authored now.
|
|
1342
|
+
*
|
|
1343
|
+
* ## What changes, stated plainly
|
|
1344
|
+
*
|
|
1345
|
+
* The graph is published as `ready` without a person having declared it
|
|
1346
|
+
* finished, because refusing to publish would leave the pipeline not running —
|
|
1347
|
+
* an upgrade that silently stops twelve loads is a worse outcome than one that
|
|
1348
|
+
* carries a decision forward on the operator's behalf. It is validated first
|
|
1349
|
+
* and the adoption is **refused** if the wrap does not validate, so nothing is
|
|
1350
|
+
* published that could not have been drawn.
|
|
1351
|
+
*
|
|
1352
|
+
* Idempotent: a connector that already has a `workflowId` is left alone and
|
|
1353
|
+
* answered `undefined`. That is what makes it safe to run at every boot.
|
|
1354
|
+
*/
|
|
1355
|
+
adoptConnector(connectorId: string, adoptedBy: string): Promise<{
|
|
1356
|
+
workflow: CatalogWorkflow;
|
|
1357
|
+
connector: CatalogConnector;
|
|
1358
|
+
} | undefined>;
|
|
1002
1359
|
}
|
|
1003
1360
|
/**
|
|
1004
1361
|
* Where the rows between two nodes actually sit.
|