@dudousxd/nestjs-catalog 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.events.d.ts +26 -1
- package/dist/catalog.events.js +8 -0
- package/dist/catalog.pipeline.d.ts +713 -6
- package/dist/catalog.pipeline.js +487 -11
- package/dist/client.d.ts +2 -2
- package/dist/client.js +34 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +18 -3
- package/package.json +1 -1
package/dist/catalog.pipeline.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* systems each believing they decide when a load runs.
|
|
10
10
|
*/
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.REDACTED_SECRET = exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_SKIP_REASONS = exports.CODE_CONTEXT_CONTRACT = exports.TRANSFORM_RUNNER = exports.TRANSFORM_LANGUAGES = exports.CONNECTOR_KINDS = void 0;
|
|
12
|
+
exports.REDACTED_SECRET = exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_BRANCH_LABELS = exports.NODE_KIND_IS_REUSABLE = exports.REUSABLE_NODE_KINDS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_SKIP_REASONS = exports.CODE_CONTEXT_CONTRACT = exports.TRANSFORM_RUNNER = exports.TRANSFORM_LANGUAGES = exports.CONNECTOR_KINDS = void 0;
|
|
13
13
|
exports.isConnectorKind = isConnectorKind;
|
|
14
14
|
exports.isTransformLanguage = isTransformLanguage;
|
|
15
15
|
exports.isWorkflowSkipReason = isWorkflowSkipReason;
|
|
@@ -27,6 +27,14 @@ exports.unreachableFilterOperator = unreachableFilterOperator;
|
|
|
27
27
|
exports.isWorkflowFilterValue = isWorkflowFilterValue;
|
|
28
28
|
exports.isWorkflowFilterPredicate = isWorkflowFilterPredicate;
|
|
29
29
|
exports.workflowFilterMatches = workflowFilterMatches;
|
|
30
|
+
exports.isReusableNodeKind = isReusableNodeKind;
|
|
31
|
+
exports.nodeKindIsReusable = nodeKindIsReusable;
|
|
32
|
+
exports.unreachableReusableNodeKind = unreachableReusableNodeKind;
|
|
33
|
+
exports.applyReusableNode = applyReusableNode;
|
|
34
|
+
exports.isReusableNodeBody = isReusableNodeBody;
|
|
35
|
+
exports.reusableNodeBodyOf = reusableNodeBodyOf;
|
|
36
|
+
exports.describeVersionPin = describeVersionPin;
|
|
37
|
+
exports.describeLiveVersion = describeLiveVersion;
|
|
30
38
|
exports.isWorkflowBranchLabel = isWorkflowBranchLabel;
|
|
31
39
|
exports.isWorkflowStatus = isWorkflowStatus;
|
|
32
40
|
exports.isWorkflowExecutionMode = isWorkflowExecutionMode;
|
|
@@ -40,10 +48,20 @@ exports.workflowGraphHash = workflowGraphHash;
|
|
|
40
48
|
exports.isWorkflowNode = isWorkflowNode;
|
|
41
49
|
exports.isWorkflowEdge = isWorkflowEdge;
|
|
42
50
|
exports.supportsWorkflows = supportsWorkflows;
|
|
51
|
+
exports.supportsWorkflowReleases = supportsWorkflowReleases;
|
|
52
|
+
exports.liveWorkflowVersion = liveWorkflowVersion;
|
|
43
53
|
exports.supportsTransformRevisions = supportsTransformRevisions;
|
|
54
|
+
exports.supportsTransformPins = supportsTransformPins;
|
|
55
|
+
exports.supportsReusableNodes = supportsReusableNodes;
|
|
44
56
|
exports.supportsWorkflowStages = supportsWorkflowStages;
|
|
45
57
|
exports.supportsLoadExpectations = supportsLoadExpectations;
|
|
46
58
|
exports.isPipelineStore = isPipelineStore;
|
|
59
|
+
// The revision shape is declared beside the audit trail rather than here,
|
|
60
|
+
// because it is one shape over two subjects — a transform's code and a saved
|
|
61
|
+
// query's SQL — and neither of them owns it. This is a type-only import, and the
|
|
62
|
+
// edge only ever points this way: `catalog.workspace.ts` knows nothing about
|
|
63
|
+
// pipelines.
|
|
64
|
+
const catalog_workspace_1 = require("./catalog.workspace");
|
|
47
65
|
/**
|
|
48
66
|
* Where a connector pulls from.
|
|
49
67
|
*
|
|
@@ -767,6 +785,239 @@ function orderedHolds(operator, held, wanted) {
|
|
|
767
785
|
return held < wanted;
|
|
768
786
|
return held <= wanted;
|
|
769
787
|
}
|
|
788
|
+
/* --- reusable nodes ------------------------------------------------------ */
|
|
789
|
+
/**
|
|
790
|
+
* The node kinds that can be saved once and used in several graphs.
|
|
791
|
+
*
|
|
792
|
+
* Source and sink, and the reason is that those two are the only kinds whose
|
|
793
|
+
* *composition* is worth a name. A connection is already a shared object, and it
|
|
794
|
+
* answers "which database" — but nobody reaches for "the warehouse" when they
|
|
795
|
+
* draw a graph, they reach for "the nightly MVR pull from the warehouse", which
|
|
796
|
+
* is the connection **plus** the query, plus whether it reads everything or only
|
|
797
|
+
* what changed, plus what the thing is called. That composition had nowhere to
|
|
798
|
+
* live, so it was retyped per graph and the fourteenth copy was the one with the
|
|
799
|
+
* typo in the `WHERE` clause.
|
|
800
|
+
*
|
|
801
|
+
* A transform is deliberately **not** here, and that is not an omission: a
|
|
802
|
+
* transform is already a stored object referenced by id
|
|
803
|
+
* ({@link WorkflowTransformNode.transformId}), so a reusable transform node
|
|
804
|
+
* would be a second way to say the same thing. What it was missing is a version
|
|
805
|
+
* pin, which is {@link WorkflowTransformNode.transformVersion}, not this.
|
|
806
|
+
*
|
|
807
|
+
* `call`, `if` and `filter` are not here either, and the record below is where
|
|
808
|
+
* each of them says so — see {@link NODE_KIND_IS_REUSABLE}.
|
|
809
|
+
*/
|
|
810
|
+
exports.REUSABLE_NODE_KINDS = ['source', 'sink'];
|
|
811
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
812
|
+
function isReusableNodeKind(value) {
|
|
813
|
+
return exports.REUSABLE_NODE_KINDS.some((kind) => kind === value);
|
|
814
|
+
}
|
|
815
|
+
/**
|
|
816
|
+
* Whether each node kind may be saved as a reusable node.
|
|
817
|
+
*
|
|
818
|
+
* A record over every kind rather than a shorter list of the two that can,
|
|
819
|
+
* because this codebase keeps being bitten by hand-maintained lists going quiet
|
|
820
|
+
* — most recently the add-node row that shipped the `filter` node with no way to
|
|
821
|
+
* create it. A kind added to {@link WORKFLOW_NODE_KINDS} without an entry here
|
|
822
|
+
* is a type error in this file naming the decision it has not made.
|
|
823
|
+
*
|
|
824
|
+
* The second `satisfies` is the other half of the same guard, in the other
|
|
825
|
+
* direction: anything named in {@link REUSABLE_NODE_KINDS} has to be `true`
|
|
826
|
+
* here, so the list and this table cannot come apart. Adding `'filter'` to that
|
|
827
|
+
* list without a `ReusableFilterBody` therefore fails to compile twice — once
|
|
828
|
+
* here, and once at every narrowing over {@link ReusableNodeBody}.
|
|
829
|
+
*
|
|
830
|
+
* Why each `false`:
|
|
831
|
+
*
|
|
832
|
+
* - `transform` — already a reference to a stored object. See
|
|
833
|
+
* {@link REUSABLE_NODE_KINDS}.
|
|
834
|
+
* - `call` — already a reference to somebody else's registered workflow, pinned
|
|
835
|
+
* by name and version. There is nothing left to name.
|
|
836
|
+
* - `if` and `filter` — a predicate is *about* the rows in front of it. A gate
|
|
837
|
+
* saved under a name and dropped into another graph tests a column that graph
|
|
838
|
+
* may not have, and a filter is worse: {@link WorkflowFilterNode.narrows} is
|
|
839
|
+
* an acknowledgement about *this* graph's sinks, so a shared one would carry
|
|
840
|
+
* somebody else's acknowledgement into a graph they never saw.
|
|
841
|
+
*/
|
|
842
|
+
exports.NODE_KIND_IS_REUSABLE = {
|
|
843
|
+
source: true,
|
|
844
|
+
transform: false,
|
|
845
|
+
sink: true,
|
|
846
|
+
call: false,
|
|
847
|
+
if: false,
|
|
848
|
+
filter: false,
|
|
849
|
+
};
|
|
850
|
+
/** Whether this kind can be saved as a reusable node. Reads {@link NODE_KIND_IS_REUSABLE}. */
|
|
851
|
+
function nodeKindIsReusable(kind) {
|
|
852
|
+
return exports.NODE_KIND_IS_REUSABLE[kind];
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* {@link unreachableNodeKind}, for reusable bodies, and for the identical
|
|
856
|
+
* reason: every branch over {@link ReusableNodeBody} ends here, so a body added
|
|
857
|
+
* to the union without a rule for folding it onto a node is a type error naming
|
|
858
|
+
* the file rather than a graph that saves and then runs a node nobody
|
|
859
|
+
* configured. It throws as well, because these arrive as JSON out of a column.
|
|
860
|
+
*/
|
|
861
|
+
function unreachableReusableNodeKind(body, where) {
|
|
862
|
+
const kind = typeof body === 'string' ? body : Reflect.get(Object(body), 'kind');
|
|
863
|
+
throw new Error(`${where} does not handle a reusable node body of kind ${JSON.stringify(kind)}. The reusable kinds and every decision made per kind are meant to move together.`);
|
|
864
|
+
}
|
|
865
|
+
/**
|
|
866
|
+
* Fold a reusable body onto the node that references it.
|
|
867
|
+
*
|
|
868
|
+
* The one implementation, called by the store when a graph is saved and by the
|
|
869
|
+
* runner when one is executed, so the node a canvas draws and the node that runs
|
|
870
|
+
* cannot describe different reads. Pure, and it takes the body rather than
|
|
871
|
+
* fetching one, for the reason `validateWorkflow` is pure: this file is imported
|
|
872
|
+
* by the browser entry point.
|
|
873
|
+
*
|
|
874
|
+
* ## What it refuses
|
|
875
|
+
*
|
|
876
|
+
* A sink body whose `targetType` differs from the one already on the node. That
|
|
877
|
+
* is not a tidiness check — it is the same shape as `WorkflowRunSteps.checkCall`
|
|
878
|
+
* and it is load-bearing for the same reason. A graph's sinks are checked
|
|
879
|
+
* against the author's write grants (`assertMayWriteTypes`) using the type on
|
|
880
|
+
* the node, at save time. If a reusable body could repoint that afterwards, then
|
|
881
|
+
* editing a shared sink would write into a type that nobody with access to this
|
|
882
|
+
* graph was ever granted — and it would do it on a schedule, with the graph's
|
|
883
|
+
* own diff showing nothing. So the disagreement fails, naming both types, and
|
|
884
|
+
* the repair is that the referencing graph is re-saved and re-checked.
|
|
885
|
+
*
|
|
886
|
+
* A mismatched *kind* is refused for the plainer reason that there is nothing
|
|
887
|
+
* sensible to do with it: a sink body on a source node is a reference somebody
|
|
888
|
+
* repointed at the wrong object, and folding half of it in would produce a node
|
|
889
|
+
* that is neither.
|
|
890
|
+
*/
|
|
891
|
+
function applyReusableNode(node, body) {
|
|
892
|
+
if (body.kind === 'source') {
|
|
893
|
+
if (node.kind !== 'source') {
|
|
894
|
+
throw new Error(reusableKindMismatch(node, body.kind));
|
|
895
|
+
}
|
|
896
|
+
return {
|
|
897
|
+
...node,
|
|
898
|
+
sourceKind: body.sourceKind,
|
|
899
|
+
connectionId: body.connectionId,
|
|
900
|
+
config: body.config,
|
|
901
|
+
secretEnvVar: body.secretEnvVar,
|
|
902
|
+
mode: body.mode,
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
if (body.kind === 'sink') {
|
|
906
|
+
if (node.kind !== 'sink') {
|
|
907
|
+
throw new Error(reusableKindMismatch(node, body.kind));
|
|
908
|
+
}
|
|
909
|
+
if (node.targetType.length > 0 && node.targetType !== body.targetType) {
|
|
910
|
+
throw new Error(`Sink "${node.name}" (${node.id}) commits ${node.targetType}, and the reusable node it uses now commits ${body.targetType}. A graph is checked against the types its sinks write at the moment it is saved, so a shared sink is not allowed to repoint one afterwards — that would write into a type nobody here was granted. Re-save this graph to adopt ${body.targetType}, which checks the grants again, or pin this node to the version that still commits ${node.targetType}.`);
|
|
911
|
+
}
|
|
912
|
+
return { ...node, targetType: body.targetType, mode: body.mode };
|
|
913
|
+
}
|
|
914
|
+
return unreachableReusableNodeKind(body, 'applyReusableNode');
|
|
915
|
+
}
|
|
916
|
+
function reusableKindMismatch(node, bodyKind) {
|
|
917
|
+
return `Node "${node.name}" (${node.id}) is a ${node.kind} node and the reusable node it names is a ${bodyKind}. A reference that changed kind under a graph would leave a node that is neither, so this is refused rather than half-applied.`;
|
|
918
|
+
}
|
|
919
|
+
/** Whether a stored value is a reusable body this build can execute. */
|
|
920
|
+
function isReusableNodeBody(value) {
|
|
921
|
+
if (typeof value !== 'object' || value === null)
|
|
922
|
+
return false;
|
|
923
|
+
const kind = Reflect.get(value, 'kind');
|
|
924
|
+
if (kind === 'source') {
|
|
925
|
+
const config = Reflect.get(value, 'config');
|
|
926
|
+
return (isConnectorKind(Reflect.get(value, 'sourceKind')) &&
|
|
927
|
+
typeof config === 'object' &&
|
|
928
|
+
config !== null &&
|
|
929
|
+
!Array.isArray(config));
|
|
930
|
+
}
|
|
931
|
+
if (kind === 'sink') {
|
|
932
|
+
const targetType = Reflect.get(value, 'targetType');
|
|
933
|
+
return typeof targetType === 'string' && targetType.length > 0;
|
|
934
|
+
}
|
|
935
|
+
// Refused rather than defaulted, exactly as `isWorkflowNode` refuses a kind it
|
|
936
|
+
// does not know: a body read back as something this build has no rule for
|
|
937
|
+
// would be folded onto a node as nothing at all, and the node would then run
|
|
938
|
+
// whatever was cached on it while claiming to be an instance of the library's.
|
|
939
|
+
return false;
|
|
940
|
+
}
|
|
941
|
+
/**
|
|
942
|
+
* The body a node is currently carrying, ready to be saved under a name.
|
|
943
|
+
*
|
|
944
|
+
* The other direction of {@link applyReusableNode}, and the reason
|
|
945
|
+
* save-as-reusable cannot silently deep-copy: this is what gets stored, the node
|
|
946
|
+
* keeps a `useId` pointing at it, and nothing anywhere duplicates a graph.
|
|
947
|
+
*
|
|
948
|
+
* Answers `undefined` for a kind that cannot be reusable rather than throwing,
|
|
949
|
+
* because the caller is a route answering a person who pressed a button on a
|
|
950
|
+
* node — {@link nodeKindIsReusable} is what a screen asks before offering it,
|
|
951
|
+
* and the route repeats the question rather than trusting the screen asked.
|
|
952
|
+
*/
|
|
953
|
+
function reusableNodeBodyOf(node) {
|
|
954
|
+
if (node.kind === 'source') {
|
|
955
|
+
return {
|
|
956
|
+
kind: 'source',
|
|
957
|
+
sourceKind: node.sourceKind,
|
|
958
|
+
connectionId: node.connectionId,
|
|
959
|
+
config: node.config,
|
|
960
|
+
secretEnvVar: node.secretEnvVar,
|
|
961
|
+
mode: node.mode,
|
|
962
|
+
};
|
|
963
|
+
}
|
|
964
|
+
if (node.kind === 'sink') {
|
|
965
|
+
return { kind: 'sink', targetType: node.targetType, mode: node.mode };
|
|
966
|
+
}
|
|
967
|
+
return undefined;
|
|
968
|
+
}
|
|
969
|
+
function describeVersionPin(version, subject) {
|
|
970
|
+
if (version === undefined) {
|
|
971
|
+
return {
|
|
972
|
+
pinned: false,
|
|
973
|
+
label: 'follows the latest',
|
|
974
|
+
detail: `This node runs whatever ${subject} says today. An edit to it reaches this graph on the next run, with no new version of this graph and nothing in its diff — which is what you want when the point is that everybody moves together, and is worth pinning against when it is not.`,
|
|
975
|
+
};
|
|
976
|
+
}
|
|
977
|
+
return {
|
|
978
|
+
pinned: true,
|
|
979
|
+
label: `pinned to v${version}`,
|
|
980
|
+
detail: `This node runs v${version} of ${subject} and stays there while it is edited elsewhere. Moving to a newer version is an edit to this graph, so it has a diff and a version of its own. A pin to a version that has been superseded more than ${catalog_workspace_1.CATALOG_REVISION_LIMIT} times can no longer be produced, and the run fails saying so rather than quietly using the latest.`,
|
|
981
|
+
};
|
|
982
|
+
}
|
|
983
|
+
/**
|
|
984
|
+
* The same question about a whole graph: does it follow its latest save, or does
|
|
985
|
+
* it run a version somebody chose?
|
|
986
|
+
*
|
|
987
|
+
* Shares {@link VersionPinCopy} and its two labels with
|
|
988
|
+
* {@link describeVersionPin} deliberately. A console that said "pinned to v6"
|
|
989
|
+
* about a transform node and invented different words for a graph would be two
|
|
990
|
+
* vocabularies for one idea, and the reader would have to work out whether they
|
|
991
|
+
* meant the same thing — which is the confusion the whole notion of a pin exists
|
|
992
|
+
* to remove.
|
|
993
|
+
*
|
|
994
|
+
* The *detail* differs, and only where the facts do. Two of them:
|
|
995
|
+
*
|
|
996
|
+
* - A node's pin can outlive the revision it names, because `catalog_revision`
|
|
997
|
+
* is capped. A graph's cannot: releases are never evicted, precisely because
|
|
998
|
+
* the one a live pointer names is the graph production is running. So this
|
|
999
|
+
* copy makes no eviction caveat, and must not acquire one.
|
|
1000
|
+
* - Following the latest is *cheap* for a node — everybody moves together, which
|
|
1001
|
+
* is often the point. For a graph it means editing is deploying, which is the
|
|
1002
|
+
* hazard this field was added to remove. So the unpinned sentence here is a
|
|
1003
|
+
* warning where the node's is a trade-off.
|
|
1004
|
+
*/
|
|
1005
|
+
function describeLiveVersion(workflow) {
|
|
1006
|
+
if (workflow.liveVersion === undefined) {
|
|
1007
|
+
return {
|
|
1008
|
+
pinned: false,
|
|
1009
|
+
label: 'follows the latest',
|
|
1010
|
+
detail: `This graph runs whatever its latest save holds, currently v${workflow.version}. Editing it is therefore the same act as deploying it: the next scheduled window runs what was last saved, with nobody having decided that it should. Releasing a version and setting it live is what separates the two.`,
|
|
1011
|
+
};
|
|
1012
|
+
}
|
|
1013
|
+
return {
|
|
1014
|
+
pinned: true,
|
|
1015
|
+
label: `running v${workflow.liveVersion}`,
|
|
1016
|
+
detail: workflow.liveVersion === workflow.version
|
|
1017
|
+
? `This graph runs the released v${workflow.liveVersion}, which is also its latest save. Editing it from here changes nothing about what runs until a new version is released and set live.`
|
|
1018
|
+
: `This graph runs the released v${workflow.liveVersion} while its latest save is v${workflow.version}. The edits since then are stored and are not running; setting a newer release live is what deploys them, and setting an older one live is a rollback.`,
|
|
1019
|
+
};
|
|
1020
|
+
}
|
|
770
1021
|
/**
|
|
771
1022
|
* Which side of an {@link WorkflowIfNode} a wire leaves by.
|
|
772
1023
|
*
|
|
@@ -980,6 +1231,15 @@ exports.WORKFLOW_ISSUE_CODES = [
|
|
|
980
1231
|
'filter-predicate-invalid',
|
|
981
1232
|
'filter-narrows-unacknowledged',
|
|
982
1233
|
'filter-narrows-nothing',
|
|
1234
|
+
/**
|
|
1235
|
+
* A version pin that is not a version — `0`, `2.5`, `"3"`, `-1`.
|
|
1236
|
+
*
|
|
1237
|
+
* One code for both pins, because they are one mistake: a threshold that
|
|
1238
|
+
* cannot name a version can only ever fail to resolve, and it fails inside a
|
|
1239
|
+
* durable step halfway through a load rather than on the canvas. The same
|
|
1240
|
+
* argument `if-threshold-invalid` makes one field along.
|
|
1241
|
+
*/
|
|
1242
|
+
'version-pin-invalid',
|
|
983
1243
|
];
|
|
984
1244
|
/**
|
|
985
1245
|
* Everything that makes a graph unrunnable, in one pure function.
|
|
@@ -1165,8 +1425,74 @@ function checkNodeWiring(nodes, incoming, outgoing, issues) {
|
|
|
1165
1425
|
const unconfigured = nodeIsUnconfigured(node);
|
|
1166
1426
|
if (unconfigured)
|
|
1167
1427
|
issues.push(unconfigured);
|
|
1428
|
+
checkVersionPins(node, issues);
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
/**
|
|
1432
|
+
* Every version pin on this node names a version that could exist.
|
|
1433
|
+
*
|
|
1434
|
+
* Both pins in one place, because they are one rule and splitting it is how one
|
|
1435
|
+
* of the two ends up accepting `0`. Checked here rather than only at the HTTP
|
|
1436
|
+
* boundary because `validateWorkflow` is what the canvas runs, so this is the
|
|
1437
|
+
* difference between a refusal on the screen where the number was typed and a
|
|
1438
|
+
* 400 after pressing Save.
|
|
1439
|
+
*
|
|
1440
|
+
* A pin has to be a whole number of at least one, matching what a version
|
|
1441
|
+
* actually is: {@link CatalogTransform.version} and
|
|
1442
|
+
* {@link CatalogReusableNode.version} both start at 1 and count up in ones.
|
|
1443
|
+
* `"3"` — which is what an unparsed form field sends — is refused rather than
|
|
1444
|
+
* coerced, for the reason `if-threshold-invalid` refuses it: a pin that no
|
|
1445
|
+
* stored version can equal resolves to nothing, and it does so inside a durable
|
|
1446
|
+
* step in the middle of a load.
|
|
1447
|
+
*
|
|
1448
|
+
* A `useVersion` with no `useId` is caught here too, as an invalid pin rather
|
|
1449
|
+
* than as its own code: it pins a reference that does not exist, so the number
|
|
1450
|
+
* can never be looked up.
|
|
1451
|
+
*/
|
|
1452
|
+
function checkVersionPins(node, issues) {
|
|
1453
|
+
if (node.kind === 'transform') {
|
|
1454
|
+
const invalid = badPin(node.transformVersion);
|
|
1455
|
+
if (invalid) {
|
|
1456
|
+
issues.push({
|
|
1457
|
+
code: 'version-pin-invalid',
|
|
1458
|
+
nodeIds: [node.id],
|
|
1459
|
+
message: `Transform node "${node.name}" (${node.id}) is pinned to version ${invalid} of its code, and a version is a whole number of at least 1. Leave it unset for the node to follow the latest.`,
|
|
1460
|
+
});
|
|
1461
|
+
}
|
|
1462
|
+
return;
|
|
1463
|
+
}
|
|
1464
|
+
if (!nodeKindIsReusable(node.kind))
|
|
1465
|
+
return;
|
|
1466
|
+
// Narrowed off the union rather than read off `node` with a property check,
|
|
1467
|
+
// so a kind that becomes reusable without gaining the fields is a type error
|
|
1468
|
+
// here and not a check that silently passes.
|
|
1469
|
+
if (node.kind !== 'source' && node.kind !== 'sink')
|
|
1470
|
+
return;
|
|
1471
|
+
const invalid = badPin(node.useVersion);
|
|
1472
|
+
if (invalid) {
|
|
1473
|
+
issues.push({
|
|
1474
|
+
code: 'version-pin-invalid',
|
|
1475
|
+
nodeIds: [node.id],
|
|
1476
|
+
message: `Node "${node.name}" (${node.id}) is pinned to version ${invalid} of the reusable node it uses, and a version is a whole number of at least 1. Leave it unset for the node to follow the latest.`,
|
|
1477
|
+
});
|
|
1478
|
+
return;
|
|
1479
|
+
}
|
|
1480
|
+
if (node.useVersion !== undefined && node.useId === undefined) {
|
|
1481
|
+
issues.push({
|
|
1482
|
+
code: 'version-pin-invalid',
|
|
1483
|
+
nodeIds: [node.id],
|
|
1484
|
+
message: `Node "${node.name}" (${node.id}) is pinned to version ${node.useVersion} but names no reusable node, so there is nothing for that version to be a version of.`,
|
|
1485
|
+
});
|
|
1168
1486
|
}
|
|
1169
1487
|
}
|
|
1488
|
+
/** The offending value, rendered, or `undefined` when the pin is fine or absent. */
|
|
1489
|
+
function badPin(version) {
|
|
1490
|
+
if (version === undefined)
|
|
1491
|
+
return undefined;
|
|
1492
|
+
if (typeof version === 'number' && Number.isInteger(version) && version >= 1)
|
|
1493
|
+
return undefined;
|
|
1494
|
+
return JSON.stringify(version);
|
|
1495
|
+
}
|
|
1170
1496
|
/**
|
|
1171
1497
|
* Every wire out of an `if` names a branch, and no other wire does.
|
|
1172
1498
|
*
|
|
@@ -1735,14 +2061,30 @@ function canonicalNode(node) {
|
|
|
1735
2061
|
// Sorted keys, so a canvas that rewrites the object in a different order
|
|
1736
2062
|
// does not look like an edit.
|
|
1737
2063
|
sortedEntries(node.config),
|
|
2064
|
+
// Appended only when there is a reference, exactly as `edge.branch` above
|
|
2065
|
+
// is appended only when there is a label, and for the same reason: adding
|
|
2066
|
+
// reusable nodes to this file must not renumber the version of a single
|
|
2067
|
+
// graph that did not change. Every source drawn before they existed
|
|
2068
|
+
// hashes to the string it always did.
|
|
2069
|
+
...canonicalReuse(node),
|
|
1738
2070
|
]);
|
|
1739
2071
|
}
|
|
1740
2072
|
if (node.kind === 'transform') {
|
|
1741
|
-
// The transform's *version* is deliberately not in here
|
|
1742
|
-
// transform is
|
|
1743
|
-
// in would bump every graph that references it
|
|
1744
|
-
// changed when it did not.
|
|
1745
|
-
|
|
2073
|
+
// The transform's *version as stored* is deliberately not in here, and that
|
|
2074
|
+
// has not changed: editing a transform is recorded as a new transform
|
|
2075
|
+
// version, and folding it in would bump every graph that references it,
|
|
2076
|
+
// which would claim the wiring changed when it did not.
|
|
2077
|
+
//
|
|
2078
|
+
// What IS in here is the *pin* — which is the opposite choice for the
|
|
2079
|
+
// opposite reason, and the same one `call` makes one branch down. Moving a
|
|
2080
|
+
// node from v3 to v5, or off a pin onto the latest, changes what the load
|
|
2081
|
+
// runs as surely as rewiring it does, and is a decision somebody made in
|
|
2082
|
+
// this graph. Appended rather than always present, so an unpinned node —
|
|
2083
|
+
// which is every transform node in every deployment today — hashes to
|
|
2084
|
+
// exactly the string it always did.
|
|
2085
|
+
return node.transformVersion === undefined
|
|
2086
|
+
? JSON.stringify([node.id, node.kind, node.transformId])
|
|
2087
|
+
: JSON.stringify([node.id, node.kind, node.transformId, node.transformVersion]);
|
|
1746
2088
|
}
|
|
1747
2089
|
if (node.kind === 'call') {
|
|
1748
2090
|
// The called version IS in here, and that is the opposite choice from a
|
|
@@ -1778,10 +2120,35 @@ function canonicalNode(node) {
|
|
|
1778
2120
|
]);
|
|
1779
2121
|
}
|
|
1780
2122
|
if (node.kind === 'sink') {
|
|
1781
|
-
return JSON.stringify([
|
|
2123
|
+
return JSON.stringify([
|
|
2124
|
+
node.id,
|
|
2125
|
+
node.kind,
|
|
2126
|
+
node.targetType,
|
|
2127
|
+
node.mode ?? 'full',
|
|
2128
|
+
...canonicalReuse(node),
|
|
2129
|
+
]);
|
|
1782
2130
|
}
|
|
1783
2131
|
return unreachableNodeKind(node, 'workflowGraphHash');
|
|
1784
2132
|
}
|
|
2133
|
+
/**
|
|
2134
|
+
* The reusable reference, as zero, one or two trailing hash components.
|
|
2135
|
+
*
|
|
2136
|
+
* Zero when there is no reference, which is what makes this additive: every
|
|
2137
|
+
* graph stored before reusable nodes existed produces the same canonical string
|
|
2138
|
+
* it always did, so no version is renumbered by a deployment picking up this
|
|
2139
|
+
* release. One when the reference follows the latest. Two when it is pinned.
|
|
2140
|
+
*
|
|
2141
|
+
* "Follows the latest" and "pinned to v1" hash differently, and they must:
|
|
2142
|
+
* moving a node off a pin is a real change to what it will run next month, and a
|
|
2143
|
+
* fingerprint that could not see it would let somebody unpin a shared sink with
|
|
2144
|
+
* no version bump and no diff — which is precisely the silence this feature was
|
|
2145
|
+
* built to end.
|
|
2146
|
+
*/
|
|
2147
|
+
function canonicalReuse(node) {
|
|
2148
|
+
if (node.useId === undefined)
|
|
2149
|
+
return [];
|
|
2150
|
+
return node.useVersion === undefined ? [node.useId] : [node.useId, node.useVersion];
|
|
2151
|
+
}
|
|
1785
2152
|
/**
|
|
1786
2153
|
* The parts of a predicate that decide which branch runs.
|
|
1787
2154
|
*
|
|
@@ -1872,13 +2239,13 @@ function fnv1a(input, offset) {
|
|
|
1872
2239
|
function isWorkflowNode(value) {
|
|
1873
2240
|
if (typeof value !== 'object' || value === null)
|
|
1874
2241
|
return false;
|
|
1875
|
-
const id = Reflect.get(value, 'id');
|
|
1876
|
-
const name = Reflect.get(value, 'name');
|
|
1877
2242
|
const kind = Reflect.get(value, 'kind');
|
|
1878
|
-
if (typeof id !== 'string' || typeof name !== 'string')
|
|
1879
|
-
return false;
|
|
1880
2243
|
if (!isWorkflowNodeKind(kind))
|
|
1881
2244
|
return false;
|
|
2245
|
+
// Everything every kind carries, checked once before the narrowing below
|
|
2246
|
+
// rather than inside the branches that could carry it. See {@link hasNodeBase}.
|
|
2247
|
+
if (!hasNodeBase(value))
|
|
2248
|
+
return false;
|
|
1882
2249
|
if (kind === 'transform') {
|
|
1883
2250
|
return typeof Reflect.get(value, 'transformId') === 'string';
|
|
1884
2251
|
}
|
|
@@ -1934,6 +2301,49 @@ function isNarrowsList(value) {
|
|
|
1934
2301
|
return true;
|
|
1935
2302
|
return Array.isArray(value) && value.every((type) => typeof type === 'string');
|
|
1936
2303
|
}
|
|
2304
|
+
/**
|
|
2305
|
+
* A version pin as it comes back out of a JSON column: absent, or a whole number
|
|
2306
|
+
* of at least one.
|
|
2307
|
+
*
|
|
2308
|
+
* The same rule {@link checkVersionPins} states, applied at the read boundary,
|
|
2309
|
+
* because the two answer different questions about the same field. The validator
|
|
2310
|
+
* tells an author their graph will not run; this decides whether a graph stored
|
|
2311
|
+
* by some other build can be read at all. Absent is accepted and always will be
|
|
2312
|
+
* — it is what every node written before pins existed carries, and it means
|
|
2313
|
+
* "follows the latest", which is exactly what those nodes have always done.
|
|
2314
|
+
*/
|
|
2315
|
+
function isOptionalVersion(value) {
|
|
2316
|
+
if (value === undefined)
|
|
2317
|
+
return true;
|
|
2318
|
+
return typeof value === 'number' && Number.isInteger(value) && value >= 1;
|
|
2319
|
+
}
|
|
2320
|
+
/**
|
|
2321
|
+
* Everything every node kind carries, whatever kind it is: its identity, and
|
|
2322
|
+
* the things it points at outside itself.
|
|
2323
|
+
*
|
|
2324
|
+
* Checked once, before {@link isWorkflowNode} narrows per kind, rather than in
|
|
2325
|
+
* the branches that could carry each field. The references are declared on two
|
|
2326
|
+
* members of the union and a node arriving as JSON does not respect that, so a
|
|
2327
|
+
* single check is both cheaper and stricter than two — and a kind that becomes
|
|
2328
|
+
* reusable later inherits it rather than having to remember it.
|
|
2329
|
+
*
|
|
2330
|
+
* A pin read back as `"3"` or as `0` names no stored version, and the resolution
|
|
2331
|
+
* happens inside a durable step in the middle of a load. So it is refused where
|
|
2332
|
+
* a graph is read out of a column rather than surviving as far as the run that
|
|
2333
|
+
* cannot honour it.
|
|
2334
|
+
*/
|
|
2335
|
+
function hasNodeBase(value) {
|
|
2336
|
+
if (typeof Reflect.get(value, 'id') !== 'string')
|
|
2337
|
+
return false;
|
|
2338
|
+
if (typeof Reflect.get(value, 'name') !== 'string')
|
|
2339
|
+
return false;
|
|
2340
|
+
if (!isOptionalVersion(Reflect.get(value, 'transformVersion')))
|
|
2341
|
+
return false;
|
|
2342
|
+
if (!isOptionalVersion(Reflect.get(value, 'useVersion')))
|
|
2343
|
+
return false;
|
|
2344
|
+
const useId = Reflect.get(value, 'useId');
|
|
2345
|
+
return useId === undefined || typeof useId === 'string';
|
|
2346
|
+
}
|
|
1937
2347
|
/**
|
|
1938
2348
|
* The narrowing counterpart of {@link unreachableNodeKind}.
|
|
1939
2349
|
*
|
|
@@ -1983,6 +2393,43 @@ function supportsWorkflows(store) {
|
|
|
1983
2393
|
// is the surface a scheduling incident already came through once.
|
|
1984
2394
|
typeof store.saveWorkflowSchedule === 'function');
|
|
1985
2395
|
}
|
|
2396
|
+
/**
|
|
2397
|
+
* Whether this store can mint a release and be pointed at one.
|
|
2398
|
+
*
|
|
2399
|
+
* All four asked for by name, for the reason `supportsWorkflows` asks for
|
|
2400
|
+
* `publishWorkflow` and `saveWorkflowSchedule` by name rather than assuming they
|
|
2401
|
+
* arrive together: a store with the mint and not the pointer would narrow
|
|
2402
|
+
* cleanly here, let somebody release a graph, and then fail on the call that was
|
|
2403
|
+
* supposed to make it run.
|
|
2404
|
+
*
|
|
2405
|
+
* {@link getWorkflowAt} is the one whose absence is least visible and most
|
|
2406
|
+
* expensive. A store that could hold a `liveVersion` and not resolve it would
|
|
2407
|
+
* point a scheduled load at a version it cannot produce — and the only place
|
|
2408
|
+
* that shows up is a cron window that stops firing.
|
|
2409
|
+
*/
|
|
2410
|
+
function supportsWorkflowReleases(store) {
|
|
2411
|
+
return (typeof store.releaseWorkflow === 'function' &&
|
|
2412
|
+
typeof store.listWorkflowReleases === 'function' &&
|
|
2413
|
+
typeof store.getWorkflowAt === 'function' &&
|
|
2414
|
+
typeof store.setLiveWorkflowVersion === 'function');
|
|
2415
|
+
}
|
|
2416
|
+
/**
|
|
2417
|
+
* Which version of this graph a run gets when the caller names none.
|
|
2418
|
+
*
|
|
2419
|
+
* The one implementation of "follow the latest unless something is live", shared
|
|
2420
|
+
* by the scheduler and by the manual run route so the two cannot disagree about
|
|
2421
|
+
* what a cron does and what the button next to it does. That divergence is not
|
|
2422
|
+
* hypothetical: the schedule used to live on the connector and on the workflow
|
|
2423
|
+
* at once, and the whole of `ConnectorScheduler`'s docblock is about what it
|
|
2424
|
+
* cost to have two copies of one answer.
|
|
2425
|
+
*
|
|
2426
|
+
* Not a fallback in the defensive sense. `liveVersion` absent is a stated
|
|
2427
|
+
* position — this graph follows its head — and this function is where that
|
|
2428
|
+
* position is turned into a number, not where a missing value is patched over.
|
|
2429
|
+
*/
|
|
2430
|
+
function liveWorkflowVersion(workflow) {
|
|
2431
|
+
return workflow.liveVersion ?? workflow.version;
|
|
2432
|
+
}
|
|
1986
2433
|
/**
|
|
1987
2434
|
* Whether this store keeps a transform's history.
|
|
1988
2435
|
*
|
|
@@ -1992,6 +2439,35 @@ function supportsWorkflows(store) {
|
|
|
1992
2439
|
function supportsTransformRevisions(store) {
|
|
1993
2440
|
return typeof store.listTransformRevisions === 'function';
|
|
1994
2441
|
}
|
|
2442
|
+
/**
|
|
2443
|
+
* Whether this store can produce one particular version of a transform's code.
|
|
2444
|
+
*
|
|
2445
|
+
* Separate from {@link supportsTransformRevisions}, and not implied by it: one
|
|
2446
|
+
* answers "can a screen show the history", the other "can a run honour a pin".
|
|
2447
|
+
* A store could reasonably have the first and not the second, and folding them
|
|
2448
|
+
* together would let a graph be saved with a pin this deployment cannot resolve
|
|
2449
|
+
* — discovered mid-load rather than at the moment the pin was set.
|
|
2450
|
+
*/
|
|
2451
|
+
function supportsTransformPins(store) {
|
|
2452
|
+
return typeof store.getTransformAt === 'function';
|
|
2453
|
+
}
|
|
2454
|
+
/**
|
|
2455
|
+
* Whether this store can hold reusable nodes.
|
|
2456
|
+
*
|
|
2457
|
+
* All six, and never a subset. A store with `getReusableNode` but no
|
|
2458
|
+
* `reusableNodeUses` could serve a picker and could not answer the question the
|
|
2459
|
+
* feature exists for — and the shape of that failure is a console offering to
|
|
2460
|
+
* share a node while being unable to say who already depends on it, which is
|
|
2461
|
+
* worse than not offering at all.
|
|
2462
|
+
*/
|
|
2463
|
+
function supportsReusableNodes(store) {
|
|
2464
|
+
return (typeof store.listReusableNodes === 'function' &&
|
|
2465
|
+
typeof store.getReusableNode === 'function' &&
|
|
2466
|
+
typeof store.getReusableNodeAt === 'function' &&
|
|
2467
|
+
typeof store.saveReusableNode === 'function' &&
|
|
2468
|
+
typeof store.deleteReusableNode === 'function' &&
|
|
2469
|
+
typeof store.reusableNodeUses === 'function');
|
|
2470
|
+
}
|
|
1995
2471
|
function supportsWorkflowStages(store) {
|
|
1996
2472
|
return typeof store.writeStage === 'function' && typeof store.readStage === 'function';
|
|
1997
2473
|
}
|
package/dist/client.d.ts
CHANGED
|
@@ -153,8 +153,8 @@ export declare const catalogRoutes: {
|
|
|
153
153
|
readonly traces: () => string;
|
|
154
154
|
readonly trace: (id: string) => string;
|
|
155
155
|
};
|
|
156
|
-
export type { CatalogConnection, CatalogConnector, ConnectionCheck, CatalogTransform, CatalogWorkflow, CatalogWorkflowCapabilities, ConnectorKind, ConnectorRun, TransformLanguage, TransformResult, CallableWorkflowRef, WorkflowBranchLabel, WorkflowCallEnvelope, WorkflowCallNode, WorkflowCallOutput, WorkflowEdge, WorkflowExecutionMode, WorkflowFilterAll, WorkflowFilterAny, WorkflowFilterComparison, WorkflowFilterGroup, WorkflowFilterNode, WorkflowFilterOneOf, WorkflowFilterOperator, WorkflowFilterPredicate, WorkflowFilterPredicateKind, WorkflowFilterPresence, WorkflowFilterValue, WorkflowGraph, WorkflowEnvPredicate, WorkflowIfNode, WorkflowIfPredicate, WorkflowIssueCode, WorkflowNode, WorkflowNodeKind, WorkflowNodeOutcome, WorkflowPredicateKind, WorkflowRowCountPredicate, WorkflowRunOrderEntry, WorkflowSinkNode, WorkflowSkipReason, WorkflowSourceNode, WorkflowStageRef, WorkflowTransformNode, WorkflowValidationIssue, } from './catalog.pipeline';
|
|
157
|
-
export { CONNECTOR_KINDS, isConnectorKind, isTransformLanguage, isWorkflowEdge, isWorkflowNode, REDACTED_SECRET, TRANSFORM_LANGUAGES, readWorkflowCallOutput, WORKFLOW_CALL_CONTRACT, } from './catalog.pipeline';
|
|
156
|
+
export type { CatalogConnection, CatalogConnector, ConnectionCheck, CatalogTransform, CatalogWorkflow, CatalogWorkflowCapabilities, CatalogWorkflowRelease, ConnectorKind, ConnectorRun, TransformLanguage, TransformResult, CallableWorkflowRef, WorkflowBranchLabel, WorkflowCallEnvelope, WorkflowCallNode, WorkflowCallOutput, WorkflowEdge, WorkflowExecutionMode, WorkflowFilterAll, WorkflowFilterAny, WorkflowFilterComparison, WorkflowFilterGroup, WorkflowFilterNode, WorkflowFilterOneOf, WorkflowFilterOperator, WorkflowFilterPredicate, WorkflowFilterPredicateKind, WorkflowFilterPresence, WorkflowFilterValue, WorkflowGraph, WorkflowEnvPredicate, WorkflowIfNode, WorkflowIfPredicate, WorkflowIssueCode, WorkflowNode, WorkflowNodeKind, WorkflowNodeOutcome, WorkflowPredicateKind, WorkflowRowCountPredicate, WorkflowRunOrderEntry, WorkflowSinkNode, WorkflowSkipReason, WorkflowSourceNode, WorkflowStageRef, WorkflowTransformNode, WorkflowValidationIssue, CatalogReusableNode, CatalogReusableNodeUse, ReusableNodeBody, ReusableNodeKind, ReusableNodeRef, ReusableSinkBody, ReusableSourceBody, VersionPinCopy, } from './catalog.pipeline';
|
|
157
|
+
export { CONNECTOR_KINDS, isConnectorKind, isTransformLanguage, isWorkflowEdge, isWorkflowNode, liveWorkflowVersion, REDACTED_SECRET, TRANSFORM_LANGUAGES, readWorkflowCallOutput, WORKFLOW_CALL_CONTRACT, applyReusableNode, reusableNodeBodyOf, isReusableNodeBody, isReusableNodeKind, REUSABLE_NODE_KINDS, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, unreachableReusableNodeKind, describeLiveVersion, describeVersionPin, } from './catalog.pipeline';
|
|
158
158
|
/**
|
|
159
159
|
* The workflow validator, shipped to the browser deliberately.
|
|
160
160
|
*
|