@dudousxd/nestjs-catalog 0.15.0 → 0.16.0-preview.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.events.d.ts +14 -0
- package/dist/catalog.pipeline.d.ts +685 -71
- package/dist/catalog.pipeline.js +505 -10
- package/dist/client.d.ts +14 -4
- package/dist/client.js +61 -12
- package/dist/index.d.ts +1 -1
- package/dist/index.js +22 -3
- package/dist/transform-runner.d.ts +19 -1
- package/dist/transform-runner.js +63 -7
- package/package.json +6 -6
- package/LICENSE +0 -21
package/dist/catalog.pipeline.js
CHANGED
|
@@ -9,15 +9,25 @@
|
|
|
9
9
|
* systems each believing they decide when a load runs.
|
|
10
10
|
*/
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.TRANSFORM_RUNNER = exports.TRANSFORM_LANGUAGES = exports.CONNECTOR_KINDS = void 0;
|
|
12
|
+
exports.REDACTED_SECRET = exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_SKIP_REASONS = exports.CODE_CONTEXT_CONTRACT = exports.TRANSFORM_RUNNER = exports.TRANSFORM_LANGUAGES = exports.CONNECTOR_KINDS = void 0;
|
|
13
13
|
exports.isConnectorKind = isConnectorKind;
|
|
14
14
|
exports.isTransformLanguage = isTransformLanguage;
|
|
15
|
+
exports.isWorkflowSkipReason = isWorkflowSkipReason;
|
|
15
16
|
exports.isWorkflowNodeKind = isWorkflowNodeKind;
|
|
17
|
+
exports.unreachableNodeKind = unreachableNodeKind;
|
|
18
|
+
exports.workflowColumnX = workflowColumnX;
|
|
19
|
+
exports.workflowRowY = workflowRowY;
|
|
20
|
+
exports.isWorkflowPredicateKind = isWorkflowPredicateKind;
|
|
21
|
+
exports.unreachablePredicateKind = unreachablePredicateKind;
|
|
22
|
+
exports.isWorkflowIfPredicate = isWorkflowIfPredicate;
|
|
23
|
+
exports.isWorkflowBranchLabel = isWorkflowBranchLabel;
|
|
16
24
|
exports.isWorkflowStatus = isWorkflowStatus;
|
|
17
25
|
exports.isWorkflowExecutionMode = isWorkflowExecutionMode;
|
|
18
26
|
exports.readWorkflowCallOutput = readWorkflowCallOutput;
|
|
27
|
+
exports.callableWorkflowBlock = callableWorkflowBlock;
|
|
19
28
|
exports.validateWorkflow = validateWorkflow;
|
|
20
29
|
exports.workflowRunOrder = workflowRunOrder;
|
|
30
|
+
exports.workflowNodeRuns = workflowNodeRuns;
|
|
21
31
|
exports.workflowGraphHash = workflowGraphHash;
|
|
22
32
|
exports.isWorkflowNode = isWorkflowNode;
|
|
23
33
|
exports.isWorkflowEdge = isWorkflowEdge;
|
|
@@ -68,6 +78,39 @@ function isTransformLanguage(value) {
|
|
|
68
78
|
return exports.TRANSFORM_LANGUAGES.some((language) => language === value);
|
|
69
79
|
}
|
|
70
80
|
exports.TRANSFORM_RUNNER = Symbol('TRANSFORM_RUNNER');
|
|
81
|
+
/**
|
|
82
|
+
* The number in {@link CatalogCodeContext.contract}.
|
|
83
|
+
*
|
|
84
|
+
* Its own version, separate from {@link WORKFLOW_CALL_CONTRACT}, because the
|
|
85
|
+
* two travel to different places for different reasons — a call envelope
|
|
86
|
+
* crosses to another SDK over the durable wire, a code context crosses to a
|
|
87
|
+
* child process this repository spawned — and a transform written against one
|
|
88
|
+
* has to be able to say which it read.
|
|
89
|
+
*/
|
|
90
|
+
exports.CODE_CONTEXT_CONTRACT = 1;
|
|
91
|
+
/**
|
|
92
|
+
* Why a `skipped` node did not run, when there is more to say than "the run
|
|
93
|
+
* stopped".
|
|
94
|
+
*
|
|
95
|
+
* **One entry, and the omission is the point.** `skipped` already meant one
|
|
96
|
+
* thing before branches existed — the run failed upstream and never reached
|
|
97
|
+
* this node — and every outcome ever stored records that meaning by *not*
|
|
98
|
+
* carrying a reason. Adding `run-stopped` to this list would not describe those
|
|
99
|
+
* rows, it would leave them describing an unknown reason, so the pre-existing
|
|
100
|
+
* meaning stays the absent one and this names only the new fact: the node is on
|
|
101
|
+
* a branch an {@link WorkflowIfNode} did not take.
|
|
102
|
+
*
|
|
103
|
+
* The distinction is not cosmetic. A sink skipped by a branch **committed
|
|
104
|
+
* nothing and left the live snapshot alone**, which is a correct, successful
|
|
105
|
+
* outcome; a sink skipped by a failure is part of a load that went wrong. A run
|
|
106
|
+
* panel that rendered both as "did not run" would answer "why is there no data
|
|
107
|
+
* in X" with the same shrug in both cases.
|
|
108
|
+
*/
|
|
109
|
+
exports.WORKFLOW_SKIP_REASONS = ['branch-not-taken'];
|
|
110
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
111
|
+
function isWorkflowSkipReason(value) {
|
|
112
|
+
return exports.WORKFLOW_SKIP_REASONS.some((reason) => reason === value);
|
|
113
|
+
}
|
|
71
114
|
/* ---------------------------------------------------------------------------
|
|
72
115
|
* Workflows: a graph of steps that ends in exactly one commit.
|
|
73
116
|
*
|
|
@@ -108,9 +151,12 @@ exports.TRANSFORM_RUNNER = Symbol('TRANSFORM_RUNNER');
|
|
|
108
151
|
* It needs no new execution path, only a different body, and adding the kind
|
|
109
152
|
* would mean two ways to drop rows and two places to look when rows go
|
|
110
153
|
* missing.
|
|
111
|
-
* - **branch / split** — already expressible
|
|
112
|
-
* read by both successors, each of which
|
|
113
|
-
* nothing for
|
|
154
|
+
* - **branch / split (unconditional)** — already expressible, and still is: a
|
|
155
|
+
* node with two outbound edges is read by both successors, each of which
|
|
156
|
+
* filters differently. There is nothing for an *unconditional* split to do.
|
|
157
|
+
* {@link WorkflowIfNode} is the conditional one, and it earns its kind by
|
|
158
|
+
* doing something no wiring can express — deciding that one of those
|
|
159
|
+
* successors, and everything only it feeds, does not run at all.
|
|
114
160
|
* - **merge / join** — a node with several inbound edges receives its inputs
|
|
115
161
|
* concatenated in edge order (see {@link WorkflowEdge}). A keyed join is then
|
|
116
162
|
* ordinary code inside the transform, which can already see every record.
|
|
@@ -136,11 +182,38 @@ exports.WORKFLOW_NODE_KINDS = [
|
|
|
136
182
|
'sink',
|
|
137
183
|
/** Hands this position to an existing durable workflow. See {@link WorkflowCallNode}. */
|
|
138
184
|
'call',
|
|
185
|
+
/** Sends the rows down one of its outbound branches. See {@link WorkflowIfNode}. */
|
|
186
|
+
'if',
|
|
139
187
|
];
|
|
140
188
|
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
141
189
|
function isWorkflowNodeKind(value) {
|
|
142
190
|
return exports.WORKFLOW_NODE_KINDS.some((kind) => kind === value);
|
|
143
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* The kind that never compiles quietly.
|
|
194
|
+
*
|
|
195
|
+
* Every place that decides something *per kind* ends in a call to this, so a
|
|
196
|
+
* kind added to {@link WORKFLOW_NODE_KINDS} without a branch there is a type
|
|
197
|
+
* error naming the file rather than a graph that saves, validates, draws and
|
|
198
|
+
* then does the wrong thing. This codebase has been bitten by exactly that
|
|
199
|
+
* shape — a `toGraph` branch forgetting a field, a node-kind map missing a kind
|
|
200
|
+
* — and the fix each time was to make the omission impossible rather than to
|
|
201
|
+
* remember harder.
|
|
202
|
+
*
|
|
203
|
+
* It throws as well as failing to compile, because the narrowing that reaches
|
|
204
|
+
* it is over data that arrives as JSON: a node whose `kind` passed
|
|
205
|
+
* {@link isWorkflowNodeKind} in an older build and reaches a newer one is
|
|
206
|
+
* possible, and returning a default for it would be the silent path this exists
|
|
207
|
+
* to close.
|
|
208
|
+
*/
|
|
209
|
+
function unreachableNodeKind(node, where) {
|
|
210
|
+
// Takes either the node or its kind, because the call sites differ: a
|
|
211
|
+
// narrowing chain over a union hands it the node, while one over the kind
|
|
212
|
+
// string — which is what a boundary reading JSON has before it has a node —
|
|
213
|
+
// hands it the string.
|
|
214
|
+
const kind = typeof node === 'string' ? node : Reflect.get(Object(node), 'kind');
|
|
215
|
+
throw new Error(`${where} does not handle a workflow node of kind ${JSON.stringify(kind)}. The kind list and every decision made per kind are meant to move together.`);
|
|
216
|
+
}
|
|
144
217
|
/**
|
|
145
218
|
* The longest a node id may be, and the alphabet it may use.
|
|
146
219
|
*
|
|
@@ -152,6 +225,160 @@ function isWorkflowNodeKind(value) {
|
|
|
152
225
|
* separator would let one node read another's rows.
|
|
153
226
|
*/
|
|
154
227
|
exports.WORKFLOW_NODE_ID_PATTERN = /^[A-Za-z0-9_-]{1,64}$/;
|
|
228
|
+
/**
|
|
229
|
+
* How big a node is on the canvas, and therefore how far apart two of them have
|
|
230
|
+
* to be.
|
|
231
|
+
*
|
|
232
|
+
* ## Why the server owns a number about pixels
|
|
233
|
+
*
|
|
234
|
+
* Because the server *writes positions*. `adoptConnector` lays a connector out
|
|
235
|
+
* as a graph, and anything else that mints a graph without a person drawing it —
|
|
236
|
+
* a promotion, a template, a script — has the same job. A writer that does not
|
|
237
|
+
* know how wide a node is can only guess, and the guess was wrong: adoption used
|
|
238
|
+
* to place columns 220 apart against a node 224 wide, so every adopted graph
|
|
239
|
+
* drew each box overlapping the next by four pixels. Nothing was stacked and
|
|
240
|
+
* nothing was missing, so it read as boxes glued together rather than as a bug,
|
|
241
|
+
* and it survived until somebody opened thirteen of them.
|
|
242
|
+
*
|
|
243
|
+
* The fix is not a bigger number. 220 was not too small, it was **derived from
|
|
244
|
+
* nothing** — it had no relationship to the width it was supposed to clear, so
|
|
245
|
+
* it was only ever correct by luck and would go wrong again the next time the
|
|
246
|
+
* node's styling changed. These constants are the relationship, stated once, in
|
|
247
|
+
* the one package both the writer and the canvas already depend on.
|
|
248
|
+
*
|
|
249
|
+
* ## What pins them to the drawing
|
|
250
|
+
*
|
|
251
|
+
* {@link WORKFLOW_NODE_WIDTH} is not a description of the node — it is the
|
|
252
|
+
* *source of* the node's width. `WorkflowNodeBody` in
|
|
253
|
+
* `@dudousxd/nestjs-catalog-react` sets its own width from this constant rather
|
|
254
|
+
* than from a Tailwind class, so the two cannot drift: changing this changes the
|
|
255
|
+
* box, and there is no second number to forget.
|
|
256
|
+
*/
|
|
257
|
+
exports.WORKFLOW_NODE_WIDTH = 224;
|
|
258
|
+
/** How tall a node is. The other half of {@link WORKFLOW_NODE_WIDTH}'s contract. */
|
|
259
|
+
exports.WORKFLOW_NODE_HEIGHT = 80;
|
|
260
|
+
/**
|
|
261
|
+
* Clear space between one column and the next, on top of the node's own width.
|
|
262
|
+
*
|
|
263
|
+
* Wide enough for the edge between two nodes to be read as a line with a
|
|
264
|
+
* direction rather than as a join. This is the part that is taste; the width it
|
|
265
|
+
* is added to is not.
|
|
266
|
+
*/
|
|
267
|
+
exports.WORKFLOW_COLUMN_GAP = 96;
|
|
268
|
+
/** Clear space between two nodes sharing a column. */
|
|
269
|
+
exports.WORKFLOW_ROW_GAP = 32;
|
|
270
|
+
/**
|
|
271
|
+
* The x of the nth column, counting from zero.
|
|
272
|
+
*
|
|
273
|
+
* Every generator of a layout goes through this rather than multiplying by a
|
|
274
|
+
* literal, which is what makes "columns never overlap" a property of one
|
|
275
|
+
* function instead of a coincidence repeated at each call site.
|
|
276
|
+
*/
|
|
277
|
+
function workflowColumnX(column) {
|
|
278
|
+
return column * (exports.WORKFLOW_NODE_WIDTH + exports.WORKFLOW_COLUMN_GAP);
|
|
279
|
+
}
|
|
280
|
+
/** The y of the nth row within a column, counting from zero. */
|
|
281
|
+
function workflowRowY(row) {
|
|
282
|
+
return row * (exports.WORKFLOW_NODE_HEIGHT + exports.WORKFLOW_ROW_GAP);
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* The kinds of test an {@link WorkflowIfNode} can make.
|
|
286
|
+
*
|
|
287
|
+
* A second predicate shape was always going to arrive — the note on
|
|
288
|
+
* {@link WorkflowIfNode} says so about `code` — and the shape it arrives into is
|
|
289
|
+
* the decision worth arguing about, because the alternative was to keep both
|
|
290
|
+
* tests' fields flat on the node and mark them optional. That version types a
|
|
291
|
+
* gate as "an env var, maybe, and a threshold, maybe": a node carrying both is
|
|
292
|
+
* representable, a node carrying neither is representable, and every reader has
|
|
293
|
+
* to invent its own rule for which one wins. It is the same mistake the note on
|
|
294
|
+
* {@link WorkflowNode} refuses for node kinds, one level down.
|
|
295
|
+
*
|
|
296
|
+
* So the predicate is a union with a discriminant of its own, and every decision
|
|
297
|
+
* made per predicate kind ends in {@link unreachablePredicateKind} — a third
|
|
298
|
+
* shape is then a build failure listing the files that have to answer for it,
|
|
299
|
+
* rather than a gate that saves, draws, and quietly always takes the `else`.
|
|
300
|
+
*/
|
|
301
|
+
exports.WORKFLOW_PREDICATE_KINDS = [
|
|
302
|
+
/** Reads a variable where the load runs. See {@link WorkflowEnvPredicate}. */
|
|
303
|
+
'env',
|
|
304
|
+
/** Counts the rows handed to the gate. See {@link WorkflowRowCountPredicate}. */
|
|
305
|
+
'rowCount',
|
|
306
|
+
];
|
|
307
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
308
|
+
function isWorkflowPredicateKind(value) {
|
|
309
|
+
return exports.WORKFLOW_PREDICATE_KINDS.some((kind) => kind === value);
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* {@link unreachableNodeKind}, one level down, and for the identical reason.
|
|
313
|
+
*
|
|
314
|
+
* Every branch over {@link WorkflowIfPredicate} ends here, so a predicate kind
|
|
315
|
+
* added to the list without a rule for hashing it, validating it or evaluating
|
|
316
|
+
* it is a type error naming the file — not a graph that runs and decides
|
|
317
|
+
* something nobody authored. It throws as well, because predicates arrive as
|
|
318
|
+
* JSON out of a column and a build older than the data is a thing that happens.
|
|
319
|
+
*/
|
|
320
|
+
function unreachablePredicateKind(predicate, where) {
|
|
321
|
+
const kind = typeof predicate === 'string' ? predicate : Reflect.get(Object(predicate), 'kind');
|
|
322
|
+
throw new Error(`${where} does not handle an if-node predicate of kind ${JSON.stringify(kind)}. The predicate kinds and every decision made per kind are meant to move together.`);
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Narrow a stored predicate, for the same reason {@link isWorkflowNode} narrows
|
|
326
|
+
* a stored node: it arrives as JSON out of a column, and a gate read back
|
|
327
|
+
* without its test is a gate that has to invent one.
|
|
328
|
+
*
|
|
329
|
+
* A row count that is not a whole number — `NaN` from a JSON round trip of an
|
|
330
|
+
* unparsed field, an `Infinity` that serialised as `null` — is refused rather
|
|
331
|
+
* than kept, because every comparison against it is false and the symptom is a
|
|
332
|
+
* `then` branch that silently never runs again.
|
|
333
|
+
*/
|
|
334
|
+
function isWorkflowIfPredicate(value) {
|
|
335
|
+
if (typeof value !== 'object' || value === null)
|
|
336
|
+
return false;
|
|
337
|
+
const kind = Reflect.get(value, 'kind');
|
|
338
|
+
if (!isWorkflowPredicateKind(kind))
|
|
339
|
+
return false;
|
|
340
|
+
if (kind === 'env') {
|
|
341
|
+
// The variable name is required and its expected value is not, exactly as
|
|
342
|
+
// the type says: an absent `equals` is the "is it set at all" test, so a
|
|
343
|
+
// stored predicate without one is complete rather than half-narrowed.
|
|
344
|
+
const equals = Reflect.get(value, 'equals');
|
|
345
|
+
return (typeof Reflect.get(value, 'envVar') === 'string' &&
|
|
346
|
+
(equals === undefined || typeof equals === 'string'));
|
|
347
|
+
}
|
|
348
|
+
if (kind === 'rowCount') {
|
|
349
|
+
const atLeast = Reflect.get(value, 'atLeast');
|
|
350
|
+
return typeof atLeast === 'number' && Number.isInteger(atLeast) && atLeast >= 0;
|
|
351
|
+
}
|
|
352
|
+
return isWorkflowPredicateKindUnhandled(kind);
|
|
353
|
+
}
|
|
354
|
+
/** The narrowing counterpart of {@link unreachablePredicateKind}. */
|
|
355
|
+
function isWorkflowPredicateKindUnhandled(kind) {
|
|
356
|
+
void kind;
|
|
357
|
+
return false;
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Which side of an {@link WorkflowIfNode} a wire leaves by.
|
|
361
|
+
*
|
|
362
|
+
* **Two closed values rather than free-form labels**, which was the other design
|
|
363
|
+
* and is worse in the one way that matters here: an unlabelled branch is a
|
|
364
|
+
* branch that never runs, and a *misspelled* branch is one too. With a free
|
|
365
|
+
* string, `thn` is a subtree that silently never executes and a graph that
|
|
366
|
+
* validates perfectly; with these, it is refused at the boundary and is a type
|
|
367
|
+
* error in the console. The failure a branch introduces is "nothing loaded and
|
|
368
|
+
* nothing complained", so the vocabulary is the place to make it impossible.
|
|
369
|
+
*
|
|
370
|
+
* An N-way `switch` node was considered and is deliberately not this: it would
|
|
371
|
+
* have to carry its own case list plus a default, the cases would have to be
|
|
372
|
+
* validated against the labels, and an unmatched value would need a rule. That
|
|
373
|
+
* is a different node, and it can be added later without changing this one —
|
|
374
|
+
* because a boolean question is what a predicate answers, and an `if` is exactly
|
|
375
|
+
* the shape of a boolean question.
|
|
376
|
+
*/
|
|
377
|
+
exports.WORKFLOW_BRANCH_LABELS = ['then', 'else'];
|
|
378
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
379
|
+
function isWorkflowBranchLabel(value) {
|
|
380
|
+
return exports.WORKFLOW_BRANCH_LABELS.some((label) => label === value);
|
|
381
|
+
}
|
|
155
382
|
/**
|
|
156
383
|
* An authored graph of steps ending in one commit.
|
|
157
384
|
*
|
|
@@ -299,6 +526,23 @@ function isCount(value) {
|
|
|
299
526
|
function describeCount(value) {
|
|
300
527
|
return value === undefined ? 'nothing' : JSON.stringify(value);
|
|
301
528
|
}
|
|
529
|
+
function callableWorkflowBlock(ref) {
|
|
530
|
+
const version = typeof ref.version === 'string' ? ref.version.trim() : '';
|
|
531
|
+
if (version.length === 0) {
|
|
532
|
+
return {
|
|
533
|
+
code: 'no-version',
|
|
534
|
+
message: `A live worker announces "${ref.name}" without saying which version it runs, which is what a worker announces before it has been upgraded to publish its registrations in full. A name with no version cannot be pinned, so this cannot be chosen — type the name and the version you mean.`,
|
|
535
|
+
};
|
|
536
|
+
}
|
|
537
|
+
const groups = ref.disagreements?.find((entry) => entry.axis === 'group')?.values ?? [];
|
|
538
|
+
if (groups.length > 1) {
|
|
539
|
+
return {
|
|
540
|
+
code: 'ambiguous-group',
|
|
541
|
+
message: `Live workers announce ${ref.name}@${version} from ${groups.length} different groups (${groups.join(', ')}), so nothing can say which queue a run would be dispatched to — or whether the two are even the same code. This cannot be chosen until the deployment stops claiming it twice.`,
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
return undefined;
|
|
545
|
+
}
|
|
302
546
|
/** Every way a graph can be refused. Exported so a canvas can key off the code. */
|
|
303
547
|
exports.WORKFLOW_ISSUE_CODES = [
|
|
304
548
|
'empty',
|
|
@@ -317,6 +561,11 @@ exports.WORKFLOW_ISSUE_CODES = [
|
|
|
317
561
|
'dead-end',
|
|
318
562
|
'transform-not-named',
|
|
319
563
|
'call-not-named',
|
|
564
|
+
'if-not-named',
|
|
565
|
+
'if-threshold-invalid',
|
|
566
|
+
'if-needs-one-input',
|
|
567
|
+
'branch-not-labelled',
|
|
568
|
+
'branch-on-plain-edge',
|
|
320
569
|
];
|
|
321
570
|
/**
|
|
322
571
|
* Everything that makes a graph unrunnable, in one pure function.
|
|
@@ -359,6 +608,7 @@ function validateWorkflow(graph) {
|
|
|
359
608
|
const sinks = nodes.filter((node) => node.kind === 'sink');
|
|
360
609
|
checkNodeWiring(nodes, incoming, outgoing, issues);
|
|
361
610
|
checkEndpoints(originators, sinks, issues);
|
|
611
|
+
checkBranches(edges, byId, issues);
|
|
362
612
|
const looped = findCycle(nodes, incoming, outgoing);
|
|
363
613
|
if (looped) {
|
|
364
614
|
issues.push({
|
|
@@ -485,11 +735,59 @@ function checkNodeWiring(nodes, incoming, outgoing, issues) {
|
|
|
485
735
|
message: `Sink "${node.name}" (${node.id}) has an outbound edge. The sink commits the snapshot, so nothing can run after it.`,
|
|
486
736
|
});
|
|
487
737
|
}
|
|
738
|
+
// Exactly one, not "at least one". A gate hands its successors the ref of
|
|
739
|
+
// the stage it was given rather than staging a copy — see `WorkflowIfNode`
|
|
740
|
+
// — and one output ref cannot name two inputs, so a second inbound edge
|
|
741
|
+
// would be silently dropped. Zero is caught by `unreachable` instead, which
|
|
742
|
+
// points at the same fix with the better message.
|
|
743
|
+
if (node.kind === 'if' && (incoming.get(node.id)?.length ?? 0) > 1) {
|
|
744
|
+
issues.push({
|
|
745
|
+
code: 'if-needs-one-input',
|
|
746
|
+
nodeIds: [node.id],
|
|
747
|
+
message: `If "${node.name}" (${node.id}) has ${incoming.get(node.id)?.length} inbound edges, and it can only carry one through. An if node is a gate: it passes the rows it is given straight down whichever branch it takes, so it has one output to hand on and cannot merge. Wire those inputs into a transform and gate the transform instead.`,
|
|
748
|
+
});
|
|
749
|
+
}
|
|
488
750
|
const unconfigured = nodeIsUnconfigured(node);
|
|
489
751
|
if (unconfigured)
|
|
490
752
|
issues.push(unconfigured);
|
|
491
753
|
}
|
|
492
754
|
}
|
|
755
|
+
/**
|
|
756
|
+
* Every wire out of an `if` names a branch, and no other wire does.
|
|
757
|
+
*
|
|
758
|
+
* Both halves, because the two failures are opposite and both are silent. An
|
|
759
|
+
* unlabelled wire out of an `if` belongs to no branch, so nothing would ever
|
|
760
|
+
* take it and the subtree behind it would be skipped on every run of every
|
|
761
|
+
* deployment — a graph that draws correctly and quietly does half its work. A
|
|
762
|
+
* label on a wire whose source does not branch is the reverse: a decision
|
|
763
|
+
* somebody wrote down that nothing reads, which is worse than no decision
|
|
764
|
+
* because the canvas draws it.
|
|
765
|
+
*
|
|
766
|
+
* Run after the structural checks, so `byId` has every endpoint and this cannot
|
|
767
|
+
* report a wire whose real problem is that it points at a node that was deleted.
|
|
768
|
+
*/
|
|
769
|
+
function checkBranches(edges, byId, issues) {
|
|
770
|
+
for (const edge of edges) {
|
|
771
|
+
const from = byId.get(edge.from);
|
|
772
|
+
if (!from)
|
|
773
|
+
continue;
|
|
774
|
+
if (from.kind === 'if' && edge.branch === undefined) {
|
|
775
|
+
issues.push({
|
|
776
|
+
code: 'branch-not-labelled',
|
|
777
|
+
nodeIds: [edge.from, edge.to],
|
|
778
|
+
message: `The wire from "${from.name}" (${from.id}) to "${byId.get(edge.to)?.name ?? edge.to}" does not say which branch it is on. Every wire out of an if node belongs to "then" or to "else", because that is what decides whether it runs — an unlabelled one would never be taken, and everything only it feeds would be skipped on every run with nothing to say why.`,
|
|
779
|
+
});
|
|
780
|
+
continue;
|
|
781
|
+
}
|
|
782
|
+
if (from.kind !== 'if' && edge.branch !== undefined) {
|
|
783
|
+
issues.push({
|
|
784
|
+
code: 'branch-on-plain-edge',
|
|
785
|
+
nodeIds: [edge.from, edge.to],
|
|
786
|
+
message: `The wire from "${from.name}" (${from.id}) is labelled "${edge.branch}", but "${from.name}" is a ${from.kind} node and does not branch. A label nothing decides on is a decision that is drawn and never read; remove it, or put an if node where the choice is meant to be made.`,
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
}
|
|
493
791
|
/**
|
|
494
792
|
* A node that names none of the thing it exists to run.
|
|
495
793
|
*
|
|
@@ -508,8 +806,46 @@ function nodeIsUnconfigured(node) {
|
|
|
508
806
|
}
|
|
509
807
|
if (node.kind === 'call')
|
|
510
808
|
return callIsUnnamed(node);
|
|
809
|
+
if (node.kind === 'if')
|
|
810
|
+
return ifIsUnconfigured(node);
|
|
511
811
|
return undefined;
|
|
512
812
|
}
|
|
813
|
+
/**
|
|
814
|
+
* A gate whose test cannot decide anything.
|
|
815
|
+
*
|
|
816
|
+
* The third of the same mistake {@link nodeIsUnconfigured} describes, once per
|
|
817
|
+
* predicate kind, because "unconfigured" means something different for each and
|
|
818
|
+
* a single check would have to pick one. Both refusals exist for one reason: a
|
|
819
|
+
* gate that cannot really choose still picks a branch, and whichever it picks is
|
|
820
|
+
* a decision the graph appears to make and nobody authored — with half the
|
|
821
|
+
* pipeline silently not running as the only symptom.
|
|
822
|
+
*/
|
|
823
|
+
function ifIsUnconfigured(node) {
|
|
824
|
+
const predicate = node.predicate;
|
|
825
|
+
if (predicate.kind === 'env') {
|
|
826
|
+
if (predicate.envVar.trim().length > 0)
|
|
827
|
+
return undefined;
|
|
828
|
+
return {
|
|
829
|
+
code: 'if-not-named',
|
|
830
|
+
nodeIds: [node.id],
|
|
831
|
+
message: `If "${node.name}" (${node.id}) names no environment variable, so there is nothing for it to decide on. It reads the *name* of a variable on the machine that runs the load — that is how a graph tells a deployment with a ClickHouse apart from one without.`,
|
|
832
|
+
};
|
|
833
|
+
}
|
|
834
|
+
if (predicate.kind === 'rowCount') {
|
|
835
|
+
// At least one, so both answers are reachable. A threshold of zero is
|
|
836
|
+
// satisfied by every run including an empty one, so the `else` subtree would
|
|
837
|
+
// never execute on any deployment — the silent half-graph, arrived at by
|
|
838
|
+
// typing a number rather than by mislabelling a wire.
|
|
839
|
+
if (Number.isInteger(predicate.atLeast) && predicate.atLeast >= 1)
|
|
840
|
+
return undefined;
|
|
841
|
+
return {
|
|
842
|
+
code: 'if-threshold-invalid',
|
|
843
|
+
nodeIds: [node.id],
|
|
844
|
+
message: `If "${node.name}" (${node.id}) branches on a row count of ${JSON.stringify(predicate.atLeast)}, and a threshold has to be a whole number of at least 1. "At least 1" is the "did anything arrive at all" test; 0 would be satisfied by every run, so the else branch — and everything only it feeds — would never run on any deployment.`,
|
|
845
|
+
};
|
|
846
|
+
}
|
|
847
|
+
return unreachablePredicateKind(predicate, 'validateWorkflow');
|
|
848
|
+
}
|
|
513
849
|
/**
|
|
514
850
|
* A call that names half of what it needs, or nothing at all.
|
|
515
851
|
*
|
|
@@ -684,6 +1020,7 @@ function workflowRunOrder(graph) {
|
|
|
684
1020
|
throw new Error(`Refusing to order an invalid workflow: ${issues.map((issue) => issue.message).join(' ')}`);
|
|
685
1021
|
}
|
|
686
1022
|
const byId = new Map(graph.nodes.map((node) => [node.id, node]));
|
|
1023
|
+
const labels = branchLabels(graph.edges);
|
|
687
1024
|
// The same adjacency the validator walks, from the same builder. Two copies of
|
|
688
1025
|
// "what is wired into what" is exactly how a graph that validated comes out
|
|
689
1026
|
// executing differently, which is the thing this function's contract rules out.
|
|
@@ -702,7 +1039,8 @@ function workflowRunOrder(graph) {
|
|
|
702
1039
|
// from, and it is part of the fingerprint precisely because it is visible in
|
|
703
1040
|
// the output. `buildAdjacency` fills `incoming` by walking the edges in
|
|
704
1041
|
// order, so that is what this already is.
|
|
705
|
-
|
|
1042
|
+
const inputs = [...(incoming.get(id) ?? [])];
|
|
1043
|
+
order.push({ node, inputs, inputBranches: labelsInto(id, inputs, labels) });
|
|
706
1044
|
for (const next of outgoing.get(id) ?? []) {
|
|
707
1045
|
const remaining = (indegree.get(next) ?? 0) - 1;
|
|
708
1046
|
indegree.set(next, remaining);
|
|
@@ -712,6 +1050,81 @@ function workflowRunOrder(graph) {
|
|
|
712
1050
|
}
|
|
713
1051
|
return order;
|
|
714
1052
|
}
|
|
1053
|
+
/** Every labelled wire, keyed by the pair it joins. */
|
|
1054
|
+
function branchLabels(edges) {
|
|
1055
|
+
const labels = new Map();
|
|
1056
|
+
for (const edge of edges) {
|
|
1057
|
+
if (edge.branch === undefined)
|
|
1058
|
+
continue;
|
|
1059
|
+
labels.set(`${edge.from}\0${edge.to}`, edge.branch);
|
|
1060
|
+
}
|
|
1061
|
+
return labels;
|
|
1062
|
+
}
|
|
1063
|
+
/** The labels on the wires into one node, with the unlabelled ones left out. */
|
|
1064
|
+
function labelsInto(to, inputs, labels) {
|
|
1065
|
+
const into = {};
|
|
1066
|
+
for (const from of inputs) {
|
|
1067
|
+
const label = labels.get(`${from}\0${to}`);
|
|
1068
|
+
if (label !== undefined)
|
|
1069
|
+
into[from] = label;
|
|
1070
|
+
}
|
|
1071
|
+
return into;
|
|
1072
|
+
}
|
|
1073
|
+
/**
|
|
1074
|
+
* Whether a node runs, given what the nodes before it did.
|
|
1075
|
+
*
|
|
1076
|
+
* ## The rule
|
|
1077
|
+
*
|
|
1078
|
+
* A node runs when **at least one wire into it is live**, where a wire is live
|
|
1079
|
+
* if its source ran and — when the wire carries a branch label — the source is
|
|
1080
|
+
* an `if` that took that branch. A node with no inbound wires always runs, which
|
|
1081
|
+
* is every source and every originating call.
|
|
1082
|
+
*
|
|
1083
|
+
* ## Why the obvious rule is wrong
|
|
1084
|
+
*
|
|
1085
|
+
* The naive version is "mark everything downstream of the untaken edge as
|
|
1086
|
+
* skipped", and it is wrong on the shape branches are most often drawn in:
|
|
1087
|
+
*
|
|
1088
|
+
* ```
|
|
1089
|
+
* ┌ then → A ┐
|
|
1090
|
+
* if ──┤ ├→ C → sink
|
|
1091
|
+
* └ else → B ┘
|
|
1092
|
+
* ```
|
|
1093
|
+
*
|
|
1094
|
+
* `C` is downstream of `B`. Take the `then` branch and the naive rule walks from
|
|
1095
|
+
* the untaken `else` edge, reaches `B`, reaches `C`, and skips it — so the sink
|
|
1096
|
+
* never runs and the load silently commits nothing, on a graph whose whole
|
|
1097
|
+
* purpose was that both branches converge. Reachability from the **taken** edges
|
|
1098
|
+
* gets it right: `C` is reached through `A`, so it runs, and `B` — reached only
|
|
1099
|
+
* through the untaken edge — does not. `C` then sees an empty stage ref for `B`,
|
|
1100
|
+
* which is exactly what `stageRefsFor` already does for an upstream that
|
|
1101
|
+
* produced nothing, so the positions a merge reads stay aligned with the wires
|
|
1102
|
+
* that were drawn.
|
|
1103
|
+
*
|
|
1104
|
+
* It is evaluated incrementally rather than as a graph walk because the answers
|
|
1105
|
+
* arrive as the run goes: `workflowRunOrder` is topological, so by the time a
|
|
1106
|
+
* node is reached every node feeding it has an outcome. That also means this
|
|
1107
|
+
* reads **only** what was recorded — no predicate is re-evaluated here, which is
|
|
1108
|
+
* the property the whole branch feature rests on.
|
|
1109
|
+
*/
|
|
1110
|
+
function workflowNodeRuns(entry, outcomes) {
|
|
1111
|
+
if (entry.inputs.length === 0)
|
|
1112
|
+
return true;
|
|
1113
|
+
return entry.inputs.some((from) => {
|
|
1114
|
+
const upstream = outcomes[from];
|
|
1115
|
+
// Anything other than a clean success means this wire carried nothing: a
|
|
1116
|
+
// skipped upstream is not on a live path, and a failed one aborts the run
|
|
1117
|
+
// before this is ever asked.
|
|
1118
|
+
if (upstream?.status !== 'succeeded')
|
|
1119
|
+
return false;
|
|
1120
|
+
const label = entry.inputBranches?.[from];
|
|
1121
|
+
// A plain wire. Live because its source ran, which is what every graph
|
|
1122
|
+
// drawn before branches existed relies on.
|
|
1123
|
+
if (label === undefined)
|
|
1124
|
+
return true;
|
|
1125
|
+
return upstream.branch === label;
|
|
1126
|
+
});
|
|
1127
|
+
}
|
|
715
1128
|
/**
|
|
716
1129
|
* A stable fingerprint of what a graph *does*.
|
|
717
1130
|
*
|
|
@@ -737,7 +1150,12 @@ function workflowGraphHash(graph) {
|
|
|
737
1150
|
const nodes = [...graph.nodes]
|
|
738
1151
|
.sort((left, right) => (left.id < right.id ? -1 : left.id > right.id ? 1 : 0))
|
|
739
1152
|
.map((node) => canonicalNode(node));
|
|
740
|
-
|
|
1153
|
+
// The branch is appended only when there is one, so every edge drawn before
|
|
1154
|
+
// branches existed hashes to exactly the string it always did. Adding a graph
|
|
1155
|
+
// to this file must not renumber the versions of graphs that did not change.
|
|
1156
|
+
const edges = graph.edges.map((edge) => edge.branch === undefined
|
|
1157
|
+
? `${edge.from}>${edge.to}`
|
|
1158
|
+
: `${edge.from}>${edge.to}:${edge.branch}`);
|
|
741
1159
|
const canonical = JSON.stringify({ nodes, edges });
|
|
742
1160
|
return `${fnv1a(canonical, 0x811c9dc5)}${fnv1a(canonical, 0x01000193)}`;
|
|
743
1161
|
}
|
|
@@ -779,7 +1197,34 @@ function canonicalNode(node) {
|
|
|
779
1197
|
sortedEntries(node.config),
|
|
780
1198
|
]);
|
|
781
1199
|
}
|
|
782
|
-
|
|
1200
|
+
if (node.kind === 'if') {
|
|
1201
|
+
return JSON.stringify([node.id, node.kind, ...canonicalPredicate(node.predicate)]);
|
|
1202
|
+
}
|
|
1203
|
+
if (node.kind === 'sink') {
|
|
1204
|
+
return JSON.stringify([node.id, node.kind, node.targetType, node.mode ?? 'full']);
|
|
1205
|
+
}
|
|
1206
|
+
return unreachableNodeKind(node, 'workflowGraphHash');
|
|
1207
|
+
}
|
|
1208
|
+
/**
|
|
1209
|
+
* The parts of a predicate that decide which branch runs.
|
|
1210
|
+
*
|
|
1211
|
+
* The kind leads, so the two tests can never canonicalise to the same string —
|
|
1212
|
+
* a gate switched from "is CLICKHOUSE_URL set" to "did 1 row arrive" is a
|
|
1213
|
+
* different pipeline on every deployment, and a hash that missed it would leave
|
|
1214
|
+
* two runs claiming the same graph version while having taken different halves
|
|
1215
|
+
* of it.
|
|
1216
|
+
*/
|
|
1217
|
+
function canonicalPredicate(predicate) {
|
|
1218
|
+
if (predicate.kind === 'env') {
|
|
1219
|
+
// Both halves, because both decide which branch runs. `equals` being absent
|
|
1220
|
+
// is a *different* test from `equals` being the empty string — "set to
|
|
1221
|
+
// anything" against "set to nothing" — so the two must not fold together.
|
|
1222
|
+
return [predicate.kind, predicate.envVar, predicate.equals ?? null];
|
|
1223
|
+
}
|
|
1224
|
+
if (predicate.kind === 'rowCount') {
|
|
1225
|
+
return [predicate.kind, predicate.atLeast];
|
|
1226
|
+
}
|
|
1227
|
+
return unreachablePredicateKind(predicate, 'workflowGraphHash');
|
|
783
1228
|
}
|
|
784
1229
|
function sortedEntries(config) {
|
|
785
1230
|
return Object.keys(config)
|
|
@@ -835,13 +1280,42 @@ function isWorkflowNode(value) {
|
|
|
835
1280
|
config !== null &&
|
|
836
1281
|
!Array.isArray(config));
|
|
837
1282
|
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
1283
|
+
if (kind === 'if') {
|
|
1284
|
+
// The predicate in full, refused rather than defaulted: a gate read back
|
|
1285
|
+
// without a test it recognises would have to invent one, and inventing one
|
|
1286
|
+
// means half the graph runs on a decision nobody made.
|
|
1287
|
+
return isWorkflowIfPredicate(Reflect.get(value, 'predicate'));
|
|
1288
|
+
}
|
|
1289
|
+
if (kind === 'source') {
|
|
1290
|
+
const sourceKind = Reflect.get(value, 'sourceKind');
|
|
1291
|
+
const config = Reflect.get(value, 'config');
|
|
1292
|
+
return isConnectorKind(sourceKind) && typeof config === 'object' && config !== null;
|
|
1293
|
+
}
|
|
1294
|
+
return isWorkflowNodeKindUnhandled(kind);
|
|
1295
|
+
}
|
|
1296
|
+
/**
|
|
1297
|
+
* The narrowing counterpart of {@link unreachableNodeKind}.
|
|
1298
|
+
*
|
|
1299
|
+
* A guard cannot take a `never` — `kind` here is a string that
|
|
1300
|
+
* {@link isWorkflowNodeKind} already accepted — so exhaustiveness is bought by
|
|
1301
|
+
* assigning it to one, which is the compile error a new kind has to answer, and
|
|
1302
|
+
* refusing the value at run time, which is what a build that skipped this file
|
|
1303
|
+
* would do to a node it has no rule for.
|
|
1304
|
+
*/
|
|
1305
|
+
function isWorkflowNodeKindUnhandled(kind) {
|
|
1306
|
+
void kind;
|
|
1307
|
+
return false;
|
|
841
1308
|
}
|
|
842
1309
|
function isWorkflowEdge(value) {
|
|
843
1310
|
if (typeof value !== 'object' || value === null)
|
|
844
1311
|
return false;
|
|
1312
|
+
// A `branch` that is present and unrecognised is refused rather than dropped,
|
|
1313
|
+
// for the reason `isWorkflowNode` refuses an unknown node: an edge silently
|
|
1314
|
+
// read back without its label is a wire that stops belonging to a branch, and
|
|
1315
|
+
// everything behind it stops running with nothing to point at.
|
|
1316
|
+
const branch = Reflect.get(value, 'branch');
|
|
1317
|
+
if (branch !== undefined && !isWorkflowBranchLabel(branch))
|
|
1318
|
+
return false;
|
|
845
1319
|
return (typeof Reflect.get(value, 'from') === 'string' && typeof Reflect.get(value, 'to') === 'string');
|
|
846
1320
|
}
|
|
847
1321
|
/**
|
|
@@ -909,3 +1383,24 @@ function isPipelineStore(store) {
|
|
|
909
1383
|
store !== null &&
|
|
910
1384
|
typeof Reflect.get(store, 'listConnectors') === 'function');
|
|
911
1385
|
}
|
|
1386
|
+
/**
|
|
1387
|
+
* What a redacted password reads as, on the wire.
|
|
1388
|
+
*
|
|
1389
|
+
* Declared here rather than in `@dudousxd/nestjs-catalog-pipeline`, where the
|
|
1390
|
+
* redaction itself lives, because this literal is not an implementation detail
|
|
1391
|
+
* of the redaction: it is part of what `GET pipeline/connections` answers, and
|
|
1392
|
+
* a browser is the audience it was invented for. A form that lets somebody
|
|
1393
|
+
* paste an address has to be able to recognise the string it was shown — a
|
|
1394
|
+
* `url` whose password is exactly this came out of a read, and posting it back
|
|
1395
|
+
* as a NEW connection stores the word "REDACTED" as the password. There is no
|
|
1396
|
+
* stored row behind a create for `restoreRedactedSecrets` to put the real one
|
|
1397
|
+
* back from, so nothing downstream can catch it: the row saves, and the failure
|
|
1398
|
+
* arrives at the first scheduled load as an authentication error against a
|
|
1399
|
+
* password nobody typed.
|
|
1400
|
+
*
|
|
1401
|
+
* The pipeline package re-exports this rather than declaring its own, so the
|
|
1402
|
+
* two halves cannot drift. A fixed literal rather than a run of asterisks, so
|
|
1403
|
+
* it is greppable in a bug report and cannot be mistaken for a password
|
|
1404
|
+
* somebody actually chose.
|
|
1405
|
+
*/
|
|
1406
|
+
exports.REDACTED_SECRET = 'REDACTED';
|