@dudousxd/nestjs-catalog 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.events.d.ts +14 -0
- package/dist/catalog.pipeline.d.ts +1124 -75
- package/dist/catalog.pipeline.js +1150 -14
- package/dist/catalog.stage-encoding.d.ts +185 -0
- package/dist/catalog.stage-encoding.js +314 -0
- package/dist/client.d.ts +14 -4
- package/dist/client.js +85 -12
- package/dist/index.d.ts +2 -1
- package/dist/index.js +48 -3
- package/dist/transform-runner.d.ts +19 -1
- package/dist/transform-runner.js +63 -7
- package/package.json +6 -6
- package/LICENSE +0 -21
package/dist/catalog.pipeline.js
CHANGED
|
@@ -9,15 +9,33 @@
|
|
|
9
9
|
* systems each believing they decide when a load runs.
|
|
10
10
|
*/
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.TRANSFORM_RUNNER = exports.TRANSFORM_LANGUAGES = exports.CONNECTOR_KINDS = void 0;
|
|
12
|
+
exports.REDACTED_SECRET = exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_SKIP_REASONS = exports.CODE_CONTEXT_CONTRACT = exports.TRANSFORM_RUNNER = exports.TRANSFORM_LANGUAGES = exports.CONNECTOR_KINDS = void 0;
|
|
13
13
|
exports.isConnectorKind = isConnectorKind;
|
|
14
14
|
exports.isTransformLanguage = isTransformLanguage;
|
|
15
|
+
exports.isWorkflowSkipReason = isWorkflowSkipReason;
|
|
15
16
|
exports.isWorkflowNodeKind = isWorkflowNodeKind;
|
|
17
|
+
exports.unreachableNodeKind = unreachableNodeKind;
|
|
18
|
+
exports.workflowColumnX = workflowColumnX;
|
|
19
|
+
exports.workflowRowY = workflowRowY;
|
|
20
|
+
exports.isWorkflowPredicateKind = isWorkflowPredicateKind;
|
|
21
|
+
exports.unreachablePredicateKind = unreachablePredicateKind;
|
|
22
|
+
exports.isWorkflowIfPredicate = isWorkflowIfPredicate;
|
|
23
|
+
exports.isWorkflowFilterPredicateKind = isWorkflowFilterPredicateKind;
|
|
24
|
+
exports.unreachableFilterPredicateKind = unreachableFilterPredicateKind;
|
|
25
|
+
exports.isWorkflowFilterOperator = isWorkflowFilterOperator;
|
|
26
|
+
exports.unreachableFilterOperator = unreachableFilterOperator;
|
|
27
|
+
exports.isWorkflowFilterValue = isWorkflowFilterValue;
|
|
28
|
+
exports.isWorkflowFilterPredicate = isWorkflowFilterPredicate;
|
|
29
|
+
exports.workflowFilterMatches = workflowFilterMatches;
|
|
30
|
+
exports.isWorkflowBranchLabel = isWorkflowBranchLabel;
|
|
16
31
|
exports.isWorkflowStatus = isWorkflowStatus;
|
|
17
32
|
exports.isWorkflowExecutionMode = isWorkflowExecutionMode;
|
|
18
33
|
exports.readWorkflowCallOutput = readWorkflowCallOutput;
|
|
34
|
+
exports.callableWorkflowBlock = callableWorkflowBlock;
|
|
19
35
|
exports.validateWorkflow = validateWorkflow;
|
|
36
|
+
exports.workflowNarrowedTypes = workflowNarrowedTypes;
|
|
20
37
|
exports.workflowRunOrder = workflowRunOrder;
|
|
38
|
+
exports.workflowNodeRuns = workflowNodeRuns;
|
|
21
39
|
exports.workflowGraphHash = workflowGraphHash;
|
|
22
40
|
exports.isWorkflowNode = isWorkflowNode;
|
|
23
41
|
exports.isWorkflowEdge = isWorkflowEdge;
|
|
@@ -68,6 +86,39 @@ function isTransformLanguage(value) {
|
|
|
68
86
|
return exports.TRANSFORM_LANGUAGES.some((language) => language === value);
|
|
69
87
|
}
|
|
70
88
|
exports.TRANSFORM_RUNNER = Symbol('TRANSFORM_RUNNER');
|
|
89
|
+
/**
|
|
90
|
+
* The number in {@link CatalogCodeContext.contract}.
|
|
91
|
+
*
|
|
92
|
+
* Its own version, separate from {@link WORKFLOW_CALL_CONTRACT}, because the
|
|
93
|
+
* two travel to different places for different reasons — a call envelope
|
|
94
|
+
* crosses to another SDK over the durable wire, a code context crosses to a
|
|
95
|
+
* child process this repository spawned — and a transform written against one
|
|
96
|
+
* has to be able to say which it read.
|
|
97
|
+
*/
|
|
98
|
+
exports.CODE_CONTEXT_CONTRACT = 1;
|
|
99
|
+
/**
|
|
100
|
+
* Why a `skipped` node did not run, when there is more to say than "the run
|
|
101
|
+
* stopped".
|
|
102
|
+
*
|
|
103
|
+
* **One entry, and the omission is the point.** `skipped` already meant one
|
|
104
|
+
* thing before branches existed — the run failed upstream and never reached
|
|
105
|
+
* this node — and every outcome ever stored records that meaning by *not*
|
|
106
|
+
* carrying a reason. Adding `run-stopped` to this list would not describe those
|
|
107
|
+
* rows, it would leave them describing an unknown reason, so the pre-existing
|
|
108
|
+
* meaning stays the absent one and this names only the new fact: the node is on
|
|
109
|
+
* a branch an {@link WorkflowIfNode} did not take.
|
|
110
|
+
*
|
|
111
|
+
* The distinction is not cosmetic. A sink skipped by a branch **committed
|
|
112
|
+
* nothing and left the live snapshot alone**, which is a correct, successful
|
|
113
|
+
* outcome; a sink skipped by a failure is part of a load that went wrong. A run
|
|
114
|
+
* panel that rendered both as "did not run" would answer "why is there no data
|
|
115
|
+
* in X" with the same shrug in both cases.
|
|
116
|
+
*/
|
|
117
|
+
exports.WORKFLOW_SKIP_REASONS = ['branch-not-taken'];
|
|
118
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
119
|
+
function isWorkflowSkipReason(value) {
|
|
120
|
+
return exports.WORKFLOW_SKIP_REASONS.some((reason) => reason === value);
|
|
121
|
+
}
|
|
71
122
|
/* ---------------------------------------------------------------------------
|
|
72
123
|
* Workflows: a graph of steps that ends in exactly one commit.
|
|
73
124
|
*
|
|
@@ -104,13 +155,28 @@ exports.TRANSFORM_RUNNER = Symbol('TRANSFORM_RUNNER');
|
|
|
104
155
|
* The kinds that were considered and rejected, since a small vocabulary is only
|
|
105
156
|
* defensible if the omissions are:
|
|
106
157
|
*
|
|
107
|
-
* - **filter** —
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
* missing.
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
158
|
+
* - **filter** — *this entry used to be a rejection, and it is worth leaving the
|
|
159
|
+
* reversal visible rather than editing the history out.* The argument was that
|
|
160
|
+
* a transform whose code returns a subset already filters, so the kind bought
|
|
161
|
+
* a second way to drop rows and a second place to look when rows went missing.
|
|
162
|
+
* That argument is sound about *code* and it is the reason
|
|
163
|
+
* {@link WorkflowFilterNode} does not take any: what changed is that the
|
|
164
|
+
* predicate is a **closed structure** rather than a body, and a closed
|
|
165
|
+
* structure can be read by something other than a JavaScript engine. Only a
|
|
166
|
+
* declarative predicate can be translated into a `WHERE` and pushed into the
|
|
167
|
+
* query the source already runs, and that is not a micro-optimisation — a
|
|
168
|
+
* transform filtering `obj_pribuybuylistdetail` reads all 7,637,391 rows off
|
|
169
|
+
* disk, over the network, and into JS objects of ~80 properties each before
|
|
170
|
+
* anything decides they were unwanted. See {@link WorkflowFilterNode} for what
|
|
171
|
+
* is actually built today (an in-memory, per-batch pass) and for where the
|
|
172
|
+
* pushdown seam is, which is a promise about a shape rather than a claim about
|
|
173
|
+
* a measurement.
|
|
174
|
+
* - **branch / split (unconditional)** — already expressible, and still is: a
|
|
175
|
+
* node with two outbound edges is read by both successors, each of which
|
|
176
|
+
* filters differently. There is nothing for an *unconditional* split to do.
|
|
177
|
+
* {@link WorkflowIfNode} is the conditional one, and it earns its kind by
|
|
178
|
+
* doing something no wiring can express — deciding that one of those
|
|
179
|
+
* successors, and everything only it feeds, does not run at all.
|
|
114
180
|
* - **merge / join** — a node with several inbound edges receives its inputs
|
|
115
181
|
* concatenated in edge order (see {@link WorkflowEdge}). A keyed join is then
|
|
116
182
|
* ordinary code inside the transform, which can already see every record.
|
|
@@ -136,11 +202,40 @@ exports.WORKFLOW_NODE_KINDS = [
|
|
|
136
202
|
'sink',
|
|
137
203
|
/** Hands this position to an existing durable workflow. See {@link WorkflowCallNode}. */
|
|
138
204
|
'call',
|
|
205
|
+
/** Sends the rows down one of its outbound branches. See {@link WorkflowIfNode}. */
|
|
206
|
+
'if',
|
|
207
|
+
/** Drops the rows that fail a declarative test. See {@link WorkflowFilterNode}. */
|
|
208
|
+
'filter',
|
|
139
209
|
];
|
|
140
210
|
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
141
211
|
function isWorkflowNodeKind(value) {
|
|
142
212
|
return exports.WORKFLOW_NODE_KINDS.some((kind) => kind === value);
|
|
143
213
|
}
|
|
214
|
+
/**
|
|
215
|
+
* The kind that never compiles quietly.
|
|
216
|
+
*
|
|
217
|
+
* Every place that decides something *per kind* ends in a call to this, so a
|
|
218
|
+
* kind added to {@link WORKFLOW_NODE_KINDS} without a branch there is a type
|
|
219
|
+
* error naming the file rather than a graph that saves, validates, draws and
|
|
220
|
+
* then does the wrong thing. This codebase has been bitten by exactly that
|
|
221
|
+
* shape — a `toGraph` branch forgetting a field, a node-kind map missing a kind
|
|
222
|
+
* — and the fix each time was to make the omission impossible rather than to
|
|
223
|
+
* remember harder.
|
|
224
|
+
*
|
|
225
|
+
* It throws as well as failing to compile, because the narrowing that reaches
|
|
226
|
+
* it is over data that arrives as JSON: a node whose `kind` passed
|
|
227
|
+
* {@link isWorkflowNodeKind} in an older build and reaches a newer one is
|
|
228
|
+
* possible, and returning a default for it would be the silent path this exists
|
|
229
|
+
* to close.
|
|
230
|
+
*/
|
|
231
|
+
function unreachableNodeKind(node, where) {
|
|
232
|
+
// Takes either the node or its kind, because the call sites differ: a
|
|
233
|
+
// narrowing chain over a union hands it the node, while one over the kind
|
|
234
|
+
// string — which is what a boundary reading JSON has before it has a node —
|
|
235
|
+
// hands it the string.
|
|
236
|
+
const kind = typeof node === 'string' ? node : Reflect.get(Object(node), 'kind');
|
|
237
|
+
throw new Error(`${where} does not handle a workflow node of kind ${JSON.stringify(kind)}. The kind list and every decision made per kind are meant to move together.`);
|
|
238
|
+
}
|
|
144
239
|
/**
|
|
145
240
|
* The longest a node id may be, and the alphabet it may use.
|
|
146
241
|
*
|
|
@@ -152,6 +247,549 @@ function isWorkflowNodeKind(value) {
|
|
|
152
247
|
* separator would let one node read another's rows.
|
|
153
248
|
*/
|
|
154
249
|
exports.WORKFLOW_NODE_ID_PATTERN = /^[A-Za-z0-9_-]{1,64}$/;
|
|
250
|
+
/**
|
|
251
|
+
* How big a node is on the canvas, and therefore how far apart two of them have
|
|
252
|
+
* to be.
|
|
253
|
+
*
|
|
254
|
+
* ## Why the server owns a number about pixels
|
|
255
|
+
*
|
|
256
|
+
* Because the server *writes positions*. `adoptConnector` lays a connector out
|
|
257
|
+
* as a graph, and anything else that mints a graph without a person drawing it —
|
|
258
|
+
* a promotion, a template, a script — has the same job. A writer that does not
|
|
259
|
+
* know how wide a node is can only guess, and the guess was wrong: adoption used
|
|
260
|
+
* to place columns 220 apart against a node 224 wide, so every adopted graph
|
|
261
|
+
* drew each box overlapping the next by four pixels. Nothing was stacked and
|
|
262
|
+
* nothing was missing, so it read as boxes glued together rather than as a bug,
|
|
263
|
+
* and it survived until somebody opened thirteen of them.
|
|
264
|
+
*
|
|
265
|
+
* The fix is not a bigger number. 220 was not too small, it was **derived from
|
|
266
|
+
* nothing** — it had no relationship to the width it was supposed to clear, so
|
|
267
|
+
* it was only ever correct by luck and would go wrong again the next time the
|
|
268
|
+
* node's styling changed. These constants are the relationship, stated once, in
|
|
269
|
+
* the one package both the writer and the canvas already depend on.
|
|
270
|
+
*
|
|
271
|
+
* ## What pins them to the drawing
|
|
272
|
+
*
|
|
273
|
+
* {@link WORKFLOW_NODE_WIDTH} is not a description of the node — it is the
|
|
274
|
+
* *source of* the node's width. `WorkflowNodeBody` in
|
|
275
|
+
* `@dudousxd/nestjs-catalog-react` sets its own width from this constant rather
|
|
276
|
+
* than from a Tailwind class, so the two cannot drift: changing this changes the
|
|
277
|
+
* box, and there is no second number to forget.
|
|
278
|
+
*/
|
|
279
|
+
exports.WORKFLOW_NODE_WIDTH = 224;
|
|
280
|
+
/** How tall a node is. The other half of {@link WORKFLOW_NODE_WIDTH}'s contract. */
|
|
281
|
+
exports.WORKFLOW_NODE_HEIGHT = 80;
|
|
282
|
+
/**
|
|
283
|
+
* Clear space between one column and the next, on top of the node's own width.
|
|
284
|
+
*
|
|
285
|
+
* Wide enough for the edge between two nodes to be read as a line with a
|
|
286
|
+
* direction rather than as a join. This is the part that is taste; the width it
|
|
287
|
+
* is added to is not.
|
|
288
|
+
*/
|
|
289
|
+
exports.WORKFLOW_COLUMN_GAP = 96;
|
|
290
|
+
/** Clear space between two nodes sharing a column. */
|
|
291
|
+
exports.WORKFLOW_ROW_GAP = 32;
|
|
292
|
+
/**
|
|
293
|
+
* The x of the nth column, counting from zero.
|
|
294
|
+
*
|
|
295
|
+
* Every generator of a layout goes through this rather than multiplying by a
|
|
296
|
+
* literal, which is what makes "columns never overlap" a property of one
|
|
297
|
+
* function instead of a coincidence repeated at each call site.
|
|
298
|
+
*/
|
|
299
|
+
function workflowColumnX(column) {
|
|
300
|
+
return column * (exports.WORKFLOW_NODE_WIDTH + exports.WORKFLOW_COLUMN_GAP);
|
|
301
|
+
}
|
|
302
|
+
/** The y of the nth row within a column, counting from zero. */
|
|
303
|
+
function workflowRowY(row) {
|
|
304
|
+
return row * (exports.WORKFLOW_NODE_HEIGHT + exports.WORKFLOW_ROW_GAP);
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* The kinds of test an {@link WorkflowIfNode} can make.
|
|
308
|
+
*
|
|
309
|
+
* A second predicate shape was always going to arrive — the note on
|
|
310
|
+
* {@link WorkflowIfNode} says so about `code` — and the shape it arrives into is
|
|
311
|
+
* the decision worth arguing about, because the alternative was to keep both
|
|
312
|
+
* tests' fields flat on the node and mark them optional. That version types a
|
|
313
|
+
* gate as "an env var, maybe, and a threshold, maybe": a node carrying both is
|
|
314
|
+
* representable, a node carrying neither is representable, and every reader has
|
|
315
|
+
* to invent its own rule for which one wins. It is the same mistake the note on
|
|
316
|
+
* {@link WorkflowNode} refuses for node kinds, one level down.
|
|
317
|
+
*
|
|
318
|
+
* So the predicate is a union with a discriminant of its own, and every decision
|
|
319
|
+
* made per predicate kind ends in {@link unreachablePredicateKind} — a third
|
|
320
|
+
* shape is then a build failure listing the files that have to answer for it,
|
|
321
|
+
* rather than a gate that saves, draws, and quietly always takes the `else`.
|
|
322
|
+
*/
|
|
323
|
+
exports.WORKFLOW_PREDICATE_KINDS = [
|
|
324
|
+
/** Reads a variable where the load runs. See {@link WorkflowEnvPredicate}. */
|
|
325
|
+
'env',
|
|
326
|
+
/** Counts the rows handed to the gate. See {@link WorkflowRowCountPredicate}. */
|
|
327
|
+
'rowCount',
|
|
328
|
+
];
|
|
329
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
330
|
+
function isWorkflowPredicateKind(value) {
|
|
331
|
+
return exports.WORKFLOW_PREDICATE_KINDS.some((kind) => kind === value);
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* {@link unreachableNodeKind}, one level down, and for the identical reason.
|
|
335
|
+
*
|
|
336
|
+
* Every branch over {@link WorkflowIfPredicate} ends here, so a predicate kind
|
|
337
|
+
* added to the list without a rule for hashing it, validating it or evaluating
|
|
338
|
+
* it is a type error naming the file — not a graph that runs and decides
|
|
339
|
+
* something nobody authored. It throws as well, because predicates arrive as
|
|
340
|
+
* JSON out of a column and a build older than the data is a thing that happens.
|
|
341
|
+
*/
|
|
342
|
+
function unreachablePredicateKind(predicate, where) {
|
|
343
|
+
const kind = typeof predicate === 'string' ? predicate : Reflect.get(Object(predicate), 'kind');
|
|
344
|
+
throw new Error(`${where} does not handle an if-node predicate of kind ${JSON.stringify(kind)}. The predicate kinds and every decision made per kind are meant to move together.`);
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Narrow a stored predicate, for the same reason {@link isWorkflowNode} narrows
|
|
348
|
+
* a stored node: it arrives as JSON out of a column, and a gate read back
|
|
349
|
+
* without its test is a gate that has to invent one.
|
|
350
|
+
*
|
|
351
|
+
* A row count that is not a whole number — `NaN` from a JSON round trip of an
|
|
352
|
+
* unparsed field, an `Infinity` that serialised as `null` — is refused rather
|
|
353
|
+
* than kept, because every comparison against it is false and the symptom is a
|
|
354
|
+
* `then` branch that silently never runs again.
|
|
355
|
+
*/
|
|
356
|
+
function isWorkflowIfPredicate(value) {
|
|
357
|
+
if (typeof value !== 'object' || value === null)
|
|
358
|
+
return false;
|
|
359
|
+
const kind = Reflect.get(value, 'kind');
|
|
360
|
+
if (!isWorkflowPredicateKind(kind))
|
|
361
|
+
return false;
|
|
362
|
+
if (kind === 'env') {
|
|
363
|
+
// The variable name is required and its expected value is not, exactly as
|
|
364
|
+
// the type says: an absent `equals` is the "is it set at all" test, so a
|
|
365
|
+
// stored predicate without one is complete rather than half-narrowed.
|
|
366
|
+
const equals = Reflect.get(value, 'equals');
|
|
367
|
+
return (typeof Reflect.get(value, 'envVar') === 'string' &&
|
|
368
|
+
(equals === undefined || typeof equals === 'string'));
|
|
369
|
+
}
|
|
370
|
+
if (kind === 'rowCount') {
|
|
371
|
+
const atLeast = Reflect.get(value, 'atLeast');
|
|
372
|
+
return typeof atLeast === 'number' && Number.isInteger(atLeast) && atLeast >= 0;
|
|
373
|
+
}
|
|
374
|
+
return isWorkflowPredicateKindUnhandled(kind);
|
|
375
|
+
}
|
|
376
|
+
/** The narrowing counterpart of {@link unreachablePredicateKind}. */
|
|
377
|
+
function isWorkflowPredicateKindUnhandled(kind) {
|
|
378
|
+
void kind;
|
|
379
|
+
return false;
|
|
380
|
+
}
|
|
381
|
+
/* ---------------------------------------------------------------------------
|
|
382
|
+
* The filter node, and the predicate language it drops rows by.
|
|
383
|
+
* ------------------------------------------------------------------------- */
|
|
384
|
+
/**
|
|
385
|
+
* The shapes a {@link WorkflowFilterPredicate} can take.
|
|
386
|
+
*
|
|
387
|
+
* Two leaf kinds plus a presence test plus two ways to combine them, and the
|
|
388
|
+
* list is closed for the same reason {@link WORKFLOW_PREDICATE_KINDS} is: every
|
|
389
|
+
* decision made per kind ends in {@link unreachableFilterPredicateKind}, so a
|
|
390
|
+
* sixth shape is a build failure naming the files that owe it an answer rather
|
|
391
|
+
* than a filter that saves, draws, and quietly keeps everything.
|
|
392
|
+
*
|
|
393
|
+
* **There is deliberately no `not`.** Every leaf carries its own inverse
|
|
394
|
+
* ({@link WORKFLOW_FILTER_OPERATORS} pairs each operator with one, `oneOf` has
|
|
395
|
+
* `notIn`, `present` has `isNotNull`) and `all`/`any` are duals, so De Morgan
|
|
396
|
+
* already writes any negation with the kinds here. A `not` node would be a
|
|
397
|
+
* second spelling of every predicate — two shapes to read when a load comes out
|
|
398
|
+
* short, and a UI with a checkbox nobody agrees on the placement of. The one
|
|
399
|
+
* case it does not cover is stated on {@link workflowFilterMatches}: a value the
|
|
400
|
+
* test cannot compare fails *both* a form and its inverse, on purpose.
|
|
401
|
+
*
|
|
402
|
+
* **There is deliberately no free-form expression and no code.** That is the
|
|
403
|
+
* whole argument for the node existing — see {@link WorkflowFilterNode}.
|
|
404
|
+
*/
|
|
405
|
+
exports.WORKFLOW_FILTER_PREDICATE_KINDS = [
|
|
406
|
+
/** One column against one value. See {@link WorkflowFilterComparison}. */
|
|
407
|
+
'compare',
|
|
408
|
+
/** One column against a list. See {@link WorkflowFilterOneOf}. */
|
|
409
|
+
'oneOf',
|
|
410
|
+
/** Whether a column has a value at all. See {@link WorkflowFilterPresence}. */
|
|
411
|
+
'present',
|
|
412
|
+
/** Every child holds. See {@link WorkflowFilterGroup}. */
|
|
413
|
+
'all',
|
|
414
|
+
/** At least one child holds. See {@link WorkflowFilterGroup}. */
|
|
415
|
+
'any',
|
|
416
|
+
];
|
|
417
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
418
|
+
function isWorkflowFilterPredicateKind(value) {
|
|
419
|
+
return exports.WORKFLOW_FILTER_PREDICATE_KINDS.some((kind) => kind === value);
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* {@link unreachablePredicateKind}, for the filter language, and for the
|
|
423
|
+
* identical reason.
|
|
424
|
+
*
|
|
425
|
+
* It throws as well as failing to compile: a predicate arrives as JSON out of a
|
|
426
|
+
* column, so a graph saved by a newer build and read by an older one is a thing
|
|
427
|
+
* that happens, and returning a default for a shape this build has no rule for
|
|
428
|
+
* would mean silently keeping — or silently dropping — every row.
|
|
429
|
+
*/
|
|
430
|
+
function unreachableFilterPredicateKind(predicate, where) {
|
|
431
|
+
const kind = typeof predicate === 'string' ? predicate : Reflect.get(Object(predicate), 'kind');
|
|
432
|
+
throw new Error(`${where} does not handle a filter predicate of kind ${JSON.stringify(kind)}. The predicate kinds and every decision made per kind are meant to move together.`);
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* What a {@link WorkflowFilterComparison} does with its column and its value.
|
|
436
|
+
*
|
|
437
|
+
* Ten, in five inverse pairs, and the pairing is the reason there is no `not`
|
|
438
|
+
* kind. Each one has an obvious single-expression form in both dialects this
|
|
439
|
+
* repository speaks, which is not a coincidence — it is the constraint the list
|
|
440
|
+
* was chosen under, so that the day a predicate is pushed into a source query
|
|
441
|
+
* the translation is a `switch` and not a design.
|
|
442
|
+
*
|
|
443
|
+
* The omissions, since a closed list is only defensible if they are:
|
|
444
|
+
*
|
|
445
|
+
* - **`between`** — `all` of a `greaterThanOrEqual` and a `lessThanOrEqual`, in
|
|
446
|
+
* one more click and with no second shape to validate, hash and translate.
|
|
447
|
+
* - **`endsWith`** — asked for far less than the other two, and unlike them it
|
|
448
|
+
* has no index that could ever serve it in either dialect, so offering it in a
|
|
449
|
+
* palette would advertise something that is a full scan by construction.
|
|
450
|
+
* `contains` covers it at the same cost.
|
|
451
|
+
* - **regular expressions** — the dialects disagree about the syntax, the
|
|
452
|
+
* engines disagree about the semantics, and a predicate whose meaning depends
|
|
453
|
+
* on which database answered it is a predicate that cannot be pushed down
|
|
454
|
+
* without changing what the load returns. That is the one property this list
|
|
455
|
+
* exists to hold.
|
|
456
|
+
* - **case-insensitive variants** — collation is a property of the column in
|
|
457
|
+
* both dialects, so a `equalsIgnoreCase` evaluated in memory and the same
|
|
458
|
+
* predicate evaluated in a `WHERE` would legitimately disagree. Normalise in a
|
|
459
|
+
* transform, where the disagreement is visible.
|
|
460
|
+
*/
|
|
461
|
+
exports.WORKFLOW_FILTER_OPERATORS = [
|
|
462
|
+
'equals',
|
|
463
|
+
'notEquals',
|
|
464
|
+
'greaterThan',
|
|
465
|
+
'lessThanOrEqual',
|
|
466
|
+
'greaterThanOrEqual',
|
|
467
|
+
'lessThan',
|
|
468
|
+
'contains',
|
|
469
|
+
'notContains',
|
|
470
|
+
'startsWith',
|
|
471
|
+
'notStartsWith',
|
|
472
|
+
];
|
|
473
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
474
|
+
function isWorkflowFilterOperator(value) {
|
|
475
|
+
return exports.WORKFLOW_FILTER_OPERATORS.some((operator) => operator === value);
|
|
476
|
+
}
|
|
477
|
+
/**
|
|
478
|
+
* {@link unreachableFilterPredicateKind}, one level further down.
|
|
479
|
+
*
|
|
480
|
+
* An operator added to the list without a rule for evaluating it, describing it
|
|
481
|
+
* or hashing it is a type error naming the file — not a comparison that silently
|
|
482
|
+
* answers false for every row, which is a filter that drops the whole dataset
|
|
483
|
+
* and reports success.
|
|
484
|
+
*/
|
|
485
|
+
function unreachableFilterOperator(operator, where) {
|
|
486
|
+
throw new Error(`${where} does not handle the filter operator ${JSON.stringify(operator)}. The operator list and every decision made per operator are meant to move together.`);
|
|
487
|
+
}
|
|
488
|
+
/** Whether a stored value is one this language can compare against. */
|
|
489
|
+
function isWorkflowFilterValue(value) {
|
|
490
|
+
if (typeof value === 'string' || typeof value === 'boolean')
|
|
491
|
+
return true;
|
|
492
|
+
// `NaN` and the infinities are refused rather than kept: every comparison
|
|
493
|
+
// against `NaN` is false, so a predicate holding one is a filter that drops
|
|
494
|
+
// every row while looking perfectly well configured, and `Infinity` does not
|
|
495
|
+
// survive a JSON round trip at all — it comes back as `null`.
|
|
496
|
+
return typeof value === 'number' && Number.isFinite(value);
|
|
497
|
+
}
|
|
498
|
+
/**
|
|
499
|
+
* The column names a predicate may name.
|
|
500
|
+
*
|
|
501
|
+
* **The same pattern `boundStatement` requires of a watermark column**, and that
|
|
502
|
+
* is the point rather than a coincidence: an identifier cannot be bound by any
|
|
503
|
+
* driver, so pushing a predicate into a query means quoting the column into the
|
|
504
|
+
* SQL, and a name carrying a quote, a dot or a space is refused rather than
|
|
505
|
+
* escaped. Requiring it *now*, while the predicate is only ever evaluated in
|
|
506
|
+
* memory, is what stops a graph being authored today that could never be pushed
|
|
507
|
+
* down tomorrow.
|
|
508
|
+
*
|
|
509
|
+
* The cost, and it is real: a source whose column is called `Part Number` cannot
|
|
510
|
+
* be filtered directly. Rename it in a transform first — which is a node that
|
|
511
|
+
* already exists and whose output is a column this can name.
|
|
512
|
+
*/
|
|
513
|
+
exports.WORKFLOW_FILTER_COLUMN_PATTERN = /^[A-Za-z_][A-Za-z0-9_$]*$/;
|
|
514
|
+
/**
|
|
515
|
+
* How deep `all`/`any` may nest.
|
|
516
|
+
*
|
|
517
|
+
* A bound rather than a trust, because a predicate arrives as JSON out of a
|
|
518
|
+
* column and every function that walks one is recursive: a graph carrying a
|
|
519
|
+
* thousand-deep tree would be a stack overflow inside a durable step rather than
|
|
520
|
+
* a refusal naming the node. Six is past anything a person builds in a form and
|
|
521
|
+
* far short of anything that costs a frame to walk.
|
|
522
|
+
*/
|
|
523
|
+
exports.WORKFLOW_FILTER_MAX_DEPTH = 6;
|
|
524
|
+
/**
|
|
525
|
+
* How many values one {@link WorkflowFilterOneOf} may list.
|
|
526
|
+
*
|
|
527
|
+
* Bounded because the list travels in the graph, into the graph fingerprint, and
|
|
528
|
+
* eventually into an `IN (...)` — and past a few hundred entries all three of
|
|
529
|
+
* those stop being reasonable and the thing being expressed is a join against
|
|
530
|
+
* another dataset rather than a filter. Wire that dataset in as a second source
|
|
531
|
+
* and join it in a transform, which is the node that can.
|
|
532
|
+
*/
|
|
533
|
+
exports.WORKFLOW_FILTER_MAX_VALUES = 500;
|
|
534
|
+
/**
|
|
535
|
+
* Narrow a stored filter predicate, refusing rather than repairing.
|
|
536
|
+
*
|
|
537
|
+
* The same contract {@link isWorkflowIfPredicate} has and for a sharper version
|
|
538
|
+
* of the same reason: a gate read back without its test picks a branch nobody
|
|
539
|
+
* authored, and a *filter* read back without its test decides which rows exist.
|
|
540
|
+
* Repairing a broken predicate — dropping an unreadable child out of an `all`,
|
|
541
|
+
* say — would silently widen or narrow what a load publishes, which is precisely
|
|
542
|
+
* the failure the node is built to make visible.
|
|
543
|
+
*
|
|
544
|
+
* Depth is carried rather than tracked globally so that the bound is on the
|
|
545
|
+
* *tree*, not on how many predicates have been checked: two sibling branches
|
|
546
|
+
* five deep are fine, and one branch seven deep is not.
|
|
547
|
+
*/
|
|
548
|
+
function isWorkflowFilterPredicate(value, depth = 0) {
|
|
549
|
+
if (depth > exports.WORKFLOW_FILTER_MAX_DEPTH)
|
|
550
|
+
return false;
|
|
551
|
+
if (typeof value !== 'object' || value === null)
|
|
552
|
+
return false;
|
|
553
|
+
const kind = Reflect.get(value, 'kind');
|
|
554
|
+
if (!isWorkflowFilterPredicateKind(kind))
|
|
555
|
+
return false;
|
|
556
|
+
if (kind === 'all' || kind === 'any') {
|
|
557
|
+
const children = Reflect.get(value, 'children');
|
|
558
|
+
if (!Array.isArray(children) || children.length === 0)
|
|
559
|
+
return false;
|
|
560
|
+
return children.every((child) => isWorkflowFilterPredicate(child, depth + 1));
|
|
561
|
+
}
|
|
562
|
+
const column = Reflect.get(value, 'column');
|
|
563
|
+
if (typeof column !== 'string' || !exports.WORKFLOW_FILTER_COLUMN_PATTERN.test(column))
|
|
564
|
+
return false;
|
|
565
|
+
return isFilterLeaf(value, kind);
|
|
566
|
+
}
|
|
567
|
+
/**
|
|
568
|
+
* The half of {@link isWorkflowFilterPredicate} that is about one condition.
|
|
569
|
+
*
|
|
570
|
+
* Split off because the two halves have nothing to do with each other: above is
|
|
571
|
+
* a tree walk with a depth bound, and this is three shapes checked field by
|
|
572
|
+
* field. Reached only with the column already accepted, which is why it does not
|
|
573
|
+
* check one.
|
|
574
|
+
*/
|
|
575
|
+
function isFilterLeaf(value, kind) {
|
|
576
|
+
const operator = Reflect.get(value, 'operator');
|
|
577
|
+
if (kind === 'present') {
|
|
578
|
+
return operator === 'isNull' || operator === 'isNotNull';
|
|
579
|
+
}
|
|
580
|
+
if (kind === 'oneOf') {
|
|
581
|
+
const values = Reflect.get(value, 'values');
|
|
582
|
+
if (operator !== 'in' && operator !== 'notIn')
|
|
583
|
+
return false;
|
|
584
|
+
return (Array.isArray(values) &&
|
|
585
|
+
values.length > 0 &&
|
|
586
|
+
values.length <= exports.WORKFLOW_FILTER_MAX_VALUES &&
|
|
587
|
+
values.every((entry) => isWorkflowFilterValue(entry)));
|
|
588
|
+
}
|
|
589
|
+
if (kind === 'compare') {
|
|
590
|
+
return isWorkflowFilterOperator(operator) && isWorkflowFilterValue(Reflect.get(value, 'value'));
|
|
591
|
+
}
|
|
592
|
+
return isWorkflowFilterPredicateKindUnhandled(kind);
|
|
593
|
+
}
|
|
594
|
+
/** The narrowing counterpart of {@link unreachableFilterPredicateKind}. */
|
|
595
|
+
function isWorkflowFilterPredicateKindUnhandled(kind) {
|
|
596
|
+
void kind;
|
|
597
|
+
return false;
|
|
598
|
+
}
|
|
599
|
+
/**
|
|
600
|
+
* Whether one row passes a predicate.
|
|
601
|
+
*
|
|
602
|
+
* Pure, allocation-free on the common path, and in core rather than in the
|
|
603
|
+
* runner so that the console can describe — and one day preview — exactly what
|
|
604
|
+
* the load will do, from the same function that does it. It is called once per
|
|
605
|
+
* row, so everything about it is arranged to be cheap: no closures built per
|
|
606
|
+
* row, no array built per row, and `note` is optional so a caller that does not
|
|
607
|
+
* want the diagnostics pays nothing for them.
|
|
608
|
+
*
|
|
609
|
+
* ## Null is unknown, and unknown does not pass
|
|
610
|
+
*
|
|
611
|
+
* A column that is `null`, `undefined`, or simply absent from the row fails
|
|
612
|
+
* **every** `compare` and **every** `oneOf`, *including the negative forms*.
|
|
613
|
+
* `status notEquals "CLOSED"` does not pass a row with no status.
|
|
614
|
+
*
|
|
615
|
+
* That is SQL's three-valued logic rather than JavaScript's, and it is chosen
|
|
616
|
+
* deliberately over the more intuitive JS answer for one reason: this predicate
|
|
617
|
+
* is meant to become a `WHERE`, and the day it does, the database will answer
|
|
618
|
+
* this way. A filter that kept those rows in memory and dropped them once pushed
|
|
619
|
+
* down would be a performance change that quietly altered what a type contains —
|
|
620
|
+
* which is the one thing a pushdown must never be able to do. {@link
|
|
621
|
+
* WorkflowFilterPresence} is how absence is tested on purpose.
|
|
622
|
+
*
|
|
623
|
+
* ## A value it cannot compare fails both a test and its inverse
|
|
624
|
+
*
|
|
625
|
+
* `qty greaterThan 10` against a `qty` holding `"n/a"` is not false because the
|
|
626
|
+
* string is small; there is no ordering between a string and a number that any
|
|
627
|
+
* two systems would agree on. Rather than invent one, the leaf answers false and
|
|
628
|
+
* calls `note` with the column, and the runner turns those counts into a line on
|
|
629
|
+
* the run: *"12,431 rows held a value in `qty` the test could not compare."*
|
|
630
|
+
*
|
|
631
|
+
* A row nothing can judge is therefore dropped rather than kept. That is the
|
|
632
|
+
* direction with a backstop — the sink's row-count bound sees the shrink, and a
|
|
633
|
+
* full sink refuses to commit nothing at all — whereas keeping unjudged rows
|
|
634
|
+
* would publish them into the type with nothing anywhere to notice.
|
|
635
|
+
*/
|
|
636
|
+
function workflowFilterMatches(predicate, row, note) {
|
|
637
|
+
if (predicate.kind === 'all' || predicate.kind === 'any') {
|
|
638
|
+
// One loop for both, because they differ only in which answer is decisive:
|
|
639
|
+
// an `all` stops at the first false and an `any` stops at the first true.
|
|
640
|
+
//
|
|
641
|
+
// **Short-circuiting is why `note` may under-count**, and that is the right
|
|
642
|
+
// trade rather than an accident: an `any` that matched on its first child
|
|
643
|
+
// never looks at the second, so a value it could not have compared there is
|
|
644
|
+
// not reported. Evaluating every child to make the diagnostics complete
|
|
645
|
+
// would run every comparison against every row for a count nobody acts on
|
|
646
|
+
// per row — and this function is called seven million times.
|
|
647
|
+
const decisive = predicate.kind === 'any';
|
|
648
|
+
for (const child of predicate.children) {
|
|
649
|
+
if (workflowFilterMatches(child, row, note) === decisive)
|
|
650
|
+
return decisive;
|
|
651
|
+
}
|
|
652
|
+
return !decisive;
|
|
653
|
+
}
|
|
654
|
+
return matchFilterLeaf(predicate, row, note);
|
|
655
|
+
}
|
|
656
|
+
/** One condition against one row. Split from the recursion above, which is all it is. */
|
|
657
|
+
function matchFilterLeaf(predicate, row, note) {
|
|
658
|
+
// `Object.hasOwn` rather than a bare index, because `row` is somebody else's
|
|
659
|
+
// record and a column named `constructor` would otherwise reach up the
|
|
660
|
+
// prototype chain and compare against a function.
|
|
661
|
+
const held = Object.hasOwn(row, predicate.column)
|
|
662
|
+
? Reflect.get(row, predicate.column)
|
|
663
|
+
: undefined;
|
|
664
|
+
if (predicate.kind === 'present') {
|
|
665
|
+
const missing = held === null || held === undefined;
|
|
666
|
+
return predicate.operator === 'isNull' ? missing : !missing;
|
|
667
|
+
}
|
|
668
|
+
// Three-valued logic — see the docblock above. Not noted, because an absent
|
|
669
|
+
// value is an ordinary fact about data rather than a predicate that does not
|
|
670
|
+
// fit it.
|
|
671
|
+
if (held === null || held === undefined)
|
|
672
|
+
return false;
|
|
673
|
+
if (predicate.kind === 'oneOf')
|
|
674
|
+
return matchFilterList(predicate, held, note);
|
|
675
|
+
if (predicate.kind === 'compare')
|
|
676
|
+
return compareFilterValue(predicate, held, note);
|
|
677
|
+
return unreachableFilterPredicateKind(predicate, 'workflowFilterMatches');
|
|
678
|
+
}
|
|
679
|
+
/** One `IN`/`NOT IN`, over a value already known to be present. */
|
|
680
|
+
function matchFilterList(predicate, held, note) {
|
|
681
|
+
if (!isWorkflowFilterValue(held)) {
|
|
682
|
+
note?.(predicate.column);
|
|
683
|
+
return false;
|
|
684
|
+
}
|
|
685
|
+
const found = predicate.values.includes(held);
|
|
686
|
+
return predicate.operator === 'in' ? found : !found;
|
|
687
|
+
}
|
|
688
|
+
/**
|
|
689
|
+
* One leaf comparison, with the type rules written out once.
|
|
690
|
+
*
|
|
691
|
+
* Split from {@link workflowFilterMatches} so the recursion above stays readable
|
|
692
|
+
* and so the ten operators are answered in one place that ends in
|
|
693
|
+
* {@link unreachableFilterOperator}.
|
|
694
|
+
*/
|
|
695
|
+
function compareFilterValue(predicate, held, note) {
|
|
696
|
+
const operator = predicate.operator;
|
|
697
|
+
const wanted = predicate.value;
|
|
698
|
+
if (operator === 'equals' || operator === 'notEquals') {
|
|
699
|
+
// Same type or nothing. `"5" === 5` is false in JavaScript and `'5' = 5` is
|
|
700
|
+
// *true* in MySQL, so a cross-type equality is precisely a comparison whose
|
|
701
|
+
// answer would change under pushdown — reported rather than picked.
|
|
702
|
+
if (typeof held !== typeof wanted) {
|
|
703
|
+
note?.(predicate.column);
|
|
704
|
+
return false;
|
|
705
|
+
}
|
|
706
|
+
return operator === 'equals' ? held === wanted : held !== wanted;
|
|
707
|
+
}
|
|
708
|
+
if (operator === 'greaterThan' ||
|
|
709
|
+
operator === 'greaterThanOrEqual' ||
|
|
710
|
+
operator === 'lessThan' ||
|
|
711
|
+
operator === 'lessThanOrEqual') {
|
|
712
|
+
// Ordered types only, and both sides the same one. Booleans are excluded on
|
|
713
|
+
// purpose: `true > false` is an answer JavaScript will give and no reader of
|
|
714
|
+
// a filter ever meant to ask for.
|
|
715
|
+
const ordered = (typeof held === 'number' && typeof wanted === 'number') ||
|
|
716
|
+
(typeof held === 'string' && typeof wanted === 'string');
|
|
717
|
+
if (!ordered) {
|
|
718
|
+
note?.(predicate.column);
|
|
719
|
+
return false;
|
|
720
|
+
}
|
|
721
|
+
return orderedHolds(operator, held, wanted);
|
|
722
|
+
}
|
|
723
|
+
// The four string tests. A non-string on either side is not coerced: coercing
|
|
724
|
+
// would make `contains` match the digits of a number, which is a full scan
|
|
725
|
+
// producing rows nobody asked for rather than a comparison.
|
|
726
|
+
if (typeof held !== 'string' || typeof wanted !== 'string') {
|
|
727
|
+
note?.(predicate.column);
|
|
728
|
+
return false;
|
|
729
|
+
}
|
|
730
|
+
return textHolds(operator, held, wanted);
|
|
731
|
+
}
|
|
732
|
+
/**
|
|
733
|
+
* The four string tests, over two values already known to be strings.
|
|
734
|
+
*
|
|
735
|
+
* Split out for the same reason {@link orderedHolds} is: the type rule and the
|
|
736
|
+
* comparison are separate concerns, and keeping them in one function put a
|
|
737
|
+
* chain of ten operators and three type rules into one place the linter would
|
|
738
|
+
* not hold in one head — which is a fair description of how a comparison ends up
|
|
739
|
+
* silently answering the wrong way.
|
|
740
|
+
*/
|
|
741
|
+
function textHolds(operator, held, wanted) {
|
|
742
|
+
if (operator === 'contains')
|
|
743
|
+
return held.includes(wanted);
|
|
744
|
+
if (operator === 'notContains')
|
|
745
|
+
return !held.includes(wanted);
|
|
746
|
+
if (operator === 'startsWith')
|
|
747
|
+
return held.startsWith(wanted);
|
|
748
|
+
if (operator === 'notStartsWith')
|
|
749
|
+
return !held.startsWith(wanted);
|
|
750
|
+
return unreachableFilterOperator(operator, 'workflowFilterMatches');
|
|
751
|
+
}
|
|
752
|
+
/**
|
|
753
|
+
* The four orderings, over two values already known to be the same ordered type.
|
|
754
|
+
*
|
|
755
|
+
* Its own function so the type check above and the comparison here are separate
|
|
756
|
+
* concerns rather than one branch doing both. Generic over the pair so that
|
|
757
|
+
* `held` and `wanted` are compared as the *same* type — the signature is what
|
|
758
|
+
* stops a future edit passing a string and a number to `>`, which is the exact
|
|
759
|
+
* silent coercion the caller went to trouble to rule out.
|
|
760
|
+
*/
|
|
761
|
+
function orderedHolds(operator, held, wanted) {
|
|
762
|
+
if (operator === 'greaterThan')
|
|
763
|
+
return held > wanted;
|
|
764
|
+
if (operator === 'greaterThanOrEqual')
|
|
765
|
+
return held >= wanted;
|
|
766
|
+
if (operator === 'lessThan')
|
|
767
|
+
return held < wanted;
|
|
768
|
+
return held <= wanted;
|
|
769
|
+
}
|
|
770
|
+
/**
|
|
771
|
+
* Which side of an {@link WorkflowIfNode} a wire leaves by.
|
|
772
|
+
*
|
|
773
|
+
* **Two closed values rather than free-form labels**, which was the other design
|
|
774
|
+
* and is worse in the one way that matters here: an unlabelled branch is a
|
|
775
|
+
* branch that never runs, and a *misspelled* branch is one too. With a free
|
|
776
|
+
* string, `thn` is a subtree that silently never executes and a graph that
|
|
777
|
+
* validates perfectly; with these, it is refused at the boundary and is a type
|
|
778
|
+
* error in the console. The failure a branch introduces is "nothing loaded and
|
|
779
|
+
* nothing complained", so the vocabulary is the place to make it impossible.
|
|
780
|
+
*
|
|
781
|
+
* An N-way `switch` node was considered and is deliberately not this: it would
|
|
782
|
+
* have to carry its own case list plus a default, the cases would have to be
|
|
783
|
+
* validated against the labels, and an unmatched value would need a rule. That
|
|
784
|
+
* is a different node, and it can be added later without changing this one —
|
|
785
|
+
* because a boolean question is what a predicate answers, and an `if` is exactly
|
|
786
|
+
* the shape of a boolean question.
|
|
787
|
+
*/
|
|
788
|
+
exports.WORKFLOW_BRANCH_LABELS = ['then', 'else'];
|
|
789
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
790
|
+
function isWorkflowBranchLabel(value) {
|
|
791
|
+
return exports.WORKFLOW_BRANCH_LABELS.some((label) => label === value);
|
|
792
|
+
}
|
|
155
793
|
/**
|
|
156
794
|
* An authored graph of steps ending in one commit.
|
|
157
795
|
*
|
|
@@ -299,6 +937,23 @@ function isCount(value) {
|
|
|
299
937
|
function describeCount(value) {
|
|
300
938
|
return value === undefined ? 'nothing' : JSON.stringify(value);
|
|
301
939
|
}
|
|
940
|
+
function callableWorkflowBlock(ref) {
|
|
941
|
+
const version = typeof ref.version === 'string' ? ref.version.trim() : '';
|
|
942
|
+
if (version.length === 0) {
|
|
943
|
+
return {
|
|
944
|
+
code: 'no-version',
|
|
945
|
+
message: `A live worker announces "${ref.name}" without saying which version it runs, which is what a worker announces before it has been upgraded to publish its registrations in full. A name with no version cannot be pinned, so this cannot be chosen — type the name and the version you mean.`,
|
|
946
|
+
};
|
|
947
|
+
}
|
|
948
|
+
const groups = ref.disagreements?.find((entry) => entry.axis === 'group')?.values ?? [];
|
|
949
|
+
if (groups.length > 1) {
|
|
950
|
+
return {
|
|
951
|
+
code: 'ambiguous-group',
|
|
952
|
+
message: `Live workers announce ${ref.name}@${version} from ${groups.length} different groups (${groups.join(', ')}), so nothing can say which queue a run would be dispatched to — or whether the two are even the same code. This cannot be chosen until the deployment stops claiming it twice.`,
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
return undefined;
|
|
956
|
+
}
|
|
302
957
|
/** Every way a graph can be refused. Exported so a canvas can key off the code. */
|
|
303
958
|
exports.WORKFLOW_ISSUE_CODES = [
|
|
304
959
|
'empty',
|
|
@@ -317,6 +972,14 @@ exports.WORKFLOW_ISSUE_CODES = [
|
|
|
317
972
|
'dead-end',
|
|
318
973
|
'transform-not-named',
|
|
319
974
|
'call-not-named',
|
|
975
|
+
'if-not-named',
|
|
976
|
+
'if-threshold-invalid',
|
|
977
|
+
'if-needs-one-input',
|
|
978
|
+
'branch-not-labelled',
|
|
979
|
+
'branch-on-plain-edge',
|
|
980
|
+
'filter-predicate-invalid',
|
|
981
|
+
'filter-narrows-unacknowledged',
|
|
982
|
+
'filter-narrows-nothing',
|
|
320
983
|
];
|
|
321
984
|
/**
|
|
322
985
|
* Everything that makes a graph unrunnable, in one pure function.
|
|
@@ -359,6 +1022,8 @@ function validateWorkflow(graph) {
|
|
|
359
1022
|
const sinks = nodes.filter((node) => node.kind === 'sink');
|
|
360
1023
|
checkNodeWiring(nodes, incoming, outgoing, issues);
|
|
361
1024
|
checkEndpoints(originators, sinks, issues);
|
|
1025
|
+
checkBranches(edges, byId, issues);
|
|
1026
|
+
checkFilterNarrowing({ nodes, edges }, originators, outgoing, issues);
|
|
362
1027
|
const looped = findCycle(nodes, incoming, outgoing);
|
|
363
1028
|
if (looped) {
|
|
364
1029
|
issues.push({
|
|
@@ -485,11 +1150,59 @@ function checkNodeWiring(nodes, incoming, outgoing, issues) {
|
|
|
485
1150
|
message: `Sink "${node.name}" (${node.id}) has an outbound edge. The sink commits the snapshot, so nothing can run after it.`,
|
|
486
1151
|
});
|
|
487
1152
|
}
|
|
1153
|
+
// Exactly one, not "at least one". A gate hands its successors the ref of
|
|
1154
|
+
// the stage it was given rather than staging a copy — see `WorkflowIfNode`
|
|
1155
|
+
// — and one output ref cannot name two inputs, so a second inbound edge
|
|
1156
|
+
// would be silently dropped. Zero is caught by `unreachable` instead, which
|
|
1157
|
+
// points at the same fix with the better message.
|
|
1158
|
+
if (node.kind === 'if' && (incoming.get(node.id)?.length ?? 0) > 1) {
|
|
1159
|
+
issues.push({
|
|
1160
|
+
code: 'if-needs-one-input',
|
|
1161
|
+
nodeIds: [node.id],
|
|
1162
|
+
message: `If "${node.name}" (${node.id}) has ${incoming.get(node.id)?.length} inbound edges, and it can only carry one through. An if node is a gate: it passes the rows it is given straight down whichever branch it takes, so it has one output to hand on and cannot merge. Wire those inputs into a transform and gate the transform instead.`,
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
488
1165
|
const unconfigured = nodeIsUnconfigured(node);
|
|
489
1166
|
if (unconfigured)
|
|
490
1167
|
issues.push(unconfigured);
|
|
491
1168
|
}
|
|
492
1169
|
}
|
|
1170
|
+
/**
|
|
1171
|
+
* Every wire out of an `if` names a branch, and no other wire does.
|
|
1172
|
+
*
|
|
1173
|
+
* Both halves, because the two failures are opposite and both are silent. An
|
|
1174
|
+
* unlabelled wire out of an `if` belongs to no branch, so nothing would ever
|
|
1175
|
+
* take it and the subtree behind it would be skipped on every run of every
|
|
1176
|
+
* deployment — a graph that draws correctly and quietly does half its work. A
|
|
1177
|
+
* label on a wire whose source does not branch is the reverse: a decision
|
|
1178
|
+
* somebody wrote down that nothing reads, which is worse than no decision
|
|
1179
|
+
* because the canvas draws it.
|
|
1180
|
+
*
|
|
1181
|
+
* Run after the structural checks, so `byId` has every endpoint and this cannot
|
|
1182
|
+
* report a wire whose real problem is that it points at a node that was deleted.
|
|
1183
|
+
*/
|
|
1184
|
+
function checkBranches(edges, byId, issues) {
|
|
1185
|
+
for (const edge of edges) {
|
|
1186
|
+
const from = byId.get(edge.from);
|
|
1187
|
+
if (!from)
|
|
1188
|
+
continue;
|
|
1189
|
+
if (from.kind === 'if' && edge.branch === undefined) {
|
|
1190
|
+
issues.push({
|
|
1191
|
+
code: 'branch-not-labelled',
|
|
1192
|
+
nodeIds: [edge.from, edge.to],
|
|
1193
|
+
message: `The wire from "${from.name}" (${from.id}) to "${byId.get(edge.to)?.name ?? edge.to}" does not say which branch it is on. Every wire out of an if node belongs to "then" or to "else", because that is what decides whether it runs — an unlabelled one would never be taken, and everything only it feeds would be skipped on every run with nothing to say why.`,
|
|
1194
|
+
});
|
|
1195
|
+
continue;
|
|
1196
|
+
}
|
|
1197
|
+
if (from.kind !== 'if' && edge.branch !== undefined) {
|
|
1198
|
+
issues.push({
|
|
1199
|
+
code: 'branch-on-plain-edge',
|
|
1200
|
+
nodeIds: [edge.from, edge.to],
|
|
1201
|
+
message: `The wire from "${from.name}" (${from.id}) is labelled "${edge.branch}", but "${from.name}" is a ${from.kind} node and does not branch. A label nothing decides on is a decision that is drawn and never read; remove it, or put an if node where the choice is meant to be made.`,
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
493
1206
|
/**
|
|
494
1207
|
* A node that names none of the thing it exists to run.
|
|
495
1208
|
*
|
|
@@ -508,8 +1221,194 @@ function nodeIsUnconfigured(node) {
|
|
|
508
1221
|
}
|
|
509
1222
|
if (node.kind === 'call')
|
|
510
1223
|
return callIsUnnamed(node);
|
|
1224
|
+
if (node.kind === 'if')
|
|
1225
|
+
return ifIsUnconfigured(node);
|
|
1226
|
+
if (node.kind === 'filter')
|
|
1227
|
+
return filterIsUnconfigured(node);
|
|
511
1228
|
return undefined;
|
|
512
1229
|
}
|
|
1230
|
+
/**
|
|
1231
|
+
* A filter whose test cannot decide anything.
|
|
1232
|
+
*
|
|
1233
|
+
* The whole predicate, checked by the same guard that reads one back out of a
|
|
1234
|
+
* database, rather than a field-by-field re-implementation here. That is not
|
|
1235
|
+
* only tidiness: the failures it catches are all *silent* ones and they point in
|
|
1236
|
+
* opposite directions — an empty `all` keeps every row so the filter does
|
|
1237
|
+
* nothing, an empty `any` drops every row so the load comes out empty, and a
|
|
1238
|
+
* predicate whose operator this build does not recognise would do neither
|
|
1239
|
+
* because {@link workflowFilterMatches} throws inside a step. A canvas has to
|
|
1240
|
+
* say so before any of that.
|
|
1241
|
+
*/
|
|
1242
|
+
function filterIsUnconfigured(node) {
|
|
1243
|
+
if (isWorkflowFilterPredicate(node.predicate))
|
|
1244
|
+
return undefined;
|
|
1245
|
+
return {
|
|
1246
|
+
code: 'filter-predicate-invalid',
|
|
1247
|
+
nodeIds: [node.id],
|
|
1248
|
+
message: `Filter "${node.name}" (${node.id}) has no test this service can run. A filter tests bare column names (letters, digits and underscore, starting with a letter or underscore) against plain values, combined with "all" and "any"; a group has to have at least one condition in it, a list at least one value and no more than ${exports.WORKFLOW_FILTER_MAX_VALUES}, and the whole tree may nest ${exports.WORKFLOW_FILTER_MAX_DEPTH} deep. An empty "all" keeps every row and an empty "any" drops every row, which is why neither is stored.`,
|
|
1249
|
+
};
|
|
1250
|
+
}
|
|
1251
|
+
/**
|
|
1252
|
+
* That every filter names the published types it narrows, and names no others.
|
|
1253
|
+
*
|
|
1254
|
+
* See {@link WorkflowFilterNode.narrows} for the argument. This is the rule
|
|
1255
|
+
* that turns dragging a filter onto a working `source → sink` wire into a graph
|
|
1256
|
+
* that refuses to save until somebody writes down the name of the type they are
|
|
1257
|
+
* about to shrink.
|
|
1258
|
+
*
|
|
1259
|
+
* Run after the structural checks and after the cycle check would have returned,
|
|
1260
|
+
* so the reachability walks below are over a graph that has both ends and no
|
|
1261
|
+
* loop.
|
|
1262
|
+
*/
|
|
1263
|
+
function checkFilterNarrowing(graph, originators, outgoing, issues) {
|
|
1264
|
+
const rootIds = originators.map((one) => one.id);
|
|
1265
|
+
for (const node of graph.nodes) {
|
|
1266
|
+
if (node.kind !== 'filter')
|
|
1267
|
+
continue;
|
|
1268
|
+
const required = workflowNarrowedTypes(graph, node.id, {
|
|
1269
|
+
originators: rootIds,
|
|
1270
|
+
outgoing,
|
|
1271
|
+
}).sort();
|
|
1272
|
+
const declared = [...new Set(node.narrows ?? [])].sort();
|
|
1273
|
+
const missing = required.filter((type) => !declared.includes(type));
|
|
1274
|
+
if (missing.length > 0)
|
|
1275
|
+
issues.push(filterNarrowsUnacknowledged(node, missing));
|
|
1276
|
+
const spurious = declared.filter((type) => !required.includes(type));
|
|
1277
|
+
if (spurious.length > 0)
|
|
1278
|
+
issues.push(filterNarrowsNothing(node, spurious));
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
/** The refusal that stops a filter quietly shrinking a type somebody else reads. */
|
|
1282
|
+
function filterNarrowsUnacknowledged(node, missing) {
|
|
1283
|
+
const one = missing.length === 1;
|
|
1284
|
+
return {
|
|
1285
|
+
code: 'filter-narrows-unacknowledged',
|
|
1286
|
+
nodeIds: [node.id],
|
|
1287
|
+
message: `Filter "${node.name}" (${node.id}) is the only thing feeding the sink that commits ${listTypes(missing)}, and that sink replaces the whole snapshot. So whatever this filter drops disappears from ${one ? 'that type' : 'those types'} the moment this graph runs — and the run reports success, because from its point of view nothing went wrong. If that is what you mean, acknowledge it on the node by naming ${one ? 'the type' : 'the types'}. If you meant to build something new out of a subset, point the sink at a different object type; if you meant to add rows rather than replace them, set the sink to incremental.`,
|
|
1288
|
+
};
|
|
1289
|
+
}
|
|
1290
|
+
/** The opposite refusal: an acknowledgement the graph no longer backs up. */
|
|
1291
|
+
function filterNarrowsNothing(node, spurious) {
|
|
1292
|
+
const one = spurious.length === 1;
|
|
1293
|
+
return {
|
|
1294
|
+
code: 'filter-narrows-nothing',
|
|
1295
|
+
nodeIds: [node.id],
|
|
1296
|
+
message: `Filter "${node.name}" (${node.id}) says it narrows ${listTypes(spurious)}, and it does not: nothing it drops is missing from ${one ? 'that snapshot' : 'those snapshots'}, either because rows also reach the sink by another path or because the sink merges rather than replaces. An acknowledgement that nothing reads is worse than none, because the canvas draws it and the next reader believes it.`,
|
|
1297
|
+
};
|
|
1298
|
+
}
|
|
1299
|
+
/** `"A"`, `"A" and "B"`, `"A", "B" and "C"` — for a message, not for a machine. */
|
|
1300
|
+
function listTypes(types) {
|
|
1301
|
+
const quoted = types.map((type) => `"${type}"`);
|
|
1302
|
+
if (quoted.length <= 1)
|
|
1303
|
+
return quoted.join('');
|
|
1304
|
+
return `${quoted.slice(0, -1).join(', ')} and ${quoted[quoted.length - 1]}`;
|
|
1305
|
+
}
|
|
1306
|
+
/**
|
|
1307
|
+
* The object types whose whole published snapshot a node stands in front of.
|
|
1308
|
+
*
|
|
1309
|
+
* A sink is in this list when it commits in `full` mode — replacing what is
|
|
1310
|
+
* served rather than merging into it — **and** removing the named node would
|
|
1311
|
+
* make it unreachable from everything that originates rows. That second half is
|
|
1312
|
+
* the load-bearing one: a filter on one of two paths into a sink narrows
|
|
1313
|
+
* nothing, because the other path still delivers, and a rule that ignored it
|
|
1314
|
+
* would demand an acknowledgement for a graph where nothing is lost.
|
|
1315
|
+
*
|
|
1316
|
+
* Removal-reachability rather than a dominator algorithm, deliberately. The
|
|
1317
|
+
* graphs here are a screenful of boxes, this runs once per filter node, and the
|
|
1318
|
+
* cheaper version would be a second, subtler implementation of "does this node
|
|
1319
|
+
* decide whether that one runs" living next to `checkReachability` — which is
|
|
1320
|
+
* exactly the kind of duplication that eventually disagrees with the walk the
|
|
1321
|
+
* rest of this file does.
|
|
1322
|
+
*
|
|
1323
|
+
* Exported because the console needs the same answer to offer the right
|
|
1324
|
+
* acknowledgements, and a canvas that computed its own would offer a set the
|
|
1325
|
+
* server then refuses.
|
|
1326
|
+
*
|
|
1327
|
+
* `precomputed` exists only so the validator, which has already built the
|
|
1328
|
+
* adjacency it needs, does not build it twice per filter node. Callers outside
|
|
1329
|
+
* this file pass a graph and nothing else.
|
|
1330
|
+
*/
|
|
1331
|
+
function workflowNarrowedTypes(graph, nodeId, precomputed) {
|
|
1332
|
+
const nodes = graph.nodes ?? [];
|
|
1333
|
+
const outgoing = precomputed?.outgoing ?? buildAdjacency(nodes, graph.edges ?? []).outgoing;
|
|
1334
|
+
const originators = precomputed?.originators ?? nodes.filter((node) => originatesRows(node)).map((node) => node.id);
|
|
1335
|
+
// Everything still reachable once this node is taken out. A source that *is*
|
|
1336
|
+
// the node cannot originate anything, which falls out of the filter below
|
|
1337
|
+
// rather than needing its own case.
|
|
1338
|
+
const roots = originators.filter((id) => id !== nodeId);
|
|
1339
|
+
const reachableWithout = walk(roots, stripNode(outgoing, nodeId));
|
|
1340
|
+
const types = [];
|
|
1341
|
+
for (const node of nodes) {
|
|
1342
|
+
if (!narrowedByRemoval(node, reachableWithout))
|
|
1343
|
+
continue;
|
|
1344
|
+
const type = node.targetType.trim();
|
|
1345
|
+
if (type.length > 0 && !types.includes(type))
|
|
1346
|
+
types.push(type);
|
|
1347
|
+
}
|
|
1348
|
+
return types;
|
|
1349
|
+
}
|
|
1350
|
+
/**
|
|
1351
|
+
* Whether this node is a full-mode sink that the removed node was the only way
|
|
1352
|
+
* to reach.
|
|
1353
|
+
*
|
|
1354
|
+
* A sink with no explicit mode is a **full** sink — the same default `runSink`
|
|
1355
|
+
* applies, read the same way here, because a graph that validated under one
|
|
1356
|
+
* meaning of the absent field and ran under the other is precisely the failure
|
|
1357
|
+
* this whole check exists to prevent.
|
|
1358
|
+
*/
|
|
1359
|
+
function narrowedByRemoval(node, reachableWithout) {
|
|
1360
|
+
if (node.kind !== 'sink')
|
|
1361
|
+
return false;
|
|
1362
|
+
if ((node.mode ?? 'full') === 'incremental')
|
|
1363
|
+
return false;
|
|
1364
|
+
return !reachableWithout.has(node.id);
|
|
1365
|
+
}
|
|
1366
|
+
/** The same adjacency with one node's outgoing edges cut, leaving the original alone. */
|
|
1367
|
+
function stripNode(outgoing, nodeId) {
|
|
1368
|
+
const without = new Map();
|
|
1369
|
+
for (const [from, targets] of outgoing) {
|
|
1370
|
+
if (from === nodeId)
|
|
1371
|
+
continue;
|
|
1372
|
+
without.set(from, targets.filter((to) => to !== nodeId));
|
|
1373
|
+
}
|
|
1374
|
+
return without;
|
|
1375
|
+
}
|
|
1376
|
+
/**
|
|
1377
|
+
* A gate whose test cannot decide anything.
|
|
1378
|
+
*
|
|
1379
|
+
* The third of the same mistake {@link nodeIsUnconfigured} describes, once per
|
|
1380
|
+
* predicate kind, because "unconfigured" means something different for each and
|
|
1381
|
+
* a single check would have to pick one. Both refusals exist for one reason: a
|
|
1382
|
+
* gate that cannot really choose still picks a branch, and whichever it picks is
|
|
1383
|
+
* a decision the graph appears to make and nobody authored — with half the
|
|
1384
|
+
* pipeline silently not running as the only symptom.
|
|
1385
|
+
*/
|
|
1386
|
+
function ifIsUnconfigured(node) {
|
|
1387
|
+
const predicate = node.predicate;
|
|
1388
|
+
if (predicate.kind === 'env') {
|
|
1389
|
+
if (predicate.envVar.trim().length > 0)
|
|
1390
|
+
return undefined;
|
|
1391
|
+
return {
|
|
1392
|
+
code: 'if-not-named',
|
|
1393
|
+
nodeIds: [node.id],
|
|
1394
|
+
message: `If "${node.name}" (${node.id}) names no environment variable, so there is nothing for it to decide on. It reads the *name* of a variable on the machine that runs the load — that is how a graph tells a deployment with a ClickHouse apart from one without.`,
|
|
1395
|
+
};
|
|
1396
|
+
}
|
|
1397
|
+
if (predicate.kind === 'rowCount') {
|
|
1398
|
+
// At least one, so both answers are reachable. A threshold of zero is
|
|
1399
|
+
// satisfied by every run including an empty one, so the `else` subtree would
|
|
1400
|
+
// never execute on any deployment — the silent half-graph, arrived at by
|
|
1401
|
+
// typing a number rather than by mislabelling a wire.
|
|
1402
|
+
if (Number.isInteger(predicate.atLeast) && predicate.atLeast >= 1)
|
|
1403
|
+
return undefined;
|
|
1404
|
+
return {
|
|
1405
|
+
code: 'if-threshold-invalid',
|
|
1406
|
+
nodeIds: [node.id],
|
|
1407
|
+
message: `If "${node.name}" (${node.id}) branches on a row count of ${JSON.stringify(predicate.atLeast)}, and a threshold has to be a whole number of at least 1. "At least 1" is the "did anything arrive at all" test; 0 would be satisfied by every run, so the else branch — and everything only it feeds — would never run on any deployment.`,
|
|
1408
|
+
};
|
|
1409
|
+
}
|
|
1410
|
+
return unreachablePredicateKind(predicate, 'validateWorkflow');
|
|
1411
|
+
}
|
|
513
1412
|
/**
|
|
514
1413
|
* A call that names half of what it needs, or nothing at all.
|
|
515
1414
|
*
|
|
@@ -684,6 +1583,7 @@ function workflowRunOrder(graph) {
|
|
|
684
1583
|
throw new Error(`Refusing to order an invalid workflow: ${issues.map((issue) => issue.message).join(' ')}`);
|
|
685
1584
|
}
|
|
686
1585
|
const byId = new Map(graph.nodes.map((node) => [node.id, node]));
|
|
1586
|
+
const labels = branchLabels(graph.edges);
|
|
687
1587
|
// The same adjacency the validator walks, from the same builder. Two copies of
|
|
688
1588
|
// "what is wired into what" is exactly how a graph that validated comes out
|
|
689
1589
|
// executing differently, which is the thing this function's contract rules out.
|
|
@@ -702,7 +1602,8 @@ function workflowRunOrder(graph) {
|
|
|
702
1602
|
// from, and it is part of the fingerprint precisely because it is visible in
|
|
703
1603
|
// the output. `buildAdjacency` fills `incoming` by walking the edges in
|
|
704
1604
|
// order, so that is what this already is.
|
|
705
|
-
|
|
1605
|
+
const inputs = [...(incoming.get(id) ?? [])];
|
|
1606
|
+
order.push({ node, inputs, inputBranches: labelsInto(id, inputs, labels) });
|
|
706
1607
|
for (const next of outgoing.get(id) ?? []) {
|
|
707
1608
|
const remaining = (indegree.get(next) ?? 0) - 1;
|
|
708
1609
|
indegree.set(next, remaining);
|
|
@@ -712,6 +1613,81 @@ function workflowRunOrder(graph) {
|
|
|
712
1613
|
}
|
|
713
1614
|
return order;
|
|
714
1615
|
}
|
|
1616
|
+
/** Every labelled wire, keyed by the pair it joins. */
|
|
1617
|
+
function branchLabels(edges) {
|
|
1618
|
+
const labels = new Map();
|
|
1619
|
+
for (const edge of edges) {
|
|
1620
|
+
if (edge.branch === undefined)
|
|
1621
|
+
continue;
|
|
1622
|
+
labels.set(`${edge.from}\0${edge.to}`, edge.branch);
|
|
1623
|
+
}
|
|
1624
|
+
return labels;
|
|
1625
|
+
}
|
|
1626
|
+
/** The labels on the wires into one node, with the unlabelled ones left out. */
|
|
1627
|
+
function labelsInto(to, inputs, labels) {
|
|
1628
|
+
const into = {};
|
|
1629
|
+
for (const from of inputs) {
|
|
1630
|
+
const label = labels.get(`${from}\0${to}`);
|
|
1631
|
+
if (label !== undefined)
|
|
1632
|
+
into[from] = label;
|
|
1633
|
+
}
|
|
1634
|
+
return into;
|
|
1635
|
+
}
|
|
1636
|
+
/**
|
|
1637
|
+
* Whether a node runs, given what the nodes before it did.
|
|
1638
|
+
*
|
|
1639
|
+
* ## The rule
|
|
1640
|
+
*
|
|
1641
|
+
* A node runs when **at least one wire into it is live**, where a wire is live
|
|
1642
|
+
* if its source ran and — when the wire carries a branch label — the source is
|
|
1643
|
+
* an `if` that took that branch. A node with no inbound wires always runs, which
|
|
1644
|
+
* is every source and every originating call.
|
|
1645
|
+
*
|
|
1646
|
+
* ## Why the obvious rule is wrong
|
|
1647
|
+
*
|
|
1648
|
+
* The naive version is "mark everything downstream of the untaken edge as
|
|
1649
|
+
* skipped", and it is wrong on the shape branches are most often drawn in:
|
|
1650
|
+
*
|
|
1651
|
+
* ```
|
|
1652
|
+
* ┌ then → A ┐
|
|
1653
|
+
* if ──┤ ├→ C → sink
|
|
1654
|
+
* └ else → B ┘
|
|
1655
|
+
* ```
|
|
1656
|
+
*
|
|
1657
|
+
* `C` is downstream of `B`. Take the `then` branch and the naive rule walks from
|
|
1658
|
+
* the untaken `else` edge, reaches `B`, reaches `C`, and skips it — so the sink
|
|
1659
|
+
* never runs and the load silently commits nothing, on a graph whose whole
|
|
1660
|
+
* purpose was that both branches converge. Reachability from the **taken** edges
|
|
1661
|
+
* gets it right: `C` is reached through `A`, so it runs, and `B` — reached only
|
|
1662
|
+
* through the untaken edge — does not. `C` then sees an empty stage ref for `B`,
|
|
1663
|
+
* which is exactly what `stageRefsFor` already does for an upstream that
|
|
1664
|
+
* produced nothing, so the positions a merge reads stay aligned with the wires
|
|
1665
|
+
* that were drawn.
|
|
1666
|
+
*
|
|
1667
|
+
* It is evaluated incrementally rather than as a graph walk because the answers
|
|
1668
|
+
* arrive as the run goes: `workflowRunOrder` is topological, so by the time a
|
|
1669
|
+
* node is reached every node feeding it has an outcome. That also means this
|
|
1670
|
+
* reads **only** what was recorded — no predicate is re-evaluated here, which is
|
|
1671
|
+
* the property the whole branch feature rests on.
|
|
1672
|
+
*/
|
|
1673
|
+
function workflowNodeRuns(entry, outcomes) {
|
|
1674
|
+
if (entry.inputs.length === 0)
|
|
1675
|
+
return true;
|
|
1676
|
+
return entry.inputs.some((from) => {
|
|
1677
|
+
const upstream = outcomes[from];
|
|
1678
|
+
// Anything other than a clean success means this wire carried nothing: a
|
|
1679
|
+
// skipped upstream is not on a live path, and a failed one aborts the run
|
|
1680
|
+
// before this is ever asked.
|
|
1681
|
+
if (upstream?.status !== 'succeeded')
|
|
1682
|
+
return false;
|
|
1683
|
+
const label = entry.inputBranches?.[from];
|
|
1684
|
+
// A plain wire. Live because its source ran, which is what every graph
|
|
1685
|
+
// drawn before branches existed relies on.
|
|
1686
|
+
if (label === undefined)
|
|
1687
|
+
return true;
|
|
1688
|
+
return upstream.branch === label;
|
|
1689
|
+
});
|
|
1690
|
+
}
|
|
715
1691
|
/**
|
|
716
1692
|
* A stable fingerprint of what a graph *does*.
|
|
717
1693
|
*
|
|
@@ -737,7 +1713,12 @@ function workflowGraphHash(graph) {
|
|
|
737
1713
|
const nodes = [...graph.nodes]
|
|
738
1714
|
.sort((left, right) => (left.id < right.id ? -1 : left.id > right.id ? 1 : 0))
|
|
739
1715
|
.map((node) => canonicalNode(node));
|
|
740
|
-
|
|
1716
|
+
// The branch is appended only when there is one, so every edge drawn before
|
|
1717
|
+
// branches existed hashes to exactly the string it always did. Adding a graph
|
|
1718
|
+
// to this file must not renumber the versions of graphs that did not change.
|
|
1719
|
+
const edges = graph.edges.map((edge) => edge.branch === undefined
|
|
1720
|
+
? `${edge.from}>${edge.to}`
|
|
1721
|
+
: `${edge.from}>${edge.to}:${edge.branch}`);
|
|
741
1722
|
const canonical = JSON.stringify({ nodes, edges });
|
|
742
1723
|
return `${fnv1a(canonical, 0x811c9dc5)}${fnv1a(canonical, 0x01000193)}`;
|
|
743
1724
|
}
|
|
@@ -779,7 +1760,88 @@ function canonicalNode(node) {
|
|
|
779
1760
|
sortedEntries(node.config),
|
|
780
1761
|
]);
|
|
781
1762
|
}
|
|
782
|
-
|
|
1763
|
+
if (node.kind === 'if') {
|
|
1764
|
+
return JSON.stringify([node.id, node.kind, ...canonicalPredicate(node.predicate)]);
|
|
1765
|
+
}
|
|
1766
|
+
if (node.kind === 'filter') {
|
|
1767
|
+
// `narrows` is in here as well as the predicate, and that is a decision
|
|
1768
|
+
// rather than completeness. It changes nothing about what the node computes
|
|
1769
|
+
// — but it is the acknowledgement that a published type is about to become a
|
|
1770
|
+
// subset, and a run that happened before anybody acknowledged that must stay
|
|
1771
|
+
// distinguishable from one that happened after. Sorted, so ticking the same
|
|
1772
|
+
// two boxes in the other order is not an edit.
|
|
1773
|
+
return JSON.stringify([
|
|
1774
|
+
node.id,
|
|
1775
|
+
node.kind,
|
|
1776
|
+
canonicalFilterPredicate(node.predicate),
|
|
1777
|
+
[...(node.narrows ?? [])].sort(),
|
|
1778
|
+
]);
|
|
1779
|
+
}
|
|
1780
|
+
if (node.kind === 'sink') {
|
|
1781
|
+
return JSON.stringify([node.id, node.kind, node.targetType, node.mode ?? 'full']);
|
|
1782
|
+
}
|
|
1783
|
+
return unreachableNodeKind(node, 'workflowGraphHash');
|
|
1784
|
+
}
|
|
1785
|
+
/**
|
|
1786
|
+
* The parts of a predicate that decide which branch runs.
|
|
1787
|
+
*
|
|
1788
|
+
* The kind leads, so the two tests can never canonicalise to the same string —
|
|
1789
|
+
* a gate switched from "is CLICKHOUSE_URL set" to "did 1 row arrive" is a
|
|
1790
|
+
* different pipeline on every deployment, and a hash that missed it would leave
|
|
1791
|
+
* two runs claiming the same graph version while having taken different halves
|
|
1792
|
+
* of it.
|
|
1793
|
+
*/
|
|
1794
|
+
function canonicalPredicate(predicate) {
|
|
1795
|
+
if (predicate.kind === 'env') {
|
|
1796
|
+
// Both halves, because both decide which branch runs. `equals` being absent
|
|
1797
|
+
// is a *different* test from `equals` being the empty string — "set to
|
|
1798
|
+
// anything" against "set to nothing" — so the two must not fold together.
|
|
1799
|
+
return [predicate.kind, predicate.envVar, predicate.equals ?? null];
|
|
1800
|
+
}
|
|
1801
|
+
if (predicate.kind === 'rowCount') {
|
|
1802
|
+
return [predicate.kind, predicate.atLeast];
|
|
1803
|
+
}
|
|
1804
|
+
return unreachablePredicateKind(predicate, 'workflowGraphHash');
|
|
1805
|
+
}
|
|
1806
|
+
/**
|
|
1807
|
+
* The parts of a filter predicate that decide which rows survive.
|
|
1808
|
+
*
|
|
1809
|
+
* Which is all of them, so this is a faithful canonicalisation rather than a
|
|
1810
|
+
* selection — every field of every kind changes what a load publishes. What it
|
|
1811
|
+
* adds over `JSON.stringify` of the predicate is order-independence where order
|
|
1812
|
+
* is meaningless and order-*dependence* where it is not: the children of an
|
|
1813
|
+
* `all` are sorted, because `A and B` and `B and A` are the same filter and a
|
|
1814
|
+
* canvas that rewrites the array must not look like an edit; the `values` of a
|
|
1815
|
+
* `oneOf` are sorted for the same reason. The kind and the operator lead, so no
|
|
1816
|
+
* two shapes can canonicalise to the same string.
|
|
1817
|
+
*
|
|
1818
|
+
* `values` is deliberately *not* deduplicated. A duplicate changes nothing about
|
|
1819
|
+
* the result, but removing one here would make the hash disagree with what is
|
|
1820
|
+
* stored, and the fingerprint is meant to answer "is this the same graph" rather
|
|
1821
|
+
* than "is this an equivalent graph".
|
|
1822
|
+
*/
|
|
1823
|
+
function canonicalFilterPredicate(predicate) {
|
|
1824
|
+
if (predicate.kind === 'all' || predicate.kind === 'any') {
|
|
1825
|
+
return JSON.stringify([
|
|
1826
|
+
predicate.kind,
|
|
1827
|
+
predicate.children.map((child) => canonicalFilterPredicate(child)).sort(),
|
|
1828
|
+
]);
|
|
1829
|
+
}
|
|
1830
|
+
if (predicate.kind === 'oneOf') {
|
|
1831
|
+
return JSON.stringify([
|
|
1832
|
+
predicate.kind,
|
|
1833
|
+
predicate.column,
|
|
1834
|
+
predicate.operator,
|
|
1835
|
+
[...predicate.values].map((value) => JSON.stringify(value)).sort(),
|
|
1836
|
+
]);
|
|
1837
|
+
}
|
|
1838
|
+
if (predicate.kind === 'present') {
|
|
1839
|
+
return JSON.stringify([predicate.kind, predicate.column, predicate.operator]);
|
|
1840
|
+
}
|
|
1841
|
+
if (predicate.kind === 'compare') {
|
|
1842
|
+
return JSON.stringify([predicate.kind, predicate.column, predicate.operator, predicate.value]);
|
|
1843
|
+
}
|
|
1844
|
+
return unreachableFilterPredicateKind(predicate, 'workflowGraphHash');
|
|
783
1845
|
}
|
|
784
1846
|
function sortedEntries(config) {
|
|
785
1847
|
return Object.keys(config)
|
|
@@ -835,13 +1897,66 @@ function isWorkflowNode(value) {
|
|
|
835
1897
|
config !== null &&
|
|
836
1898
|
!Array.isArray(config));
|
|
837
1899
|
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
1900
|
+
if (kind === 'if') {
|
|
1901
|
+
// The predicate in full, refused rather than defaulted: a gate read back
|
|
1902
|
+
// without a test it recognises would have to invent one, and inventing one
|
|
1903
|
+
// means half the graph runs on a decision nobody made.
|
|
1904
|
+
return isWorkflowIfPredicate(Reflect.get(value, 'predicate'));
|
|
1905
|
+
}
|
|
1906
|
+
if (kind === 'filter') {
|
|
1907
|
+
return isNarrowsList(Reflect.get(value, 'narrows'))
|
|
1908
|
+
? isWorkflowFilterPredicate(Reflect.get(value, 'predicate'))
|
|
1909
|
+
: false;
|
|
1910
|
+
}
|
|
1911
|
+
if (kind === 'source') {
|
|
1912
|
+
const sourceKind = Reflect.get(value, 'sourceKind');
|
|
1913
|
+
const config = Reflect.get(value, 'config');
|
|
1914
|
+
return isConnectorKind(sourceKind) && typeof config === 'object' && config !== null;
|
|
1915
|
+
}
|
|
1916
|
+
return isWorkflowNodeKindUnhandled(kind);
|
|
1917
|
+
}
|
|
1918
|
+
/**
|
|
1919
|
+
* Whether a stored `narrows` is one this build can read.
|
|
1920
|
+
*
|
|
1921
|
+
* Checked as strictly as the predicate beside it, and it has to be:
|
|
1922
|
+
* {@link WorkflowFilterNode.narrows} is what stands between "filter into the
|
|
1923
|
+
* type that already exists" and a published snapshot quietly becoming a subset.
|
|
1924
|
+
* A value that is not a list of strings is refused rather than dropped, because
|
|
1925
|
+
* dropping it turns a graph somebody acknowledged into one that never was — and
|
|
1926
|
+
* `validateWorkflow` would then refuse it with a message about a field they did
|
|
1927
|
+
* fill in.
|
|
1928
|
+
*
|
|
1929
|
+
* Absent is accepted, because that is what every filter narrowing nothing looks
|
|
1930
|
+
* like and what every node stored before this field existed is.
|
|
1931
|
+
*/
|
|
1932
|
+
function isNarrowsList(value) {
|
|
1933
|
+
if (value === undefined)
|
|
1934
|
+
return true;
|
|
1935
|
+
return Array.isArray(value) && value.every((type) => typeof type === 'string');
|
|
1936
|
+
}
|
|
1937
|
+
/**
|
|
1938
|
+
* The narrowing counterpart of {@link unreachableNodeKind}.
|
|
1939
|
+
*
|
|
1940
|
+
* A guard cannot take a `never` — `kind` here is a string that
|
|
1941
|
+
* {@link isWorkflowNodeKind} already accepted — so exhaustiveness is bought by
|
|
1942
|
+
* assigning it to one, which is the compile error a new kind has to answer, and
|
|
1943
|
+
* refusing the value at run time, which is what a build that skipped this file
|
|
1944
|
+
* would do to a node it has no rule for.
|
|
1945
|
+
*/
|
|
1946
|
+
function isWorkflowNodeKindUnhandled(kind) {
|
|
1947
|
+
void kind;
|
|
1948
|
+
return false;
|
|
841
1949
|
}
|
|
842
1950
|
function isWorkflowEdge(value) {
|
|
843
1951
|
if (typeof value !== 'object' || value === null)
|
|
844
1952
|
return false;
|
|
1953
|
+
// A `branch` that is present and unrecognised is refused rather than dropped,
|
|
1954
|
+
// for the reason `isWorkflowNode` refuses an unknown node: an edge silently
|
|
1955
|
+
// read back without its label is a wire that stops belonging to a branch, and
|
|
1956
|
+
// everything behind it stops running with nothing to point at.
|
|
1957
|
+
const branch = Reflect.get(value, 'branch');
|
|
1958
|
+
if (branch !== undefined && !isWorkflowBranchLabel(branch))
|
|
1959
|
+
return false;
|
|
845
1960
|
return (typeof Reflect.get(value, 'from') === 'string' && typeof Reflect.get(value, 'to') === 'string');
|
|
846
1961
|
}
|
|
847
1962
|
/**
|
|
@@ -909,3 +2024,24 @@ function isPipelineStore(store) {
|
|
|
909
2024
|
store !== null &&
|
|
910
2025
|
typeof Reflect.get(store, 'listConnectors') === 'function');
|
|
911
2026
|
}
|
|
2027
|
+
/**
|
|
2028
|
+
* What a redacted password reads as, on the wire.
|
|
2029
|
+
*
|
|
2030
|
+
* Declared here rather than in `@dudousxd/nestjs-catalog-pipeline`, where the
|
|
2031
|
+
* redaction itself lives, because this literal is not an implementation detail
|
|
2032
|
+
* of the redaction: it is part of what `GET pipeline/connections` answers, and
|
|
2033
|
+
* a browser is the audience it was invented for. A form that lets somebody
|
|
2034
|
+
* paste an address has to be able to recognise the string it was shown — a
|
|
2035
|
+
* `url` whose password is exactly this came out of a read, and posting it back
|
|
2036
|
+
* as a NEW connection stores the word "REDACTED" as the password. There is no
|
|
2037
|
+
* stored row behind a create for `restoreRedactedSecrets` to put the real one
|
|
2038
|
+
* back from, so nothing downstream can catch it: the row saves, and the failure
|
|
2039
|
+
* arrives at the first scheduled load as an authentication error against a
|
|
2040
|
+
* password nobody typed.
|
|
2041
|
+
*
|
|
2042
|
+
* The pipeline package re-exports this rather than declaring its own, so the
|
|
2043
|
+
* two halves cannot drift. A fixed literal rather than a run of asterisks, so
|
|
2044
|
+
* it is greppable in a bug report and cannot be mistaken for a password
|
|
2045
|
+
* somebody actually chose.
|
|
2046
|
+
*/
|
|
2047
|
+
exports.REDACTED_SECRET = 'REDACTED';
|