@dudousxd/nestjs-catalog 0.16.0-preview.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.pipeline.d.ts +442 -7
- package/dist/catalog.pipeline.js +646 -5
- package/dist/catalog.stage-encoding.d.ts +185 -0
- package/dist/catalog.stage-encoding.js +314 -0
- package/dist/client.d.ts +2 -2
- package/dist/client.js +26 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +29 -3
- package/package.json +1 -1
|
@@ -699,6 +699,20 @@ export interface WorkflowNodeOutcome {
|
|
|
699
699
|
status: 'succeeded' | 'failed' | 'skipped';
|
|
700
700
|
/** Rows this node produced, or committed if it is the sink. */
|
|
701
701
|
rows: number;
|
|
702
|
+
/**
|
|
703
|
+
* Rows this node was *given*, for a node whose whole purpose is that the two
|
|
704
|
+
* numbers differ. Set by {@link WorkflowFilterNode} and absent everywhere else.
|
|
705
|
+
*
|
|
706
|
+
* Two numbers rather than a `dropped` counter, because `dropped` is
|
|
707
|
+
* `rowsIn - rows` and a third stored number is a third thing that can
|
|
708
|
+
* disagree with the other two. A run panel subtracts.
|
|
709
|
+
*
|
|
710
|
+
* Optional, so an outcome written before filters existed reads back as what it
|
|
711
|
+
* is — a node that never reported an input count — rather than as one that
|
|
712
|
+
* received nothing. Absent and zero are different facts here, and conflating
|
|
713
|
+
* them would make every historical transform look like it dropped everything.
|
|
714
|
+
*/
|
|
715
|
+
rowsIn?: number;
|
|
702
716
|
/** For a transform node: which version of its code ran. */
|
|
703
717
|
transformVersion?: number;
|
|
704
718
|
/**
|
|
@@ -734,10 +748,22 @@ export interface WorkflowNodeOutcome {
|
|
|
734
748
|
* The kinds that were considered and rejected, since a small vocabulary is only
|
|
735
749
|
* defensible if the omissions are:
|
|
736
750
|
*
|
|
737
|
-
* - **filter** —
|
|
738
|
-
*
|
|
739
|
-
*
|
|
740
|
-
* missing.
|
|
751
|
+
* - **filter** — *this entry used to be a rejection, and it is worth leaving the
|
|
752
|
+
* reversal visible rather than editing the history out.* The argument was that
|
|
753
|
+
* a transform whose code returns a subset already filters, so the kind bought
|
|
754
|
+
* a second way to drop rows and a second place to look when rows went missing.
|
|
755
|
+
* That argument is sound about *code* and it is the reason
|
|
756
|
+
* {@link WorkflowFilterNode} does not take any: what changed is that the
|
|
757
|
+
* predicate is a **closed structure** rather than a body, and a closed
|
|
758
|
+
* structure can be read by something other than a JavaScript engine. Only a
|
|
759
|
+
* declarative predicate can be translated into a `WHERE` and pushed into the
|
|
760
|
+
* query the source already runs, and that is not a micro-optimisation — a
|
|
761
|
+
* transform filtering `obj_pribuybuylistdetail` reads all 7,637,391 rows off
|
|
762
|
+
* disk, over the network, and into JS objects of ~80 properties each before
|
|
763
|
+
* anything decides they were unwanted. See {@link WorkflowFilterNode} for what
|
|
764
|
+
* is actually built today (an in-memory, per-batch pass) and for where the
|
|
765
|
+
* pushdown seam is, which is a promise about a shape rather than a claim about
|
|
766
|
+
* a measurement.
|
|
741
767
|
* - **branch / split (unconditional)** — already expressible, and still is: a
|
|
742
768
|
* node with two outbound edges is read by both successors, each of which
|
|
743
769
|
* filters differently. There is nothing for an *unconditional* split to do.
|
|
@@ -760,7 +786,7 @@ export interface WorkflowNodeOutcome {
|
|
|
760
786
|
* outside a run, which is why `call` names one and not a step. If a step is
|
|
761
787
|
* what you want, the thing to call is a one-step workflow wrapping it.
|
|
762
788
|
*/
|
|
763
|
-
export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if"];
|
|
789
|
+
export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter"];
|
|
764
790
|
export type WorkflowNodeKind = (typeof WORKFLOW_NODE_KINDS)[number];
|
|
765
791
|
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
766
792
|
export declare function isWorkflowNodeKind(value: unknown): value is WorkflowNodeKind;
|
|
@@ -1180,13 +1206,386 @@ export interface WorkflowIfNode extends WorkflowNodeBase {
|
|
|
1180
1206
|
*/
|
|
1181
1207
|
predicate: WorkflowIfPredicate;
|
|
1182
1208
|
}
|
|
1209
|
+
/**
|
|
1210
|
+
* The shapes a {@link WorkflowFilterPredicate} can take.
|
|
1211
|
+
*
|
|
1212
|
+
* Two leaf kinds plus a presence test plus two ways to combine them, and the
|
|
1213
|
+
* list is closed for the same reason {@link WORKFLOW_PREDICATE_KINDS} is: every
|
|
1214
|
+
* decision made per kind ends in {@link unreachableFilterPredicateKind}, so a
|
|
1215
|
+
* sixth shape is a build failure naming the files that owe it an answer rather
|
|
1216
|
+
* than a filter that saves, draws, and quietly keeps everything.
|
|
1217
|
+
*
|
|
1218
|
+
* **There is deliberately no `not`.** Every leaf carries its own inverse
|
|
1219
|
+
* ({@link WORKFLOW_FILTER_OPERATORS} pairs each operator with one, `oneOf` has
|
|
1220
|
+
* `notIn`, `present` has `isNotNull`) and `all`/`any` are duals, so De Morgan
|
|
1221
|
+
* already writes any negation with the kinds here. A `not` node would be a
|
|
1222
|
+
* second spelling of every predicate — two shapes to read when a load comes out
|
|
1223
|
+
* short, and a UI with a checkbox nobody agrees on the placement of. The one
|
|
1224
|
+
* case it does not cover is stated on {@link workflowFilterMatches}: a value the
|
|
1225
|
+
* test cannot compare fails *both* a form and its inverse, on purpose.
|
|
1226
|
+
*
|
|
1227
|
+
* **There is deliberately no free-form expression and no code.** That is the
|
|
1228
|
+
* whole argument for the node existing — see {@link WorkflowFilterNode}.
|
|
1229
|
+
*/
|
|
1230
|
+
export declare const WORKFLOW_FILTER_PREDICATE_KINDS: readonly ["compare", "oneOf", "present", "all", "any"];
|
|
1231
|
+
export type WorkflowFilterPredicateKind = (typeof WORKFLOW_FILTER_PREDICATE_KINDS)[number];
|
|
1232
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
1233
|
+
export declare function isWorkflowFilterPredicateKind(value: unknown): value is WorkflowFilterPredicateKind;
|
|
1234
|
+
/**
|
|
1235
|
+
* {@link unreachablePredicateKind}, for the filter language, and for the
|
|
1236
|
+
* identical reason.
|
|
1237
|
+
*
|
|
1238
|
+
* It throws as well as failing to compile: a predicate arrives as JSON out of a
|
|
1239
|
+
* column, so a graph saved by a newer build and read by an older one is a thing
|
|
1240
|
+
* that happens, and returning a default for a shape this build has no rule for
|
|
1241
|
+
* would mean silently keeping — or silently dropping — every row.
|
|
1242
|
+
*/
|
|
1243
|
+
export declare function unreachableFilterPredicateKind(predicate: never, where: string): never;
|
|
1244
|
+
/**
|
|
1245
|
+
* What a {@link WorkflowFilterComparison} does with its column and its value.
|
|
1246
|
+
*
|
|
1247
|
+
* Ten, in five inverse pairs, and the pairing is the reason there is no `not`
|
|
1248
|
+
* kind. Each one has an obvious single-expression form in both dialects this
|
|
1249
|
+
* repository speaks, which is not a coincidence — it is the constraint the list
|
|
1250
|
+
* was chosen under, so that the day a predicate is pushed into a source query
|
|
1251
|
+
* the translation is a `switch` and not a design.
|
|
1252
|
+
*
|
|
1253
|
+
* The omissions, since a closed list is only defensible if they are:
|
|
1254
|
+
*
|
|
1255
|
+
* - **`between`** — `all` of a `greaterThanOrEqual` and a `lessThanOrEqual`, in
|
|
1256
|
+
* one more click and with no second shape to validate, hash and translate.
|
|
1257
|
+
* - **`endsWith`** — asked for far less than the other two, and unlike them it
|
|
1258
|
+
* has no index that could ever serve it in either dialect, so offering it in a
|
|
1259
|
+
* palette would advertise something that is a full scan by construction.
|
|
1260
|
+
* `contains` covers it at the same cost.
|
|
1261
|
+
* - **regular expressions** — the dialects disagree about the syntax, the
|
|
1262
|
+
* engines disagree about the semantics, and a predicate whose meaning depends
|
|
1263
|
+
* on which database answered it is a predicate that cannot be pushed down
|
|
1264
|
+
* without changing what the load returns. That is the one property this list
|
|
1265
|
+
* exists to hold.
|
|
1266
|
+
* - **case-insensitive variants** — collation is a property of the column in
|
|
1267
|
+
* both dialects, so a `equalsIgnoreCase` evaluated in memory and the same
|
|
1268
|
+
* predicate evaluated in a `WHERE` would legitimately disagree. Normalise in a
|
|
1269
|
+
* transform, where the disagreement is visible.
|
|
1270
|
+
*/
|
|
1271
|
+
export declare const WORKFLOW_FILTER_OPERATORS: readonly ["equals", "notEquals", "greaterThan", "lessThanOrEqual", "greaterThanOrEqual", "lessThan", "contains", "notContains", "startsWith", "notStartsWith"];
|
|
1272
|
+
export type WorkflowFilterOperator = (typeof WORKFLOW_FILTER_OPERATORS)[number];
|
|
1273
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
1274
|
+
export declare function isWorkflowFilterOperator(value: unknown): value is WorkflowFilterOperator;
|
|
1275
|
+
/**
|
|
1276
|
+
* {@link unreachableFilterPredicateKind}, one level further down.
|
|
1277
|
+
*
|
|
1278
|
+
* An operator added to the list without a rule for evaluating it, describing it
|
|
1279
|
+
* or hashing it is a type error naming the file — not a comparison that silently
|
|
1280
|
+
* answers false for every row, which is a filter that drops the whole dataset
|
|
1281
|
+
* and reports success.
|
|
1282
|
+
*/
|
|
1283
|
+
export declare function unreachableFilterOperator(operator: never, where: string): never;
|
|
1284
|
+
/**
|
|
1285
|
+
* What a predicate may be compared against.
|
|
1286
|
+
*
|
|
1287
|
+
* Three scalars and nothing else. No `null` — {@link WorkflowFilterPresence} is
|
|
1288
|
+
* how absence is tested, and folding it in here would make `equals: null` and
|
|
1289
|
+
* `isNull` two spellings of one question with different answers under the
|
|
1290
|
+
* three-valued logic on {@link workflowFilterMatches}. No object and no array —
|
|
1291
|
+
* an array is {@link WorkflowFilterOneOf.values}, and an object compared with
|
|
1292
|
+
* `===` never matches anything a source produced.
|
|
1293
|
+
*
|
|
1294
|
+
* **Dates are strings.** A source hands back whatever its driver decoded, and a
|
|
1295
|
+
* catalog that tried to parse dates here would have to pick a format, get it
|
|
1296
|
+
* wrong for one dialect, and produce a comparison that silently ordered rows
|
|
1297
|
+
* differently from the `WHERE` this predicate is meant to become. ISO-8601
|
|
1298
|
+
* strings sort correctly under `<` and under every SQL collation, which is why
|
|
1299
|
+
* `boundStatement` already compares watermarks as text.
|
|
1300
|
+
*/
|
|
1301
|
+
export type WorkflowFilterValue = string | number | boolean;
|
|
1302
|
+
/** Whether a stored value is one this language can compare against. */
|
|
1303
|
+
export declare function isWorkflowFilterValue(value: unknown): value is WorkflowFilterValue;
|
|
1304
|
+
/**
|
|
1305
|
+
* The column names a predicate may name.
|
|
1306
|
+
*
|
|
1307
|
+
* **The same pattern `boundStatement` requires of a watermark column**, and that
|
|
1308
|
+
* is the point rather than a coincidence: an identifier cannot be bound by any
|
|
1309
|
+
* driver, so pushing a predicate into a query means quoting the column into the
|
|
1310
|
+
* SQL, and a name carrying a quote, a dot or a space is refused rather than
|
|
1311
|
+
* escaped. Requiring it *now*, while the predicate is only ever evaluated in
|
|
1312
|
+
* memory, is what stops a graph being authored today that could never be pushed
|
|
1313
|
+
* down tomorrow.
|
|
1314
|
+
*
|
|
1315
|
+
* The cost, and it is real: a source whose column is called `Part Number` cannot
|
|
1316
|
+
* be filtered directly. Rename it in a transform first — which is a node that
|
|
1317
|
+
* already exists and whose output is a column this can name.
|
|
1318
|
+
*/
|
|
1319
|
+
export declare const WORKFLOW_FILTER_COLUMN_PATTERN: RegExp;
|
|
1320
|
+
/**
|
|
1321
|
+
* How deep `all`/`any` may nest.
|
|
1322
|
+
*
|
|
1323
|
+
* A bound rather than a trust, because a predicate arrives as JSON out of a
|
|
1324
|
+
* column and every function that walks one is recursive: a graph carrying a
|
|
1325
|
+
* thousand-deep tree would be a stack overflow inside a durable step rather than
|
|
1326
|
+
* a refusal naming the node. Six is past anything a person builds in a form and
|
|
1327
|
+
* far short of anything that costs a frame to walk.
|
|
1328
|
+
*/
|
|
1329
|
+
export declare const WORKFLOW_FILTER_MAX_DEPTH = 6;
|
|
1330
|
+
/**
|
|
1331
|
+
* How many values one {@link WorkflowFilterOneOf} may list.
|
|
1332
|
+
*
|
|
1333
|
+
* Bounded because the list travels in the graph, into the graph fingerprint, and
|
|
1334
|
+
* eventually into an `IN (...)` — and past a few hundred entries all three of
|
|
1335
|
+
* those stop being reasonable and the thing being expressed is a join against
|
|
1336
|
+
* another dataset rather than a filter. Wire that dataset in as a second source
|
|
1337
|
+
* and join it in a transform, which is the node that can.
|
|
1338
|
+
*/
|
|
1339
|
+
export declare const WORKFLOW_FILTER_MAX_VALUES = 500;
|
|
1340
|
+
/** One column against one value. The leaf almost every filter is made of. */
|
|
1341
|
+
export interface WorkflowFilterComparison {
|
|
1342
|
+
kind: 'compare';
|
|
1343
|
+
/** A bare column name. See {@link WORKFLOW_FILTER_COLUMN_PATTERN}. */
|
|
1344
|
+
column: string;
|
|
1345
|
+
operator: WorkflowFilterOperator;
|
|
1346
|
+
value: WorkflowFilterValue;
|
|
1347
|
+
}
|
|
1348
|
+
/**
|
|
1349
|
+
* One column against a list of values.
|
|
1350
|
+
*
|
|
1351
|
+
* Its own kind rather than an `equals` whose value is allowed to be an array,
|
|
1352
|
+
* because that shape makes `value` mean two things and every reader has to test
|
|
1353
|
+
* which — the flat-optional-fields mistake {@link WORKFLOW_PREDICATE_KINDS}
|
|
1354
|
+
* argues against, one level down again. It also has no honest single spelling in
|
|
1355
|
+
* SQL as a comparison: `IN (…)` takes a parenthesised list and one bound
|
|
1356
|
+
* parameter per entry.
|
|
1357
|
+
*/
|
|
1358
|
+
export interface WorkflowFilterOneOf {
|
|
1359
|
+
kind: 'oneOf';
|
|
1360
|
+
column: string;
|
|
1361
|
+
/** `notIn` rather than a `negated` flag, so the two read alike in a palette. */
|
|
1362
|
+
operator: 'in' | 'notIn';
|
|
1363
|
+
/** At most {@link WORKFLOW_FILTER_MAX_VALUES}, and never empty. */
|
|
1364
|
+
values: WorkflowFilterValue[];
|
|
1365
|
+
}
|
|
1366
|
+
/**
|
|
1367
|
+
* Whether a column has a value at all.
|
|
1368
|
+
*
|
|
1369
|
+
* The one test the other two cannot express, and the reason they cannot is the
|
|
1370
|
+
* three-valued logic on {@link workflowFilterMatches}: every `compare` and every
|
|
1371
|
+
* `oneOf` is false when the column is null, *including the inverses*, exactly as
|
|
1372
|
+
* a `WHERE` would answer. So "the ones with no delivery date" has to be its own
|
|
1373
|
+
* shape or it would be unaskable.
|
|
1374
|
+
*/
|
|
1375
|
+
export interface WorkflowFilterPresence {
|
|
1376
|
+
kind: 'present';
|
|
1377
|
+
column: string;
|
|
1378
|
+
operator: 'isNull' | 'isNotNull';
|
|
1379
|
+
}
|
|
1380
|
+
/**
|
|
1381
|
+
* What `all` and `any` share, which is everything except the word.
|
|
1382
|
+
*
|
|
1383
|
+
* **An empty group is refused**, by `validateWorkflow`, by
|
|
1384
|
+
* {@link isWorkflowFilterPredicate} and at the HTTP boundary. It is not a
|
|
1385
|
+
* pedantic refusal: `all` of nothing is vacuously true, so it keeps every row
|
|
1386
|
+
* and the filter does nothing; `any` of nothing is vacuously false, so it drops
|
|
1387
|
+
* every row and the load comes out empty. Both are silent, both are reached by
|
|
1388
|
+
* deleting the last condition in a form, and they are opposite catastrophes.
|
|
1389
|
+
*/
|
|
1390
|
+
interface WorkflowFilterGroupBase {
|
|
1391
|
+
/** Never empty. Nested no deeper than {@link WORKFLOW_FILTER_MAX_DEPTH}. */
|
|
1392
|
+
children: WorkflowFilterPredicate[];
|
|
1393
|
+
}
|
|
1394
|
+
/**
|
|
1395
|
+
* Every child holds.
|
|
1396
|
+
*
|
|
1397
|
+
* Two interfaces rather than one carrying `kind: 'all' | 'any'`, which was
|
|
1398
|
+
* written first and does not work: TypeScript will not remove a union member
|
|
1399
|
+
* from the union when its discriminant is itself a union of literals and the two
|
|
1400
|
+
* are tested separately, so `unreachableFilterPredicateKind` at the end of every
|
|
1401
|
+
* `switch`-by-`if` chain stopped compiling — which is to say the exhaustiveness
|
|
1402
|
+
* this whole file is arranged around was silently unavailable for these two
|
|
1403
|
+
* kinds. Two declarations over a shared base costs one line and buys the
|
|
1404
|
+
* property back.
|
|
1405
|
+
*/
|
|
1406
|
+
export interface WorkflowFilterAll extends WorkflowFilterGroupBase {
|
|
1407
|
+
kind: 'all';
|
|
1408
|
+
}
|
|
1409
|
+
/** At least one child holds. Two declarations, for the reason on {@link WorkflowFilterAll}. */
|
|
1410
|
+
export interface WorkflowFilterAny extends WorkflowFilterGroupBase {
|
|
1411
|
+
kind: 'any';
|
|
1412
|
+
}
|
|
1413
|
+
export type WorkflowFilterGroup = WorkflowFilterAll | WorkflowFilterAny;
|
|
1414
|
+
/** What a filter node tests. See {@link WORKFLOW_FILTER_PREDICATE_KINDS}. */
|
|
1415
|
+
export type WorkflowFilterPredicate = WorkflowFilterComparison | WorkflowFilterOneOf | WorkflowFilterPresence | WorkflowFilterAll | WorkflowFilterAny;
|
|
1416
|
+
/**
|
|
1417
|
+
* Narrow a stored filter predicate, refusing rather than repairing.
|
|
1418
|
+
*
|
|
1419
|
+
* The same contract {@link isWorkflowIfPredicate} has and for a sharper version
|
|
1420
|
+
* of the same reason: a gate read back without its test picks a branch nobody
|
|
1421
|
+
* authored, and a *filter* read back without its test decides which rows exist.
|
|
1422
|
+
* Repairing a broken predicate — dropping an unreadable child out of an `all`,
|
|
1423
|
+
* say — would silently widen or narrow what a load publishes, which is precisely
|
|
1424
|
+
* the failure the node is built to make visible.
|
|
1425
|
+
*
|
|
1426
|
+
* Depth is carried rather than tracked globally so that the bound is on the
|
|
1427
|
+
* *tree*, not on how many predicates have been checked: two sibling branches
|
|
1428
|
+
* five deep are fine, and one branch seven deep is not.
|
|
1429
|
+
*/
|
|
1430
|
+
export declare function isWorkflowFilterPredicate(value: unknown, depth?: number): value is WorkflowFilterPredicate;
|
|
1431
|
+
/**
|
|
1432
|
+
* Whether one row passes a predicate.
|
|
1433
|
+
*
|
|
1434
|
+
* Pure, allocation-free on the common path, and in core rather than in the
|
|
1435
|
+
* runner so that the console can describe — and one day preview — exactly what
|
|
1436
|
+
* the load will do, from the same function that does it. It is called once per
|
|
1437
|
+
* row, so everything about it is arranged to be cheap: no closures built per
|
|
1438
|
+
* row, no array built per row, and `note` is optional so a caller that does not
|
|
1439
|
+
* want the diagnostics pays nothing for them.
|
|
1440
|
+
*
|
|
1441
|
+
* ## Null is unknown, and unknown does not pass
|
|
1442
|
+
*
|
|
1443
|
+
* A column that is `null`, `undefined`, or simply absent from the row fails
|
|
1444
|
+
* **every** `compare` and **every** `oneOf`, *including the negative forms*.
|
|
1445
|
+
* `status notEquals "CLOSED"` does not pass a row with no status.
|
|
1446
|
+
*
|
|
1447
|
+
* That is SQL's three-valued logic rather than JavaScript's, and it is chosen
|
|
1448
|
+
* deliberately over the more intuitive JS answer for one reason: this predicate
|
|
1449
|
+
* is meant to become a `WHERE`, and the day it does, the database will answer
|
|
1450
|
+
* this way. A filter that kept those rows in memory and dropped them once pushed
|
|
1451
|
+
* down would be a performance change that quietly altered what a type contains —
|
|
1452
|
+
* which is the one thing a pushdown must never be able to do. {@link
|
|
1453
|
+
* WorkflowFilterPresence} is how absence is tested on purpose.
|
|
1454
|
+
*
|
|
1455
|
+
* ## A value it cannot compare fails both a test and its inverse
|
|
1456
|
+
*
|
|
1457
|
+
* `qty greaterThan 10` against a `qty` holding `"n/a"` is not false because the
|
|
1458
|
+
* string is small; there is no ordering between a string and a number that any
|
|
1459
|
+
* two systems would agree on. Rather than invent one, the leaf answers false and
|
|
1460
|
+
* calls `note` with the column, and the runner turns those counts into a line on
|
|
1461
|
+
* the run: *"12,431 rows held a value in `qty` the test could not compare."*
|
|
1462
|
+
*
|
|
1463
|
+
* A row nothing can judge is therefore dropped rather than kept. That is the
|
|
1464
|
+
* direction with a backstop — the sink's row-count bound sees the shrink, and a
|
|
1465
|
+
* full sink refuses to commit nothing at all — whereas keeping unjudged rows
|
|
1466
|
+
* would publish them into the type with nothing anywhere to notice.
|
|
1467
|
+
*/
|
|
1468
|
+
export declare function workflowFilterMatches(predicate: WorkflowFilterPredicate, row: Record<string, unknown>, note?: (column: string) => void): boolean;
|
|
1469
|
+
/**
|
|
1470
|
+
* Drops the rows that fail a declarative test, and reports how many.
|
|
1471
|
+
*
|
|
1472
|
+
* ## Why this is a node and not a transform that returns a subset
|
|
1473
|
+
*
|
|
1474
|
+
* A transform can already filter, so the kind has to earn itself. Three reasons,
|
|
1475
|
+
* in increasing order of how much they decide the shape:
|
|
1476
|
+
*
|
|
1477
|
+
* 1. **It is legible on the canvas.** "Keep the open orders" is readable from
|
|
1478
|
+
* the box; the same rule inside a transform is readable only by opening the
|
|
1479
|
+
* code, and only by somebody who can read the language it is written in.
|
|
1480
|
+
* 2. **Its effect is reportable.** A filter records rows in *and* rows out
|
|
1481
|
+
* ({@link WorkflowNodeOutcome.rowsIn}), so a run panel can say what was
|
|
1482
|
+
* dropped. A transform records one number, and a transform that quietly
|
|
1483
|
+
* started dropping 90% of its input looks exactly like a source that got
|
|
1484
|
+
* smaller. A filter whose effect is invisible is how data goes missing.
|
|
1485
|
+
* 3. **Only a declarative predicate can be pushed into the source.** This is the
|
|
1486
|
+
* one that fixes the design. Arbitrary code cannot be translated to SQL, so a
|
|
1487
|
+
* code filter is *always* the expensive path: every row is read off disk, sent
|
|
1488
|
+
* over the network and turned into a JS object before anything decides it was
|
|
1489
|
+
* unwanted. A closed structure of column, operator and value can become a
|
|
1490
|
+
* `WHERE`, and then the rows are never read at all.
|
|
1491
|
+
*
|
|
1492
|
+
* ## Where it runs today, and where it does not yet
|
|
1493
|
+
*
|
|
1494
|
+
* **Today: in memory, one staged batch at a time, always.** The predicate is
|
|
1495
|
+
* shaped so it *could* be pushed into a SQL source's query, and it is not, and
|
|
1496
|
+
* saying so plainly is better than implying a win that has not been measured.
|
|
1497
|
+
* What stands in the way is not the predicate — `boundStatement` in
|
|
1498
|
+
* `sources.ts` already wraps an author's SQL in `SELECT * FROM (…) WHERE …`
|
|
1499
|
+
* with a quoted identifier and a bound parameter, which is exactly the mechanism
|
|
1500
|
+
* a pushdown would reuse — but the fetcher contract: `SourceFetcher` receives a
|
|
1501
|
+
* connector, a secret, a watermark and a mode, and knows nothing about the
|
|
1502
|
+
* graph, while the runner that *does* know the graph dispatches by connector
|
|
1503
|
+
* kind alone. Threading a graph-derived predicate through that also drags in the
|
|
1504
|
+
* schema-discovery path, which shares `sqlTarget`.
|
|
1505
|
+
*
|
|
1506
|
+
* There is a second reason, and it is the more interesting one: a pushed-down
|
|
1507
|
+
* filter **cannot honestly report rows in**. The rows it dropped were never
|
|
1508
|
+
* fetched, so "7,637,391 in, 96,204 out" would become "96,204 in, 96,204 out,
|
|
1509
|
+
* nothing dropped" — reason 2 above, deleted by reason 3. Recovering the number
|
|
1510
|
+
* means a second `COUNT(*)` over the unfiltered query, which is the scan the
|
|
1511
|
+
* pushdown existed to avoid. Whichever way that is resolved, it is a decision
|
|
1512
|
+
* about what a run reports and not a refactor, so it belongs in the change that
|
|
1513
|
+
* makes the move rather than in a field added speculatively here.
|
|
1514
|
+
*
|
|
1515
|
+
* The seam is therefore one marked place in `WorkflowRunnerService.runFilter`
|
|
1516
|
+
* rather than an unused translator sitting in this file waiting to rot.
|
|
1517
|
+
*
|
|
1518
|
+
* ## The trap this node had to be designed against: {@link narrows}
|
|
1519
|
+
*
|
|
1520
|
+
* Dropping a filter onto an existing `source → sink` wire replaces the published
|
|
1521
|
+
* snapshot of that type with a subset — and every part of the run reports
|
|
1522
|
+
* success, because from the run's point of view everything did succeed. See
|
|
1523
|
+
* {@link narrows} for how the graph is made to tell that apart from filtering to
|
|
1524
|
+
* derive something new, and why it cannot be told apart structurally.
|
|
1525
|
+
*/
|
|
1526
|
+
export interface WorkflowFilterNode extends WorkflowNodeBase {
|
|
1527
|
+
kind: 'filter';
|
|
1528
|
+
/**
|
|
1529
|
+
* What a row has to satisfy to pass. See
|
|
1530
|
+
* {@link WORKFLOW_FILTER_PREDICATE_KINDS}.
|
|
1531
|
+
*/
|
|
1532
|
+
predicate: WorkflowFilterPredicate;
|
|
1533
|
+
/**
|
|
1534
|
+
* The object types whose published snapshot this filter is acknowledged to
|
|
1535
|
+
* narrow.
|
|
1536
|
+
*
|
|
1537
|
+
* ## The two intentions, and why the graph cannot tell them apart
|
|
1538
|
+
*
|
|
1539
|
+
* Filtering to **derive a new type** — `source → filter → sink(OpenOrders)` —
|
|
1540
|
+
* and filtering before **recommitting the same type** —
|
|
1541
|
+
* `source → filter → sink(PriBuy)`, where `PriBuy` was until this morning the
|
|
1542
|
+
* whole table — are *structurally identical graphs*. The only thing that
|
|
1543
|
+
* differs is what the type on the sink already means to everybody reading it,
|
|
1544
|
+
* and there is nothing in the nodes or the edges that knows that. Any rule
|
|
1545
|
+
* claiming to distinguish them from the shape alone would be inventing a
|
|
1546
|
+
* signal, and would then either refuse the safe case or wave the dangerous one
|
|
1547
|
+
* through.
|
|
1548
|
+
*
|
|
1549
|
+
* So the graph makes the author **name the types**, and that naming is the
|
|
1550
|
+
* whole mechanism: typing `OpenOrders` is a different act from typing
|
|
1551
|
+
* `PriBuy`, and nobody types the second one by accident. What makes it a
|
|
1552
|
+
* safeguard rather than a checkbox is that it is *required exactly where it
|
|
1553
|
+
* matters and refused everywhere else*, both checked by `validateWorkflow`:
|
|
1554
|
+
*
|
|
1555
|
+
* - It must list **every** full-mode sink this filter stands in front of *on
|
|
1556
|
+
* every path* — that is, every sink whose entire snapshot would be a subset
|
|
1557
|
+
* because of this node. Removing the node would make that sink unreachable;
|
|
1558
|
+
* see `workflowNarrowedTypes`, which is the one implementation both the
|
|
1559
|
+
* validator and the console call.
|
|
1560
|
+
* - It must list **nothing else**. A type named here that this filter does not
|
|
1561
|
+
* in fact narrow is refused, for the reason a branch label on a plain wire
|
|
1562
|
+
* is: an acknowledgement nothing reads is worse than none, because it is
|
|
1563
|
+
* drawn.
|
|
1564
|
+
*
|
|
1565
|
+
* A filter on one of several paths into a sink narrows nothing — other rows
|
|
1566
|
+
* still reach it — and a filter in front of an incremental sink narrows
|
|
1567
|
+
* nothing either, because an incremental commit merges into what is already
|
|
1568
|
+
* there rather than replacing it. Neither has anything to declare, and neither
|
|
1569
|
+
* may declare it.
|
|
1570
|
+
*
|
|
1571
|
+
* The consequence is the intended one: dragging a filter onto a working
|
|
1572
|
+
* `source → sink` wire produces a graph that **will not save** until somebody
|
|
1573
|
+
* writes down the name of the type they are about to shrink. It is in the
|
|
1574
|
+
* graph fingerprint, so acknowledging it is a new version of the graph.
|
|
1575
|
+
*
|
|
1576
|
+
* The run-time backstop is unchanged and still the last word: the sink's
|
|
1577
|
+
* row-count bound (`maxShrink`) refuses a commit that loses more of the served
|
|
1578
|
+
* snapshot than the type allows, whatever this field says.
|
|
1579
|
+
*/
|
|
1580
|
+
narrows?: string[];
|
|
1581
|
+
}
|
|
1183
1582
|
/**
|
|
1184
1583
|
* A discriminated union, so narrowing a node is `node.kind === "sink"` and
|
|
1185
1584
|
* never a type assertion. This is why the kind list is not simply a string on
|
|
1186
1585
|
* one node shape with every field optional: that shape lets a source node carry
|
|
1187
1586
|
* a `transformId` and nothing catches it.
|
|
1188
1587
|
*/
|
|
1189
|
-
export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode;
|
|
1588
|
+
export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode;
|
|
1190
1589
|
/**
|
|
1191
1590
|
* Which side of an {@link WorkflowIfNode} a wire leaves by.
|
|
1192
1591
|
*
|
|
@@ -1474,6 +1873,13 @@ export interface WorkflowNodeStepOutput {
|
|
|
1474
1873
|
*/
|
|
1475
1874
|
branch?: WorkflowBranchLabel;
|
|
1476
1875
|
rows: number;
|
|
1876
|
+
/**
|
|
1877
|
+
* What a {@link WorkflowFilterNode} was handed, against `rows` above, which is
|
|
1878
|
+
* what it passed on. See {@link WorkflowNodeOutcome.rowsIn}: it travels on the
|
|
1879
|
+
* step's output so that a replayed node reports the same drop the first
|
|
1880
|
+
* attempt did, rather than reporting nothing because it was not re-run.
|
|
1881
|
+
*/
|
|
1882
|
+
rowsIn?: number;
|
|
1477
1883
|
elapsedMs: number;
|
|
1478
1884
|
/**
|
|
1479
1885
|
* The one thing here that is not a counter, and the one exception worth
|
|
@@ -1682,7 +2088,7 @@ export interface CallableWorkflowBlock {
|
|
|
1682
2088
|
}
|
|
1683
2089
|
export declare function callableWorkflowBlock(ref: CallableWorkflowRef): CallableWorkflowBlock | undefined;
|
|
1684
2090
|
/** Every way a graph can be refused. Exported so a canvas can key off the code. */
|
|
1685
|
-
export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "call-not-named", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge"];
|
|
2091
|
+
export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "call-not-named", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing"];
|
|
1686
2092
|
export type WorkflowIssueCode = (typeof WORKFLOW_ISSUE_CODES)[number];
|
|
1687
2093
|
export interface WorkflowValidationIssue {
|
|
1688
2094
|
code: WorkflowIssueCode;
|
|
@@ -1713,6 +2119,35 @@ export interface WorkflowValidationIssue {
|
|
|
1713
2119
|
* being read.
|
|
1714
2120
|
*/
|
|
1715
2121
|
export declare function validateWorkflow(graph: WorkflowGraph): WorkflowValidationIssue[];
|
|
2122
|
+
/**
|
|
2123
|
+
* The object types whose whole published snapshot a node stands in front of.
|
|
2124
|
+
*
|
|
2125
|
+
* A sink is in this list when it commits in `full` mode — replacing what is
|
|
2126
|
+
* served rather than merging into it — **and** removing the named node would
|
|
2127
|
+
* make it unreachable from everything that originates rows. That second half is
|
|
2128
|
+
* the load-bearing one: a filter on one of two paths into a sink narrows
|
|
2129
|
+
* nothing, because the other path still delivers, and a rule that ignored it
|
|
2130
|
+
* would demand an acknowledgement for a graph where nothing is lost.
|
|
2131
|
+
*
|
|
2132
|
+
* Removal-reachability rather than a dominator algorithm, deliberately. The
|
|
2133
|
+
* graphs here are a screenful of boxes, this runs once per filter node, and the
|
|
2134
|
+
* cheaper version would be a second, subtler implementation of "does this node
|
|
2135
|
+
* decide whether that one runs" living next to `checkReachability` — which is
|
|
2136
|
+
* exactly the kind of duplication that eventually disagrees with the walk the
|
|
2137
|
+
* rest of this file does.
|
|
2138
|
+
*
|
|
2139
|
+
* Exported because the console needs the same answer to offer the right
|
|
2140
|
+
* acknowledgements, and a canvas that computed its own would offer a set the
|
|
2141
|
+
* server then refuses.
|
|
2142
|
+
*
|
|
2143
|
+
* `precomputed` exists only so the validator, which has already built the
|
|
2144
|
+
* adjacency it needs, does not build it twice per filter node. Callers outside
|
|
2145
|
+
* this file pass a graph and nothing else.
|
|
2146
|
+
*/
|
|
2147
|
+
export declare function workflowNarrowedTypes(graph: WorkflowGraph, nodeId: string, precomputed?: {
|
|
2148
|
+
originators: string[];
|
|
2149
|
+
outgoing: ReadonlyMap<string, string[]>;
|
|
2150
|
+
}): string[];
|
|
1716
2151
|
/**
|
|
1717
2152
|
* The order the nodes run in, and the inputs each one gets.
|
|
1718
2153
|
*
|