@dudousxd/nestjs-catalog 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.events.d.ts +14 -0
- package/dist/catalog.pipeline.d.ts +1124 -75
- package/dist/catalog.pipeline.js +1150 -14
- package/dist/catalog.stage-encoding.d.ts +185 -0
- package/dist/catalog.stage-encoding.js +314 -0
- package/dist/client.d.ts +14 -4
- package/dist/client.js +85 -12
- package/dist/index.d.ts +2 -1
- package/dist/index.js +48 -3
- package/dist/transform-runner.d.ts +19 -1
- package/dist/transform-runner.js +63 -7
- package/package.json +6 -6
- package/LICENSE +0 -21
|
@@ -406,12 +406,18 @@ export interface CatalogTransform {
|
|
|
406
406
|
description?: string;
|
|
407
407
|
language: TransformLanguage;
|
|
408
408
|
/**
|
|
409
|
-
* The body of a function over one batch. It receives `records` and
|
|
410
|
-
* the rows to store.
|
|
409
|
+
* The body of a function over one batch. It receives `records` and
|
|
410
|
+
* `context`, and returns the rows to store.
|
|
411
411
|
*
|
|
412
412
|
* A batch rather than a record at a time: a transform that needs to look up,
|
|
413
413
|
* deduplicate or aggregate cannot do it one row at a time, and paying one
|
|
414
414
|
* process spawn per record would make any real load unusable.
|
|
415
|
+
*
|
|
416
|
+
* `context` is a {@link CatalogCodeContext} — the run, the node, the counts
|
|
417
|
+
* of what fed it, and the environment variables this deployment admits.
|
|
418
|
+
* Second rather than first, so that every transform written before it existed
|
|
419
|
+
* still runs: the harness supplies the parameter, and code that never names
|
|
420
|
+
* it is unaffected.
|
|
415
421
|
*/
|
|
416
422
|
code: string;
|
|
417
423
|
version: number;
|
|
@@ -460,6 +466,7 @@ export interface TransformResult {
|
|
|
460
466
|
export interface TransformRunner {
|
|
461
467
|
run(transform: Pick<CatalogTransform, 'language' | 'code'>, records: unknown[], options?: {
|
|
462
468
|
timeoutMs?: number;
|
|
469
|
+
context?: CatalogCodeContext;
|
|
463
470
|
}): Promise<TransformResult>;
|
|
464
471
|
/** Languages this runner can actually execute in this environment. */
|
|
465
472
|
available(): Promise<TransformLanguage[]>;
|
|
@@ -473,6 +480,123 @@ export interface TransformRunner {
|
|
|
473
480
|
pythonPackages?(): Promise<string[]>;
|
|
474
481
|
}
|
|
475
482
|
export declare const TRANSFORM_RUNNER: unique symbol;
|
|
483
|
+
/**
|
|
484
|
+
* The number in {@link CatalogCodeContext.contract}.
|
|
485
|
+
*
|
|
486
|
+
* Its own version, separate from {@link WORKFLOW_CALL_CONTRACT}, because the
|
|
487
|
+
* two travel to different places for different reasons — a call envelope
|
|
488
|
+
* crosses to another SDK over the durable wire, a code context crosses to a
|
|
489
|
+
* child process this repository spawned — and a transform written against one
|
|
490
|
+
* has to be able to say which it read.
|
|
491
|
+
*/
|
|
492
|
+
export declare const CODE_CONTEXT_CONTRACT = 1;
|
|
493
|
+
/**
|
|
494
|
+
* The second argument every code-bearing node's code is handed.
|
|
495
|
+
*
|
|
496
|
+
* ## Why this exists at all
|
|
497
|
+
*
|
|
498
|
+
* A transform is a function over a batch, and a batch is not the whole of what
|
|
499
|
+
* the code needs to know. It needs the credential for the API it enriches
|
|
500
|
+
* against; it needs to say which run it belongs to when it logs; and — the case
|
|
501
|
+
* that motivated this — a conditional node's predicate has no `records` at all
|
|
502
|
+
* and still has to answer "did the source return anything", which is the guard
|
|
503
|
+
* that stops an empty snapshot being committed over live data.
|
|
504
|
+
*
|
|
505
|
+
* ## Why it is not `process.env`
|
|
506
|
+
*
|
|
507
|
+
* The obvious answer to "code needs environment variables" is to stop trimming
|
|
508
|
+
* the child's environment. It is the wrong one, and this repository has already
|
|
509
|
+
* argued the case in `secret-env-allowlist.ts`: a `secretEnvVar` is chosen by
|
|
510
|
+
* whoever writes the connector, so `process.env[name]` with nothing in between
|
|
511
|
+
* let anyone holding `catalog:write` on one narrow object type read the host
|
|
512
|
+
* application's own `DATABASE_URL`. Transform code is *the same principal by a
|
|
513
|
+
* shorter route* — it is a string that principal saved, and it can print
|
|
514
|
+
* whatever it reads into `logs`, which land in the run record and are served at
|
|
515
|
+
* `catalog:read`.
|
|
516
|
+
*
|
|
517
|
+
* So {@link env} is the allow-listed environment and nothing else: the same
|
|
518
|
+
* policy, the same two levers, the same boot warning. One rule across the
|
|
519
|
+
* product rather than a second one that quietly undoes the first. What the
|
|
520
|
+
* child's *own* `process.env` holds is unchanged — still `{PATH, NODE_ENV}` —
|
|
521
|
+
* and the admitted values arrive as data on stdin instead.
|
|
522
|
+
*
|
|
523
|
+
* ## Replay
|
|
524
|
+
*
|
|
525
|
+
* Everything here is plain JSON, and everything except {@link env} and
|
|
526
|
+
* {@link environment} is derived from a durable step's checkpointed input: the
|
|
527
|
+
* run id, the node, the graph's version and the stage handles are byte-
|
|
528
|
+
* identical on every attempt and on every replay. Those two are reads of
|
|
529
|
+
* pod-local state, which for a transform is harmless — a transform runs inside
|
|
530
|
+
* a step whose *output* is checkpointed, so replay returns the answer rather
|
|
531
|
+
* than re-running the code — and for a predicate evaluated in a workflow body
|
|
532
|
+
* is not. A caller in that position must resolve the context inside a step and
|
|
533
|
+
* let the checkpoint carry it, so that a redeploy between the original run and
|
|
534
|
+
* the replay cannot move the branch. The shape is JSON precisely so that it can
|
|
535
|
+
* be.
|
|
536
|
+
*/
|
|
537
|
+
export interface CatalogCodeContext {
|
|
538
|
+
/** {@link CODE_CONTEXT_CONTRACT} at the time the code ran. */
|
|
539
|
+
contract: number;
|
|
540
|
+
/**
|
|
541
|
+
* The run this code belongs to, which is also the snapshot id.
|
|
542
|
+
*
|
|
543
|
+
* Absent means there is no run: the editor's try pane executes code against
|
|
544
|
+
* sample records and stores nothing. Anything derived from this — an
|
|
545
|
+
* idempotency key, a filename — should refuse rather than invent one, and the
|
|
546
|
+
* absence is the signal that lets it.
|
|
547
|
+
*/
|
|
548
|
+
runId?: string;
|
|
549
|
+
/** The graph, when the code is a node in one. Absent for a connector's single transform. */
|
|
550
|
+
workflow?: {
|
|
551
|
+
id: string;
|
|
552
|
+
name: string;
|
|
553
|
+
version: number;
|
|
554
|
+
};
|
|
555
|
+
/** The node, when the code is a node. Absent for a connector's single transform. */
|
|
556
|
+
node?: {
|
|
557
|
+
id: string;
|
|
558
|
+
name: string;
|
|
559
|
+
};
|
|
560
|
+
/** The connector, when the code runs as a connector's transform rather than in a graph. */
|
|
561
|
+
connectorId?: string;
|
|
562
|
+
/**
|
|
563
|
+
* The host's name for this copy of the world — `dev`, `prod` — when the host
|
|
564
|
+
* declared one.
|
|
565
|
+
*
|
|
566
|
+
* Nothing in the catalog can work this out for itself: a process may serve
|
|
567
|
+
* several environments and the choice arrives per request, so the only
|
|
568
|
+
* honest source is the host saying so. Present because branching on a named
|
|
569
|
+
* environment is legible in a way that sniffing a variable is not, and absent
|
|
570
|
+
* whenever the host stayed silent, which is a different thing from `dev`.
|
|
571
|
+
*/
|
|
572
|
+
environment?: string;
|
|
573
|
+
/**
|
|
574
|
+
* How many records reached this code.
|
|
575
|
+
*
|
|
576
|
+
* Redundant with `records.length` for a transform and the whole payload for a
|
|
577
|
+
* predicate, which has no records. Stated as one number because the line a
|
|
578
|
+
* predicate is written to hold is "did anything arrive", and making that a
|
|
579
|
+
* sum over {@link inputs} invites `inputs[0].rowCount`, which is wrong the
|
|
580
|
+
* moment somebody draws a second edge.
|
|
581
|
+
*/
|
|
582
|
+
rowCount: number;
|
|
583
|
+
/**
|
|
584
|
+
* Per inbound edge, in edge order — handles and counts, never the rows.
|
|
585
|
+
*
|
|
586
|
+
* The same {@link WorkflowStageRef} the call node hands a callee, deliberately
|
|
587
|
+
* rather than a second vocabulary for the same fact. Empty outside a graph.
|
|
588
|
+
*/
|
|
589
|
+
inputs: WorkflowStageRef[];
|
|
590
|
+
/**
|
|
591
|
+
* The environment variables this deployment admits, and only those.
|
|
592
|
+
*
|
|
593
|
+
* Filtered by the credential allow-list — see `secret-env-allowlist.ts` — so
|
|
594
|
+
* a name nobody admitted is not here, and an empty object is the ordinary
|
|
595
|
+
* answer on a deployment that has declared no policy. The run's logs say
|
|
596
|
+
* which it was.
|
|
597
|
+
*/
|
|
598
|
+
env: Record<string, string>;
|
|
599
|
+
}
|
|
476
600
|
export interface ConnectorRun {
|
|
477
601
|
id: string;
|
|
478
602
|
connectorId: string;
|
|
@@ -539,18 +663,74 @@ export interface ConnectorRun {
|
|
|
539
663
|
*/
|
|
540
664
|
nodeOutcomes?: Record<string, WorkflowNodeOutcome>;
|
|
541
665
|
}
|
|
666
|
+
/**
|
|
667
|
+
* Why a `skipped` node did not run, when there is more to say than "the run
|
|
668
|
+
* stopped".
|
|
669
|
+
*
|
|
670
|
+
* **One entry, and the omission is the point.** `skipped` already meant one
|
|
671
|
+
* thing before branches existed — the run failed upstream and never reached
|
|
672
|
+
* this node — and every outcome ever stored records that meaning by *not*
|
|
673
|
+
* carrying a reason. Adding `run-stopped` to this list would not describe those
|
|
674
|
+
* rows, it would leave them describing an unknown reason, so the pre-existing
|
|
675
|
+
* meaning stays the absent one and this names only the new fact: the node is on
|
|
676
|
+
* a branch an {@link WorkflowIfNode} did not take.
|
|
677
|
+
*
|
|
678
|
+
* The distinction is not cosmetic. A sink skipped by a branch **committed
|
|
679
|
+
* nothing and left the live snapshot alone**, which is a correct, successful
|
|
680
|
+
* outcome; a sink skipped by a failure is part of a load that went wrong. A run
|
|
681
|
+
* panel that rendered both as "did not run" would answer "why is there no data
|
|
682
|
+
* in X" with the same shrug in both cases.
|
|
683
|
+
*/
|
|
684
|
+
export declare const WORKFLOW_SKIP_REASONS: readonly ["branch-not-taken"];
|
|
685
|
+
export type WorkflowSkipReason = (typeof WORKFLOW_SKIP_REASONS)[number];
|
|
686
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
687
|
+
export declare function isWorkflowSkipReason(value: unknown): value is WorkflowSkipReason;
|
|
542
688
|
/** What one node did during a run. Small by construction — counters, not rows. */
|
|
543
689
|
export interface WorkflowNodeOutcome {
|
|
544
690
|
/**
|
|
545
691
|
* `skipped` exists for the nodes downstream of a failure. Without it, a
|
|
546
692
|
* ten-node graph that died at node seven records three nodes with no entry at
|
|
547
693
|
* all, which reads the same as three nodes nobody has looked at yet.
|
|
694
|
+
*
|
|
695
|
+
* It now carries a second, legitimate meaning as well — a node on the branch
|
|
696
|
+
* an `if` did not take — and {@link skippedBecause} is what tells the two
|
|
697
|
+
* apart.
|
|
548
698
|
*/
|
|
549
699
|
status: 'succeeded' | 'failed' | 'skipped';
|
|
550
700
|
/** Rows this node produced, or committed if it is the sink. */
|
|
551
701
|
rows: number;
|
|
702
|
+
/**
|
|
703
|
+
* Rows this node was *given*, for a node whose whole purpose is that the two
|
|
704
|
+
* numbers differ. Set by {@link WorkflowFilterNode} and absent everywhere else.
|
|
705
|
+
*
|
|
706
|
+
* Two numbers rather than a `dropped` counter, because `dropped` is
|
|
707
|
+
* `rowsIn - rows` and a third stored number is a third thing that can
|
|
708
|
+
* disagree with the other two. A run panel subtracts.
|
|
709
|
+
*
|
|
710
|
+
* Optional, so an outcome written before filters existed reads back as what it
|
|
711
|
+
* is — a node that never reported an input count — rather than as one that
|
|
712
|
+
* received nothing. Absent and zero are different facts here, and conflating
|
|
713
|
+
* them would make every historical transform look like it dropped everything.
|
|
714
|
+
*/
|
|
715
|
+
rowsIn?: number;
|
|
552
716
|
/** For a transform node: which version of its code ran. */
|
|
553
717
|
transformVersion?: number;
|
|
718
|
+
/**
|
|
719
|
+
* For an {@link WorkflowIfNode}: the branch this run took.
|
|
720
|
+
*
|
|
721
|
+
* **Written once, on the first evaluation, and read back on every replay.**
|
|
722
|
+
* This is the record the durable path replays from rather than re-evaluating:
|
|
723
|
+
* a predicate reads the environment, and an environment is pod-local, so a
|
|
724
|
+
* replay landing on another pod could otherwise take the other branch halfway
|
|
725
|
+
* through a run and load through a shape nobody chose. See
|
|
726
|
+
* `WorkflowRunnerService.runIf`.
|
|
727
|
+
*
|
|
728
|
+
* It is also the answer to "why did nothing load into X", which is the
|
|
729
|
+
* question a branch makes askable and nothing else on a run can answer.
|
|
730
|
+
*/
|
|
731
|
+
branch?: WorkflowBranchLabel;
|
|
732
|
+
/** See {@link WORKFLOW_SKIP_REASONS}. Absent means the run stopped short. */
|
|
733
|
+
skippedBecause?: WorkflowSkipReason;
|
|
554
734
|
elapsedMs?: number;
|
|
555
735
|
error?: string;
|
|
556
736
|
}
|
|
@@ -568,13 +748,28 @@ export interface WorkflowNodeOutcome {
|
|
|
568
748
|
* The kinds that were considered and rejected, since a small vocabulary is only
|
|
569
749
|
* defensible if the omissions are:
|
|
570
750
|
*
|
|
571
|
-
* - **filter** —
|
|
572
|
-
*
|
|
573
|
-
*
|
|
574
|
-
* missing.
|
|
575
|
-
*
|
|
576
|
-
*
|
|
577
|
-
*
|
|
751
|
+
* - **filter** — *this entry used to be a rejection, and it is worth leaving the
|
|
752
|
+
* reversal visible rather than editing the history out.* The argument was that
|
|
753
|
+
* a transform whose code returns a subset already filters, so the kind bought
|
|
754
|
+
* a second way to drop rows and a second place to look when rows went missing.
|
|
755
|
+
* That argument is sound about *code* and it is the reason
|
|
756
|
+
* {@link WorkflowFilterNode} does not take any: what changed is that the
|
|
757
|
+
* predicate is a **closed structure** rather than a body, and a closed
|
|
758
|
+
* structure can be read by something other than a JavaScript engine. Only a
|
|
759
|
+
* declarative predicate can be translated into a `WHERE` and pushed into the
|
|
760
|
+
* query the source already runs, and that is not a micro-optimisation — a
|
|
761
|
+
* transform filtering `obj_pribuybuylistdetail` reads all 7,637,391 rows off
|
|
762
|
+
* disk, over the network, and into JS objects of ~80 properties each before
|
|
763
|
+
* anything decides they were unwanted. See {@link WorkflowFilterNode} for what
|
|
764
|
+
* is actually built today (an in-memory, per-batch pass) and for where the
|
|
765
|
+
* pushdown seam is, which is a promise about a shape rather than a claim about
|
|
766
|
+
* a measurement.
|
|
767
|
+
* - **branch / split (unconditional)** — already expressible, and still is: a
|
|
768
|
+
* node with two outbound edges is read by both successors, each of which
|
|
769
|
+
* filters differently. There is nothing for an *unconditional* split to do.
|
|
770
|
+
* {@link WorkflowIfNode} is the conditional one, and it earns its kind by
|
|
771
|
+
* doing something no wiring can express — deciding that one of those
|
|
772
|
+
* successors, and everything only it feeds, does not run at all.
|
|
578
773
|
* - **merge / join** — a node with several inbound edges receives its inputs
|
|
579
774
|
* concatenated in edge order (see {@link WorkflowEdge}). A keyed join is then
|
|
580
775
|
* ordinary code inside the transform, which can already see every record.
|
|
@@ -591,10 +786,28 @@ export interface WorkflowNodeOutcome {
|
|
|
591
786
|
* outside a run, which is why `call` names one and not a step. If a step is
|
|
592
787
|
* what you want, the thing to call is a one-step workflow wrapping it.
|
|
593
788
|
*/
|
|
594
|
-
export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call"];
|
|
789
|
+
export declare const WORKFLOW_NODE_KINDS: readonly ["source", "transform", "sink", "call", "if", "filter"];
|
|
595
790
|
export type WorkflowNodeKind = (typeof WORKFLOW_NODE_KINDS)[number];
|
|
596
791
|
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
597
792
|
export declare function isWorkflowNodeKind(value: unknown): value is WorkflowNodeKind;
|
|
793
|
+
/**
|
|
794
|
+
* The kind that never compiles quietly.
|
|
795
|
+
*
|
|
796
|
+
* Every place that decides something *per kind* ends in a call to this, so a
|
|
797
|
+
* kind added to {@link WORKFLOW_NODE_KINDS} without a branch there is a type
|
|
798
|
+
* error naming the file rather than a graph that saves, validates, draws and
|
|
799
|
+
* then does the wrong thing. This codebase has been bitten by exactly that
|
|
800
|
+
* shape — a `toGraph` branch forgetting a field, a node-kind map missing a kind
|
|
801
|
+
* — and the fix each time was to make the omission impossible rather than to
|
|
802
|
+
* remember harder.
|
|
803
|
+
*
|
|
804
|
+
* It throws as well as failing to compile, because the narrowing that reaches
|
|
805
|
+
* it is over data that arrives as JSON: a node whose `kind` passed
|
|
806
|
+
* {@link isWorkflowNodeKind} in an older build and reaches a newer one is
|
|
807
|
+
* possible, and returning a default for it would be the silent path this exists
|
|
808
|
+
* to close.
|
|
809
|
+
*/
|
|
810
|
+
export declare function unreachableNodeKind(node: never, where: string): never;
|
|
598
811
|
/**
|
|
599
812
|
* The longest a node id may be, and the alphabet it may use.
|
|
600
813
|
*
|
|
@@ -618,12 +831,68 @@ interface WorkflowNodeBase {
|
|
|
618
831
|
* a person arranged and the server then forgot is a canvas that loses work.
|
|
619
832
|
* Excluded from the graph fingerprint for the same reason a rename is: moving
|
|
620
833
|
* a box is not a new version of the graph.
|
|
834
|
+
*
|
|
835
|
+
* A position is in the units {@link WORKFLOW_NODE_WIDTH} is measured in, and
|
|
836
|
+
* anything **generating** one should space it with {@link workflowColumnX} and
|
|
837
|
+
* {@link workflowRowY} rather than by picking a number.
|
|
621
838
|
*/
|
|
622
839
|
position?: {
|
|
623
840
|
x: number;
|
|
624
841
|
y: number;
|
|
625
842
|
};
|
|
626
843
|
}
|
|
844
|
+
/**
|
|
845
|
+
* How big a node is on the canvas, and therefore how far apart two of them have
|
|
846
|
+
* to be.
|
|
847
|
+
*
|
|
848
|
+
* ## Why the server owns a number about pixels
|
|
849
|
+
*
|
|
850
|
+
* Because the server *writes positions*. `adoptConnector` lays a connector out
|
|
851
|
+
* as a graph, and anything else that mints a graph without a person drawing it —
|
|
852
|
+
* a promotion, a template, a script — has the same job. A writer that does not
|
|
853
|
+
* know how wide a node is can only guess, and the guess was wrong: adoption used
|
|
854
|
+
* to place columns 220 apart against a node 224 wide, so every adopted graph
|
|
855
|
+
* drew each box overlapping the next by four pixels. Nothing was stacked and
|
|
856
|
+
* nothing was missing, so it read as boxes glued together rather than as a bug,
|
|
857
|
+
* and it survived until somebody opened thirteen of them.
|
|
858
|
+
*
|
|
859
|
+
* The fix is not a bigger number. 220 was not too small, it was **derived from
|
|
860
|
+
* nothing** — it had no relationship to the width it was supposed to clear, so
|
|
861
|
+
* it was only ever correct by luck and would go wrong again the next time the
|
|
862
|
+
* node's styling changed. These constants are the relationship, stated once, in
|
|
863
|
+
* the one package both the writer and the canvas already depend on.
|
|
864
|
+
*
|
|
865
|
+
* ## What pins them to the drawing
|
|
866
|
+
*
|
|
867
|
+
* {@link WORKFLOW_NODE_WIDTH} is not a description of the node — it is the
|
|
868
|
+
* *source of* the node's width. `WorkflowNodeBody` in
|
|
869
|
+
* `@dudousxd/nestjs-catalog-react` sets its own width from this constant rather
|
|
870
|
+
* than from a Tailwind class, so the two cannot drift: changing this changes the
|
|
871
|
+
* box, and there is no second number to forget.
|
|
872
|
+
*/
|
|
873
|
+
export declare const WORKFLOW_NODE_WIDTH = 224;
|
|
874
|
+
/** How tall a node is. The other half of {@link WORKFLOW_NODE_WIDTH}'s contract. */
|
|
875
|
+
export declare const WORKFLOW_NODE_HEIGHT = 80;
|
|
876
|
+
/**
|
|
877
|
+
* Clear space between one column and the next, on top of the node's own width.
|
|
878
|
+
*
|
|
879
|
+
* Wide enough for the edge between two nodes to be read as a line with a
|
|
880
|
+
* direction rather than as a join. This is the part that is taste; the width it
|
|
881
|
+
* is added to is not.
|
|
882
|
+
*/
|
|
883
|
+
export declare const WORKFLOW_COLUMN_GAP = 96;
|
|
884
|
+
/** Clear space between two nodes sharing a column. */
|
|
885
|
+
export declare const WORKFLOW_ROW_GAP = 32;
|
|
886
|
+
/**
|
|
887
|
+
* The x of the nth column, counting from zero.
|
|
888
|
+
*
|
|
889
|
+
* Every generator of a layout goes through this rather than multiplying by a
|
|
890
|
+
* literal, which is what makes "columns never overlap" a property of one
|
|
891
|
+
* function instead of a coincidence repeated at each call site.
|
|
892
|
+
*/
|
|
893
|
+
export declare function workflowColumnX(column: number): number;
|
|
894
|
+
/** The y of the nth row within a column, counting from zero. */
|
|
895
|
+
export declare function workflowRowY(row: number): number;
|
|
627
896
|
/**
|
|
628
897
|
* Reads records out of a system.
|
|
629
898
|
*
|
|
@@ -743,13 +1012,602 @@ export interface WorkflowCallNode extends WorkflowNodeBase {
|
|
|
743
1012
|
/** Parameters the author typed, handed to the child under `input`. */
|
|
744
1013
|
config: Record<string, unknown>;
|
|
745
1014
|
}
|
|
1015
|
+
/**
|
|
1016
|
+
* The kinds of test an {@link WorkflowIfNode} can make.
|
|
1017
|
+
*
|
|
1018
|
+
* A second predicate shape was always going to arrive — the note on
|
|
1019
|
+
* {@link WorkflowIfNode} says so about `code` — and the shape it arrives into is
|
|
1020
|
+
* the decision worth arguing about, because the alternative was to keep both
|
|
1021
|
+
* tests' fields flat on the node and mark them optional. That version types a
|
|
1022
|
+
* gate as "an env var, maybe, and a threshold, maybe": a node carrying both is
|
|
1023
|
+
* representable, a node carrying neither is representable, and every reader has
|
|
1024
|
+
* to invent its own rule for which one wins. It is the same mistake the note on
|
|
1025
|
+
* {@link WorkflowNode} refuses for node kinds, one level down.
|
|
1026
|
+
*
|
|
1027
|
+
* So the predicate is a union with a discriminant of its own, and every decision
|
|
1028
|
+
* made per predicate kind ends in {@link unreachablePredicateKind} — a third
|
|
1029
|
+
* shape is then a build failure listing the files that have to answer for it,
|
|
1030
|
+
* rather than a gate that saves, draws, and quietly always takes the `else`.
|
|
1031
|
+
*/
|
|
1032
|
+
export declare const WORKFLOW_PREDICATE_KINDS: readonly ["env", "rowCount"];
|
|
1033
|
+
export type WorkflowPredicateKind = (typeof WORKFLOW_PREDICATE_KINDS)[number];
|
|
1034
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
1035
|
+
export declare function isWorkflowPredicateKind(value: unknown): value is WorkflowPredicateKind;
|
|
1036
|
+
/**
|
|
1037
|
+
* {@link unreachableNodeKind}, one level down, and for the identical reason.
|
|
1038
|
+
*
|
|
1039
|
+
* Every branch over {@link WorkflowIfPredicate} ends here, so a predicate kind
|
|
1040
|
+
* added to the list without a rule for hashing it, validating it or evaluating
|
|
1041
|
+
* it is a type error naming the file — not a graph that runs and decides
|
|
1042
|
+
* something nobody authored. It throws as well, because predicates arrive as
|
|
1043
|
+
* JSON out of a column and a build older than the data is a thing that happens.
|
|
1044
|
+
*/
|
|
1045
|
+
export declare function unreachablePredicateKind(predicate: never, where: string): never;
|
|
1046
|
+
/**
|
|
1047
|
+
* Tests a variable on the machine that runs the load.
|
|
1048
|
+
*
|
|
1049
|
+
* The original predicate, and the one the node was built for: a deployment that
|
|
1050
|
+
* has a ClickHouse has its URL and one that does not has nothing. It names the
|
|
1051
|
+
* variable and never its value — see the note on {@link WorkflowIfNode} for why
|
|
1052
|
+
* that is a safety property rather than a convenience.
|
|
1053
|
+
*/
|
|
1054
|
+
export interface WorkflowEnvPredicate {
|
|
1055
|
+
kind: 'env';
|
|
1056
|
+
/**
|
|
1057
|
+
* The name of the environment variable to read on the machine that runs the
|
|
1058
|
+
* node. **The name, never the value** — nothing about a credential is stored
|
|
1059
|
+
* in a catalog.
|
|
1060
|
+
*/
|
|
1061
|
+
envVar: string;
|
|
1062
|
+
/**
|
|
1063
|
+
* What it has to equal for the `then` branch to be taken.
|
|
1064
|
+
*
|
|
1065
|
+
* Absent means "is it set to anything non-empty", which is the ClickHouse
|
|
1066
|
+
* case: a deployment that has one has the URL, a deployment that does not has
|
|
1067
|
+
* nothing. Present means an exact string comparison, which is the
|
|
1068
|
+
* `DEPLOY_ENV = local` case.
|
|
1069
|
+
*/
|
|
1070
|
+
equals?: string;
|
|
1071
|
+
}
|
|
1072
|
+
/**
|
|
1073
|
+
* Tests how many rows reached the gate.
|
|
1074
|
+
*
|
|
1075
|
+
* ## The case
|
|
1076
|
+
*
|
|
1077
|
+
* "Only run the sink if the source returned anything." A nightly export that
|
|
1078
|
+
* comes back empty because the upstream system is mid-maintenance is not a
|
|
1079
|
+
* failure — nothing is broken, there is simply nothing to load — but committing
|
|
1080
|
+
* it repoints the live view of a type at an empty snapshot, and the run reports
|
|
1081
|
+
* success while doing it. A gate in front of the sink turns that into a skip,
|
|
1082
|
+
* and a skipped node is never executed, so nothing commits. That is the same
|
|
1083
|
+
* guarantee the `else` branch already gives, pointed at the case that actually
|
|
1084
|
+
* happens.
|
|
1085
|
+
*
|
|
1086
|
+
* ## Which rows
|
|
1087
|
+
*
|
|
1088
|
+
* The ones on the single inbound edge. `validateWorkflow` refuses a gate with
|
|
1089
|
+
* more than one (`if-needs-one-input`) and refuses one with none as unreachable,
|
|
1090
|
+
* so "how many rows" has exactly one answer — and it is the count on the very
|
|
1091
|
+
* {@link WorkflowStageRef} the gate hands on, so the number tested and the rows
|
|
1092
|
+
* carried cannot disagree.
|
|
1093
|
+
*
|
|
1094
|
+
* ## Where the number comes from, which is the replay argument
|
|
1095
|
+
*
|
|
1096
|
+
* `WorkflowStageRef.rowCount`, off {@link WorkflowNodeStepInput.inputs}, which
|
|
1097
|
+
* is part of the step's checkpointed input and was itself produced by an
|
|
1098
|
+
* upstream step's checkpointed output. Nothing counts rows at evaluation time
|
|
1099
|
+
* and nothing reads the stage store: a resumed run on another pod sees the same
|
|
1100
|
+
* number the first attempt saw, and the branch it produced is read back off
|
|
1101
|
+
* {@link WorkflowNodeStepOutput.branch} anyway.
|
|
1102
|
+
*
|
|
1103
|
+
* ## Why a threshold and not "greater than zero"
|
|
1104
|
+
*
|
|
1105
|
+
* `atLeast: 1` *is* "did anything arrive", so the common case costs nothing to
|
|
1106
|
+
* express — and "a full export is never under ten thousand rows, so treat a
|
|
1107
|
+
* hundred as a broken upstream rather than as data" is the next thing anybody
|
|
1108
|
+
* asks for, and it would otherwise need a second predicate kind for one integer.
|
|
1109
|
+
*
|
|
1110
|
+
* One comparison and one direction, deliberately. `atMost`, `equals` and a
|
|
1111
|
+
* chosen operator were all considered and are all the same mistake the node's
|
|
1112
|
+
* own `negate` flag would have been: the inverse test is already expressible by
|
|
1113
|
+
* swapping which successor is on `then` and which is on `else`, and two ways to
|
|
1114
|
+
* say one thing is two places to look when a load takes the branch nobody
|
|
1115
|
+
* expected.
|
|
1116
|
+
*/
|
|
1117
|
+
export interface WorkflowRowCountPredicate {
|
|
1118
|
+
kind: 'rowCount';
|
|
1119
|
+
/**
|
|
1120
|
+
* How many rows have to reach the gate for the `then` branch to be taken.
|
|
1121
|
+
*
|
|
1122
|
+
* A whole number of at least one, and `validateWorkflow` says so. Zero is
|
|
1123
|
+
* refused rather than treated as "always" because it is a gate that can only
|
|
1124
|
+
* ever answer one way — the `else` subtree would never run on any deployment,
|
|
1125
|
+
* which is the silent half-graph this node's whole design is arranged against.
|
|
1126
|
+
*/
|
|
1127
|
+
atLeast: number;
|
|
1128
|
+
}
|
|
1129
|
+
/**
|
|
1130
|
+
* What an `if` node tests. See {@link WORKFLOW_PREDICATE_KINDS}.
|
|
1131
|
+
*/
|
|
1132
|
+
export type WorkflowIfPredicate = WorkflowEnvPredicate | WorkflowRowCountPredicate;
|
|
1133
|
+
/**
|
|
1134
|
+
* Narrow a stored predicate, for the same reason {@link isWorkflowNode} narrows
|
|
1135
|
+
* a stored node: it arrives as JSON out of a column, and a gate read back
|
|
1136
|
+
* without its test is a gate that has to invent one.
|
|
1137
|
+
*
|
|
1138
|
+
* A row count that is not a whole number — `NaN` from a JSON round trip of an
|
|
1139
|
+
* unparsed field, an `Infinity` that serialised as `null` — is refused rather
|
|
1140
|
+
* than kept, because every comparison against it is false and the symptom is a
|
|
1141
|
+
* `then` branch that silently never runs again.
|
|
1142
|
+
*/
|
|
1143
|
+
export declare function isWorkflowIfPredicate(value: unknown): value is WorkflowIfPredicate;
|
|
1144
|
+
/**
|
|
1145
|
+
* Sends the rows down one of its two outbound branches, and skips the other.
|
|
1146
|
+
*
|
|
1147
|
+
* The node the maintainer asked for, and the reason it is a node rather than a
|
|
1148
|
+
* flag on an edge: a deployment that has a ClickHouse and a deployment that does
|
|
1149
|
+
* not are the *same graph*, and the difference between them is one decision made
|
|
1150
|
+
* at run time. Modelling it as two workflows means two things to keep in step;
|
|
1151
|
+
* modelling it as an edge flag means the decision has nowhere to be recorded and
|
|
1152
|
+
* nothing on the canvas to open.
|
|
1153
|
+
*
|
|
1154
|
+
* ## What it does to the rows: nothing
|
|
1155
|
+
*
|
|
1156
|
+
* An `if` is a gate, not a stage. It passes its input through untouched — its
|
|
1157
|
+
* output ref *is* its input's ref — so a branch costs no copy of the dataset and
|
|
1158
|
+
* no second write into the stage store. That is also why it takes **exactly one
|
|
1159
|
+
* inbound edge**: a node hands its successors one {@link WorkflowStageRef}, so a
|
|
1160
|
+
* gate fed by two inputs could only either copy the rows to merge them (paying
|
|
1161
|
+
* for the whole dataset to make a decision that reads none of it) or silently
|
|
1162
|
+
* drop one. Merging is what a transform is for; put one in front.
|
|
1163
|
+
*
|
|
1164
|
+
* That holds for {@link WorkflowRowCountPredicate} too, which is the one
|
|
1165
|
+
* predicate that sounds like it reads the data and does not: the count it tests
|
|
1166
|
+
* is the number already written on the {@link WorkflowStageRef} it was handed,
|
|
1167
|
+
* so a gate still touches no rows — and with one inbound edge there is exactly
|
|
1168
|
+
* one count for "how many rows" to mean.
|
|
1169
|
+
*
|
|
1170
|
+
* ## The predicate is declarative, and that is the safety property
|
|
1171
|
+
*
|
|
1172
|
+
* It is one of {@link WORKFLOW_PREDICATE_KINDS} — a variable's name, or a count
|
|
1173
|
+
* off a checkpoint — and never a value and never code. Three reasons, in order
|
|
1174
|
+
* of how much they cost to get wrong:
|
|
1175
|
+
*
|
|
1176
|
+
* 1. **Replay.** The durable engine replays a run, possibly on another pod. A
|
|
1177
|
+
* predicate is by definition the thing whose answer decides which half of the
|
|
1178
|
+
* graph exists, so an answer that can differ between the run and its replay
|
|
1179
|
+
* is a run that loads through a shape nobody chose — and it would show up as
|
|
1180
|
+
* a non-determinism error two nodes later, naming neither the branch nor the
|
|
1181
|
+
* variable. The outcome is therefore recorded on first evaluation
|
|
1182
|
+
* ({@link WorkflowNodeOutcome.branch}) and read back afterwards, and the
|
|
1183
|
+
* declarative form is what keeps that record small enough to be a checkpoint.
|
|
1184
|
+
* Both predicate kinds are answerable from what a step was handed:
|
|
1185
|
+
* {@link WorkflowRowCountPredicate} reads a number that arrived on the step's
|
|
1186
|
+
* own checkpointed input rather than counting anything.
|
|
1187
|
+
* 2. **A predicate is not a place for a secret.** A name is stored; a value
|
|
1188
|
+
* never is. This is the same rule {@link WorkflowSourceNode.secretEnvVar}
|
|
1189
|
+
* follows and for the same reason.
|
|
1190
|
+
* 3. **Code would need a context to read, and this node does not own it.** What
|
|
1191
|
+
* code-bearing nodes may see — allow-listed variables, a `catalog` object —
|
|
1192
|
+
* is a question being answered elsewhere. A `code` predicate is additive the
|
|
1193
|
+
* day it lands: it becomes another member of {@link WorkflowIfPredicate}, and
|
|
1194
|
+
* everything that reads a branch reads it off the recorded outcome exactly as
|
|
1195
|
+
* it does now.
|
|
1196
|
+
*
|
|
1197
|
+
* To invert the test, swap which successor is on `then` and which is on `else`.
|
|
1198
|
+
* There is deliberately no `negate` flag: two ways to say one thing is two
|
|
1199
|
+
* places to look when a load takes the branch you did not expect.
|
|
1200
|
+
*/
|
|
1201
|
+
export interface WorkflowIfNode extends WorkflowNodeBase {
|
|
1202
|
+
kind: 'if';
|
|
1203
|
+
/**
|
|
1204
|
+
* What it tests. A union rather than a field per test — see
|
|
1205
|
+
* {@link WORKFLOW_PREDICATE_KINDS} for why that is the whole point.
|
|
1206
|
+
*/
|
|
1207
|
+
predicate: WorkflowIfPredicate;
|
|
1208
|
+
}
|
|
1209
|
+
/**
|
|
1210
|
+
* The shapes a {@link WorkflowFilterPredicate} can take.
|
|
1211
|
+
*
|
|
1212
|
+
* Two leaf kinds plus a presence test plus two ways to combine them, and the
|
|
1213
|
+
* list is closed for the same reason {@link WORKFLOW_PREDICATE_KINDS} is: every
|
|
1214
|
+
* decision made per kind ends in {@link unreachableFilterPredicateKind}, so a
|
|
1215
|
+
* sixth shape is a build failure naming the files that owe it an answer rather
|
|
1216
|
+
* than a filter that saves, draws, and quietly keeps everything.
|
|
1217
|
+
*
|
|
1218
|
+
* **There is deliberately no `not`.** Every leaf carries its own inverse
|
|
1219
|
+
* ({@link WORKFLOW_FILTER_OPERATORS} pairs each operator with one, `oneOf` has
|
|
1220
|
+
* `notIn`, `present` has `isNotNull`) and `all`/`any` are duals, so De Morgan
|
|
1221
|
+
* already writes any negation with the kinds here. A `not` node would be a
|
|
1222
|
+
* second spelling of every predicate — two shapes to read when a load comes out
|
|
1223
|
+
* short, and a UI with a checkbox nobody agrees on the placement of. The one
|
|
1224
|
+
* case it does not cover is stated on {@link workflowFilterMatches}: a value the
|
|
1225
|
+
* test cannot compare fails *both* a form and its inverse, on purpose.
|
|
1226
|
+
*
|
|
1227
|
+
* **There is deliberately no free-form expression and no code.** That is the
|
|
1228
|
+
* whole argument for the node existing — see {@link WorkflowFilterNode}.
|
|
1229
|
+
*/
|
|
1230
|
+
export declare const WORKFLOW_FILTER_PREDICATE_KINDS: readonly ["compare", "oneOf", "present", "all", "any"];
|
|
1231
|
+
export type WorkflowFilterPredicateKind = (typeof WORKFLOW_FILTER_PREDICATE_KINDS)[number];
|
|
1232
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
1233
|
+
export declare function isWorkflowFilterPredicateKind(value: unknown): value is WorkflowFilterPredicateKind;
|
|
1234
|
+
/**
|
|
1235
|
+
* {@link unreachablePredicateKind}, for the filter language, and for the
|
|
1236
|
+
* identical reason.
|
|
1237
|
+
*
|
|
1238
|
+
* It throws as well as failing to compile: a predicate arrives as JSON out of a
|
|
1239
|
+
* column, so a graph saved by a newer build and read by an older one is a thing
|
|
1240
|
+
* that happens, and returning a default for a shape this build has no rule for
|
|
1241
|
+
* would mean silently keeping — or silently dropping — every row.
|
|
1242
|
+
*/
|
|
1243
|
+
export declare function unreachableFilterPredicateKind(predicate: never, where: string): never;
|
|
1244
|
+
/**
|
|
1245
|
+
* What a {@link WorkflowFilterComparison} does with its column and its value.
|
|
1246
|
+
*
|
|
1247
|
+
* Ten, in five inverse pairs, and the pairing is the reason there is no `not`
|
|
1248
|
+
* kind. Each one has an obvious single-expression form in both dialects this
|
|
1249
|
+
* repository speaks, which is not a coincidence — it is the constraint the list
|
|
1250
|
+
* was chosen under, so that the day a predicate is pushed into a source query
|
|
1251
|
+
* the translation is a `switch` and not a design.
|
|
1252
|
+
*
|
|
1253
|
+
* The omissions, since a closed list is only defensible if they are:
|
|
1254
|
+
*
|
|
1255
|
+
* - **`between`** — `all` of a `greaterThanOrEqual` and a `lessThanOrEqual`, in
|
|
1256
|
+
* one more click and with no second shape to validate, hash and translate.
|
|
1257
|
+
* - **`endsWith`** — asked for far less than the other two, and unlike them it
|
|
1258
|
+
* has no index that could ever serve it in either dialect, so offering it in a
|
|
1259
|
+
* palette would advertise something that is a full scan by construction.
|
|
1260
|
+
* `contains` covers it at the same cost.
|
|
1261
|
+
* - **regular expressions** — the dialects disagree about the syntax, the
|
|
1262
|
+
* engines disagree about the semantics, and a predicate whose meaning depends
|
|
1263
|
+
* on which database answered it is a predicate that cannot be pushed down
|
|
1264
|
+
* without changing what the load returns. That is the one property this list
|
|
1265
|
+
* exists to hold.
|
|
1266
|
+
* - **case-insensitive variants** — collation is a property of the column in
|
|
1267
|
+
* both dialects, so a `equalsIgnoreCase` evaluated in memory and the same
|
|
1268
|
+
* predicate evaluated in a `WHERE` would legitimately disagree. Normalise in a
|
|
1269
|
+
* transform, where the disagreement is visible.
|
|
1270
|
+
*/
|
|
1271
|
+
export declare const WORKFLOW_FILTER_OPERATORS: readonly ["equals", "notEquals", "greaterThan", "lessThanOrEqual", "greaterThanOrEqual", "lessThan", "contains", "notContains", "startsWith", "notStartsWith"];
|
|
1272
|
+
export type WorkflowFilterOperator = (typeof WORKFLOW_FILTER_OPERATORS)[number];
|
|
1273
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
1274
|
+
export declare function isWorkflowFilterOperator(value: unknown): value is WorkflowFilterOperator;
|
|
1275
|
+
/**
|
|
1276
|
+
* {@link unreachableFilterPredicateKind}, one level further down.
|
|
1277
|
+
*
|
|
1278
|
+
* An operator added to the list without a rule for evaluating it, describing it
|
|
1279
|
+
* or hashing it is a type error naming the file — not a comparison that silently
|
|
1280
|
+
* answers false for every row, which is a filter that drops the whole dataset
|
|
1281
|
+
* and reports success.
|
|
1282
|
+
*/
|
|
1283
|
+
export declare function unreachableFilterOperator(operator: never, where: string): never;
|
|
1284
|
+
/**
|
|
1285
|
+
* What a predicate may be compared against.
|
|
1286
|
+
*
|
|
1287
|
+
* Three scalars and nothing else. No `null` — {@link WorkflowFilterPresence} is
|
|
1288
|
+
* how absence is tested, and folding it in here would make `equals: null` and
|
|
1289
|
+
* `isNull` two spellings of one question with different answers under the
|
|
1290
|
+
* three-valued logic on {@link workflowFilterMatches}. No object and no array —
|
|
1291
|
+
* an array is {@link WorkflowFilterOneOf.values}, and an object compared with
|
|
1292
|
+
* `===` never matches anything a source produced.
|
|
1293
|
+
*
|
|
1294
|
+
* **Dates are strings.** A source hands back whatever its driver decoded, and a
|
|
1295
|
+
* catalog that tried to parse dates here would have to pick a format, get it
|
|
1296
|
+
* wrong for one dialect, and produce a comparison that silently ordered rows
|
|
1297
|
+
* differently from the `WHERE` this predicate is meant to become. ISO-8601
|
|
1298
|
+
* strings sort correctly under `<` and under every SQL collation, which is why
|
|
1299
|
+
* `boundStatement` already compares watermarks as text.
|
|
1300
|
+
*/
|
|
1301
|
+
export type WorkflowFilterValue = string | number | boolean;
|
|
1302
|
+
/** Whether a stored value is one this language can compare against. */
|
|
1303
|
+
export declare function isWorkflowFilterValue(value: unknown): value is WorkflowFilterValue;
|
|
1304
|
+
/**
|
|
1305
|
+
* The column names a predicate may name.
|
|
1306
|
+
*
|
|
1307
|
+
* **The same pattern `boundStatement` requires of a watermark column**, and that
|
|
1308
|
+
* is the point rather than a coincidence: an identifier cannot be bound by any
|
|
1309
|
+
* driver, so pushing a predicate into a query means quoting the column into the
|
|
1310
|
+
* SQL, and a name carrying a quote, a dot or a space is refused rather than
|
|
1311
|
+
* escaped. Requiring it *now*, while the predicate is only ever evaluated in
|
|
1312
|
+
* memory, is what stops a graph being authored today that could never be pushed
|
|
1313
|
+
* down tomorrow.
|
|
1314
|
+
*
|
|
1315
|
+
* The cost, and it is real: a source whose column is called `Part Number` cannot
|
|
1316
|
+
* be filtered directly. Rename it in a transform first — which is a node that
|
|
1317
|
+
* already exists and whose output is a column this can name.
|
|
1318
|
+
*/
|
|
1319
|
+
export declare const WORKFLOW_FILTER_COLUMN_PATTERN: RegExp;
|
|
1320
|
+
/**
|
|
1321
|
+
* How deep `all`/`any` may nest.
|
|
1322
|
+
*
|
|
1323
|
+
* A bound rather than a trust, because a predicate arrives as JSON out of a
|
|
1324
|
+
* column and every function that walks one is recursive: a graph carrying a
|
|
1325
|
+
* thousand-deep tree would be a stack overflow inside a durable step rather than
|
|
1326
|
+
* a refusal naming the node. Six is past anything a person builds in a form and
|
|
1327
|
+
* far short of anything that costs a frame to walk.
|
|
1328
|
+
*/
|
|
1329
|
+
export declare const WORKFLOW_FILTER_MAX_DEPTH = 6;
|
|
1330
|
+
/**
|
|
1331
|
+
* How many values one {@link WorkflowFilterOneOf} may list.
|
|
1332
|
+
*
|
|
1333
|
+
* Bounded because the list travels in the graph, into the graph fingerprint, and
|
|
1334
|
+
* eventually into an `IN (...)` — and past a few hundred entries all three of
|
|
1335
|
+
* those stop being reasonable and the thing being expressed is a join against
|
|
1336
|
+
* another dataset rather than a filter. Wire that dataset in as a second source
|
|
1337
|
+
* and join it in a transform, which is the node that can.
|
|
1338
|
+
*/
|
|
1339
|
+
export declare const WORKFLOW_FILTER_MAX_VALUES = 500;
|
|
1340
|
+
/** One column against one value. The leaf almost every filter is made of. */
|
|
1341
|
+
export interface WorkflowFilterComparison {
|
|
1342
|
+
kind: 'compare';
|
|
1343
|
+
/** A bare column name. See {@link WORKFLOW_FILTER_COLUMN_PATTERN}. */
|
|
1344
|
+
column: string;
|
|
1345
|
+
operator: WorkflowFilterOperator;
|
|
1346
|
+
value: WorkflowFilterValue;
|
|
1347
|
+
}
|
|
1348
|
+
/**
|
|
1349
|
+
* One column against a list of values.
|
|
1350
|
+
*
|
|
1351
|
+
* Its own kind rather than an `equals` whose value is allowed to be an array,
|
|
1352
|
+
* because that shape makes `value` mean two things and every reader has to test
|
|
1353
|
+
* which — the flat-optional-fields mistake {@link WORKFLOW_PREDICATE_KINDS}
|
|
1354
|
+
* argues against, one level down again. It also has no honest single spelling in
|
|
1355
|
+
* SQL as a comparison: `IN (…)` takes a parenthesised list and one bound
|
|
1356
|
+
* parameter per entry.
|
|
1357
|
+
*/
|
|
1358
|
+
export interface WorkflowFilterOneOf {
|
|
1359
|
+
kind: 'oneOf';
|
|
1360
|
+
column: string;
|
|
1361
|
+
/** `notIn` rather than a `negated` flag, so the two read alike in a palette. */
|
|
1362
|
+
operator: 'in' | 'notIn';
|
|
1363
|
+
/** At most {@link WORKFLOW_FILTER_MAX_VALUES}, and never empty. */
|
|
1364
|
+
values: WorkflowFilterValue[];
|
|
1365
|
+
}
|
|
1366
|
+
/**
|
|
1367
|
+
* Whether a column has a value at all.
|
|
1368
|
+
*
|
|
1369
|
+
* The one test the other two cannot express, and the reason they cannot is the
|
|
1370
|
+
* three-valued logic on {@link workflowFilterMatches}: every `compare` and every
|
|
1371
|
+
* `oneOf` is false when the column is null, *including the inverses*, exactly as
|
|
1372
|
+
* a `WHERE` would answer. So "the ones with no delivery date" has to be its own
|
|
1373
|
+
* shape or it would be unaskable.
|
|
1374
|
+
*/
|
|
1375
|
+
export interface WorkflowFilterPresence {
|
|
1376
|
+
kind: 'present';
|
|
1377
|
+
column: string;
|
|
1378
|
+
operator: 'isNull' | 'isNotNull';
|
|
1379
|
+
}
|
|
1380
|
+
/**
|
|
1381
|
+
* What `all` and `any` share, which is everything except the word.
|
|
1382
|
+
*
|
|
1383
|
+
* **An empty group is refused**, by `validateWorkflow`, by
|
|
1384
|
+
* {@link isWorkflowFilterPredicate} and at the HTTP boundary. It is not a
|
|
1385
|
+
* pedantic refusal: `all` of nothing is vacuously true, so it keeps every row
|
|
1386
|
+
* and the filter does nothing; `any` of nothing is vacuously false, so it drops
|
|
1387
|
+
* every row and the load comes out empty. Both are silent, both are reached by
|
|
1388
|
+
* deleting the last condition in a form, and they are opposite catastrophes.
|
|
1389
|
+
*/
|
|
1390
|
+
interface WorkflowFilterGroupBase {
|
|
1391
|
+
/** Never empty. Nested no deeper than {@link WORKFLOW_FILTER_MAX_DEPTH}. */
|
|
1392
|
+
children: WorkflowFilterPredicate[];
|
|
1393
|
+
}
|
|
1394
|
+
/**
|
|
1395
|
+
* Every child holds.
|
|
1396
|
+
*
|
|
1397
|
+
* Two interfaces rather than one carrying `kind: 'all' | 'any'`, which was
|
|
1398
|
+
* written first and does not work: TypeScript will not remove a union member
|
|
1399
|
+
* from the union when its discriminant is itself a union of literals and the two
|
|
1400
|
+
* are tested separately, so `unreachableFilterPredicateKind` at the end of every
|
|
1401
|
+
* `switch`-by-`if` chain stopped compiling — which is to say the exhaustiveness
|
|
1402
|
+
* this whole file is arranged around was silently unavailable for these two
|
|
1403
|
+
* kinds. Two declarations over a shared base costs one line and buys the
|
|
1404
|
+
* property back.
|
|
1405
|
+
*/
|
|
1406
|
+
export interface WorkflowFilterAll extends WorkflowFilterGroupBase {
|
|
1407
|
+
kind: 'all';
|
|
1408
|
+
}
|
|
1409
|
+
/** At least one child holds. Two declarations, for the reason on {@link WorkflowFilterAll}. */
|
|
1410
|
+
export interface WorkflowFilterAny extends WorkflowFilterGroupBase {
|
|
1411
|
+
kind: 'any';
|
|
1412
|
+
}
|
|
1413
|
+
export type WorkflowFilterGroup = WorkflowFilterAll | WorkflowFilterAny;
|
|
1414
|
+
/** What a filter node tests. See {@link WORKFLOW_FILTER_PREDICATE_KINDS}. */
|
|
1415
|
+
export type WorkflowFilterPredicate = WorkflowFilterComparison | WorkflowFilterOneOf | WorkflowFilterPresence | WorkflowFilterAll | WorkflowFilterAny;
|
|
1416
|
+
/**
|
|
1417
|
+
* Narrow a stored filter predicate, refusing rather than repairing.
|
|
1418
|
+
*
|
|
1419
|
+
* The same contract {@link isWorkflowIfPredicate} has and for a sharper version
|
|
1420
|
+
* of the same reason: a gate read back without its test picks a branch nobody
|
|
1421
|
+
* authored, and a *filter* read back without its test decides which rows exist.
|
|
1422
|
+
* Repairing a broken predicate — dropping an unreadable child out of an `all`,
|
|
1423
|
+
* say — would silently widen or narrow what a load publishes, which is precisely
|
|
1424
|
+
* the failure the node is built to make visible.
|
|
1425
|
+
*
|
|
1426
|
+
* Depth is carried rather than tracked globally so that the bound is on the
|
|
1427
|
+
* *tree*, not on how many predicates have been checked: two sibling branches
|
|
1428
|
+
* five deep are fine, and one branch seven deep is not.
|
|
1429
|
+
*/
|
|
1430
|
+
export declare function isWorkflowFilterPredicate(value: unknown, depth?: number): value is WorkflowFilterPredicate;
|
|
1431
|
+
/**
|
|
1432
|
+
* Whether one row passes a predicate.
|
|
1433
|
+
*
|
|
1434
|
+
* Pure, allocation-free on the common path, and in core rather than in the
|
|
1435
|
+
* runner so that the console can describe — and one day preview — exactly what
|
|
1436
|
+
* the load will do, from the same function that does it. It is called once per
|
|
1437
|
+
* row, so everything about it is arranged to be cheap: no closures built per
|
|
1438
|
+
* row, no array built per row, and `note` is optional so a caller that does not
|
|
1439
|
+
* want the diagnostics pays nothing for them.
|
|
1440
|
+
*
|
|
1441
|
+
* ## Null is unknown, and unknown does not pass
|
|
1442
|
+
*
|
|
1443
|
+
* A column that is `null`, `undefined`, or simply absent from the row fails
|
|
1444
|
+
* **every** `compare` and **every** `oneOf`, *including the negative forms*.
|
|
1445
|
+
* `status notEquals "CLOSED"` does not pass a row with no status.
|
|
1446
|
+
*
|
|
1447
|
+
* That is SQL's three-valued logic rather than JavaScript's, and it is chosen
|
|
1448
|
+
* deliberately over the more intuitive JS answer for one reason: this predicate
|
|
1449
|
+
* is meant to become a `WHERE`, and the day it does, the database will answer
|
|
1450
|
+
* this way. A filter that kept those rows in memory and dropped them once pushed
|
|
1451
|
+
* down would be a performance change that quietly altered what a type contains —
|
|
1452
|
+
* which is the one thing a pushdown must never be able to do. {@link
|
|
1453
|
+
* WorkflowFilterPresence} is how absence is tested on purpose.
|
|
1454
|
+
*
|
|
1455
|
+
* ## A value it cannot compare fails both a test and its inverse
|
|
1456
|
+
*
|
|
1457
|
+
* `qty greaterThan 10` against a `qty` holding `"n/a"` is not false because the
|
|
1458
|
+
* string is small; there is no ordering between a string and a number that any
|
|
1459
|
+
* two systems would agree on. Rather than invent one, the leaf answers false and
|
|
1460
|
+
* calls `note` with the column, and the runner turns those counts into a line on
|
|
1461
|
+
* the run: *"12,431 rows held a value in `qty` the test could not compare."*
|
|
1462
|
+
*
|
|
1463
|
+
* A row nothing can judge is therefore dropped rather than kept. That is the
|
|
1464
|
+
* direction with a backstop — the sink's row-count bound sees the shrink, and a
|
|
1465
|
+
* full sink refuses to commit nothing at all — whereas keeping unjudged rows
|
|
1466
|
+
* would publish them into the type with nothing anywhere to notice.
|
|
1467
|
+
*/
|
|
1468
|
+
export declare function workflowFilterMatches(predicate: WorkflowFilterPredicate, row: Record<string, unknown>, note?: (column: string) => void): boolean;
|
|
1469
|
+
/**
|
|
1470
|
+
* Drops the rows that fail a declarative test, and reports how many.
|
|
1471
|
+
*
|
|
1472
|
+
* ## Why this is a node and not a transform that returns a subset
|
|
1473
|
+
*
|
|
1474
|
+
* A transform can already filter, so the kind has to earn itself. Three reasons,
|
|
1475
|
+
* in increasing order of how much they decide the shape:
|
|
1476
|
+
*
|
|
1477
|
+
* 1. **It is legible on the canvas.** "Keep the open orders" is readable from
|
|
1478
|
+
* the box; the same rule inside a transform is readable only by opening the
|
|
1479
|
+
* code, and only by somebody who can read the language it is written in.
|
|
1480
|
+
* 2. **Its effect is reportable.** A filter records rows in *and* rows out
|
|
1481
|
+
* ({@link WorkflowNodeOutcome.rowsIn}), so a run panel can say what was
|
|
1482
|
+
* dropped. A transform records one number, and a transform that quietly
|
|
1483
|
+
* started dropping 90% of its input looks exactly like a source that got
|
|
1484
|
+
* smaller. A filter whose effect is invisible is how data goes missing.
|
|
1485
|
+
* 3. **Only a declarative predicate can be pushed into the source.** This is the
|
|
1486
|
+
* one that fixes the design. Arbitrary code cannot be translated to SQL, so a
|
|
1487
|
+
* code filter is *always* the expensive path: every row is read off disk, sent
|
|
1488
|
+
* over the network and turned into a JS object before anything decides it was
|
|
1489
|
+
* unwanted. A closed structure of column, operator and value can become a
|
|
1490
|
+
* `WHERE`, and then the rows are never read at all.
|
|
1491
|
+
*
|
|
1492
|
+
* ## Where it runs today, and where it does not yet
|
|
1493
|
+
*
|
|
1494
|
+
* **Today: in memory, one staged batch at a time, always.** The predicate is
|
|
1495
|
+
* shaped so it *could* be pushed into a SQL source's query, and it is not, and
|
|
1496
|
+
* saying so plainly is better than implying a win that has not been measured.
|
|
1497
|
+
* What stands in the way is not the predicate — `boundStatement` in
|
|
1498
|
+
* `sources.ts` already wraps an author's SQL in `SELECT * FROM (…) WHERE …`
|
|
1499
|
+
* with a quoted identifier and a bound parameter, which is exactly the mechanism
|
|
1500
|
+
* a pushdown would reuse — but the fetcher contract: `SourceFetcher` receives a
|
|
1501
|
+
* connector, a secret, a watermark and a mode, and knows nothing about the
|
|
1502
|
+
* graph, while the runner that *does* know the graph dispatches by connector
|
|
1503
|
+
* kind alone. Threading a graph-derived predicate through that also drags in the
|
|
1504
|
+
* schema-discovery path, which shares `sqlTarget`.
|
|
1505
|
+
*
|
|
1506
|
+
* There is a second reason, and it is the more interesting one: a pushed-down
|
|
1507
|
+
* filter **cannot honestly report rows in**. The rows it dropped were never
|
|
1508
|
+
* fetched, so "7,637,391 in, 96,204 out" would become "96,204 in, 96,204 out,
|
|
1509
|
+
* nothing dropped" — reason 2 above, deleted by reason 3. Recovering the number
|
|
1510
|
+
* means a second `COUNT(*)` over the unfiltered query, which is the scan the
|
|
1511
|
+
* pushdown existed to avoid. Whichever way that is resolved, it is a decision
|
|
1512
|
+
* about what a run reports and not a refactor, so it belongs in the change that
|
|
1513
|
+
* makes the move rather than in a field added speculatively here.
|
|
1514
|
+
*
|
|
1515
|
+
* The seam is therefore one marked place in `WorkflowRunnerService.runFilter`
|
|
1516
|
+
* rather than an unused translator sitting in this file waiting to rot.
|
|
1517
|
+
*
|
|
1518
|
+
* ## The trap this node had to be designed against: {@link narrows}
|
|
1519
|
+
*
|
|
1520
|
+
* Dropping a filter onto an existing `source → sink` wire replaces the published
|
|
1521
|
+
* snapshot of that type with a subset — and every part of the run reports
|
|
1522
|
+
* success, because from the run's point of view everything did succeed. See
|
|
1523
|
+
* {@link narrows} for how the graph is made to tell that apart from filtering to
|
|
1524
|
+
* derive something new, and why it cannot be told apart structurally.
|
|
1525
|
+
*/
|
|
1526
|
+
export interface WorkflowFilterNode extends WorkflowNodeBase {
|
|
1527
|
+
kind: 'filter';
|
|
1528
|
+
/**
|
|
1529
|
+
* What a row has to satisfy to pass. See
|
|
1530
|
+
* {@link WORKFLOW_FILTER_PREDICATE_KINDS}.
|
|
1531
|
+
*/
|
|
1532
|
+
predicate: WorkflowFilterPredicate;
|
|
1533
|
+
/**
|
|
1534
|
+
* The object types whose published snapshot this filter is acknowledged to
|
|
1535
|
+
* narrow.
|
|
1536
|
+
*
|
|
1537
|
+
* ## The two intentions, and why the graph cannot tell them apart
|
|
1538
|
+
*
|
|
1539
|
+
* Filtering to **derive a new type** — `source → filter → sink(OpenOrders)` —
|
|
1540
|
+
* and filtering before **recommitting the same type** —
|
|
1541
|
+
* `source → filter → sink(PriBuy)`, where `PriBuy` was until this morning the
|
|
1542
|
+
* whole table — are *structurally identical graphs*. The only thing that
|
|
1543
|
+
* differs is what the type on the sink already means to everybody reading it,
|
|
1544
|
+
* and there is nothing in the nodes or the edges that knows that. Any rule
|
|
1545
|
+
* claiming to distinguish them from the shape alone would be inventing a
|
|
1546
|
+
* signal, and would then either refuse the safe case or wave the dangerous one
|
|
1547
|
+
* through.
|
|
1548
|
+
*
|
|
1549
|
+
* So the graph makes the author **name the types**, and that naming is the
|
|
1550
|
+
* whole mechanism: typing `OpenOrders` is a different act from typing
|
|
1551
|
+
* `PriBuy`, and nobody types the second one by accident. What makes it a
|
|
1552
|
+
* safeguard rather than a checkbox is that it is *required exactly where it
|
|
1553
|
+
* matters and refused everywhere else*, both checked by `validateWorkflow`:
|
|
1554
|
+
*
|
|
1555
|
+
* - It must list **every** full-mode sink this filter stands in front of *on
|
|
1556
|
+
* every path* — that is, every sink whose entire snapshot would be a subset
|
|
1557
|
+
* because of this node. Removing the node would make that sink unreachable;
|
|
1558
|
+
* see `workflowNarrowedTypes`, which is the one implementation both the
|
|
1559
|
+
* validator and the console call.
|
|
1560
|
+
* - It must list **nothing else**. A type named here that this filter does not
|
|
1561
|
+
* in fact narrow is refused, for the reason a branch label on a plain wire
|
|
1562
|
+
* is: an acknowledgement nothing reads is worse than none, because it is
|
|
1563
|
+
* drawn.
|
|
1564
|
+
*
|
|
1565
|
+
* A filter on one of several paths into a sink narrows nothing — other rows
|
|
1566
|
+
* still reach it — and a filter in front of an incremental sink narrows
|
|
1567
|
+
* nothing either, because an incremental commit merges into what is already
|
|
1568
|
+
* there rather than replacing it. Neither has anything to declare, and neither
|
|
1569
|
+
* may declare it.
|
|
1570
|
+
*
|
|
1571
|
+
* The consequence is the intended one: dragging a filter onto a working
|
|
1572
|
+
* `source → sink` wire produces a graph that **will not save** until somebody
|
|
1573
|
+
* writes down the name of the type they are about to shrink. It is in the
|
|
1574
|
+
* graph fingerprint, so acknowledging it is a new version of the graph.
|
|
1575
|
+
*
|
|
1576
|
+
* The run-time backstop is unchanged and still the last word: the sink's
|
|
1577
|
+
* row-count bound (`maxShrink`) refuses a commit that loses more of the served
|
|
1578
|
+
* snapshot than the type allows, whatever this field says.
|
|
1579
|
+
*/
|
|
1580
|
+
narrows?: string[];
|
|
1581
|
+
}
|
|
746
1582
|
/**
|
|
747
1583
|
* A discriminated union, so narrowing a node is `node.kind === "sink"` and
|
|
748
1584
|
* never a type assertion. This is why the kind list is not simply a string on
|
|
749
1585
|
* one node shape with every field optional: that shape lets a source node carry
|
|
750
1586
|
* a `transformId` and nothing catches it.
|
|
751
1587
|
*/
|
|
752
|
-
export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode;
|
|
1588
|
+
export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | WorkflowSinkNode | WorkflowCallNode | WorkflowIfNode | WorkflowFilterNode;
|
|
1589
|
+
/**
|
|
1590
|
+
* Which side of an {@link WorkflowIfNode} a wire leaves by.
|
|
1591
|
+
*
|
|
1592
|
+
* **Two closed values rather than free-form labels**, which was the other design
|
|
1593
|
+
* and is worse in the one way that matters here: an unlabelled branch is a
|
|
1594
|
+
* branch that never runs, and a *misspelled* branch is one too. With a free
|
|
1595
|
+
* string, `thn` is a subtree that silently never executes and a graph that
|
|
1596
|
+
* validates perfectly; with these, it is refused at the boundary and is a type
|
|
1597
|
+
* error in the console. The failure a branch introduces is "nothing loaded and
|
|
1598
|
+
* nothing complained", so the vocabulary is the place to make it impossible.
|
|
1599
|
+
*
|
|
1600
|
+
* An N-way `switch` node was considered and is deliberately not this: it would
|
|
1601
|
+
* have to carry its own case list plus a default, the cases would have to be
|
|
1602
|
+
* validated against the labels, and an unmatched value would need a rule. That
|
|
1603
|
+
* is a different node, and it can be added later without changing this one —
|
|
1604
|
+
* because a boolean question is what a predicate answers, and an `if` is exactly
|
|
1605
|
+
* the shape of a boolean question.
|
|
1606
|
+
*/
|
|
1607
|
+
export declare const WORKFLOW_BRANCH_LABELS: readonly ["then", "else"];
|
|
1608
|
+
export type WorkflowBranchLabel = (typeof WORKFLOW_BRANCH_LABELS)[number];
|
|
1609
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
1610
|
+
export declare function isWorkflowBranchLabel(value: unknown): value is WorkflowBranchLabel;
|
|
753
1611
|
/**
|
|
754
1612
|
* One wire.
|
|
755
1613
|
*
|
|
@@ -766,6 +1624,22 @@ export type WorkflowNode = WorkflowSourceNode | WorkflowTransformNode | Workflow
|
|
|
766
1624
|
export interface WorkflowEdge {
|
|
767
1625
|
from: string;
|
|
768
1626
|
to: string;
|
|
1627
|
+
/**
|
|
1628
|
+
* Which branch of an {@link WorkflowIfNode} this wire leaves by.
|
|
1629
|
+
*
|
|
1630
|
+
* **Optional, and absent on every edge that existed before branches did.** An
|
|
1631
|
+
* edge with no label is a plain wire that always carries rows when its source
|
|
1632
|
+
* ran — which is what every stored edge is, so nothing about an existing graph
|
|
1633
|
+
* changes, including its fingerprint. The label is *required* on an edge
|
|
1634
|
+
* leaving an `if` and *refused* on any other, both by `validateWorkflow`: a
|
|
1635
|
+
* label on a wire nothing branches at would look like a decision and be
|
|
1636
|
+
* ignored, and an unlabelled wire out of an `if` has no branch to belong to.
|
|
1637
|
+
*
|
|
1638
|
+
* Several wires may share a label. An `if` whose `then` side fans out to two
|
|
1639
|
+
* nodes is ordinary fan-out that happens to be conditional; what is refused is
|
|
1640
|
+
* two wires between the *same pair*, which was already refused as a duplicate.
|
|
1641
|
+
*/
|
|
1642
|
+
branch?: WorkflowBranchLabel;
|
|
769
1643
|
}
|
|
770
1644
|
/** Just the executable part of a workflow, for validating a canvas draft. */
|
|
771
1645
|
export interface WorkflowGraph {
|
|
@@ -987,7 +1861,25 @@ export interface WorkflowNodeStepOutput {
|
|
|
987
1861
|
};
|
|
988
1862
|
/** Which transform version ran, for a transform node. */
|
|
989
1863
|
transformVersion?: number;
|
|
1864
|
+
/**
|
|
1865
|
+
* Which branch an `if` node took, for an `if` node.
|
|
1866
|
+
*
|
|
1867
|
+
* **On the step's output, which is the whole mechanism.** A step's output is
|
|
1868
|
+
* what the durable engine checkpoints and hands back on replay without running
|
|
1869
|
+
* the step again, so putting the evaluated branch here means the decision is
|
|
1870
|
+
* made exactly once, in the run's own history, and every later turn reads the
|
|
1871
|
+
* recorded one. A body that asked the predicate itself would be re-evaluating
|
|
1872
|
+
* a pod-local fact on a pod that may not be the same one.
|
|
1873
|
+
*/
|
|
1874
|
+
branch?: WorkflowBranchLabel;
|
|
990
1875
|
rows: number;
|
|
1876
|
+
/**
|
|
1877
|
+
* What a {@link WorkflowFilterNode} was handed, against `rows` above, which is
|
|
1878
|
+
* what it passed on. See {@link WorkflowNodeOutcome.rowsIn}: it travels on the
|
|
1879
|
+
* step's output so that a replayed node reports the same drop the first
|
|
1880
|
+
* attempt did, rather than reporting nothing because it was not re-run.
|
|
1881
|
+
*/
|
|
1882
|
+
rowsIn?: number;
|
|
991
1883
|
elapsedMs: number;
|
|
992
1884
|
/**
|
|
993
1885
|
* The one thing here that is not a counter, and the one exception worth
|
|
@@ -1076,35 +1968,127 @@ export declare function readWorkflowCallOutput(value: unknown): WorkflowCallOutp
|
|
|
1076
1968
|
/**
|
|
1077
1969
|
* One workflow a call node could name.
|
|
1078
1970
|
*
|
|
1079
|
-
*
|
|
1080
|
-
*
|
|
1081
|
-
*
|
|
1082
|
-
*
|
|
1083
|
-
*
|
|
1084
|
-
*
|
|
1085
|
-
*
|
|
1086
|
-
*
|
|
1087
|
-
*
|
|
1088
|
-
*
|
|
1089
|
-
*
|
|
1090
|
-
*
|
|
1091
|
-
*
|
|
1971
|
+
* ## What this used to say, and what changed
|
|
1972
|
+
*
|
|
1973
|
+
* This shape was declared with nothing producing it, because a picker needs a
|
|
1974
|
+
* list and no list existed. The durable engine could answer
|
|
1975
|
+
* `workflowBody(name, version)` for the process asking and nothing more, and a
|
|
1976
|
+
* missing body is ambiguous by construction: it equally means "not registered",
|
|
1977
|
+
* "registered through `registerRemote` against another SDK", or "a group
|
|
1978
|
+
* resolved by convention against a live worker". A list inferred from that
|
|
1979
|
+
* would have omitted precisely the cross-SDK workflows this node exists to
|
|
1980
|
+
* call, and would have omitted them silently.
|
|
1981
|
+
*
|
|
1982
|
+
* `@dudousxd/nestjs-durable-core` **0.65.0** closed that with
|
|
1983
|
+
* `WorkflowEngine.announcedWorkflows()`: live workers publish what they can
|
|
1984
|
+
* execute on the worker-descriptor keyspace, and every pod folds the same
|
|
1985
|
+
* published statements, so the answer no longer depends on which replica
|
|
1986
|
+
* served the request. That is what fills this shape now — see
|
|
1987
|
+
* `WorkflowLauncher.callableWorkflows` in `@dudousxd/nestjs-catalog-pipeline`
|
|
1988
|
+
* for the adapter, and keep three properties of the aggregate in mind, because
|
|
1989
|
+
* every field below exists to carry one of them:
|
|
1990
|
+
*
|
|
1991
|
+
* - **It is what is runnable, not what is known about.** The announcer is
|
|
1992
|
+
* always the process that CONSUMES the queue, so a `registerRemote` entry is
|
|
1993
|
+
* never announced by the engine that declared it. An operator running bodies
|
|
1994
|
+
* inline with no queue announces nothing at all — its workflows are real and
|
|
1995
|
+
* startable, and simply not in this list.
|
|
1996
|
+
* - **It is a snapshot.** An announcement lives on a descriptor key with the
|
|
1997
|
+
* worker-heartbeat TTL, so a worker that dies takes its entries with it
|
|
1998
|
+
* within about one beat. Nothing built on this should be cached past that,
|
|
1999
|
+
* and nothing should present it as authoritative-forever.
|
|
2000
|
+
* - **Disagreements are kept, not merged.** Two live workers may claim the same
|
|
2001
|
+
* `name@version` from different groups. The aggregate refuses to pick a
|
|
2002
|
+
* winner, and so does this shape: see {@link disagreements}.
|
|
2003
|
+
*
|
|
2004
|
+
* One entry per **version**, never per name — a picker that listed names and
|
|
2005
|
+
* resolved the version for you would undo the pin the call node exists for.
|
|
1092
2006
|
*/
|
|
1093
2007
|
export interface CallableWorkflowRef {
|
|
1094
2008
|
name: string;
|
|
1095
|
-
|
|
2009
|
+
/**
|
|
2010
|
+
* The version to pin, and **absent is a real answer**.
|
|
2011
|
+
*
|
|
2012
|
+
* A worker that has not been upgraded announces a bare name with no version
|
|
2013
|
+
* and no group. Silence is not a claim, so no version is invented for it from
|
|
2014
|
+
* another announcer's — and an entry without one cannot satisfy the pin, which
|
|
2015
|
+
* is why {@link callableWorkflowBlock} refuses it rather than letting a picker
|
|
2016
|
+
* offer a name that would run whatever is newest on the day it runs.
|
|
2017
|
+
*/
|
|
2018
|
+
version?: string;
|
|
1096
2019
|
/** What it does, if the deployment publishes one. Shown beside the name. */
|
|
1097
2020
|
description?: string;
|
|
1098
2021
|
/**
|
|
1099
|
-
* The worker group its turns are dispatched to, when
|
|
1100
|
-
* that says "this one's body is not in this
|
|
1101
|
-
* a separate TS worker — which is precisely
|
|
1102
|
-
* tell from a missing body.
|
|
2022
|
+
* The worker group its turns are dispatched to, when the live announcers
|
|
2023
|
+
* agree on exactly one. The signal that says "this one's body is not in this
|
|
2024
|
+
* process" — a Python workflow, or a separate TS worker — which is precisely
|
|
2025
|
+
* what a caller cannot otherwise tell from a missing body.
|
|
2026
|
+
*
|
|
2027
|
+
* Absent means either nobody stated one or the announcers disagree, and those
|
|
2028
|
+
* two are not the same: the second puts a `group` entry in
|
|
2029
|
+
* {@link disagreements} and the first does not.
|
|
1103
2030
|
*/
|
|
1104
2031
|
group?: string;
|
|
2032
|
+
/**
|
|
2033
|
+
* How many live workers announce it. `1` is a single point of failure, and it
|
|
2034
|
+
* is never `0` — an entry exists only because somebody announced it.
|
|
2035
|
+
*/
|
|
2036
|
+
workers?: number;
|
|
2037
|
+
/**
|
|
2038
|
+
* The axes the live announcers do not agree on, empty or absent when they
|
|
2039
|
+
* speak with one voice. Carried rather than resolved: the registry refuses to
|
|
2040
|
+
* guess and so does everything downstream of it.
|
|
2041
|
+
*/
|
|
2042
|
+
disagreements?: CallableWorkflowDisagreement[];
|
|
2043
|
+
}
|
|
2044
|
+
/**
|
|
2045
|
+
* One axis on which the live announcers of a workflow differ.
|
|
2046
|
+
*
|
|
2047
|
+
* Mirrors the durable engine's own `Disagreement` rather than re-deriving it,
|
|
2048
|
+
* and `values` holds every distinct **declared** value: an announcer that stated
|
|
2049
|
+
* nothing on the axis contributes nothing, because silence is not a claim.
|
|
2050
|
+
*/
|
|
2051
|
+
export interface CallableWorkflowDisagreement {
|
|
2052
|
+
axis: 'group' | 'origin' | 'requires';
|
|
2053
|
+
values: string[];
|
|
1105
2054
|
}
|
|
2055
|
+
/**
|
|
2056
|
+
* Why an announced entry must not be committed onto a call node, or `undefined`
|
|
2057
|
+
* when it can be.
|
|
2058
|
+
*
|
|
2059
|
+
* Pure and exported from the browser entry point as well as this one, for the
|
|
2060
|
+
* reason {@link validateWorkflow} is: the canvas that greys the option out and
|
|
2061
|
+
* anything server-side that reasons about the same list must apply *the same*
|
|
2062
|
+
* rule. A picker with its own copy of it is a picker that eventually offers
|
|
2063
|
+
* something the rest of the system considers unusable.
|
|
2064
|
+
*
|
|
2065
|
+
* Two refusals, and they are refusals rather than warnings because in both cases
|
|
2066
|
+
* committing the entry would write a node whose meaning nobody can state:
|
|
2067
|
+
*
|
|
2068
|
+
* - `no-version` — an un-upgraded worker announced a bare name. The call node's
|
|
2069
|
+
* whole point is the pin; a node holding a name and no version follows
|
|
2070
|
+
* whatever gets deployed next, which is the failure the version field exists
|
|
2071
|
+
* to prevent. The name is still perfectly typeable by hand *with* a version
|
|
2072
|
+
* the author knows, so this refuses the one-click commit and not the workflow.
|
|
2073
|
+
* - `ambiguous-group` — two live workers claim this exact `name@version` from
|
|
2074
|
+
* different groups. Two groups means two queues, and nothing here can know
|
|
2075
|
+
* which one a run would land on, so the two bodies may not even be the same
|
|
2076
|
+
* code. Picking one on the author's behalf would be acting on a claim nobody
|
|
2077
|
+
* made.
|
|
2078
|
+
*
|
|
2079
|
+
* A disagreement on `origin` or `requires` is deliberately **not** a refusal. It
|
|
2080
|
+
* is worth showing — two packages declaring one name is a mess somebody should
|
|
2081
|
+
* clean up — but it does not change which queue the run goes to, and refusing on
|
|
2082
|
+
* it would block a pin that is otherwise exactly determined.
|
|
2083
|
+
*/
|
|
2084
|
+
export interface CallableWorkflowBlock {
|
|
2085
|
+
code: 'no-version' | 'ambiguous-group';
|
|
2086
|
+
/** A full sentence, addressed to whoever is looking at the picker. */
|
|
2087
|
+
message: string;
|
|
2088
|
+
}
|
|
2089
|
+
export declare function callableWorkflowBlock(ref: CallableWorkflowRef): CallableWorkflowBlock | undefined;
|
|
1106
2090
|
/** Every way a graph can be refused. Exported so a canvas can key off the code. */
|
|
1107
|
-
export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "call-not-named"];
|
|
2091
|
+
export declare const WORKFLOW_ISSUE_CODES: readonly ["empty", "invalid-node-id", "duplicate-node-id", "edge-endpoint-missing", "self-edge", "duplicate-edge", "cycle", "no-source", "source-has-input", "no-sink", "duplicate-sink-type", "sink-has-output", "unreachable", "dead-end", "transform-not-named", "call-not-named", "if-not-named", "if-threshold-invalid", "if-needs-one-input", "branch-not-labelled", "branch-on-plain-edge", "filter-predicate-invalid", "filter-narrows-unacknowledged", "filter-narrows-nothing"];
|
|
1108
2092
|
export type WorkflowIssueCode = (typeof WORKFLOW_ISSUE_CODES)[number];
|
|
1109
2093
|
export interface WorkflowValidationIssue {
|
|
1110
2094
|
code: WorkflowIssueCode;
|
|
@@ -1135,6 +2119,35 @@ export interface WorkflowValidationIssue {
|
|
|
1135
2119
|
* being read.
|
|
1136
2120
|
*/
|
|
1137
2121
|
export declare function validateWorkflow(graph: WorkflowGraph): WorkflowValidationIssue[];
|
|
2122
|
+
/**
|
|
2123
|
+
* The object types whose whole published snapshot a node stands in front of.
|
|
2124
|
+
*
|
|
2125
|
+
* A sink is in this list when it commits in `full` mode — replacing what is
|
|
2126
|
+
* served rather than merging into it — **and** removing the named node would
|
|
2127
|
+
* make it unreachable from everything that originates rows. That second half is
|
|
2128
|
+
* the load-bearing one: a filter on one of two paths into a sink narrows
|
|
2129
|
+
* nothing, because the other path still delivers, and a rule that ignored it
|
|
2130
|
+
* would demand an acknowledgement for a graph where nothing is lost.
|
|
2131
|
+
*
|
|
2132
|
+
* Removal-reachability rather than a dominator algorithm, deliberately. The
|
|
2133
|
+
* graphs here are a screenful of boxes, this runs once per filter node, and the
|
|
2134
|
+
* cheaper version would be a second, subtler implementation of "does this node
|
|
2135
|
+
* decide whether that one runs" living next to `checkReachability` — which is
|
|
2136
|
+
* exactly the kind of duplication that eventually disagrees with the walk the
|
|
2137
|
+
* rest of this file does.
|
|
2138
|
+
*
|
|
2139
|
+
* Exported because the console needs the same answer to offer the right
|
|
2140
|
+
* acknowledgements, and a canvas that computed its own would offer a set the
|
|
2141
|
+
* server then refuses.
|
|
2142
|
+
*
|
|
2143
|
+
* `precomputed` exists only so the validator, which has already built the
|
|
2144
|
+
* adjacency it needs, does not build it twice per filter node. Callers outside
|
|
2145
|
+
* this file pass a graph and nothing else.
|
|
2146
|
+
*/
|
|
2147
|
+
export declare function workflowNarrowedTypes(graph: WorkflowGraph, nodeId: string, precomputed?: {
|
|
2148
|
+
originators: string[];
|
|
2149
|
+
outgoing: ReadonlyMap<string, string[]>;
|
|
2150
|
+
}): string[];
|
|
1138
2151
|
/**
|
|
1139
2152
|
* The order the nodes run in, and the inputs each one gets.
|
|
1140
2153
|
*
|
|
@@ -1147,10 +2160,68 @@ export declare function validateWorkflow(graph: WorkflowGraph): WorkflowValidati
|
|
|
1147
2160
|
* order over a broken graph is a load that half-happens, which is harder to
|
|
1148
2161
|
* recover from than one that never started.
|
|
1149
2162
|
*/
|
|
1150
|
-
export declare function workflowRunOrder(graph: WorkflowGraph):
|
|
2163
|
+
export declare function workflowRunOrder(graph: WorkflowGraph): WorkflowRunOrderEntry[];
|
|
2164
|
+
/** One position in the order: which node, what feeds it, and on which branches. */
|
|
2165
|
+
export interface WorkflowRunOrderEntry {
|
|
1151
2166
|
node: WorkflowNode;
|
|
2167
|
+
/** Upstream node ids, in inbound-edge order. */
|
|
1152
2168
|
inputs: string[];
|
|
1153
|
-
|
|
2169
|
+
/**
|
|
2170
|
+
* The branch label of each inbound edge that has one, keyed by the upstream
|
|
2171
|
+
* node id.
|
|
2172
|
+
*
|
|
2173
|
+
* A record rather than an array positionally aligned with {@link inputs},
|
|
2174
|
+
* which was the obvious shape and is unsafe for one specific reason: this
|
|
2175
|
+
* travels through a durable checkpoint as JSON, and `JSON.stringify` turns an
|
|
2176
|
+
* `undefined` hole in an array into `null`. A plain wire would come back from
|
|
2177
|
+
* a replay as `null` rather than absent, `null !== 'then'` would be false, and
|
|
2178
|
+
* the node would be treated as dead — the graph would silently stop running
|
|
2179
|
+
* half of itself on resumed runs only. A key that is simply not there
|
|
2180
|
+
* round-trips exactly. The key is unique because duplicate edges are refused.
|
|
2181
|
+
*/
|
|
2182
|
+
inputBranches: Record<string, WorkflowBranchLabel>;
|
|
2183
|
+
}
|
|
2184
|
+
/**
|
|
2185
|
+
* Whether a node runs, given what the nodes before it did.
|
|
2186
|
+
*
|
|
2187
|
+
* ## The rule
|
|
2188
|
+
*
|
|
2189
|
+
* A node runs when **at least one wire into it is live**, where a wire is live
|
|
2190
|
+
* if its source ran and — when the wire carries a branch label — the source is
|
|
2191
|
+
* an `if` that took that branch. A node with no inbound wires always runs, which
|
|
2192
|
+
* is every source and every originating call.
|
|
2193
|
+
*
|
|
2194
|
+
* ## Why the obvious rule is wrong
|
|
2195
|
+
*
|
|
2196
|
+
* The naive version is "mark everything downstream of the untaken edge as
|
|
2197
|
+
* skipped", and it is wrong on the shape branches are most often drawn in:
|
|
2198
|
+
*
|
|
2199
|
+
* ```
|
|
2200
|
+
* ┌ then → A ┐
|
|
2201
|
+
* if ──┤ ├→ C → sink
|
|
2202
|
+
* └ else → B ┘
|
|
2203
|
+
* ```
|
|
2204
|
+
*
|
|
2205
|
+
* `C` is downstream of `B`. Take the `then` branch and the naive rule walks from
|
|
2206
|
+
* the untaken `else` edge, reaches `B`, reaches `C`, and skips it — so the sink
|
|
2207
|
+
* never runs and the load silently commits nothing, on a graph whose whole
|
|
2208
|
+
* purpose was that both branches converge. Reachability from the **taken** edges
|
|
2209
|
+
* gets it right: `C` is reached through `A`, so it runs, and `B` — reached only
|
|
2210
|
+
* through the untaken edge — does not. `C` then sees an empty stage ref for `B`,
|
|
2211
|
+
* which is exactly what `stageRefsFor` already does for an upstream that
|
|
2212
|
+
* produced nothing, so the positions a merge reads stay aligned with the wires
|
|
2213
|
+
* that were drawn.
|
|
2214
|
+
*
|
|
2215
|
+
* It is evaluated incrementally rather than as a graph walk because the answers
|
|
2216
|
+
* arrive as the run goes: `workflowRunOrder` is topological, so by the time a
|
|
2217
|
+
* node is reached every node feeding it has an outcome. That also means this
|
|
2218
|
+
* reads **only** what was recorded — no predicate is re-evaluated here, which is
|
|
2219
|
+
* the property the whole branch feature rests on.
|
|
2220
|
+
*/
|
|
2221
|
+
export declare function workflowNodeRuns(entry: {
|
|
2222
|
+
inputs: readonly string[];
|
|
2223
|
+
inputBranches?: Readonly<Record<string, WorkflowBranchLabel>>;
|
|
2224
|
+
}, outcomes: Readonly<Record<string, WorkflowNodeOutcome>>): boolean;
|
|
1154
2225
|
/**
|
|
1155
2226
|
* A stable fingerprint of what a graph *does*.
|
|
1156
2227
|
*
|
|
@@ -1313,49 +2384,6 @@ export interface CatalogWorkflowStore {
|
|
|
1313
2384
|
schedule?: string;
|
|
1314
2385
|
enabled?: boolean;
|
|
1315
2386
|
}, changedBy: string): Promise<CatalogWorkflow>;
|
|
1316
|
-
/**
|
|
1317
|
-
* Wrap a connector that predates workflows into the graph it always was.
|
|
1318
|
-
*
|
|
1319
|
-
* **The upgrade path, and the reason removing `POST connectors` does not
|
|
1320
|
-
* orphan anything.** A deployment upgrading into this change has connectors
|
|
1321
|
-
* that were authored directly — a kind, a config, optionally one transform,
|
|
1322
|
-
* and a target type — and no route can edit them any more. Three outcomes
|
|
1323
|
-
* were possible: migrate them, leave them readable but frozen, or make
|
|
1324
|
-
* somebody rebuild each by hand. This is the first, and it is chosen because
|
|
1325
|
-
* the shape was already exactly representable: a connector *is* a
|
|
1326
|
-
* single-source, single-sink graph with at most one transform in the middle,
|
|
1327
|
-
* and it has been describable as one since `workflowId` existed.
|
|
1328
|
-
*
|
|
1329
|
-
* ## What is preserved, and why each matters
|
|
1330
|
-
*
|
|
1331
|
-
* - **The connector id.** It is the mutex key, the owner of every
|
|
1332
|
-
* `ConnectorRun` and the holder of the watermark. Minting a fresh connector
|
|
1333
|
-
* and leaving the old one would split one pipeline's history in two and
|
|
1334
|
-
* leave nothing serialising the halves against each other.
|
|
1335
|
-
* - **The watermark, re-keyed.** A plain connector kept a flat `state`; a
|
|
1336
|
-
* graph keys it by source-node id. Carrying the blob over unchanged would
|
|
1337
|
-
* leave the new source node with no watermark at all, and the first run
|
|
1338
|
-
* after the upgrade would re-read an incremental source from the beginning —
|
|
1339
|
-
* which is not data loss, but is a surprise measured in hours on a large
|
|
1340
|
-
* source.
|
|
1341
|
-
* - **The schedule**, moved to the graph, which is where it is authored now.
|
|
1342
|
-
*
|
|
1343
|
-
* ## What changes, stated plainly
|
|
1344
|
-
*
|
|
1345
|
-
* The graph is published as `ready` without a person having declared it
|
|
1346
|
-
* finished, because refusing to publish would leave the pipeline not running —
|
|
1347
|
-
* an upgrade that silently stops twelve loads is a worse outcome than one that
|
|
1348
|
-
* carries a decision forward on the operator's behalf. It is validated first
|
|
1349
|
-
* and the adoption is **refused** if the wrap does not validate, so nothing is
|
|
1350
|
-
* published that could not have been drawn.
|
|
1351
|
-
*
|
|
1352
|
-
* Idempotent: a connector that already has a `workflowId` is left alone and
|
|
1353
|
-
* answered `undefined`. That is what makes it safe to run at every boot.
|
|
1354
|
-
*/
|
|
1355
|
-
adoptConnector(connectorId: string, adoptedBy: string): Promise<{
|
|
1356
|
-
workflow: CatalogWorkflow;
|
|
1357
|
-
connector: CatalogConnector;
|
|
1358
|
-
} | undefined>;
|
|
1359
2387
|
}
|
|
1360
2388
|
/**
|
|
1361
2389
|
* Where the rows between two nodes actually sit.
|
|
@@ -1612,6 +2640,27 @@ export interface CatalogConnection {
|
|
|
1612
2640
|
lastCheckOk?: boolean;
|
|
1613
2641
|
lastCheckError?: string;
|
|
1614
2642
|
}
|
|
2643
|
+
/**
|
|
2644
|
+
* What a redacted password reads as, on the wire.
|
|
2645
|
+
*
|
|
2646
|
+
* Declared here rather than in `@dudousxd/nestjs-catalog-pipeline`, where the
|
|
2647
|
+
* redaction itself lives, because this literal is not an implementation detail
|
|
2648
|
+
* of the redaction: it is part of what `GET pipeline/connections` answers, and
|
|
2649
|
+
* a browser is the audience it was invented for. A form that lets somebody
|
|
2650
|
+
* paste an address has to be able to recognise the string it was shown — a
|
|
2651
|
+
* `url` whose password is exactly this came out of a read, and posting it back
|
|
2652
|
+
* as a NEW connection stores the word "REDACTED" as the password. There is no
|
|
2653
|
+
* stored row behind a create for `restoreRedactedSecrets` to put the real one
|
|
2654
|
+
* back from, so nothing downstream can catch it: the row saves, and the failure
|
|
2655
|
+
* arrives at the first scheduled load as an authentication error against a
|
|
2656
|
+
* password nobody typed.
|
|
2657
|
+
*
|
|
2658
|
+
* The pipeline package re-exports this rather than declaring its own, so the
|
|
2659
|
+
* two halves cannot drift. A fixed literal rather than a run of asterisks, so
|
|
2660
|
+
* it is greppable in a bug report and cannot be mistaken for a password
|
|
2661
|
+
* somebody actually chose.
|
|
2662
|
+
*/
|
|
2663
|
+
export declare const REDACTED_SECRET = "REDACTED";
|
|
1615
2664
|
/** What checking a connection found. */
|
|
1616
2665
|
export interface ConnectionCheck {
|
|
1617
2666
|
ok: boolean;
|