@dudousxd/nestjs-catalog 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/catalog.environment.d.ts +12 -1
- package/dist/catalog.environment.js +12 -1
- package/dist/catalog.pipeline.d.ts +252 -0
- package/dist/catalog.pipeline.js +133 -1
- package/dist/client.d.ts +2 -2
- package/dist/client.js +13 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +10 -4
- package/dist/transform-runner.d.ts +42 -1
- package/dist/transform-runner.js +677 -28
- package/package.json +1 -1
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
* its own durable keyspace ({@link durableKeyspaceFor}), for the reason spelled
|
|
25
25
|
* out there.
|
|
26
26
|
*/
|
|
27
|
-
import type { ConnectorKind, TransformLanguage, WorkflowEdge, WorkflowNode } from './catalog.pipeline';
|
|
27
|
+
import type { ConnectorKind, TransformLanguage, TransformMode, WorkflowEdge, WorkflowNode } from './catalog.pipeline';
|
|
28
28
|
/**
|
|
29
29
|
* Where a request declares its environment.
|
|
30
30
|
*
|
|
@@ -337,6 +337,17 @@ export interface PromotableTransform {
|
|
|
337
337
|
description?: string;
|
|
338
338
|
language: TransformLanguage;
|
|
339
339
|
code: string;
|
|
340
|
+
/**
|
|
341
|
+
* Whether the code is called once with the batch or once per record.
|
|
342
|
+
*
|
|
343
|
+
* Carried, and **resolved** rather than left absent, unlike most optional
|
|
344
|
+
* fields on this shape. The mode decides what the same text means, so a
|
|
345
|
+
* promotion that dropped it would run the identical transform under a
|
|
346
|
+
* different contract in production than in dev — the one failure a promotion
|
|
347
|
+
* exists to make impossible. Resolved because a plan is a description of what
|
|
348
|
+
* the apply will do, and "absent, and the target will decide" is not one.
|
|
349
|
+
*/
|
|
350
|
+
mode: TransformMode;
|
|
340
351
|
/** The source's version, carried for the record only — see {@link planPromotion}. */
|
|
341
352
|
version: number;
|
|
342
353
|
}
|
|
@@ -407,7 +407,18 @@ function planTransforms(source, target, selected) {
|
|
|
407
407
|
if (!selected('transform', transform.id))
|
|
408
408
|
continue;
|
|
409
409
|
const existing = targetTransforms.get(transform.id);
|
|
410
|
-
|
|
410
|
+
// `mode` is compared, and forgetting it would be the quiet failure this
|
|
411
|
+
// whole field exists to prevent: a transform switched from batch to
|
|
412
|
+
// per-record with its code untouched is a real change to what it computes,
|
|
413
|
+
// and a plan that reported "nothing to release" would leave production
|
|
414
|
+
// running the other contract.
|
|
415
|
+
const fields = diffFields(existing, transform, [
|
|
416
|
+
'name',
|
|
417
|
+
'description',
|
|
418
|
+
'language',
|
|
419
|
+
'code',
|
|
420
|
+
'mode',
|
|
421
|
+
]);
|
|
411
422
|
changes.push({
|
|
412
423
|
kind: 'transform',
|
|
413
424
|
id: transform.id,
|
|
@@ -425,6 +425,65 @@ export declare const TRANSFORM_LANGUAGES: readonly ["javascript", "typescript",
|
|
|
425
425
|
export type TransformLanguage = (typeof TRANSFORM_LANGUAGES)[number];
|
|
426
426
|
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
427
427
|
export declare function isTransformLanguage(value: unknown): value is TransformLanguage;
|
|
428
|
+
/**
|
|
429
|
+
* Whether a transform is a function over the whole batch or over one record.
|
|
430
|
+
*
|
|
431
|
+
* ## Why this is declared and never inferred
|
|
432
|
+
*
|
|
433
|
+
* The two are not interchangeable and the difference is invisible in the code.
|
|
434
|
+
* `records.map(...)` and a body that returns one object read almost identically,
|
|
435
|
+
* and a detector that guessed from destructuring — `{ records }` versus
|
|
436
|
+
* `{ record }` — would be reading a *parameter name*, which is the author's to
|
|
437
|
+
* choose and which minification, a rename, or a rest parameter changes without
|
|
438
|
+
* changing what the function computes. Guess wrong towards `record` and an
|
|
439
|
+
* aggregation is called 102,520 times and returns 102,520 partial answers, none
|
|
440
|
+
* of which fails; guess wrong towards `batch` and a per-record function is handed
|
|
441
|
+
* an array and reads `undefined` off every property. Both commit. Neither errors.
|
|
442
|
+
* So the mode is a field somebody set, and the cost of setting it is one control
|
|
443
|
+
* in the editor.
|
|
444
|
+
*
|
|
445
|
+
* ## Why a closed list rather than `streaming?: boolean`
|
|
446
|
+
*
|
|
447
|
+
* The identical argument {@link WORKFLOW_CALL_MODES} makes one level up. A flag
|
|
448
|
+
* beside a future third calling convention — a windowed transform, a keyed one —
|
|
449
|
+
* is two optional booleans whose combinations nobody defined, and each reader
|
|
450
|
+
* invents its own rule for which wins. A closed list with an exhaustiveness guard
|
|
451
|
+
* ({@link unreachableTransformMode}) makes a third convention a compile error
|
|
452
|
+
* naming the files that have to answer for it: the harness that generates the
|
|
453
|
+
* call, the runner that chooses a transport, and the two runners that consume
|
|
454
|
+
* the result.
|
|
455
|
+
*
|
|
456
|
+
* ## What the default has to be, and why it is not a choice
|
|
457
|
+
*
|
|
458
|
+
* Absent means {@link CatalogTransform.mode} was never set, which is every
|
|
459
|
+
* transform stored before this field existed, and every one of them is a function
|
|
460
|
+
* over the whole batch — the harness handed it `records` and there was no other
|
|
461
|
+
* shape to write. Reading absence as anything else would silently change what a
|
|
462
|
+
* deployment's existing loads compute. Read it through {@link transformMode}
|
|
463
|
+
* rather than defaulting it a second time.
|
|
464
|
+
*/
|
|
465
|
+
export declare const TRANSFORM_MODES: readonly ["batch", "record"];
|
|
466
|
+
export type TransformMode = (typeof TRANSFORM_MODES)[number];
|
|
467
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
468
|
+
export declare function isTransformMode(value: unknown): value is TransformMode;
|
|
469
|
+
/**
|
|
470
|
+
* {@link unreachableCallMode}, for transforms, and for the identical reason.
|
|
471
|
+
*
|
|
472
|
+
* Every branch over {@link TransformMode} ends here, so a third calling
|
|
473
|
+
* convention added to the list without a harness to generate it, a transport to
|
|
474
|
+
* carry it and a consumer to read its output is a type error naming the file. It
|
|
475
|
+
* throws as well, because a mode arrives as JSON out of a column and a build
|
|
476
|
+
* older than the data is a thing that happens.
|
|
477
|
+
*/
|
|
478
|
+
export declare function unreachableTransformMode(mode: never, where: string): never;
|
|
479
|
+
/**
|
|
480
|
+
* The mode this transform runs in, with the default applied once.
|
|
481
|
+
*
|
|
482
|
+
* Absent means `'batch'` — see {@link TRANSFORM_MODES}. One function so that the
|
|
483
|
+
* store, the runner, the two consumers, the editor and the try pane cannot each
|
|
484
|
+
* carry their own `?? 'batch'` and have one of them drift.
|
|
485
|
+
*/
|
|
486
|
+
export declare function transformMode(transform: Pick<CatalogTransform, 'mode'>): TransformMode;
|
|
428
487
|
/**
|
|
429
488
|
* User code that maps a source record to a row.
|
|
430
489
|
*
|
|
@@ -481,11 +540,40 @@ export interface CatalogTransform {
|
|
|
481
540
|
* a new field costs one generated line rather than an edit to stored code.
|
|
482
541
|
*/
|
|
483
542
|
code: string;
|
|
543
|
+
/**
|
|
544
|
+
* Whether {@link code} is called once with the batch or once per record.
|
|
545
|
+
*
|
|
546
|
+
* Absent means `'batch'`, which is what every transform stored before this
|
|
547
|
+
* field existed is. See {@link TRANSFORM_MODES} for why it is declared rather
|
|
548
|
+
* than inferred, and read it through {@link transformMode}.
|
|
549
|
+
*
|
|
550
|
+
* A `'record'` transform is constrained in two ways the mode alone does not
|
|
551
|
+
* say, and both are refused rather than discovered at run time — see
|
|
552
|
+
* {@link recordModeRefusal}: it must be a **module**, because a bare body has
|
|
553
|
+
* `records` in scope by the harness's own construction, and it cannot be
|
|
554
|
+
* **Python**, because that harness writes the `def` and has no second one yet.
|
|
555
|
+
*/
|
|
556
|
+
mode?: TransformMode;
|
|
484
557
|
version: number;
|
|
485
558
|
createdBy: string;
|
|
486
559
|
createdAt: string;
|
|
487
560
|
updatedAt: string;
|
|
488
561
|
}
|
|
562
|
+
/**
|
|
563
|
+
* Why this transform cannot run in the mode it declares, if it cannot.
|
|
564
|
+
*
|
|
565
|
+
* Two combinations are representable and neither can work, so both are refused
|
|
566
|
+
* at the point somebody presses save rather than at three in the morning when a
|
|
567
|
+
* schedule fires. `undefined` means there is nothing wrong.
|
|
568
|
+
*
|
|
569
|
+
* Asked in both places on purpose. The controller asks so the author is told
|
|
570
|
+
* while they are still looking at the code; the runner asks because a row can
|
|
571
|
+
* reach it that no controller in this build ever validated — promoted from
|
|
572
|
+
* another environment, restored from a backup, written by an older version — and
|
|
573
|
+
* the failure a runner must never have is the silent one where a per-record
|
|
574
|
+
* module is handed an array and quietly reads `undefined` off every property.
|
|
575
|
+
*/
|
|
576
|
+
export declare function recordModeRefusal(transform: Pick<CatalogTransform, 'language' | 'code' | 'mode'>): string | undefined;
|
|
489
577
|
/**
|
|
490
578
|
* The single argument a module-shaped transform is called with.
|
|
491
579
|
*
|
|
@@ -557,6 +645,77 @@ export interface CatalogTransformInput<TRecord = Record<string, unknown>> {
|
|
|
557
645
|
* ```
|
|
558
646
|
*/
|
|
559
647
|
export type CatalogTransformFunction<TRecord = Record<string, unknown>> = (input: CatalogTransformInput<TRecord>) => Array<Record<string, unknown>> | Promise<Array<Record<string, unknown>>>;
|
|
648
|
+
/**
|
|
649
|
+
* The single argument a `'record'`-mode transform is called with, once per
|
|
650
|
+
* record.
|
|
651
|
+
*
|
|
652
|
+
* One object, for the reason {@link CatalogTransformInput} gives and not a
|
|
653
|
+
* second time: a field can be added later without redefining what any signature
|
|
654
|
+
* already written means.
|
|
655
|
+
*
|
|
656
|
+
* ## `record` rather than `records`, deliberately one letter apart
|
|
657
|
+
*
|
|
658
|
+
* Which is a real risk and was weighed against the alternatives. A name like
|
|
659
|
+
* `row` or `item` would be further from its sibling and would be *wrong*: what
|
|
660
|
+
* arrives is a record exactly as the source produced it, and a row is what the
|
|
661
|
+
* transform returns — the two words already mean different things everywhere
|
|
662
|
+
* else in this package, and borrowing one of them here to reduce a typo would
|
|
663
|
+
* make the vocabulary lie.
|
|
664
|
+
*
|
|
665
|
+
* The typo it invites is also the one mistake in this area that cannot go quiet.
|
|
666
|
+
* `{ records }` in a per-record transform destructures `undefined`, and the first
|
|
667
|
+
* thing anybody does with it — `.map`, `.length`, `.filter` — throws on the very
|
|
668
|
+
* first record, with a stack frame in the author's own file. The dangerous
|
|
669
|
+
* direction is the other one, and that is exactly what {@link TRANSFORM_MODES}
|
|
670
|
+
* refuses to guess about.
|
|
671
|
+
*
|
|
672
|
+
* ## What is not on it
|
|
673
|
+
*
|
|
674
|
+
* No index, no total, no `isFirst`. Each of those is a way to write a transform
|
|
675
|
+
* whose answer depends on where a record fell in the stream, which is the
|
|
676
|
+
* property this mode exists to *not* have — a record's row must be a function of
|
|
677
|
+
* that record. `context.rowCount` is a count of what reached the node and is
|
|
678
|
+
* already there for anything that legitimately needs the size of the load.
|
|
679
|
+
*/
|
|
680
|
+
export interface CatalogRecordTransformInput<TRecord = Record<string, unknown>> {
|
|
681
|
+
/** One record, exactly as the source produced it. */
|
|
682
|
+
record: TRecord;
|
|
683
|
+
/** The run, the node, the counts, and the admitted environment variables. */
|
|
684
|
+
context: CatalogCodeContext;
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* The function a `'record'`-mode transform exports.
|
|
688
|
+
*
|
|
689
|
+
* **For the editor and nothing else**, exactly as {@link CatalogTransformFunction}
|
|
690
|
+
* is: TypeScript transforms run through Node's own type *stripping*, so the
|
|
691
|
+
* annotations are erased on the way in and a wrong one is a squiggle rather than
|
|
692
|
+
* a failed run.
|
|
693
|
+
*
|
|
694
|
+
* The return type is the whole contract of the mode and it is deliberately four
|
|
695
|
+
* things at once:
|
|
696
|
+
*
|
|
697
|
+
* - **an object** — one row, the ordinary case;
|
|
698
|
+
* - **an array** — several rows, so one record can fan out;
|
|
699
|
+
* - **an empty array** — no rows, so a record can be dropped;
|
|
700
|
+
* - **`null` or `undefined`** — no rows either, because that is what a function
|
|
701
|
+
* with a bare `return` or a missed branch produces and reading it as anything
|
|
702
|
+
* else would invent a row nobody wrote.
|
|
703
|
+
*
|
|
704
|
+
* Map, filter and flatMap under one rule, and no ambiguity between the first two
|
|
705
|
+
* cases: an array is never a row, because a row is a plain object everywhere in
|
|
706
|
+
* this package and the runners have always dropped anything else.
|
|
707
|
+
*
|
|
708
|
+
* ```ts
|
|
709
|
+
* import type { CatalogRecordTransformFunction } from '@dudousxd/nestjs-catalog/client';
|
|
710
|
+
*
|
|
711
|
+
* const transform: CatalogRecordTransformFunction<{ 'Mgmt Cd': string }> = ({ record }) => ({
|
|
712
|
+
* mgmtCd: record['Mgmt Cd'],
|
|
713
|
+
* });
|
|
714
|
+
*
|
|
715
|
+
* export default transform;
|
|
716
|
+
* ```
|
|
717
|
+
*/
|
|
718
|
+
export type CatalogRecordTransformFunction<TRecord = Record<string, unknown>> = (input: CatalogRecordTransformInput<TRecord>) => Record<string, unknown> | Array<Record<string, unknown>> | null | undefined | Promise<Record<string, unknown> | Array<Record<string, unknown>> | null | undefined>;
|
|
560
719
|
export interface TransformResult {
|
|
561
720
|
rows: Array<Record<string, unknown>>;
|
|
562
721
|
/**
|
|
@@ -595,11 +754,87 @@ export interface TransformResult {
|
|
|
595
754
|
* people who are not already trusted with the database needs a container or a
|
|
596
755
|
* sandboxed runtime, and this interface is where that gets plugged in.
|
|
597
756
|
*/
|
|
757
|
+
/**
|
|
758
|
+
* What a per-record run produced, asked **only after {@link TransformStream.rows}
|
|
759
|
+
* is exhausted**.
|
|
760
|
+
*
|
|
761
|
+
* The stream equivalent of {@link TransformResult}, and it is a separate shape
|
|
762
|
+
* rather than the same one because the fields genuinely differ in *when they are
|
|
763
|
+
* knowable*. `rows` is not a value here — the whole point is that nothing holds
|
|
764
|
+
* them — so what is left is the counts and the log, and neither is final until
|
|
765
|
+
* the last record has gone past. `recordsIn` is new and is the one number a
|
|
766
|
+
* batch call never needed: a caller that streamed its source has no `.length` to
|
|
767
|
+
* report as `fetched`.
|
|
768
|
+
*/
|
|
769
|
+
export interface TransformStreamSummary {
|
|
770
|
+
/** How many records the runner fed the code. */
|
|
771
|
+
recordsIn: number;
|
|
772
|
+
/** How many rows came back, over every record. */
|
|
773
|
+
rowsOut: number;
|
|
774
|
+
/** {@link TransformResult.logs}, bounded by the runner in exactly the same way. */
|
|
775
|
+
logs: string[];
|
|
776
|
+
elapsedMs: number;
|
|
777
|
+
}
|
|
778
|
+
/**
|
|
779
|
+
* A per-record run in progress: the rows as they arrive, and the counts once
|
|
780
|
+
* they have.
|
|
781
|
+
*
|
|
782
|
+
* The same two-part shape `StreamedFetchResult` uses in the pipeline package —
|
|
783
|
+
* an iterable plus a function asked afterwards — and copied from it on purpose
|
|
784
|
+
* rather than invented. The reason it gives is the reason here: a stream is not
|
|
785
|
+
* complete until it has been drained, so anything computed *over* it is not yet
|
|
786
|
+
* known when the call returns, and a field would hand a caller a number that
|
|
787
|
+
* stops short of the rows they have already written.
|
|
788
|
+
*
|
|
789
|
+
* {@link summary} before {@link rows} is exhausted is a programming error and
|
|
790
|
+
* the bundled runner throws rather than answering with a running total, because
|
|
791
|
+
* a running total is exactly what somebody would then record as `fetched`.
|
|
792
|
+
*/
|
|
793
|
+
export interface TransformStream {
|
|
794
|
+
/**
|
|
795
|
+
* The rows, in record order, in the order the code emitted them.
|
|
796
|
+
*
|
|
797
|
+
* Pulled, not pushed: the next record is not fed to the code until the row
|
|
798
|
+
* before it has been taken, so a consumer that writes to a database
|
|
799
|
+
* back-pressures all the way to the source. That is the property the mode
|
|
800
|
+
* exists for and it is the caller's to keep — a consumer that collects this
|
|
801
|
+
* into an array has re-created the whole-batch memory profile with extra
|
|
802
|
+
* steps.
|
|
803
|
+
*
|
|
804
|
+
* Throws where the code threw, naming the record. See the runner.
|
|
805
|
+
*/
|
|
806
|
+
rows: AsyncIterable<Record<string, unknown>>;
|
|
807
|
+
/** The counts and the log. Call only after {@link rows} is exhausted. */
|
|
808
|
+
summary(): TransformStreamSummary;
|
|
809
|
+
}
|
|
598
810
|
export interface TransformRunner {
|
|
599
811
|
run(transform: Pick<CatalogTransform, 'language' | 'code'>, records: unknown[], options?: {
|
|
600
812
|
timeoutMs?: number;
|
|
601
813
|
context?: CatalogCodeContext;
|
|
602
814
|
}): Promise<TransformResult>;
|
|
815
|
+
/**
|
|
816
|
+
* Run a `'record'`-mode transform over a stream of records, streaming the rows
|
|
817
|
+
* back.
|
|
818
|
+
*
|
|
819
|
+
* **Optional**, mixed in for the reason every optional member of
|
|
820
|
+
* {@link CatalogPipelineStore} is: a runner written against the previous shape
|
|
821
|
+
* of this interface still satisfies it, and a purely additive capability must
|
|
822
|
+
* not turn that into a compile error. {@link supportsTransformStreaming} is how
|
|
823
|
+
* a caller asks; a deployment whose runner cannot stream runs a per-record
|
|
824
|
+
* transform through {@link run} against a buffered batch instead, which is
|
|
825
|
+
* slower and holds more but computes the identical rows.
|
|
826
|
+
*
|
|
827
|
+
* Not a widening of {@link run}. The two differ in what the caller must hand
|
|
828
|
+
* over (an array against an iterable), in what comes back (rows against a
|
|
829
|
+
* stream of them), in when the counts are knowable, and in what the timeout
|
|
830
|
+
* measures — see the bundled runner, where a stream is bounded by a stall
|
|
831
|
+
* rather than by total wall clock. One method doing both would have four
|
|
832
|
+
* optional fields and a reader could not tell which combination was legal.
|
|
833
|
+
*/
|
|
834
|
+
runStream?(transform: Pick<CatalogTransform, 'language' | 'code' | 'mode'>, records: AsyncIterable<unknown>, options?: {
|
|
835
|
+
timeoutMs?: number;
|
|
836
|
+
context?: CatalogCodeContext;
|
|
837
|
+
}): Promise<TransformStream>;
|
|
603
838
|
/** Languages this runner can actually execute in this environment. */
|
|
604
839
|
available(): Promise<TransformLanguage[]>;
|
|
605
840
|
/**
|
|
@@ -611,6 +846,16 @@ export interface TransformRunner {
|
|
|
611
846
|
*/
|
|
612
847
|
pythonPackages?(): Promise<string[]>;
|
|
613
848
|
}
|
|
849
|
+
/**
|
|
850
|
+
* Whether this runner can stream a per-record transform.
|
|
851
|
+
*
|
|
852
|
+
* The method rather than a flag, exactly as {@link supportsTransformRevisions}
|
|
853
|
+
* argues one interface along: a flag is a claim and a method is the thing
|
|
854
|
+
* itself. A runner that answers `false` still runs `'record'` transforms — the
|
|
855
|
+
* consumers buffer and call {@link TransformRunner.run} — so this is a question
|
|
856
|
+
* about *how much is held*, never about whether the load works.
|
|
857
|
+
*/
|
|
858
|
+
export declare function supportsTransformStreaming(runner: TransformRunner): runner is TransformRunner & Required<Pick<TransformRunner, 'runStream'>>;
|
|
614
859
|
export declare const TRANSFORM_RUNNER: unique symbol;
|
|
615
860
|
/**
|
|
616
861
|
* The number in {@link CatalogCodeContext.contract}.
|
|
@@ -3677,6 +3922,13 @@ export interface CatalogPipelineStore extends Partial<CatalogWorkflowStore>, Par
|
|
|
3677
3922
|
saveTransform(input: Pick<CatalogTransform, 'name' | 'language' | 'code'> & {
|
|
3678
3923
|
id?: string;
|
|
3679
3924
|
description?: string;
|
|
3925
|
+
/**
|
|
3926
|
+
* Absent leaves the stored mode alone rather than resetting it to
|
|
3927
|
+
* `'batch'`, which is what a caller written before this field existed
|
|
3928
|
+
* means and the only reading under which such a caller cannot silently
|
|
3929
|
+
* change what a transform computes. See {@link TRANSFORM_MODES}.
|
|
3930
|
+
*/
|
|
3931
|
+
mode?: TransformMode;
|
|
3680
3932
|
}, createdBy: string): Promise<CatalogTransform>;
|
|
3681
3933
|
deleteTransform(id: string): Promise<boolean>;
|
|
3682
3934
|
/**
|
package/dist/catalog.pipeline.js
CHANGED
|
@@ -9,11 +9,16 @@
|
|
|
9
9
|
* systems each believing they decide when a load runs.
|
|
10
10
|
*/
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.REDACTED_SECRET = exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_BRANCH_LABELS = exports.NODE_KIND_IS_REUSABLE = exports.REUSABLE_NODE_KINDS = exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_RENAME_UNNAMED = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_SKIP_REASONS = exports.CODE_CONTEXT_CONTRACT = exports.TRANSFORM_RUNNER = exports.TRANSFORM_LANGUAGES = exports.SOURCE_FORMATS = exports.CONNECTOR_KINDS = void 0;
|
|
12
|
+
exports.REDACTED_SECRET = exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_BRANCH_LABELS = exports.NODE_KIND_IS_REUSABLE = exports.REUSABLE_NODE_KINDS = exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_RENAME_UNNAMED = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_COLUMN_GAP = exports.WORKFLOW_NODE_HEIGHT = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_SKIP_REASONS = exports.CODE_CONTEXT_CONTRACT = exports.TRANSFORM_RUNNER = exports.TRANSFORM_MODES = exports.TRANSFORM_LANGUAGES = exports.SOURCE_FORMATS = exports.CONNECTOR_KINDS = void 0;
|
|
13
13
|
exports.isConnectorKind = isConnectorKind;
|
|
14
14
|
exports.isSourceFormat = isSourceFormat;
|
|
15
15
|
exports.unreachableSourceFormat = unreachableSourceFormat;
|
|
16
16
|
exports.isTransformLanguage = isTransformLanguage;
|
|
17
|
+
exports.isTransformMode = isTransformMode;
|
|
18
|
+
exports.unreachableTransformMode = unreachableTransformMode;
|
|
19
|
+
exports.transformMode = transformMode;
|
|
20
|
+
exports.recordModeRefusal = recordModeRefusal;
|
|
21
|
+
exports.supportsTransformStreaming = supportsTransformStreaming;
|
|
17
22
|
exports.isWorkflowSkipReason = isWorkflowSkipReason;
|
|
18
23
|
exports.isWorkflowNodeKind = isWorkflowNodeKind;
|
|
19
24
|
exports.unreachableNodeKind = unreachableNodeKind;
|
|
@@ -75,6 +80,11 @@ exports.isPipelineStore = isPipelineStore;
|
|
|
75
80
|
// edge only ever points this way: `catalog.workspace.ts` knows nothing about
|
|
76
81
|
// pipelines.
|
|
77
82
|
const catalog_workspace_1 = require("./catalog.workspace");
|
|
83
|
+
// The one rule that tells a module-shaped transform from a bare body, used here
|
|
84
|
+
// to refuse a per-record transform written as a body. The edge only points this
|
|
85
|
+
// way: `transform-shape.ts` imports nothing at all, so it can be read on its own
|
|
86
|
+
// and cannot be dragged into a cycle.
|
|
87
|
+
const transform_shape_1 = require("./transform-shape");
|
|
78
88
|
/**
|
|
79
89
|
* Where a connector pulls from.
|
|
80
90
|
*
|
|
@@ -183,6 +193,128 @@ exports.TRANSFORM_LANGUAGES = ['javascript', 'typescript', 'python'];
|
|
|
183
193
|
function isTransformLanguage(value) {
|
|
184
194
|
return exports.TRANSFORM_LANGUAGES.some((language) => language === value);
|
|
185
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* Whether a transform is a function over the whole batch or over one record.
|
|
198
|
+
*
|
|
199
|
+
* ## Why this is declared and never inferred
|
|
200
|
+
*
|
|
201
|
+
* The two are not interchangeable and the difference is invisible in the code.
|
|
202
|
+
* `records.map(...)` and a body that returns one object read almost identically,
|
|
203
|
+
* and a detector that guessed from destructuring — `{ records }` versus
|
|
204
|
+
* `{ record }` — would be reading a *parameter name*, which is the author's to
|
|
205
|
+
* choose and which minification, a rename, or a rest parameter changes without
|
|
206
|
+
* changing what the function computes. Guess wrong towards `record` and an
|
|
207
|
+
* aggregation is called 102,520 times and returns 102,520 partial answers, none
|
|
208
|
+
* of which fails; guess wrong towards `batch` and a per-record function is handed
|
|
209
|
+
* an array and reads `undefined` off every property. Both commit. Neither errors.
|
|
210
|
+
* So the mode is a field somebody set, and the cost of setting it is one control
|
|
211
|
+
* in the editor.
|
|
212
|
+
*
|
|
213
|
+
* ## Why a closed list rather than `streaming?: boolean`
|
|
214
|
+
*
|
|
215
|
+
* The identical argument {@link WORKFLOW_CALL_MODES} makes one level up. A flag
|
|
216
|
+
* beside a future third calling convention — a windowed transform, a keyed one —
|
|
217
|
+
* is two optional booleans whose combinations nobody defined, and each reader
|
|
218
|
+
* invents its own rule for which wins. A closed list with an exhaustiveness guard
|
|
219
|
+
* ({@link unreachableTransformMode}) makes a third convention a compile error
|
|
220
|
+
* naming the files that have to answer for it: the harness that generates the
|
|
221
|
+
* call, the runner that chooses a transport, and the two runners that consume
|
|
222
|
+
* the result.
|
|
223
|
+
*
|
|
224
|
+
* ## What the default has to be, and why it is not a choice
|
|
225
|
+
*
|
|
226
|
+
* Absent means {@link CatalogTransform.mode} was never set, which is every
|
|
227
|
+
* transform stored before this field existed, and every one of them is a function
|
|
228
|
+
* over the whole batch — the harness handed it `records` and there was no other
|
|
229
|
+
* shape to write. Reading absence as anything else would silently change what a
|
|
230
|
+
* deployment's existing loads compute. Read it through {@link transformMode}
|
|
231
|
+
* rather than defaulting it a second time.
|
|
232
|
+
*/
|
|
233
|
+
exports.TRANSFORM_MODES = [
|
|
234
|
+
/**
|
|
235
|
+
* The code is called **once**, with every record the node received.
|
|
236
|
+
*
|
|
237
|
+
* What every stored transform is, and what aggregating, deduplicating,
|
|
238
|
+
* sorting and joining require: none of them can be done a row at a time, and
|
|
239
|
+
* chunking the calls would not fail — it would return one answer per chunk and
|
|
240
|
+
* commit. The whole input is therefore in the running process's heap, which is
|
|
241
|
+
* the honest cost of the promise.
|
|
242
|
+
*/
|
|
243
|
+
'batch',
|
|
244
|
+
/**
|
|
245
|
+
* The code is called **once per record**, and never sees the batch.
|
|
246
|
+
*
|
|
247
|
+
* What a rename, a projection or a normalisation is by construction — and,
|
|
248
|
+
* since `WORKFLOW_FILTER_COLUMN_PATTERN` and `property-names.ts` both refuse a
|
|
249
|
+
* header with a space in it, what every DPAS file this catalog ingests needs
|
|
250
|
+
* before anything else can touch it. The records arrive as a stream and the
|
|
251
|
+
* rows leave as one, so nothing anywhere holds the dataset.
|
|
252
|
+
*/
|
|
253
|
+
'record',
|
|
254
|
+
];
|
|
255
|
+
/** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
|
|
256
|
+
function isTransformMode(value) {
|
|
257
|
+
return exports.TRANSFORM_MODES.some((mode) => mode === value);
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* {@link unreachableCallMode}, for transforms, and for the identical reason.
|
|
261
|
+
*
|
|
262
|
+
* Every branch over {@link TransformMode} ends here, so a third calling
|
|
263
|
+
* convention added to the list without a harness to generate it, a transport to
|
|
264
|
+
* carry it and a consumer to read its output is a type error naming the file. It
|
|
265
|
+
* throws as well, because a mode arrives as JSON out of a column and a build
|
|
266
|
+
* older than the data is a thing that happens.
|
|
267
|
+
*/
|
|
268
|
+
function unreachableTransformMode(mode, where) {
|
|
269
|
+
throw new Error(`${where} has no rule for the transform mode ${JSON.stringify(mode)}. It was added to TRANSFORM_MODES without teaching this code how to call user code written for it, and guessing would run somebody's transform under a contract they did not write it against.`);
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* The mode this transform runs in, with the default applied once.
|
|
273
|
+
*
|
|
274
|
+
* Absent means `'batch'` — see {@link TRANSFORM_MODES}. One function so that the
|
|
275
|
+
* store, the runner, the two consumers, the editor and the try pane cannot each
|
|
276
|
+
* carry their own `?? 'batch'` and have one of them drift.
|
|
277
|
+
*/
|
|
278
|
+
function transformMode(transform) {
|
|
279
|
+
return transform.mode ?? 'batch';
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Why this transform cannot run in the mode it declares, if it cannot.
|
|
283
|
+
*
|
|
284
|
+
* Two combinations are representable and neither can work, so both are refused
|
|
285
|
+
* at the point somebody presses save rather than at three in the morning when a
|
|
286
|
+
* schedule fires. `undefined` means there is nothing wrong.
|
|
287
|
+
*
|
|
288
|
+
* Asked in both places on purpose. The controller asks so the author is told
|
|
289
|
+
* while they are still looking at the code; the runner asks because a row can
|
|
290
|
+
* reach it that no controller in this build ever validated — promoted from
|
|
291
|
+
* another environment, restored from a backup, written by an older version — and
|
|
292
|
+
* the failure a runner must never have is the silent one where a per-record
|
|
293
|
+
* module is handed an array and quietly reads `undefined` off every property.
|
|
294
|
+
*/
|
|
295
|
+
function recordModeRefusal(transform) {
|
|
296
|
+
if (transformMode(transform) !== 'record')
|
|
297
|
+
return undefined;
|
|
298
|
+
if (transform.language === 'python') {
|
|
299
|
+
return 'A per-record transform cannot be written in Python yet. The Python harness writes `def transform(records, context):` around the code, so a Python transform never states its own signature and there is no second `def` for the per-record shape — see TRANSFORM_MODES. Use javascript or typescript for a per-record transform, or leave this one as a whole-batch transform.';
|
|
300
|
+
}
|
|
301
|
+
if ((0, transform_shape_1.transformShape)(transform.code) === 'body') {
|
|
302
|
+
return 'A per-record transform must be a module that exports a function — `export default function transform({ record, context }) { … }` — and this code has no `export` at the start of a statement, so the catalog would run it as a bare function body. A body is handed `records`, the whole array, by the wrapper the catalog writes around it; there is no honest way to give it one record under a name it never wrote. Add the `export`, or set this transform back to whole-batch.';
|
|
303
|
+
}
|
|
304
|
+
return undefined;
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Whether this runner can stream a per-record transform.
|
|
308
|
+
*
|
|
309
|
+
* The method rather than a flag, exactly as {@link supportsTransformRevisions}
|
|
310
|
+
* argues one interface along: a flag is a claim and a method is the thing
|
|
311
|
+
* itself. A runner that answers `false` still runs `'record'` transforms — the
|
|
312
|
+
* consumers buffer and call {@link TransformRunner.run} — so this is a question
|
|
313
|
+
* about *how much is held*, never about whether the load works.
|
|
314
|
+
*/
|
|
315
|
+
function supportsTransformStreaming(runner) {
|
|
316
|
+
return typeof runner.runStream === 'function';
|
|
317
|
+
}
|
|
186
318
|
exports.TRANSFORM_RUNNER = Symbol('TRANSFORM_RUNNER');
|
|
187
319
|
/**
|
|
188
320
|
* The number in {@link CatalogCodeContext.contract}.
|
package/dist/client.d.ts
CHANGED
|
@@ -153,9 +153,9 @@ export declare const catalogRoutes: {
|
|
|
153
153
|
readonly traces: () => string;
|
|
154
154
|
readonly trace: (id: string) => string;
|
|
155
155
|
};
|
|
156
|
-
export type { CatalogConnection, CatalogConnector, ConnectionCheck, CatalogTransform, CatalogTransformFunction, CatalogTransformInput, CatalogWorkflow, CatalogWorkflowCapabilities, CatalogWorkflowRelease, ConnectorKind, ConnectorRun, TransformLanguage, TransformResult, CallableWorkflowRef, WorkflowBranchLabel, WorkflowCallEnvelope, WorkflowCallMode, WorkflowCallNode, WorkflowCallOutput, WorkflowEdge, WorkflowExecutionMode, WorkflowFilterAll, WorkflowFilterAny, WorkflowFilterComparison, WorkflowFilterGroup, WorkflowFilterNode, WorkflowFilterOneOf, WorkflowFilterOperator, WorkflowFilterPredicate, WorkflowFilterPredicateKind, WorkflowFilterPresence, WorkflowFilterValue, WorkflowGraph, WorkflowEnvPredicate, WorkflowIfNode, WorkflowIfPredicate, WorkflowIssueCode, WorkflowNode, WorkflowNodeKind, WorkflowNodeOutcome, WorkflowPredicateKind, WorkflowRenameNode, WorkflowRenameUnnamed, WorkflowRowCountPredicate, WorkflowRunOrderEntry, WorkflowSinkNode, WorkflowSkipReason, WorkflowSourceNode, WorkflowStageRef, WorkflowTransformNode, WorkflowValidationIssue, CatalogReusableNode, CatalogReusableNodeUse, ReusableNodeBody, ReusableNodeKind, ReusableNodeRef, ReusableSinkBody, ReusableSourceBody, SourceFormat, VersionPinCopy, } from './catalog.pipeline';
|
|
156
|
+
export type { CatalogConnection, CatalogConnector, ConnectionCheck, CatalogTransform, CatalogTransformFunction, CatalogTransformInput, CatalogRecordTransformFunction, CatalogRecordTransformInput, CatalogWorkflow, CatalogWorkflowCapabilities, CatalogWorkflowRelease, ConnectorKind, ConnectorRun, TransformLanguage, TransformMode, TransformResult, CallableWorkflowRef, WorkflowBranchLabel, WorkflowCallEnvelope, WorkflowCallMode, WorkflowCallNode, WorkflowCallOutput, WorkflowEdge, WorkflowExecutionMode, WorkflowFilterAll, WorkflowFilterAny, WorkflowFilterComparison, WorkflowFilterGroup, WorkflowFilterNode, WorkflowFilterOneOf, WorkflowFilterOperator, WorkflowFilterPredicate, WorkflowFilterPredicateKind, WorkflowFilterPresence, WorkflowFilterValue, WorkflowGraph, WorkflowEnvPredicate, WorkflowIfNode, WorkflowIfPredicate, WorkflowIssueCode, WorkflowNode, WorkflowNodeKind, WorkflowNodeOutcome, WorkflowPredicateKind, WorkflowRenameNode, WorkflowRenameUnnamed, WorkflowRowCountPredicate, WorkflowRunOrderEntry, WorkflowSinkNode, WorkflowSkipReason, WorkflowSourceNode, WorkflowStageRef, WorkflowTransformNode, WorkflowValidationIssue, CatalogReusableNode, CatalogReusableNodeUse, ReusableNodeBody, ReusableNodeKind, ReusableNodeRef, ReusableSinkBody, ReusableSourceBody, SourceFormat, VersionPinCopy, } from './catalog.pipeline';
|
|
157
157
|
export { type TransformShape, transformDeclaresModule, transformShape, } from './transform-shape';
|
|
158
|
-
export { CONNECTOR_KINDS, isConnectorKind, isSourceFormat, isTransformLanguage, SOURCE_FORMATS, isWorkflowEdge, isWorkflowNode, liveWorkflowVersion, REDACTED_SECRET, TRANSFORM_LANGUAGES, readWorkflowCallOutput, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, isWorkflowCallMode, unreachableCallMode, workflowCallMode, applyReusableNode, reusableNodeBodyOf, isReusableNodeBody, isReusableNodeKind, REUSABLE_NODE_KINDS, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, unreachableReusableNodeKind, describeLiveVersion, describeVersionPin, } from './catalog.pipeline';
|
|
158
|
+
export { CONNECTOR_KINDS, isConnectorKind, isSourceFormat, isTransformLanguage, isTransformMode, TRANSFORM_MODES, transformMode, recordModeRefusal, SOURCE_FORMATS, isWorkflowEdge, isWorkflowNode, liveWorkflowVersion, REDACTED_SECRET, TRANSFORM_LANGUAGES, readWorkflowCallOutput, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, isWorkflowCallMode, unreachableCallMode, workflowCallMode, applyReusableNode, reusableNodeBodyOf, isReusableNodeBody, isReusableNodeKind, REUSABLE_NODE_KINDS, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, unreachableReusableNodeKind, describeLiveVersion, describeVersionPin, } from './catalog.pipeline';
|
|
159
159
|
/**
|
|
160
160
|
* The workflow validator, shipped to the browser deliberately.
|
|
161
161
|
*
|
package/dist/client.js
CHANGED
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* types are.
|
|
12
12
|
*/
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
-
exports.
|
|
15
|
-
exports.DELETE_RECONCILIATION_STRATEGIES = exports.callableWorkflowBlock = exports.isWorkflowStatus = exports.isWorkflowNodeKind = exports.isWorkflowExecutionMode = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_STATUSES = exports.unreachableRenameUnnamed = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableFilterPredicateKind = exports.unreachableFilterOperator = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowKnownColumns = exports.workflowRenameUnnamed = exports.renameColumnRefusals = exports.workflowFilterMatches = exports.workflowFilterColumns = exports.isWorkflowSkipReason = exports.isWorkflowRenameUnnamed = exports.isWorkflowRenameColumns = exports.isWorkflowPredicateKind = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowBranchLabel = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_RENAME_UNNAMED = exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.workflowRowY = void 0;
|
|
14
|
+
exports.WORKFLOW_COLUMN_GAP = exports.describeVersionPin = exports.describeLiveVersion = exports.unreachableReusableNodeKind = exports.nodeKindIsReusable = exports.NODE_KIND_IS_REUSABLE = exports.REUSABLE_NODE_KINDS = exports.isReusableNodeKind = exports.isReusableNodeBody = exports.reusableNodeBodyOf = exports.applyReusableNode = exports.workflowCallMode = exports.unreachableCallMode = exports.isWorkflowCallMode = exports.WORKFLOW_CALL_MODES = exports.WORKFLOW_CALL_CONTRACT = exports.readWorkflowCallOutput = exports.TRANSFORM_LANGUAGES = exports.REDACTED_SECRET = exports.liveWorkflowVersion = exports.isWorkflowNode = exports.isWorkflowEdge = exports.SOURCE_FORMATS = exports.recordModeRefusal = exports.transformMode = exports.TRANSFORM_MODES = exports.isTransformMode = exports.isTransformLanguage = exports.isSourceFormat = exports.isConnectorKind = exports.CONNECTOR_KINDS = exports.transformShape = exports.transformDeclaresModule = exports.catalogRoutes = exports.UnsafeIdentifierError = exports.physicalColumn = exports.outputAlias = exports.isSafeIdentifier = exports.VALUELESS_FILTER_OPERATORS = exports.resolveObjectFilters = exports.parseObjectFilter = exports.offeredFilterOperators = exports.isCatalogFilterOperator = exports.filterOperatorsFor = exports.filterOperatorTakesValue = exports.encodeObjectFilter = exports.coerceFilterValue = exports.CATALOG_FILTER_OPERATORS = exports.CATALOG_FILTER_LIMIT = exports.CATALOG_REVISION_LIMIT = void 0;
|
|
15
|
+
exports.DELETE_RECONCILIATION_STRATEGIES = exports.callableWorkflowBlock = exports.isWorkflowStatus = exports.isWorkflowNodeKind = exports.isWorkflowExecutionMode = exports.workflowRunOrder = exports.workflowGraphHash = exports.WORKFLOW_STATUSES = exports.unreachableRenameUnnamed = exports.unreachablePredicateKind = exports.unreachableNodeKind = exports.unreachableFilterPredicateKind = exports.unreachableFilterOperator = exports.workflowNodeRuns = exports.workflowNarrowedTypes = exports.workflowKnownColumns = exports.workflowRenameUnnamed = exports.renameColumnRefusals = exports.workflowFilterMatches = exports.workflowFilterColumns = exports.isWorkflowSkipReason = exports.isWorkflowRenameUnnamed = exports.isWorkflowRenameColumns = exports.isWorkflowPredicateKind = exports.isWorkflowIfPredicate = exports.isWorkflowFilterValue = exports.isWorkflowFilterPredicateKind = exports.isWorkflowFilterPredicate = exports.isWorkflowFilterOperator = exports.isWorkflowBranchLabel = exports.WORKFLOW_SKIP_REASONS = exports.WORKFLOW_RENAME_UNNAMED = exports.WORKFLOW_RENAME_MAX_COLUMNS = exports.WORKFLOW_FILTER_PREDICATE_KINDS = exports.WORKFLOW_FILTER_OPERATORS = exports.WORKFLOW_FILTER_MAX_VALUES = exports.WORKFLOW_FILTER_MAX_DEPTH = exports.WORKFLOW_FILTER_COLUMN_PATTERN = exports.WORKFLOW_PREDICATE_KINDS = exports.WORKFLOW_BRANCH_LABELS = exports.WORKFLOW_NODE_KINDS = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.validateWorkflow = exports.workflowRowY = exports.workflowColumnX = exports.WORKFLOW_ROW_GAP = exports.WORKFLOW_NODE_WIDTH = exports.WORKFLOW_NODE_HEIGHT = void 0;
|
|
16
16
|
exports.isDeleteReconciliationStrategy = isDeleteReconciliationStrategy;
|
|
17
17
|
exports.pipelineExpectationRoutes = pipelineExpectationRoutes;
|
|
18
18
|
// A value, not a type: a screen saying how far back the history goes should read
|
|
@@ -144,6 +144,17 @@ Object.defineProperty(exports, "CONNECTOR_KINDS", { enumerable: true, get: funct
|
|
|
144
144
|
Object.defineProperty(exports, "isConnectorKind", { enumerable: true, get: function () { return catalog_pipeline_1.isConnectorKind; } });
|
|
145
145
|
Object.defineProperty(exports, "isSourceFormat", { enumerable: true, get: function () { return catalog_pipeline_1.isSourceFormat; } });
|
|
146
146
|
Object.defineProperty(exports, "isTransformLanguage", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformLanguage; } });
|
|
147
|
+
// The editor offers exactly the modes the runner can call, and applies the
|
|
148
|
+
// default through the same function the store and both runners do. A screen
|
|
149
|
+
// carrying its own `?? 'batch'` is the second copy that decides a stored
|
|
150
|
+
// transform means something else.
|
|
151
|
+
Object.defineProperty(exports, "isTransformMode", { enumerable: true, get: function () { return catalog_pipeline_1.isTransformMode; } });
|
|
152
|
+
Object.defineProperty(exports, "TRANSFORM_MODES", { enumerable: true, get: function () { return catalog_pipeline_1.TRANSFORM_MODES; } });
|
|
153
|
+
Object.defineProperty(exports, "transformMode", { enumerable: true, get: function () { return catalog_pipeline_1.transformMode; } });
|
|
154
|
+
// Whether the two impossible combinations are impossible, asked while the
|
|
155
|
+
// author is still looking at the code rather than at three in the morning when
|
|
156
|
+
// a schedule fires.
|
|
157
|
+
Object.defineProperty(exports, "recordModeRefusal", { enumerable: true, get: function () { return catalog_pipeline_1.recordModeRefusal; } });
|
|
147
158
|
Object.defineProperty(exports, "SOURCE_FORMATS", { enumerable: true, get: function () { return catalog_pipeline_1.SOURCE_FORMATS; } });
|
|
148
159
|
// The canvas narrows nodes and edges it reads back from HTTP. Without these
|
|
149
160
|
// it either imports them from the package root — dragging NestJS and MikroORM
|
package/dist/index.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export { type CatalogOverlayStore, FileCatalogOverlayStore, InMemoryCatalogOverl
|
|
|
8
8
|
export { CATALOG_OVERLAY_STORE } from './catalog.overlay-store.token';
|
|
9
9
|
export { MikroOrmCatalogRegistry } from './catalog.registry';
|
|
10
10
|
export { CatalogRegistry } from './catalog.registry.base';
|
|
11
|
-
export { CATALOG_PIPELINE_STORE, CODE_CONTEXT_CONTRACT, type CatalogCodeContext, type CatalogConnection, type CatalogConnector, type ConnectionCheck, CONNECTOR_KINDS, type CatalogLoadExpectations, type CatalogLoadExpectationStore, type CatalogPipelineStore, type CatalogStageStore, type CatalogTransform, type CatalogTransformFunction, type CatalogTransformInput, type CatalogWorkflow, type CatalogWorkflowCapabilities, type CatalogWorkflowRelease, type CatalogWorkflowReleaseStore, type CatalogWorkflowStore, type CallableWorkflowBlock, type CallableWorkflowDisagreement, type CallableWorkflowRef, callableWorkflowBlock, type ConnectorKind, type ConnectorRun, type DeleteReconciliation, isConnectorKind, isPipelineStore, isSourceFormat, isTransformLanguage, type LoadExpectation, type RowCountBound, SOURCE_FORMATS, type SourceFormat, unreachableSourceFormat, type StoredLoadExpectation, readWorkflowCallOutput, REDACTED_SECRET, supportsLoadExpectations, supportsReusableNodes, supportsTransformPins, supportsTransformRevisions, applyReusableNode, type CatalogReusableNode, type CatalogReusableNodeStore, type CatalogReusableNodeUse, describeLiveVersion, describeVersionPin, isReusableNodeBody, isReusableNodeKind, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, REUSABLE_NODE_KINDS, type ReusableNodeBody, type ReusableNodeKind, type ReusableNodeRef, type ReusableSinkBody, type ReusableSourceBody, reusableNodeBodyOf, unreachableReusableNodeKind, type VersionPinCopy, isWorkflowBranchLabel, isWorkflowEdge, isWorkflowExecutionMode, isWorkflowFilterOperator, isWorkflowFilterPredicate, isWorkflowFilterPredicateKind, isWorkflowFilterValue, isWorkflowIfPredicate, isWorkflowNode, isWorkflowCallMode, isWorkflowNodeKind, isWorkflowPredicateKind, isWorkflowRenameColumns, isWorkflowRenameUnnamed, isWorkflowSkipReason, isWorkflowStatus, liveWorkflowVersion, supportsWorkflowReleases, supportsWorkflows, supportsWorkflowStages, supportsStagePayloads, TRANSFORM_RUNNER, TRANSFORM_LANGUAGES, type TransformLanguage, type TransformResult, type TransformRunner, unreachableFilterOperator, unreachableFilterPredicateKind, unreachableCallMode, unreachableNodeKind, unreachablePredicateKind, unreachableRenameUnnamed, validateWorkflow, WORKFLOW_BRANCH_LABELS, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, WORKFLOW_COLUMN_GAP, WORKFLOW_EXECUTION_MODES, WORKFLOW_FILTER_COLUMN_PATTERN, WORKFLOW_FILTER_MAX_DEPTH, WORKFLOW_FILTER_MAX_VALUES, WORKFLOW_FILTER_OPERATORS, WORKFLOW_FILTER_PREDICATE_KINDS, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_HEIGHT, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_NODE_WIDTH, WORKFLOW_PREDICATE_KINDS, WORKFLOW_RENAME_MAX_COLUMNS, WORKFLOW_RENAME_UNNAMED, WORKFLOW_ROW_GAP, WORKFLOW_SKIP_REASONS, WORKFLOW_STATUSES, type WorkflowBranchLabel, workflowColumnX, workflowRowY, type WorkflowCallEnvelope, type WorkflowCallMode, workflowCallMode, type WorkflowCallNode, type WorkflowCallOutput, type WorkflowEdge, type WorkflowExecutionMode, type WorkflowFilterAll, type WorkflowFilterAny, type WorkflowFilterComparison, type WorkflowFilterGroup, workflowFilterColumns, workflowFilterMatches, type WorkflowFilterNode, type WorkflowFilterOneOf, type WorkflowFilterOperator, type WorkflowFilterPredicate, type WorkflowFilterPredicateKind, type WorkflowFilterPresence, type WorkflowFilterValue, type WorkflowGraph, workflowGraphHash, type WorkflowEnvPredicate, type WorkflowIfNode, type WorkflowIfPredicate, type WorkflowIssueCode, type WorkflowNode, type WorkflowNodeKind, workflowKnownColumns, workflowNarrowedTypes, type WorkflowNodeOutcome, workflowNodeRuns, type WorkflowNodeStepInput, type WorkflowNodeStepOutput, workflowRunOrder, type WorkflowRunOrderEntry, type WorkflowPredicateKind, renameColumnRefusals, type WorkflowRenameNode, type WorkflowRenameUnnamed, workflowRenameUnnamed, type WorkflowRowCountPredicate, type WorkflowSinkNode, type WorkflowSkipReason, type WorkflowSourceNode, type WorkflowStageRef, type WorkflowStatus, type WorkflowTransformNode, type WorkflowValidationIssue, } from './catalog.pipeline';
|
|
11
|
+
export { CATALOG_PIPELINE_STORE, CODE_CONTEXT_CONTRACT, type CatalogCodeContext, type CatalogConnection, type CatalogConnector, type ConnectionCheck, CONNECTOR_KINDS, type CatalogLoadExpectations, type CatalogLoadExpectationStore, type CatalogPipelineStore, type CatalogStageStore, type CatalogRecordTransformFunction, type CatalogRecordTransformInput, type CatalogTransform, type CatalogTransformFunction, type CatalogTransformInput, type CatalogWorkflow, type CatalogWorkflowCapabilities, type CatalogWorkflowRelease, type CatalogWorkflowReleaseStore, type CatalogWorkflowStore, type CallableWorkflowBlock, type CallableWorkflowDisagreement, type CallableWorkflowRef, callableWorkflowBlock, type ConnectorKind, type ConnectorRun, type DeleteReconciliation, isConnectorKind, isPipelineStore, isSourceFormat, isTransformLanguage, isTransformMode, type LoadExpectation, type RowCountBound, SOURCE_FORMATS, type SourceFormat, unreachableSourceFormat, type StoredLoadExpectation, readWorkflowCallOutput, REDACTED_SECRET, supportsLoadExpectations, supportsReusableNodes, supportsTransformPins, supportsTransformRevisions, supportsTransformStreaming, applyReusableNode, type CatalogReusableNode, type CatalogReusableNodeStore, type CatalogReusableNodeUse, describeLiveVersion, describeVersionPin, isReusableNodeBody, isReusableNodeKind, NODE_KIND_IS_REUSABLE, nodeKindIsReusable, REUSABLE_NODE_KINDS, type ReusableNodeBody, type ReusableNodeKind, type ReusableNodeRef, type ReusableSinkBody, type ReusableSourceBody, reusableNodeBodyOf, unreachableReusableNodeKind, type VersionPinCopy, isWorkflowBranchLabel, isWorkflowEdge, isWorkflowExecutionMode, isWorkflowFilterOperator, isWorkflowFilterPredicate, isWorkflowFilterPredicateKind, isWorkflowFilterValue, isWorkflowIfPredicate, isWorkflowNode, isWorkflowCallMode, isWorkflowNodeKind, isWorkflowPredicateKind, isWorkflowRenameColumns, isWorkflowRenameUnnamed, isWorkflowSkipReason, isWorkflowStatus, liveWorkflowVersion, supportsWorkflowReleases, supportsWorkflows, supportsWorkflowStages, supportsStagePayloads, TRANSFORM_RUNNER, TRANSFORM_LANGUAGES, TRANSFORM_MODES, recordModeRefusal, transformMode, type TransformLanguage, type TransformMode, type TransformResult, type TransformRunner, type TransformStream, type TransformStreamSummary, unreachableFilterOperator, unreachableTransformMode, unreachableFilterPredicateKind, unreachableCallMode, unreachableNodeKind, unreachablePredicateKind, unreachableRenameUnnamed, validateWorkflow, WORKFLOW_BRANCH_LABELS, WORKFLOW_CALL_CONTRACT, WORKFLOW_CALL_MODES, WORKFLOW_COLUMN_GAP, WORKFLOW_EXECUTION_MODES, WORKFLOW_FILTER_COLUMN_PATTERN, WORKFLOW_FILTER_MAX_DEPTH, WORKFLOW_FILTER_MAX_VALUES, WORKFLOW_FILTER_OPERATORS, WORKFLOW_FILTER_PREDICATE_KINDS, WORKFLOW_ISSUE_CODES, WORKFLOW_NODE_HEIGHT, WORKFLOW_NODE_ID_PATTERN, WORKFLOW_NODE_KINDS, WORKFLOW_NODE_WIDTH, WORKFLOW_PREDICATE_KINDS, WORKFLOW_RENAME_MAX_COLUMNS, WORKFLOW_RENAME_UNNAMED, WORKFLOW_ROW_GAP, WORKFLOW_SKIP_REASONS, WORKFLOW_STATUSES, type WorkflowBranchLabel, workflowColumnX, workflowRowY, type WorkflowCallEnvelope, type WorkflowCallMode, workflowCallMode, type WorkflowCallNode, type WorkflowCallOutput, type WorkflowEdge, type WorkflowExecutionMode, type WorkflowFilterAll, type WorkflowFilterAny, type WorkflowFilterComparison, type WorkflowFilterGroup, workflowFilterColumns, workflowFilterMatches, type WorkflowFilterNode, type WorkflowFilterOneOf, type WorkflowFilterOperator, type WorkflowFilterPredicate, type WorkflowFilterPredicateKind, type WorkflowFilterPresence, type WorkflowFilterValue, type WorkflowGraph, workflowGraphHash, type WorkflowEnvPredicate, type WorkflowIfNode, type WorkflowIfPredicate, type WorkflowIssueCode, type WorkflowNode, type WorkflowNodeKind, workflowKnownColumns, workflowNarrowedTypes, type WorkflowNodeOutcome, workflowNodeRuns, type WorkflowNodeStepInput, type WorkflowNodeStepOutput, workflowRunOrder, type WorkflowRunOrderEntry, type WorkflowPredicateKind, renameColumnRefusals, type WorkflowRenameNode, type WorkflowRenameUnnamed, workflowRenameUnnamed, type WorkflowRowCountPredicate, type WorkflowSinkNode, type WorkflowSkipReason, type WorkflowSourceNode, type WorkflowStageRef, type WorkflowStatus, type WorkflowTransformNode, type WorkflowValidationIssue, } from './catalog.pipeline';
|
|
12
12
|
export { type ColumnarStageBatch, STAGE_ENCODING, STAGE_ENCODING_VERSION, type StagePayload, classifyStagePayload, decodeStageRows, encodeStageRows, isColumnarStageBatch, renameStagePayload, type StageRenamePlan, type StageRenameResult, } from './catalog.stage-encoding';
|
|
13
13
|
export * from './catalog.environment';
|
|
14
14
|
export { QueryCache } from './catalog.query-cache';
|