@dudousxd/nestjs-catalog-react 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -13
- package/dist/ConnectionPanel.d.ts +11 -7
- package/dist/ConnectionPanel.js +65 -28
- package/dist/PipelineConsole.d.ts +23 -126
- package/dist/PipelineConsole.js +39 -551
- package/dist/WorkflowCanvas.js +401 -29
- package/dist/context.d.ts +161 -37
- package/dist/context.js +35 -11
- package/dist/index.d.ts +6 -3
- package/dist/index.js +40 -14
- package/dist/routes.d.ts +54 -13
- package/dist/routes.js +5 -4
- package/dist/schema-discovery.d.ts +216 -0
- package/dist/schema-discovery.js +225 -0
- package/dist/source-fields.d.ts +1 -12
- package/dist/source-fields.js +13 -16
- package/dist/ui/code-editor.d.ts +44 -2
- package/dist/ui/code-editor.js +10 -2
- package/dist/ui/dialog.d.ts +13 -1
- package/dist/ui/dialog.js +13 -2
- package/dist/workflow/graph.js +31 -9
- package/dist/workflow/index.d.ts +2 -0
- package/dist/workflow/index.js +7 -0
- package/dist/workflow/lifecycle.d.ts +164 -0
- package/dist/workflow/lifecycle.js +249 -0
- package/dist/workflow/model.d.ts +1 -1
- package/dist/workflow/nodes.js +10 -1
- package/dist/workflow/runs.d.ts +38 -0
- package/dist/workflow/runs.js +101 -0
- package/dist/workflow/shape.d.ts +169 -0
- package/dist/workflow/shape.js +249 -0
- package/dist/workflow/templates.d.ts +623 -0
- package/dist/workflow/templates.js +1092 -0
- package/dist/workflow/validate.d.ts +20 -1
- package/dist/workflow/validate.js +41 -3
- package/package.json +2 -2
|
@@ -0,0 +1,623 @@
|
|
|
1
|
+
import { type ConnectorKind, type LoadExpectationInput, type TransformLanguage, type WorkflowEdge, type WorkflowNode } from '@dudousxd/nestjs-catalog/client';
|
|
2
|
+
/**
|
|
3
|
+
* The half-dozen graphs people actually draw, with the decisions already made.
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS FILE EXISTS, WHICH IS NOT CONVENIENCE
|
|
6
|
+
* ---------------------------------------------
|
|
7
|
+
* Thirteen types were loaded into a dev catalog in one evening by hand-building
|
|
8
|
+
* one pipeline per type. Six came out with **every renamed column 100% null** —
|
|
9
|
+
* `Subwo` has 313,833 rows and 73 of its 84 columns empty. Nothing caught it:
|
|
10
|
+
* the loads committed, the row counts were right, the runs were green. The same
|
|
11
|
+
* wrong decision about property naming was simply made six times, because it was
|
|
12
|
+
* being re-derived per type by somebody trying to get data in.
|
|
13
|
+
*
|
|
14
|
+
* A template is therefore not sugar. It is the place a decision that is easy to
|
|
15
|
+
* get wrong is made **once, by somebody who thought about it**, and written down
|
|
16
|
+
* where it can be reviewed. Every template here states what it assumes and what
|
|
17
|
+
* it declares on the operator's behalf, and both travel with the plan it
|
|
18
|
+
* produces so a screen can show them rather than bury them.
|
|
19
|
+
*
|
|
20
|
+
* THREE RULES EVERY TEMPLATE HERE OBEYS
|
|
21
|
+
* -------------------------------------
|
|
22
|
+
* **It does not hide the decision.** A plan is plain nodes, edges, transform
|
|
23
|
+
* bodies and expectation payloads. There is no template object left behind in
|
|
24
|
+
* the saved workflow and nothing a screen cannot render as an ordinary graph —
|
|
25
|
+
* after applying one, every choice it made is an editable field. That is the
|
|
26
|
+
* point: a template that produced an opaque graph would be worse than no
|
|
27
|
+
* template, because the next person could not tell what had been decided for
|
|
28
|
+
* them.
|
|
29
|
+
*
|
|
30
|
+
* **It does not claim a mode it cannot justify.** Nothing here offers
|
|
31
|
+
* `incremental`. An incremental load is refused outright unless the type carries
|
|
32
|
+
* a delete-reconciliation declaration, and it needs a watermark column the
|
|
33
|
+
* template cannot know — so offering it would mean either shipping a graph that
|
|
34
|
+
* fails at its first run or inventing a declaration on somebody's behalf.
|
|
35
|
+
* {@link periodicFullReload} is the one template that touches reconciliation at
|
|
36
|
+
* all, and it declares the strategy and the cadence *together*, from one input,
|
|
37
|
+
* because that is exactly the pair nothing currently links.
|
|
38
|
+
*
|
|
39
|
+
* **It does not restate a list.** Source kinds come from {@link CONNECTOR_KINDS}
|
|
40
|
+
* through {@link SOURCE_KINDS}, starter code is keyed by `TransformLanguage`,
|
|
41
|
+
* nodes are built through {@link NODE_FACTORIES} keyed by `WorkflowNodeKind`,
|
|
42
|
+
* and the naming rule is `isSafeIdentifier(physicalColumn(…))` from the catalog
|
|
43
|
+
* itself — the same two calls, in the same order, that the DDL makes and that
|
|
44
|
+
* the publish-time refusal makes. All four are exhaustive mappings over an
|
|
45
|
+
* exported union, so a kind or a language added to the library without a line
|
|
46
|
+
* here is a **compile error**, not a template that quietly stops covering it.
|
|
47
|
+
* This codebase has been bitten repeatedly by hand-maintained lists that go
|
|
48
|
+
* silent.
|
|
49
|
+
*
|
|
50
|
+
* THE NAMING PROBLEM, WHICH IS THE WHOLE REASON FOR THE REFUSALS
|
|
51
|
+
* -------------------------------------------------------------
|
|
52
|
+
* The warehouse matches records to properties **by property name** —
|
|
53
|
+
* `row[property.name]`, in both the MySQL and ClickHouse stores. So on a graph
|
|
54
|
+
* with no transform on the path, where a record arrives keyed by the source's
|
|
55
|
+
* own column spelling, the property has to be named that spelling exactly.
|
|
56
|
+
* `columnName` is display metadata; it is never a lookup key and never becomes
|
|
57
|
+
* SQL, so recording the source's spelling there does not redirect anything.
|
|
58
|
+
*
|
|
59
|
+
* That used to make the whole class of awkward columns unloadable. A property
|
|
60
|
+
* name was written verbatim as the view's output column and as the alias of
|
|
61
|
+
* every read, both through `ident`, which refuses rather than escapes — so a
|
|
62
|
+
* source column called `Asset Id` could be neither kept (publishing refused the
|
|
63
|
+
* name) nor renamed to `Asset_Id` (the record still arrives keyed `Asset Id`,
|
|
64
|
+
* the store asks for `Asset_Id`, gets `undefined`, and writes null into every
|
|
65
|
+
* row of every run while reporting success). The second is the naive fix and is
|
|
66
|
+
* precisely what produced the six null types.
|
|
67
|
+
*
|
|
68
|
+
* **`fix/view-alias-sanitised` has since landed, and it closed the first door.**
|
|
69
|
+
* Both alias sites now go through `outputAlias`, which cleans a name SQL cannot
|
|
70
|
+
* take and leaves one it can exactly as it is, so a property may keep the
|
|
71
|
+
* source's own spelling end to end: `Asset Id` is now a perfectly good property
|
|
72
|
+
* name, lands in a column called `Asset_Id`, and reads back under `Asset Id`.
|
|
73
|
+
* `Asset LIN/TAMCN` likewise. Refusing those would now be refusing a graph the
|
|
74
|
+
* server would happily accept, so the templates no longer do.
|
|
75
|
+
*
|
|
76
|
+
* **What is left is narrower and is still a trap.** The publish-time refusal
|
|
77
|
+
* asks whether the name *cleans* to an identifier, not whether it is one, and a
|
|
78
|
+
* name can still fail that: `2024 Total` cleans to `2024_Total`, and no store
|
|
79
|
+
* will quote a column starting with a digit. For those columns both doors are
|
|
80
|
+
* still shut in exactly the old way — the name is refused at publish, and the
|
|
81
|
+
* rename that gets reached for instead is the one that writes nulls forever. So
|
|
82
|
+
* {@link replicateTable} and {@link loadFileDrop} still **refuse**, on precisely
|
|
83
|
+
* that set, and name every offending column. The refusal did not go away; its
|
|
84
|
+
* subject got smaller and its reason got sharper.
|
|
85
|
+
*/
|
|
86
|
+
/**
|
|
87
|
+
* One decision a template made so that the operator did not have to.
|
|
88
|
+
*
|
|
89
|
+
* Three fields rather than one sentence, because the three are read at different
|
|
90
|
+
* moments. `what` is scanned in a list before applying; `why` is the argument,
|
|
91
|
+
* and is what somebody needs in order to *disagree*; `changeAt` is what they
|
|
92
|
+
* need afterwards, and its existence is the promise that nothing here is
|
|
93
|
+
* permanent. A declaration with no `changeAt` would be a decision the template
|
|
94
|
+
* had hidden, which is the one thing a template may not do.
|
|
95
|
+
*/
|
|
96
|
+
export interface TemplateDeclaration {
|
|
97
|
+
/** The decision, in one line. */
|
|
98
|
+
what: string;
|
|
99
|
+
/** The reasoning, addressed to somebody who may want to overrule it. */
|
|
100
|
+
why: string;
|
|
101
|
+
/** Where on the screen this is changed after the template is applied. */
|
|
102
|
+
changeAt: string;
|
|
103
|
+
}
|
|
104
|
+
export type TemplateRefusalCode =
|
|
105
|
+
/** Columns the source spells in a way that cannot become a property name. */
|
|
106
|
+
'columns-not-identifiers'
|
|
107
|
+
/** Nobody has run discovery, so the question above cannot be answered. */
|
|
108
|
+
| 'columns-unknown'
|
|
109
|
+
/** A required field of the template's own form was left blank. */
|
|
110
|
+
| 'incomplete'
|
|
111
|
+
/** Two branches of one graph aimed at the same object type. */
|
|
112
|
+
| 'duplicate-target-type';
|
|
113
|
+
/**
|
|
114
|
+
* Why a template will not produce a graph.
|
|
115
|
+
*
|
|
116
|
+
* A refusal, not a warning, and never a graph-with-a-note-attached. The failure
|
|
117
|
+
* this whole file exists for is a load that looks successful, so a template that
|
|
118
|
+
* emitted the null-producing graph alongside a caution would be reproducing it:
|
|
119
|
+
* the graph would get applied and the caution would get scrolled past.
|
|
120
|
+
*/
|
|
121
|
+
export interface TemplateRefusal {
|
|
122
|
+
code: TemplateRefusalCode;
|
|
123
|
+
/** A full sentence, addressed to whoever chose the template. */
|
|
124
|
+
message: string;
|
|
125
|
+
/** The column or type names this is about, when it is about named things. */
|
|
126
|
+
subjects: readonly string[];
|
|
127
|
+
}
|
|
128
|
+
/** A transform the plan needs created before its graph can run. */
|
|
129
|
+
export interface TemplateTransformRequest {
|
|
130
|
+
/** The transform node that will point at it once it has a server id. */
|
|
131
|
+
forNodeId: string;
|
|
132
|
+
name: string;
|
|
133
|
+
language: TransformLanguage;
|
|
134
|
+
/** A function body over `records`, ready to run and meant to be edited. */
|
|
135
|
+
code: string;
|
|
136
|
+
/** What this code is for, in the words the editor should show above it. */
|
|
137
|
+
why: string;
|
|
138
|
+
}
|
|
139
|
+
/** A per-type load expectation the template declares. */
|
|
140
|
+
export interface TemplateExpectation {
|
|
141
|
+
typeName: string;
|
|
142
|
+
input: LoadExpectationInput;
|
|
143
|
+
why: string;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* How often the graph must run, as one object holding both halves.
|
|
147
|
+
*
|
|
148
|
+
* The pair is the entire point of {@link periodicFullReload}: a cron expression
|
|
149
|
+
* and a `withinMs` are two statements about the same cadence, and an operator
|
|
150
|
+
* setting one and forgetting the other is how a type ends up declaring a
|
|
151
|
+
* reconciliation interval nothing honours. Both come from one entry of
|
|
152
|
+
* {@link RELOAD_CADENCES}, so they cannot disagree.
|
|
153
|
+
*/
|
|
154
|
+
export interface TemplateSchedule {
|
|
155
|
+
/** Cron-ish, in whatever `CATALOG_SCHEDULE_TZ` names. */
|
|
156
|
+
cron: string;
|
|
157
|
+
/** The same cadence in milliseconds, which is what `withinMs` is derived from. */
|
|
158
|
+
everyMs: number;
|
|
159
|
+
/** For a screen: "every six hours". */
|
|
160
|
+
human: string;
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Everything a template produces, as data a screen applies rather than as a
|
|
164
|
+
* side effect.
|
|
165
|
+
*
|
|
166
|
+
* Nothing in here is written anywhere by this module. A plan is a proposal: the
|
|
167
|
+
* screen shows it, the operator edits it, and only then does anything get
|
|
168
|
+
* POSTed. That ordering is what makes {@link declarations} meaningful — a
|
|
169
|
+
* decision somebody was told about *after* it had been saved is not a decision
|
|
170
|
+
* they were offered.
|
|
171
|
+
*/
|
|
172
|
+
export interface TemplatePlan {
|
|
173
|
+
templateId: WorkflowTemplateId;
|
|
174
|
+
/** A suggested workflow name. Editable like everything else. */
|
|
175
|
+
name: string;
|
|
176
|
+
description: string;
|
|
177
|
+
nodes: WorkflowNode[];
|
|
178
|
+
edges: WorkflowEdge[];
|
|
179
|
+
/**
|
|
180
|
+
* Transforms to create first. The matching transform nodes carry
|
|
181
|
+
* `transformId: ''` until they exist — see {@link attachTransformIds}, and see
|
|
182
|
+
* {@link planIsRunnable} for why an unattached plan is deliberately not valid.
|
|
183
|
+
*/
|
|
184
|
+
transforms: readonly TemplateTransformRequest[];
|
|
185
|
+
expectations: readonly TemplateExpectation[];
|
|
186
|
+
schedule?: TemplateSchedule;
|
|
187
|
+
declarations: readonly TemplateDeclaration[];
|
|
188
|
+
/** What the operator still has to fill in. Placeholders, connections, queries. */
|
|
189
|
+
todo: readonly string[];
|
|
190
|
+
}
|
|
191
|
+
export type TemplateOutcome = {
|
|
192
|
+
ok: true;
|
|
193
|
+
plan: TemplatePlan;
|
|
194
|
+
} | {
|
|
195
|
+
ok: false;
|
|
196
|
+
refusals: readonly TemplateRefusal[];
|
|
197
|
+
};
|
|
198
|
+
/**
|
|
199
|
+
* What each source kind needs configured, and — the load-bearing part — whether
|
|
200
|
+
* a graph reading through it can go straight into a sink.
|
|
201
|
+
*
|
|
202
|
+
* A `Record` over {@link ConnectorKind} rather than a list of the kinds this
|
|
203
|
+
* file happens to care about. Adding a sixth kind to `CONNECTOR_KINDS` without
|
|
204
|
+
* adding an entry here is a type error, which is the only way a template can be
|
|
205
|
+
* trusted to still be talking about the whole set a year from now.
|
|
206
|
+
*
|
|
207
|
+
* `keysRecordsBySourceName` is the field the refusals turn on. Every fetcher
|
|
208
|
+
* that reads a tabular source hands records over keyed by the source's own
|
|
209
|
+
* column spelling, so with no transform on the path the property names have to
|
|
210
|
+
* *be* those spellings — which is answerable, and is what
|
|
211
|
+
* {@link refuseUnusableColumnNames} answers. `inline` is the one kind where the
|
|
212
|
+
* question does not arise: the records were pasted in by whoever is authoring
|
|
213
|
+
* the graph, so their keys are already whatever that person chose.
|
|
214
|
+
*/
|
|
215
|
+
interface SourceKindProfile {
|
|
216
|
+
/** For a screen, and for the sentences below. */
|
|
217
|
+
label: string;
|
|
218
|
+
/** Config keys the fetcher requires. Shipped as placeholders and as `todo`. */
|
|
219
|
+
required: readonly string[];
|
|
220
|
+
/** Config keys worth offering, but which have working defaults. */
|
|
221
|
+
optional: readonly string[];
|
|
222
|
+
/** Whether records arrive keyed by the source's own column names. */
|
|
223
|
+
keysRecordsBySourceName: boolean;
|
|
224
|
+
}
|
|
225
|
+
export declare const SOURCE_KINDS: Record<ConnectorKind, SourceKindProfile>;
|
|
226
|
+
/**
|
|
227
|
+
* The kinds a file drop can arrive as, derived rather than listed.
|
|
228
|
+
*
|
|
229
|
+
* `CONNECTOR_KINDS` filtered by what {@link SOURCE_KINDS} says about it, so a
|
|
230
|
+
* new file-ish kind is picked up by having its profile filled in rather than by
|
|
231
|
+
* somebody remembering this line exists.
|
|
232
|
+
*/
|
|
233
|
+
export declare const FILE_DROP_KINDS: ("file" | "s3")[];
|
|
234
|
+
/**
|
|
235
|
+
* Refuse the column names that cannot become property names, or say nothing.
|
|
236
|
+
*
|
|
237
|
+
* The single most important function in this file, and the one place the naming
|
|
238
|
+
* rule is applied. See the header for the full argument; the short version is
|
|
239
|
+
* that with no transform on the path a record arrives keyed by the source's own
|
|
240
|
+
* spelling and the store asks for `row[property.name]`, so the two have to be
|
|
241
|
+
* the same string — and while that string no longer has to be a SQL identifier
|
|
242
|
+
* itself, it still has to *clean* to one.
|
|
243
|
+
*
|
|
244
|
+
* Three outcomes rather than two, and the third is not a formality:
|
|
245
|
+
*
|
|
246
|
+
* - Every column is usable — nothing is returned. This is now the answer for
|
|
247
|
+
* `Asset Id` and `Asset LIN/TAMCN`, which it was not before
|
|
248
|
+
* `fix/view-alias-sanitised` landed.
|
|
249
|
+
* - Some are not — a refusal naming every one of them. All of them, not the
|
|
250
|
+
* first, because a legacy export is usually wrong about forty columns in the
|
|
251
|
+
* same way and one refusal per attempt would make fixing it a morning.
|
|
252
|
+
* - **Nobody knows what the columns are** — also a refusal, with its own code,
|
|
253
|
+
* and it survived the alias fix on purpose. The set it protects against got
|
|
254
|
+
* much smaller and did not empty: a source with a `2024 Total` column, applied
|
|
255
|
+
* without anybody looking, still ends with that property refused at publish,
|
|
256
|
+
* renamed to `2024_Total` by the obvious next move, and loading null into
|
|
257
|
+
* every row of every run while reporting success. That is the original
|
|
258
|
+
* disaster, still reachable, one column narrower. A template that proceeded on
|
|
259
|
+
* an unknown column list would be asserting the names are fine because nobody
|
|
260
|
+
* looked. Silence is not a pass.
|
|
261
|
+
*/
|
|
262
|
+
export declare function refuseUnusableColumnNames(columns: readonly string[] | undefined, context: {
|
|
263
|
+
what: string;
|
|
264
|
+
remedy: string;
|
|
265
|
+
}): TemplateRefusal | undefined;
|
|
266
|
+
/**
|
|
267
|
+
* The cadences a periodic full reload may run at, each carrying both halves.
|
|
268
|
+
*
|
|
269
|
+
* **This closed list is the template.** A cron expression and a `withinMs` are
|
|
270
|
+
* two statements about one cadence, and today nothing links them: an operator
|
|
271
|
+
* sets a schedule on one screen and a reconciliation interval on another, and
|
|
272
|
+
* nothing anywhere notices when they stop agreeing. `withinMs` is only honest if
|
|
273
|
+
* the schedule really does reload that often — once the newest full load of a
|
|
274
|
+
* type is older than it, incremental loads of that type stop committing — so a
|
|
275
|
+
* `withinMs` larger than the real cadence is a guard that never fires and a
|
|
276
|
+
* smaller one is a guard that fires constantly.
|
|
277
|
+
*
|
|
278
|
+
* Deriving both from one entry makes disagreement impossible. Somebody wanting a
|
|
279
|
+
* cadence that is not here has to add it here, in code, where both halves are on
|
|
280
|
+
* the same line and a reviewer sees them together — which is the whole doctrine
|
|
281
|
+
* of this file applied to itself.
|
|
282
|
+
*
|
|
283
|
+
* The cron is evaluated in whatever `CATALOG_SCHEDULE_TZ` names and `everyMs` is
|
|
284
|
+
* wall-clock, so the two drift by an hour twice a year across a DST boundary.
|
|
285
|
+
* The slack in {@link withinMsFor} absorbs that; it is called out here because a
|
|
286
|
+
* reader comparing `86_400_000` against `0 2 * * *` deserves to know it is not
|
|
287
|
+
* an exact identity.
|
|
288
|
+
*/
|
|
289
|
+
export declare const RELOAD_CADENCES: {
|
|
290
|
+
readonly hourly: {
|
|
291
|
+
readonly cron: "0 * * * *";
|
|
292
|
+
readonly everyMs: 3600000;
|
|
293
|
+
readonly human: "every hour";
|
|
294
|
+
};
|
|
295
|
+
readonly 'six-hourly': {
|
|
296
|
+
readonly cron: "0 */6 * * *";
|
|
297
|
+
readonly everyMs: 21600000;
|
|
298
|
+
readonly human: "every six hours";
|
|
299
|
+
};
|
|
300
|
+
readonly daily: {
|
|
301
|
+
readonly cron: "0 2 * * *";
|
|
302
|
+
readonly everyMs: 86400000;
|
|
303
|
+
readonly human: "every day at 02:00";
|
|
304
|
+
};
|
|
305
|
+
readonly weekly: {
|
|
306
|
+
readonly cron: "0 2 * * 0";
|
|
307
|
+
readonly everyMs: 604800000;
|
|
308
|
+
readonly human: "every Sunday at 02:00";
|
|
309
|
+
};
|
|
310
|
+
};
|
|
311
|
+
export type ReloadCadence = keyof typeof RELOAD_CADENCES;
|
|
312
|
+
export declare const RELOAD_CADENCE_IDS: readonly ReloadCadence[];
|
|
313
|
+
/**
|
|
314
|
+
* How stale a type may get before its reconciliation claim stops being true.
|
|
315
|
+
*
|
|
316
|
+
* Twice the cadence, and the factor is a decision rather than a round number.
|
|
317
|
+
* At exactly the cadence, any run that starts a minute late — or simply takes a
|
|
318
|
+
* few minutes, since the age is measured from the snapshot's creation — makes
|
|
319
|
+
* the type stale and stops incremental loads of it committing. A guard that
|
|
320
|
+
* fires on an ordinary Tuesday is a guard somebody switches off, taking the real
|
|
321
|
+
* failure with it.
|
|
322
|
+
*
|
|
323
|
+
* Twice means one missed cycle is tolerated and the second consecutive miss is
|
|
324
|
+
* refused, which is the smallest slack that distinguishes "late" from "not
|
|
325
|
+
* happening". It also covers the DST hour noted on {@link RELOAD_CADENCES}.
|
|
326
|
+
*/
|
|
327
|
+
export declare function withinMsFor(cadence: ReloadCadence): number;
|
|
328
|
+
export type WorkflowTemplateId = 'replicate-table' | 'load-file-drop' | 'fan-out-types' | 'join-sources' | 'enrich-with-lookup' | 'periodic-full-reload';
|
|
329
|
+
/**
|
|
330
|
+
* What a picker shows before anything is filled in.
|
|
331
|
+
*
|
|
332
|
+
* `assumes` is separate from the declarations on a plan, and the split matters.
|
|
333
|
+
* A declaration is something the template *did* and can be undone; an assumption
|
|
334
|
+
* is something it takes as given and cannot check, so somebody for whom it is
|
|
335
|
+
* false should pick a different template rather than apply this one and edit it.
|
|
336
|
+
*/
|
|
337
|
+
export interface WorkflowTemplateMeta {
|
|
338
|
+
id: WorkflowTemplateId;
|
|
339
|
+
title: string;
|
|
340
|
+
summary: string;
|
|
341
|
+
assumes: readonly string[];
|
|
342
|
+
}
|
|
343
|
+
/**
|
|
344
|
+
* The catalogue, keyed by id so that a new id without metadata fails the build.
|
|
345
|
+
*
|
|
346
|
+
* Shipped by the library rather than stored per deployment, deliberately: these
|
|
347
|
+
* are decisions, decisions belong in code where they are reviewed and carry
|
|
348
|
+
* their reasoning, and a template stored in a database is one nobody reviews.
|
|
349
|
+
* Per-deployment templates are a store concern and a separate change, and this
|
|
350
|
+
* sentence is here so that the absence reads as a decision rather than an
|
|
351
|
+
* oversight.
|
|
352
|
+
*/
|
|
353
|
+
export declare const WORKFLOW_TEMPLATES: Record<WorkflowTemplateId, WorkflowTemplateMeta>;
|
|
354
|
+
export declare const WORKFLOW_TEMPLATE_IDS: readonly WorkflowTemplateId[];
|
|
355
|
+
export interface ReplicateTableOptions {
|
|
356
|
+
/** The object type the sink commits. */
|
|
357
|
+
targetType: string;
|
|
358
|
+
/**
|
|
359
|
+
* The columns the query returns, as the source spells them.
|
|
360
|
+
*
|
|
361
|
+
* `undefined` means nobody has run discovery, which is a refusal rather than a
|
|
362
|
+
* pass — see {@link refuseUnusableColumnNames}.
|
|
363
|
+
*/
|
|
364
|
+
sourceColumns?: readonly string[];
|
|
365
|
+
/** An existing connection to read through, when there is one. */
|
|
366
|
+
connectionId?: string;
|
|
367
|
+
/** A starting query. Shipped blank when absent, never guessed. */
|
|
368
|
+
query?: string;
|
|
369
|
+
workflowName?: string;
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* SQL straight into a type — the named case, and the one that produced the
|
|
373
|
+
* nulls.
|
|
374
|
+
*
|
|
375
|
+
* The shape is trivial and the value is entirely in the refusal. Two nodes and
|
|
376
|
+
* one edge is not something anybody needs help drawing; what they need help with
|
|
377
|
+
* is knowing that this shape is only correct when every column the query returns
|
|
378
|
+
* can be a property name, and that when it is not, the graph still saves, still
|
|
379
|
+
* runs, still commits and still reports the right row count while writing null
|
|
380
|
+
* into most of the columns.
|
|
381
|
+
*
|
|
382
|
+
* So this template checks, and refuses, and names every offending column. It
|
|
383
|
+
* does **not** offer to sanitise them: that is the naive fix and it is what was
|
|
384
|
+
* done six times in one evening. For the columns that now reach this refusal
|
|
385
|
+
* there is no spelling that is both the key the record arrives under and a name
|
|
386
|
+
* a store can create, so the rename has to be paired with a transform that
|
|
387
|
+
* renames the record's keys — which is a decision about the data rather than a
|
|
388
|
+
* shape, and belongs to whoever knows the data.
|
|
389
|
+
*
|
|
390
|
+
* **What it no longer refuses.** `Asset Id` and `Asset LIN/TAMCN` pass now.
|
|
391
|
+
* `fix/view-alias-sanitised` made the view's output alias and the read's lookup
|
|
392
|
+
* key clean together, so a property keeps the source's spelling end to end and a
|
|
393
|
+
* two-node graph over those columns is simply correct. Refusing them would be
|
|
394
|
+
* this template sending somebody off to do the exact rename that caused the
|
|
395
|
+
* incident it exists to prevent. See the header.
|
|
396
|
+
*/
|
|
397
|
+
export declare function replicateTable(options: ReplicateTableOptions): TemplateOutcome;
|
|
398
|
+
export interface FileDropOptions {
|
|
399
|
+
targetType: string;
|
|
400
|
+
/** `file` for a path or URL, `s3` for a bucket. Both parse the same formats. */
|
|
401
|
+
kind: (typeof FILE_DROP_KINDS)[number];
|
|
402
|
+
/** The header row, as the drop spells it. `undefined` refuses — see below. */
|
|
403
|
+
sourceColumns?: readonly string[];
|
|
404
|
+
connectionId?: string;
|
|
405
|
+
workflowName?: string;
|
|
406
|
+
}
|
|
407
|
+
/**
|
|
408
|
+
* A DoD/DPAS-style drop into a type.
|
|
409
|
+
*
|
|
410
|
+
* Structurally the same two nodes as {@link replicateTable} and separated from
|
|
411
|
+
* it because the config, the assumptions and the failure mode differ enough that
|
|
412
|
+
* one template covering both would have to hedge every sentence. A drop is a
|
|
413
|
+
* full extract that lands on a schedule somebody else controls, its columns come
|
|
414
|
+
* from a spreadsheet header rather than a `SELECT` list, and a spreadsheet header
|
|
415
|
+
* is overwhelmingly the most likely place to meet a column headed `2024 Total`
|
|
416
|
+
* or `2024 Q1 Hours` — which is why the same refusal matters more here, not less.
|
|
417
|
+
*
|
|
418
|
+
* `Asset LIN/TAMCN`, the header this template was written around, is no longer
|
|
419
|
+
* one of them: it cleans to `Asset_LIN_TAMCN` and is published under its own
|
|
420
|
+
* spelling. Nor is `FY24 Hours`, which cleans to `FY24_Hours` — the year has to
|
|
421
|
+
* be at the *front* to be a problem. What is refused is a header whose cleaned
|
|
422
|
+
* form starts with a digit, and that really does need a transform.
|
|
423
|
+
*/
|
|
424
|
+
export declare function loadFileDrop(options: FileDropOptions): TemplateOutcome;
|
|
425
|
+
export interface FanOutTarget {
|
|
426
|
+
targetType: string;
|
|
427
|
+
/** The property names this type commits. Also what the projection keeps. */
|
|
428
|
+
properties: readonly string[];
|
|
429
|
+
}
|
|
430
|
+
export interface FanOutOptions {
|
|
431
|
+
sourceKind: ConnectorKind;
|
|
432
|
+
targets: readonly FanOutTarget[];
|
|
433
|
+
language: TransformLanguage;
|
|
434
|
+
connectionId?: string;
|
|
435
|
+
workflowName?: string;
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* One expensive read feeding several types.
|
|
439
|
+
*
|
|
440
|
+
* The reason the model allows several sinks at all: pulling ten million rows
|
|
441
|
+
* once and deriving vehicles and work orders from them, rather than pulling
|
|
442
|
+
* twice. The validator endorses it explicitly and guards the one genuinely
|
|
443
|
+
* ambiguous case — two sinks writing the *same* type, which would be two
|
|
444
|
+
* snapshots of one type in one run with nothing to say which readers should get.
|
|
445
|
+
* This template refuses that up front rather than letting the canvas report it,
|
|
446
|
+
* because at the point of choosing it is a typo and after applying it is a graph
|
|
447
|
+
* to unpick.
|
|
448
|
+
*
|
|
449
|
+
* **A transform per branch, not one shared transform.** A node with two outbound
|
|
450
|
+
* edges is read by both successors, so a single transform feeding two sinks
|
|
451
|
+
* would commit identical wide rows into both types. The narrowing has to happen
|
|
452
|
+
* per branch, which is what makes this a fan-*out* rather than a duplication.
|
|
453
|
+
*/
|
|
454
|
+
export declare function fanOutTypes(options: FanOutOptions): TemplateOutcome;
|
|
455
|
+
export interface JoinOptions {
|
|
456
|
+
targetType: string;
|
|
457
|
+
language: TransformLanguage;
|
|
458
|
+
/** A field only the left-hand side carries. How the batch is split. */
|
|
459
|
+
discriminator: string;
|
|
460
|
+
/** The field on each side the join matches on. */
|
|
461
|
+
leftKey: string;
|
|
462
|
+
rightKey: string;
|
|
463
|
+
leftSourceKind: ConnectorKind;
|
|
464
|
+
rightSourceKind: ConnectorKind;
|
|
465
|
+
leftName?: string;
|
|
466
|
+
rightName?: string;
|
|
467
|
+
workflowName?: string;
|
|
468
|
+
}
|
|
469
|
+
export type EnrichOptions = JoinOptions;
|
|
470
|
+
/**
|
|
471
|
+
* Two reads into one type, joined inside the transform.
|
|
472
|
+
*
|
|
473
|
+
* Expressible only because the transform contract is set-wise: the code is a
|
|
474
|
+
* function over a whole batch precisely so that it can deduplicate, aggregate
|
|
475
|
+
* and join, none of which can be done a row at a time.
|
|
476
|
+
*
|
|
477
|
+
* **The decision this template exists to make is what the batch looks like.** A
|
|
478
|
+
* node with several inbound edges receives its inputs *concatenated in edge
|
|
479
|
+
* order, in one array, with nothing marking which record came from which side*.
|
|
480
|
+
* The natural assumption is two arrays or two arguments, and code written under
|
|
481
|
+
* that assumption silently processes the wrong half. So the starter body
|
|
482
|
+
* partitions the batch explicitly, with the reason in a comment directly above
|
|
483
|
+
* the loop, and the template asks for the discriminator up front rather than
|
|
484
|
+
* leaving somebody to discover the problem from a result that is merely odd.
|
|
485
|
+
*
|
|
486
|
+
* Edge order is not decoration: it is what decides which side is left, and it is
|
|
487
|
+
* part of the graph fingerprint, so swapping the two inputs is a new version of
|
|
488
|
+
* the graph. The left source is added first, deliberately and stably.
|
|
489
|
+
*/
|
|
490
|
+
export declare function joinSources(options: JoinOptions): TemplateOutcome;
|
|
491
|
+
/**
|
|
492
|
+
* A feed plus a dictionary, where a row the dictionary has not heard of
|
|
493
|
+
* survives.
|
|
494
|
+
*
|
|
495
|
+
* The same graph and the same code as {@link joinSources} with one flag flipped,
|
|
496
|
+
* and it is a separate template because that flag is the decision. Enriching
|
|
497
|
+
* against a lookup and joining two feeds look identical on a canvas and differ
|
|
498
|
+
* entirely in what happens to a record with no counterpart: dropping unmatched
|
|
499
|
+
* rows from an enrichment means a load silently loses every record whose key the
|
|
500
|
+
* dictionary has not caught up with yet — a load that succeeds, reports a
|
|
501
|
+
* plausible row count, and is missing data. That is the same shape of failure as
|
|
502
|
+
* the null columns, and it is the reason the two are offered as two choices
|
|
503
|
+
* rather than one template with a checkbox nobody reads.
|
|
504
|
+
*
|
|
505
|
+
* Concretely useful here: this deployment has a `unit_dictionary` type and a
|
|
506
|
+
* unit-normalisation problem it was built for.
|
|
507
|
+
*/
|
|
508
|
+
export declare function enrichWithLookup(options: EnrichOptions): TemplateOutcome;
|
|
509
|
+
export interface PeriodicFullReloadOptions {
|
|
510
|
+
targetType: string;
|
|
511
|
+
sourceKind: ConnectorKind;
|
|
512
|
+
cadence: ReloadCadence;
|
|
513
|
+
/** Why this cadence reconciles deletes acceptably. Required, and not by accident. */
|
|
514
|
+
because: string;
|
|
515
|
+
sourceColumns?: readonly string[];
|
|
516
|
+
connectionId?: string;
|
|
517
|
+
workflowName?: string;
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* A full read on a schedule, with the reconciliation it implies declared from
|
|
521
|
+
* the same input.
|
|
522
|
+
*
|
|
523
|
+
* The template most worth having, because the two things it produces are the two
|
|
524
|
+
* that must agree and that nothing currently links. A `periodic-full-reload`
|
|
525
|
+
* strategy says deletions reach this type because it is fully reloaded every
|
|
526
|
+
* `withinMs`; a schedule says how often it is actually reloaded. An operator
|
|
527
|
+
* sets one on the Model screen and the other on the connector, and nothing
|
|
528
|
+
* anywhere notices when they drift apart — at which point the type carries a
|
|
529
|
+
* declaration that reads as a guarantee and polices nothing.
|
|
530
|
+
*
|
|
531
|
+
* Here they come from one {@link RELOAD_CADENCES} entry, so they cannot
|
|
532
|
+
* disagree, and `withinMs` gets its slack from {@link withinMsFor} rather than
|
|
533
|
+
* from whatever somebody typed.
|
|
534
|
+
*
|
|
535
|
+
* **`because` is required and is not decoration.** `refuseUndeclaredDeletes`
|
|
536
|
+
* rejects a declaration whose reason is blank, on every strategy, because the
|
|
537
|
+
* control was never "it must be in code" — it is that somebody chose a strategy
|
|
538
|
+
* and wrote down why. A template that supplied a default reason would be
|
|
539
|
+
* defeating exactly the mechanism it is here to satisfy.
|
|
540
|
+
*
|
|
541
|
+
* **Where the schedule goes, and why the plan does not say.** The plan carries
|
|
542
|
+
* the cadence as data — {@link TemplatePlan.schedule} — and leaves *placing* it
|
|
543
|
+
* to whatever applies the plan. That was a deliberate bet, and it paid.
|
|
544
|
+
*
|
|
545
|
+
* When this was written, a workflow was put on a timer by a connector that
|
|
546
|
+
* pointed at it with `workflowId` and carried the `schedule` string. A template
|
|
547
|
+
* that had written that hop into its plan or its `todo` would have had to be
|
|
548
|
+
* rewritten when the workflow became the only authored object: connectors stopped
|
|
549
|
+
* being authored at all, and a schedule now lives on the workflow, set through
|
|
550
|
+
* `PUT workflows/:id/schedule`. Because the plan only ever declared "this runs
|
|
551
|
+
* `0 2 * * *`" and never "create a connector that says so", the move cost this
|
|
552
|
+
* template nothing — the same plan, rebased with no conflict, describes the new
|
|
553
|
+
* arrangement as accurately as it described the old one.
|
|
554
|
+
*
|
|
555
|
+
* What did not change, and is the part worth stating: only a `ready` graph is
|
|
556
|
+
* scheduled. A schedule written against a draft is stored and does not fire.
|
|
557
|
+
*/
|
|
558
|
+
export declare function periodicFullReload(options: PeriodicFullReloadOptions): TemplateOutcome;
|
|
559
|
+
export type TemplateRequest = {
|
|
560
|
+
templateId: 'replicate-table';
|
|
561
|
+
options: ReplicateTableOptions;
|
|
562
|
+
} | {
|
|
563
|
+
templateId: 'load-file-drop';
|
|
564
|
+
options: FileDropOptions;
|
|
565
|
+
} | {
|
|
566
|
+
templateId: 'fan-out-types';
|
|
567
|
+
options: FanOutOptions;
|
|
568
|
+
} | {
|
|
569
|
+
templateId: 'join-sources';
|
|
570
|
+
options: JoinOptions;
|
|
571
|
+
} | {
|
|
572
|
+
templateId: 'enrich-with-lookup';
|
|
573
|
+
options: EnrichOptions;
|
|
574
|
+
} | {
|
|
575
|
+
templateId: 'periodic-full-reload';
|
|
576
|
+
options: PeriodicFullReloadOptions;
|
|
577
|
+
};
|
|
578
|
+
/**
|
|
579
|
+
* Build whichever template was asked for, with the options that template needs.
|
|
580
|
+
*
|
|
581
|
+
* A discriminated request rather than one options bag with everything optional,
|
|
582
|
+
* so asking for a join without a discriminator is a type error at the call site
|
|
583
|
+
* instead of a refusal at run time. The refusals still exist because a form
|
|
584
|
+
* hands over strings a user typed, and an empty string satisfies `string`.
|
|
585
|
+
*/
|
|
586
|
+
export declare function buildWorkflowTemplate(request: TemplateRequest): TemplateOutcome;
|
|
587
|
+
/**
|
|
588
|
+
* Point each transform node at the transform that was created for it.
|
|
589
|
+
*
|
|
590
|
+
* Separate from building the plan, and pure, because the ids do not exist until
|
|
591
|
+
* a server has stored the code — a template cannot invent them and must not
|
|
592
|
+
* pretend to. Until this has run, a plan carrying transforms is deliberately
|
|
593
|
+
* **not** a valid graph: `validateWorkflow` reports `transform-not-named` for
|
|
594
|
+
* every node with an empty `transformId`, which is the correct answer, because a
|
|
595
|
+
* node pointing at no code cannot run. See {@link planIsRunnable}.
|
|
596
|
+
*
|
|
597
|
+
* Nodes whose id is absent from the map are left exactly as they were, so a
|
|
598
|
+
* partial failure part-way through creating three transforms leaves a graph that
|
|
599
|
+
* still says which node is unfinished rather than one that silently claims to be
|
|
600
|
+
* complete.
|
|
601
|
+
*/
|
|
602
|
+
export declare function attachTransformIds(plan: TemplatePlan, transformIdByNodeId: ReadonlyMap<string, string>): TemplatePlan;
|
|
603
|
+
/** The authored half of a plan, in the shape `saveWorkflow` takes. */
|
|
604
|
+
export declare function planToWorkflowInput(plan: TemplatePlan): {
|
|
605
|
+
name: string;
|
|
606
|
+
description: string;
|
|
607
|
+
nodes: WorkflowNode[];
|
|
608
|
+
edges: Array<{
|
|
609
|
+
from: string;
|
|
610
|
+
to: string;
|
|
611
|
+
}>;
|
|
612
|
+
};
|
|
613
|
+
/**
|
|
614
|
+
* Whether every transform node in the plan has been pointed at real code.
|
|
615
|
+
*
|
|
616
|
+
* The one question a screen has to ask between creating the transforms and
|
|
617
|
+
* saving the graph. Named rather than left as an inline `.every`, because "the
|
|
618
|
+
* plan is ready to save" and "the plan validates" are different questions and
|
|
619
|
+
* conflating them is how a save gets attempted with an empty `transformId` and
|
|
620
|
+
* comes back as a validation error about a field nobody typed.
|
|
621
|
+
*/
|
|
622
|
+
export declare function planIsRunnable(plan: TemplatePlan): boolean;
|
|
623
|
+
export {};
|