@dudousxd/nestjs-catalog-react 0.17.0 → 0.19.0-preview.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -13
- package/dist/CatalogManager.d.ts +23 -1
- package/dist/CatalogManager.js +76 -13
- package/dist/ConnectionPanel.d.ts +11 -7
- package/dist/ConnectionPanel.js +78 -149
- package/dist/LoadedBySection.d.ts +39 -0
- package/dist/LoadedBySection.js +143 -0
- package/dist/PipelineConsole.d.ts +23 -126
- package/dist/PipelineConsole.js +39 -551
- package/dist/WorkflowCanvas.d.ts +16 -0
- package/dist/WorkflowCanvas.js +1064 -64
- package/dist/connection-form.d.ts +241 -0
- package/dist/connection-form.js +235 -0
- package/dist/context.d.ts +236 -38
- package/dist/context.js +46 -11
- package/dist/index.d.ts +10 -3
- package/dist/index.js +63 -14
- package/dist/routes.d.ts +84 -13
- package/dist/routes.js +7 -4
- package/dist/schema-discovery.d.ts +232 -0
- package/dist/schema-discovery.js +225 -0
- package/dist/source-connection.d.ts +82 -0
- package/dist/source-connection.js +176 -0
- package/dist/source-fields.d.ts +16 -14
- package/dist/source-fields.js +28 -18
- package/dist/ui/code-editor.d.ts +44 -2
- package/dist/ui/code-editor.js +10 -2
- package/dist/ui/combobox.d.ts +97 -0
- package/dist/ui/combobox.js +80 -0
- package/dist/ui/dialog.d.ts +13 -1
- package/dist/ui/dialog.js +13 -2
- package/dist/workflow/edges.d.ts +110 -0
- package/dist/workflow/edges.js +112 -0
- package/dist/workflow/graph.d.ts +39 -1
- package/dist/workflow/graph.js +212 -52
- package/dist/workflow/index.d.ts +8 -0
- package/dist/workflow/index.js +12 -0
- package/dist/workflow/lifecycle.d.ts +135 -0
- package/dist/workflow/lifecycle.js +216 -0
- package/dist/workflow/model.d.ts +17 -3
- package/dist/workflow/model.js +7 -2
- package/dist/workflow/nodes.d.ts +4 -1
- package/dist/workflow/nodes.js +198 -10
- package/dist/workflow/runs.d.ts +38 -0
- package/dist/workflow/runs.js +101 -0
- package/dist/workflow/shape.d.ts +169 -0
- package/dist/workflow/shape.js +249 -0
- package/dist/workflow/templates.d.ts +623 -0
- package/dist/workflow/templates.js +1102 -0
- package/dist/workflow/validate.d.ts +20 -1
- package/dist/workflow/validate.js +52 -3
- package/dist/workflow/wiring.d.ts +84 -0
- package/dist/workflow/wiring.js +254 -0
- package/package.json +9 -9
|
@@ -0,0 +1,1102 @@
|
|
|
1
|
+
import { CONNECTOR_KINDS, isSafeIdentifier, physicalColumn, } from '@dudousxd/nestjs-catalog/client';
|
|
2
|
+
export const SOURCE_KINDS = {
|
|
3
|
+
http: {
|
|
4
|
+
label: 'an HTTP endpoint',
|
|
5
|
+
required: ['url'],
|
|
6
|
+
optional: ['headers', 'path'],
|
|
7
|
+
keysRecordsBySourceName: true,
|
|
8
|
+
},
|
|
9
|
+
sql: {
|
|
10
|
+
label: 'a SQL database',
|
|
11
|
+
required: ['url', 'query'],
|
|
12
|
+
optional: ['watermarkColumn'],
|
|
13
|
+
keysRecordsBySourceName: true,
|
|
14
|
+
},
|
|
15
|
+
file: {
|
|
16
|
+
label: 'a file',
|
|
17
|
+
required: ['path'],
|
|
18
|
+
optional: ['format', 'jsonPath', 'delimiter'],
|
|
19
|
+
keysRecordsBySourceName: true,
|
|
20
|
+
},
|
|
21
|
+
s3: {
|
|
22
|
+
label: 'an object store',
|
|
23
|
+
required: ['bucket'],
|
|
24
|
+
optional: ['prefix', 'suffix', 'format', 'delimiter', 'region', 'maxObjectsPerRun'],
|
|
25
|
+
keysRecordsBySourceName: true,
|
|
26
|
+
},
|
|
27
|
+
inline: {
|
|
28
|
+
label: 'records pasted into the graph',
|
|
29
|
+
required: ['records'],
|
|
30
|
+
optional: [],
|
|
31
|
+
// The keys are whatever the author typed, so there is no source spelling to
|
|
32
|
+
// disagree with a property name and nothing for the refusal to check.
|
|
33
|
+
keysRecordsBySourceName: false,
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* The kinds a file drop can arrive as, derived rather than listed.
|
|
38
|
+
*
|
|
39
|
+
* `CONNECTOR_KINDS` filtered by what {@link SOURCE_KINDS} says about it, so a
|
|
40
|
+
* new file-ish kind is picked up by having its profile filled in rather than by
|
|
41
|
+
* somebody remembering this line exists.
|
|
42
|
+
*/
|
|
43
|
+
export const FILE_DROP_KINDS = CONNECTOR_KINDS.filter((kind) => kind === 'file' || kind === 's3');
|
|
44
|
+
const NODE_FACTORIES = {
|
|
45
|
+
source: (id, name, init) => ({ id, name, kind: 'source', ...init }),
|
|
46
|
+
transform: (id, name, init) => ({ id, name, kind: 'transform', ...init }),
|
|
47
|
+
sink: (id, name, init) => ({ id, name, kind: 'sink', ...init }),
|
|
48
|
+
// `call` arrived after these templates were written, and the map above is
|
|
49
|
+
// exactly why: the build stopped rather than the kind going quietly missing.
|
|
50
|
+
//
|
|
51
|
+
// No template below builds one, and that is a statement rather than an
|
|
52
|
+
// omission. A call node hands a step to a workflow this graph does not own,
|
|
53
|
+
// pinned by name and version — which is a decision about somebody else's
|
|
54
|
+
// code, and a template exists to make a decision once on the reader's behalf.
|
|
55
|
+
// There is nothing here to decide for them, and no way to enumerate a
|
|
56
|
+
// deployment's registrations to offer a choice from. The factory is complete
|
|
57
|
+
// so that a template CAN build one the day a case is worth prefilling.
|
|
58
|
+
call: (id, name, init) => ({ id, name, kind: 'call', ...init }),
|
|
59
|
+
// `if` arrived the same way `call` did, and the map stopped the build again.
|
|
60
|
+
//
|
|
61
|
+
// Also unused by every template below, for a reason of its own: which
|
|
62
|
+
// environment variable tells one deployment apart from another is the entire
|
|
63
|
+
// content of a gate, and it is knowledge about a *deployment* rather than
|
|
64
|
+
// about a pipeline shape. A template that guessed one would prefill a decision
|
|
65
|
+
// that is wrong everywhere except where it was written — and a branch that is
|
|
66
|
+
// wrong is a half of the graph that silently does not run, which is the one
|
|
67
|
+
// failure this node had to be built not to have.
|
|
68
|
+
if: (id, name, init) => ({ id, name, kind: 'if', ...init }),
|
|
69
|
+
};
|
|
70
|
+
/* -------------------------------------------------------------------------- */
|
|
71
|
+
/* Starter transform code */
|
|
72
|
+
/* -------------------------------------------------------------------------- */
|
|
73
|
+
/**
|
|
74
|
+
* What every starter body has to say out loud, and why it is said in a comment
|
|
75
|
+
* rather than only in a docblock nobody will be looking at.
|
|
76
|
+
*
|
|
77
|
+
* A transform node with several inbound edges receives its inputs **concatenated
|
|
78
|
+
* in edge order, in one array, with nothing marking which record came from
|
|
79
|
+
* which side**. That is the model's documented behaviour and it is what makes a
|
|
80
|
+
* join expressible at all — the code can see every record at once — but it is
|
|
81
|
+
* also the single most surprising thing about writing one. The obvious
|
|
82
|
+
* assumption is that two inputs arrive as two arguments, or as two arrays, and
|
|
83
|
+
* code written under that assumption produces a plausible-looking result from
|
|
84
|
+
* the wrong half of the data.
|
|
85
|
+
*
|
|
86
|
+
* So the starter code partitions the batch explicitly and the comment naming the
|
|
87
|
+
* behaviour sits directly above the loop that depends on it. Somebody editing
|
|
88
|
+
* this a month later reads it in the editor; they do not read this file.
|
|
89
|
+
*/
|
|
90
|
+
const JOIN_NOTE = [
|
|
91
|
+
'Both inbound edges arrive here as ONE array, concatenated in edge order.',
|
|
92
|
+
'Nothing marks which source a record came from, so they are told apart by a',
|
|
93
|
+
'field only one side carries. Change the discriminator if either shape does.',
|
|
94
|
+
];
|
|
95
|
+
/** Render a note as comment lines in the language's own syntax. */
|
|
96
|
+
function comment(lines, language) {
|
|
97
|
+
const prefix = language === 'python' ? '# ' : '// ';
|
|
98
|
+
return lines.map((line) => `${prefix}${line}`).join('\n');
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* A list literal that reads the same in all three languages.
|
|
102
|
+
*
|
|
103
|
+
* `JSON.stringify` on each element rather than a template with quotes around it:
|
|
104
|
+
* a column called `O'Brien` would otherwise close the string and produce a
|
|
105
|
+
* starter body that does not parse, which is a confusing first impression of a
|
|
106
|
+
* template whose entire subject is awkward column names.
|
|
107
|
+
*/
|
|
108
|
+
function quoteList(values) {
|
|
109
|
+
return `[${values.map((value) => JSON.stringify(value)).join(', ')}]`;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* A keyed join over the concatenated batch, per language.
|
|
113
|
+
*
|
|
114
|
+
* Keyed by {@link TransformLanguage} so a language added to the library without
|
|
115
|
+
* a body here is a type error rather than a transform that saves empty and fails
|
|
116
|
+
* at run time — the same argument `IDENTITY_TRANSFORM` in the canvas makes.
|
|
117
|
+
*
|
|
118
|
+
* `keepUnmatched` is the one genuinely contested choice and it is why enrichment
|
|
119
|
+
* and joining are separate templates over identical code. Enriching against a
|
|
120
|
+
* lookup keeps a row whose key is not in the dictionary, because dropping it
|
|
121
|
+
* would mean a load silently loses every record the lookup has not heard of yet
|
|
122
|
+
* — which is the null-shaped failure this file exists for, wearing a different
|
|
123
|
+
* hat. Joining two feeds into one type drops it, because a record with no
|
|
124
|
+
* counterpart is not a record of the joined thing.
|
|
125
|
+
*/
|
|
126
|
+
const JOIN_CODE = {
|
|
127
|
+
javascript: (options) => javascriptJoin(options),
|
|
128
|
+
// Deliberately the same body. The runner strips types and never checks them,
|
|
129
|
+
// so a TypeScript transform is JavaScript with annotations allowed; shipping
|
|
130
|
+
// an annotated copy would be two bodies to fix when the logic is wrong.
|
|
131
|
+
typescript: (options) => javascriptJoin(options),
|
|
132
|
+
python: (options) => pythonJoin(options),
|
|
133
|
+
};
|
|
134
|
+
function javascriptJoin(options) {
|
|
135
|
+
const { discriminator, leftKey, rightKey, keepUnmatched } = options;
|
|
136
|
+
const miss = keepUnmatched
|
|
137
|
+
? ' // No match: the row is kept as it arrived, with nothing added.\n if (!found) return record;'
|
|
138
|
+
: ' // No match: this row is not a record of the joined thing, so it is dropped.\n if (!found) return undefined;';
|
|
139
|
+
return `${comment(JOIN_NOTE, 'javascript')}
|
|
140
|
+
const left = [];
|
|
141
|
+
const right = [];
|
|
142
|
+
for (const record of records) {
|
|
143
|
+
(Object.hasOwn(record, ${JSON.stringify(discriminator)}) ? left : right).push(record);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// A missing key is not a key, so a record without one is never indexed. Left it
|
|
147
|
+
// in and every row that is ALSO missing the key would join to it: a full result
|
|
148
|
+
// set, a plausible row count, entirely wrong rows. Skipping it here covers both
|
|
149
|
+
// sides at once — nothing is stored under a missing key, so nothing can be
|
|
150
|
+
// looked up by one either.
|
|
151
|
+
const byKey = new Map();
|
|
152
|
+
for (const record of right) {
|
|
153
|
+
const key = record[${JSON.stringify(rightKey)}];
|
|
154
|
+
if (key === undefined || key === null) continue;
|
|
155
|
+
byKey.set(key, record);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
return left
|
|
159
|
+
.map((record) => {
|
|
160
|
+
const found = byKey.get(record[${JSON.stringify(leftKey)}]);
|
|
161
|
+
${miss}
|
|
162
|
+
// The left record wins on any field both sides carry: it is the record this
|
|
163
|
+
// row is about, and the lookup is only filling in what it did not know.
|
|
164
|
+
return { ...found, ...record };
|
|
165
|
+
})
|
|
166
|
+
.filter((record) => record !== undefined);`;
|
|
167
|
+
}
|
|
168
|
+
function pythonJoin(options) {
|
|
169
|
+
const { discriminator, leftKey, rightKey, keepUnmatched } = options;
|
|
170
|
+
const miss = keepUnmatched
|
|
171
|
+
? ' # No match: the row is kept as it arrived, with nothing added.\n out.append(record)\n continue'
|
|
172
|
+
: ' # No match: this row is not a record of the joined thing, so it is dropped.\n continue';
|
|
173
|
+
return `${comment(JOIN_NOTE, 'python')}
|
|
174
|
+
left = []
|
|
175
|
+
right = []
|
|
176
|
+
for record in records:
|
|
177
|
+
(left if ${JSON.stringify(discriminator)} in record else right).append(record)
|
|
178
|
+
|
|
179
|
+
# A missing key is not a key, so a record without one is never indexed. Left it
|
|
180
|
+
# in and every row that is ALSO missing the key would join to it: a full result
|
|
181
|
+
# set, a plausible row count, entirely wrong rows. Skipping it here covers both
|
|
182
|
+
# sides at once — nothing is stored under a missing key, so nothing can be
|
|
183
|
+
# looked up by one either.
|
|
184
|
+
by_key = {}
|
|
185
|
+
for record in right:
|
|
186
|
+
key = record.get(${JSON.stringify(rightKey)})
|
|
187
|
+
if key is None:
|
|
188
|
+
continue
|
|
189
|
+
by_key[key] = record
|
|
190
|
+
|
|
191
|
+
out = []
|
|
192
|
+
for record in left:
|
|
193
|
+
found = by_key.get(record.get(${JSON.stringify(leftKey)}))
|
|
194
|
+
if found is None:
|
|
195
|
+
${miss}
|
|
196
|
+
# The left record wins on any field both sides carry: it is the record this
|
|
197
|
+
# row is about, and the lookup is only filling in what it did not know.
|
|
198
|
+
merged = dict(found)
|
|
199
|
+
merged.update(record)
|
|
200
|
+
out.append(merged)
|
|
201
|
+
return out`;
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Narrow one shared batch down to the columns a single type commits.
|
|
205
|
+
*
|
|
206
|
+
* Every branch of a fan-out reads the **same** rows — a node with two outbound
|
|
207
|
+
* edges is read by both successors — so each branch has to do its own narrowing.
|
|
208
|
+
* A fan-out wired straight from one source into several sinks would commit
|
|
209
|
+
* identical wide rows into every type, which is not what anybody drawing that
|
|
210
|
+
* shape means.
|
|
211
|
+
*/
|
|
212
|
+
const PROJECT_CODE = {
|
|
213
|
+
javascript: (typeName, keep) => javascriptProject(typeName, keep),
|
|
214
|
+
typescript: (typeName, keep) => javascriptProject(typeName, keep),
|
|
215
|
+
python: (typeName, keep) => `${comment(projectNote(typeName), 'python')}
|
|
216
|
+
keep = ${quoteList(keep)}
|
|
217
|
+
return [{key: record.get(key) for key in keep} for record in records]`,
|
|
218
|
+
};
|
|
219
|
+
function projectNote(typeName) {
|
|
220
|
+
return [
|
|
221
|
+
`Keep only the columns ${typeName} commits.`,
|
|
222
|
+
'Every branch of this graph reads the SAME source rows, so each one narrows',
|
|
223
|
+
'them itself. These names are property names, and the warehouse looks each',
|
|
224
|
+
'field up by property name — so what this returns has to be keyed by them.',
|
|
225
|
+
];
|
|
226
|
+
}
|
|
227
|
+
function javascriptProject(typeName, keep) {
|
|
228
|
+
return `${comment(projectNote(typeName), 'javascript')}
|
|
229
|
+
const keep = ${quoteList(keep)};
|
|
230
|
+
return records.map((record) =>
|
|
231
|
+
Object.fromEntries(keep.map((key) => [key, record[key]])),
|
|
232
|
+
);`;
|
|
233
|
+
}
|
|
234
|
+
/* -------------------------------------------------------------------------- */
|
|
235
|
+
/* The identifier refusal */
|
|
236
|
+
/* -------------------------------------------------------------------------- */
|
|
237
|
+
/**
|
|
238
|
+
* Whether a column name could be published as a property name at all.
|
|
239
|
+
*
|
|
240
|
+
* `isSafeIdentifier(physicalColumn(name))` — the same two calls, in the same
|
|
241
|
+
* order, that `identifierRefusal` makes at publish time and that every store
|
|
242
|
+
* makes on the way to a column. Both come from `@dudousxd/nestjs-catalog/client`
|
|
243
|
+
* and are the functions themselves, not a copy of either: a canvas whose idea of
|
|
244
|
+
* a legal name differs from the server's by one character is a canvas that
|
|
245
|
+
* promises a load the publisher then refuses, or refuses one it would have
|
|
246
|
+
* taken.
|
|
247
|
+
*
|
|
248
|
+
* **It is the composition and not `isSafeIdentifier` alone**, and getting that
|
|
249
|
+
* wrong in either direction is a real failure rather than a nicety. Asking only
|
|
250
|
+
* `isSafeIdentifier(name)` is the obsolete, stricter question — it refuses
|
|
251
|
+
* `Asset Id`, which now works end to end, and the operator sent away by that
|
|
252
|
+
* refusal is being sent to do the rename that writes nulls. Asking only whether
|
|
253
|
+
* the cleaning *ran* would pass everything, since `physicalColumn` always
|
|
254
|
+
* returns something. What has to be asked is whether what it returns is a name a
|
|
255
|
+
* store can create.
|
|
256
|
+
*
|
|
257
|
+
* **Which is a narrow set, and worth naming exactly so nobody re-derives it.**
|
|
258
|
+
* The cleaning already guarantees the character set and the length, so the only
|
|
259
|
+
* things left for `isSafeIdentifier` to reject are a cleaned form that starts
|
|
260
|
+
* with a digit and one that is empty. In practice: `2024 Total` → `2024_Total`
|
|
261
|
+
* is refused; `FY24 Hours` → `FY24_Hours` is not, because the digits are not
|
|
262
|
+
* first; `#Rank` → `_Rank` is not, because punctuation becomes an underscore and
|
|
263
|
+
* an underscore may lead; `Asset LIN/TAMCN` → `Asset_LIN_TAMCN` is not. A column
|
|
264
|
+
* named with the empty string is the other refusal, and a source producing one
|
|
265
|
+
* has a problem this template is not the right place to explain.
|
|
266
|
+
*/
|
|
267
|
+
function canBePropertyName(column) {
|
|
268
|
+
return isSafeIdentifier(physicalColumn(column));
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Refuse the column names that cannot become property names, or say nothing.
|
|
272
|
+
*
|
|
273
|
+
* The single most important function in this file, and the one place the naming
|
|
274
|
+
* rule is applied. See the header for the full argument; the short version is
|
|
275
|
+
* that with no transform on the path a record arrives keyed by the source's own
|
|
276
|
+
* spelling and the store asks for `row[property.name]`, so the two have to be
|
|
277
|
+
* the same string — and while that string no longer has to be a SQL identifier
|
|
278
|
+
* itself, it still has to *clean* to one.
|
|
279
|
+
*
|
|
280
|
+
* Three outcomes rather than two, and the third is not a formality:
|
|
281
|
+
*
|
|
282
|
+
* - Every column is usable — nothing is returned. This is now the answer for
|
|
283
|
+
* `Asset Id` and `Asset LIN/TAMCN`, which it was not before
|
|
284
|
+
* `fix/view-alias-sanitised` landed.
|
|
285
|
+
* - Some are not — a refusal naming every one of them. All of them, not the
|
|
286
|
+
* first, because a legacy export is usually wrong about forty columns in the
|
|
287
|
+
* same way and one refusal per attempt would make fixing it a morning.
|
|
288
|
+
* - **Nobody knows what the columns are** — also a refusal, with its own code,
|
|
289
|
+
* and it survived the alias fix on purpose. The set it protects against got
|
|
290
|
+
* much smaller and did not empty: a source with a `2024 Total` column, applied
|
|
291
|
+
* without anybody looking, still ends with that property refused at publish,
|
|
292
|
+
* renamed to `2024_Total` by the obvious next move, and loading null into
|
|
293
|
+
* every row of every run while reporting success. That is the original
|
|
294
|
+
* disaster, still reachable, one column narrower. A template that proceeded on
|
|
295
|
+
* an unknown column list would be asserting the names are fine because nobody
|
|
296
|
+
* looked. Silence is not a pass.
|
|
297
|
+
*/
|
|
298
|
+
export function refuseUnusableColumnNames(columns, context) {
|
|
299
|
+
if (columns === undefined) {
|
|
300
|
+
return {
|
|
301
|
+
code: 'columns-unknown',
|
|
302
|
+
subjects: [],
|
|
303
|
+
message: `Nobody has said which columns ${context.what} produces, so this template cannot check the one thing that would make it commit nulls. With no transform on the path a record arrives keyed by the source's own column spelling and the warehouse looks every field up by property name, so the two have to match exactly — and while a property name may now be spelled however the source spells it, it still has to clean to something a store can create as a column. Run schema discovery on the source first, or ${context.remedy}`,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
const offenders = columns.filter((column) => !canBePropertyName(column));
|
|
307
|
+
if (offenders.length === 0)
|
|
308
|
+
return undefined;
|
|
309
|
+
return {
|
|
310
|
+
code: 'columns-not-identifiers',
|
|
311
|
+
subjects: offenders,
|
|
312
|
+
message: `${offenders.length === 1 ? 'One column' : `${offenders.length} columns`} of ${context.what} ${offenders.length === 1 ? 'is' : 'are'} spelled in a way that cannot be a property name: ${offenders.map((name) => `${JSON.stringify(name)} (cleans to ${JSON.stringify(physicalColumn(name))})`).join(', ')}. A property name does not have to be a SQL identifier — a store cleans it, so \`Asset Id\` is a perfectly good name and lands in \`Asset_Id\` — but what the cleaning produces still has to be one, and ${offenders.length === 1 ? 'this one is not' : 'these are not'}. Publishing refuses the name, and the rename reached for instead is the trap: the warehouse matches records to properties by name, so a property called \`${physicalColumn(offenders[0])}\` while the record still arrives keyed \`${offenders[0]}\` writes null into that column on every run and reports success. That is what happened to six types in one evening, so this template refuses rather than guessing which way you meant. ${context.remedy}`,
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
/** The remedy sentence, which is the same one every no-transform template gives. */
|
|
316
|
+
const RENAME_REMEDY = "Put a transform between the source and the sink whose job is to rename the record keys to the property names — the model's own answer, and the reason `columnName` exists is to record what the source called it. For these columns the rename is genuinely unavoidable, which is why it is worth writing rather than a chore: there is no spelling that is both what the source sends and something a store can create.";
|
|
317
|
+
/* -------------------------------------------------------------------------- */
|
|
318
|
+
/* Cadences */
|
|
319
|
+
/* -------------------------------------------------------------------------- */
|
|
320
|
+
/**
|
|
321
|
+
* The cadences a periodic full reload may run at, each carrying both halves.
|
|
322
|
+
*
|
|
323
|
+
* **This closed list is the template.** A cron expression and a `withinMs` are
|
|
324
|
+
* two statements about one cadence, and today nothing links them: an operator
|
|
325
|
+
* sets a schedule on one screen and a reconciliation interval on another, and
|
|
326
|
+
* nothing anywhere notices when they stop agreeing. `withinMs` is only honest if
|
|
327
|
+
* the schedule really does reload that often — once the newest full load of a
|
|
328
|
+
* type is older than it, incremental loads of that type stop committing — so a
|
|
329
|
+
* `withinMs` larger than the real cadence is a guard that never fires and a
|
|
330
|
+
* smaller one is a guard that fires constantly.
|
|
331
|
+
*
|
|
332
|
+
* Deriving both from one entry makes disagreement impossible. Somebody wanting a
|
|
333
|
+
* cadence that is not here has to add it here, in code, where both halves are on
|
|
334
|
+
* the same line and a reviewer sees them together — which is the whole doctrine
|
|
335
|
+
* of this file applied to itself.
|
|
336
|
+
*
|
|
337
|
+
* The cron is evaluated in whatever `CATALOG_SCHEDULE_TZ` names and `everyMs` is
|
|
338
|
+
* wall-clock, so the two drift by an hour twice a year across a DST boundary.
|
|
339
|
+
* The slack in {@link withinMsFor} absorbs that; it is called out here because a
|
|
340
|
+
* reader comparing `86_400_000` against `0 2 * * *` deserves to know it is not
|
|
341
|
+
* an exact identity.
|
|
342
|
+
*/
|
|
343
|
+
export const RELOAD_CADENCES = {
|
|
344
|
+
hourly: { cron: '0 * * * *', everyMs: 3_600_000, human: 'every hour' },
|
|
345
|
+
'six-hourly': { cron: '0 */6 * * *', everyMs: 21_600_000, human: 'every six hours' },
|
|
346
|
+
daily: { cron: '0 2 * * *', everyMs: 86_400_000, human: 'every day at 02:00' },
|
|
347
|
+
weekly: { cron: '0 2 * * 0', everyMs: 604_800_000, human: 'every Sunday at 02:00' },
|
|
348
|
+
};
|
|
349
|
+
export const RELOAD_CADENCE_IDS = Object.keys(RELOAD_CADENCES);
|
|
350
|
+
/**
|
|
351
|
+
* How stale a type may get before its reconciliation claim stops being true.
|
|
352
|
+
*
|
|
353
|
+
* Twice the cadence, and the factor is a decision rather than a round number.
|
|
354
|
+
* At exactly the cadence, any run that starts a minute late — or simply takes a
|
|
355
|
+
* few minutes, since the age is measured from the snapshot's creation — makes
|
|
356
|
+
* the type stale and stops incremental loads of it committing. A guard that
|
|
357
|
+
* fires on an ordinary Tuesday is a guard somebody switches off, taking the real
|
|
358
|
+
* failure with it.
|
|
359
|
+
*
|
|
360
|
+
* Twice means one missed cycle is tolerated and the second consecutive miss is
|
|
361
|
+
* refused, which is the smallest slack that distinguishes "late" from "not
|
|
362
|
+
* happening". It also covers the DST hour noted on {@link RELOAD_CADENCES}.
|
|
363
|
+
*/
|
|
364
|
+
export function withinMsFor(cadence) {
|
|
365
|
+
return RELOAD_CADENCES[cadence].everyMs * 2;
|
|
366
|
+
}
|
|
367
|
+
/* -------------------------------------------------------------------------- */
|
|
368
|
+
/* Node construction */
|
|
369
|
+
/* -------------------------------------------------------------------------- */
|
|
370
|
+
/**
|
|
371
|
+
* A node id that is stable across re-applications of the same template.
|
|
372
|
+
*
|
|
373
|
+
* Deliberately derived from what the node is *for* rather than randomised. A
|
|
374
|
+
* node id becomes a durable step name and part of the key its staged rows are
|
|
375
|
+
* stored under, so an id that changes between one application and the next
|
|
376
|
+
* re-executes work a replay would otherwise have skipped. The canvas randomises
|
|
377
|
+
* ids for nodes somebody draws by hand, which is right for those — there is no
|
|
378
|
+
* meaning to preserve — and wrong for these.
|
|
379
|
+
*
|
|
380
|
+
* `WORKFLOW_NODE_ID_PATTERN` allows letters, digits, underscore and hyphen up to
|
|
381
|
+
* 64 characters; anything else is folded to `_` and the whole thing is bounded,
|
|
382
|
+
* with an ordinal appended so that two types whose names differ only outside the
|
|
383
|
+
* alphabet cannot collide into one id.
|
|
384
|
+
*/
|
|
385
|
+
function nodeId(prefix, subject, ordinal) {
|
|
386
|
+
const cleaned = subject.replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 40);
|
|
387
|
+
return `${prefix}_${cleaned}_${ordinal}`.slice(0, 64);
|
|
388
|
+
}
|
|
389
|
+
/** Placeholder config for a source kind, from the profile rather than by hand. */
|
|
390
|
+
function placeholderConfig(kind) {
|
|
391
|
+
const config = {};
|
|
392
|
+
for (const key of SOURCE_KINDS[kind].required)
|
|
393
|
+
config[key] = '';
|
|
394
|
+
return config;
|
|
395
|
+
}
|
|
396
|
+
function configTodo(kind, nodeName) {
|
|
397
|
+
const profile = SOURCE_KINDS[kind];
|
|
398
|
+
return `"${nodeName}" reads ${profile.label} and needs ${profile.required
|
|
399
|
+
.map((key) => `\`${key}\``)
|
|
400
|
+
.join(' and ')} filled in — the template ships ${profile.required.length === 1 ? 'it' : 'them'} blank on purpose, because a placeholder that looks like a real address is worse than an empty one.`;
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* The declaration every template here makes about load mode, stated once.
|
|
404
|
+
*
|
|
405
|
+
* Shared rather than repeated because it is the same decision for the same
|
|
406
|
+
* reason in all six, and a reader comparing two templates should be able to see
|
|
407
|
+
* that it is literally the same sentence rather than having to diff two
|
|
408
|
+
* paraphrases.
|
|
409
|
+
*/
|
|
410
|
+
const FULL_MODE_DECLARATION = {
|
|
411
|
+
what: 'This graph reads and commits in `full` mode.',
|
|
412
|
+
why: 'An incremental load is blind to deletes — a row removed at the source never changes again, so it is never returned again and is carried forward into every later snapshot indefinitely, with nothing going wrong at any single step. Because that failure is silent the pipeline refuses an incremental load of a type that has not declared how deletes reach it, and it also needs a watermark column no template can know. Offering `incremental` here would mean shipping a graph that fails at its first run, or inventing a declaration on your behalf.',
|
|
413
|
+
changeAt: 'The `mode` field on the source node and on the sink node.',
|
|
414
|
+
};
|
|
415
|
+
/**
|
|
416
|
+
* The catalogue, keyed by id so that a new id without metadata fails the build.
|
|
417
|
+
*
|
|
418
|
+
* Shipped by the library rather than stored per deployment, deliberately: these
|
|
419
|
+
* are decisions, decisions belong in code where they are reviewed and carry
|
|
420
|
+
* their reasoning, and a template stored in a database is one nobody reviews.
|
|
421
|
+
* Per-deployment templates are a store concern and a separate change, and this
|
|
422
|
+
* sentence is here so that the absence reads as a decision rather than an
|
|
423
|
+
* oversight.
|
|
424
|
+
*/
|
|
425
|
+
export const WORKFLOW_TEMPLATES = {
|
|
426
|
+
'replicate-table': {
|
|
427
|
+
id: 'replicate-table',
|
|
428
|
+
title: 'Replicate a table',
|
|
429
|
+
summary: 'One SQL query straight into one object type, with no transform in between.',
|
|
430
|
+
assumes: [
|
|
431
|
+
'The query returns one row per record of the type, already shaped the way the type wants it.',
|
|
432
|
+
'Every column it returns is spelled in a way that can be a property name — which allows `Asset Id`, and not `2024 Total`. The template checks this and refuses if not.',
|
|
433
|
+
],
|
|
434
|
+
},
|
|
435
|
+
'load-file-drop': {
|
|
436
|
+
id: 'load-file-drop',
|
|
437
|
+
title: 'Load a file drop',
|
|
438
|
+
summary: 'A CSV, NDJSON or JSON drop — on disk or in an object store — into one object type.',
|
|
439
|
+
assumes: [
|
|
440
|
+
'The drop is a full extract each time, not a delta.',
|
|
441
|
+
'Its header row is spelled in a way that can be a property name — which allows `Asset LIN/TAMCN`, and not a column headed with a year. The template checks this and refuses if not.',
|
|
442
|
+
],
|
|
443
|
+
},
|
|
444
|
+
'fan-out-types': {
|
|
445
|
+
id: 'fan-out-types',
|
|
446
|
+
title: 'Fan one source into several types',
|
|
447
|
+
summary: 'One expensive read, narrowed by a transform per type into a sink per type.',
|
|
448
|
+
assumes: [
|
|
449
|
+
'The source is worth reading once rather than once per type — that is the entire reason for this shape.',
|
|
450
|
+
'Each type is a projection of the same rows. A type needing different rows needs its own source.',
|
|
451
|
+
],
|
|
452
|
+
},
|
|
453
|
+
'join-sources': {
|
|
454
|
+
id: 'join-sources',
|
|
455
|
+
title: 'Join two sources into one type',
|
|
456
|
+
summary: 'Two reads, joined on a key inside one transform, committed as a single type.',
|
|
457
|
+
assumes: [
|
|
458
|
+
'One field tells the two sides apart, because the runner concatenates them into one untagged batch.',
|
|
459
|
+
'The whole of both sides fits in memory at once — the transform contract is set-wise by design.',
|
|
460
|
+
],
|
|
461
|
+
},
|
|
462
|
+
'enrich-with-lookup': {
|
|
463
|
+
id: 'enrich-with-lookup',
|
|
464
|
+
title: 'Enrich against a lookup table',
|
|
465
|
+
summary: 'A main feed plus a dictionary, joined so that unmatched rows survive.',
|
|
466
|
+
assumes: [
|
|
467
|
+
'The lookup is small enough to hold beside the feed it enriches.',
|
|
468
|
+
'A row whose key is not in the lookup is still a row worth keeping. That is the difference from a join.',
|
|
469
|
+
],
|
|
470
|
+
},
|
|
471
|
+
'periodic-full-reload': {
|
|
472
|
+
id: 'periodic-full-reload',
|
|
473
|
+
title: 'Periodic full reload',
|
|
474
|
+
summary: 'A full read on a schedule, with the matching delete-reconciliation declared from the same cadence.',
|
|
475
|
+
assumes: [
|
|
476
|
+
'A full read is the right mode here rather than a stepping stone to an incremental one.',
|
|
477
|
+
'The source can be read in its entirety at the chosen cadence without hurting it.',
|
|
478
|
+
],
|
|
479
|
+
},
|
|
480
|
+
};
|
|
481
|
+
export const WORKFLOW_TEMPLATE_IDS = Object.keys(WORKFLOW_TEMPLATES);
|
|
482
|
+
/**
|
|
483
|
+
* SQL straight into a type — the named case, and the one that produced the
|
|
484
|
+
* nulls.
|
|
485
|
+
*
|
|
486
|
+
* The shape is trivial and the value is entirely in the refusal. Two nodes and
|
|
487
|
+
* one edge is not something anybody needs help drawing; what they need help with
|
|
488
|
+
* is knowing that this shape is only correct when every column the query returns
|
|
489
|
+
* can be a property name, and that when it is not, the graph still saves, still
|
|
490
|
+
* runs, still commits and still reports the right row count while writing null
|
|
491
|
+
* into most of the columns.
|
|
492
|
+
*
|
|
493
|
+
* So this template checks, and refuses, and names every offending column. It
|
|
494
|
+
* does **not** offer to sanitise them: that is the naive fix and it is what was
|
|
495
|
+
* done six times in one evening. For the columns that now reach this refusal
|
|
496
|
+
* there is no spelling that is both the key the record arrives under and a name
|
|
497
|
+
* a store can create, so the rename has to be paired with a transform that
|
|
498
|
+
* renames the record's keys — which is a decision about the data rather than a
|
|
499
|
+
* shape, and belongs to whoever knows the data.
|
|
500
|
+
*
|
|
501
|
+
* **What it no longer refuses.** `Asset Id` and `Asset LIN/TAMCN` pass now.
|
|
502
|
+
* `fix/view-alias-sanitised` made the view's output alias and the read's lookup
|
|
503
|
+
* key clean together, so a property keeps the source's spelling end to end and a
|
|
504
|
+
* two-node graph over those columns is simply correct. Refusing them would be
|
|
505
|
+
* this template sending somebody off to do the exact rename that caused the
|
|
506
|
+
* incident it exists to prevent. See the header.
|
|
507
|
+
*/
|
|
508
|
+
export function replicateTable(options) {
|
|
509
|
+
const targetType = options.targetType.trim();
|
|
510
|
+
if (targetType.length === 0) {
|
|
511
|
+
return {
|
|
512
|
+
ok: false,
|
|
513
|
+
refusals: [
|
|
514
|
+
{
|
|
515
|
+
code: 'incomplete',
|
|
516
|
+
subjects: [],
|
|
517
|
+
message: 'This template needs the object type the sink will commit. The type is chosen on the sink because the sink is the node that commits.',
|
|
518
|
+
},
|
|
519
|
+
],
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
const refusal = refuseUnusableColumnNames(options.sourceColumns, {
|
|
523
|
+
what: 'the query',
|
|
524
|
+
remedy: RENAME_REMEDY,
|
|
525
|
+
});
|
|
526
|
+
if (refusal)
|
|
527
|
+
return { ok: false, refusals: [refusal] };
|
|
528
|
+
const source = NODE_FACTORIES.source(nodeId('source', targetType, 1), `${targetType} query`, {
|
|
529
|
+
sourceKind: 'sql',
|
|
530
|
+
config: { ...placeholderConfig('sql'), ...(options.query ? { query: options.query } : {}) },
|
|
531
|
+
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
|
532
|
+
mode: 'full',
|
|
533
|
+
});
|
|
534
|
+
const sink = NODE_FACTORIES.sink(nodeId('sink', targetType, 1), targetType, {
|
|
535
|
+
targetType,
|
|
536
|
+
mode: 'full',
|
|
537
|
+
});
|
|
538
|
+
return {
|
|
539
|
+
ok: true,
|
|
540
|
+
plan: {
|
|
541
|
+
templateId: 'replicate-table',
|
|
542
|
+
name: options.workflowName?.trim() || `Replicate ${targetType}`,
|
|
543
|
+
description: `Reads ${targetType} from a SQL source in one query and commits it whole.`,
|
|
544
|
+
nodes: [source, sink],
|
|
545
|
+
edges: [{ from: source.id, to: sink.id }],
|
|
546
|
+
transforms: [],
|
|
547
|
+
expectations: [],
|
|
548
|
+
declarations: [
|
|
549
|
+
FULL_MODE_DECLARATION,
|
|
550
|
+
{
|
|
551
|
+
what: 'No transform sits between the source and the sink.',
|
|
552
|
+
why: 'Every column the query returns was checked with the same two calls the store and the publish API make — cleaning the name, then asking whether the result is an identifier — and all of them can be property names, so the records arrive keyed exactly as the type will ask for them. Name the properties as the query spells them; a column called `Asset Id` wants a property called `Asset Id`, not `Asset_Id`. Had any column failed, this template would have refused instead of adding a transform, because the rename it would have to write is a decision about your data rather than a shape.',
|
|
553
|
+
changeAt: 'Add a transform node on the edge if the mapping ever stops being one-to-one.',
|
|
554
|
+
},
|
|
555
|
+
],
|
|
556
|
+
todo: [
|
|
557
|
+
configTodo('sql', source.name),
|
|
558
|
+
`The sink commits \`${targetType}\`. Publish that type first if it does not exist — a sink naming a type nobody has published fails at the commit, after the whole source has been read.`,
|
|
559
|
+
],
|
|
560
|
+
},
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* A DoD/DPAS-style drop into a type.
|
|
565
|
+
*
|
|
566
|
+
* Structurally the same two nodes as {@link replicateTable} and separated from
|
|
567
|
+
* it because the config, the assumptions and the failure mode differ enough that
|
|
568
|
+
* one template covering both would have to hedge every sentence. A drop is a
|
|
569
|
+
* full extract that lands on a schedule somebody else controls, its columns come
|
|
570
|
+
* from a spreadsheet header rather than a `SELECT` list, and a spreadsheet header
|
|
571
|
+
* is overwhelmingly the most likely place to meet a column headed `2024 Total`
|
|
572
|
+
* or `2024 Q1 Hours` — which is why the same refusal matters more here, not less.
|
|
573
|
+
*
|
|
574
|
+
* `Asset LIN/TAMCN`, the header this template was written around, is no longer
|
|
575
|
+
* one of them: it cleans to `Asset_LIN_TAMCN` and is published under its own
|
|
576
|
+
* spelling. Nor is `FY24 Hours`, which cleans to `FY24_Hours` — the year has to
|
|
577
|
+
* be at the *front* to be a problem. What is refused is a header whose cleaned
|
|
578
|
+
* form starts with a digit, and that really does need a transform.
|
|
579
|
+
*/
|
|
580
|
+
export function loadFileDrop(options) {
|
|
581
|
+
const targetType = options.targetType.trim();
|
|
582
|
+
if (targetType.length === 0) {
|
|
583
|
+
return {
|
|
584
|
+
ok: false,
|
|
585
|
+
refusals: [
|
|
586
|
+
{
|
|
587
|
+
code: 'incomplete',
|
|
588
|
+
subjects: [],
|
|
589
|
+
message: 'This template needs the object type the sink will commit.',
|
|
590
|
+
},
|
|
591
|
+
],
|
|
592
|
+
};
|
|
593
|
+
}
|
|
594
|
+
const refusal = refuseUnusableColumnNames(options.sourceColumns, {
|
|
595
|
+
what: "the drop's header row",
|
|
596
|
+
remedy: RENAME_REMEDY,
|
|
597
|
+
});
|
|
598
|
+
if (refusal)
|
|
599
|
+
return { ok: false, refusals: [refusal] };
|
|
600
|
+
const source = NODE_FACTORIES.source(nodeId('source', targetType, 1), `${targetType} drop`, {
|
|
601
|
+
sourceKind: options.kind,
|
|
602
|
+
config: placeholderConfig(options.kind),
|
|
603
|
+
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
|
604
|
+
mode: 'full',
|
|
605
|
+
});
|
|
606
|
+
const sink = NODE_FACTORIES.sink(nodeId('sink', targetType, 1), targetType, {
|
|
607
|
+
targetType,
|
|
608
|
+
mode: 'full',
|
|
609
|
+
});
|
|
610
|
+
return {
|
|
611
|
+
ok: true,
|
|
612
|
+
plan: {
|
|
613
|
+
templateId: 'load-file-drop',
|
|
614
|
+
name: options.workflowName?.trim() || `Load ${targetType} drop`,
|
|
615
|
+
description: `Reads a ${targetType} drop from ${SOURCE_KINDS[options.kind].label} and commits it whole.`,
|
|
616
|
+
nodes: [source, sink],
|
|
617
|
+
edges: [{ from: source.id, to: sink.id }],
|
|
618
|
+
transforms: [],
|
|
619
|
+
expectations: [],
|
|
620
|
+
declarations: [
|
|
621
|
+
FULL_MODE_DECLARATION,
|
|
622
|
+
{
|
|
623
|
+
what: 'Each run treats the drop as the complete dataset.',
|
|
624
|
+
why: 'A drop is replaced rather than appended to, and a full commit repoints the view atomically, so a run that reads a short file replaces the good data with the short file. That is the failure worth knowing about here — the row-count bound on the type is what catches it, and it is worth setting before this runs unattended.',
|
|
625
|
+
changeAt: "The type's load expectation, on the Model screen.",
|
|
626
|
+
},
|
|
627
|
+
],
|
|
628
|
+
todo: [
|
|
629
|
+
configTodo(options.kind, source.name),
|
|
630
|
+
options.kind === 's3'
|
|
631
|
+
? 'Set `prefix` and `suffix` so a run reads only the drop and not everything else in the bucket. Without them every object under the bucket is read on every run.'
|
|
632
|
+
: 'Set `format` if the extension does not say — a `.txt` holding CSV is parsed as JSON and yields nothing.',
|
|
633
|
+
],
|
|
634
|
+
},
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* One expensive read feeding several types.
|
|
639
|
+
*
|
|
640
|
+
* The reason the model allows several sinks at all: pulling ten million rows
|
|
641
|
+
* once and deriving vehicles and work orders from them, rather than pulling
|
|
642
|
+
* twice. The validator endorses it explicitly and guards the one genuinely
|
|
643
|
+
* ambiguous case — two sinks writing the *same* type, which would be two
|
|
644
|
+
* snapshots of one type in one run with nothing to say which readers should get.
|
|
645
|
+
* This template refuses that up front rather than letting the canvas report it,
|
|
646
|
+
* because at the point of choosing it is a typo and after applying it is a graph
|
|
647
|
+
* to unpick.
|
|
648
|
+
*
|
|
649
|
+
* **A transform per branch, not one shared transform.** A node with two outbound
|
|
650
|
+
* edges is read by both successors, so a single transform feeding two sinks
|
|
651
|
+
* would commit identical wide rows into both types. The narrowing has to happen
|
|
652
|
+
* per branch, which is what makes this a fan-*out* rather than a duplication.
|
|
653
|
+
*/
|
|
654
|
+
export function fanOutTypes(options) {
|
|
655
|
+
const refusals = [];
|
|
656
|
+
const targets = options.targets.filter((target) => target.targetType.trim().length > 0);
|
|
657
|
+
if (targets.length < 2) {
|
|
658
|
+
refusals.push({
|
|
659
|
+
code: 'incomplete',
|
|
660
|
+
subjects: [],
|
|
661
|
+
message: 'This template needs at least two object types. With one type it is a plain source-to-sink graph, and "Replicate a table" is that graph with a check this one cannot make.',
|
|
662
|
+
});
|
|
663
|
+
}
|
|
664
|
+
const seen = new Set();
|
|
665
|
+
const duplicated = new Set();
|
|
666
|
+
for (const target of targets) {
|
|
667
|
+
const name = target.targetType.trim();
|
|
668
|
+
if (seen.has(name))
|
|
669
|
+
duplicated.add(name);
|
|
670
|
+
seen.add(name);
|
|
671
|
+
}
|
|
672
|
+
if (duplicated.size > 0) {
|
|
673
|
+
refusals.push({
|
|
674
|
+
code: 'duplicate-target-type',
|
|
675
|
+
subjects: [...duplicated],
|
|
676
|
+
message: `Two branches both commit ${[...duplicated].map((name) => JSON.stringify(name)).join(', ')}. Two snapshots of one type in a single run leaves nothing to say which one readers should get, so the graph would be refused — send them to different types, or narrow them in one branch instead of two.`,
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
for (const target of targets) {
|
|
680
|
+
const refusal = refuseUnusableColumnNames(target.properties, {
|
|
681
|
+
what: `the properties of ${target.targetType.trim()}`,
|
|
682
|
+
remedy: "A property name is cleaned into the column a store creates for it, and what the cleaning produces has to be an identifier — so publishing refuses this one. Rename it in the type, and record the source spelling in that property's `columnName`. This branch already has a transform on it, so make that transform emit the new key too.",
|
|
683
|
+
});
|
|
684
|
+
if (refusal)
|
|
685
|
+
refusals.push(refusal);
|
|
686
|
+
}
|
|
687
|
+
if (refusals.length > 0)
|
|
688
|
+
return { ok: false, refusals };
|
|
689
|
+
const source = NODE_FACTORIES.source(nodeId('source', 'shared', 1), 'Shared read', {
|
|
690
|
+
sourceKind: options.sourceKind,
|
|
691
|
+
config: placeholderConfig(options.sourceKind),
|
|
692
|
+
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
|
693
|
+
mode: 'full',
|
|
694
|
+
});
|
|
695
|
+
const nodes = [source];
|
|
696
|
+
const edges = [];
|
|
697
|
+
const transforms = [];
|
|
698
|
+
targets.forEach((target, index) => {
|
|
699
|
+
const typeName = target.targetType.trim();
|
|
700
|
+
const transform = NODE_FACTORIES.transform(nodeId('transform', typeName, index + 1), `Narrow to ${typeName}`, { transformId: '' });
|
|
701
|
+
const sink = NODE_FACTORIES.sink(nodeId('sink', typeName, index + 1), typeName, {
|
|
702
|
+
targetType: typeName,
|
|
703
|
+
mode: 'full',
|
|
704
|
+
});
|
|
705
|
+
nodes.push(transform, sink);
|
|
706
|
+
edges.push({ from: source.id, to: transform.id }, { from: transform.id, to: sink.id });
|
|
707
|
+
transforms.push({
|
|
708
|
+
forNodeId: transform.id,
|
|
709
|
+
name: `Narrow to ${typeName}`,
|
|
710
|
+
language: options.language,
|
|
711
|
+
code: PROJECT_CODE[options.language](typeName, target.properties),
|
|
712
|
+
why: `Keeps only the columns ${typeName} commits, out of the rows every branch of this graph shares.`,
|
|
713
|
+
});
|
|
714
|
+
});
|
|
715
|
+
return {
|
|
716
|
+
ok: true,
|
|
717
|
+
plan: {
|
|
718
|
+
templateId: 'fan-out-types',
|
|
719
|
+
name: options.workflowName?.trim() ||
|
|
720
|
+
`Split into ${targets.map((t) => t.targetType.trim()).join(', ')}`,
|
|
721
|
+
description: `Reads once and derives ${targets.length} types from the same rows.`,
|
|
722
|
+
nodes,
|
|
723
|
+
edges,
|
|
724
|
+
transforms,
|
|
725
|
+
expectations: [],
|
|
726
|
+
declarations: [
|
|
727
|
+
FULL_MODE_DECLARATION,
|
|
728
|
+
{
|
|
729
|
+
what: 'Each type gets its own transform, rather than sharing one.',
|
|
730
|
+
why: 'Both successors of the source read the same rows, so one shared transform would commit identical wide rows into every type. Narrowing per branch is what makes this a fan-out and not a duplication.',
|
|
731
|
+
changeAt: 'The code on each transform node, in the editor.',
|
|
732
|
+
},
|
|
733
|
+
{
|
|
734
|
+
what: 'Each sink commits independently.',
|
|
735
|
+
why: 'There is no distributed transaction across sinks and the model does not pretend otherwise: a run that commits one type and fails on another is a failed run that has nonetheless written something. That is the price of the single read, and it is the same thing two separate connectors would do today.',
|
|
736
|
+
changeAt: 'Split into one workflow per type if a partial commit is unacceptable.',
|
|
737
|
+
},
|
|
738
|
+
],
|
|
739
|
+
todo: [
|
|
740
|
+
configTodo(options.sourceKind, source.name),
|
|
741
|
+
`${transforms.length} transforms have to be created before this graph will run — their nodes name no code until they exist.`,
|
|
742
|
+
'Check each projection against what the type actually publishes: a property name in the list that the type does not have writes a column nobody reads, and one the type has that is missing from the list is committed as null.',
|
|
743
|
+
],
|
|
744
|
+
},
|
|
745
|
+
};
|
|
746
|
+
}
|
|
747
|
+
/**
|
|
748
|
+
* Two reads into one type, joined inside the transform.
|
|
749
|
+
*
|
|
750
|
+
* Expressible only because the transform contract is set-wise: the code is a
|
|
751
|
+
* function over a whole batch precisely so that it can deduplicate, aggregate
|
|
752
|
+
* and join, none of which can be done a row at a time.
|
|
753
|
+
*
|
|
754
|
+
* **The decision this template exists to make is what the batch looks like.** A
|
|
755
|
+
* node with several inbound edges receives its inputs *concatenated in edge
|
|
756
|
+
* order, in one array, with nothing marking which record came from which side*.
|
|
757
|
+
* The natural assumption is two arrays or two arguments, and code written under
|
|
758
|
+
* that assumption silently processes the wrong half. So the starter body
|
|
759
|
+
* partitions the batch explicitly, with the reason in a comment directly above
|
|
760
|
+
* the loop, and the template asks for the discriminator up front rather than
|
|
761
|
+
* leaving somebody to discover the problem from a result that is merely odd.
|
|
762
|
+
*
|
|
763
|
+
* Edge order is not decoration: it is what decides which side is left, and it is
|
|
764
|
+
* part of the graph fingerprint, so swapping the two inputs is a new version of
|
|
765
|
+
* the graph. The left source is added first, deliberately and stably.
|
|
766
|
+
*/
|
|
767
|
+
export function joinSources(options) {
|
|
768
|
+
return buildJoin(options, 'join-sources');
|
|
769
|
+
}
|
|
770
|
+
/**
|
|
771
|
+
* A feed plus a dictionary, where a row the dictionary has not heard of
|
|
772
|
+
* survives.
|
|
773
|
+
*
|
|
774
|
+
* The same graph and the same code as {@link joinSources} with one flag flipped,
|
|
775
|
+
* and it is a separate template because that flag is the decision. Enriching
|
|
776
|
+
* against a lookup and joining two feeds look identical on a canvas and differ
|
|
777
|
+
* entirely in what happens to a record with no counterpart: dropping unmatched
|
|
778
|
+
* rows from an enrichment means a load silently loses every record whose key the
|
|
779
|
+
* dictionary has not caught up with yet — a load that succeeds, reports a
|
|
780
|
+
* plausible row count, and is missing data. That is the same shape of failure as
|
|
781
|
+
* the null columns, and it is the reason the two are offered as two choices
|
|
782
|
+
* rather than one template with a checkbox nobody reads.
|
|
783
|
+
*
|
|
784
|
+
* Concretely useful here: this deployment has a `unit_dictionary` type and a
|
|
785
|
+
* unit-normalisation problem it was built for.
|
|
786
|
+
*/
|
|
787
|
+
export function enrichWithLookup(options) {
|
|
788
|
+
return buildJoin(options, 'enrich-with-lookup');
|
|
789
|
+
}
|
|
790
|
+
/**
|
|
791
|
+
* The fields a join cannot be built without, in the words the refusal uses.
|
|
792
|
+
*
|
|
793
|
+
* Split out so that adding a field is one line in one list rather than another
|
|
794
|
+
* branch in a function that is already deciding four other things — and so that
|
|
795
|
+
* the refusal names *every* blank field at once. A form that reports its
|
|
796
|
+
* required fields one round trip at a time is a form people abandon.
|
|
797
|
+
*/
|
|
798
|
+
function missingJoinFields(options) {
|
|
799
|
+
const required = [
|
|
800
|
+
[options.targetType, 'the object type the sink commits'],
|
|
801
|
+
[options.discriminator, 'the field that tells the two sides apart'],
|
|
802
|
+
[options.leftKey, 'the key on the left-hand side'],
|
|
803
|
+
[options.rightKey, 'the key on the right-hand side'],
|
|
804
|
+
];
|
|
805
|
+
return required.filter(([value]) => value.trim().length === 0).map(([, described]) => described);
|
|
806
|
+
}
|
|
807
|
+
function buildJoin(options, templateId) {
|
|
808
|
+
const enriching = templateId === 'enrich-with-lookup';
|
|
809
|
+
const targetType = options.targetType.trim();
|
|
810
|
+
const missing = missingJoinFields(options);
|
|
811
|
+
if (missing.length > 0) {
|
|
812
|
+
return {
|
|
813
|
+
ok: false,
|
|
814
|
+
refusals: [
|
|
815
|
+
{
|
|
816
|
+
code: 'incomplete',
|
|
817
|
+
subjects: missing,
|
|
818
|
+
message: `This template still needs ${missing.join(', ')}. The discriminator especially is not optional: the runner hands both inputs to the transform as one concatenated array with nothing marking which side a record came from, so without a field that only one side carries there is no way for the code to tell them apart — and code that guesses produces a plausible result from the wrong half of the data.`,
|
|
819
|
+
},
|
|
820
|
+
],
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
const leftName = options.leftName?.trim() || (enriching ? 'Main feed' : 'Left source');
|
|
824
|
+
const rightName = options.rightName?.trim() || (enriching ? 'Lookup' : 'Right source');
|
|
825
|
+
const left = NODE_FACTORIES.source(nodeId('source', 'left', 1), leftName, {
|
|
826
|
+
sourceKind: options.leftSourceKind,
|
|
827
|
+
config: placeholderConfig(options.leftSourceKind),
|
|
828
|
+
mode: 'full',
|
|
829
|
+
});
|
|
830
|
+
const right = NODE_FACTORIES.source(nodeId('source', 'right', 2), rightName, {
|
|
831
|
+
sourceKind: options.rightSourceKind,
|
|
832
|
+
config: placeholderConfig(options.rightSourceKind),
|
|
833
|
+
mode: 'full',
|
|
834
|
+
});
|
|
835
|
+
const transform = NODE_FACTORIES.transform(nodeId('transform', targetType, 1), enriching ? `Enrich ${targetType}` : `Join into ${targetType}`, { transformId: '' });
|
|
836
|
+
const sink = NODE_FACTORIES.sink(nodeId('sink', targetType, 1), targetType, {
|
|
837
|
+
targetType,
|
|
838
|
+
mode: 'full',
|
|
839
|
+
});
|
|
840
|
+
const code = JOIN_CODE[options.language]({
|
|
841
|
+
discriminator: options.discriminator.trim(),
|
|
842
|
+
leftKey: options.leftKey.trim(),
|
|
843
|
+
rightKey: options.rightKey.trim(),
|
|
844
|
+
keepUnmatched: enriching,
|
|
845
|
+
});
|
|
846
|
+
return {
|
|
847
|
+
ok: true,
|
|
848
|
+
plan: {
|
|
849
|
+
templateId,
|
|
850
|
+
name: options.workflowName?.trim() ||
|
|
851
|
+
(enriching ? `Enrich ${targetType}` : `Join into ${targetType}`),
|
|
852
|
+
description: enriching
|
|
853
|
+
? `Reads ${leftName} and ${rightName}, fills each row in from the lookup, and commits ${targetType}.`
|
|
854
|
+
: `Reads ${leftName} and ${rightName}, joins them on a key, and commits ${targetType}.`,
|
|
855
|
+
nodes: [left, right, transform, sink],
|
|
856
|
+
// Left first. Edge order decides which side is which and is part of the
|
|
857
|
+
// graph fingerprint, so this ordering is load-bearing rather than tidy.
|
|
858
|
+
edges: [
|
|
859
|
+
{ from: left.id, to: transform.id },
|
|
860
|
+
{ from: right.id, to: transform.id },
|
|
861
|
+
{ from: transform.id, to: sink.id },
|
|
862
|
+
],
|
|
863
|
+
transforms: [
|
|
864
|
+
{
|
|
865
|
+
forNodeId: transform.id,
|
|
866
|
+
name: enriching ? `Enrich ${targetType}` : `Join into ${targetType}`,
|
|
867
|
+
language: options.language,
|
|
868
|
+
code,
|
|
869
|
+
why: enriching
|
|
870
|
+
? `Splits the concatenated batch, indexes ${rightName} by \`${options.rightKey.trim()}\`, and fills in each ${leftName} row. A row with no match is kept.`
|
|
871
|
+
: `Splits the concatenated batch, indexes ${rightName} by \`${options.rightKey.trim()}\`, and joins. A row with no match is dropped.`,
|
|
872
|
+
},
|
|
873
|
+
],
|
|
874
|
+
expectations: [],
|
|
875
|
+
declarations: [
|
|
876
|
+
FULL_MODE_DECLARATION,
|
|
877
|
+
{
|
|
878
|
+
what: `Both inputs are read into one batch and told apart by \`${options.discriminator.trim()}\`.`,
|
|
879
|
+
why: 'A transform node receives its inbound edges concatenated in edge order, in one array, with nothing marking the origin of a record. That is what makes a join possible at all — the code can see everything at once — but it means the split has to be written explicitly, and a discriminator chosen by somebody who knows both shapes is the only reliable way to do it.',
|
|
880
|
+
changeAt: 'The partition at the top of the transform body.',
|
|
881
|
+
},
|
|
882
|
+
enriching
|
|
883
|
+
? {
|
|
884
|
+
what: 'A row whose key is not in the lookup is kept, unenriched.',
|
|
885
|
+
why: 'Dropping it would mean the load silently loses every record the dictionary has not caught up with — a run that succeeds, reports a plausible count and is quietly missing data. If an unmatched row genuinely is not a record of this type, use "Join two sources" instead, which drops it and says so.',
|
|
886
|
+
changeAt: 'The `if (!found)` branch in the transform body.',
|
|
887
|
+
}
|
|
888
|
+
: {
|
|
889
|
+
what: 'A row whose key has no counterpart is dropped.',
|
|
890
|
+
why: 'A record with nothing on the other side is not a record of the joined thing. If it is — if the right-hand side is a dictionary rather than a second feed — use "Enrich against a lookup table" instead, which keeps it.',
|
|
891
|
+
changeAt: 'The `if (!found)` branch in the transform body.',
|
|
892
|
+
},
|
|
893
|
+
{
|
|
894
|
+
what: 'The left record wins where both sides carry the same field.',
|
|
895
|
+
why: 'The left side is the record the row is about; the other side is filling in what it did not know. The reverse would let a stale dictionary overwrite live values, which is unrecoverable once committed.',
|
|
896
|
+
changeAt: 'The spread order on the last line of the transform body.',
|
|
897
|
+
},
|
|
898
|
+
],
|
|
899
|
+
todo: [
|
|
900
|
+
configTodo(options.leftSourceKind, left.name),
|
|
901
|
+
configTodo(options.rightSourceKind, right.name),
|
|
902
|
+
'The transform has to be created before this graph will run — its node names no code until it exists.',
|
|
903
|
+
`Both sides are held in memory at once while the transform runs. That is inherent to a set-wise contract, but it means ${rightName} wants to be the smaller of the two.`,
|
|
904
|
+
],
|
|
905
|
+
},
|
|
906
|
+
};
|
|
907
|
+
}
|
|
908
|
+
/**
|
|
909
|
+
* A full read on a schedule, with the reconciliation it implies declared from
|
|
910
|
+
* the same input.
|
|
911
|
+
*
|
|
912
|
+
* The template most worth having, because the two things it produces are the two
|
|
913
|
+
* that must agree and that nothing currently links. A `periodic-full-reload`
|
|
914
|
+
* strategy says deletions reach this type because it is fully reloaded every
|
|
915
|
+
* `withinMs`; a schedule says how often it is actually reloaded. An operator
|
|
916
|
+
* sets one on the Model screen and the other on the connector, and nothing
|
|
917
|
+
* anywhere notices when they drift apart — at which point the type carries a
|
|
918
|
+
* declaration that reads as a guarantee and polices nothing.
|
|
919
|
+
*
|
|
920
|
+
* Here they come from one {@link RELOAD_CADENCES} entry, so they cannot
|
|
921
|
+
* disagree, and `withinMs` gets its slack from {@link withinMsFor} rather than
|
|
922
|
+
* from whatever somebody typed.
|
|
923
|
+
*
|
|
924
|
+
* **`because` is required and is not decoration.** `refuseUndeclaredDeletes`
|
|
925
|
+
* rejects a declaration whose reason is blank, on every strategy, because the
|
|
926
|
+
* control was never "it must be in code" — it is that somebody chose a strategy
|
|
927
|
+
* and wrote down why. A template that supplied a default reason would be
|
|
928
|
+
* defeating exactly the mechanism it is here to satisfy.
|
|
929
|
+
*
|
|
930
|
+
* **Where the schedule goes, and why the plan does not say.** The plan carries
|
|
931
|
+
* the cadence as data — {@link TemplatePlan.schedule} — and leaves *placing* it
|
|
932
|
+
* to whatever applies the plan. That was a deliberate bet, and it paid.
|
|
933
|
+
*
|
|
934
|
+
* When this was written, a workflow was put on a timer by a connector that
|
|
935
|
+
* pointed at it with `workflowId` and carried the `schedule` string. A template
|
|
936
|
+
* that had written that hop into its plan or its `todo` would have had to be
|
|
937
|
+
* rewritten when the workflow became the only authored object: connectors stopped
|
|
938
|
+
* being authored at all, and a schedule now lives on the workflow, set through
|
|
939
|
+
* `PUT workflows/:id/schedule`. Because the plan only ever declared "this runs
|
|
940
|
+
* `0 2 * * *`" and never "create a connector that says so", the move cost this
|
|
941
|
+
* template nothing — the same plan, rebased with no conflict, describes the new
|
|
942
|
+
* arrangement as accurately as it described the old one.
|
|
943
|
+
*
|
|
944
|
+
* What did not change, and is the part worth stating: only a `ready` graph is
|
|
945
|
+
* scheduled. A schedule written against a draft is stored and does not fire.
|
|
946
|
+
*/
|
|
947
|
+
export function periodicFullReload(options) {
|
|
948
|
+
const targetType = options.targetType.trim();
|
|
949
|
+
const because = options.because.trim();
|
|
950
|
+
const missing = [];
|
|
951
|
+
if (targetType.length === 0)
|
|
952
|
+
missing.push('the object type the sink commits');
|
|
953
|
+
if (because.length === 0)
|
|
954
|
+
missing.push('a reason this cadence reconciles deletes acceptably');
|
|
955
|
+
if (missing.length > 0) {
|
|
956
|
+
return {
|
|
957
|
+
ok: false,
|
|
958
|
+
refusals: [
|
|
959
|
+
{
|
|
960
|
+
code: 'incomplete',
|
|
961
|
+
subjects: missing,
|
|
962
|
+
message: `This template still needs ${missing.join(' and ')}. The reason is not optional and is not rhetorical: a delete-reconciliation declaration with a blank reason is refused by the pipeline on every strategy, because what makes the declaration worth anything is that somebody chose it and wrote down why. Supplying a default here would defeat the mechanism this template exists to satisfy.`,
|
|
963
|
+
},
|
|
964
|
+
],
|
|
965
|
+
};
|
|
966
|
+
}
|
|
967
|
+
const refusal = refuseUnusableColumnNames(options.sourceColumns, {
|
|
968
|
+
what: 'the source',
|
|
969
|
+
remedy: RENAME_REMEDY,
|
|
970
|
+
});
|
|
971
|
+
if (refusal)
|
|
972
|
+
return { ok: false, refusals: [refusal] };
|
|
973
|
+
const schedule = RELOAD_CADENCES[options.cadence];
|
|
974
|
+
const withinMs = withinMsFor(options.cadence);
|
|
975
|
+
const source = NODE_FACTORIES.source(nodeId('source', targetType, 1), `${targetType} full read`, {
|
|
976
|
+
sourceKind: options.sourceKind,
|
|
977
|
+
config: placeholderConfig(options.sourceKind),
|
|
978
|
+
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
|
979
|
+
mode: 'full',
|
|
980
|
+
});
|
|
981
|
+
const sink = NODE_FACTORIES.sink(nodeId('sink', targetType, 1), targetType, {
|
|
982
|
+
targetType,
|
|
983
|
+
mode: 'full',
|
|
984
|
+
});
|
|
985
|
+
const deletes = {
|
|
986
|
+
strategy: 'periodic-full-reload',
|
|
987
|
+
because,
|
|
988
|
+
withinMs,
|
|
989
|
+
};
|
|
990
|
+
return {
|
|
991
|
+
ok: true,
|
|
992
|
+
plan: {
|
|
993
|
+
templateId: 'periodic-full-reload',
|
|
994
|
+
name: options.workflowName?.trim() || `Reload ${targetType} ${schedule.human}`,
|
|
995
|
+
description: `Reads ${targetType} whole ${schedule.human} and commits it, reconciling deletes by replacement.`,
|
|
996
|
+
nodes: [source, sink],
|
|
997
|
+
edges: [{ from: source.id, to: sink.id }],
|
|
998
|
+
transforms: [],
|
|
999
|
+
expectations: [
|
|
1000
|
+
{
|
|
1001
|
+
typeName: targetType,
|
|
1002
|
+
input: { deletes },
|
|
1003
|
+
why: `Declares that deletions reach ${targetType} by full replacement within ${withinMs}ms, which is twice the ${schedule.human} cadence this same template set.`,
|
|
1004
|
+
},
|
|
1005
|
+
],
|
|
1006
|
+
schedule,
|
|
1007
|
+
declarations: [
|
|
1008
|
+
{
|
|
1009
|
+
what: `The schedule is \`${schedule.cron}\` and the reconciliation interval is ${withinMs}ms. Both come from one choice.`,
|
|
1010
|
+
why: 'A `withinMs` is only honest if the schedule really does reload that often — once the newest full load of a type is older than it, incremental loads of that type stop committing. Set separately, the two drift and the declaration becomes a sentence that polices nothing. Derived from one cadence, they cannot.',
|
|
1011
|
+
changeAt: 'Re-apply with a different cadence, or edit the schedule and the expectation together.',
|
|
1012
|
+
},
|
|
1013
|
+
{
|
|
1014
|
+
what: 'The interval is twice the cadence rather than equal to it.',
|
|
1015
|
+
why: 'At exactly the cadence, any run that starts a minute late — or simply takes a few minutes, since the age is measured from the snapshot — makes the type stale and blocks loads of it. A guard that fires on an ordinary Tuesday is a guard somebody switches off, taking the real failure with it. Twice tolerates one missed cycle and refuses the second, and covers the hour a DST boundary moves.',
|
|
1016
|
+
changeAt: "The type's load expectation, on the Model screen.",
|
|
1017
|
+
},
|
|
1018
|
+
{
|
|
1019
|
+
what: 'The graph reads and commits in `full` mode, and that is the point rather than a stepping stone.',
|
|
1020
|
+
why: 'This is the mode the reconciliation strategy describes. An incremental read here would be blind to deletes, which is the exact hole the declaration is claiming to have closed — and where the upstream ingest deletes and reinserts per base, every row gets a fresh timestamp in one batch anyway, so an incremental read returns the whole table while still missing the deletions.',
|
|
1021
|
+
changeAt: 'The `mode` field on the source node and on the sink node.',
|
|
1022
|
+
},
|
|
1023
|
+
],
|
|
1024
|
+
todo: [
|
|
1025
|
+
configTodo(options.sourceKind, source.name),
|
|
1026
|
+
`The schedule is part of this plan but is not part of the graph: send \`${schedule.cron}\` to \`PUT workflows/:id/schedule\` once the workflow exists, and publish it first — a schedule on a draft is stored and never fires.`,
|
|
1027
|
+
`The expectation is a separate write against \`${targetType}\`. Applying the graph without it leaves the type undeclared, which is the state this template exists to avoid.`,
|
|
1028
|
+
],
|
|
1029
|
+
},
|
|
1030
|
+
};
|
|
1031
|
+
}
|
|
1032
|
+
/**
|
|
1033
|
+
* Build whichever template was asked for, with the options that template needs.
|
|
1034
|
+
*
|
|
1035
|
+
* A discriminated request rather than one options bag with everything optional,
|
|
1036
|
+
* so asking for a join without a discriminator is a type error at the call site
|
|
1037
|
+
* instead of a refusal at run time. The refusals still exist because a form
|
|
1038
|
+
* hands over strings a user typed, and an empty string satisfies `string`.
|
|
1039
|
+
*/
|
|
1040
|
+
export function buildWorkflowTemplate(request) {
|
|
1041
|
+
switch (request.templateId) {
|
|
1042
|
+
case 'replicate-table':
|
|
1043
|
+
return replicateTable(request.options);
|
|
1044
|
+
case 'load-file-drop':
|
|
1045
|
+
return loadFileDrop(request.options);
|
|
1046
|
+
case 'fan-out-types':
|
|
1047
|
+
return fanOutTypes(request.options);
|
|
1048
|
+
case 'join-sources':
|
|
1049
|
+
return joinSources(request.options);
|
|
1050
|
+
case 'enrich-with-lookup':
|
|
1051
|
+
return enrichWithLookup(request.options);
|
|
1052
|
+
case 'periodic-full-reload':
|
|
1053
|
+
return periodicFullReload(request.options);
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
/**
|
|
1057
|
+
* Point each transform node at the transform that was created for it.
|
|
1058
|
+
*
|
|
1059
|
+
* Separate from building the plan, and pure, because the ids do not exist until
|
|
1060
|
+
* a server has stored the code — a template cannot invent them and must not
|
|
1061
|
+
* pretend to. Until this has run, a plan carrying transforms is deliberately
|
|
1062
|
+
* **not** a valid graph: `validateWorkflow` reports `transform-not-named` for
|
|
1063
|
+
* every node with an empty `transformId`, which is the correct answer, because a
|
|
1064
|
+
* node pointing at no code cannot run. See {@link planIsRunnable}.
|
|
1065
|
+
*
|
|
1066
|
+
* Nodes whose id is absent from the map are left exactly as they were, so a
|
|
1067
|
+
* partial failure part-way through creating three transforms leaves a graph that
|
|
1068
|
+
* still says which node is unfinished rather than one that silently claims to be
|
|
1069
|
+
* complete.
|
|
1070
|
+
*/
|
|
1071
|
+
export function attachTransformIds(plan, transformIdByNodeId) {
|
|
1072
|
+
return {
|
|
1073
|
+
...plan,
|
|
1074
|
+
nodes: plan.nodes.map((node) => {
|
|
1075
|
+
if (node.kind !== 'transform')
|
|
1076
|
+
return node;
|
|
1077
|
+
const id = transformIdByNodeId.get(node.id);
|
|
1078
|
+
return id === undefined ? node : { ...node, transformId: id };
|
|
1079
|
+
}),
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
/** The authored half of a plan, in the shape `saveWorkflow` takes. */
|
|
1083
|
+
export function planToWorkflowInput(plan) {
|
|
1084
|
+
return {
|
|
1085
|
+
name: plan.name,
|
|
1086
|
+
description: plan.description,
|
|
1087
|
+
nodes: plan.nodes,
|
|
1088
|
+
edges: plan.edges.map((edge) => ({ from: edge.from, to: edge.to })),
|
|
1089
|
+
};
|
|
1090
|
+
}
|
|
1091
|
+
/**
|
|
1092
|
+
* Whether every transform node in the plan has been pointed at real code.
|
|
1093
|
+
*
|
|
1094
|
+
* The one question a screen has to ask between creating the transforms and
|
|
1095
|
+
* saving the graph. Named rather than left as an inline `.every`, because "the
|
|
1096
|
+
* plan is ready to save" and "the plan validates" are different questions and
|
|
1097
|
+
* conflating them is how a save gets attempted with an empty `transformId` and
|
|
1098
|
+
* comes back as a validation error about a field nobody typed.
|
|
1099
|
+
*/
|
|
1100
|
+
export function planIsRunnable(plan) {
|
|
1101
|
+
return plan.nodes.every((node) => node.kind !== 'transform' || node.transformId.length > 0);
|
|
1102
|
+
}
|