@dudousxd/nestjs-catalog-react 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1092 @@
1
+ import { CONNECTOR_KINDS, isSafeIdentifier, physicalColumn, } from '@dudousxd/nestjs-catalog/client';
2
+ export const SOURCE_KINDS = {
3
+ http: {
4
+ label: 'an HTTP endpoint',
5
+ required: ['url'],
6
+ optional: ['headers', 'path'],
7
+ keysRecordsBySourceName: true,
8
+ },
9
+ sql: {
10
+ label: 'a SQL database',
11
+ required: ['url', 'query'],
12
+ optional: ['watermarkColumn'],
13
+ keysRecordsBySourceName: true,
14
+ },
15
+ file: {
16
+ label: 'a file',
17
+ required: ['path'],
18
+ optional: ['format', 'jsonPath', 'delimiter'],
19
+ keysRecordsBySourceName: true,
20
+ },
21
+ s3: {
22
+ label: 'an object store',
23
+ required: ['bucket'],
24
+ optional: ['prefix', 'suffix', 'format', 'delimiter', 'region', 'maxObjectsPerRun'],
25
+ keysRecordsBySourceName: true,
26
+ },
27
+ inline: {
28
+ label: 'records pasted into the graph',
29
+ required: ['records'],
30
+ optional: [],
31
+ // The keys are whatever the author typed, so there is no source spelling to
32
+ // disagree with a property name and nothing for the refusal to check.
33
+ keysRecordsBySourceName: false,
34
+ },
35
+ };
36
+ /**
37
+ * The kinds a file drop can arrive as, derived rather than listed.
38
+ *
39
+ * `CONNECTOR_KINDS` filtered by what {@link SOURCE_KINDS} says about it, so a
40
+ * new file-ish kind is picked up by having its profile filled in rather than by
41
+ * somebody remembering this line exists.
42
+ */
43
+ export const FILE_DROP_KINDS = CONNECTOR_KINDS.filter((kind) => kind === 'file' || kind === 's3');
44
+ const NODE_FACTORIES = {
45
+ source: (id, name, init) => ({ id, name, kind: 'source', ...init }),
46
+ transform: (id, name, init) => ({ id, name, kind: 'transform', ...init }),
47
+ sink: (id, name, init) => ({ id, name, kind: 'sink', ...init }),
48
+ // `call` arrived after these templates were written, and the map above is
49
+ // exactly why: the build stopped rather than the kind going quietly missing.
50
+ //
51
+ // No template below builds one, and that is a statement rather than an
52
+ // omission. A call node hands a step to a workflow this graph does not own,
53
+ // pinned by name and version — which is a decision about somebody else's
54
+ // code, and a template exists to make a decision once on the reader's behalf.
55
+ // There is nothing here to decide for them, and no way to enumerate a
56
+ // deployment's registrations to offer a choice from. The factory is complete
57
+ // so that a template CAN build one the day a case is worth prefilling.
58
+ call: (id, name, init) => ({ id, name, kind: 'call', ...init }),
59
+ };
60
+ /* -------------------------------------------------------------------------- */
61
+ /* Starter transform code */
62
+ /* -------------------------------------------------------------------------- */
63
+ /**
64
+ * What every starter body has to say out loud, and why it is said in a comment
65
+ * rather than only in a docblock nobody will be looking at.
66
+ *
67
+ * A transform node with several inbound edges receives its inputs **concatenated
68
+ * in edge order, in one array, with nothing marking which record came from
69
+ * which side**. That is the model's documented behaviour and it is what makes a
70
+ * join expressible at all — the code can see every record at once — but it is
71
+ * also the single most surprising thing about writing one. The obvious
72
+ * assumption is that two inputs arrive as two arguments, or as two arrays, and
73
+ * code written under that assumption produces a plausible-looking result from
74
+ * the wrong half of the data.
75
+ *
76
+ * So the starter code partitions the batch explicitly and the comment naming the
77
+ * behaviour sits directly above the loop that depends on it. Somebody editing
78
+ * this a month later reads it in the editor; they do not read this file.
79
+ */
80
+ const JOIN_NOTE = [
81
+ 'Both inbound edges arrive here as ONE array, concatenated in edge order.',
82
+ 'Nothing marks which source a record came from, so they are told apart by a',
83
+ 'field only one side carries. Change the discriminator if either shape does.',
84
+ ];
85
+ /** Render a note as comment lines in the language's own syntax. */
86
+ function comment(lines, language) {
87
+ const prefix = language === 'python' ? '# ' : '// ';
88
+ return lines.map((line) => `${prefix}${line}`).join('\n');
89
+ }
90
+ /**
91
+ * A list literal that reads the same in all three languages.
92
+ *
93
+ * `JSON.stringify` on each element rather than a template with quotes around it:
94
+ * a column called `O'Brien` would otherwise close the string and produce a
95
+ * starter body that does not parse, which is a confusing first impression of a
96
+ * template whose entire subject is awkward column names.
97
+ */
98
+ function quoteList(values) {
99
+ return `[${values.map((value) => JSON.stringify(value)).join(', ')}]`;
100
+ }
101
+ /**
102
+ * A keyed join over the concatenated batch, per language.
103
+ *
104
+ * Keyed by {@link TransformLanguage} so a language added to the library without
105
+ * a body here is a type error rather than a transform that saves empty and fails
106
+ * at run time — the same argument `IDENTITY_TRANSFORM` in the canvas makes.
107
+ *
108
+ * `keepUnmatched` is the one genuinely contested choice and it is why enrichment
109
+ * and joining are separate templates over identical code. Enriching against a
110
+ * lookup keeps a row whose key is not in the dictionary, because dropping it
111
+ * would mean a load silently loses every record the lookup has not heard of yet
112
+ * — which is the null-shaped failure this file exists for, wearing a different
113
+ * hat. Joining two feeds into one type drops it, because a record with no
114
+ * counterpart is not a record of the joined thing.
115
+ */
116
+ const JOIN_CODE = {
117
+ javascript: (options) => javascriptJoin(options),
118
+ // Deliberately the same body. The runner strips types and never checks them,
119
+ // so a TypeScript transform is JavaScript with annotations allowed; shipping
120
+ // an annotated copy would be two bodies to fix when the logic is wrong.
121
+ typescript: (options) => javascriptJoin(options),
122
+ python: (options) => pythonJoin(options),
123
+ };
124
+ function javascriptJoin(options) {
125
+ const { discriminator, leftKey, rightKey, keepUnmatched } = options;
126
+ const miss = keepUnmatched
127
+ ? ' // No match: the row is kept as it arrived, with nothing added.\n if (!found) return record;'
128
+ : ' // No match: this row is not a record of the joined thing, so it is dropped.\n if (!found) return undefined;';
129
+ return `${comment(JOIN_NOTE, 'javascript')}
130
+ const left = [];
131
+ const right = [];
132
+ for (const record of records) {
133
+ (Object.hasOwn(record, ${JSON.stringify(discriminator)}) ? left : right).push(record);
134
+ }
135
+
136
+ // A missing key is not a key, so a record without one is never indexed. Left it
137
+ // in and every row that is ALSO missing the key would join to it: a full result
138
+ // set, a plausible row count, entirely wrong rows. Skipping it here covers both
139
+ // sides at once — nothing is stored under a missing key, so nothing can be
140
+ // looked up by one either.
141
+ const byKey = new Map();
142
+ for (const record of right) {
143
+ const key = record[${JSON.stringify(rightKey)}];
144
+ if (key === undefined || key === null) continue;
145
+ byKey.set(key, record);
146
+ }
147
+
148
+ return left
149
+ .map((record) => {
150
+ const found = byKey.get(record[${JSON.stringify(leftKey)}]);
151
+ ${miss}
152
+ // The left record wins on any field both sides carry: it is the record this
153
+ // row is about, and the lookup is only filling in what it did not know.
154
+ return { ...found, ...record };
155
+ })
156
+ .filter((record) => record !== undefined);`;
157
+ }
158
+ function pythonJoin(options) {
159
+ const { discriminator, leftKey, rightKey, keepUnmatched } = options;
160
+ const miss = keepUnmatched
161
+ ? ' # No match: the row is kept as it arrived, with nothing added.\n out.append(record)\n continue'
162
+ : ' # No match: this row is not a record of the joined thing, so it is dropped.\n continue';
163
+ return `${comment(JOIN_NOTE, 'python')}
164
+ left = []
165
+ right = []
166
+ for record in records:
167
+ (left if ${JSON.stringify(discriminator)} in record else right).append(record)
168
+
169
+ # A missing key is not a key, so a record without one is never indexed. Left it
170
+ # in and every row that is ALSO missing the key would join to it: a full result
171
+ # set, a plausible row count, entirely wrong rows. Skipping it here covers both
172
+ # sides at once — nothing is stored under a missing key, so nothing can be
173
+ # looked up by one either.
174
+ by_key = {}
175
+ for record in right:
176
+ key = record.get(${JSON.stringify(rightKey)})
177
+ if key is None:
178
+ continue
179
+ by_key[key] = record
180
+
181
+ out = []
182
+ for record in left:
183
+ found = by_key.get(record.get(${JSON.stringify(leftKey)}))
184
+ if found is None:
185
+ ${miss}
186
+ # The left record wins on any field both sides carry: it is the record this
187
+ # row is about, and the lookup is only filling in what it did not know.
188
+ merged = dict(found)
189
+ merged.update(record)
190
+ out.append(merged)
191
+ return out`;
192
+ }
193
+ /**
194
+ * Narrow one shared batch down to the columns a single type commits.
195
+ *
196
+ * Every branch of a fan-out reads the **same** rows — a node with two outbound
197
+ * edges is read by both successors — so each branch has to do its own narrowing.
198
+ * A fan-out wired straight from one source into several sinks would commit
199
+ * identical wide rows into every type, which is not what anybody drawing that
200
+ * shape means.
201
+ */
202
+ const PROJECT_CODE = {
203
+ javascript: (typeName, keep) => javascriptProject(typeName, keep),
204
+ typescript: (typeName, keep) => javascriptProject(typeName, keep),
205
+ python: (typeName, keep) => `${comment(projectNote(typeName), 'python')}
206
+ keep = ${quoteList(keep)}
207
+ return [{key: record.get(key) for key in keep} for record in records]`,
208
+ };
209
+ function projectNote(typeName) {
210
+ return [
211
+ `Keep only the columns ${typeName} commits.`,
212
+ 'Every branch of this graph reads the SAME source rows, so each one narrows',
213
+ 'them itself. These names are property names, and the warehouse looks each',
214
+ 'field up by property name — so what this returns has to be keyed by them.',
215
+ ];
216
+ }
217
+ function javascriptProject(typeName, keep) {
218
+ return `${comment(projectNote(typeName), 'javascript')}
219
+ const keep = ${quoteList(keep)};
220
+ return records.map((record) =>
221
+ Object.fromEntries(keep.map((key) => [key, record[key]])),
222
+ );`;
223
+ }
224
+ /* -------------------------------------------------------------------------- */
225
+ /* The identifier refusal */
226
+ /* -------------------------------------------------------------------------- */
227
+ /**
228
+ * Whether a column name could be published as a property name at all.
229
+ *
230
+ * `isSafeIdentifier(physicalColumn(name))` — the same two calls, in the same
231
+ * order, that `identifierRefusal` makes at publish time and that every store
232
+ * makes on the way to a column. Both come from `@dudousxd/nestjs-catalog/client`
233
+ * and are the functions themselves, not a copy of either: a canvas whose idea of
234
+ * a legal name differs from the server's by one character is a canvas that
235
+ * promises a load the publisher then refuses, or refuses one it would have
236
+ * taken.
237
+ *
238
+ * **It is the composition and not `isSafeIdentifier` alone**, and getting that
239
+ * wrong in either direction is a real failure rather than a nicety. Asking only
240
+ * `isSafeIdentifier(name)` is the obsolete, stricter question — it refuses
241
+ * `Asset Id`, which now works end to end, and the operator sent away by that
242
+ * refusal is being sent to do the rename that writes nulls. Asking only whether
243
+ * the cleaning *ran* would pass everything, since `physicalColumn` always
244
+ * returns something. What has to be asked is whether what it returns is a name a
245
+ * store can create.
246
+ *
247
+ * **Which is a narrow set, and worth naming exactly so nobody re-derives it.**
248
+ * The cleaning already guarantees the character set and the length, so the only
249
+ * things left for `isSafeIdentifier` to reject are a cleaned form that starts
250
+ * with a digit and one that is empty. In practice: `2024 Total` → `2024_Total`
251
+ * is refused; `FY24 Hours` → `FY24_Hours` is not, because the digits are not
252
+ * first; `#Rank` → `_Rank` is not, because punctuation becomes an underscore and
253
+ * an underscore may lead; `Asset LIN/TAMCN` → `Asset_LIN_TAMCN` is not. A column
254
+ * named with the empty string is the other refusal, and a source producing one
255
+ * has a problem this template is not the right place to explain.
256
+ */
257
+ function canBePropertyName(column) {
258
+ return isSafeIdentifier(physicalColumn(column));
259
+ }
260
+ /**
261
+ * Refuse the column names that cannot become property names, or say nothing.
262
+ *
263
+ * The single most important function in this file, and the one place the naming
264
+ * rule is applied. See the header for the full argument; the short version is
265
+ * that with no transform on the path a record arrives keyed by the source's own
266
+ * spelling and the store asks for `row[property.name]`, so the two have to be
267
+ * the same string — and while that string no longer has to be a SQL identifier
268
+ * itself, it still has to *clean* to one.
269
+ *
270
+ * Three outcomes rather than two, and the third is not a formality:
271
+ *
272
+ * - Every column is usable — nothing is returned. This is now the answer for
273
+ * `Asset Id` and `Asset LIN/TAMCN`, which it was not before
274
+ * `fix/view-alias-sanitised` landed.
275
+ * - Some are not — a refusal naming every one of them. All of them, not the
276
+ * first, because a legacy export is usually wrong about forty columns in the
277
+ * same way and one refusal per attempt would make fixing it a morning.
278
+ * - **Nobody knows what the columns are** — also a refusal, with its own code,
279
+ * and it survived the alias fix on purpose. The set it protects against got
280
+ * much smaller and did not empty: a source with a `2024 Total` column, applied
281
+ * without anybody looking, still ends with that property refused at publish,
282
+ * renamed to `2024_Total` by the obvious next move, and loading null into
283
+ * every row of every run while reporting success. That is the original
284
+ * disaster, still reachable, one column narrower. A template that proceeded on
285
+ * an unknown column list would be asserting the names are fine because nobody
286
+ * looked. Silence is not a pass.
287
+ */
288
+ export function refuseUnusableColumnNames(columns, context) {
289
+ if (columns === undefined) {
290
+ return {
291
+ code: 'columns-unknown',
292
+ subjects: [],
293
+ message: `Nobody has said which columns ${context.what} produces, so this template cannot check the one thing that would make it commit nulls. With no transform on the path a record arrives keyed by the source's own column spelling and the warehouse looks every field up by property name, so the two have to match exactly — and while a property name may now be spelled however the source spells it, it still has to clean to something a store can create as a column. Run schema discovery on the source first, or ${context.remedy}`,
294
+ };
295
+ }
296
+ const offenders = columns.filter((column) => !canBePropertyName(column));
297
+ if (offenders.length === 0)
298
+ return undefined;
299
+ return {
300
+ code: 'columns-not-identifiers',
301
+ subjects: offenders,
302
+ message: `${offenders.length === 1 ? 'One column' : `${offenders.length} columns`} of ${context.what} ${offenders.length === 1 ? 'is' : 'are'} spelled in a way that cannot be a property name: ${offenders.map((name) => `${JSON.stringify(name)} (cleans to ${JSON.stringify(physicalColumn(name))})`).join(', ')}. A property name does not have to be a SQL identifier — a store cleans it, so \`Asset Id\` is a perfectly good name and lands in \`Asset_Id\` — but what the cleaning produces still has to be one, and ${offenders.length === 1 ? 'this one is not' : 'these are not'}. Publishing refuses the name, and the rename reached for instead is the trap: the warehouse matches records to properties by name, so a property called \`${physicalColumn(offenders[0])}\` while the record still arrives keyed \`${offenders[0]}\` writes null into that column on every run and reports success. That is what happened to six types in one evening, so this template refuses rather than guessing which way you meant. ${context.remedy}`,
303
+ };
304
+ }
305
+ /** The remedy sentence, which is the same one every no-transform template gives. */
306
+ const RENAME_REMEDY = "Put a transform between the source and the sink whose job is to rename the record keys to the property names — the model's own answer, and the reason `columnName` exists is to record what the source called it. For these columns the rename is genuinely unavoidable, which is why it is worth writing rather than a chore: there is no spelling that is both what the source sends and something a store can create.";
307
+ /* -------------------------------------------------------------------------- */
308
+ /* Cadences */
309
+ /* -------------------------------------------------------------------------- */
310
+ /**
311
+ * The cadences a periodic full reload may run at, each carrying both halves.
312
+ *
313
+ * **This closed list is the template.** A cron expression and a `withinMs` are
314
+ * two statements about one cadence, and today nothing links them: an operator
315
+ * sets a schedule on one screen and a reconciliation interval on another, and
316
+ * nothing anywhere notices when they stop agreeing. `withinMs` is only honest if
317
+ * the schedule really does reload that often — once the newest full load of a
318
+ * type is older than it, incremental loads of that type stop committing — so a
319
+ * `withinMs` larger than the real cadence is a guard that never fires and a
320
+ * smaller one is a guard that fires constantly.
321
+ *
322
+ * Deriving both from one entry makes disagreement impossible. Somebody wanting a
323
+ * cadence that is not here has to add it here, in code, where both halves are on
324
+ * the same line and a reviewer sees them together — which is the whole doctrine
325
+ * of this file applied to itself.
326
+ *
327
+ * The cron is evaluated in whatever `CATALOG_SCHEDULE_TZ` names and `everyMs` is
328
+ * wall-clock, so the two drift by an hour twice a year across a DST boundary.
329
+ * The slack in {@link withinMsFor} absorbs that; it is called out here because a
330
+ * reader comparing `86_400_000` against `0 2 * * *` deserves to know it is not
331
+ * an exact identity.
332
+ */
333
+ export const RELOAD_CADENCES = {
334
+ hourly: { cron: '0 * * * *', everyMs: 3_600_000, human: 'every hour' },
335
+ 'six-hourly': { cron: '0 */6 * * *', everyMs: 21_600_000, human: 'every six hours' },
336
+ daily: { cron: '0 2 * * *', everyMs: 86_400_000, human: 'every day at 02:00' },
337
+ weekly: { cron: '0 2 * * 0', everyMs: 604_800_000, human: 'every Sunday at 02:00' },
338
+ };
339
+ export const RELOAD_CADENCE_IDS = Object.keys(RELOAD_CADENCES);
340
+ /**
341
+ * How stale a type may get before its reconciliation claim stops being true.
342
+ *
343
+ * Twice the cadence, and the factor is a decision rather than a round number.
344
+ * At exactly the cadence, any run that starts a minute late — or simply takes a
345
+ * few minutes, since the age is measured from the snapshot's creation — makes
346
+ * the type stale and stops incremental loads of it committing. A guard that
347
+ * fires on an ordinary Tuesday is a guard somebody switches off, taking the real
348
+ * failure with it.
349
+ *
350
+ * Twice means one missed cycle is tolerated and the second consecutive miss is
351
+ * refused, which is the smallest slack that distinguishes "late" from "not
352
+ * happening". It also covers the DST hour noted on {@link RELOAD_CADENCES}.
353
+ */
354
+ export function withinMsFor(cadence) {
355
+ return RELOAD_CADENCES[cadence].everyMs * 2;
356
+ }
357
+ /* -------------------------------------------------------------------------- */
358
+ /* Node construction */
359
+ /* -------------------------------------------------------------------------- */
360
+ /**
361
+ * A node id that is stable across re-applications of the same template.
362
+ *
363
+ * Deliberately derived from what the node is *for* rather than randomised. A
364
+ * node id becomes a durable step name and part of the key its staged rows are
365
+ * stored under, so an id that changes between one application and the next
366
+ * re-executes work a replay would otherwise have skipped. The canvas randomises
367
+ * ids for nodes somebody draws by hand, which is right for those — there is no
368
+ * meaning to preserve — and wrong for these.
369
+ *
370
+ * `WORKFLOW_NODE_ID_PATTERN` allows letters, digits, underscore and hyphen up to
371
+ * 64 characters; anything else is folded to `_` and the whole thing is bounded,
372
+ * with an ordinal appended so that two types whose names differ only outside the
373
+ * alphabet cannot collide into one id.
374
+ */
375
+ function nodeId(prefix, subject, ordinal) {
376
+ const cleaned = subject.replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 40);
377
+ return `${prefix}_${cleaned}_${ordinal}`.slice(0, 64);
378
+ }
379
+ /** Placeholder config for a source kind, from the profile rather than by hand. */
380
+ function placeholderConfig(kind) {
381
+ const config = {};
382
+ for (const key of SOURCE_KINDS[kind].required)
383
+ config[key] = '';
384
+ return config;
385
+ }
386
+ function configTodo(kind, nodeName) {
387
+ const profile = SOURCE_KINDS[kind];
388
+ return `"${nodeName}" reads ${profile.label} and needs ${profile.required
389
+ .map((key) => `\`${key}\``)
390
+ .join(' and ')} filled in — the template ships ${profile.required.length === 1 ? 'it' : 'them'} blank on purpose, because a placeholder that looks like a real address is worse than an empty one.`;
391
+ }
392
+ /**
393
+ * The declaration every template here makes about load mode, stated once.
394
+ *
395
+ * Shared rather than repeated because it is the same decision for the same
396
+ * reason in all six, and a reader comparing two templates should be able to see
397
+ * that it is literally the same sentence rather than having to diff two
398
+ * paraphrases.
399
+ */
400
+ const FULL_MODE_DECLARATION = {
401
+ what: 'This graph reads and commits in `full` mode.',
402
+ why: 'An incremental load is blind to deletes — a row removed at the source never changes again, so it is never returned again and is carried forward into every later snapshot indefinitely, with nothing going wrong at any single step. Because that failure is silent the pipeline refuses an incremental load of a type that has not declared how deletes reach it, and it also needs a watermark column no template can know. Offering `incremental` here would mean shipping a graph that fails at its first run, or inventing a declaration on your behalf.',
403
+ changeAt: 'The `mode` field on the source node and on the sink node.',
404
+ };
405
+ /**
406
+ * The catalogue, keyed by id so that a new id without metadata fails the build.
407
+ *
408
+ * Shipped by the library rather than stored per deployment, deliberately: these
409
+ * are decisions, decisions belong in code where they are reviewed and carry
410
+ * their reasoning, and a template stored in a database is one nobody reviews.
411
+ * Per-deployment templates are a store concern and a separate change, and this
412
+ * sentence is here so that the absence reads as a decision rather than an
413
+ * oversight.
414
+ */
415
+ export const WORKFLOW_TEMPLATES = {
416
+ 'replicate-table': {
417
+ id: 'replicate-table',
418
+ title: 'Replicate a table',
419
+ summary: 'One SQL query straight into one object type, with no transform in between.',
420
+ assumes: [
421
+ 'The query returns one row per record of the type, already shaped the way the type wants it.',
422
+ 'Every column it returns is spelled in a way that can be a property name — which allows `Asset Id`, and not `2024 Total`. The template checks this and refuses if not.',
423
+ ],
424
+ },
425
+ 'load-file-drop': {
426
+ id: 'load-file-drop',
427
+ title: 'Load a file drop',
428
+ summary: 'A CSV, NDJSON or JSON drop — on disk or in an object store — into one object type.',
429
+ assumes: [
430
+ 'The drop is a full extract each time, not a delta.',
431
+ 'Its header row is spelled in a way that can be a property name — which allows `Asset LIN/TAMCN`, and not a column headed with a year. The template checks this and refuses if not.',
432
+ ],
433
+ },
434
+ 'fan-out-types': {
435
+ id: 'fan-out-types',
436
+ title: 'Fan one source into several types',
437
+ summary: 'One expensive read, narrowed by a transform per type into a sink per type.',
438
+ assumes: [
439
+ 'The source is worth reading once rather than once per type — that is the entire reason for this shape.',
440
+ 'Each type is a projection of the same rows. A type needing different rows needs its own source.',
441
+ ],
442
+ },
443
+ 'join-sources': {
444
+ id: 'join-sources',
445
+ title: 'Join two sources into one type',
446
+ summary: 'Two reads, joined on a key inside one transform, committed as a single type.',
447
+ assumes: [
448
+ 'One field tells the two sides apart, because the runner concatenates them into one untagged batch.',
449
+ 'The whole of both sides fits in memory at once — the transform contract is set-wise by design.',
450
+ ],
451
+ },
452
+ 'enrich-with-lookup': {
453
+ id: 'enrich-with-lookup',
454
+ title: 'Enrich against a lookup table',
455
+ summary: 'A main feed plus a dictionary, joined so that unmatched rows survive.',
456
+ assumes: [
457
+ 'The lookup is small enough to hold beside the feed it enriches.',
458
+ 'A row whose key is not in the lookup is still a row worth keeping. That is the difference from a join.',
459
+ ],
460
+ },
461
+ 'periodic-full-reload': {
462
+ id: 'periodic-full-reload',
463
+ title: 'Periodic full reload',
464
+ summary: 'A full read on a schedule, with the matching delete-reconciliation declared from the same cadence.',
465
+ assumes: [
466
+ 'A full read is the right mode here rather than a stepping stone to an incremental one.',
467
+ 'The source can be read in its entirety at the chosen cadence without hurting it.',
468
+ ],
469
+ },
470
+ };
471
+ export const WORKFLOW_TEMPLATE_IDS = Object.keys(WORKFLOW_TEMPLATES);
472
+ /**
473
+ * SQL straight into a type — the named case, and the one that produced the
474
+ * nulls.
475
+ *
476
+ * The shape is trivial and the value is entirely in the refusal. Two nodes and
477
+ * one edge is not something anybody needs help drawing; what they need help with
478
+ * is knowing that this shape is only correct when every column the query returns
479
+ * can be a property name, and that when it is not, the graph still saves, still
480
+ * runs, still commits and still reports the right row count while writing null
481
+ * into most of the columns.
482
+ *
483
+ * So this template checks, and refuses, and names every offending column. It
484
+ * does **not** offer to sanitise them: that is the naive fix and it is what was
485
+ * done six times in one evening. For the columns that now reach this refusal
486
+ * there is no spelling that is both the key the record arrives under and a name
487
+ * a store can create, so the rename has to be paired with a transform that
488
+ * renames the record's keys — which is a decision about the data rather than a
489
+ * shape, and belongs to whoever knows the data.
490
+ *
491
+ * **What it no longer refuses.** `Asset Id` and `Asset LIN/TAMCN` pass now.
492
+ * `fix/view-alias-sanitised` made the view's output alias and the read's lookup
493
+ * key clean together, so a property keeps the source's spelling end to end and a
494
+ * two-node graph over those columns is simply correct. Refusing them would be
495
+ * this template sending somebody off to do the exact rename that caused the
496
+ * incident it exists to prevent. See the header.
497
+ */
498
+ export function replicateTable(options) {
499
+ const targetType = options.targetType.trim();
500
+ if (targetType.length === 0) {
501
+ return {
502
+ ok: false,
503
+ refusals: [
504
+ {
505
+ code: 'incomplete',
506
+ subjects: [],
507
+ message: 'This template needs the object type the sink will commit. The type is chosen on the sink because the sink is the node that commits.',
508
+ },
509
+ ],
510
+ };
511
+ }
512
+ const refusal = refuseUnusableColumnNames(options.sourceColumns, {
513
+ what: 'the query',
514
+ remedy: RENAME_REMEDY,
515
+ });
516
+ if (refusal)
517
+ return { ok: false, refusals: [refusal] };
518
+ const source = NODE_FACTORIES.source(nodeId('source', targetType, 1), `${targetType} query`, {
519
+ sourceKind: 'sql',
520
+ config: { ...placeholderConfig('sql'), ...(options.query ? { query: options.query } : {}) },
521
+ ...(options.connectionId ? { connectionId: options.connectionId } : {}),
522
+ mode: 'full',
523
+ });
524
+ const sink = NODE_FACTORIES.sink(nodeId('sink', targetType, 1), targetType, {
525
+ targetType,
526
+ mode: 'full',
527
+ });
528
+ return {
529
+ ok: true,
530
+ plan: {
531
+ templateId: 'replicate-table',
532
+ name: options.workflowName?.trim() || `Replicate ${targetType}`,
533
+ description: `Reads ${targetType} from a SQL source in one query and commits it whole.`,
534
+ nodes: [source, sink],
535
+ edges: [{ from: source.id, to: sink.id }],
536
+ transforms: [],
537
+ expectations: [],
538
+ declarations: [
539
+ FULL_MODE_DECLARATION,
540
+ {
541
+ what: 'No transform sits between the source and the sink.',
542
+ why: 'Every column the query returns was checked with the same two calls the store and the publish API make — cleaning the name, then asking whether the result is an identifier — and all of them can be property names, so the records arrive keyed exactly as the type will ask for them. Name the properties as the query spells them; a column called `Asset Id` wants a property called `Asset Id`, not `Asset_Id`. Had any column failed, this template would have refused instead of adding a transform, because the rename it would have to write is a decision about your data rather than a shape.',
543
+ changeAt: 'Add a transform node on the edge if the mapping ever stops being one-to-one.',
544
+ },
545
+ ],
546
+ todo: [
547
+ configTodo('sql', source.name),
548
+ `The sink commits \`${targetType}\`. Publish that type first if it does not exist — a sink naming a type nobody has published fails at the commit, after the whole source has been read.`,
549
+ ],
550
+ },
551
+ };
552
+ }
553
+ /**
554
+ * A DoD/DPAS-style drop into a type.
555
+ *
556
+ * Structurally the same two nodes as {@link replicateTable} and separated from
557
+ * it because the config, the assumptions and the failure mode differ enough that
558
+ * one template covering both would have to hedge every sentence. A drop is a
559
+ * full extract that lands on a schedule somebody else controls, its columns come
560
+ * from a spreadsheet header rather than a `SELECT` list, and a spreadsheet header
561
+ * is overwhelmingly the most likely place to meet a column headed `2024 Total`
562
+ * or `2024 Q1 Hours` — which is why the same refusal matters more here, not less.
563
+ *
564
+ * `Asset LIN/TAMCN`, the header this template was written around, is no longer
565
+ * one of them: it cleans to `Asset_LIN_TAMCN` and is published under its own
566
+ * spelling. Nor is `FY24 Hours`, which cleans to `FY24_Hours` — the year has to
567
+ * be at the *front* to be a problem. What is refused is a header whose cleaned
568
+ * form starts with a digit, and that really does need a transform.
569
+ */
570
+ export function loadFileDrop(options) {
571
+ const targetType = options.targetType.trim();
572
+ if (targetType.length === 0) {
573
+ return {
574
+ ok: false,
575
+ refusals: [
576
+ {
577
+ code: 'incomplete',
578
+ subjects: [],
579
+ message: 'This template needs the object type the sink will commit.',
580
+ },
581
+ ],
582
+ };
583
+ }
584
+ const refusal = refuseUnusableColumnNames(options.sourceColumns, {
585
+ what: "the drop's header row",
586
+ remedy: RENAME_REMEDY,
587
+ });
588
+ if (refusal)
589
+ return { ok: false, refusals: [refusal] };
590
+ const source = NODE_FACTORIES.source(nodeId('source', targetType, 1), `${targetType} drop`, {
591
+ sourceKind: options.kind,
592
+ config: placeholderConfig(options.kind),
593
+ ...(options.connectionId ? { connectionId: options.connectionId } : {}),
594
+ mode: 'full',
595
+ });
596
+ const sink = NODE_FACTORIES.sink(nodeId('sink', targetType, 1), targetType, {
597
+ targetType,
598
+ mode: 'full',
599
+ });
600
+ return {
601
+ ok: true,
602
+ plan: {
603
+ templateId: 'load-file-drop',
604
+ name: options.workflowName?.trim() || `Load ${targetType} drop`,
605
+ description: `Reads a ${targetType} drop from ${SOURCE_KINDS[options.kind].label} and commits it whole.`,
606
+ nodes: [source, sink],
607
+ edges: [{ from: source.id, to: sink.id }],
608
+ transforms: [],
609
+ expectations: [],
610
+ declarations: [
611
+ FULL_MODE_DECLARATION,
612
+ {
613
+ what: 'Each run treats the drop as the complete dataset.',
614
+ why: 'A drop is replaced rather than appended to, and a full commit repoints the view atomically, so a run that reads a short file replaces the good data with the short file. That is the failure worth knowing about here — the row-count bound on the type is what catches it, and it is worth setting before this runs unattended.',
615
+ changeAt: "The type's load expectation, on the Model screen.",
616
+ },
617
+ ],
618
+ todo: [
619
+ configTodo(options.kind, source.name),
620
+ options.kind === 's3'
621
+ ? 'Set `prefix` and `suffix` so a run reads only the drop and not everything else in the bucket. Without them every object under the bucket is read on every run.'
622
+ : 'Set `format` if the extension does not say — a `.txt` holding CSV is parsed as JSON and yields nothing.',
623
+ ],
624
+ },
625
+ };
626
+ }
627
+ /**
628
+ * One expensive read feeding several types.
629
+ *
630
+ * The reason the model allows several sinks at all: pulling ten million rows
631
+ * once and deriving vehicles and work orders from them, rather than pulling
632
+ * twice. The validator endorses it explicitly and guards the one genuinely
633
+ * ambiguous case — two sinks writing the *same* type, which would be two
634
+ * snapshots of one type in one run with nothing to say which readers should get.
635
+ * This template refuses that up front rather than letting the canvas report it,
636
+ * because at the point of choosing it is a typo and after applying it is a graph
637
+ * to unpick.
638
+ *
639
+ * **A transform per branch, not one shared transform.** A node with two outbound
640
+ * edges is read by both successors, so a single transform feeding two sinks
641
+ * would commit identical wide rows into both types. The narrowing has to happen
642
+ * per branch, which is what makes this a fan-*out* rather than a duplication.
643
+ */
644
+ export function fanOutTypes(options) {
645
+ const refusals = [];
646
+ const targets = options.targets.filter((target) => target.targetType.trim().length > 0);
647
+ if (targets.length < 2) {
648
+ refusals.push({
649
+ code: 'incomplete',
650
+ subjects: [],
651
+ message: 'This template needs at least two object types. With one type it is a plain source-to-sink graph, and "Replicate a table" is that graph with a check this one cannot make.',
652
+ });
653
+ }
654
+ const seen = new Set();
655
+ const duplicated = new Set();
656
+ for (const target of targets) {
657
+ const name = target.targetType.trim();
658
+ if (seen.has(name))
659
+ duplicated.add(name);
660
+ seen.add(name);
661
+ }
662
+ if (duplicated.size > 0) {
663
+ refusals.push({
664
+ code: 'duplicate-target-type',
665
+ subjects: [...duplicated],
666
+ message: `Two branches both commit ${[...duplicated].map((name) => JSON.stringify(name)).join(', ')}. Two snapshots of one type in a single run leaves nothing to say which one readers should get, so the graph would be refused — send them to different types, or narrow them in one branch instead of two.`,
667
+ });
668
+ }
669
+ for (const target of targets) {
670
+ const refusal = refuseUnusableColumnNames(target.properties, {
671
+ what: `the properties of ${target.targetType.trim()}`,
672
+ remedy: "A property name is cleaned into the column a store creates for it, and what the cleaning produces has to be an identifier — so publishing refuses this one. Rename it in the type, and record the source spelling in that property's `columnName`. This branch already has a transform on it, so make that transform emit the new key too.",
673
+ });
674
+ if (refusal)
675
+ refusals.push(refusal);
676
+ }
677
+ if (refusals.length > 0)
678
+ return { ok: false, refusals };
679
+ const source = NODE_FACTORIES.source(nodeId('source', 'shared', 1), 'Shared read', {
680
+ sourceKind: options.sourceKind,
681
+ config: placeholderConfig(options.sourceKind),
682
+ ...(options.connectionId ? { connectionId: options.connectionId } : {}),
683
+ mode: 'full',
684
+ });
685
+ const nodes = [source];
686
+ const edges = [];
687
+ const transforms = [];
688
+ targets.forEach((target, index) => {
689
+ const typeName = target.targetType.trim();
690
+ const transform = NODE_FACTORIES.transform(nodeId('transform', typeName, index + 1), `Narrow to ${typeName}`, { transformId: '' });
691
+ const sink = NODE_FACTORIES.sink(nodeId('sink', typeName, index + 1), typeName, {
692
+ targetType: typeName,
693
+ mode: 'full',
694
+ });
695
+ nodes.push(transform, sink);
696
+ edges.push({ from: source.id, to: transform.id }, { from: transform.id, to: sink.id });
697
+ transforms.push({
698
+ forNodeId: transform.id,
699
+ name: `Narrow to ${typeName}`,
700
+ language: options.language,
701
+ code: PROJECT_CODE[options.language](typeName, target.properties),
702
+ why: `Keeps only the columns ${typeName} commits, out of the rows every branch of this graph shares.`,
703
+ });
704
+ });
705
+ return {
706
+ ok: true,
707
+ plan: {
708
+ templateId: 'fan-out-types',
709
+ name: options.workflowName?.trim() ||
710
+ `Split into ${targets.map((t) => t.targetType.trim()).join(', ')}`,
711
+ description: `Reads once and derives ${targets.length} types from the same rows.`,
712
+ nodes,
713
+ edges,
714
+ transforms,
715
+ expectations: [],
716
+ declarations: [
717
+ FULL_MODE_DECLARATION,
718
+ {
719
+ what: 'Each type gets its own transform, rather than sharing one.',
720
+ why: 'Both successors of the source read the same rows, so one shared transform would commit identical wide rows into every type. Narrowing per branch is what makes this a fan-out and not a duplication.',
721
+ changeAt: 'The code on each transform node, in the editor.',
722
+ },
723
+ {
724
+ what: 'Each sink commits independently.',
725
+ why: 'There is no distributed transaction across sinks and the model does not pretend otherwise: a run that commits one type and fails on another is a failed run that has nonetheless written something. That is the price of the single read, and it is the same thing two separate connectors would do today.',
726
+ changeAt: 'Split into one workflow per type if a partial commit is unacceptable.',
727
+ },
728
+ ],
729
+ todo: [
730
+ configTodo(options.sourceKind, source.name),
731
+ `${transforms.length} transforms have to be created before this graph will run — their nodes name no code until they exist.`,
732
+ 'Check each projection against what the type actually publishes: a property name in the list that the type does not have writes a column nobody reads, and one the type has that is missing from the list is committed as null.',
733
+ ],
734
+ },
735
+ };
736
+ }
737
+ /**
738
+ * Two reads into one type, joined inside the transform.
739
+ *
740
+ * Expressible only because the transform contract is set-wise: the code is a
741
+ * function over a whole batch precisely so that it can deduplicate, aggregate
742
+ * and join, none of which can be done a row at a time.
743
+ *
744
+ * **The decision this template exists to make is what the batch looks like.** A
745
+ * node with several inbound edges receives its inputs *concatenated in edge
746
+ * order, in one array, with nothing marking which record came from which side*.
747
+ * The natural assumption is two arrays or two arguments, and code written under
748
+ * that assumption silently processes the wrong half. So the starter body
749
+ * partitions the batch explicitly, with the reason in a comment directly above
750
+ * the loop, and the template asks for the discriminator up front rather than
751
+ * leaving somebody to discover the problem from a result that is merely odd.
752
+ *
753
+ * Edge order is not decoration: it is what decides which side is left, and it is
754
+ * part of the graph fingerprint, so swapping the two inputs is a new version of
755
+ * the graph. The left source is added first, deliberately and stably.
756
+ */
757
+ export function joinSources(options) {
758
+ return buildJoin(options, 'join-sources');
759
+ }
760
+ /**
761
+ * A feed plus a dictionary, where a row the dictionary has not heard of
762
+ * survives.
763
+ *
764
+ * The same graph and the same code as {@link joinSources} with one flag flipped,
765
+ * and it is a separate template because that flag is the decision. Enriching
766
+ * against a lookup and joining two feeds look identical on a canvas and differ
767
+ * entirely in what happens to a record with no counterpart: dropping unmatched
768
+ * rows from an enrichment means a load silently loses every record whose key the
769
+ * dictionary has not caught up with yet — a load that succeeds, reports a
770
+ * plausible row count, and is missing data. That is the same shape of failure as
771
+ * the null columns, and it is the reason the two are offered as two choices
772
+ * rather than one template with a checkbox nobody reads.
773
+ *
774
+ * Concretely useful here: this deployment has a `unit_dictionary` type and a
775
+ * unit-normalisation problem it was built for.
776
+ */
777
+ export function enrichWithLookup(options) {
778
+ return buildJoin(options, 'enrich-with-lookup');
779
+ }
780
+ /**
781
+ * The fields a join cannot be built without, in the words the refusal uses.
782
+ *
783
+ * Split out so that adding a field is one line in one list rather than another
784
+ * branch in a function that is already deciding four other things — and so that
785
+ * the refusal names *every* blank field at once. A form that reports its
786
+ * required fields one round trip at a time is a form people abandon.
787
+ */
788
+ function missingJoinFields(options) {
789
+ const required = [
790
+ [options.targetType, 'the object type the sink commits'],
791
+ [options.discriminator, 'the field that tells the two sides apart'],
792
+ [options.leftKey, 'the key on the left-hand side'],
793
+ [options.rightKey, 'the key on the right-hand side'],
794
+ ];
795
+ return required.filter(([value]) => value.trim().length === 0).map(([, described]) => described);
796
+ }
797
+ function buildJoin(options, templateId) {
798
+ const enriching = templateId === 'enrich-with-lookup';
799
+ const targetType = options.targetType.trim();
800
+ const missing = missingJoinFields(options);
801
+ if (missing.length > 0) {
802
+ return {
803
+ ok: false,
804
+ refusals: [
805
+ {
806
+ code: 'incomplete',
807
+ subjects: missing,
808
+ message: `This template still needs ${missing.join(', ')}. The discriminator especially is not optional: the runner hands both inputs to the transform as one concatenated array with nothing marking which side a record came from, so without a field that only one side carries there is no way for the code to tell them apart — and code that guesses produces a plausible result from the wrong half of the data.`,
809
+ },
810
+ ],
811
+ };
812
+ }
813
+ const leftName = options.leftName?.trim() || (enriching ? 'Main feed' : 'Left source');
814
+ const rightName = options.rightName?.trim() || (enriching ? 'Lookup' : 'Right source');
815
+ const left = NODE_FACTORIES.source(nodeId('source', 'left', 1), leftName, {
816
+ sourceKind: options.leftSourceKind,
817
+ config: placeholderConfig(options.leftSourceKind),
818
+ mode: 'full',
819
+ });
820
+ const right = NODE_FACTORIES.source(nodeId('source', 'right', 2), rightName, {
821
+ sourceKind: options.rightSourceKind,
822
+ config: placeholderConfig(options.rightSourceKind),
823
+ mode: 'full',
824
+ });
825
+ const transform = NODE_FACTORIES.transform(nodeId('transform', targetType, 1), enriching ? `Enrich ${targetType}` : `Join into ${targetType}`, { transformId: '' });
826
+ const sink = NODE_FACTORIES.sink(nodeId('sink', targetType, 1), targetType, {
827
+ targetType,
828
+ mode: 'full',
829
+ });
830
+ const code = JOIN_CODE[options.language]({
831
+ discriminator: options.discriminator.trim(),
832
+ leftKey: options.leftKey.trim(),
833
+ rightKey: options.rightKey.trim(),
834
+ keepUnmatched: enriching,
835
+ });
836
+ return {
837
+ ok: true,
838
+ plan: {
839
+ templateId,
840
+ name: options.workflowName?.trim() ||
841
+ (enriching ? `Enrich ${targetType}` : `Join into ${targetType}`),
842
+ description: enriching
843
+ ? `Reads ${leftName} and ${rightName}, fills each row in from the lookup, and commits ${targetType}.`
844
+ : `Reads ${leftName} and ${rightName}, joins them on a key, and commits ${targetType}.`,
845
+ nodes: [left, right, transform, sink],
846
+ // Left first. Edge order decides which side is which and is part of the
847
+ // graph fingerprint, so this ordering is load-bearing rather than tidy.
848
+ edges: [
849
+ { from: left.id, to: transform.id },
850
+ { from: right.id, to: transform.id },
851
+ { from: transform.id, to: sink.id },
852
+ ],
853
+ transforms: [
854
+ {
855
+ forNodeId: transform.id,
856
+ name: enriching ? `Enrich ${targetType}` : `Join into ${targetType}`,
857
+ language: options.language,
858
+ code,
859
+ why: enriching
860
+ ? `Splits the concatenated batch, indexes ${rightName} by \`${options.rightKey.trim()}\`, and fills in each ${leftName} row. A row with no match is kept.`
861
+ : `Splits the concatenated batch, indexes ${rightName} by \`${options.rightKey.trim()}\`, and joins. A row with no match is dropped.`,
862
+ },
863
+ ],
864
+ expectations: [],
865
+ declarations: [
866
+ FULL_MODE_DECLARATION,
867
+ {
868
+ what: `Both inputs are read into one batch and told apart by \`${options.discriminator.trim()}\`.`,
869
+ why: 'A transform node receives its inbound edges concatenated in edge order, in one array, with nothing marking the origin of a record. That is what makes a join possible at all — the code can see everything at once — but it means the split has to be written explicitly, and a discriminator chosen by somebody who knows both shapes is the only reliable way to do it.',
870
+ changeAt: 'The partition at the top of the transform body.',
871
+ },
872
+ enriching
873
+ ? {
874
+ what: 'A row whose key is not in the lookup is kept, unenriched.',
875
+ why: 'Dropping it would mean the load silently loses every record the dictionary has not caught up with — a run that succeeds, reports a plausible count and is quietly missing data. If an unmatched row genuinely is not a record of this type, use "Join two sources" instead, which drops it and says so.',
876
+ changeAt: 'The `if (!found)` branch in the transform body.',
877
+ }
878
+ : {
879
+ what: 'A row whose key has no counterpart is dropped.',
880
+ why: 'A record with nothing on the other side is not a record of the joined thing. If it is — if the right-hand side is a dictionary rather than a second feed — use "Enrich against a lookup table" instead, which keeps it.',
881
+ changeAt: 'The `if (!found)` branch in the transform body.',
882
+ },
883
+ {
884
+ what: 'The left record wins where both sides carry the same field.',
885
+ why: 'The left side is the record the row is about; the other side is filling in what it did not know. The reverse would let a stale dictionary overwrite live values, which is unrecoverable once committed.',
886
+ changeAt: 'The spread order on the last line of the transform body.',
887
+ },
888
+ ],
889
+ todo: [
890
+ configTodo(options.leftSourceKind, left.name),
891
+ configTodo(options.rightSourceKind, right.name),
892
+ 'The transform has to be created before this graph will run — its node names no code until it exists.',
893
+ `Both sides are held in memory at once while the transform runs. That is inherent to a set-wise contract, but it means ${rightName} wants to be the smaller of the two.`,
894
+ ],
895
+ },
896
+ };
897
+ }
898
+ /**
899
+ * A full read on a schedule, with the reconciliation it implies declared from
900
+ * the same input.
901
+ *
902
+ * The template most worth having, because the two things it produces are the two
903
+ * that must agree and that nothing currently links. A `periodic-full-reload`
904
+ * strategy says deletions reach this type because it is fully reloaded every
905
+ * `withinMs`; a schedule says how often it is actually reloaded. An operator
906
+ * sets one on the Model screen and the other on the connector, and nothing
907
+ * anywhere notices when they drift apart — at which point the type carries a
908
+ * declaration that reads as a guarantee and polices nothing.
909
+ *
910
+ * Here they come from one {@link RELOAD_CADENCES} entry, so they cannot
911
+ * disagree, and `withinMs` gets its slack from {@link withinMsFor} rather than
912
+ * from whatever somebody typed.
913
+ *
914
+ * **`because` is required and is not decoration.** `refuseUndeclaredDeletes`
915
+ * rejects a declaration whose reason is blank, on every strategy, because the
916
+ * control was never "it must be in code" — it is that somebody chose a strategy
917
+ * and wrote down why. A template that supplied a default reason would be
918
+ * defeating exactly the mechanism it is here to satisfy.
919
+ *
920
+ * **Where the schedule goes, and why the plan does not say.** The plan carries
921
+ * the cadence as data — {@link TemplatePlan.schedule} — and leaves *placing* it
922
+ * to whatever applies the plan. That was a deliberate bet, and it paid.
923
+ *
924
+ * When this was written, a workflow was put on a timer by a connector that
925
+ * pointed at it with `workflowId` and carried the `schedule` string. A template
926
+ * that had written that hop into its plan or its `todo` would have had to be
927
+ * rewritten when the workflow became the only authored object: connectors stopped
928
+ * being authored at all, and a schedule now lives on the workflow, set through
929
+ * `PUT workflows/:id/schedule`. Because the plan only ever declared "this runs
930
+ * `0 2 * * *`" and never "create a connector that says so", the move cost this
931
+ * template nothing — the same plan, rebased with no conflict, describes the new
932
+ * arrangement as accurately as it described the old one.
933
+ *
934
+ * What did not change, and is the part worth stating: only a `ready` graph is
935
+ * scheduled. A schedule written against a draft is stored and does not fire.
936
+ */
937
+ export function periodicFullReload(options) {
938
+ const targetType = options.targetType.trim();
939
+ const because = options.because.trim();
940
+ const missing = [];
941
+ if (targetType.length === 0)
942
+ missing.push('the object type the sink commits');
943
+ if (because.length === 0)
944
+ missing.push('a reason this cadence reconciles deletes acceptably');
945
+ if (missing.length > 0) {
946
+ return {
947
+ ok: false,
948
+ refusals: [
949
+ {
950
+ code: 'incomplete',
951
+ subjects: missing,
952
+ message: `This template still needs ${missing.join(' and ')}. The reason is not optional and is not rhetorical: a delete-reconciliation declaration with a blank reason is refused by the pipeline on every strategy, because what makes the declaration worth anything is that somebody chose it and wrote down why. Supplying a default here would defeat the mechanism this template exists to satisfy.`,
953
+ },
954
+ ],
955
+ };
956
+ }
957
+ const refusal = refuseUnusableColumnNames(options.sourceColumns, {
958
+ what: 'the source',
959
+ remedy: RENAME_REMEDY,
960
+ });
961
+ if (refusal)
962
+ return { ok: false, refusals: [refusal] };
963
+ const schedule = RELOAD_CADENCES[options.cadence];
964
+ const withinMs = withinMsFor(options.cadence);
965
+ const source = NODE_FACTORIES.source(nodeId('source', targetType, 1), `${targetType} full read`, {
966
+ sourceKind: options.sourceKind,
967
+ config: placeholderConfig(options.sourceKind),
968
+ ...(options.connectionId ? { connectionId: options.connectionId } : {}),
969
+ mode: 'full',
970
+ });
971
+ const sink = NODE_FACTORIES.sink(nodeId('sink', targetType, 1), targetType, {
972
+ targetType,
973
+ mode: 'full',
974
+ });
975
+ const deletes = {
976
+ strategy: 'periodic-full-reload',
977
+ because,
978
+ withinMs,
979
+ };
980
+ return {
981
+ ok: true,
982
+ plan: {
983
+ templateId: 'periodic-full-reload',
984
+ name: options.workflowName?.trim() || `Reload ${targetType} ${schedule.human}`,
985
+ description: `Reads ${targetType} whole ${schedule.human} and commits it, reconciling deletes by replacement.`,
986
+ nodes: [source, sink],
987
+ edges: [{ from: source.id, to: sink.id }],
988
+ transforms: [],
989
+ expectations: [
990
+ {
991
+ typeName: targetType,
992
+ input: { deletes },
993
+ why: `Declares that deletions reach ${targetType} by full replacement within ${withinMs}ms, which is twice the ${schedule.human} cadence this same template set.`,
994
+ },
995
+ ],
996
+ schedule,
997
+ declarations: [
998
+ {
999
+ what: `The schedule is \`${schedule.cron}\` and the reconciliation interval is ${withinMs}ms. Both come from one choice.`,
1000
+ why: 'A `withinMs` is only honest if the schedule really does reload that often — once the newest full load of a type is older than it, incremental loads of that type stop committing. Set separately, the two drift and the declaration becomes a sentence that polices nothing. Derived from one cadence, they cannot.',
1001
+ changeAt: 'Re-apply with a different cadence, or edit the schedule and the expectation together.',
1002
+ },
1003
+ {
1004
+ what: 'The interval is twice the cadence rather than equal to it.',
1005
+ why: 'At exactly the cadence, any run that starts a minute late — or simply takes a few minutes, since the age is measured from the snapshot — makes the type stale and blocks loads of it. A guard that fires on an ordinary Tuesday is a guard somebody switches off, taking the real failure with it. Twice tolerates one missed cycle and refuses the second, and covers the hour a DST boundary moves.',
1006
+ changeAt: "The type's load expectation, on the Model screen.",
1007
+ },
1008
+ {
1009
+ what: 'The graph reads and commits in `full` mode, and that is the point rather than a stepping stone.',
1010
+ why: 'This is the mode the reconciliation strategy describes. An incremental read here would be blind to deletes, which is the exact hole the declaration is claiming to have closed — and where the upstream ingest deletes and reinserts per base, every row gets a fresh timestamp in one batch anyway, so an incremental read returns the whole table while still missing the deletions.',
1011
+ changeAt: 'The `mode` field on the source node and on the sink node.',
1012
+ },
1013
+ ],
1014
+ todo: [
1015
+ configTodo(options.sourceKind, source.name),
1016
+ `The schedule is part of this plan but is not part of the graph: send \`${schedule.cron}\` to \`PUT workflows/:id/schedule\` once the workflow exists, and publish it first — a schedule on a draft is stored and never fires.`,
1017
+ `The expectation is a separate write against \`${targetType}\`. Applying the graph without it leaves the type undeclared, which is the state this template exists to avoid.`,
1018
+ ],
1019
+ },
1020
+ };
1021
+ }
1022
+ /**
1023
+ * Build whichever template was asked for, with the options that template needs.
1024
+ *
1025
+ * A discriminated request rather than one options bag with everything optional,
1026
+ * so asking for a join without a discriminator is a type error at the call site
1027
+ * instead of a refusal at run time. The refusals still exist because a form
1028
+ * hands over strings a user typed, and an empty string satisfies `string`.
1029
+ */
1030
+ export function buildWorkflowTemplate(request) {
1031
+ switch (request.templateId) {
1032
+ case 'replicate-table':
1033
+ return replicateTable(request.options);
1034
+ case 'load-file-drop':
1035
+ return loadFileDrop(request.options);
1036
+ case 'fan-out-types':
1037
+ return fanOutTypes(request.options);
1038
+ case 'join-sources':
1039
+ return joinSources(request.options);
1040
+ case 'enrich-with-lookup':
1041
+ return enrichWithLookup(request.options);
1042
+ case 'periodic-full-reload':
1043
+ return periodicFullReload(request.options);
1044
+ }
1045
+ }
1046
+ /**
1047
+ * Point each transform node at the transform that was created for it.
1048
+ *
1049
+ * Separate from building the plan, and pure, because the ids do not exist until
1050
+ * a server has stored the code — a template cannot invent them and must not
1051
+ * pretend to. Until this has run, a plan carrying transforms is deliberately
1052
+ * **not** a valid graph: `validateWorkflow` reports `transform-not-named` for
1053
+ * every node with an empty `transformId`, which is the correct answer, because a
1054
+ * node pointing at no code cannot run. See {@link planIsRunnable}.
1055
+ *
1056
+ * Nodes whose id is absent from the map are left exactly as they were, so a
1057
+ * partial failure part-way through creating three transforms leaves a graph that
1058
+ * still says which node is unfinished rather than one that silently claims to be
1059
+ * complete.
1060
+ */
1061
+ export function attachTransformIds(plan, transformIdByNodeId) {
1062
+ return {
1063
+ ...plan,
1064
+ nodes: plan.nodes.map((node) => {
1065
+ if (node.kind !== 'transform')
1066
+ return node;
1067
+ const id = transformIdByNodeId.get(node.id);
1068
+ return id === undefined ? node : { ...node, transformId: id };
1069
+ }),
1070
+ };
1071
+ }
1072
+ /** The authored half of a plan, in the shape `saveWorkflow` takes. */
1073
+ export function planToWorkflowInput(plan) {
1074
+ return {
1075
+ name: plan.name,
1076
+ description: plan.description,
1077
+ nodes: plan.nodes,
1078
+ edges: plan.edges.map((edge) => ({ from: edge.from, to: edge.to })),
1079
+ };
1080
+ }
1081
+ /**
1082
+ * Whether every transform node in the plan has been pointed at real code.
1083
+ *
1084
+ * The one question a screen has to ask between creating the transforms and
1085
+ * saving the graph. Named rather than left as an inline `.every`, because "the
1086
+ * plan is ready to save" and "the plan validates" are different questions and
1087
+ * conflating them is how a save gets attempted with an empty `transformId` and
1088
+ * comes back as a validation error about a field nobody typed.
1089
+ */
1090
+ export function planIsRunnable(plan) {
1091
+ return plan.nodes.every((node) => node.kind !== 'transform' || node.transformId.length > 0);
1092
+ }