@grepr/cli 1.14.13-15e1f25 → 1.15.0-c0f7761

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/build/dist/commands/job-draft-command.d.ts.map +1 -1
  2. package/build/dist/commands/job-draft-command.js +6 -71
  3. package/build/dist/commands/job-draft-command.js.map +1 -1
  4. package/build/dist/docs-index/index.json +1 -1
  5. package/build/dist/grepr.d.ts.map +1 -1
  6. package/build/dist/grepr.js +0 -2
  7. package/build/dist/grepr.js.map +1 -1
  8. package/build/dist/lib/backfill.js +4 -8
  9. package/build/dist/lib/backfill.js.map +1 -1
  10. package/build/dist/lib/job-patch.d.ts +15 -31
  11. package/build/dist/lib/job-patch.d.ts.map +1 -1
  12. package/build/dist/lib/job-patch.js +39 -709
  13. package/build/dist/lib/job-patch.js.map +1 -1
  14. package/build/dist/lib/job-plan.d.ts +3 -6
  15. package/build/dist/lib/job-plan.d.ts.map +1 -1
  16. package/build/dist/lib/job-plan.js +5 -55
  17. package/build/dist/lib/job-plan.js.map +1 -1
  18. package/build/dist/lib/option-parsers.d.ts +0 -5
  19. package/build/dist/lib/option-parsers.d.ts.map +1 -1
  20. package/build/dist/lib/option-parsers.js +0 -7
  21. package/build/dist/lib/option-parsers.js.map +1 -1
  22. package/build/dist/lib/signal-source.d.ts +1 -1
  23. package/build/dist/lib/signal-source.d.ts.map +1 -1
  24. package/build/dist/lib/signal-source.js +10 -69
  25. package/build/dist/lib/signal-source.js.map +1 -1
  26. package/build/dist/openapi/openApiTypes.d.ts +44 -1
  27. package/build/dist/openapi/openApiTypes.d.ts.map +1 -1
  28. package/build/dist/openapi/openApiTypes.js +5 -0
  29. package/build/dist/openapi/openApiTypes.js.map +1 -1
  30. package/build/dist/openapi-docs/schemas/ChangeFinding.md +1 -1
  31. package/build/dist/openapi-docs/schemas/PipelineChangeRequest.md +4 -0
  32. package/build/dist/types.d.ts +8 -37
  33. package/build/dist/types.d.ts.map +1 -1
  34. package/build/dist/types.js +0 -69
  35. package/build/dist/types.js.map +1 -1
  36. package/package.json +1 -1
  37. package/build/dist/commands/job-to-test-command.d.ts +0 -74
  38. package/build/dist/commands/job-to-test-command.d.ts.map +0 -1
  39. package/build/dist/commands/job-to-test-command.js +0 -161
  40. package/build/dist/commands/job-to-test-command.js.map +0 -1
  41. package/build/dist/lib/job-graph-log-pipeline-constants.d.ts +0 -12
  42. package/build/dist/lib/job-graph-log-pipeline-constants.d.ts.map +0 -1
  43. package/build/dist/lib/job-graph-log-pipeline-constants.js +0 -16
  44. package/build/dist/lib/job-graph-log-pipeline-constants.js.map +0 -1
  45. package/build/dist/lib/job-graph-transformer.d.ts +0 -119
  46. package/build/dist/lib/job-graph-transformer.d.ts.map +0 -1
  47. package/build/dist/lib/job-graph-transformer.js +0 -767
  48. package/build/dist/lib/job-graph-transformer.js.map +0 -1
  49. package/build/dist/lib/job-graph-utils.d.ts +0 -50
  50. package/build/dist/lib/job-graph-utils.d.ts.map +0 -1
  51. package/build/dist/lib/job-graph-utils.js +0 -90
  52. package/build/dist/lib/job-graph-utils.js.map +0 -1
@@ -1,7 +1,5 @@
1
- /** Structured patch format for grepr pipelines: a list of ops applied locally before any production write (see {@link JobBackend} for the two substrates). */
2
- import { ConditionNodeKind, MaskingOperatorType, DropNodeKind, PassthroughNodeKind, SqlNodeKind, SqlNodeOutputRouting, SqlOperationType, SqlOutputStatementType, LogAttributesRemapperType, LogReducerType, GrokParserType, LogsFilterType, DatadogQueryPredicateType, DatadogLogSinkType, SplunkLogSinkType, NewRelicLogSinkType, SumoLogSinkType, OtlpLogSinkType, LogsIcebergTableSinkType, TemplateOperationType, TemplateQueryExceptionType, SumAttributesMergeStrategyType, MinAttributesMergeStrategyType, MaxAttributesMergeStrategyType, AverageAttributesMergeStrategyType, } from '../openapi/openApiTypes.js';
3
- import { parseEdge } from './job-graph-utils.js';
4
- import { RAW_ATTRIBUTES_REMAPPER, RAW_ATTRIBUTES_REMAPPER_TYPE, RAW_JSON_PROCESSOR, RAW_JSON_PROCESSOR_TYPE, RAW_LOG_REDUCER, RAW_PARSER_TYPES, RAW_PRE_EXCEPTIONS_FILTER, RAW_PRE_PARSER_FILTER, RAW_PRE_WAREHOUSE_FILTER, } from './job-graph-log-pipeline-constants.js';
1
+ /** Structured patch format for template-backed grepr pipelines: a list of ops applied to `templateInputs.input` locally before any production write. */
2
+ import { ConditionNodeKind, MaskingOperatorType, DropNodeKind, PassthroughNodeKind, SqlNodeKind, SqlNodeOutputRouting, SqlOperationType, SqlOutputStatementType, LogAttributesRemapperType, GrokParserType, DatadogQueryPredicateType, DatadogLogSinkType, SplunkLogSinkType, NewRelicLogSinkType, SumoLogSinkType, OtlpLogSinkType, LogsIcebergTableSinkType, TemplateOperationType, TemplateQueryExceptionType, SumAttributesMergeStrategyType, MinAttributesMergeStrategyType, MaxAttributesMergeStrategyType, AverageAttributesMergeStrategyType, } from '../openapi/openApiTypes.js';
5
3
  /** The accepted-phases suffix shared by every template transform-phase error. */
6
4
  const ACCEPTED_TRANSFORM_PHASES_MESSAGE = 'Accepted phases are pre-parser, pre-warehouse, and pre-exceptions.';
7
5
  /** The downstream pipeline steps a `sql_output` may route to (the generated {@link SqlNodeOutputRouting} values). */
@@ -16,11 +14,6 @@ const VENDOR_LOG_SINK_TYPES = new Set([
16
14
  OtlpLogSinkType.otlp_log_sink,
17
15
  ]);
18
16
  const LOGS_ICEBERG_TABLE_SINK_TYPE = LogsIcebergTableSinkType.logs_iceberg_table_sink;
19
- // Raw data-lake and processed-logs sinks share the logs-iceberg-table-sink type
20
- // and are distinguished only by these vertex-name prefixes — a naming convention
21
- // this CLI must match, not invent.
22
- const RAW_DATA_LAKE_SINK_NAME_PREFIX = 'raw_data_sink';
23
- const PROCESSED_LOGS_SINK_NAME_PREFIX = 'processed_logs_';
24
17
  /**
25
18
  * Required fields per op, keyed by op name. Typed as `Record<JobPatchOp['op'], …>`
26
19
  * so a new op added to the union fails to compile until its entry is added here —
@@ -131,23 +124,12 @@ export function parsePatch(raw) {
131
124
  }
132
125
  return { operations: parsedOperations };
133
126
  }
134
- /** Detect whether a job is template-backed (a `template-operation` vertex) or job-graph. */
135
- export function detectBackend(job) {
136
- const vertices = job.jobGraph?.vertices;
137
- if (!Array.isArray(vertices))
138
- return 'job-graph';
139
- return vertices.some(v => v.type === TemplateOperationType.template_operation) ? 'template' : 'job-graph';
140
- }
141
127
  /**
142
128
  * Apply a patch to a fetched job, producing the `SchemaUpdateJob` payload for
143
- * `job:apply`. Pure (clones input); dispatches on backend. Throws if an op
144
- * targets a missing field/vertex or a topology op hits a non-UI raw graph.
129
+ * `job:apply`. Pure (clones input). Throws if the job is not template-backed or
130
+ * an op targets a missing field.
145
131
  */
146
132
  export function applyPatch(job, patch) {
147
- const backend = detectBackend(job);
148
- return backend === 'template' ? applyPatchToTemplate(job, patch) : applyPatchToJobGraph(job, patch);
149
- }
150
- function applyPatchToTemplate(job, patch) {
151
133
  const cloned = structuredClone(job);
152
134
  const templateOp = findTemplateOperation(cloned);
153
135
  const input = readTemplateInput(templateOp);
@@ -167,44 +149,14 @@ function applyPatchToTemplate(job, patch) {
167
149
  teamIds: cloned.teamIds,
168
150
  };
169
151
  }
170
- /**
171
- * Apply a patch to a job-graph (non-template) pipeline. Clones the job and
172
- * mutates the resolved graph on the clone (the input is left untouched). Field
173
- * names match the template inputs except `add-reducer-exception`, which appends
174
- * a raw `EventPredicate` to the reducer vertex's `logReducerExceptions`. Topology
175
- * ops require the canonical UI log-pipeline chain.
176
- */
177
- function applyPatchToJobGraph(job, patch) {
178
- const cloned = structuredClone(job);
179
- const jobGraph = cloned.jobGraph;
180
- const vertices = jobGraph?.vertices;
181
- if (!jobGraph || !Array.isArray(vertices) || vertices.length === 0) {
182
- throw new Error('Job has no jobGraph.vertices to patch');
183
- }
184
- if (!Array.isArray(jobGraph.edges)) {
185
- jobGraph.edges = [];
186
- }
187
- for (let i = 0; i < patch.operations.length; i++) {
188
- applyJobGraphOperation(jobGraph, patch.operations[i], i);
189
- }
190
- if (patch.operations.some(touchesSourceConfig)) {
191
- assertProposedJobGraphHasSource(jobGraph);
192
- }
193
- return {
194
- desiredState: cloned.desiredState,
195
- fromVersion: cloned.version,
196
- jobGraph: cloned.jobGraph,
197
- teamIds: cloned.teamIds,
198
- };
199
- }
200
152
  function applyOperation(input, op, index) {
201
153
  switch (op.op) {
202
154
  case 'add-message-attribute':
203
- return applyAddMessageAttribute(input, op.attributePath, index);
155
+ return applyMessageAttributeToRemapper(requireRemapper(input, 'add-message-attribute', index, ADD_PARSER_REMAPPER_HINT), op.attributePath, index);
204
156
  case 'add-group-by':
205
- return applyAddGroupBy(input, op.attributePath, index);
157
+ return applyGroupByToReducer(input.reducer, op.attributePath, index);
206
158
  case 'add-aggregation-strategy':
207
- return applyAddAggregation(input, op.attributePath, op.strategies, index);
159
+ return applyAggregationToReducer(input.reducer, op.attributePath, op.strategies, index);
208
160
  case 'add-reducer-exception':
209
161
  return applyAddReducerException(input, op.predicate);
210
162
  case 'add-grok-rule':
@@ -244,11 +196,11 @@ function applyOperation(input, op, index) {
244
196
  case 'set-raw-dataset':
245
197
  return applySetRawDataset(input, op.datasetId);
246
198
  case 'remove-message-attribute':
247
- return applyRemoveMessageAttribute(input, op.attributePath, index);
199
+ return removeMessageAttributeFromRemapper(requireRemapper(input, 'remove-message-attribute', index), op.attributePath, index);
248
200
  case 'remove-group-by':
249
- return applyRemoveGroupBy(input, op.attributePath, index);
201
+ return removeGroupByFromReducer(input.reducer, op.attributePath, index);
250
202
  case 'remove-aggregation-strategy':
251
- return applyRemoveAggregation(input, op.attributePath, op.strategies, index);
203
+ return removeAggregationFromReducer(input.reducer, op.attributePath, op.strategies, index);
252
204
  case 'remove-reducer-exception':
253
205
  return applyRemoveReducerException(input, op.predicate, index);
254
206
  case 'remove-grok-rule':
@@ -261,11 +213,11 @@ function applyOperation(input, op, index) {
261
213
  // See add-sink: read the vendor-only `filter` via `in` so a JSON patch's stray field still reaches the guard.
262
214
  return applyUpdateSink(input, op.target, op.sink, 'filter' in op ? op.filter : undefined, index);
263
215
  case 'update-message-attribute':
264
- return applyUpdateMessageAttribute(input, op.from, op.to, index);
216
+ return updateMessageAttributeOnRemapper(requireRemapper(input, 'update-message-attribute', index), op.from, op.to, index);
265
217
  case 'update-group-by':
266
- return applyUpdateGroupBy(input, op.from, op.to, index);
218
+ return updateGroupByOnReducer(input.reducer, op.from, op.to, index);
267
219
  case 'update-aggregation-strategy':
268
- return applyUpdateAggregation(input, op.attributePath, op.strategies, index);
220
+ return updateAggregationOnReducer(input.reducer, op.attributePath, op.strategies, index);
269
221
  case 'update-reducer-exception':
270
222
  return applyUpdateReducerException(input, op.from, op.to, index);
271
223
  default:
@@ -277,143 +229,6 @@ function throwUnknownOp(op, index) {
277
229
  const unknownOp = op;
278
230
  throw new Error(`Operation ${index}: unknown op "${unknownOp.op ?? '(missing)'}"`);
279
231
  }
280
- function applyJobGraphOperation(jobGraph, op, index) {
281
- const vertices = jobGraph.vertices;
282
- switch (op.op) {
283
- case 'add-message-attribute':
284
- return jobGraphAddMessageAttribute(vertices, op.attributePath, index);
285
- case 'add-group-by':
286
- return jobGraphAddGroupBy(vertices, op.attributePath, index);
287
- case 'add-aggregation-strategy':
288
- return jobGraphAddAggregation(vertices, op.attributePath, op.strategies, index);
289
- case 'add-reducer-exception':
290
- return jobGraphAddReducerException(vertices, op.predicate, index);
291
- case 'add-grok-rule':
292
- return jobGraphAddGrokRule(vertices, op.pattern, op.parserName, op.extractAttribute, index);
293
- case 'add-parser':
294
- return jobGraphAddParser(jobGraph, op.parser, index);
295
- case 'remove-parser':
296
- return jobGraphRemoveParser(jobGraph, op.name, index);
297
- case 'set-filter':
298
- return jobGraphSetFilter(jobGraph, op.phase, op.filter, index);
299
- case 'clear-filter':
300
- return jobGraphClearFilter(jobGraph, op.phase, index);
301
- case 'set-sql-transform':
302
- throw new Error(`Operation ${index} (set-sql-transform): SQL transform edits are only supported for ` +
303
- `template-backed log-reducer jobs in the CLI. This job is a direct job graph. `);
304
- case 'set-transform-chain':
305
- case 'clear-transform-chain':
306
- throw new Error(`Operation ${index} (${op.op}): transform-chain edits are only supported for ` +
307
- `template-backed log-reducer jobs in the CLI. This job is a direct job graph. `);
308
- case 'set-masking':
309
- case 'clear-masking':
310
- throw new Error(`Operation ${index} (${op.op}): masking operator edits are only supported for ` +
311
- `template-backed log-reducer jobs in the CLI. This job is a direct job graph. `);
312
- case 'add-source':
313
- return jobGraphAddSource(jobGraph, op.source, index);
314
- case 'remove-source':
315
- return jobGraphRemoveSource(jobGraph, op.name, index);
316
- case 'add-sink':
317
- // See the template dispatch: read the target-specific field via `in`.
318
- return jobGraphAddSink(jobGraph, op.target, op.sink, 'filter' in op ? op.filter : undefined, index);
319
- case 'remove-sink':
320
- return jobGraphRemoveSink(jobGraph, op.target, 'name' in op ? op.name : undefined, index);
321
- case 'set-raw-dataset':
322
- return jobGraphSetRawDataset(jobGraph, op.datasetId, index);
323
- case 'remove-message-attribute':
324
- return jobGraphRemoveMessageAttribute(vertices, op.attributePath, index);
325
- case 'remove-group-by':
326
- return jobGraphRemoveGroupBy(vertices, op.attributePath, index);
327
- case 'remove-aggregation-strategy':
328
- return jobGraphRemoveAggregation(vertices, op.attributePath, op.strategies, index);
329
- case 'remove-reducer-exception':
330
- return jobGraphRemoveReducerException(vertices, op.predicate, index);
331
- case 'remove-grok-rule':
332
- return jobGraphRemoveGrokRule(vertices, op.pattern, op.parserName, index);
333
- case 'update-source':
334
- return jobGraphUpdateSource(jobGraph, op.source, index);
335
- case 'update-parser':
336
- return jobGraphUpdateParser(jobGraph, op.parser, index);
337
- case 'update-sink':
338
- // See add-sink: read the vendor-only `filter` via `in`.
339
- return jobGraphUpdateSink(jobGraph, op.target, op.sink, 'filter' in op ? op.filter : undefined, index);
340
- case 'update-message-attribute':
341
- return jobGraphUpdateMessageAttribute(vertices, op.from, op.to, index);
342
- case 'update-group-by':
343
- return jobGraphUpdateGroupBy(vertices, op.from, op.to, index);
344
- case 'update-aggregation-strategy':
345
- return jobGraphUpdateAggregation(vertices, op.attributePath, op.strategies, index);
346
- case 'update-reducer-exception':
347
- return jobGraphUpdateReducerException(vertices, op.from, op.to, index);
348
- case 'set-input-field':
349
- case 'unset-input-field':
350
- throw new Error(`Operation ${index} (${op.op}): generic template-input paths are not supported on raw job graphs. ` +
351
- `Use a semantic operation, or apply this change directly via ` +
352
- `'grepr job:get' + manual edit + 'grepr job:update'.`);
353
- default:
354
- return throwUnknownOp(op, index);
355
- }
356
- }
357
- function findUniqueVertexByType(vertices, type, opLabel, index) {
358
- const matches = vertices.filter(v => v.type === type);
359
- if (matches.length === 0) {
360
- throw new Error(`Operation ${index} (${opLabel}): no ${type} vertex found in jobGraph.vertices. ` +
361
- `Non-template pipelines must already include this vertex; this CLI doesn't add new vertices to job-graph pipelines.`);
362
- }
363
- if (matches.length > 1) {
364
- throw new Error(`Operation ${index} (${opLabel}): expected exactly one ${type} vertex but found ${matches.length}. ` +
365
- `Ambiguous target; resolve by hand via 'grepr job:get' + 'grepr job:update'.`);
366
- }
367
- return matches[0];
368
- }
369
- function jobGraphAddMessageAttribute(vertices, attributePath, index) {
370
- const remapper = findUniqueVertexByType(vertices, LogAttributesRemapperType.log_attributes_remapper, 'add-message-attribute', index);
371
- applyMessageAttributeToRemapper(remapper, attributePath, index);
372
- }
373
- function jobGraphAddGroupBy(vertices, attributePath, index) {
374
- const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'add-group-by', index);
375
- applyGroupByToReducer(reducer, attributePath, index);
376
- }
377
- function jobGraphAddAggregation(vertices, attributePath, strategies, index) {
378
- const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'add-aggregation-strategy', index);
379
- applyAggregationToReducer(reducer, attributePath, strategies, index);
380
- }
381
- /** Append a predicate to the reducer's `logReducerExceptions` (stored raw, unlike the template path's `TemplateQueryException` wrapper). */
382
- function jobGraphAddReducerException(vertices, predicate, index) {
383
- const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'add-reducer-exception', index);
384
- const existing = reducer.logReducerExceptions ?? [];
385
- const serialized = JSON.stringify(predicate);
386
- if (existing.some(p => JSON.stringify(p) === serialized)) {
387
- reducer.logReducerExceptions = existing;
388
- return; // idempotent
389
- }
390
- existing.push(predicate);
391
- reducer.logReducerExceptions = existing;
392
- }
393
- function jobGraphAddGrokRule(vertices, pattern, parserName, extractAttribute, index) {
394
- const grok = findJobGraphGrokParser(vertices, parserName, 'add-grok-rule', index);
395
- applyGrokRuleToParser(grok, pattern, extractAttribute);
396
- }
397
- /** Resolve the target grok-parser vertex by name (or the sole grok-parser); shared by add/remove-grok-rule. Throws if absent or ambiguous. */
398
- function findJobGraphGrokParser(vertices, parserName, opLabel, index) {
399
- const groks = vertices.filter(v => v.type === GrokParserType.grok_parser);
400
- const grok = selectGrokParser(groks, parserName, 'jobGraph.vertices', opLabel, index);
401
- if (!grok) {
402
- const hint = opLabel === 'add-grok-rule' ? " Add a grok-parser vertex via 'grepr job:update' before using add-grok-rule on this pipeline." : '';
403
- throw new Error(`Operation ${index} (${opLabel}): ${parserName ? `grok-parser "${parserName}" not found` : 'no grok-parser vertex found in jobGraph.vertices'}.${hint}`);
404
- }
405
- return grok;
406
- }
407
- function selectGrokParser(groks, parserName, location, opLabel, index) {
408
- if (parserName) {
409
- return groks.find(parser => parser.name === parserName);
410
- }
411
- if (groks.length > 1) {
412
- throw new Error(`Operation ${index} (${opLabel}): found ${groks.length} grok parsers in ${location}; ` +
413
- `pass parserName to choose the target parser.`);
414
- }
415
- return groks[0];
416
- }
417
232
  function applyMessageAttributeToRemapper(remapper, attributePath, index) {
418
233
  const parts = splitPath(attributePath, index);
419
234
  if (parts.length === 1) {
@@ -471,7 +286,7 @@ function entryNotFoundError(index, opLabel, what) {
471
286
  return new Error(`Operation ${index} (${opLabel}): ${what} not found.`);
472
287
  }
473
288
  // remove-* counterparts of the add-* reducer/remapper/grok writers: mutate the same
474
- // vertex field, shared verbatim by both backends, throw if the entry is absent.
289
+ // field and throw if the entry is absent.
475
290
  function removeMessageAttributeFromRemapper(remapper, attributePath, index) {
476
291
  const parts = splitPath(attributePath, index);
477
292
  const removed = parts.length === 1
@@ -579,339 +394,6 @@ function reducerExceptionUpdateIndex(serialized, from, to, index) {
579
394
  }
580
395
  return at;
581
396
  }
582
- const DEFAULT_EDGE_OUTPUT = 'output';
583
- const DEFAULT_EDGE_INPUT = 'input';
584
- function jobGraphAddSource(jobGraph, source, index) {
585
- assertRawUiLogGraph(jobGraph, 'add-source', index, [RAW_PRE_PARSER_FILTER]);
586
- assertOperationIdentity(source, 'add-source', index, 'source');
587
- if (findVertexIndexByName(jobGraph, source.name) !== -1) {
588
- throw new Error(`Operation ${index} (add-source): source "${source.name}" already exists`);
589
- }
590
- jobGraph.vertices.push(source);
591
- addEdgeIfMissing(jobGraph, source.name, DEFAULT_EDGE_OUTPUT, RAW_PRE_PARSER_FILTER, DEFAULT_EDGE_INPUT);
592
- }
593
- function jobGraphRemoveSource(jobGraph, name, index) {
594
- assertRawUiLogGraph(jobGraph, 'remove-source', index, [RAW_PRE_PARSER_FILTER]);
595
- const idx = findVertexIndexByName(jobGraph, name);
596
- if (idx === -1) {
597
- throw new Error(`Operation ${index} (remove-source): source "${name}" not found in jobGraph.vertices`);
598
- }
599
- if (!isCanonicalRawSource(jobGraph, name)) {
600
- throw unsupportedRawShapeError(index, 'remove-source', `vertex "${name}" is not a canonical UI source feeding ${RAW_PRE_PARSER_FILTER}`);
601
- }
602
- jobGraph.vertices.splice(idx, 1);
603
- removeEdgesTouching(jobGraph, name);
604
- }
605
- function jobGraphAddSink(jobGraph, target, sink, filter, index) {
606
- assertOperationIdentity(sink, 'add-sink', index, 'sink');
607
- assertSinkTargetShape(target, sink, filter, index, 'add-sink');
608
- // Anchors on a unique log_reducer (assertRawUiLogGraph rejects missing/duplicate).
609
- assertRawUiLogGraph(jobGraph, 'add-sink', index, [RAW_LOG_REDUCER]);
610
- if (findVertexIndexByName(jobGraph, sink.name) !== -1) {
611
- throw new Error(`Operation ${index} (add-sink): sink "${sink.name}" already exists`);
612
- }
613
- // processed-logs is a singular slot — mirror the template-backend guard.
614
- // Only reject a reducer-fed iceberg sink (direct or 1-hop through a filter);
615
- // a raw data-lake sink fed by pre_data_warehouse_filter is a separate slot.
616
- if (target === 'processed-logs' && hasReducerFedIcebergSink(jobGraph)) {
617
- throw new Error(`Operation ${index} (add-sink): a logs-iceberg-table-sink already exists. ` +
618
- `Remove it first with remove-sink (target: processed-logs) before adding a new one.`);
619
- }
620
- if (filter !== undefined) {
621
- if (typeof filter !== 'object') {
622
- throw new Error(`Operation ${index} (add-sink): filter must be an object`);
623
- }
624
- const filterName = `${sink.name}_filter`;
625
- if (findVertexIndexByName(jobGraph, filterName) !== -1) {
626
- throw new Error(`Operation ${index} (add-sink): generated filter vertex "${filterName}" already exists`);
627
- }
628
- const filterVertex = {
629
- ...filter,
630
- name: filterName,
631
- type: LogsFilterType.logs_filter,
632
- };
633
- jobGraph.vertices.push(filterVertex, sink);
634
- addEdgeIfMissing(jobGraph, RAW_LOG_REDUCER, DEFAULT_EDGE_OUTPUT, filterName, DEFAULT_EDGE_INPUT);
635
- addEdgeIfMissing(jobGraph, filterName, DEFAULT_EDGE_OUTPUT, sink.name, DEFAULT_EDGE_INPUT);
636
- return;
637
- }
638
- jobGraph.vertices.push(sink);
639
- addEdgeIfMissing(jobGraph, RAW_LOG_REDUCER, DEFAULT_EDGE_OUTPUT, sink.name, DEFAULT_EDGE_INPUT);
640
- }
641
- function jobGraphRemoveSink(jobGraph, target, name, index) {
642
- assertValidSinkTarget(target, 'remove-sink', index);
643
- const sinkName = resolveRawSinkToRemove(jobGraph, target, name, index);
644
- // If the sink is fed by its generated single-use filter, drop that too.
645
- const generatedFilterName = `${sinkName}_filter`;
646
- const fedByGeneratedFilter = parsedEdges(jobGraph).some(edge => edge.targetVertex === sinkName && edge.sourceVertex === generatedFilterName);
647
- removeVertexByName(jobGraph, sinkName);
648
- removeEdgesTouching(jobGraph, sinkName);
649
- if (fedByGeneratedFilter && findVertexByName(jobGraph, generatedFilterName)) {
650
- removeVertexByName(jobGraph, generatedFilterName);
651
- removeEdgesTouching(jobGraph, generatedFilterName);
652
- }
653
- }
654
- /**
655
- * Resolve which raw-graph sink a `remove-sink` op targets. `vendor` requires a
656
- * `name` matching a vendor sink type; `processed-logs` ignores `name` and
657
- * removes the unique `logs-iceberg-table-sink` (rejecting 0 or multiple).
658
- */
659
- function resolveRawSinkToRemove(jobGraph, target, name, index) {
660
- if (target === 'vendor') {
661
- if (typeof name !== 'string' || name.length === 0) {
662
- throw new Error(`Operation ${index} (remove-sink): name is required for target "vendor"`);
663
- }
664
- const vertex = findVertexByName(jobGraph, name);
665
- if (!vertex) {
666
- throw new Error(`Operation ${index} (remove-sink): sink "${name}" not found in jobGraph.vertices`);
667
- }
668
- if (!VENDOR_LOG_SINK_TYPES.has(vertex.type)) {
669
- throw new Error(`Operation ${index} (remove-sink): vertex "${name}" is not a vendor sink (type "${vertex.type}")`);
670
- }
671
- return name;
672
- }
673
- return findIcebergSinkByRole(jobGraph, PROCESSED_LOGS_SINK_NAME_PREFIX, 'processed-logs', 'remove-sink', 'remove the sink', index).name;
674
- }
675
- /**
676
- * The single `logs-iceberg-table-sink` vertex playing the given role, identified
677
- * by its name prefix. Selecting by type alone is ambiguous — raw and processed
678
- * sinks share the type — so a count-based pick can target the wrong dataset sink.
679
- * Errors on 0 or >1 matches for the role.
680
- */
681
- function findIcebergSinkByRole(jobGraph, namePrefix, role, opLabel, action, index) {
682
- const matches = jobGraph.vertices.filter(v => v.type === LOGS_ICEBERG_TABLE_SINK_TYPE && v.name.startsWith(namePrefix));
683
- if (matches.length !== 1) {
684
- throw new Error(`Operation ${index} (${opLabel}): found ${matches.length} ${role} sinks (logs-iceberg-table-sink named "${namePrefix}*"); ` +
685
- `expected exactly one to ${action}. Use 'grepr job:get' + 'grepr job:update' to ${action} explicitly.`);
686
- }
687
- return matches[0];
688
- }
689
- function jobGraphSetRawDataset(jobGraph, datasetId, index) {
690
- const sink = findIcebergSinkByRole(jobGraph, RAW_DATA_LAKE_SINK_NAME_PREFIX, 'raw data-lake', 'set-raw-dataset', 'set the dataset', index);
691
- sink.datasetId = datasetId;
692
- }
693
- function jobGraphSetFilter(jobGraph, phase, filter, index) {
694
- if (!filter || typeof filter !== 'object') {
695
- throw new Error(`Operation ${index} (set-filter): filter must be an object`);
696
- }
697
- const rawName = rawFilterNameForPhase(phase, index, 'set-filter');
698
- assertRawUiLogGraph(jobGraph, 'set-filter', index, [rawName]);
699
- replaceRawFilterVertex(jobGraph, rawName, filter, 'set-filter', index);
700
- }
701
- function jobGraphClearFilter(jobGraph, phase, index) {
702
- const rawName = rawFilterNameForPhase(phase, index, 'clear-filter');
703
- assertRawUiLogGraph(jobGraph, 'clear-filter', index, [rawName]);
704
- replaceRawFilterVertex(jobGraph, rawName, { predicate: { type: DatadogQueryPredicateType.datadog_query, query: '' } }, 'clear-filter', index);
705
- }
706
- function replaceRawFilterVertex(jobGraph, rawName, patch, opLabel, index) {
707
- const existing = findVertexByName(jobGraph, rawName);
708
- if (!existing) {
709
- throw unsupportedRawShapeError(index, opLabel, `vertex "${rawName}" not found`);
710
- }
711
- replaceVertexByName(jobGraph, rawName, {
712
- ...existing,
713
- ...patch,
714
- type: LogsFilterType.logs_filter,
715
- name: rawName,
716
- }, opLabel, index);
717
- }
718
- function jobGraphAddParser(jobGraph, parser, index) {
719
- assertRawUiLogGraph(jobGraph, 'add-parser', index, [RAW_PRE_PARSER_FILTER, RAW_PRE_WAREHOUSE_FILTER]);
720
- assertOperationIdentity(parser, 'add-parser', index, 'parser');
721
- if (!RAW_PARSER_TYPES.has(parser.type)) {
722
- throw new Error(`Operation ${index} (add-parser): parser type "${parser.type}" is not supported for raw UI graph insertion`);
723
- }
724
- if (findVertexIndexByName(jobGraph, parser.name) !== -1) {
725
- throw new Error(`Operation ${index} (add-parser): parser "${parser.name}" already exists`);
726
- }
727
- if (parser.type === RAW_JSON_PROCESSOR_TYPE) {
728
- if (parser.name !== RAW_JSON_PROCESSOR) {
729
- throw unsupportedRawShapeError(index, 'add-parser', `json processor must be named "${RAW_JSON_PROCESSOR}"`);
730
- }
731
- if (findVertexByName(jobGraph, RAW_JSON_PROCESSOR) || findVertexByType(jobGraph, RAW_JSON_PROCESSOR_TYPE)) {
732
- throw new Error(`Operation ${index} (add-parser): json-log-processor already exists in the raw graph`);
733
- }
734
- const successor = orderedParserNames(jobGraph, 'add-parser', index)[0] ?? RAW_PRE_WAREHOUSE_FILTER;
735
- return insertRawVertexBetween(jobGraph, parser, RAW_PRE_PARSER_FILTER, successor, 'add-parser', index);
736
- }
737
- if (parser.type === RAW_ATTRIBUTES_REMAPPER_TYPE) {
738
- if (parser.name !== RAW_ATTRIBUTES_REMAPPER) {
739
- throw unsupportedRawShapeError(index, 'add-parser', `attributes remapper must be named "${RAW_ATTRIBUTES_REMAPPER}"`);
740
- }
741
- if (findVertexByName(jobGraph, RAW_ATTRIBUTES_REMAPPER) || findVertexByType(jobGraph, RAW_ATTRIBUTES_REMAPPER_TYPE)) {
742
- throw new Error(`Operation ${index} (add-parser): log-attributes-remapper already exists in the raw graph`);
743
- }
744
- // Locate the json processor by type, not name: live UI graphs use suffixed
745
- // names (e.g. json_log_processor_1), and a name-only lookup would silently
746
- // fall back to RAW_PRE_PARSER_FILTER and wire the remapper before it.
747
- const jsonProcessor = findVertexByType(jobGraph, RAW_JSON_PROCESSOR_TYPE);
748
- const predecessor = jsonProcessor ? jsonProcessor.name : RAW_PRE_PARSER_FILTER;
749
- const successor = singleMainSuccessor(jobGraph, predecessor, 'add-parser', index);
750
- return insertRawVertexBetween(jobGraph, parser, predecessor, successor, 'add-parser', index);
751
- }
752
- const existingParserNames = orderedParserNames(jobGraph, 'add-parser', index);
753
- const predecessor = existingParserNames[existingParserNames.length - 1] ?? RAW_PRE_PARSER_FILTER;
754
- insertRawVertexBetween(jobGraph, parser, predecessor, RAW_PRE_WAREHOUSE_FILTER, 'add-parser', index);
755
- }
756
- function jobGraphRemoveParser(jobGraph, name, index) {
757
- assertRawUiLogGraph(jobGraph, 'remove-parser', index, [RAW_PRE_PARSER_FILTER, RAW_PRE_WAREHOUSE_FILTER]);
758
- const parser = findVertexByName(jobGraph, name);
759
- if (!parser) {
760
- throw new Error(`Operation ${index} (remove-parser): parser "${name}" not found in jobGraph.vertices`);
761
- }
762
- if (!RAW_PARSER_TYPES.has(parser.type)) {
763
- throw new Error(`Operation ${index} (remove-parser): vertex "${name}" is not a supported parser type`);
764
- }
765
- const incoming = parsedEdges(jobGraph).filter(edge => edge.targetVertex === name);
766
- const outgoing = parsedEdges(jobGraph).filter(edge => edge.sourceVertex === name);
767
- if (incoming.length !== 1 || outgoing.length !== 1) {
768
- throw unsupportedRawShapeError(index, 'remove-parser', `parser "${name}" must have exactly one incoming and one outgoing edge`);
769
- }
770
- removeVertexByName(jobGraph, name);
771
- removeEdgesTouching(jobGraph, name);
772
- const from = incoming[0];
773
- const to = outgoing[0];
774
- addEdgeIfMissing(jobGraph, from.sourceVertex, from.sourcePort, to.targetVertex, to.targetPort);
775
- }
776
- // --- Job-graph per-entry removals: anchor on the unique reducer/remapper/grok
777
- // vertex (the add-path guards), then delegate to the shared remover. ---
778
- function jobGraphRemoveMessageAttribute(vertices, attributePath, index) {
779
- const remapper = findUniqueVertexByType(vertices, LogAttributesRemapperType.log_attributes_remapper, 'remove-message-attribute', index);
780
- removeMessageAttributeFromRemapper(remapper, attributePath, index);
781
- }
782
- function jobGraphRemoveGroupBy(vertices, attributePath, index) {
783
- const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'remove-group-by', index);
784
- removeGroupByFromReducer(reducer, attributePath, index);
785
- }
786
- function jobGraphRemoveAggregation(vertices, attributePath, strategies, index) {
787
- const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'remove-aggregation-strategy', index);
788
- removeAggregationFromReducer(reducer, attributePath, strategies, index);
789
- }
790
- /** Remove a raw `EventPredicate` from the reducer's `logReducerExceptions` (mirror of jobGraphAddReducerException). */
791
- function jobGraphRemoveReducerException(vertices, predicate, index) {
792
- const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'remove-reducer-exception', index);
793
- const serialized = JSON.stringify(predicate);
794
- const existing = reducer.logReducerExceptions ?? [];
795
- const kept = existing.filter(p => JSON.stringify(p) !== serialized);
796
- if (kept.length === existing.length) {
797
- throw entryNotFoundError(index, 'remove-reducer-exception', 'matching reducer exception');
798
- }
799
- reducer.logReducerExceptions = kept;
800
- }
801
- function jobGraphRemoveGrokRule(vertices, pattern, parserName, index) {
802
- const grok = findJobGraphGrokParser(vertices, parserName, 'remove-grok-rule', index);
803
- removeGrokRuleFromParser(grok, pattern, index);
804
- }
805
- // --- Job-graph in-place vertex updates: replace the named vertex's config via
806
- // replaceVertexByName, leaving edges/position untouched. ---
807
- function jobGraphUpdateSource(jobGraph, source, index) {
808
- assertOperationIdentity(source, 'update-source', index, 'source');
809
- assertRawUiLogGraph(jobGraph, 'update-source', index, [RAW_PRE_PARSER_FILTER]);
810
- if (findVertexIndexByName(jobGraph, source.name) === -1) {
811
- throw new Error(`Operation ${index} (update-source): source "${source.name}" not found in jobGraph.vertices.`);
812
- }
813
- if (!isCanonicalRawSource(jobGraph, source.name)) {
814
- throw unsupportedRawShapeError(index, 'update-source', `vertex "${source.name}" is not a canonical UI source feeding ${RAW_PRE_PARSER_FILTER}`);
815
- }
816
- replaceVertexByName(jobGraph, source.name, source, 'update-source', index);
817
- }
818
- function jobGraphUpdateParser(jobGraph, parser, index) {
819
- assertOperationIdentity(parser, 'update-parser', index, 'parser');
820
- assertRawUiLogGraph(jobGraph, 'update-parser', index, [RAW_PRE_PARSER_FILTER, RAW_PRE_WAREHOUSE_FILTER]);
821
- const existing = findVertexByName(jobGraph, parser.name);
822
- if (!existing) {
823
- throw new Error(`Operation ${index} (update-parser): parser "${parser.name}" not found in jobGraph.vertices.`);
824
- }
825
- if (!RAW_PARSER_TYPES.has(existing.type)) {
826
- throw new Error(`Operation ${index} (update-parser): vertex "${parser.name}" is not a supported parser type.`);
827
- }
828
- if (!RAW_PARSER_TYPES.has(parser.type)) {
829
- throw new Error(`Operation ${index} (update-parser): replacement parser type "${parser.type}" is not supported for raw UI graph parsers.`);
830
- }
831
- replaceVertexByName(jobGraph, parser.name, parser, 'update-parser', index);
832
- }
833
- function jobGraphUpdateSink(jobGraph, target, sink, filter, index) {
834
- assertOperationIdentity(sink, 'update-sink', index, 'sink');
835
- assertSinkTargetShape(target, sink, filter, index, 'update-sink');
836
- if (target === 'processed-logs') {
837
- const existing = findIcebergSinkByRole(jobGraph, PROCESSED_LOGS_SINK_NAME_PREFIX, 'processed-logs', 'update-sink', 'update the sink', index);
838
- replaceVertexByName(jobGraph, existing.name, { ...sink, name: existing.name }, 'update-sink', index);
839
- return;
840
- }
841
- // vendor: anchor on the unique reducer (as add-sink does), then replace the named vendor sink.
842
- assertRawUiLogGraph(jobGraph, 'update-sink', index, [RAW_LOG_REDUCER]);
843
- const existing = findVertexByName(jobGraph, sink.name);
844
- if (!existing) {
845
- throw new Error(`Operation ${index} (update-sink): sink "${sink.name}" not found in jobGraph.vertices.`);
846
- }
847
- if (!VENDOR_LOG_SINK_TYPES.has(existing.type)) {
848
- throw new Error(`Operation ${index} (update-sink): vertex "${sink.name}" is not a vendor sink (type "${existing.type}").`);
849
- }
850
- replaceVertexByName(jobGraph, sink.name, sink, 'update-sink', index);
851
- if (filter !== undefined) {
852
- const filterName = `${sink.name}_filter`;
853
- if (!findVertexByName(jobGraph, filterName)) {
854
- throw new Error(`Operation ${index} (update-sink): sink "${sink.name}" has no generated gating filter "${filterName}" to update; ` +
855
- `use remove-sink + add-sink to introduce a gate.`);
856
- }
857
- replaceRawFilterVertex(jobGraph, filterName, filter, 'update-sink', index);
858
- }
859
- }
860
- // --- Job-graph in-place updates of reducer-list entries: anchor on the unique
861
- // reducer/remapper (as the add/remove paths do), then delegate to the shared updater. ---
862
- function jobGraphUpdateMessageAttribute(vertices, from, to, index) {
863
- const remapper = findUniqueVertexByType(vertices, LogAttributesRemapperType.log_attributes_remapper, 'update-message-attribute', index);
864
- updateMessageAttributeOnRemapper(remapper, from, to, index);
865
- }
866
- function jobGraphUpdateGroupBy(vertices, from, to, index) {
867
- const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'update-group-by', index);
868
- updateGroupByOnReducer(reducer, from, to, index);
869
- }
870
- function jobGraphUpdateAggregation(vertices, attributePath, strategies, index) {
871
- const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'update-aggregation-strategy', index);
872
- updateAggregationOnReducer(reducer, attributePath, strategies, index);
873
- }
874
- /** Replace the matching raw predicate in the reducer's `logReducerExceptions` in place (mirror of jobGraphAddReducerException). */
875
- function jobGraphUpdateReducerException(vertices, from, to, index) {
876
- const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'update-reducer-exception', index);
877
- const existing = reducer.logReducerExceptions ?? [];
878
- const idx = reducerExceptionUpdateIndex(existing.map(p => JSON.stringify(p)), from, to, index);
879
- existing[idx] = to;
880
- reducer.logReducerExceptions = existing;
881
- }
882
- function rawFilterNameForPhase(phase, index, opLabel) {
883
- switch (phase) {
884
- case 'pre-parser':
885
- return RAW_PRE_PARSER_FILTER;
886
- case 'pre-warehouse':
887
- return RAW_PRE_WAREHOUSE_FILTER;
888
- case 'pre-exceptions':
889
- return RAW_PRE_EXCEPTIONS_FILTER;
890
- default:
891
- throw new Error(`Operation ${index} (${opLabel}): phase "${phase}" has no canonical UI raw-graph stage. ` +
892
- ACCEPTED_TRANSFORM_PHASES_MESSAGE);
893
- }
894
- }
895
- function assertRawUiLogGraph(jobGraph, opLabel, index, requiredNames) {
896
- const seen = new Set();
897
- for (const vertex of jobGraph.vertices) {
898
- if (typeof vertex.name !== 'string' || vertex.name.length === 0) {
899
- throw unsupportedRawShapeError(index, opLabel, 'every vertex must have a non-empty name');
900
- }
901
- if (seen.has(vertex.name)) {
902
- throw unsupportedRawShapeError(index, opLabel, `duplicate vertex name "${vertex.name}"`);
903
- }
904
- seen.add(vertex.name);
905
- }
906
- const missing = requiredNames.filter(name => !seen.has(name));
907
- if (missing.length > 0) {
908
- throw unsupportedRawShapeError(index, opLabel, `missing canonical vertex: ${missing.join(', ')}`);
909
- }
910
- }
911
- function unsupportedRawShapeError(index, opLabel, detail) {
912
- return new Error(`Operation ${index} (${opLabel}): unsupported raw job graph shape. ` +
913
- `UI-level topology edits require a canonical UI log pipeline graph; ${detail}.`);
914
- }
915
397
  function assertOperationIdentity(operation, opLabel, index, fieldName) {
916
398
  if (!operation || typeof operation !== 'object') {
917
399
  throw new Error(`Operation ${index} (${opLabel}): ${fieldName} must be an object`);
@@ -923,145 +405,11 @@ function assertOperationIdentity(operation, opLabel, index, fieldName) {
923
405
  throw new Error(`Operation ${index} (${opLabel}): ${fieldName}.type must be a non-empty string`);
924
406
  }
925
407
  }
926
- function findVertexByName(jobGraph, name) {
927
- return jobGraph.vertices.find(vertex => vertex.name === name);
928
- }
929
- function findVertexByType(jobGraph, type) {
930
- return jobGraph.vertices.find(vertex => vertex.type === type);
931
- }
932
- function findVertexIndexByName(jobGraph, name) {
933
- return jobGraph.vertices.findIndex(vertex => vertex.name === name);
934
- }
935
- function replaceVertexByName(jobGraph, name, replacement, opLabel, index) {
936
- const idx = findVertexIndexByName(jobGraph, name);
937
- if (idx === -1) {
938
- throw unsupportedRawShapeError(index, opLabel, `vertex "${name}" not found`);
939
- }
940
- jobGraph.vertices[idx] = replacement;
941
- }
942
- function removeVertexByName(jobGraph, name) {
943
- const idx = findVertexIndexByName(jobGraph, name);
944
- if (idx !== -1) {
945
- jobGraph.vertices.splice(idx, 1);
946
- }
947
- }
948
- function graphEdges(jobGraph) {
949
- const graph = jobGraph;
950
- if (!Array.isArray(graph.edges)) {
951
- graph.edges = []; // Lazily initializes edges to [] if absent — normalizes the schema type omission.
952
- }
953
- return graph.edges;
954
- }
955
- function parsedEdges(jobGraph) {
956
- return graphEdges(jobGraph).map((edge, index) => ({ edge, index, ...parseEdge(edge) }));
957
- }
958
- /** True if any `logs-iceberg-table-sink` is reachable from `RAW_LOG_REDUCER` within two hops. */
959
- function hasReducerFedIcebergSink(jobGraph) {
960
- const edges = parsedEdges(jobGraph);
961
- const reducerNeighbors = new Set(edges.filter(e => e.sourceVertex === RAW_LOG_REDUCER).map(e => e.targetVertex));
962
- return jobGraph.vertices.some(v => v.type === LOGS_ICEBERG_TABLE_SINK_TYPE &&
963
- (reducerNeighbors.has(v.name) ||
964
- edges.some(e => reducerNeighbors.has(e.sourceVertex) && e.targetVertex === v.name)));
965
- }
966
- function formatEdge(sourceVertex, sourcePort, targetVertex, targetPort) {
967
- const source = sourcePort === DEFAULT_EDGE_OUTPUT ? sourceVertex : `${sourceVertex}:${sourcePort}`;
968
- const target = targetPort === DEFAULT_EDGE_INPUT ? targetVertex : `${targetVertex}:${targetPort}`;
969
- return `${source} -> ${target}`;
970
- }
971
- function addEdgeIfMissing(jobGraph, sourceVertex, sourcePort, targetVertex, targetPort) {
972
- const candidate = formatEdge(sourceVertex, sourcePort, targetVertex, targetPort);
973
- const exists = parsedEdges(jobGraph).some(edge => edge.sourceVertex === sourceVertex &&
974
- edge.sourcePort === sourcePort &&
975
- edge.targetVertex === targetVertex &&
976
- edge.targetPort === targetPort);
977
- if (!exists) {
978
- graphEdges(jobGraph).push(candidate);
979
- }
980
- }
981
- function removeEdgesTouching(jobGraph, name) {
982
- const keep = graphEdges(jobGraph).filter(edge => {
983
- const parsed = parseEdge(edge);
984
- return parsed.sourceVertex !== name && parsed.targetVertex !== name;
985
- });
986
- jobGraph.edges = keep;
987
- }
988
- function removeEdgeAt(jobGraph, index) {
989
- graphEdges(jobGraph).splice(index, 1);
990
- }
991
- function insertRawVertexBetween(jobGraph, vertex, predecessor, successor, opLabel, index) {
992
- const directEdges = parsedEdges(jobGraph).filter(edge => edge.sourceVertex === predecessor && edge.targetVertex === successor);
993
- if (directEdges.length !== 1) {
994
- throw unsupportedRawShapeError(index, opLabel, `expected exactly one parser-chain edge ${predecessor} -> ${successor}`);
995
- }
996
- const directEdge = directEdges[0];
997
- jobGraph.vertices.push(vertex);
998
- removeEdgeAt(jobGraph, directEdge.index);
999
- addEdgeIfMissing(jobGraph, predecessor, directEdge.sourcePort, vertex.name, DEFAULT_EDGE_INPUT);
1000
- addEdgeIfMissing(jobGraph, vertex.name, DEFAULT_EDGE_OUTPUT, successor, directEdge.targetPort);
1001
- }
1002
- function orderedParserNames(jobGraph, opLabel, index) {
1003
- const names = [];
1004
- let current = RAW_PRE_PARSER_FILTER;
1005
- const seen = new Set([current]);
1006
- while (current !== RAW_PRE_WAREHOUSE_FILTER) {
1007
- const outgoing = parsedEdges(jobGraph).filter(edge => edge.sourceVertex === current);
1008
- if (outgoing.length !== 1) {
1009
- throw unsupportedRawShapeError(index, opLabel, `parser chain vertex "${current}" must have exactly one outgoing main-chain edge`);
1010
- }
1011
- const next = outgoing[0].targetVertex;
1012
- if (next === RAW_PRE_WAREHOUSE_FILTER) {
1013
- return names;
1014
- }
1015
- const nextVertex = findVertexByName(jobGraph, next);
1016
- if (!nextVertex || !RAW_PARSER_TYPES.has(nextVertex.type)) {
1017
- throw unsupportedRawShapeError(index, opLabel, `vertex "${next}" between ${RAW_PRE_PARSER_FILTER} and ${RAW_PRE_WAREHOUSE_FILTER} is not a supported parser`);
1018
- }
1019
- if (seen.has(next)) {
1020
- throw unsupportedRawShapeError(index, opLabel, `cycle detected at parser vertex "${next}"`);
1021
- }
1022
- names.push(next);
1023
- seen.add(next);
1024
- current = next;
1025
- }
1026
- return names;
1027
- }
1028
- function singleMainSuccessor(jobGraph, name, opLabel, index) {
1029
- const outgoing = parsedEdges(jobGraph).filter(edge => edge.sourceVertex === name);
1030
- if (outgoing.length !== 1) {
1031
- throw unsupportedRawShapeError(index, opLabel, `vertex "${name}" must have exactly one outgoing main-chain edge`);
1032
- }
1033
- return outgoing[0].targetVertex;
1034
- }
1035
- function isCanonicalRawSource(jobGraph, name) {
1036
- const edges = parsedEdges(jobGraph);
1037
- const hasIncoming = edges.some(edge => edge.targetVertex === name);
1038
- const feedsPreParser = edges.some(edge => edge.sourceVertex === name && edge.targetVertex === RAW_PRE_PARSER_FILTER);
1039
- return !hasIncoming && feedsPreParser;
1040
- }
1041
408
  function assertProposedTemplateHasSource(input) {
1042
409
  if (!Array.isArray(input.sources) || input.sources.length === 0) {
1043
410
  throw new Error('Proposed job graph has zero sources; a log pipeline must keep at least one source.');
1044
411
  }
1045
412
  }
1046
- function assertProposedJobGraphHasSource(jobGraph) {
1047
- if (!jobGraph.vertices.some(vertex => typeof vertex.name === 'string' && isCanonicalRawSource(jobGraph, vertex.name))) {
1048
- throw new Error('Proposed job graph has zero sources; a log pipeline must keep at least one source.');
1049
- }
1050
- }
1051
- function applyAddMessageAttribute(input, attributePath, index) {
1052
- const remapper = findRemapper(input);
1053
- if (!remapper) {
1054
- throw new Error(`Operation ${index} (add-message-attribute): no log-attributes-remapper in input.parsers. ` +
1055
- `Template-backed pipelines normally include a remapper by default; if yours doesn't, add one via add-parser first.`);
1056
- }
1057
- applyMessageAttributeToRemapper(remapper, attributePath, index);
1058
- }
1059
- function applyAddGroupBy(input, attributePath, index) {
1060
- applyGroupByToReducer(input.reducer, attributePath, index);
1061
- }
1062
- function applyAddAggregation(input, attributePath, strategies, index) {
1063
- applyAggregationToReducer(input.reducer, attributePath, strategies, index);
1064
- }
1065
413
  function applyAddReducerException(input, predicate) {
1066
414
  // Wrapped in a TemplateQueryException; matching logs bypass aggregation.
1067
415
  const serialized = JSON.stringify(predicate);
@@ -1081,9 +429,12 @@ function applyAddGrokRule(input, pattern, parserName, extractAttribute, index) {
1081
429
  }
1082
430
  /** Resolve the target grok-parser in `input.parsers` by name (or the sole grok-parser); shared by add/remove-grok-rule. Throws if absent, ambiguous, or not a grok-parser. */
1083
431
  function findTemplateGrokParser(parsers, parserName, opLabel, index) {
1084
- const grok = parserName
1085
- ? parsers.find(p => p.name === parserName)
1086
- : selectGrokParser(parsers.filter(p => p.type === GrokParserType.grok_parser), undefined, 'input.parsers', opLabel, index);
432
+ const groks = parsers.filter(p => p.type === GrokParserType.grok_parser);
433
+ if (!parserName && groks.length > 1) {
434
+ throw new Error(`Operation ${index} (${opLabel}): found ${groks.length} grok parsers in input.parsers; ` +
435
+ `pass parserName to choose the target parser.`);
436
+ }
437
+ const grok = parserName ? parsers.find(p => p.name === parserName) : groks[0];
1087
438
  if (!grok) {
1088
439
  const hint = opLabel === 'add-grok-rule' ? ' If you need to introduce one, use add-parser first.' : '';
1089
440
  throw new Error(`Operation ${index} (${opLabel}): ${parserName ? `parser "${parserName}" not found` : 'no grok-parser found in input.parsers'}.${hint}`);
@@ -1247,8 +598,8 @@ function findSimpleRootFilterGate(root) {
1247
598
  * Install or update a filter gate at the phase's transform slot:
1248
599
  * - empty slot: `IF filter THEN PASSTHROUGH ELSE DROP`
1249
600
  * - existing simple gate: rebuild with the new predicate, keeping the pass side
1250
- * and inheriting `inverted`/`lateDataDuration` when the patch omits them (raw
1251
- * merge semantics — a predicate-only edit must not flip keep↔drop)
601
+ * and inheriting `inverted`/`lateDataDuration` when the patch omits them, so a
602
+ * predicate-only edit does not flip keep and drop
1252
603
  * - any other existing chain (e.g. a SqlNode): `IF filter THEN <chain> ELSE DROP`
1253
604
  */
1254
605
  function applySetFilter(input, phase, filter, index) {
@@ -1425,21 +776,7 @@ function applyRemoveSource(input, name, index) {
1425
776
  }
1426
777
  input.sources.splice(idx, 1);
1427
778
  }
1428
- // --- Template per-entry removals: resolve the same reducer/remapper/grok the add
1429
- // path writes, then delegate to the shared remover. ---
1430
- function applyRemoveMessageAttribute(input, attributePath, index) {
1431
- const remapper = findRemapper(input);
1432
- if (!remapper) {
1433
- throw new Error(`Operation ${index} (remove-message-attribute): no log-attributes-remapper in input.parsers.`);
1434
- }
1435
- removeMessageAttributeFromRemapper(remapper, attributePath, index);
1436
- }
1437
- function applyRemoveGroupBy(input, attributePath, index) {
1438
- removeGroupByFromReducer(input.reducer, attributePath, index);
1439
- }
1440
- function applyRemoveAggregation(input, attributePath, strategies, index) {
1441
- removeAggregationFromReducer(input.reducer, attributePath, strategies, index);
1442
- }
779
+ // --- Template per-entry removals. ---
1443
780
  /** Remove the `TemplateQueryException` whose predicate matches (mirror of applyAddReducerException). */
1444
781
  function applyRemoveReducerException(input, predicate, index) {
1445
782
  const serialized = JSON.stringify(predicate);
@@ -1503,21 +840,7 @@ function applyUpdateSink(input, target, sink, filter, index) {
1503
840
  // Validated as a logs-iceberg-table-sink by assertSinkTargetShape.
1504
841
  input.processedLogsSink = sink;
1505
842
  }
1506
- // --- Template in-place updates of reducer-list entries: resolve the same
1507
- // reducer/remapper the add/remove paths use, then delegate to the shared updater. ---
1508
- function applyUpdateMessageAttribute(input, from, to, index) {
1509
- const remapper = findRemapper(input);
1510
- if (!remapper) {
1511
- throw new Error(`Operation ${index} (update-message-attribute): no log-attributes-remapper in input.parsers.`);
1512
- }
1513
- updateMessageAttributeOnRemapper(remapper, from, to, index);
1514
- }
1515
- function applyUpdateGroupBy(input, from, to, index) {
1516
- updateGroupByOnReducer(input.reducer, from, to, index);
1517
- }
1518
- function applyUpdateAggregation(input, attributePath, strategies, index) {
1519
- updateAggregationOnReducer(input.reducer, attributePath, strategies, index);
1520
- }
843
+ // --- Template in-place updates of reducer-list entries. ---
1521
844
  /** Replace the matching `TemplateQueryException`'s predicate in place (mirror of applyAddReducerException). */
1522
845
  function applyUpdateReducerException(input, from, to, index) {
1523
846
  const exceptions = input.exceptions ?? [];
@@ -1534,7 +857,7 @@ function assertValidSinkTarget(target, opLabel, index) {
1534
857
  throw new Error(`Operation ${index} (${opLabel}): target must be "vendor" or "processed-logs", got ${JSON.stringify(target)}.`);
1535
858
  }
1536
859
  }
1537
- /** Validate the sink type matches the target and `filter` is vendor-only; shared by add-sink and update-sink on both backends. */
860
+ /** Validate the sink type matches the target and `filter` is vendor-only; shared by add-sink and update-sink. */
1538
861
  function assertSinkTargetShape(target, sink, filter, index, opLabel) {
1539
862
  assertValidSinkTarget(target, opLabel, index);
1540
863
  if (target === 'vendor') {
@@ -1711,9 +1034,9 @@ function touchesSinkConfig(op, index) {
1711
1034
  }
1712
1035
  }
1713
1036
  /**
1714
- * Find the single `template-operation` vertex (exactly one per template-backed
1715
- * job); throws on 0 (non-template) or >1 (unsupported). Accepts `SchemaReadJob`
1716
- * or `SchemaUpdateJob` without casts.
1037
+ * Find the single `template-operation` vertex of a template-backed job. This is
1038
+ * the guard that rejects a raw job graph pipeline (0 matches); it also rejects
1039
+ * more than 1 match. Accepts `SchemaReadJob` or `SchemaUpdateJob` without casts.
1717
1040
  */
1718
1041
  export function findTemplateOperation(job) {
1719
1042
  const vertices = job.jobGraph?.vertices;
@@ -1722,8 +1045,9 @@ export function findTemplateOperation(job) {
1722
1045
  }
1723
1046
  const matches = vertices.filter(v => v.type === TemplateOperationType.template_operation);
1724
1047
  if (matches.length === 0) {
1725
- throw new Error(`Pipeline is not template-backed (no template-operation vertex). ` +
1726
- `This helper only applies to template-backed pipelines.`);
1048
+ throw new Error('This pipeline uses a raw job graph (no template-operation vertex). ' +
1049
+ 'job:plan, job:draft and job:apply support only template-backed pipelines. ' +
1050
+ 'To change a raw job graph, edit it and run job:update.');
1727
1051
  }
1728
1052
  if (matches.length > 1) {
1729
1053
  throw new Error(`Pipeline has ${matches.length} template-operation vertices; expected exactly 1. ` +
@@ -1748,8 +1072,14 @@ function writeTemplateInput(templateOp, input) {
1748
1072
  // templateInputs values are typed `unknown`, so the structured input assigns directly — no cast needed.
1749
1073
  templateOp.templateInputs['input'] = input;
1750
1074
  }
1751
- function findRemapper(input) {
1752
- return input.parsers.find(p => p.type === LogAttributesRemapperType.log_attributes_remapper);
1075
+ const ADD_PARSER_REMAPPER_HINT = `Template-backed pipelines normally include a remapper by default; if yours doesn't, add one via add-parser first.`;
1076
+ function requireRemapper(input, opLabel, index, hint) {
1077
+ const remapper = input.parsers.find(p => p.type === LogAttributesRemapperType.log_attributes_remapper);
1078
+ if (!remapper) {
1079
+ const message = `Operation ${index} (${opLabel}): no log-attributes-remapper in input.parsers.`;
1080
+ throw new Error(hint === undefined ? message : `${message} ${hint}`);
1081
+ }
1082
+ return remapper;
1753
1083
  }
1754
1084
  function splitPath(path, index) {
1755
1085
  if (path.length === 0) {