@grepr/cli 1.14.13-7f3738f → 1.15.0-f5dcf28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/dist/commands/job-draft-command.d.ts.map +1 -1
- package/build/dist/commands/job-draft-command.js +6 -71
- package/build/dist/commands/job-draft-command.js.map +1 -1
- package/build/dist/docs-index/index.json +1 -1
- package/build/dist/grepr.d.ts.map +1 -1
- package/build/dist/grepr.js +0 -2
- package/build/dist/grepr.js.map +1 -1
- package/build/dist/lib/backfill.js +4 -8
- package/build/dist/lib/backfill.js.map +1 -1
- package/build/dist/lib/job-patch.d.ts +15 -31
- package/build/dist/lib/job-patch.d.ts.map +1 -1
- package/build/dist/lib/job-patch.js +39 -709
- package/build/dist/lib/job-patch.js.map +1 -1
- package/build/dist/lib/job-plan.d.ts +3 -6
- package/build/dist/lib/job-plan.d.ts.map +1 -1
- package/build/dist/lib/job-plan.js +5 -55
- package/build/dist/lib/job-plan.js.map +1 -1
- package/build/dist/lib/option-parsers.d.ts +0 -5
- package/build/dist/lib/option-parsers.d.ts.map +1 -1
- package/build/dist/lib/option-parsers.js +0 -7
- package/build/dist/lib/option-parsers.js.map +1 -1
- package/build/dist/lib/signal-source.d.ts +1 -1
- package/build/dist/lib/signal-source.d.ts.map +1 -1
- package/build/dist/lib/signal-source.js +10 -69
- package/build/dist/lib/signal-source.js.map +1 -1
- package/build/dist/openapi/openApiTypes.d.ts +188 -9
- package/build/dist/openapi/openApiTypes.d.ts.map +1 -1
- package/build/dist/openapi/openApiTypes.js +21 -0
- package/build/dist/openapi/openApiTypes.js.map +1 -1
- package/build/dist/openapi-docs/api/Agents/create_2.md +1 -1
- package/build/dist/openapi-docs/api/Agents/investigations.md +1 -1
- package/build/dist/openapi-docs/api/Agents/update.md +1 -1
- package/build/dist/openapi-docs/api/Datasets/getDatasetDataTypes.md +25 -0
- package/build/dist/openapi-docs/schemas/AgentConfigCreate.md +3 -0
- package/build/dist/openapi-docs/schemas/AgentConfigPayload.md +3 -0
- package/build/dist/openapi-docs/schemas/AgentConfigUpdate.md +3 -0
- package/build/dist/openapi-docs/schemas/AgentSchedule.md +16 -0
- package/build/dist/openapi-docs/schemas/AnalyticsDiagnostic.md +4 -1
- package/build/dist/openapi-docs/schemas/AnalyticsDiagnosticRange.md +19 -0
- package/build/dist/openapi-docs/schemas/AnalyticsDraftRequest.md +2 -2
- package/build/dist/openapi-docs/schemas/AnalyticsField.md +6 -0
- package/build/dist/openapi-docs/schemas/AnalyticsQuery.md +3 -0
- package/build/dist/openapi-docs/schemas/AnalyticsResultGrouping.md +23 -0
- package/build/dist/openapi-docs/schemas/AnalyticsResultShape.md +22 -0
- package/build/dist/openapi-docs/schemas/AnalyticsTableBinding.md +1 -1
- package/build/dist/openapi-docs/schemas/AnalyticsVocabularyResponse.md +4 -0
- package/build/dist/openapi-docs/schemas/AthenaAnalyticsSource.md +2 -2
- package/build/dist/openapi-docs/schemas/DatasetDataTypes.md +9 -0
- package/build/dist/openapi-docs/schemas/EventRecord.md +11 -0
- package/build/dist/openapi-docs/schemas/SqlAnalyticsQuery.md +29 -0
- package/build/dist/openapi-docs/schemas/SqlAnalyticsTable.md +16 -0
- package/build/dist/openapi-docs/schemas/TrinoAnalyticsSource.md +2 -2
- package/build/dist/types.d.ts +8 -37
- package/build/dist/types.d.ts.map +1 -1
- package/build/dist/types.js +0 -69
- package/build/dist/types.js.map +1 -1
- package/package.json +1 -1
- package/build/dist/commands/job-to-test-command.d.ts +0 -74
- package/build/dist/commands/job-to-test-command.d.ts.map +0 -1
- package/build/dist/commands/job-to-test-command.js +0 -161
- package/build/dist/commands/job-to-test-command.js.map +0 -1
- package/build/dist/lib/job-graph-log-pipeline-constants.d.ts +0 -12
- package/build/dist/lib/job-graph-log-pipeline-constants.d.ts.map +0 -1
- package/build/dist/lib/job-graph-log-pipeline-constants.js +0 -16
- package/build/dist/lib/job-graph-log-pipeline-constants.js.map +0 -1
- package/build/dist/lib/job-graph-transformer.d.ts +0 -119
- package/build/dist/lib/job-graph-transformer.d.ts.map +0 -1
- package/build/dist/lib/job-graph-transformer.js +0 -767
- package/build/dist/lib/job-graph-transformer.js.map +0 -1
- package/build/dist/lib/job-graph-utils.d.ts +0 -50
- package/build/dist/lib/job-graph-utils.d.ts.map +0 -1
- package/build/dist/lib/job-graph-utils.js +0 -90
- package/build/dist/lib/job-graph-utils.js.map +0 -1
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
/** Structured patch format for grepr pipelines: a list of ops applied locally before any production write
|
|
2
|
-
import { ConditionNodeKind, MaskingOperatorType, DropNodeKind, PassthroughNodeKind, SqlNodeKind, SqlNodeOutputRouting, SqlOperationType, SqlOutputStatementType, LogAttributesRemapperType,
|
|
3
|
-
import { parseEdge } from './job-graph-utils.js';
|
|
4
|
-
import { RAW_ATTRIBUTES_REMAPPER, RAW_ATTRIBUTES_REMAPPER_TYPE, RAW_JSON_PROCESSOR, RAW_JSON_PROCESSOR_TYPE, RAW_LOG_REDUCER, RAW_PARSER_TYPES, RAW_PRE_EXCEPTIONS_FILTER, RAW_PRE_PARSER_FILTER, RAW_PRE_WAREHOUSE_FILTER, } from './job-graph-log-pipeline-constants.js';
|
|
1
|
+
/** Structured patch format for template-backed grepr pipelines: a list of ops applied to `templateInputs.input` locally before any production write. */
|
|
2
|
+
import { ConditionNodeKind, MaskingOperatorType, DropNodeKind, PassthroughNodeKind, SqlNodeKind, SqlNodeOutputRouting, SqlOperationType, SqlOutputStatementType, LogAttributesRemapperType, GrokParserType, DatadogQueryPredicateType, DatadogLogSinkType, SplunkLogSinkType, NewRelicLogSinkType, SumoLogSinkType, OtlpLogSinkType, LogsIcebergTableSinkType, TemplateOperationType, TemplateQueryExceptionType, SumAttributesMergeStrategyType, MinAttributesMergeStrategyType, MaxAttributesMergeStrategyType, AverageAttributesMergeStrategyType, } from '../openapi/openApiTypes.js';
|
|
5
3
|
/** The accepted-phases suffix shared by every template transform-phase error. */
|
|
6
4
|
const ACCEPTED_TRANSFORM_PHASES_MESSAGE = 'Accepted phases are pre-parser, pre-warehouse, and pre-exceptions.';
|
|
7
5
|
/** The downstream pipeline steps a `sql_output` may route to (the generated {@link SqlNodeOutputRouting} values). */
|
|
@@ -16,11 +14,6 @@ const VENDOR_LOG_SINK_TYPES = new Set([
|
|
|
16
14
|
OtlpLogSinkType.otlp_log_sink,
|
|
17
15
|
]);
|
|
18
16
|
const LOGS_ICEBERG_TABLE_SINK_TYPE = LogsIcebergTableSinkType.logs_iceberg_table_sink;
|
|
19
|
-
// Raw data-lake and processed-logs sinks share the logs-iceberg-table-sink type
|
|
20
|
-
// and are distinguished only by these vertex-name prefixes — a naming convention
|
|
21
|
-
// this CLI must match, not invent.
|
|
22
|
-
const RAW_DATA_LAKE_SINK_NAME_PREFIX = 'raw_data_sink';
|
|
23
|
-
const PROCESSED_LOGS_SINK_NAME_PREFIX = 'processed_logs_';
|
|
24
17
|
/**
|
|
25
18
|
* Required fields per op, keyed by op name. Typed as `Record<JobPatchOp['op'], …>`
|
|
26
19
|
* so a new op added to the union fails to compile until its entry is added here —
|
|
@@ -131,23 +124,12 @@ export function parsePatch(raw) {
|
|
|
131
124
|
}
|
|
132
125
|
return { operations: parsedOperations };
|
|
133
126
|
}
|
|
134
|
-
/** Detect whether a job is template-backed (a `template-operation` vertex) or job-graph. */
|
|
135
|
-
export function detectBackend(job) {
|
|
136
|
-
const vertices = job.jobGraph?.vertices;
|
|
137
|
-
if (!Array.isArray(vertices))
|
|
138
|
-
return 'job-graph';
|
|
139
|
-
return vertices.some(v => v.type === TemplateOperationType.template_operation) ? 'template' : 'job-graph';
|
|
140
|
-
}
|
|
141
127
|
/**
|
|
142
128
|
* Apply a patch to a fetched job, producing the `SchemaUpdateJob` payload for
|
|
143
|
-
* `job:apply`. Pure (clones input)
|
|
144
|
-
* targets a missing field
|
|
129
|
+
* `job:apply`. Pure (clones input). Throws if the job is not template-backed or
|
|
130
|
+
* an op targets a missing field.
|
|
145
131
|
*/
|
|
146
132
|
export function applyPatch(job, patch) {
|
|
147
|
-
const backend = detectBackend(job);
|
|
148
|
-
return backend === 'template' ? applyPatchToTemplate(job, patch) : applyPatchToJobGraph(job, patch);
|
|
149
|
-
}
|
|
150
|
-
function applyPatchToTemplate(job, patch) {
|
|
151
133
|
const cloned = structuredClone(job);
|
|
152
134
|
const templateOp = findTemplateOperation(cloned);
|
|
153
135
|
const input = readTemplateInput(templateOp);
|
|
@@ -167,44 +149,14 @@ function applyPatchToTemplate(job, patch) {
|
|
|
167
149
|
teamIds: cloned.teamIds,
|
|
168
150
|
};
|
|
169
151
|
}
|
|
170
|
-
/**
|
|
171
|
-
* Apply a patch to a job-graph (non-template) pipeline. Clones the job and
|
|
172
|
-
* mutates the resolved graph on the clone (the input is left untouched). Field
|
|
173
|
-
* names match the template inputs except `add-reducer-exception`, which appends
|
|
174
|
-
* a raw `EventPredicate` to the reducer vertex's `logReducerExceptions`. Topology
|
|
175
|
-
* ops require the canonical UI log-pipeline chain.
|
|
176
|
-
*/
|
|
177
|
-
function applyPatchToJobGraph(job, patch) {
|
|
178
|
-
const cloned = structuredClone(job);
|
|
179
|
-
const jobGraph = cloned.jobGraph;
|
|
180
|
-
const vertices = jobGraph?.vertices;
|
|
181
|
-
if (!jobGraph || !Array.isArray(vertices) || vertices.length === 0) {
|
|
182
|
-
throw new Error('Job has no jobGraph.vertices to patch');
|
|
183
|
-
}
|
|
184
|
-
if (!Array.isArray(jobGraph.edges)) {
|
|
185
|
-
jobGraph.edges = [];
|
|
186
|
-
}
|
|
187
|
-
for (let i = 0; i < patch.operations.length; i++) {
|
|
188
|
-
applyJobGraphOperation(jobGraph, patch.operations[i], i);
|
|
189
|
-
}
|
|
190
|
-
if (patch.operations.some(touchesSourceConfig)) {
|
|
191
|
-
assertProposedJobGraphHasSource(jobGraph);
|
|
192
|
-
}
|
|
193
|
-
return {
|
|
194
|
-
desiredState: cloned.desiredState,
|
|
195
|
-
fromVersion: cloned.version,
|
|
196
|
-
jobGraph: cloned.jobGraph,
|
|
197
|
-
teamIds: cloned.teamIds,
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
152
|
function applyOperation(input, op, index) {
|
|
201
153
|
switch (op.op) {
|
|
202
154
|
case 'add-message-attribute':
|
|
203
|
-
return
|
|
155
|
+
return applyMessageAttributeToRemapper(requireRemapper(input, 'add-message-attribute', index, ADD_PARSER_REMAPPER_HINT), op.attributePath, index);
|
|
204
156
|
case 'add-group-by':
|
|
205
|
-
return
|
|
157
|
+
return applyGroupByToReducer(input.reducer, op.attributePath, index);
|
|
206
158
|
case 'add-aggregation-strategy':
|
|
207
|
-
return
|
|
159
|
+
return applyAggregationToReducer(input.reducer, op.attributePath, op.strategies, index);
|
|
208
160
|
case 'add-reducer-exception':
|
|
209
161
|
return applyAddReducerException(input, op.predicate);
|
|
210
162
|
case 'add-grok-rule':
|
|
@@ -244,11 +196,11 @@ function applyOperation(input, op, index) {
|
|
|
244
196
|
case 'set-raw-dataset':
|
|
245
197
|
return applySetRawDataset(input, op.datasetId);
|
|
246
198
|
case 'remove-message-attribute':
|
|
247
|
-
return
|
|
199
|
+
return removeMessageAttributeFromRemapper(requireRemapper(input, 'remove-message-attribute', index), op.attributePath, index);
|
|
248
200
|
case 'remove-group-by':
|
|
249
|
-
return
|
|
201
|
+
return removeGroupByFromReducer(input.reducer, op.attributePath, index);
|
|
250
202
|
case 'remove-aggregation-strategy':
|
|
251
|
-
return
|
|
203
|
+
return removeAggregationFromReducer(input.reducer, op.attributePath, op.strategies, index);
|
|
252
204
|
case 'remove-reducer-exception':
|
|
253
205
|
return applyRemoveReducerException(input, op.predicate, index);
|
|
254
206
|
case 'remove-grok-rule':
|
|
@@ -261,11 +213,11 @@ function applyOperation(input, op, index) {
|
|
|
261
213
|
// See add-sink: read the vendor-only `filter` via `in` so a JSON patch's stray field still reaches the guard.
|
|
262
214
|
return applyUpdateSink(input, op.target, op.sink, 'filter' in op ? op.filter : undefined, index);
|
|
263
215
|
case 'update-message-attribute':
|
|
264
|
-
return
|
|
216
|
+
return updateMessageAttributeOnRemapper(requireRemapper(input, 'update-message-attribute', index), op.from, op.to, index);
|
|
265
217
|
case 'update-group-by':
|
|
266
|
-
return
|
|
218
|
+
return updateGroupByOnReducer(input.reducer, op.from, op.to, index);
|
|
267
219
|
case 'update-aggregation-strategy':
|
|
268
|
-
return
|
|
220
|
+
return updateAggregationOnReducer(input.reducer, op.attributePath, op.strategies, index);
|
|
269
221
|
case 'update-reducer-exception':
|
|
270
222
|
return applyUpdateReducerException(input, op.from, op.to, index);
|
|
271
223
|
default:
|
|
@@ -277,143 +229,6 @@ function throwUnknownOp(op, index) {
|
|
|
277
229
|
const unknownOp = op;
|
|
278
230
|
throw new Error(`Operation ${index}: unknown op "${unknownOp.op ?? '(missing)'}"`);
|
|
279
231
|
}
|
|
280
|
-
function applyJobGraphOperation(jobGraph, op, index) {
|
|
281
|
-
const vertices = jobGraph.vertices;
|
|
282
|
-
switch (op.op) {
|
|
283
|
-
case 'add-message-attribute':
|
|
284
|
-
return jobGraphAddMessageAttribute(vertices, op.attributePath, index);
|
|
285
|
-
case 'add-group-by':
|
|
286
|
-
return jobGraphAddGroupBy(vertices, op.attributePath, index);
|
|
287
|
-
case 'add-aggregation-strategy':
|
|
288
|
-
return jobGraphAddAggregation(vertices, op.attributePath, op.strategies, index);
|
|
289
|
-
case 'add-reducer-exception':
|
|
290
|
-
return jobGraphAddReducerException(vertices, op.predicate, index);
|
|
291
|
-
case 'add-grok-rule':
|
|
292
|
-
return jobGraphAddGrokRule(vertices, op.pattern, op.parserName, op.extractAttribute, index);
|
|
293
|
-
case 'add-parser':
|
|
294
|
-
return jobGraphAddParser(jobGraph, op.parser, index);
|
|
295
|
-
case 'remove-parser':
|
|
296
|
-
return jobGraphRemoveParser(jobGraph, op.name, index);
|
|
297
|
-
case 'set-filter':
|
|
298
|
-
return jobGraphSetFilter(jobGraph, op.phase, op.filter, index);
|
|
299
|
-
case 'clear-filter':
|
|
300
|
-
return jobGraphClearFilter(jobGraph, op.phase, index);
|
|
301
|
-
case 'set-sql-transform':
|
|
302
|
-
throw new Error(`Operation ${index} (set-sql-transform): SQL transform edits are only supported for ` +
|
|
303
|
-
`template-backed log-reducer jobs in the CLI. This job is a direct job graph. `);
|
|
304
|
-
case 'set-transform-chain':
|
|
305
|
-
case 'clear-transform-chain':
|
|
306
|
-
throw new Error(`Operation ${index} (${op.op}): transform-chain edits are only supported for ` +
|
|
307
|
-
`template-backed log-reducer jobs in the CLI. This job is a direct job graph. `);
|
|
308
|
-
case 'set-masking':
|
|
309
|
-
case 'clear-masking':
|
|
310
|
-
throw new Error(`Operation ${index} (${op.op}): masking operator edits are only supported for ` +
|
|
311
|
-
`template-backed log-reducer jobs in the CLI. This job is a direct job graph. `);
|
|
312
|
-
case 'add-source':
|
|
313
|
-
return jobGraphAddSource(jobGraph, op.source, index);
|
|
314
|
-
case 'remove-source':
|
|
315
|
-
return jobGraphRemoveSource(jobGraph, op.name, index);
|
|
316
|
-
case 'add-sink':
|
|
317
|
-
// See the template dispatch: read the target-specific field via `in`.
|
|
318
|
-
return jobGraphAddSink(jobGraph, op.target, op.sink, 'filter' in op ? op.filter : undefined, index);
|
|
319
|
-
case 'remove-sink':
|
|
320
|
-
return jobGraphRemoveSink(jobGraph, op.target, 'name' in op ? op.name : undefined, index);
|
|
321
|
-
case 'set-raw-dataset':
|
|
322
|
-
return jobGraphSetRawDataset(jobGraph, op.datasetId, index);
|
|
323
|
-
case 'remove-message-attribute':
|
|
324
|
-
return jobGraphRemoveMessageAttribute(vertices, op.attributePath, index);
|
|
325
|
-
case 'remove-group-by':
|
|
326
|
-
return jobGraphRemoveGroupBy(vertices, op.attributePath, index);
|
|
327
|
-
case 'remove-aggregation-strategy':
|
|
328
|
-
return jobGraphRemoveAggregation(vertices, op.attributePath, op.strategies, index);
|
|
329
|
-
case 'remove-reducer-exception':
|
|
330
|
-
return jobGraphRemoveReducerException(vertices, op.predicate, index);
|
|
331
|
-
case 'remove-grok-rule':
|
|
332
|
-
return jobGraphRemoveGrokRule(vertices, op.pattern, op.parserName, index);
|
|
333
|
-
case 'update-source':
|
|
334
|
-
return jobGraphUpdateSource(jobGraph, op.source, index);
|
|
335
|
-
case 'update-parser':
|
|
336
|
-
return jobGraphUpdateParser(jobGraph, op.parser, index);
|
|
337
|
-
case 'update-sink':
|
|
338
|
-
// See add-sink: read the vendor-only `filter` via `in`.
|
|
339
|
-
return jobGraphUpdateSink(jobGraph, op.target, op.sink, 'filter' in op ? op.filter : undefined, index);
|
|
340
|
-
case 'update-message-attribute':
|
|
341
|
-
return jobGraphUpdateMessageAttribute(vertices, op.from, op.to, index);
|
|
342
|
-
case 'update-group-by':
|
|
343
|
-
return jobGraphUpdateGroupBy(vertices, op.from, op.to, index);
|
|
344
|
-
case 'update-aggregation-strategy':
|
|
345
|
-
return jobGraphUpdateAggregation(vertices, op.attributePath, op.strategies, index);
|
|
346
|
-
case 'update-reducer-exception':
|
|
347
|
-
return jobGraphUpdateReducerException(vertices, op.from, op.to, index);
|
|
348
|
-
case 'set-input-field':
|
|
349
|
-
case 'unset-input-field':
|
|
350
|
-
throw new Error(`Operation ${index} (${op.op}): generic template-input paths are not supported on raw job graphs. ` +
|
|
351
|
-
`Use a semantic operation, or apply this change directly via ` +
|
|
352
|
-
`'grepr job:get' + manual edit + 'grepr job:update'.`);
|
|
353
|
-
default:
|
|
354
|
-
return throwUnknownOp(op, index);
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
function findUniqueVertexByType(vertices, type, opLabel, index) {
|
|
358
|
-
const matches = vertices.filter(v => v.type === type);
|
|
359
|
-
if (matches.length === 0) {
|
|
360
|
-
throw new Error(`Operation ${index} (${opLabel}): no ${type} vertex found in jobGraph.vertices. ` +
|
|
361
|
-
`Non-template pipelines must already include this vertex; this CLI doesn't add new vertices to job-graph pipelines.`);
|
|
362
|
-
}
|
|
363
|
-
if (matches.length > 1) {
|
|
364
|
-
throw new Error(`Operation ${index} (${opLabel}): expected exactly one ${type} vertex but found ${matches.length}. ` +
|
|
365
|
-
`Ambiguous target; resolve by hand via 'grepr job:get' + 'grepr job:update'.`);
|
|
366
|
-
}
|
|
367
|
-
return matches[0];
|
|
368
|
-
}
|
|
369
|
-
function jobGraphAddMessageAttribute(vertices, attributePath, index) {
|
|
370
|
-
const remapper = findUniqueVertexByType(vertices, LogAttributesRemapperType.log_attributes_remapper, 'add-message-attribute', index);
|
|
371
|
-
applyMessageAttributeToRemapper(remapper, attributePath, index);
|
|
372
|
-
}
|
|
373
|
-
function jobGraphAddGroupBy(vertices, attributePath, index) {
|
|
374
|
-
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'add-group-by', index);
|
|
375
|
-
applyGroupByToReducer(reducer, attributePath, index);
|
|
376
|
-
}
|
|
377
|
-
function jobGraphAddAggregation(vertices, attributePath, strategies, index) {
|
|
378
|
-
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'add-aggregation-strategy', index);
|
|
379
|
-
applyAggregationToReducer(reducer, attributePath, strategies, index);
|
|
380
|
-
}
|
|
381
|
-
/** Append a predicate to the reducer's `logReducerExceptions` (stored raw, unlike the template path's `TemplateQueryException` wrapper). */
|
|
382
|
-
function jobGraphAddReducerException(vertices, predicate, index) {
|
|
383
|
-
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'add-reducer-exception', index);
|
|
384
|
-
const existing = reducer.logReducerExceptions ?? [];
|
|
385
|
-
const serialized = JSON.stringify(predicate);
|
|
386
|
-
if (existing.some(p => JSON.stringify(p) === serialized)) {
|
|
387
|
-
reducer.logReducerExceptions = existing;
|
|
388
|
-
return; // idempotent
|
|
389
|
-
}
|
|
390
|
-
existing.push(predicate);
|
|
391
|
-
reducer.logReducerExceptions = existing;
|
|
392
|
-
}
|
|
393
|
-
function jobGraphAddGrokRule(vertices, pattern, parserName, extractAttribute, index) {
|
|
394
|
-
const grok = findJobGraphGrokParser(vertices, parserName, 'add-grok-rule', index);
|
|
395
|
-
applyGrokRuleToParser(grok, pattern, extractAttribute);
|
|
396
|
-
}
|
|
397
|
-
/** Resolve the target grok-parser vertex by name (or the sole grok-parser); shared by add/remove-grok-rule. Throws if absent or ambiguous. */
|
|
398
|
-
function findJobGraphGrokParser(vertices, parserName, opLabel, index) {
|
|
399
|
-
const groks = vertices.filter(v => v.type === GrokParserType.grok_parser);
|
|
400
|
-
const grok = selectGrokParser(groks, parserName, 'jobGraph.vertices', opLabel, index);
|
|
401
|
-
if (!grok) {
|
|
402
|
-
const hint = opLabel === 'add-grok-rule' ? " Add a grok-parser vertex via 'grepr job:update' before using add-grok-rule on this pipeline." : '';
|
|
403
|
-
throw new Error(`Operation ${index} (${opLabel}): ${parserName ? `grok-parser "${parserName}" not found` : 'no grok-parser vertex found in jobGraph.vertices'}.${hint}`);
|
|
404
|
-
}
|
|
405
|
-
return grok;
|
|
406
|
-
}
|
|
407
|
-
function selectGrokParser(groks, parserName, location, opLabel, index) {
|
|
408
|
-
if (parserName) {
|
|
409
|
-
return groks.find(parser => parser.name === parserName);
|
|
410
|
-
}
|
|
411
|
-
if (groks.length > 1) {
|
|
412
|
-
throw new Error(`Operation ${index} (${opLabel}): found ${groks.length} grok parsers in ${location}; ` +
|
|
413
|
-
`pass parserName to choose the target parser.`);
|
|
414
|
-
}
|
|
415
|
-
return groks[0];
|
|
416
|
-
}
|
|
417
232
|
function applyMessageAttributeToRemapper(remapper, attributePath, index) {
|
|
418
233
|
const parts = splitPath(attributePath, index);
|
|
419
234
|
if (parts.length === 1) {
|
|
@@ -471,7 +286,7 @@ function entryNotFoundError(index, opLabel, what) {
|
|
|
471
286
|
return new Error(`Operation ${index} (${opLabel}): ${what} not found.`);
|
|
472
287
|
}
|
|
473
288
|
// remove-* counterparts of the add-* reducer/remapper/grok writers: mutate the same
|
|
474
|
-
//
|
|
289
|
+
// field and throw if the entry is absent.
|
|
475
290
|
function removeMessageAttributeFromRemapper(remapper, attributePath, index) {
|
|
476
291
|
const parts = splitPath(attributePath, index);
|
|
477
292
|
const removed = parts.length === 1
|
|
@@ -579,339 +394,6 @@ function reducerExceptionUpdateIndex(serialized, from, to, index) {
|
|
|
579
394
|
}
|
|
580
395
|
return at;
|
|
581
396
|
}
|
|
582
|
-
const DEFAULT_EDGE_OUTPUT = 'output';
|
|
583
|
-
const DEFAULT_EDGE_INPUT = 'input';
|
|
584
|
-
function jobGraphAddSource(jobGraph, source, index) {
|
|
585
|
-
assertRawUiLogGraph(jobGraph, 'add-source', index, [RAW_PRE_PARSER_FILTER]);
|
|
586
|
-
assertOperationIdentity(source, 'add-source', index, 'source');
|
|
587
|
-
if (findVertexIndexByName(jobGraph, source.name) !== -1) {
|
|
588
|
-
throw new Error(`Operation ${index} (add-source): source "${source.name}" already exists`);
|
|
589
|
-
}
|
|
590
|
-
jobGraph.vertices.push(source);
|
|
591
|
-
addEdgeIfMissing(jobGraph, source.name, DEFAULT_EDGE_OUTPUT, RAW_PRE_PARSER_FILTER, DEFAULT_EDGE_INPUT);
|
|
592
|
-
}
|
|
593
|
-
function jobGraphRemoveSource(jobGraph, name, index) {
|
|
594
|
-
assertRawUiLogGraph(jobGraph, 'remove-source', index, [RAW_PRE_PARSER_FILTER]);
|
|
595
|
-
const idx = findVertexIndexByName(jobGraph, name);
|
|
596
|
-
if (idx === -1) {
|
|
597
|
-
throw new Error(`Operation ${index} (remove-source): source "${name}" not found in jobGraph.vertices`);
|
|
598
|
-
}
|
|
599
|
-
if (!isCanonicalRawSource(jobGraph, name)) {
|
|
600
|
-
throw unsupportedRawShapeError(index, 'remove-source', `vertex "${name}" is not a canonical UI source feeding ${RAW_PRE_PARSER_FILTER}`);
|
|
601
|
-
}
|
|
602
|
-
jobGraph.vertices.splice(idx, 1);
|
|
603
|
-
removeEdgesTouching(jobGraph, name);
|
|
604
|
-
}
|
|
605
|
-
function jobGraphAddSink(jobGraph, target, sink, filter, index) {
|
|
606
|
-
assertOperationIdentity(sink, 'add-sink', index, 'sink');
|
|
607
|
-
assertSinkTargetShape(target, sink, filter, index, 'add-sink');
|
|
608
|
-
// Anchors on a unique log_reducer (assertRawUiLogGraph rejects missing/duplicate).
|
|
609
|
-
assertRawUiLogGraph(jobGraph, 'add-sink', index, [RAW_LOG_REDUCER]);
|
|
610
|
-
if (findVertexIndexByName(jobGraph, sink.name) !== -1) {
|
|
611
|
-
throw new Error(`Operation ${index} (add-sink): sink "${sink.name}" already exists`);
|
|
612
|
-
}
|
|
613
|
-
// processed-logs is a singular slot — mirror the template-backend guard.
|
|
614
|
-
// Only reject a reducer-fed iceberg sink (direct or 1-hop through a filter);
|
|
615
|
-
// a raw data-lake sink fed by pre_data_warehouse_filter is a separate slot.
|
|
616
|
-
if (target === 'processed-logs' && hasReducerFedIcebergSink(jobGraph)) {
|
|
617
|
-
throw new Error(`Operation ${index} (add-sink): a logs-iceberg-table-sink already exists. ` +
|
|
618
|
-
`Remove it first with remove-sink (target: processed-logs) before adding a new one.`);
|
|
619
|
-
}
|
|
620
|
-
if (filter !== undefined) {
|
|
621
|
-
if (typeof filter !== 'object') {
|
|
622
|
-
throw new Error(`Operation ${index} (add-sink): filter must be an object`);
|
|
623
|
-
}
|
|
624
|
-
const filterName = `${sink.name}_filter`;
|
|
625
|
-
if (findVertexIndexByName(jobGraph, filterName) !== -1) {
|
|
626
|
-
throw new Error(`Operation ${index} (add-sink): generated filter vertex "${filterName}" already exists`);
|
|
627
|
-
}
|
|
628
|
-
const filterVertex = {
|
|
629
|
-
...filter,
|
|
630
|
-
name: filterName,
|
|
631
|
-
type: LogsFilterType.logs_filter,
|
|
632
|
-
};
|
|
633
|
-
jobGraph.vertices.push(filterVertex, sink);
|
|
634
|
-
addEdgeIfMissing(jobGraph, RAW_LOG_REDUCER, DEFAULT_EDGE_OUTPUT, filterName, DEFAULT_EDGE_INPUT);
|
|
635
|
-
addEdgeIfMissing(jobGraph, filterName, DEFAULT_EDGE_OUTPUT, sink.name, DEFAULT_EDGE_INPUT);
|
|
636
|
-
return;
|
|
637
|
-
}
|
|
638
|
-
jobGraph.vertices.push(sink);
|
|
639
|
-
addEdgeIfMissing(jobGraph, RAW_LOG_REDUCER, DEFAULT_EDGE_OUTPUT, sink.name, DEFAULT_EDGE_INPUT);
|
|
640
|
-
}
|
|
641
|
-
function jobGraphRemoveSink(jobGraph, target, name, index) {
|
|
642
|
-
assertValidSinkTarget(target, 'remove-sink', index);
|
|
643
|
-
const sinkName = resolveRawSinkToRemove(jobGraph, target, name, index);
|
|
644
|
-
// If the sink is fed by its generated single-use filter, drop that too.
|
|
645
|
-
const generatedFilterName = `${sinkName}_filter`;
|
|
646
|
-
const fedByGeneratedFilter = parsedEdges(jobGraph).some(edge => edge.targetVertex === sinkName && edge.sourceVertex === generatedFilterName);
|
|
647
|
-
removeVertexByName(jobGraph, sinkName);
|
|
648
|
-
removeEdgesTouching(jobGraph, sinkName);
|
|
649
|
-
if (fedByGeneratedFilter && findVertexByName(jobGraph, generatedFilterName)) {
|
|
650
|
-
removeVertexByName(jobGraph, generatedFilterName);
|
|
651
|
-
removeEdgesTouching(jobGraph, generatedFilterName);
|
|
652
|
-
}
|
|
653
|
-
}
|
|
654
|
-
/**
|
|
655
|
-
* Resolve which raw-graph sink a `remove-sink` op targets. `vendor` requires a
|
|
656
|
-
* `name` matching a vendor sink type; `processed-logs` ignores `name` and
|
|
657
|
-
* removes the unique `logs-iceberg-table-sink` (rejecting 0 or multiple).
|
|
658
|
-
*/
|
|
659
|
-
function resolveRawSinkToRemove(jobGraph, target, name, index) {
|
|
660
|
-
if (target === 'vendor') {
|
|
661
|
-
if (typeof name !== 'string' || name.length === 0) {
|
|
662
|
-
throw new Error(`Operation ${index} (remove-sink): name is required for target "vendor"`);
|
|
663
|
-
}
|
|
664
|
-
const vertex = findVertexByName(jobGraph, name);
|
|
665
|
-
if (!vertex) {
|
|
666
|
-
throw new Error(`Operation ${index} (remove-sink): sink "${name}" not found in jobGraph.vertices`);
|
|
667
|
-
}
|
|
668
|
-
if (!VENDOR_LOG_SINK_TYPES.has(vertex.type)) {
|
|
669
|
-
throw new Error(`Operation ${index} (remove-sink): vertex "${name}" is not a vendor sink (type "${vertex.type}")`);
|
|
670
|
-
}
|
|
671
|
-
return name;
|
|
672
|
-
}
|
|
673
|
-
return findIcebergSinkByRole(jobGraph, PROCESSED_LOGS_SINK_NAME_PREFIX, 'processed-logs', 'remove-sink', 'remove the sink', index).name;
|
|
674
|
-
}
|
|
675
|
-
/**
|
|
676
|
-
* The single `logs-iceberg-table-sink` vertex playing the given role, identified
|
|
677
|
-
* by its name prefix. Selecting by type alone is ambiguous — raw and processed
|
|
678
|
-
* sinks share the type — so a count-based pick can target the wrong dataset sink.
|
|
679
|
-
* Errors on 0 or >1 matches for the role.
|
|
680
|
-
*/
|
|
681
|
-
function findIcebergSinkByRole(jobGraph, namePrefix, role, opLabel, action, index) {
|
|
682
|
-
const matches = jobGraph.vertices.filter(v => v.type === LOGS_ICEBERG_TABLE_SINK_TYPE && v.name.startsWith(namePrefix));
|
|
683
|
-
if (matches.length !== 1) {
|
|
684
|
-
throw new Error(`Operation ${index} (${opLabel}): found ${matches.length} ${role} sinks (logs-iceberg-table-sink named "${namePrefix}*"); ` +
|
|
685
|
-
`expected exactly one to ${action}. Use 'grepr job:get' + 'grepr job:update' to ${action} explicitly.`);
|
|
686
|
-
}
|
|
687
|
-
return matches[0];
|
|
688
|
-
}
|
|
689
|
-
function jobGraphSetRawDataset(jobGraph, datasetId, index) {
|
|
690
|
-
const sink = findIcebergSinkByRole(jobGraph, RAW_DATA_LAKE_SINK_NAME_PREFIX, 'raw data-lake', 'set-raw-dataset', 'set the dataset', index);
|
|
691
|
-
sink.datasetId = datasetId;
|
|
692
|
-
}
|
|
693
|
-
function jobGraphSetFilter(jobGraph, phase, filter, index) {
|
|
694
|
-
if (!filter || typeof filter !== 'object') {
|
|
695
|
-
throw new Error(`Operation ${index} (set-filter): filter must be an object`);
|
|
696
|
-
}
|
|
697
|
-
const rawName = rawFilterNameForPhase(phase, index, 'set-filter');
|
|
698
|
-
assertRawUiLogGraph(jobGraph, 'set-filter', index, [rawName]);
|
|
699
|
-
replaceRawFilterVertex(jobGraph, rawName, filter, 'set-filter', index);
|
|
700
|
-
}
|
|
701
|
-
function jobGraphClearFilter(jobGraph, phase, index) {
|
|
702
|
-
const rawName = rawFilterNameForPhase(phase, index, 'clear-filter');
|
|
703
|
-
assertRawUiLogGraph(jobGraph, 'clear-filter', index, [rawName]);
|
|
704
|
-
replaceRawFilterVertex(jobGraph, rawName, { predicate: { type: DatadogQueryPredicateType.datadog_query, query: '' } }, 'clear-filter', index);
|
|
705
|
-
}
|
|
706
|
-
function replaceRawFilterVertex(jobGraph, rawName, patch, opLabel, index) {
|
|
707
|
-
const existing = findVertexByName(jobGraph, rawName);
|
|
708
|
-
if (!existing) {
|
|
709
|
-
throw unsupportedRawShapeError(index, opLabel, `vertex "${rawName}" not found`);
|
|
710
|
-
}
|
|
711
|
-
replaceVertexByName(jobGraph, rawName, {
|
|
712
|
-
...existing,
|
|
713
|
-
...patch,
|
|
714
|
-
type: LogsFilterType.logs_filter,
|
|
715
|
-
name: rawName,
|
|
716
|
-
}, opLabel, index);
|
|
717
|
-
}
|
|
718
|
-
function jobGraphAddParser(jobGraph, parser, index) {
|
|
719
|
-
assertRawUiLogGraph(jobGraph, 'add-parser', index, [RAW_PRE_PARSER_FILTER, RAW_PRE_WAREHOUSE_FILTER]);
|
|
720
|
-
assertOperationIdentity(parser, 'add-parser', index, 'parser');
|
|
721
|
-
if (!RAW_PARSER_TYPES.has(parser.type)) {
|
|
722
|
-
throw new Error(`Operation ${index} (add-parser): parser type "${parser.type}" is not supported for raw UI graph insertion`);
|
|
723
|
-
}
|
|
724
|
-
if (findVertexIndexByName(jobGraph, parser.name) !== -1) {
|
|
725
|
-
throw new Error(`Operation ${index} (add-parser): parser "${parser.name}" already exists`);
|
|
726
|
-
}
|
|
727
|
-
if (parser.type === RAW_JSON_PROCESSOR_TYPE) {
|
|
728
|
-
if (parser.name !== RAW_JSON_PROCESSOR) {
|
|
729
|
-
throw unsupportedRawShapeError(index, 'add-parser', `json processor must be named "${RAW_JSON_PROCESSOR}"`);
|
|
730
|
-
}
|
|
731
|
-
if (findVertexByName(jobGraph, RAW_JSON_PROCESSOR) || findVertexByType(jobGraph, RAW_JSON_PROCESSOR_TYPE)) {
|
|
732
|
-
throw new Error(`Operation ${index} (add-parser): json-log-processor already exists in the raw graph`);
|
|
733
|
-
}
|
|
734
|
-
const successor = orderedParserNames(jobGraph, 'add-parser', index)[0] ?? RAW_PRE_WAREHOUSE_FILTER;
|
|
735
|
-
return insertRawVertexBetween(jobGraph, parser, RAW_PRE_PARSER_FILTER, successor, 'add-parser', index);
|
|
736
|
-
}
|
|
737
|
-
if (parser.type === RAW_ATTRIBUTES_REMAPPER_TYPE) {
|
|
738
|
-
if (parser.name !== RAW_ATTRIBUTES_REMAPPER) {
|
|
739
|
-
throw unsupportedRawShapeError(index, 'add-parser', `attributes remapper must be named "${RAW_ATTRIBUTES_REMAPPER}"`);
|
|
740
|
-
}
|
|
741
|
-
if (findVertexByName(jobGraph, RAW_ATTRIBUTES_REMAPPER) || findVertexByType(jobGraph, RAW_ATTRIBUTES_REMAPPER_TYPE)) {
|
|
742
|
-
throw new Error(`Operation ${index} (add-parser): log-attributes-remapper already exists in the raw graph`);
|
|
743
|
-
}
|
|
744
|
-
// Locate the json processor by type, not name: live UI graphs use suffixed
|
|
745
|
-
// names (e.g. json_log_processor_1), and a name-only lookup would silently
|
|
746
|
-
// fall back to RAW_PRE_PARSER_FILTER and wire the remapper before it.
|
|
747
|
-
const jsonProcessor = findVertexByType(jobGraph, RAW_JSON_PROCESSOR_TYPE);
|
|
748
|
-
const predecessor = jsonProcessor ? jsonProcessor.name : RAW_PRE_PARSER_FILTER;
|
|
749
|
-
const successor = singleMainSuccessor(jobGraph, predecessor, 'add-parser', index);
|
|
750
|
-
return insertRawVertexBetween(jobGraph, parser, predecessor, successor, 'add-parser', index);
|
|
751
|
-
}
|
|
752
|
-
const existingParserNames = orderedParserNames(jobGraph, 'add-parser', index);
|
|
753
|
-
const predecessor = existingParserNames[existingParserNames.length - 1] ?? RAW_PRE_PARSER_FILTER;
|
|
754
|
-
insertRawVertexBetween(jobGraph, parser, predecessor, RAW_PRE_WAREHOUSE_FILTER, 'add-parser', index);
|
|
755
|
-
}
|
|
756
|
-
function jobGraphRemoveParser(jobGraph, name, index) {
|
|
757
|
-
assertRawUiLogGraph(jobGraph, 'remove-parser', index, [RAW_PRE_PARSER_FILTER, RAW_PRE_WAREHOUSE_FILTER]);
|
|
758
|
-
const parser = findVertexByName(jobGraph, name);
|
|
759
|
-
if (!parser) {
|
|
760
|
-
throw new Error(`Operation ${index} (remove-parser): parser "${name}" not found in jobGraph.vertices`);
|
|
761
|
-
}
|
|
762
|
-
if (!RAW_PARSER_TYPES.has(parser.type)) {
|
|
763
|
-
throw new Error(`Operation ${index} (remove-parser): vertex "${name}" is not a supported parser type`);
|
|
764
|
-
}
|
|
765
|
-
const incoming = parsedEdges(jobGraph).filter(edge => edge.targetVertex === name);
|
|
766
|
-
const outgoing = parsedEdges(jobGraph).filter(edge => edge.sourceVertex === name);
|
|
767
|
-
if (incoming.length !== 1 || outgoing.length !== 1) {
|
|
768
|
-
throw unsupportedRawShapeError(index, 'remove-parser', `parser "${name}" must have exactly one incoming and one outgoing edge`);
|
|
769
|
-
}
|
|
770
|
-
removeVertexByName(jobGraph, name);
|
|
771
|
-
removeEdgesTouching(jobGraph, name);
|
|
772
|
-
const from = incoming[0];
|
|
773
|
-
const to = outgoing[0];
|
|
774
|
-
addEdgeIfMissing(jobGraph, from.sourceVertex, from.sourcePort, to.targetVertex, to.targetPort);
|
|
775
|
-
}
|
|
776
|
-
// --- Job-graph per-entry removals: anchor on the unique reducer/remapper/grok
|
|
777
|
-
// vertex (the add-path guards), then delegate to the shared remover. ---
|
|
778
|
-
function jobGraphRemoveMessageAttribute(vertices, attributePath, index) {
|
|
779
|
-
const remapper = findUniqueVertexByType(vertices, LogAttributesRemapperType.log_attributes_remapper, 'remove-message-attribute', index);
|
|
780
|
-
removeMessageAttributeFromRemapper(remapper, attributePath, index);
|
|
781
|
-
}
|
|
782
|
-
function jobGraphRemoveGroupBy(vertices, attributePath, index) {
|
|
783
|
-
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'remove-group-by', index);
|
|
784
|
-
removeGroupByFromReducer(reducer, attributePath, index);
|
|
785
|
-
}
|
|
786
|
-
function jobGraphRemoveAggregation(vertices, attributePath, strategies, index) {
|
|
787
|
-
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'remove-aggregation-strategy', index);
|
|
788
|
-
removeAggregationFromReducer(reducer, attributePath, strategies, index);
|
|
789
|
-
}
|
|
790
|
-
/** Remove a raw `EventPredicate` from the reducer's `logReducerExceptions` (mirror of jobGraphAddReducerException). */
|
|
791
|
-
function jobGraphRemoveReducerException(vertices, predicate, index) {
|
|
792
|
-
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'remove-reducer-exception', index);
|
|
793
|
-
const serialized = JSON.stringify(predicate);
|
|
794
|
-
const existing = reducer.logReducerExceptions ?? [];
|
|
795
|
-
const kept = existing.filter(p => JSON.stringify(p) !== serialized);
|
|
796
|
-
if (kept.length === existing.length) {
|
|
797
|
-
throw entryNotFoundError(index, 'remove-reducer-exception', 'matching reducer exception');
|
|
798
|
-
}
|
|
799
|
-
reducer.logReducerExceptions = kept;
|
|
800
|
-
}
|
|
801
|
-
function jobGraphRemoveGrokRule(vertices, pattern, parserName, index) {
|
|
802
|
-
const grok = findJobGraphGrokParser(vertices, parserName, 'remove-grok-rule', index);
|
|
803
|
-
removeGrokRuleFromParser(grok, pattern, index);
|
|
804
|
-
}
|
|
805
|
-
// --- Job-graph in-place vertex updates: replace the named vertex's config via
|
|
806
|
-
// replaceVertexByName, leaving edges/position untouched. ---
|
|
807
|
-
function jobGraphUpdateSource(jobGraph, source, index) {
|
|
808
|
-
assertOperationIdentity(source, 'update-source', index, 'source');
|
|
809
|
-
assertRawUiLogGraph(jobGraph, 'update-source', index, [RAW_PRE_PARSER_FILTER]);
|
|
810
|
-
if (findVertexIndexByName(jobGraph, source.name) === -1) {
|
|
811
|
-
throw new Error(`Operation ${index} (update-source): source "${source.name}" not found in jobGraph.vertices.`);
|
|
812
|
-
}
|
|
813
|
-
if (!isCanonicalRawSource(jobGraph, source.name)) {
|
|
814
|
-
throw unsupportedRawShapeError(index, 'update-source', `vertex "${source.name}" is not a canonical UI source feeding ${RAW_PRE_PARSER_FILTER}`);
|
|
815
|
-
}
|
|
816
|
-
replaceVertexByName(jobGraph, source.name, source, 'update-source', index);
|
|
817
|
-
}
|
|
818
|
-
function jobGraphUpdateParser(jobGraph, parser, index) {
|
|
819
|
-
assertOperationIdentity(parser, 'update-parser', index, 'parser');
|
|
820
|
-
assertRawUiLogGraph(jobGraph, 'update-parser', index, [RAW_PRE_PARSER_FILTER, RAW_PRE_WAREHOUSE_FILTER]);
|
|
821
|
-
const existing = findVertexByName(jobGraph, parser.name);
|
|
822
|
-
if (!existing) {
|
|
823
|
-
throw new Error(`Operation ${index} (update-parser): parser "${parser.name}" not found in jobGraph.vertices.`);
|
|
824
|
-
}
|
|
825
|
-
if (!RAW_PARSER_TYPES.has(existing.type)) {
|
|
826
|
-
throw new Error(`Operation ${index} (update-parser): vertex "${parser.name}" is not a supported parser type.`);
|
|
827
|
-
}
|
|
828
|
-
if (!RAW_PARSER_TYPES.has(parser.type)) {
|
|
829
|
-
throw new Error(`Operation ${index} (update-parser): replacement parser type "${parser.type}" is not supported for raw UI graph parsers.`);
|
|
830
|
-
}
|
|
831
|
-
replaceVertexByName(jobGraph, parser.name, parser, 'update-parser', index);
|
|
832
|
-
}
|
|
833
|
-
function jobGraphUpdateSink(jobGraph, target, sink, filter, index) {
|
|
834
|
-
assertOperationIdentity(sink, 'update-sink', index, 'sink');
|
|
835
|
-
assertSinkTargetShape(target, sink, filter, index, 'update-sink');
|
|
836
|
-
if (target === 'processed-logs') {
|
|
837
|
-
const existing = findIcebergSinkByRole(jobGraph, PROCESSED_LOGS_SINK_NAME_PREFIX, 'processed-logs', 'update-sink', 'update the sink', index);
|
|
838
|
-
replaceVertexByName(jobGraph, existing.name, { ...sink, name: existing.name }, 'update-sink', index);
|
|
839
|
-
return;
|
|
840
|
-
}
|
|
841
|
-
// vendor: anchor on the unique reducer (as add-sink does), then replace the named vendor sink.
|
|
842
|
-
assertRawUiLogGraph(jobGraph, 'update-sink', index, [RAW_LOG_REDUCER]);
|
|
843
|
-
const existing = findVertexByName(jobGraph, sink.name);
|
|
844
|
-
if (!existing) {
|
|
845
|
-
throw new Error(`Operation ${index} (update-sink): sink "${sink.name}" not found in jobGraph.vertices.`);
|
|
846
|
-
}
|
|
847
|
-
if (!VENDOR_LOG_SINK_TYPES.has(existing.type)) {
|
|
848
|
-
throw new Error(`Operation ${index} (update-sink): vertex "${sink.name}" is not a vendor sink (type "${existing.type}").`);
|
|
849
|
-
}
|
|
850
|
-
replaceVertexByName(jobGraph, sink.name, sink, 'update-sink', index);
|
|
851
|
-
if (filter !== undefined) {
|
|
852
|
-
const filterName = `${sink.name}_filter`;
|
|
853
|
-
if (!findVertexByName(jobGraph, filterName)) {
|
|
854
|
-
throw new Error(`Operation ${index} (update-sink): sink "${sink.name}" has no generated gating filter "${filterName}" to update; ` +
|
|
855
|
-
`use remove-sink + add-sink to introduce a gate.`);
|
|
856
|
-
}
|
|
857
|
-
replaceRawFilterVertex(jobGraph, filterName, filter, 'update-sink', index);
|
|
858
|
-
}
|
|
859
|
-
}
|
|
860
|
-
// --- Job-graph in-place updates of reducer-list entries: anchor on the unique
|
|
861
|
-
// reducer/remapper (as the add/remove paths do), then delegate to the shared updater. ---
|
|
862
|
-
function jobGraphUpdateMessageAttribute(vertices, from, to, index) {
|
|
863
|
-
const remapper = findUniqueVertexByType(vertices, LogAttributesRemapperType.log_attributes_remapper, 'update-message-attribute', index);
|
|
864
|
-
updateMessageAttributeOnRemapper(remapper, from, to, index);
|
|
865
|
-
}
|
|
866
|
-
function jobGraphUpdateGroupBy(vertices, from, to, index) {
|
|
867
|
-
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'update-group-by', index);
|
|
868
|
-
updateGroupByOnReducer(reducer, from, to, index);
|
|
869
|
-
}
|
|
870
|
-
function jobGraphUpdateAggregation(vertices, attributePath, strategies, index) {
|
|
871
|
-
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'update-aggregation-strategy', index);
|
|
872
|
-
updateAggregationOnReducer(reducer, attributePath, strategies, index);
|
|
873
|
-
}
|
|
874
|
-
/** Replace the matching raw predicate in the reducer's `logReducerExceptions` in place (mirror of jobGraphAddReducerException). */
|
|
875
|
-
function jobGraphUpdateReducerException(vertices, from, to, index) {
|
|
876
|
-
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'update-reducer-exception', index);
|
|
877
|
-
const existing = reducer.logReducerExceptions ?? [];
|
|
878
|
-
const idx = reducerExceptionUpdateIndex(existing.map(p => JSON.stringify(p)), from, to, index);
|
|
879
|
-
existing[idx] = to;
|
|
880
|
-
reducer.logReducerExceptions = existing;
|
|
881
|
-
}
|
|
882
|
-
function rawFilterNameForPhase(phase, index, opLabel) {
|
|
883
|
-
switch (phase) {
|
|
884
|
-
case 'pre-parser':
|
|
885
|
-
return RAW_PRE_PARSER_FILTER;
|
|
886
|
-
case 'pre-warehouse':
|
|
887
|
-
return RAW_PRE_WAREHOUSE_FILTER;
|
|
888
|
-
case 'pre-exceptions':
|
|
889
|
-
return RAW_PRE_EXCEPTIONS_FILTER;
|
|
890
|
-
default:
|
|
891
|
-
throw new Error(`Operation ${index} (${opLabel}): phase "${phase}" has no canonical UI raw-graph stage. ` +
|
|
892
|
-
ACCEPTED_TRANSFORM_PHASES_MESSAGE);
|
|
893
|
-
}
|
|
894
|
-
}
|
|
895
|
-
function assertRawUiLogGraph(jobGraph, opLabel, index, requiredNames) {
|
|
896
|
-
const seen = new Set();
|
|
897
|
-
for (const vertex of jobGraph.vertices) {
|
|
898
|
-
if (typeof vertex.name !== 'string' || vertex.name.length === 0) {
|
|
899
|
-
throw unsupportedRawShapeError(index, opLabel, 'every vertex must have a non-empty name');
|
|
900
|
-
}
|
|
901
|
-
if (seen.has(vertex.name)) {
|
|
902
|
-
throw unsupportedRawShapeError(index, opLabel, `duplicate vertex name "${vertex.name}"`);
|
|
903
|
-
}
|
|
904
|
-
seen.add(vertex.name);
|
|
905
|
-
}
|
|
906
|
-
const missing = requiredNames.filter(name => !seen.has(name));
|
|
907
|
-
if (missing.length > 0) {
|
|
908
|
-
throw unsupportedRawShapeError(index, opLabel, `missing canonical vertex: ${missing.join(', ')}`);
|
|
909
|
-
}
|
|
910
|
-
}
|
|
911
|
-
function unsupportedRawShapeError(index, opLabel, detail) {
|
|
912
|
-
return new Error(`Operation ${index} (${opLabel}): unsupported raw job graph shape. ` +
|
|
913
|
-
`UI-level topology edits require a canonical UI log pipeline graph; ${detail}.`);
|
|
914
|
-
}
|
|
915
397
|
function assertOperationIdentity(operation, opLabel, index, fieldName) {
|
|
916
398
|
if (!operation || typeof operation !== 'object') {
|
|
917
399
|
throw new Error(`Operation ${index} (${opLabel}): ${fieldName} must be an object`);
|
|
@@ -923,145 +405,11 @@ function assertOperationIdentity(operation, opLabel, index, fieldName) {
|
|
|
923
405
|
throw new Error(`Operation ${index} (${opLabel}): ${fieldName}.type must be a non-empty string`);
|
|
924
406
|
}
|
|
925
407
|
}
|
|
926
|
-
function findVertexByName(jobGraph, name) {
|
|
927
|
-
return jobGraph.vertices.find(vertex => vertex.name === name);
|
|
928
|
-
}
|
|
929
|
-
function findVertexByType(jobGraph, type) {
|
|
930
|
-
return jobGraph.vertices.find(vertex => vertex.type === type);
|
|
931
|
-
}
|
|
932
|
-
function findVertexIndexByName(jobGraph, name) {
|
|
933
|
-
return jobGraph.vertices.findIndex(vertex => vertex.name === name);
|
|
934
|
-
}
|
|
935
|
-
function replaceVertexByName(jobGraph, name, replacement, opLabel, index) {
|
|
936
|
-
const idx = findVertexIndexByName(jobGraph, name);
|
|
937
|
-
if (idx === -1) {
|
|
938
|
-
throw unsupportedRawShapeError(index, opLabel, `vertex "${name}" not found`);
|
|
939
|
-
}
|
|
940
|
-
jobGraph.vertices[idx] = replacement;
|
|
941
|
-
}
|
|
942
|
-
function removeVertexByName(jobGraph, name) {
|
|
943
|
-
const idx = findVertexIndexByName(jobGraph, name);
|
|
944
|
-
if (idx !== -1) {
|
|
945
|
-
jobGraph.vertices.splice(idx, 1);
|
|
946
|
-
}
|
|
947
|
-
}
|
|
948
|
-
function graphEdges(jobGraph) {
|
|
949
|
-
const graph = jobGraph;
|
|
950
|
-
if (!Array.isArray(graph.edges)) {
|
|
951
|
-
graph.edges = []; // Lazily initializes edges to [] if absent — normalizes the schema type omission.
|
|
952
|
-
}
|
|
953
|
-
return graph.edges;
|
|
954
|
-
}
|
|
955
|
-
function parsedEdges(jobGraph) {
|
|
956
|
-
return graphEdges(jobGraph).map((edge, index) => ({ edge, index, ...parseEdge(edge) }));
|
|
957
|
-
}
|
|
958
|
-
/** True if any `logs-iceberg-table-sink` is reachable from `RAW_LOG_REDUCER` within two hops. */
|
|
959
|
-
function hasReducerFedIcebergSink(jobGraph) {
|
|
960
|
-
const edges = parsedEdges(jobGraph);
|
|
961
|
-
const reducerNeighbors = new Set(edges.filter(e => e.sourceVertex === RAW_LOG_REDUCER).map(e => e.targetVertex));
|
|
962
|
-
return jobGraph.vertices.some(v => v.type === LOGS_ICEBERG_TABLE_SINK_TYPE &&
|
|
963
|
-
(reducerNeighbors.has(v.name) ||
|
|
964
|
-
edges.some(e => reducerNeighbors.has(e.sourceVertex) && e.targetVertex === v.name)));
|
|
965
|
-
}
|
|
966
|
-
function formatEdge(sourceVertex, sourcePort, targetVertex, targetPort) {
|
|
967
|
-
const source = sourcePort === DEFAULT_EDGE_OUTPUT ? sourceVertex : `${sourceVertex}:${sourcePort}`;
|
|
968
|
-
const target = targetPort === DEFAULT_EDGE_INPUT ? targetVertex : `${targetVertex}:${targetPort}`;
|
|
969
|
-
return `${source} -> ${target}`;
|
|
970
|
-
}
|
|
971
|
-
function addEdgeIfMissing(jobGraph, sourceVertex, sourcePort, targetVertex, targetPort) {
|
|
972
|
-
const candidate = formatEdge(sourceVertex, sourcePort, targetVertex, targetPort);
|
|
973
|
-
const exists = parsedEdges(jobGraph).some(edge => edge.sourceVertex === sourceVertex &&
|
|
974
|
-
edge.sourcePort === sourcePort &&
|
|
975
|
-
edge.targetVertex === targetVertex &&
|
|
976
|
-
edge.targetPort === targetPort);
|
|
977
|
-
if (!exists) {
|
|
978
|
-
graphEdges(jobGraph).push(candidate);
|
|
979
|
-
}
|
|
980
|
-
}
|
|
981
|
-
function removeEdgesTouching(jobGraph, name) {
|
|
982
|
-
const keep = graphEdges(jobGraph).filter(edge => {
|
|
983
|
-
const parsed = parseEdge(edge);
|
|
984
|
-
return parsed.sourceVertex !== name && parsed.targetVertex !== name;
|
|
985
|
-
});
|
|
986
|
-
jobGraph.edges = keep;
|
|
987
|
-
}
|
|
988
|
-
function removeEdgeAt(jobGraph, index) {
|
|
989
|
-
graphEdges(jobGraph).splice(index, 1);
|
|
990
|
-
}
|
|
991
|
-
function insertRawVertexBetween(jobGraph, vertex, predecessor, successor, opLabel, index) {
|
|
992
|
-
const directEdges = parsedEdges(jobGraph).filter(edge => edge.sourceVertex === predecessor && edge.targetVertex === successor);
|
|
993
|
-
if (directEdges.length !== 1) {
|
|
994
|
-
throw unsupportedRawShapeError(index, opLabel, `expected exactly one parser-chain edge ${predecessor} -> ${successor}`);
|
|
995
|
-
}
|
|
996
|
-
const directEdge = directEdges[0];
|
|
997
|
-
jobGraph.vertices.push(vertex);
|
|
998
|
-
removeEdgeAt(jobGraph, directEdge.index);
|
|
999
|
-
addEdgeIfMissing(jobGraph, predecessor, directEdge.sourcePort, vertex.name, DEFAULT_EDGE_INPUT);
|
|
1000
|
-
addEdgeIfMissing(jobGraph, vertex.name, DEFAULT_EDGE_OUTPUT, successor, directEdge.targetPort);
|
|
1001
|
-
}
|
|
1002
|
-
function orderedParserNames(jobGraph, opLabel, index) {
|
|
1003
|
-
const names = [];
|
|
1004
|
-
let current = RAW_PRE_PARSER_FILTER;
|
|
1005
|
-
const seen = new Set([current]);
|
|
1006
|
-
while (current !== RAW_PRE_WAREHOUSE_FILTER) {
|
|
1007
|
-
const outgoing = parsedEdges(jobGraph).filter(edge => edge.sourceVertex === current);
|
|
1008
|
-
if (outgoing.length !== 1) {
|
|
1009
|
-
throw unsupportedRawShapeError(index, opLabel, `parser chain vertex "${current}" must have exactly one outgoing main-chain edge`);
|
|
1010
|
-
}
|
|
1011
|
-
const next = outgoing[0].targetVertex;
|
|
1012
|
-
if (next === RAW_PRE_WAREHOUSE_FILTER) {
|
|
1013
|
-
return names;
|
|
1014
|
-
}
|
|
1015
|
-
const nextVertex = findVertexByName(jobGraph, next);
|
|
1016
|
-
if (!nextVertex || !RAW_PARSER_TYPES.has(nextVertex.type)) {
|
|
1017
|
-
throw unsupportedRawShapeError(index, opLabel, `vertex "${next}" between ${RAW_PRE_PARSER_FILTER} and ${RAW_PRE_WAREHOUSE_FILTER} is not a supported parser`);
|
|
1018
|
-
}
|
|
1019
|
-
if (seen.has(next)) {
|
|
1020
|
-
throw unsupportedRawShapeError(index, opLabel, `cycle detected at parser vertex "${next}"`);
|
|
1021
|
-
}
|
|
1022
|
-
names.push(next);
|
|
1023
|
-
seen.add(next);
|
|
1024
|
-
current = next;
|
|
1025
|
-
}
|
|
1026
|
-
return names;
|
|
1027
|
-
}
|
|
1028
|
-
function singleMainSuccessor(jobGraph, name, opLabel, index) {
|
|
1029
|
-
const outgoing = parsedEdges(jobGraph).filter(edge => edge.sourceVertex === name);
|
|
1030
|
-
if (outgoing.length !== 1) {
|
|
1031
|
-
throw unsupportedRawShapeError(index, opLabel, `vertex "${name}" must have exactly one outgoing main-chain edge`);
|
|
1032
|
-
}
|
|
1033
|
-
return outgoing[0].targetVertex;
|
|
1034
|
-
}
|
|
1035
|
-
function isCanonicalRawSource(jobGraph, name) {
|
|
1036
|
-
const edges = parsedEdges(jobGraph);
|
|
1037
|
-
const hasIncoming = edges.some(edge => edge.targetVertex === name);
|
|
1038
|
-
const feedsPreParser = edges.some(edge => edge.sourceVertex === name && edge.targetVertex === RAW_PRE_PARSER_FILTER);
|
|
1039
|
-
return !hasIncoming && feedsPreParser;
|
|
1040
|
-
}
|
|
1041
408
|
function assertProposedTemplateHasSource(input) {
|
|
1042
409
|
if (!Array.isArray(input.sources) || input.sources.length === 0) {
|
|
1043
410
|
throw new Error('Proposed job graph has zero sources; a log pipeline must keep at least one source.');
|
|
1044
411
|
}
|
|
1045
412
|
}
|
|
1046
|
-
function assertProposedJobGraphHasSource(jobGraph) {
|
|
1047
|
-
if (!jobGraph.vertices.some(vertex => typeof vertex.name === 'string' && isCanonicalRawSource(jobGraph, vertex.name))) {
|
|
1048
|
-
throw new Error('Proposed job graph has zero sources; a log pipeline must keep at least one source.');
|
|
1049
|
-
}
|
|
1050
|
-
}
|
|
1051
|
-
function applyAddMessageAttribute(input, attributePath, index) {
|
|
1052
|
-
const remapper = findRemapper(input);
|
|
1053
|
-
if (!remapper) {
|
|
1054
|
-
throw new Error(`Operation ${index} (add-message-attribute): no log-attributes-remapper in input.parsers. ` +
|
|
1055
|
-
`Template-backed pipelines normally include a remapper by default; if yours doesn't, add one via add-parser first.`);
|
|
1056
|
-
}
|
|
1057
|
-
applyMessageAttributeToRemapper(remapper, attributePath, index);
|
|
1058
|
-
}
|
|
1059
|
-
function applyAddGroupBy(input, attributePath, index) {
|
|
1060
|
-
applyGroupByToReducer(input.reducer, attributePath, index);
|
|
1061
|
-
}
|
|
1062
|
-
function applyAddAggregation(input, attributePath, strategies, index) {
|
|
1063
|
-
applyAggregationToReducer(input.reducer, attributePath, strategies, index);
|
|
1064
|
-
}
|
|
1065
413
|
function applyAddReducerException(input, predicate) {
|
|
1066
414
|
// Wrapped in a TemplateQueryException; matching logs bypass aggregation.
|
|
1067
415
|
const serialized = JSON.stringify(predicate);
|
|
@@ -1081,9 +429,12 @@ function applyAddGrokRule(input, pattern, parserName, extractAttribute, index) {
|
|
|
1081
429
|
}
|
|
1082
430
|
/** Resolve the target grok-parser in `input.parsers` by name (or the sole grok-parser); shared by add/remove-grok-rule. Throws if absent, ambiguous, or not a grok-parser. */
|
|
1083
431
|
function findTemplateGrokParser(parsers, parserName, opLabel, index) {
|
|
1084
|
-
const
|
|
1085
|
-
|
|
1086
|
-
|
|
432
|
+
const groks = parsers.filter(p => p.type === GrokParserType.grok_parser);
|
|
433
|
+
if (!parserName && groks.length > 1) {
|
|
434
|
+
throw new Error(`Operation ${index} (${opLabel}): found ${groks.length} grok parsers in input.parsers; ` +
|
|
435
|
+
`pass parserName to choose the target parser.`);
|
|
436
|
+
}
|
|
437
|
+
const grok = parserName ? parsers.find(p => p.name === parserName) : groks[0];
|
|
1087
438
|
if (!grok) {
|
|
1088
439
|
const hint = opLabel === 'add-grok-rule' ? ' If you need to introduce one, use add-parser first.' : '';
|
|
1089
440
|
throw new Error(`Operation ${index} (${opLabel}): ${parserName ? `parser "${parserName}" not found` : 'no grok-parser found in input.parsers'}.${hint}`);
|
|
@@ -1247,8 +598,8 @@ function findSimpleRootFilterGate(root) {
|
|
|
1247
598
|
* Install or update a filter gate at the phase's transform slot:
|
|
1248
599
|
* - empty slot: `IF filter THEN PASSTHROUGH ELSE DROP`
|
|
1249
600
|
* - existing simple gate: rebuild with the new predicate, keeping the pass side
|
|
1250
|
-
* and inheriting `inverted`/`lateDataDuration` when the patch omits them
|
|
1251
|
-
*
|
|
601
|
+
* and inheriting `inverted`/`lateDataDuration` when the patch omits them, so a
|
|
602
|
+
* predicate-only edit does not flip keep and drop
|
|
1252
603
|
* - any other existing chain (e.g. a SqlNode): `IF filter THEN <chain> ELSE DROP`
|
|
1253
604
|
*/
|
|
1254
605
|
function applySetFilter(input, phase, filter, index) {
|
|
@@ -1425,21 +776,7 @@ function applyRemoveSource(input, name, index) {
|
|
|
1425
776
|
}
|
|
1426
777
|
input.sources.splice(idx, 1);
|
|
1427
778
|
}
|
|
1428
|
-
// --- Template per-entry removals
|
|
1429
|
-
// path writes, then delegate to the shared remover. ---
|
|
1430
|
-
function applyRemoveMessageAttribute(input, attributePath, index) {
|
|
1431
|
-
const remapper = findRemapper(input);
|
|
1432
|
-
if (!remapper) {
|
|
1433
|
-
throw new Error(`Operation ${index} (remove-message-attribute): no log-attributes-remapper in input.parsers.`);
|
|
1434
|
-
}
|
|
1435
|
-
removeMessageAttributeFromRemapper(remapper, attributePath, index);
|
|
1436
|
-
}
|
|
1437
|
-
function applyRemoveGroupBy(input, attributePath, index) {
|
|
1438
|
-
removeGroupByFromReducer(input.reducer, attributePath, index);
|
|
1439
|
-
}
|
|
1440
|
-
function applyRemoveAggregation(input, attributePath, strategies, index) {
|
|
1441
|
-
removeAggregationFromReducer(input.reducer, attributePath, strategies, index);
|
|
1442
|
-
}
|
|
779
|
+
// --- Template per-entry removals. ---
|
|
1443
780
|
/** Remove the `TemplateQueryException` whose predicate matches (mirror of applyAddReducerException). */
|
|
1444
781
|
function applyRemoveReducerException(input, predicate, index) {
|
|
1445
782
|
const serialized = JSON.stringify(predicate);
|
|
@@ -1503,21 +840,7 @@ function applyUpdateSink(input, target, sink, filter, index) {
|
|
|
1503
840
|
// Validated as a logs-iceberg-table-sink by assertSinkTargetShape.
|
|
1504
841
|
input.processedLogsSink = sink;
|
|
1505
842
|
}
|
|
1506
|
-
// --- Template in-place updates of reducer-list entries
|
|
1507
|
-
// reducer/remapper the add/remove paths use, then delegate to the shared updater. ---
|
|
1508
|
-
function applyUpdateMessageAttribute(input, from, to, index) {
|
|
1509
|
-
const remapper = findRemapper(input);
|
|
1510
|
-
if (!remapper) {
|
|
1511
|
-
throw new Error(`Operation ${index} (update-message-attribute): no log-attributes-remapper in input.parsers.`);
|
|
1512
|
-
}
|
|
1513
|
-
updateMessageAttributeOnRemapper(remapper, from, to, index);
|
|
1514
|
-
}
|
|
1515
|
-
function applyUpdateGroupBy(input, from, to, index) {
|
|
1516
|
-
updateGroupByOnReducer(input.reducer, from, to, index);
|
|
1517
|
-
}
|
|
1518
|
-
function applyUpdateAggregation(input, attributePath, strategies, index) {
|
|
1519
|
-
updateAggregationOnReducer(input.reducer, attributePath, strategies, index);
|
|
1520
|
-
}
|
|
843
|
+
// --- Template in-place updates of reducer-list entries. ---
|
|
1521
844
|
/** Replace the matching `TemplateQueryException`'s predicate in place (mirror of applyAddReducerException). */
|
|
1522
845
|
function applyUpdateReducerException(input, from, to, index) {
|
|
1523
846
|
const exceptions = input.exceptions ?? [];
|
|
@@ -1534,7 +857,7 @@ function assertValidSinkTarget(target, opLabel, index) {
|
|
|
1534
857
|
throw new Error(`Operation ${index} (${opLabel}): target must be "vendor" or "processed-logs", got ${JSON.stringify(target)}.`);
|
|
1535
858
|
}
|
|
1536
859
|
}
|
|
1537
|
-
/** Validate the sink type matches the target and `filter` is vendor-only; shared by add-sink and update-sink
|
|
860
|
+
/** Validate the sink type matches the target and `filter` is vendor-only; shared by add-sink and update-sink. */
|
|
1538
861
|
function assertSinkTargetShape(target, sink, filter, index, opLabel) {
|
|
1539
862
|
assertValidSinkTarget(target, opLabel, index);
|
|
1540
863
|
if (target === 'vendor') {
|
|
@@ -1711,9 +1034,9 @@ function touchesSinkConfig(op, index) {
|
|
|
1711
1034
|
}
|
|
1712
1035
|
}
|
|
1713
1036
|
/**
|
|
1714
|
-
* Find the single `template-operation` vertex
|
|
1715
|
-
*
|
|
1716
|
-
* or `SchemaUpdateJob` without casts.
|
|
1037
|
+
* Find the single `template-operation` vertex of a template-backed job. This is
|
|
1038
|
+
* the guard that rejects a raw job graph pipeline (0 matches); it also rejects
|
|
1039
|
+
* more than 1 match. Accepts `SchemaReadJob` or `SchemaUpdateJob` without casts.
|
|
1717
1040
|
*/
|
|
1718
1041
|
export function findTemplateOperation(job) {
|
|
1719
1042
|
const vertices = job.jobGraph?.vertices;
|
|
@@ -1722,8 +1045,9 @@ export function findTemplateOperation(job) {
|
|
|
1722
1045
|
}
|
|
1723
1046
|
const matches = vertices.filter(v => v.type === TemplateOperationType.template_operation);
|
|
1724
1047
|
if (matches.length === 0) {
|
|
1725
|
-
throw new Error(
|
|
1726
|
-
|
|
1048
|
+
throw new Error('This pipeline uses a raw job graph (no template-operation vertex). ' +
|
|
1049
|
+
'job:plan, job:draft and job:apply support only template-backed pipelines. ' +
|
|
1050
|
+
'To change a raw job graph, edit it and run job:update.');
|
|
1727
1051
|
}
|
|
1728
1052
|
if (matches.length > 1) {
|
|
1729
1053
|
throw new Error(`Pipeline has ${matches.length} template-operation vertices; expected exactly 1. ` +
|
|
@@ -1748,8 +1072,14 @@ function writeTemplateInput(templateOp, input) {
|
|
|
1748
1072
|
// templateInputs values are typed `unknown`, so the structured input assigns directly — no cast needed.
|
|
1749
1073
|
templateOp.templateInputs['input'] = input;
|
|
1750
1074
|
}
|
|
1751
|
-
|
|
1752
|
-
|
|
1075
|
+
const ADD_PARSER_REMAPPER_HINT = `Template-backed pipelines normally include a remapper by default; if yours doesn't, add one via add-parser first.`;
|
|
1076
|
+
function requireRemapper(input, opLabel, index, hint) {
|
|
1077
|
+
const remapper = input.parsers.find(p => p.type === LogAttributesRemapperType.log_attributes_remapper);
|
|
1078
|
+
if (!remapper) {
|
|
1079
|
+
const message = `Operation ${index} (${opLabel}): no log-attributes-remapper in input.parsers.`;
|
|
1080
|
+
throw new Error(hint === undefined ? message : `${message} ${hint}`);
|
|
1081
|
+
}
|
|
1082
|
+
return remapper;
|
|
1753
1083
|
}
|
|
1754
1084
|
function splitPath(path, index) {
|
|
1755
1085
|
if (path.length === 0) {
|