@grepr/cli 1.6.5-3542068 → 1.6.7-6789611
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/dist/lib/job-patch.d.ts +71 -0
- package/build/dist/lib/job-patch.d.ts.map +1 -1
- package/build/dist/lib/job-patch.js +459 -20
- package/build/dist/lib/job-patch.js.map +1 -1
- package/build/dist/openapi/openApiTypes.d.ts +96 -1
- package/build/dist/openapi/openApiTypes.d.ts.map +1 -1
- package/build/dist/openapi/openApiTypes.js +1 -0
- package/build/dist/openapi/openApiTypes.js.map +1 -1
- package/package.json +1 -1
|
@@ -39,6 +39,20 @@ const REQUIRED_OP_FIELDS = {
|
|
|
39
39
|
'add-sink': [['target', 'string'], ['sink', 'object']],
|
|
40
40
|
'remove-sink': [['target', 'string']],
|
|
41
41
|
'set-raw-dataset': [['datasetId', 'string']],
|
|
42
|
+
'remove-message-attribute': [['attributePath', 'string']],
|
|
43
|
+
'remove-group-by': [['attributePath', 'string']],
|
|
44
|
+
// `strategies` is optional (omit to drop all entries for the path), so it stays off the required list.
|
|
45
|
+
'remove-aggregation-strategy': [['attributePath', 'string']],
|
|
46
|
+
'remove-reducer-exception': [['predicate', 'object']],
|
|
47
|
+
'remove-grok-rule': [['pattern', 'string']],
|
|
48
|
+
'update-source': [['source', 'object']],
|
|
49
|
+
'update-parser': [['parser', 'object']],
|
|
50
|
+
// Target-conditional fields (the vendor variant's `filter`) stay with the apply-time guards.
|
|
51
|
+
'update-sink': [['target', 'string'], ['sink', 'object']],
|
|
52
|
+
'update-message-attribute': [['from', 'string'], ['to', 'string']],
|
|
53
|
+
'update-group-by': [['from', 'string'], ['to', 'string']],
|
|
54
|
+
'update-aggregation-strategy': [['attributePath', 'string'], ['strategies', 'array']],
|
|
55
|
+
'update-reducer-exception': [['from', 'object'], ['to', 'object']],
|
|
42
56
|
};
|
|
43
57
|
function fieldMatches(value, type) {
|
|
44
58
|
switch (type) {
|
|
@@ -202,6 +216,31 @@ function applyOperation(input, op, index) {
|
|
|
202
216
|
return applyRemoveSink(input, op.target, 'name' in op ? op.name : undefined, index);
|
|
203
217
|
case 'set-raw-dataset':
|
|
204
218
|
return applySetRawDataset(input, op.datasetId);
|
|
219
|
+
case 'remove-message-attribute':
|
|
220
|
+
return applyRemoveMessageAttribute(input, op.attributePath, index);
|
|
221
|
+
case 'remove-group-by':
|
|
222
|
+
return applyRemoveGroupBy(input, op.attributePath, index);
|
|
223
|
+
case 'remove-aggregation-strategy':
|
|
224
|
+
return applyRemoveAggregation(input, op.attributePath, op.strategies, index);
|
|
225
|
+
case 'remove-reducer-exception':
|
|
226
|
+
return applyRemoveReducerException(input, op.predicate, index);
|
|
227
|
+
case 'remove-grok-rule':
|
|
228
|
+
return applyRemoveGrokRule(input, op.pattern, op.parserName, index);
|
|
229
|
+
case 'update-source':
|
|
230
|
+
return applyUpdateSource(input, op.source, index);
|
|
231
|
+
case 'update-parser':
|
|
232
|
+
return applyUpdateParser(input, op.parser, index);
|
|
233
|
+
case 'update-sink':
|
|
234
|
+
// See add-sink: read the vendor-only `filter` via `in` so a JSON patch's stray field still reaches the guard.
|
|
235
|
+
return applyUpdateSink(input, op.target, op.sink, 'filter' in op ? op.filter : undefined, index);
|
|
236
|
+
case 'update-message-attribute':
|
|
237
|
+
return applyUpdateMessageAttribute(input, op.from, op.to, index);
|
|
238
|
+
case 'update-group-by':
|
|
239
|
+
return applyUpdateGroupBy(input, op.from, op.to, index);
|
|
240
|
+
case 'update-aggregation-strategy':
|
|
241
|
+
return applyUpdateAggregation(input, op.attributePath, op.strategies, index);
|
|
242
|
+
case 'update-reducer-exception':
|
|
243
|
+
return applyUpdateReducerException(input, op.from, op.to, index);
|
|
205
244
|
default:
|
|
206
245
|
return throwUnknownOp(op, index);
|
|
207
246
|
}
|
|
@@ -243,6 +282,31 @@ function applyJobGraphOperation(jobGraph, op, index) {
|
|
|
243
282
|
return jobGraphRemoveSink(jobGraph, op.target, 'name' in op ? op.name : undefined, index);
|
|
244
283
|
case 'set-raw-dataset':
|
|
245
284
|
return jobGraphSetRawDataset(jobGraph, op.datasetId, index);
|
|
285
|
+
case 'remove-message-attribute':
|
|
286
|
+
return jobGraphRemoveMessageAttribute(vertices, op.attributePath, index);
|
|
287
|
+
case 'remove-group-by':
|
|
288
|
+
return jobGraphRemoveGroupBy(vertices, op.attributePath, index);
|
|
289
|
+
case 'remove-aggregation-strategy':
|
|
290
|
+
return jobGraphRemoveAggregation(vertices, op.attributePath, op.strategies, index);
|
|
291
|
+
case 'remove-reducer-exception':
|
|
292
|
+
return jobGraphRemoveReducerException(vertices, op.predicate, index);
|
|
293
|
+
case 'remove-grok-rule':
|
|
294
|
+
return jobGraphRemoveGrokRule(vertices, op.pattern, op.parserName, index);
|
|
295
|
+
case 'update-source':
|
|
296
|
+
return jobGraphUpdateSource(jobGraph, op.source, index);
|
|
297
|
+
case 'update-parser':
|
|
298
|
+
return jobGraphUpdateParser(jobGraph, op.parser, index);
|
|
299
|
+
case 'update-sink':
|
|
300
|
+
// See add-sink: read the vendor-only `filter` via `in`.
|
|
301
|
+
return jobGraphUpdateSink(jobGraph, op.target, op.sink, 'filter' in op ? op.filter : undefined, index);
|
|
302
|
+
case 'update-message-attribute':
|
|
303
|
+
return jobGraphUpdateMessageAttribute(vertices, op.from, op.to, index);
|
|
304
|
+
case 'update-group-by':
|
|
305
|
+
return jobGraphUpdateGroupBy(vertices, op.from, op.to, index);
|
|
306
|
+
case 'update-aggregation-strategy':
|
|
307
|
+
return jobGraphUpdateAggregation(vertices, op.attributePath, op.strategies, index);
|
|
308
|
+
case 'update-reducer-exception':
|
|
309
|
+
return jobGraphUpdateReducerException(vertices, op.from, op.to, index);
|
|
246
310
|
case 'set-input-field':
|
|
247
311
|
case 'unset-input-field':
|
|
248
312
|
throw new Error(`Operation ${index} (${op.op}): generic template-input paths are not supported on raw job graphs. ` +
|
|
@@ -289,20 +353,25 @@ function jobGraphAddReducerException(vertices, predicate, index) {
|
|
|
289
353
|
reducer.logReducerExceptions = existing;
|
|
290
354
|
}
|
|
291
355
|
function jobGraphAddGrokRule(vertices, pattern, parserName, extractAttribute, index) {
|
|
356
|
+
const grok = findJobGraphGrokParser(vertices, parserName, 'add-grok-rule', index);
|
|
357
|
+
applyGrokRuleToParser(grok, pattern, extractAttribute);
|
|
358
|
+
}
|
|
359
|
+
/** Resolve the target grok-parser vertex by name (or the sole grok-parser); shared by add/remove-grok-rule. Throws if absent or ambiguous. */
|
|
360
|
+
function findJobGraphGrokParser(vertices, parserName, opLabel, index) {
|
|
292
361
|
const groks = vertices.filter(v => v.type === GrokParserType.grok_parser);
|
|
293
|
-
const grok = selectGrokParser(groks, parserName, 'jobGraph.vertices', index);
|
|
362
|
+
const grok = selectGrokParser(groks, parserName, 'jobGraph.vertices', opLabel, index);
|
|
294
363
|
if (!grok) {
|
|
295
|
-
|
|
296
|
-
|
|
364
|
+
const hint = opLabel === 'add-grok-rule' ? " Add a grok-parser vertex via 'grepr job:update' before using add-grok-rule on this pipeline." : '';
|
|
365
|
+
throw new Error(`Operation ${index} (${opLabel}): ${parserName ? `grok-parser "${parserName}" not found` : 'no grok-parser vertex found in jobGraph.vertices'}.${hint}`);
|
|
297
366
|
}
|
|
298
|
-
|
|
367
|
+
return grok;
|
|
299
368
|
}
|
|
300
|
-
function selectGrokParser(groks, parserName, location, index) {
|
|
369
|
+
function selectGrokParser(groks, parserName, location, opLabel, index) {
|
|
301
370
|
if (parserName) {
|
|
302
371
|
return groks.find(parser => parser.name === parserName);
|
|
303
372
|
}
|
|
304
373
|
if (groks.length > 1) {
|
|
305
|
-
throw new Error(`Operation ${index} (
|
|
374
|
+
throw new Error(`Operation ${index} (${opLabel}): found ${groks.length} grok parsers in ${location}; ` +
|
|
306
375
|
`pass parserName to choose the target parser.`);
|
|
307
376
|
}
|
|
308
377
|
return groks[0];
|
|
@@ -359,6 +428,119 @@ function applyGrokRuleToParser(parser, pattern, extractAttribute) {
|
|
|
359
428
|
parser.extractAttribute = extractAttribute;
|
|
360
429
|
}
|
|
361
430
|
}
|
|
431
|
+
/** Consistent "entry not found" error shared by the per-entry remove-* and update-* ops. */
|
|
432
|
+
function entryNotFoundError(index, opLabel, what) {
|
|
433
|
+
return new Error(`Operation ${index} (${opLabel}): ${what} not found.`);
|
|
434
|
+
}
|
|
435
|
+
// remove-* counterparts of the add-* reducer/remapper/grok writers: mutate the same
|
|
436
|
+
// vertex field, shared verbatim by both backends, throw if the entry is absent.
|
|
437
|
+
function removeMessageAttributeFromRemapper(remapper, attributePath, index) {
|
|
438
|
+
const parts = splitPath(attributePath, index);
|
|
439
|
+
const removed = parts.length === 1
|
|
440
|
+
? removeString(remapper.messageReservedAttributes, parts[0])
|
|
441
|
+
: removeStringArray(remapper.messageReservedAttributePaths, parts);
|
|
442
|
+
if (!removed)
|
|
443
|
+
throw entryNotFoundError(index, 'remove-message-attribute', `message attribute "${attributePath}"`);
|
|
444
|
+
}
|
|
445
|
+
function removeGroupByFromReducer(reducer, attributePath, index) {
|
|
446
|
+
const parts = splitPath(attributePath, index);
|
|
447
|
+
const removed = parts.length === 1
|
|
448
|
+
? removeString(reducer.partitionByAttributes, parts[0])
|
|
449
|
+
: removeStringArray(reducer.partitionByAttributePaths, parts);
|
|
450
|
+
if (!removed)
|
|
451
|
+
throw entryNotFoundError(index, 'remove-group-by', `group-by "${attributePath}"`);
|
|
452
|
+
}
|
|
453
|
+
function removeAggregationFromReducer(reducer, attributePath, strategies, index) {
|
|
454
|
+
if (strategies !== undefined && (!Array.isArray(strategies) || strategies.length === 0)) {
|
|
455
|
+
throw new Error(`Operation ${index} (remove-aggregation-strategy): strategies must be a non-empty array; ` +
|
|
456
|
+
`omit it to drop every entry for the path.`);
|
|
457
|
+
}
|
|
458
|
+
const parts = splitPath(attributePath, index);
|
|
459
|
+
const types = strategies?.map(strategyTypeFor);
|
|
460
|
+
const entries = reducer.attributeMergeStrategyEntries ?? [];
|
|
461
|
+
const kept = entries.filter(entry => {
|
|
462
|
+
if (!arraysEqual(entry.attributePath, parts))
|
|
463
|
+
return true;
|
|
464
|
+
if (types === undefined)
|
|
465
|
+
return false; // no strategies given: drop every entry for the path
|
|
466
|
+
const type = entry.strategy?.type;
|
|
467
|
+
return type === undefined || !types.includes(type); // else drop only the listed strategies
|
|
468
|
+
});
|
|
469
|
+
if (kept.length === entries.length) {
|
|
470
|
+
const what = strategies === undefined
|
|
471
|
+
? `aggregation strategy for "${attributePath}"`
|
|
472
|
+
: `aggregation strategy [${strategies.join(', ')}] for "${attributePath}"`;
|
|
473
|
+
throw entryNotFoundError(index, 'remove-aggregation-strategy', what);
|
|
474
|
+
}
|
|
475
|
+
reducer.attributeMergeStrategyEntries = kept;
|
|
476
|
+
}
|
|
477
|
+
function removeGrokRuleFromParser(parser, pattern, index) {
|
|
478
|
+
if (!removeString(parser.grokParsingRules, pattern)) {
|
|
479
|
+
throw entryNotFoundError(index, 'remove-grok-rule', `grok rule "${pattern}"`);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
// In-place updates of the dual single/multi-part path lists shared by message
|
|
483
|
+
// attributes (remapper) and group-by (reducer): replace `from` with `to` at its
|
|
484
|
+
// existing position. Both must share arity since the two lists are separate.
|
|
485
|
+
function updateDualListPath(single, multi, from, to, opLabel, noun, index) {
|
|
486
|
+
const fromParts = splitPath(from, index);
|
|
487
|
+
const toParts = splitPath(to, index);
|
|
488
|
+
if ((fromParts.length === 1) !== (toParts.length === 1)) {
|
|
489
|
+
throw new Error(`Operation ${index} (${opLabel}): cannot change "${from}" to "${to}" in place — ` +
|
|
490
|
+
`single-part and multi-part paths are stored in different lists; remove + add instead.`);
|
|
491
|
+
}
|
|
492
|
+
if (from !== to) {
|
|
493
|
+
const toExists = toParts.length === 1
|
|
494
|
+
? (single ?? []).includes(toParts[0])
|
|
495
|
+
: (multi ?? []).some(existing => arraysEqual(existing, toParts));
|
|
496
|
+
if (toExists) {
|
|
497
|
+
throw new Error(`Operation ${index} (${opLabel}): ${noun} "${to}" already exists; remove "${from}" instead.`);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
const replaced = fromParts.length === 1
|
|
501
|
+
? replaceString(single, fromParts[0], toParts[0])
|
|
502
|
+
: replaceStringArray(multi, fromParts, toParts);
|
|
503
|
+
if (!replaced)
|
|
504
|
+
throw entryNotFoundError(index, opLabel, `${noun} "${from}"`);
|
|
505
|
+
}
|
|
506
|
+
function updateMessageAttributeOnRemapper(remapper, from, to, index) {
|
|
507
|
+
updateDualListPath(remapper.messageReservedAttributes, remapper.messageReservedAttributePaths, from, to, 'update-message-attribute', 'message attribute', index);
|
|
508
|
+
}
|
|
509
|
+
function updateGroupByOnReducer(reducer, from, to, index) {
|
|
510
|
+
updateDualListPath(reducer.partitionByAttributes, reducer.partitionByAttributePaths, from, to, 'update-group-by', 'group-by', index);
|
|
511
|
+
}
|
|
512
|
+
/** Replace the merge-strategy set for `attributePath` in place, anchored at its first existing entry's position. Errors if the path has no entries. */
|
|
513
|
+
function updateAggregationOnReducer(reducer, attributePath, strategies, index) {
|
|
514
|
+
if (strategies.length === 0) {
|
|
515
|
+
throw new Error(`Operation ${index} (update-aggregation-strategy): strategies array must not be empty`);
|
|
516
|
+
}
|
|
517
|
+
const parts = splitPath(attributePath, index);
|
|
518
|
+
const entries = reducer.attributeMergeStrategyEntries ?? [];
|
|
519
|
+
const at = entries.findIndex(e => arraysEqual(e.attributePath, parts));
|
|
520
|
+
if (at === -1) {
|
|
521
|
+
throw entryNotFoundError(index, 'update-aggregation-strategy', `aggregation strategy for "${attributePath}"`);
|
|
522
|
+
}
|
|
523
|
+
const replacement = [...new Set(strategies.map(strategyTypeFor))].map(type => ({ attributePath: parts, strategy: { type } }));
|
|
524
|
+
const kept = entries.filter(e => !arraysEqual(e.attributePath, parts));
|
|
525
|
+
kept.splice(at, 0, ...replacement); // `at` is the path's first index; all earlier entries are non-path, so it survives the filter.
|
|
526
|
+
reducer.attributeMergeStrategyEntries = kept;
|
|
527
|
+
}
|
|
528
|
+
/**
|
|
529
|
+
* Locate `from` among the serialized existing predicates for update-reducer-exception
|
|
530
|
+
* (entries that aren't predicates serialize to `undefined` and never match). Returns the
|
|
531
|
+
* index to replace; throws if `from` is absent or `to` would duplicate a different entry.
|
|
532
|
+
*/
|
|
533
|
+
function reducerExceptionUpdateIndex(serialized, from, to, index) {
|
|
534
|
+
const at = serialized.indexOf(JSON.stringify(from));
|
|
535
|
+
if (at === -1) {
|
|
536
|
+
throw entryNotFoundError(index, 'update-reducer-exception', 'matching reducer exception');
|
|
537
|
+
}
|
|
538
|
+
const dup = serialized.indexOf(JSON.stringify(to));
|
|
539
|
+
if (dup !== -1 && dup !== at) {
|
|
540
|
+
throw new Error(`Operation ${index} (update-reducer-exception): an exception matching "to" already exists; remove the "from" exception instead.`);
|
|
541
|
+
}
|
|
542
|
+
return at;
|
|
543
|
+
}
|
|
362
544
|
const DEFAULT_EDGE_OUTPUT = 'output';
|
|
363
545
|
const DEFAULT_EDGE_INPUT = 'input';
|
|
364
546
|
function jobGraphAddSource(jobGraph, source, index) {
|
|
@@ -384,7 +566,7 @@ function jobGraphRemoveSource(jobGraph, name, index) {
|
|
|
384
566
|
}
|
|
385
567
|
function jobGraphAddSink(jobGraph, target, sink, filter, index) {
|
|
386
568
|
assertOperationIdentity(sink, 'add-sink', index, 'sink');
|
|
387
|
-
assertSinkTargetShape(target, sink, filter, index);
|
|
569
|
+
assertSinkTargetShape(target, sink, filter, index, 'add-sink');
|
|
388
570
|
// Anchors on a unique log_reducer (assertRawUiLogGraph rejects missing/duplicate).
|
|
389
571
|
assertRawUiLogGraph(jobGraph, 'add-sink', index, [RAW_LOG_REDUCER]);
|
|
390
572
|
if (findVertexIndexByName(jobGraph, sink.name) !== -1) {
|
|
@@ -553,6 +735,112 @@ function jobGraphRemoveParser(jobGraph, name, index) {
|
|
|
553
735
|
const to = outgoing[0];
|
|
554
736
|
addEdgeIfMissing(jobGraph, from.sourceVertex, from.sourcePort, to.targetVertex, to.targetPort);
|
|
555
737
|
}
|
|
738
|
+
// --- Job-graph per-entry removals: anchor on the unique reducer/remapper/grok
|
|
739
|
+
// vertex (the add-path guards), then delegate to the shared remover. ---
|
|
740
|
+
function jobGraphRemoveMessageAttribute(vertices, attributePath, index) {
|
|
741
|
+
const remapper = findUniqueVertexByType(vertices, LogAttributesRemapperType.log_attributes_remapper, 'remove-message-attribute', index);
|
|
742
|
+
removeMessageAttributeFromRemapper(remapper, attributePath, index);
|
|
743
|
+
}
|
|
744
|
+
function jobGraphRemoveGroupBy(vertices, attributePath, index) {
|
|
745
|
+
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'remove-group-by', index);
|
|
746
|
+
removeGroupByFromReducer(reducer, attributePath, index);
|
|
747
|
+
}
|
|
748
|
+
function jobGraphRemoveAggregation(vertices, attributePath, strategies, index) {
|
|
749
|
+
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'remove-aggregation-strategy', index);
|
|
750
|
+
removeAggregationFromReducer(reducer, attributePath, strategies, index);
|
|
751
|
+
}
|
|
752
|
+
/** Remove a raw `EventPredicate` from the reducer's `logReducerExceptions` (mirror of jobGraphAddReducerException). */
|
|
753
|
+
function jobGraphRemoveReducerException(vertices, predicate, index) {
|
|
754
|
+
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'remove-reducer-exception', index);
|
|
755
|
+
const serialized = JSON.stringify(predicate);
|
|
756
|
+
const existing = reducer.logReducerExceptions ?? [];
|
|
757
|
+
const kept = existing.filter(p => JSON.stringify(p) !== serialized);
|
|
758
|
+
if (kept.length === existing.length) {
|
|
759
|
+
throw entryNotFoundError(index, 'remove-reducer-exception', 'matching reducer exception');
|
|
760
|
+
}
|
|
761
|
+
reducer.logReducerExceptions = kept;
|
|
762
|
+
}
|
|
763
|
+
function jobGraphRemoveGrokRule(vertices, pattern, parserName, index) {
|
|
764
|
+
const grok = findJobGraphGrokParser(vertices, parserName, 'remove-grok-rule', index);
|
|
765
|
+
removeGrokRuleFromParser(grok, pattern, index);
|
|
766
|
+
}
|
|
767
|
+
// --- Job-graph in-place vertex updates: replace the named vertex's config via
|
|
768
|
+
// replaceVertexByName, leaving edges/position untouched. ---
|
|
769
|
+
function jobGraphUpdateSource(jobGraph, source, index) {
|
|
770
|
+
assertOperationIdentity(source, 'update-source', index, 'source');
|
|
771
|
+
assertRawUiLogGraph(jobGraph, 'update-source', index, [RAW_PRE_PARSER_FILTER]);
|
|
772
|
+
if (findVertexIndexByName(jobGraph, source.name) === -1) {
|
|
773
|
+
throw new Error(`Operation ${index} (update-source): source "${source.name}" not found in jobGraph.vertices.`);
|
|
774
|
+
}
|
|
775
|
+
if (!isCanonicalRawSource(jobGraph, source.name)) {
|
|
776
|
+
throw unsupportedRawShapeError(index, 'update-source', `vertex "${source.name}" is not a canonical UI source feeding ${RAW_PRE_PARSER_FILTER}`);
|
|
777
|
+
}
|
|
778
|
+
replaceVertexByName(jobGraph, source.name, source, 'update-source', index);
|
|
779
|
+
}
|
|
780
|
+
function jobGraphUpdateParser(jobGraph, parser, index) {
|
|
781
|
+
assertOperationIdentity(parser, 'update-parser', index, 'parser');
|
|
782
|
+
assertRawUiLogGraph(jobGraph, 'update-parser', index, [RAW_PRE_PARSER_FILTER, RAW_PRE_WAREHOUSE_FILTER]);
|
|
783
|
+
const existing = findVertexByName(jobGraph, parser.name);
|
|
784
|
+
if (!existing) {
|
|
785
|
+
throw new Error(`Operation ${index} (update-parser): parser "${parser.name}" not found in jobGraph.vertices.`);
|
|
786
|
+
}
|
|
787
|
+
if (!RAW_PARSER_TYPES.has(existing.type)) {
|
|
788
|
+
throw new Error(`Operation ${index} (update-parser): vertex "${parser.name}" is not a supported parser type.`);
|
|
789
|
+
}
|
|
790
|
+
if (!RAW_PARSER_TYPES.has(parser.type)) {
|
|
791
|
+
throw new Error(`Operation ${index} (update-parser): replacement parser type "${parser.type}" is not supported for raw UI graph parsers.`);
|
|
792
|
+
}
|
|
793
|
+
replaceVertexByName(jobGraph, parser.name, parser, 'update-parser', index);
|
|
794
|
+
}
|
|
795
|
+
function jobGraphUpdateSink(jobGraph, target, sink, filter, index) {
|
|
796
|
+
assertOperationIdentity(sink, 'update-sink', index, 'sink');
|
|
797
|
+
assertSinkTargetShape(target, sink, filter, index, 'update-sink');
|
|
798
|
+
if (target === 'processed-logs') {
|
|
799
|
+
const existing = findIcebergSinkByRole(jobGraph, PROCESSED_LOGS_SINK_NAME_PREFIX, 'processed-logs', 'update-sink', 'update the sink', index);
|
|
800
|
+
replaceVertexByName(jobGraph, existing.name, { ...sink, name: existing.name }, 'update-sink', index);
|
|
801
|
+
return;
|
|
802
|
+
}
|
|
803
|
+
// vendor: anchor on the unique reducer (as add-sink does), then replace the named vendor sink.
|
|
804
|
+
assertRawUiLogGraph(jobGraph, 'update-sink', index, [RAW_LOG_REDUCER]);
|
|
805
|
+
const existing = findVertexByName(jobGraph, sink.name);
|
|
806
|
+
if (!existing) {
|
|
807
|
+
throw new Error(`Operation ${index} (update-sink): sink "${sink.name}" not found in jobGraph.vertices.`);
|
|
808
|
+
}
|
|
809
|
+
if (!VENDOR_LOG_SINK_TYPES.has(existing.type)) {
|
|
810
|
+
throw new Error(`Operation ${index} (update-sink): vertex "${sink.name}" is not a vendor sink (type "${existing.type}").`);
|
|
811
|
+
}
|
|
812
|
+
replaceVertexByName(jobGraph, sink.name, sink, 'update-sink', index);
|
|
813
|
+
if (filter !== undefined) {
|
|
814
|
+
const filterName = `${sink.name}_filter`;
|
|
815
|
+
if (!findVertexByName(jobGraph, filterName)) {
|
|
816
|
+
throw new Error(`Operation ${index} (update-sink): sink "${sink.name}" has no generated gating filter "${filterName}" to update; ` +
|
|
817
|
+
`use remove-sink + add-sink to introduce a gate.`);
|
|
818
|
+
}
|
|
819
|
+
replaceRawFilterVertex(jobGraph, filterName, filter, 'update-sink', index);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
// --- Job-graph in-place updates of reducer-list entries: anchor on the unique
|
|
823
|
+
// reducer/remapper (as the add/remove paths do), then delegate to the shared updater. ---
|
|
824
|
+
function jobGraphUpdateMessageAttribute(vertices, from, to, index) {
|
|
825
|
+
const remapper = findUniqueVertexByType(vertices, LogAttributesRemapperType.log_attributes_remapper, 'update-message-attribute', index);
|
|
826
|
+
updateMessageAttributeOnRemapper(remapper, from, to, index);
|
|
827
|
+
}
|
|
828
|
+
function jobGraphUpdateGroupBy(vertices, from, to, index) {
|
|
829
|
+
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'update-group-by', index);
|
|
830
|
+
updateGroupByOnReducer(reducer, from, to, index);
|
|
831
|
+
}
|
|
832
|
+
function jobGraphUpdateAggregation(vertices, attributePath, strategies, index) {
|
|
833
|
+
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'update-aggregation-strategy', index);
|
|
834
|
+
updateAggregationOnReducer(reducer, attributePath, strategies, index);
|
|
835
|
+
}
|
|
836
|
+
/** Replace the matching raw predicate in the reducer's `logReducerExceptions` in place (mirror of jobGraphAddReducerException). */
|
|
837
|
+
function jobGraphUpdateReducerException(vertices, from, to, index) {
|
|
838
|
+
const reducer = findUniqueVertexByType(vertices, LogReducerType.log_reducer, 'update-reducer-exception', index);
|
|
839
|
+
const existing = reducer.logReducerExceptions ?? [];
|
|
840
|
+
const idx = reducerExceptionUpdateIndex(existing.map(p => JSON.stringify(p)), from, to, index);
|
|
841
|
+
existing[idx] = to;
|
|
842
|
+
reducer.logReducerExceptions = existing;
|
|
843
|
+
}
|
|
556
844
|
function rawFilterNameForPhase(phase, index, opLabel) {
|
|
557
845
|
switch (phase) {
|
|
558
846
|
case 'pre-parser':
|
|
@@ -750,18 +1038,22 @@ function applyAddReducerException(input, predicate) {
|
|
|
750
1038
|
input.exceptions.push(exception);
|
|
751
1039
|
}
|
|
752
1040
|
function applyAddGrokRule(input, pattern, parserName, extractAttribute, index) {
|
|
753
|
-
const
|
|
1041
|
+
const grok = findTemplateGrokParser(input.parsers, parserName, 'add-grok-rule', index);
|
|
1042
|
+
applyGrokRuleToParser(grok, pattern, extractAttribute);
|
|
1043
|
+
}
|
|
1044
|
+
/** Resolve the target grok-parser in `input.parsers` by name (or the sole grok-parser); shared by add/remove-grok-rule. Throws if absent, ambiguous, or not a grok-parser. */
|
|
1045
|
+
function findTemplateGrokParser(parsers, parserName, opLabel, index) {
|
|
754
1046
|
const grok = parserName
|
|
755
1047
|
? parsers.find(p => p.name === parserName)
|
|
756
|
-
: selectGrokParser(parsers.filter(p => p.type === GrokParserType.grok_parser), undefined, 'input.parsers', index);
|
|
1048
|
+
: selectGrokParser(parsers.filter(p => p.type === GrokParserType.grok_parser), undefined, 'input.parsers', opLabel, index);
|
|
757
1049
|
if (!grok) {
|
|
758
|
-
|
|
759
|
-
|
|
1050
|
+
const hint = opLabel === 'add-grok-rule' ? ' If you need to introduce one, use add-parser first.' : '';
|
|
1051
|
+
throw new Error(`Operation ${index} (${opLabel}): ${parserName ? `parser "${parserName}" not found` : 'no grok-parser found in input.parsers'}.${hint}`);
|
|
760
1052
|
}
|
|
761
1053
|
if (grok.type !== GrokParserType.grok_parser) {
|
|
762
|
-
throw new Error(`Operation ${index} (
|
|
1054
|
+
throw new Error(`Operation ${index} (${opLabel}): parser "${grok.name}" is not a grok-parser`);
|
|
763
1055
|
}
|
|
764
|
-
|
|
1056
|
+
return grok;
|
|
765
1057
|
}
|
|
766
1058
|
function applySetInputField(input, path, value, index) {
|
|
767
1059
|
const parts = splitPath(path, index);
|
|
@@ -834,34 +1126,137 @@ function applyRemoveSource(input, name, index) {
|
|
|
834
1126
|
}
|
|
835
1127
|
input.sources.splice(idx, 1);
|
|
836
1128
|
}
|
|
1129
|
+
// --- Template per-entry removals: resolve the same reducer/remapper/grok the add
|
|
1130
|
+
// path writes, then delegate to the shared remover. ---
|
|
1131
|
+
function applyRemoveMessageAttribute(input, attributePath, index) {
|
|
1132
|
+
const remapper = findRemapper(input);
|
|
1133
|
+
if (!remapper) {
|
|
1134
|
+
throw new Error(`Operation ${index} (remove-message-attribute): no log-attributes-remapper in input.parsers.`);
|
|
1135
|
+
}
|
|
1136
|
+
removeMessageAttributeFromRemapper(remapper, attributePath, index);
|
|
1137
|
+
}
|
|
1138
|
+
function applyRemoveGroupBy(input, attributePath, index) {
|
|
1139
|
+
removeGroupByFromReducer(input.reducer, attributePath, index);
|
|
1140
|
+
}
|
|
1141
|
+
function applyRemoveAggregation(input, attributePath, strategies, index) {
|
|
1142
|
+
removeAggregationFromReducer(input.reducer, attributePath, strategies, index);
|
|
1143
|
+
}
|
|
1144
|
+
/** Remove the `TemplateQueryException` whose predicate matches (mirror of applyAddReducerException). */
|
|
1145
|
+
function applyRemoveReducerException(input, predicate, index) {
|
|
1146
|
+
const serialized = JSON.stringify(predicate);
|
|
1147
|
+
const exceptions = input.exceptions ?? [];
|
|
1148
|
+
const kept = exceptions.filter(e => !(e.type === TemplateQueryExceptionType.query_exception &&
|
|
1149
|
+
JSON.stringify(e.predicate) === serialized));
|
|
1150
|
+
if (kept.length === exceptions.length) {
|
|
1151
|
+
throw entryNotFoundError(index, 'remove-reducer-exception', 'matching reducer exception');
|
|
1152
|
+
}
|
|
1153
|
+
input.exceptions = kept;
|
|
1154
|
+
}
|
|
1155
|
+
function applyRemoveGrokRule(input, pattern, parserName, index) {
|
|
1156
|
+
const grok = findTemplateGrokParser(input.parsers, parserName, 'remove-grok-rule', index);
|
|
1157
|
+
removeGrokRuleFromParser(grok, pattern, index);
|
|
1158
|
+
}
|
|
1159
|
+
// --- Template in-place vertex updates: replace the matching list entry by name. ---
|
|
1160
|
+
function applyUpdateSource(input, source, index) {
|
|
1161
|
+
assertOperationIdentity(source, 'update-source', index, 'source');
|
|
1162
|
+
const idx = input.sources.findIndex(s => s.name === source.name);
|
|
1163
|
+
if (idx === -1) {
|
|
1164
|
+
throw new Error(`Operation ${index} (update-source): source "${source.name}" not found in input.sources.`);
|
|
1165
|
+
}
|
|
1166
|
+
input.sources[idx] = source;
|
|
1167
|
+
}
|
|
1168
|
+
function applyUpdateParser(input, parser, index) {
|
|
1169
|
+
assertOperationIdentity(parser, 'update-parser', index, 'parser');
|
|
1170
|
+
const idx = input.parsers.findIndex(p => p.name === parser.name);
|
|
1171
|
+
if (idx === -1) {
|
|
1172
|
+
throw new Error(`Operation ${index} (update-parser): parser "${parser.name}" not found in input.parsers.`);
|
|
1173
|
+
}
|
|
1174
|
+
input.parsers[idx] = parser;
|
|
1175
|
+
}
|
|
1176
|
+
function applyUpdateSink(input, target, sink, filter, index) {
|
|
1177
|
+
assertOperationIdentity(sink, 'update-sink', index, 'sink');
|
|
1178
|
+
assertSinkTargetShape(target, sink, filter, index, 'update-sink');
|
|
1179
|
+
if (target === 'vendor') {
|
|
1180
|
+
const sinks = input.sinks ?? [];
|
|
1181
|
+
const existingEntry = sinks.find(entry => entry.sink?.name === sink.name);
|
|
1182
|
+
if (!existingEntry) {
|
|
1183
|
+
throw new Error(`Operation ${index} (update-sink): sink "${sink.name}" not found in input.sinks.`);
|
|
1184
|
+
}
|
|
1185
|
+
if (filter !== undefined && existingEntry.filter === undefined) {
|
|
1186
|
+
throw new Error(`Operation ${index} (update-sink): sink "${sink.name}" has no gating filter to update; ` +
|
|
1187
|
+
`use remove-sink + add-sink to introduce a gate.`);
|
|
1188
|
+
}
|
|
1189
|
+
const replacementEntry = { sink };
|
|
1190
|
+
if (filter !== undefined) {
|
|
1191
|
+
replacementEntry.filter = filter;
|
|
1192
|
+
}
|
|
1193
|
+
else if (existingEntry.filter !== undefined) {
|
|
1194
|
+
replacementEntry.filter = existingEntry.filter;
|
|
1195
|
+
}
|
|
1196
|
+
input.sinks = sinks.map(entry => (entry === existingEntry ? replacementEntry : entry));
|
|
1197
|
+
return;
|
|
1198
|
+
}
|
|
1199
|
+
// processed-logs: a singular slot, so the replacement is taken as-is — its name is
|
|
1200
|
+
// not matched against the existing sink (unlike the by-name vendor path above).
|
|
1201
|
+
if (input.processedLogsSink === undefined || input.processedLogsSink === null) {
|
|
1202
|
+
throw new Error(`Operation ${index} (update-sink): no processedLogsSink set to update.`);
|
|
1203
|
+
}
|
|
1204
|
+
// Validated as a logs-iceberg-table-sink by assertSinkTargetShape.
|
|
1205
|
+
input.processedLogsSink = sink;
|
|
1206
|
+
}
|
|
1207
|
+
// --- Template in-place updates of reducer-list entries: resolve the same
|
|
1208
|
+
// reducer/remapper the add/remove paths use, then delegate to the shared updater. ---
|
|
1209
|
+
function applyUpdateMessageAttribute(input, from, to, index) {
|
|
1210
|
+
const remapper = findRemapper(input);
|
|
1211
|
+
if (!remapper) {
|
|
1212
|
+
throw new Error(`Operation ${index} (update-message-attribute): no log-attributes-remapper in input.parsers.`);
|
|
1213
|
+
}
|
|
1214
|
+
updateMessageAttributeOnRemapper(remapper, from, to, index);
|
|
1215
|
+
}
|
|
1216
|
+
function applyUpdateGroupBy(input, from, to, index) {
|
|
1217
|
+
updateGroupByOnReducer(input.reducer, from, to, index);
|
|
1218
|
+
}
|
|
1219
|
+
function applyUpdateAggregation(input, attributePath, strategies, index) {
|
|
1220
|
+
updateAggregationOnReducer(input.reducer, attributePath, strategies, index);
|
|
1221
|
+
}
|
|
1222
|
+
/** Replace the matching `TemplateQueryException`'s predicate in place (mirror of applyAddReducerException). */
|
|
1223
|
+
function applyUpdateReducerException(input, from, to, index) {
|
|
1224
|
+
const exceptions = input.exceptions ?? [];
|
|
1225
|
+
const serialized = exceptions.map(e => e.type === TemplateQueryExceptionType.query_exception
|
|
1226
|
+
? JSON.stringify(e.predicate)
|
|
1227
|
+
: undefined);
|
|
1228
|
+
const idx = reducerExceptionUpdateIndex(serialized, from, to, index);
|
|
1229
|
+
exceptions[idx] = { type: TemplateQueryExceptionType.query_exception, predicate: to };
|
|
1230
|
+
input.exceptions = exceptions;
|
|
1231
|
+
}
|
|
837
1232
|
/** Reject a `target` not in {@link SinkTarget}; a typo like `"processed-log"` would otherwise fall through to the processed-logs branch. */
|
|
838
1233
|
function assertValidSinkTarget(target, opLabel, index) {
|
|
839
1234
|
if (target !== 'vendor' && target !== 'processed-logs') {
|
|
840
1235
|
throw new Error(`Operation ${index} (${opLabel}): target must be "vendor" or "processed-logs", got ${JSON.stringify(target)}.`);
|
|
841
1236
|
}
|
|
842
1237
|
}
|
|
843
|
-
/** Validate the sink type matches the target and `filter` is vendor-only; shared by both backends. */
|
|
844
|
-
function assertSinkTargetShape(target, sink, filter, index) {
|
|
845
|
-
assertValidSinkTarget(target,
|
|
1238
|
+
/** Validate the sink type matches the target and `filter` is vendor-only; shared by add-sink and update-sink on both backends. */
|
|
1239
|
+
function assertSinkTargetShape(target, sink, filter, index, opLabel) {
|
|
1240
|
+
assertValidSinkTarget(target, opLabel, index);
|
|
846
1241
|
if (target === 'vendor') {
|
|
847
1242
|
if (!VENDOR_LOG_SINK_TYPES.has(sink.type)) {
|
|
848
|
-
throw new Error(`Operation ${index} (
|
|
1243
|
+
throw new Error(`Operation ${index} (${opLabel}): vendor sink type "${sink.type}" is not supported. ` +
|
|
849
1244
|
`Supported: ${[...VENDOR_LOG_SINK_TYPES].join(', ')}.`);
|
|
850
1245
|
}
|
|
851
1246
|
return;
|
|
852
1247
|
}
|
|
853
1248
|
// processed-logs
|
|
854
1249
|
if (sink.type !== LOGS_ICEBERG_TABLE_SINK_TYPE) {
|
|
855
|
-
throw new Error(`Operation ${index} (
|
|
1250
|
+
throw new Error(`Operation ${index} (${opLabel}): target "processed-logs" requires a ${LOGS_ICEBERG_TABLE_SINK_TYPE} sink, got "${sink.type}".`);
|
|
856
1251
|
}
|
|
857
1252
|
if (filter !== undefined) {
|
|
858
|
-
throw new Error(`Operation ${index} (
|
|
1253
|
+
throw new Error(`Operation ${index} (${opLabel}): filter is only supported for target "vendor"; ` +
|
|
859
1254
|
`the processed-logs sink has no per-sink filter.`);
|
|
860
1255
|
}
|
|
861
1256
|
}
|
|
862
1257
|
function applyAddSink(input, target, sink, filter, index) {
|
|
863
1258
|
assertOperationIdentity(sink, 'add-sink', index, 'sink');
|
|
864
|
-
assertSinkTargetShape(target, sink, filter, index);
|
|
1259
|
+
assertSinkTargetShape(target, sink, filter, index, 'add-sink');
|
|
865
1260
|
if (target === 'vendor') {
|
|
866
1261
|
const sinks = input.sinks ?? [];
|
|
867
1262
|
if (sinks.some(entry => entry.sink?.name === sink.name)) {
|
|
@@ -970,6 +1365,7 @@ function touchesSourceConfig(op) {
|
|
|
970
1365
|
switch (op.op) {
|
|
971
1366
|
case 'add-source':
|
|
972
1367
|
case 'remove-source':
|
|
1368
|
+
case 'update-source':
|
|
973
1369
|
return true;
|
|
974
1370
|
case 'set-input-field':
|
|
975
1371
|
case 'unset-input-field': {
|
|
@@ -984,6 +1380,7 @@ function touchesSinkConfig(op) {
|
|
|
984
1380
|
switch (op.op) {
|
|
985
1381
|
case 'add-sink':
|
|
986
1382
|
case 'remove-sink':
|
|
1383
|
+
case 'update-sink':
|
|
987
1384
|
case 'set-raw-dataset':
|
|
988
1385
|
return true;
|
|
989
1386
|
case 'set-input-field':
|
|
@@ -1049,6 +1446,8 @@ function strategyTypeFor(strategy) {
|
|
|
1049
1446
|
case 'max': return MaxAttributesMergeStrategyType.max;
|
|
1050
1447
|
case 'avg': return AverageAttributesMergeStrategyType.avg;
|
|
1051
1448
|
}
|
|
1449
|
+
// Reachable only from an untyped JSON patch; the switch is exhaustive for the union.
|
|
1450
|
+
throw new Error(`Unknown aggregation strategy ${JSON.stringify(strategy)}; expected one of: sum, min, max, avg.`);
|
|
1052
1451
|
}
|
|
1053
1452
|
function addUniqueString(list, value) {
|
|
1054
1453
|
if (!list.includes(value))
|
|
@@ -1059,6 +1458,46 @@ function addUniqueStringArray(list, value) {
|
|
|
1059
1458
|
list.push(value);
|
|
1060
1459
|
}
|
|
1061
1460
|
}
|
|
1461
|
+
/** Remove the first occurrence of `value`; returns whether anything was removed. */
|
|
1462
|
+
function removeString(list, value) {
|
|
1463
|
+
if (!list)
|
|
1464
|
+
return false;
|
|
1465
|
+
const idx = list.indexOf(value);
|
|
1466
|
+
if (idx === -1)
|
|
1467
|
+
return false;
|
|
1468
|
+
list.splice(idx, 1);
|
|
1469
|
+
return true;
|
|
1470
|
+
}
|
|
1471
|
+
/** Remove the first array equal to `value`; returns whether anything was removed. */
|
|
1472
|
+
function removeStringArray(list, value) {
|
|
1473
|
+
if (!list)
|
|
1474
|
+
return false;
|
|
1475
|
+
const idx = list.findIndex(existing => arraysEqual(existing, value));
|
|
1476
|
+
if (idx === -1)
|
|
1477
|
+
return false;
|
|
1478
|
+
list.splice(idx, 1);
|
|
1479
|
+
return true;
|
|
1480
|
+
}
|
|
1481
|
+
/** Replace the first occurrence of `oldV` with `newV` in place; returns whether `oldV` was found. */
|
|
1482
|
+
function replaceString(list, oldV, newV) {
|
|
1483
|
+
if (!list)
|
|
1484
|
+
return false;
|
|
1485
|
+
const idx = list.indexOf(oldV);
|
|
1486
|
+
if (idx === -1)
|
|
1487
|
+
return false;
|
|
1488
|
+
list[idx] = newV;
|
|
1489
|
+
return true;
|
|
1490
|
+
}
|
|
1491
|
+
/** Replace the first array equal to `oldV` with `newV` in place; returns whether `oldV` was found. */
|
|
1492
|
+
function replaceStringArray(list, oldV, newV) {
|
|
1493
|
+
if (!list)
|
|
1494
|
+
return false;
|
|
1495
|
+
const idx = list.findIndex(existing => arraysEqual(existing, oldV));
|
|
1496
|
+
if (idx === -1)
|
|
1497
|
+
return false;
|
|
1498
|
+
list[idx] = newV;
|
|
1499
|
+
return true;
|
|
1500
|
+
}
|
|
1062
1501
|
function arraysEqual(a, b) {
|
|
1063
1502
|
if (a.length !== b.length)
|
|
1064
1503
|
return false;
|