@grepr/cli 1.6.29-bea24ac → 1.6.42-91f3e80

package/build/dist/lib/job-patch.js

@@ -1,7 +1,12 @@
 /** Structured patch format for grepr pipelines: a list of ops applied locally before any production write (see {@link JobBackend} for the two substrates). */
-import { LogAttributesRemapperType, LogReducerType, GrokParserType, LogsFilterType, DatadogQueryPredicateType, DatadogLogSinkType, SplunkLogSinkType, NewRelicLogSinkType, SumoLogSinkType, OtlpLogSinkType, LogsIcebergTableSinkType, TemplateOperationType, TemplateQueryExceptionType, SumAttributesMergeStrategyType, MinAttributesMergeStrategyType, MaxAttributesMergeStrategyType, AverageAttributesMergeStrategyType, } from '../openapi/openApiTypes.js';
+import { ConditionNodeKind, DropNodeKind, PassthroughNodeKind, SqlNodeKind, SqlNodeOutputRouting, SqlOperationType, SqlOutputStatementType, LogAttributesRemapperType, LogReducerType, GrokParserType, LogsFilterType, DatadogQueryPredicateType, DatadogLogSinkType, SplunkLogSinkType, NewRelicLogSinkType, SumoLogSinkType, OtlpLogSinkType, LogsIcebergTableSinkType, TemplateOperationType, TemplateQueryExceptionType, SumAttributesMergeStrategyType, MinAttributesMergeStrategyType, MaxAttributesMergeStrategyType, AverageAttributesMergeStrategyType, } from '../openapi/openApiTypes.js';
 import { parseEdge } from './job-graph-utils.js';
 import { RAW_ATTRIBUTES_REMAPPER, RAW_ATTRIBUTES_REMAPPER_TYPE, RAW_JSON_PROCESSOR, RAW_JSON_PROCESSOR_TYPE, RAW_LOG_REDUCER, RAW_PARSER_TYPES, RAW_PRE_EXCEPTIONS_FILTER, RAW_PRE_PARSER_FILTER, RAW_PRE_WAREHOUSE_FILTER, } from './job-graph-log-pipeline-constants.js';
+/** The accepted-phases suffix shared by every template transform-phase error. */
+const ACCEPTED_TRANSFORM_PHASES_MESSAGE = 'Accepted phases are pre-parser, pre-warehouse, and pre-exceptions.';
+/** The downstream pipeline steps a `sql_output` may route to (the generated {@link SqlNodeOutputRouting} values). */
+const SQL_OUTPUT_ROUTING_TARGETS = new Set(Object.values(SqlNodeOutputRouting));
+const SQL_TRANSFORM_MAIN_STREAMS = new Set(['drop', 'passthrough']);
 /** Vendor log sink types the UI supports as `add-sink target: 'vendor'`. */
 const VENDOR_LOG_SINK_TYPES = new Set([
     DatadogLogSinkType.datadog_log_sink,
@@ -33,6 +38,9 @@ const REQUIRED_OP_FIELDS = {
     'remove-parser': [['name', 'string']],
     'set-filter': [['phase', 'string'], ['filter', 'object']],
     'clear-filter': [['phase', 'string']],
+    'set-transform-chain': [['phase', 'string'], ['root', 'object']],
+    'clear-transform-chain': [['phase', 'string']],
+    'set-sql-transform': [['phase', 'string'], ['sqlOperation', 'object'], ['outputRouting', 'object'], ['mainStream', 'string']],
     'add-source': [['source', 'object']],
     'remove-source': [['name', 'string']],
     // Target-conditional fields (vendor removal's `name`, sink shape) stay with the apply-time guards.
@@ -203,7 +211,13 @@ function applyOperation(input, op, index) {
         case 'set-filter':
             return applySetFilter(input, op.phase, op.filter, index);
         case 'clear-filter':
-            return applyClearFilter(input, op.phase);
+            return applyClearFilter(input, op.phase, index);
+        case 'set-transform-chain':
+            return applySetTransformChain(input, op.phase, op.root, index);
+        case 'clear-transform-chain':
+            return applyClearTransformChain(input, op.phase, index);
+        case 'set-sql-transform':
+            return applySetSqlTransform(input, op.phase, op.sqlOperation, op.outputRouting, op.mainStream, op.gate, index);
         case 'add-source':
             return applyAddSource(input, op.source, index);
         case 'remove-source':
@@ -271,6 +285,13 @@ function applyJobGraphOperation(jobGraph, op, index) {
             return jobGraphSetFilter(jobGraph, op.phase, op.filter, index);
         case 'clear-filter':
             return jobGraphClearFilter(jobGraph, op.phase, index);
+        case 'set-sql-transform':
+            throw new Error(`Operation ${index} (set-sql-transform): SQL transform edits are only supported for ` +
+                `template-backed log-reducer jobs in the CLI. This job is a direct job graph. `);
+        case 'set-transform-chain':
+        case 'clear-transform-chain':
+            throw new Error(`Operation ${index} (${op.op}): transform-chain edits are only supported for ` +
+                `template-backed log-reducer jobs in the CLI. This job is a direct job graph. `);
         case 'add-source':
             return jobGraphAddSource(jobGraph, op.source, index);
         case 'remove-source':
@@ -849,9 +870,9 @@ function rawFilterNameForPhase(phase, index, opLabel) {
             return RAW_PRE_WAREHOUSE_FILTER;
         case 'pre-exceptions':
             return RAW_PRE_EXCEPTIONS_FILTER;
-        case 'pre-aggregation':
-            throw new Error(`Operation ${index} (${opLabel}): phase "pre-aggregation" has no canonical UI raw-graph stage. ` +
-                `Use "pre-warehouse" or "pre-exceptions" for raw UI log graphs.`);
+        default:
+            throw new Error(`Operation ${index} (${opLabel}): phase "${phase}" has no canonical UI raw-graph stage. ` +
+                ACCEPTED_TRANSFORM_PHASES_MESSAGE);
     }
 }
 function assertRawUiLogGraph(jobGraph, opLabel, index, requiredNames) {
@@ -1096,21 +1117,255 @@ function applyRemoveParser(input, name, index) {
     }
     input.parsers.splice(idx, 1);
 }
+/**
+ * Map a filter phase to its template `transforms` slot key. Only the three
+ * transform-backed phases are accepted; any other phase fails with the shared
+ * accepted-phases error.
+ */
+function phaseToTransformSlot(phase, index, opLabel) {
+    switch (phase) {
+        case 'pre-parser':
+            return 'preParser';
+        case 'pre-warehouse':
+            return 'preWarehouse';
+        case 'pre-exceptions':
+            return 'preExceptions';
+        default:
+            throw new Error(`Operation ${index} (${opLabel}): phase "${phase}" is not accepted. ${ACCEPTED_TRANSFORM_PHASES_MESSAGE}`);
+    }
+}
+function makePassthroughNode() {
+    return { kind: PassthroughNodeKind.passthrough_node };
+}
+function makeDropNode() {
+    return { kind: DropNodeKind.drop_node };
+}
+function makeMainStreamNode(mainStream) {
+    return mainStream === 'drop' ? makeDropNode() : makePassthroughNode();
+}
+function nodeKind(node) {
+    return node?.kind;
+}
+export function isDropNode(node) {
+    return nodeKind(node) === DropNodeKind.drop_node;
+}
+export function isPassthroughNode(node) {
+    return nodeKind(node) === PassthroughNodeKind.passthrough_node;
+}
+export function isSqlNode(node) {
+    return nodeKind(node) === SqlNodeKind.sql_node;
+}
+export function isConditionNode(node) {
+    return nodeKind(node) === ConditionNodeKind.condition_node;
+}
+/**
+ * Build a CLI-managed filter gate routing kept events into `passSide` and
+ * dropping the rest. A normal keep filter passes `thenAction`; an inverted
+ * filter swaps the branches. `maxLateEventTimestampDelta` becomes the gate's
+ * root `lateDataDuration`, matching the previous `set-filter` behavior.
+ */
+function makeFilterGate(filter, passSide) {
+    const drop = makeDropNode();
+    const gate = {
+        kind: ConditionNodeKind.condition_node,
+        predicate: filter.predicate,
+        thenAction: filter.inverted ? drop : passSide,
+        elseAction: filter.inverted ? passSide : drop,
+    };
+    if (filter.maxLateEventTimestampDelta !== undefined) {
+        gate.lateDataDuration = filter.maxLateEventTimestampDelta;
+    }
+    return gate;
+}
+/**
+ * Detect whether `root` is an unambiguous CLI-managed filter gate: a
+ * `ConditionNode` with exactly one terminal `DropNode` branch. Returns the pass
+ * side and inversion, or `undefined` when `root` is not a condition or has
+ * meaningful (non-drop) work on both branches — a chain the CLI must not guess
+ * at.
+ */
+function findSimpleRootFilterGate(root) {
+    if (!isConditionNode(root))
+        return undefined;
+    const cond = root;
+    const thenIsDrop = isDropNode(cond.thenAction);
+    const elseIsDrop = isDropNode(cond.elseAction);
+    // Exactly one branch must be a drop; both-drop or neither-drop is ambiguous.
+    if (thenIsDrop === elseIsDrop)
+        return undefined;
+    return {
+        passSide: elseIsDrop ? cond.thenAction : cond.elseAction,
+        inverted: !elseIsDrop,
+        lateDataDuration: cond.lateDataDuration,
+    };
+}
+/**
+ * Install or update a filter gate at the phase's transform slot:
+ * - empty slot: `IF filter THEN PASSTHROUGH ELSE DROP`
+ * - existing simple gate: rebuild with the new predicate, keeping the pass side
+ *   and inheriting `inverted`/`lateDataDuration` when the patch omits them (raw
+ *   merge semantics — a predicate-only edit must not flip keep↔drop)
+ * - any other existing chain (e.g. a SqlNode): `IF filter THEN <chain> ELSE DROP`
+ */
 function applySetFilter(input, phase, filter, index) {
     if (!filter || typeof filter !== 'object') {
         throw new Error(`Operation ${index} (set-filter): filter must be an object`);
     }
-    const filters = input.filters ?? {};
-    filters[phase] = { ...(filters[phase] ?? {}), ...filter };
-    input.filters = filters;
+    if (!isEventPredicate(filter.predicate)) {
+        throw new Error(`Operation ${index} (set-filter): filter.predicate must be an EventPredicate object with a "type"`);
+    }
+    const slot = phaseToTransformSlot(phase, index, 'set-filter');
+    const transforms = input.transforms ?? {};
+    const existing = transforms[slot];
+    const match = existing === undefined ? undefined : findSimpleRootFilterGate(existing);
+    const passSide = existing === undefined ? makePassthroughNode() : match?.passSide ?? existing;
+    const gateFilter = match
+        ? {
+            ...filter,
+            inverted: filter.inverted ?? match.inverted,
+            maxLateEventTimestampDelta: filter.maxLateEventTimestampDelta ?? match.lateDataDuration,
+        }
+        : filter;
+    transforms[slot] = makeFilterGate(gateFilter, passSide);
+    input.transforms = transforms;
 }
-function applyClearFilter(input, phase) {
-    const filters = input.filters ?? {};
-    const existing = filters[phase];
-    if (!existing || typeof existing !== 'object')
+/**
+ * Remove a root filter gate from the phase's transform slot, replacing the slot
+ * with the gate's pass side. No-ops on an empty slot. Fails clearly when the
+ * slot holds a chain that is not an unambiguous filter gate, rather than
+ * guessing which branch is the filter.
+ */
+function applyClearFilter(input, phase, index) {
+    const slot = phaseToTransformSlot(phase, index, 'clear-filter');
+    const transforms = input.transforms;
+    const existing = transforms?.[slot];
+    if (existing === undefined)
         return;
-    filters[phase] = { ...existing, predicate: { type: DatadogQueryPredicateType.datadog_query, query: '' } };
-    input.filters = filters;
+    const match = findSimpleRootFilterGate(existing);
+    if (!match) {
+        const detail = isConditionNode(existing)
+            ? 'contains a branching transform chain, not a simple filter gate'
+            : `contains a ${nodeKind(existing) ?? 'non-condition'} at its root, not a filter gate`;
+        throw new Error(`Operation ${index} (clear-filter): template transform slot "${slot}" ${detail}. ` +
+            `Use set-transform-chain to edit it explicitly.`);
+    }
+    transforms[slot] = match.passSide;
+    input.transforms = transforms;
+}
+/** True when `value` looks like an `EventPredicate` (an object carrying a string `type` discriminator). */
+function isEventPredicate(value) {
+    return isRecord(value) && typeof value['type'] === 'string';
+}
+/**
+ * Replace the phase's transform slot with a hand-authored chain root. The CLI
+ * does not introspect or validate the tree beyond requiring an object; the
+ * template compiler remains the final authority.
+ */
+function applySetTransformChain(input, phase, root, index) {
+    if (!isRecord(root)) {
+        throw new Error(`Operation ${index} (set-transform-chain): root must be a chain-node object`);
+    }
+    const slot = phaseToTransformSlot(phase, index, 'set-transform-chain');
+    const transforms = input.transforms ?? {};
+    transforms[slot] = root;
+    input.transforms = transforms;
+}
+/** Remove a phase's entire transform slot (distinct from `clear-filter`, which removes only a filter gate). */
+function applyClearTransformChain(input, phase, index) {
+    const slot = phaseToTransformSlot(phase, index, 'clear-transform-chain');
+    if (input.transforms) {
+        Reflect.deleteProperty(input.transforms, slot);
+    }
+}
+/**
+ * Replace the phase's transform slot with a generated `SqlNode` chain. Any
+ * existing chain is replaced — callers needing to preserve or merge a chain
+ * must use `set-transform-chain`.
+ */
+function applySetSqlTransform(input, phase, sqlOperation, outputRouting, mainStream, gate, index) {
+    const slot = phaseToTransformSlot(phase, index, 'set-sql-transform');
+    validateSqlTransform(sqlOperation, outputRouting, mainStream, gate, index);
+    const sqlChain = {
+        kind: SqlNodeKind.sql_node,
+        sqlOperation,
+        outputRouting,
+        next: makeMainStreamNode(mainStream),
+    };
+    // An ungated drop (`mainStream: 'drop'`, no gate) still needs a condition node
+    // wrapper with a match-all (`*`) predicate. The `elseAction` passthrough is
+    // unreachable under `*`, but it is load-bearing, NOT dead code: it is the SQL
+    // node's connected main output edge. A bare `sql-node` whose `next` is a drop
+    // compiles to a disconnected `chainOut` in the Flink graph (see ENGT-4852 fix);
+    // the passthrough else-branch is what keeps that main output connected. Do not
+    // "simplify" it away — `test_setSqlTransform_noGate_dropMainStream_buildsMatchAllConditionNode`
+    // pins this shape.
+    const effectiveGate = gate ?? { type: DatadogQueryPredicateType.datadog_query, query: '*' };
+    const root = mainStream === 'passthrough' && gate === undefined
+        ? sqlChain
+        : {
+            kind: ConditionNodeKind.condition_node,
+            predicate: effectiveGate,
+            thenAction: sqlChain,
+            elseAction: makePassthroughNode(),
+        };
+    const transforms = input.transforms ?? {};
+    transforms[slot] = root;
+    input.transforms = transforms;
+}
+/**
+ * Validate a `set-sql-transform` op before mutating the input, so malformed
+ * SQL fails with a readable CLI error ahead of `job:draft`.
+ */
+function validateSqlTransform(sqlOperation, outputRouting, mainStream, gate, index) {
+    const fail = (detail) => {
+        throw new Error(`Operation ${index} (set-sql-transform): ${detail}`);
+    };
+    if (!isRecord(sqlOperation) || sqlOperation['type'] !== SqlOperationType.sql_operation) {
+        fail(`sqlOperation must be an object with type "${SqlOperationType.sql_operation}"`);
+    }
+    const statements = sqlOperation.statements;
+    if (!Array.isArray(statements) || statements.length === 0) {
+        fail('sqlOperation.statements must be a non-empty array');
+    }
+    const outputNames = statements
+        .filter(s => s.type === SqlOutputStatementType.sql_output)
+        .map(s => s.outputName);
+    if (outputNames.length === 0) {
+        fail('sqlOperation.statements must include at least one sql_output statement');
+    }
+    const seen = new Set();
+    for (const name of outputNames) {
+        if (typeof name !== 'string' || name.length === 0) {
+            fail('every sql_output statement must have a non-empty outputName');
+        }
+        if (seen.has(name)) {
+            fail(`duplicate sql_output outputName "${name}"; output names must be unique`);
+        }
+        seen.add(name);
+    }
+    if (!isRecord(outputRouting)) {
+        fail('outputRouting must be an object');
+    }
+    if (typeof mainStream !== 'string' || !SQL_TRANSFORM_MAIN_STREAMS.has(mainStream)) {
+        fail(`mainStream must be one of: ${[...SQL_TRANSFORM_MAIN_STREAMS].join(', ')}`);
+    }
+    for (const [key, target] of Object.entries(outputRouting)) {
+        if (!seen.has(key)) {
+            fail(`outputRouting key "${key}" does not match any sql_output outputName`);
+        }
+        if (typeof target !== 'string' || !SQL_OUTPUT_ROUTING_TARGETS.has(target)) {
+            fail(`outputRouting["${key}"] = ${JSON.stringify(target)} is not a valid target step. ` +
+                `Valid targets: ${[...SQL_OUTPUT_ROUTING_TARGETS].join(', ')}.`);
+        }
+    }
+    for (const name of seen) {
+        if (!Object.prototype.hasOwnProperty.call(outputRouting, name)) {
+            fail(`sql_output "${name}" has no route in outputRouting`);
+        }
+    }
+    if (gate !== undefined && !isEventPredicate(gate)) {
+        fail('gate must be an EventPredicate object with a "type"');
+    }
 }
 function applyAddSource(input, source, index) {
     assertOperationIdentity(source, 'add-source', index, 'source');
@@ -1343,12 +1598,12 @@ const INPUT_FIELD_TOUCHES = {
     rawSinkConfig: 'sink',
     datasetId: 'sink',
     conditionalDatasets: 'sink',
+    confirmedDeliveryEnabled: 'sink',
     reducer: 'transform',
     parsers: 'transform',
-    filters: 'transform',
+    transforms: 'transform',
     exceptions: 'transform',
     sampler: 'transform',
-    sqlOperations: 'transform',
 };
 /**
  * Classify a generic `set/unset-input-field` path by its top-level field.