@grepr/cli 1.14.13-15e1f25 → 1.15.0-48d68ee
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/dist/commands/job-draft-command.d.ts.map +1 -1
- package/build/dist/commands/job-draft-command.js +6 -71
- package/build/dist/commands/job-draft-command.js.map +1 -1
- package/build/dist/docs-index/index.json +1 -1
- package/build/dist/grepr.d.ts.map +1 -1
- package/build/dist/grepr.js +0 -2
- package/build/dist/grepr.js.map +1 -1
- package/build/dist/lib/backfill.js +4 -8
- package/build/dist/lib/backfill.js.map +1 -1
- package/build/dist/lib/job-patch.d.ts +15 -31
- package/build/dist/lib/job-patch.d.ts.map +1 -1
- package/build/dist/lib/job-patch.js +39 -709
- package/build/dist/lib/job-patch.js.map +1 -1
- package/build/dist/lib/job-plan.d.ts +3 -6
- package/build/dist/lib/job-plan.d.ts.map +1 -1
- package/build/dist/lib/job-plan.js +5 -55
- package/build/dist/lib/job-plan.js.map +1 -1
- package/build/dist/lib/option-parsers.d.ts +0 -5
- package/build/dist/lib/option-parsers.d.ts.map +1 -1
- package/build/dist/lib/option-parsers.js +0 -7
- package/build/dist/lib/option-parsers.js.map +1 -1
- package/build/dist/lib/signal-source.d.ts +1 -1
- package/build/dist/lib/signal-source.d.ts.map +1 -1
- package/build/dist/lib/signal-source.js +10 -69
- package/build/dist/lib/signal-source.js.map +1 -1
- package/build/dist/openapi/openApiTypes.d.ts +50 -7
- package/build/dist/openapi/openApiTypes.d.ts.map +1 -1
- package/build/dist/openapi/openApiTypes.js +5 -0
- package/build/dist/openapi/openApiTypes.js.map +1 -1
- package/build/dist/openapi-docs/schemas/AgentConfigCreate.md +3 -0
- package/build/dist/openapi-docs/schemas/AgentConfigPayload.md +3 -0
- package/build/dist/openapi-docs/schemas/AgentConfigUpdate.md +3 -0
- package/build/dist/openapi-docs/schemas/AgentDecisionConfiguration.md +19 -0
- package/build/dist/openapi-docs/schemas/AthenaAgentSessionsSource.md +1 -1
- package/build/dist/openapi-docs/schemas/AthenaAnalyticsSource.md +1 -1
- package/build/dist/openapi-docs/schemas/ChangeFinding.md +1 -1
- package/build/dist/openapi-docs/schemas/DecisionOutcome.md +11 -0
- package/build/dist/openapi-docs/schemas/InvestigationSummary.md +1 -1
- package/build/dist/openapi-docs/schemas/InvestigationTranscript.md +4 -1
- package/build/dist/openapi-docs/schemas/IssueRun.md +1 -1
- package/build/dist/openapi-docs/schemas/PipelineChangeRequest.md +4 -0
- package/build/dist/openapi-docs/schemas/SignalDecision.md +14 -0
- package/build/dist/openapi-docs/schemas/TrinoAnalyticsSource.md +1 -1
- package/build/dist/openapi-docs/schemas/TrinoRawLogsSource.md +1 -1
- package/build/dist/openapi-docs/schemas/TrinoRawSpanSource.md +1 -1
- package/build/dist/openapi-docs/schemas/TrinoReducerLogSource.md +1 -1
- package/build/dist/types.d.ts +8 -37
- package/build/dist/types.d.ts.map +1 -1
- package/build/dist/types.js +0 -69
- package/build/dist/types.js.map +1 -1
- package/package.json +1 -1
- package/build/dist/commands/job-to-test-command.d.ts +0 -74
- package/build/dist/commands/job-to-test-command.d.ts.map +0 -1
- package/build/dist/commands/job-to-test-command.js +0 -161
- package/build/dist/commands/job-to-test-command.js.map +0 -1
- package/build/dist/lib/job-graph-log-pipeline-constants.d.ts +0 -12
- package/build/dist/lib/job-graph-log-pipeline-constants.d.ts.map +0 -1
- package/build/dist/lib/job-graph-log-pipeline-constants.js +0 -16
- package/build/dist/lib/job-graph-log-pipeline-constants.js.map +0 -1
- package/build/dist/lib/job-graph-transformer.d.ts +0 -119
- package/build/dist/lib/job-graph-transformer.d.ts.map +0 -1
- package/build/dist/lib/job-graph-transformer.js +0 -767
- package/build/dist/lib/job-graph-transformer.js.map +0 -1
- package/build/dist/lib/job-graph-utils.d.ts +0 -50
- package/build/dist/lib/job-graph-utils.d.ts.map +0 -1
- package/build/dist/lib/job-graph-utils.js +0 -90
- package/build/dist/lib/job-graph-utils.js.map +0 -1
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* CLI command for transforming production jobs into test configurations.
|
|
3
|
-
*
|
|
4
|
-
* The job:to-test command takes a production job definition and transforms it
|
|
5
|
-
* for testing purposes. This is a pure transformation - no API calls are made.
|
|
6
|
-
*
|
|
7
|
-
* Key features:
|
|
8
|
-
* - Replace sources with dataset queries or add record limits
|
|
9
|
-
* - Replace sinks with synchronous test sinks or test dataset sinks
|
|
10
|
-
* - Add test tagging to track test data
|
|
11
|
-
* - Show diff of transformations
|
|
12
|
-
* - Output to file or stdout
|
|
13
|
-
*
|
|
14
|
-
* Example usage:
|
|
15
|
-
* grepr job:to-test prod.json --execution SYNCHRONOUS --output test.json
|
|
16
|
-
* grepr job:to-test prod.json --dataset-id my_dataset --start 2025-01-01T00:00:00Z \
|
|
17
|
-
* --end 2025-01-01T01:00:00Z --show-diff
|
|
18
|
-
*/
|
|
19
|
-
import { Command } from 'commander';
|
|
20
|
-
import { ICommand } from '../lib/command-registry.js';
|
|
21
|
-
import { MergeConfiguration } from '../types.js';
|
|
22
|
-
/**
|
|
23
|
-
* Command implementation for job:to-test.
|
|
24
|
-
*
|
|
25
|
-
* This command registers itself with the CLI and handles:
|
|
26
|
-
* - Option parsing and validation
|
|
27
|
-
* - Job file loading
|
|
28
|
-
* - Transformation execution
|
|
29
|
-
* - Output formatting and writing
|
|
30
|
-
*/
|
|
31
|
-
export declare class JobToTestCommand implements ICommand {
|
|
32
|
-
/**
|
|
33
|
-
* Registers the job:to-test command with the CLI program.
|
|
34
|
-
*
|
|
35
|
-
* @param program - The commander program to add the command to
|
|
36
|
-
* @param _mergeConfiguration - Configuration merger (unused by this command)
|
|
37
|
-
*/
|
|
38
|
-
addToProgram(program: Command, _mergeConfiguration: MergeConfiguration): void;
|
|
39
|
-
/**
|
|
40
|
-
* Validates command options for consistency and completeness.
|
|
41
|
-
*
|
|
42
|
-
* Validation rules:
|
|
43
|
-
* - Cannot use both --sample-data-file and --dataset-id (mutually exclusive)
|
|
44
|
-
* - --dataset-id requires --start and --end
|
|
45
|
-
* - --execution must be SYNCHRONOUS or ASYNCHRONOUS
|
|
46
|
-
* - --processing must be BATCH or STREAMING
|
|
47
|
-
*
|
|
48
|
-
* @param options - The command options to validate
|
|
49
|
-
* @throws Error if validation fails
|
|
50
|
-
*/
|
|
51
|
-
private validateOptions;
|
|
52
|
-
/**
|
|
53
|
-
* Loads and validates a job definition from a JSON file.
|
|
54
|
-
*
|
|
55
|
-
* The file must exist and contain valid JSON with required fields:
|
|
56
|
-
* - name: Job name
|
|
57
|
-
* - execution: Execution type
|
|
58
|
-
* - processing: Processing type
|
|
59
|
-
*
|
|
60
|
-
* @param jobFile - Path to the job definition file
|
|
61
|
-
* @returns The loaded job definition
|
|
62
|
-
* @throws Error if file not found, invalid JSON, or missing required fields
|
|
63
|
-
*/
|
|
64
|
-
private loadJobFromFile;
|
|
65
|
-
/**
|
|
66
|
-
* Formats the transformed job as JSON string.
|
|
67
|
-
*
|
|
68
|
-
* @param transformedJob - The job to format
|
|
69
|
-
* @param pretty - Whether to pretty-print (2-space indentation) or compact
|
|
70
|
-
* @returns JSON string representation of the job
|
|
71
|
-
*/
|
|
72
|
-
private formatOutput;
|
|
73
|
-
}
|
|
74
|
-
//# sourceMappingURL=job-to-test-command.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"job-to-test-command.d.ts","sourceRoot":"","sources":["../../../src/main/typescript/commands/job-to-test-command.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAElD,OAAO,EAA+B,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAI1E;;;;;;;;GAQG;AACH,qBAAa,gBAAiB,YAAW,QAAQ;IAC/C;;;;;OAKG;IACH,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,GAAG,IAAI;IAoE7E;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,eAAe;IAkBvB;;;;;;;;;;;OAWG;YACW,eAAe;IAmB7B;;;;;;OAMG;IACH,OAAO,CAAC,YAAY;CAMrB"}
|
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* CLI command for transforming production jobs into test configurations.
|
|
3
|
-
*
|
|
4
|
-
* The job:to-test command takes a production job definition and transforms it
|
|
5
|
-
* for testing purposes. This is a pure transformation - no API calls are made.
|
|
6
|
-
*
|
|
7
|
-
* Key features:
|
|
8
|
-
* - Replace sources with dataset queries or add record limits
|
|
9
|
-
* - Replace sinks with synchronous test sinks or test dataset sinks
|
|
10
|
-
* - Add test tagging to track test data
|
|
11
|
-
* - Show diff of transformations
|
|
12
|
-
* - Output to file or stdout
|
|
13
|
-
*
|
|
14
|
-
* Example usage:
|
|
15
|
-
* grepr job:to-test prod.json --execution SYNCHRONOUS --output test.json
|
|
16
|
-
* grepr job:to-test prod.json --dataset-id my_dataset --start 2025-01-01T00:00:00Z \
|
|
17
|
-
* --end 2025-01-01T01:00:00Z --show-diff
|
|
18
|
-
*/
|
|
19
|
-
import fs from 'fs-extra';
|
|
20
|
-
import { parseIntArg } from '../lib/option-parsers.js';
|
|
21
|
-
import { JobExecution, JobProcessing } from '../types.js';
|
|
22
|
-
import { transformJobToTest, showDiff } from '../lib/job-graph-transformer.js';
|
|
23
|
-
/**
|
|
24
|
-
* Command implementation for job:to-test.
|
|
25
|
-
*
|
|
26
|
-
* This command registers itself with the CLI and handles:
|
|
27
|
-
* - Option parsing and validation
|
|
28
|
-
* - Job file loading
|
|
29
|
-
* - Transformation execution
|
|
30
|
-
* - Output formatting and writing
|
|
31
|
-
*/
|
|
32
|
-
export class JobToTestCommand {
|
|
33
|
-
/**
|
|
34
|
-
* Registers the job:to-test command with the CLI program.
|
|
35
|
-
*
|
|
36
|
-
* @param program - The commander program to add the command to
|
|
37
|
-
* @param _mergeConfiguration - Configuration merger (unused by this command)
|
|
38
|
-
*/
|
|
39
|
-
addToProgram(program, _mergeConfiguration) {
|
|
40
|
-
program
|
|
41
|
-
.command('job:to-test <input-file>')
|
|
42
|
-
.description('Transform a job configuration for testing')
|
|
43
|
-
// Output options (note: -o, --output is registered as a global flag on the program)
|
|
44
|
-
.option('--no-pretty', 'Disable pretty printing')
|
|
45
|
-
.option('--show-diff', 'Show transformation diff')
|
|
46
|
-
// Execution and processing type overrides
|
|
47
|
-
.option('--execution <type>', 'Execution type (SYNCHRONOUS|ASYNCHRONOUS)')
|
|
48
|
-
.option('--processing <type>', 'Processing type (BATCH|STREAMING)')
|
|
49
|
-
// Source options (mutually exclusive: sample data OR dataset)
|
|
50
|
-
.option('--sample-data-file <file>', 'Sample logs file')
|
|
51
|
-
.option('--dataset-id <id>', 'Dataset ID for source (requires --start and --end)')
|
|
52
|
-
.option('--query <query>', 'Datadog syntax query for dataset source (optional, requires --dataset-id)')
|
|
53
|
-
.option('--start <time>', 'Start time (ISO 8601)')
|
|
54
|
-
.option('--end <time>', 'End time (ISO 8601)')
|
|
55
|
-
.option('--limit-records <n>', 'Limit for batch sources', parseIntArg, 1000)
|
|
56
|
-
// Sink options
|
|
57
|
-
.option('--test-dataset <id>', 'Test dataset ID for async sinks (enables tagging)')
|
|
58
|
-
// Test metadata
|
|
59
|
-
.option('--test-tag <tag>', 'Custom test tag (default: auto-generated)')
|
|
60
|
-
.option('--test-name <name>', 'Custom test job name (default: {original}_test)')
|
|
61
|
-
.action(async (inputFile, options, command) => {
|
|
62
|
-
try {
|
|
63
|
-
// Merge global options (-o, --output, --quiet, etc.) with sub-command options.
|
|
64
|
-
// Same pattern used by ListCommand/CrudCommand for consistent flag handling.
|
|
65
|
-
const merged = { ...command.parent?.opts(), ...options };
|
|
66
|
-
// Step 1: Validate that options are valid and compatible
|
|
67
|
-
this.validateOptions(merged);
|
|
68
|
-
// Step 2: Load the original job definition from file
|
|
69
|
-
const originalJob = await this.loadJobFromFile(inputFile);
|
|
70
|
-
// Step 3: Transform the job to test configuration
|
|
71
|
-
const transformedJob = transformJobToTest(originalJob, merged);
|
|
72
|
-
// Step 4: Show diff if requested (before outputting JSON)
|
|
73
|
-
if (merged.showDiff) {
|
|
74
|
-
showDiff(originalJob, transformedJob);
|
|
75
|
-
}
|
|
76
|
-
// Step 5: Format the output (pretty or compact JSON)
|
|
77
|
-
const output = this.formatOutput(transformedJob, merged.pretty !== false);
|
|
78
|
-
// Step 6: Write to file or stdout
|
|
79
|
-
if (merged.output) {
|
|
80
|
-
await fs.writeFile(merged.output, output);
|
|
81
|
-
console.log(`✓ Test job written to ${merged.output}`);
|
|
82
|
-
}
|
|
83
|
-
else {
|
|
84
|
-
console.log(output);
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
catch (error) {
|
|
88
|
-
console.error('Error transforming job:', error.message);
|
|
89
|
-
process.exit(1);
|
|
90
|
-
}
|
|
91
|
-
});
|
|
92
|
-
}
|
|
93
|
-
/**
|
|
94
|
-
* Validates command options for consistency and completeness.
|
|
95
|
-
*
|
|
96
|
-
* Validation rules:
|
|
97
|
-
* - Cannot use both --sample-data-file and --dataset-id (mutually exclusive)
|
|
98
|
-
* - --dataset-id requires --start and --end
|
|
99
|
-
* - --execution must be SYNCHRONOUS or ASYNCHRONOUS
|
|
100
|
-
* - --processing must be BATCH or STREAMING
|
|
101
|
-
*
|
|
102
|
-
* @param options - The command options to validate
|
|
103
|
-
* @throws Error if validation fails
|
|
104
|
-
*/
|
|
105
|
-
validateOptions(options) {
|
|
106
|
-
if (options.sampleDataFile && options.datasetId) {
|
|
107
|
-
throw new Error('Cannot use both --sample-data-file and --dataset-id');
|
|
108
|
-
}
|
|
109
|
-
if (options.datasetId && (!options.start || !options.end)) {
|
|
110
|
-
throw new Error('--dataset-id requires --start and --end');
|
|
111
|
-
}
|
|
112
|
-
if (options.execution && ![JobExecution.ASYNCHRONOUS, JobExecution.SYNCHRONOUS].includes(options.execution)) {
|
|
113
|
-
throw new Error('--execution must be SYNCHRONOUS or ASYNCHRONOUS');
|
|
114
|
-
}
|
|
115
|
-
if (options.processing && ![JobProcessing.BATCH, JobProcessing.STREAMING].includes(options.processing)) {
|
|
116
|
-
throw new Error('--processing must be BATCH or STREAMING');
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
/**
|
|
120
|
-
* Loads and validates a job definition from a JSON file.
|
|
121
|
-
*
|
|
122
|
-
* The file must exist and contain valid JSON with required fields:
|
|
123
|
-
* - name: Job name
|
|
124
|
-
* - execution: Execution type
|
|
125
|
-
* - processing: Processing type
|
|
126
|
-
*
|
|
127
|
-
* @param jobFile - Path to the job definition file
|
|
128
|
-
* @returns The loaded job definition
|
|
129
|
-
* @throws Error if file not found, invalid JSON, or missing required fields
|
|
130
|
-
*/
|
|
131
|
-
async loadJobFromFile(jobFile) {
|
|
132
|
-
if (!await fs.pathExists(jobFile)) {
|
|
133
|
-
throw new Error(`Job definition file not found: ${jobFile}`);
|
|
134
|
-
}
|
|
135
|
-
let jobDefinition;
|
|
136
|
-
try {
|
|
137
|
-
jobDefinition = await fs.readJson(jobFile);
|
|
138
|
-
}
|
|
139
|
-
catch (error) {
|
|
140
|
-
throw new Error(`Failed to load job definition: ${error.message}`);
|
|
141
|
-
}
|
|
142
|
-
if (!jobDefinition.name || !jobDefinition.execution || !jobDefinition.processing) {
|
|
143
|
-
throw new Error('Invalid job definition: missing required fields (name, execution, processing)');
|
|
144
|
-
}
|
|
145
|
-
return jobDefinition;
|
|
146
|
-
}
|
|
147
|
-
/**
|
|
148
|
-
* Formats the transformed job as JSON string.
|
|
149
|
-
*
|
|
150
|
-
* @param transformedJob - The job to format
|
|
151
|
-
* @param pretty - Whether to pretty-print (2-space indentation) or compact
|
|
152
|
-
* @returns JSON string representation of the job
|
|
153
|
-
*/
|
|
154
|
-
formatOutput(transformedJob, pretty) {
|
|
155
|
-
if (pretty) {
|
|
156
|
-
return JSON.stringify(transformedJob, null, 2);
|
|
157
|
-
}
|
|
158
|
-
return JSON.stringify(transformedJob);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
//# sourceMappingURL=job-to-test-command.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"job-to-test-command.js","sourceRoot":"","sources":["../../../src/main/typescript/commands/job-to-test-command.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,MAAM,UAAU,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAsB,MAAM,SAAS,CAAC;AAE1E,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAoB,MAAM,6BAA6B,CAAC;AAE7F;;;;;;;;GAQG;AACH,MAAM,OAAO,gBAAgB;IAC3B;;;;;OAKG;IACH,YAAY,CAAC,OAAgB,EAAE,mBAAuC;QACpE,OAAO;aACJ,OAAO,CAAC,0BAA0B,CAAC;aACnC,WAAW,CAAC,2CAA2C,CAAC;YACzD,oFAAoF;aACnF,MAAM,CAAC,aAAa,EAAE,yBAAyB,CAAC;aAChD,MAAM,CAAC,aAAa,EAAE,0BAA0B,CAAC;YAElD,0CAA0C;aACzC,MAAM,CAAC,oBAAoB,EAAE,2CAA2C,CAAC;aACzE,MAAM,CAAC,qBAAqB,EAAE,mCAAmC,CAAC;YAEnE,8DAA8D;aAC7D,MAAM,CAAC,2BAA2B,EAAE,kBAAkB,CAAC;aACvD,MAAM,CAAC,mBAAmB,EAAE,oDAAoD,CAAC;aACjF,MAAM,CAAC,iBAAiB,EAAE,2EAA2E,CAAC;aACtG,MAAM,CAAC,gBAAgB,EAAE,uBAAuB,CAAC;aACjD,MAAM,CAAC,cAAc,EAAE,qBAAqB,CAAC;aAC7C,MAAM,CAAC,qBAAqB,EAAE,yBAAyB,EAAE,WAAW,EAAE,IAAI,CAAC;YAE5E,eAAe;aACd,MAAM,CAAC,qBAAqB,EAAE,mDAAmD,CAAC;YAEnF,gBAAgB;aACf,MAAM,CAAC,kBAAkB,EAAE,2CAA2C,CAAC;aACvE,MAAM,CAAC,oBAAoB,EAAE,iDAAiD,CAAC;aAC/E,MAAM,CAAC,KAAK,EACX,SAAiB,EACjB,OAAqF,EACrF,OAAgB,EAChB,EAAE;YACF,IAAI,CAAC;gBACH,+EAA+E;gBAC/E,6EAA6E;gBAC7E,MAAM,MAAM,GAAG,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC;gBAEzD,yDAAyD;gBACzD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAE7B,qDAAqD;gBACrD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;gBAE1D,kDAAkD;gBAClD,MAAM,cAAc,GAAG,kBAAkB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;gBAE/D,0DAA0D;gBAC1D,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACpB,QAAQ,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBACxC,CAAC;gBAED,qDAAqD;gBACrD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC;gBAE1E,kCAAkC;gBAClC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;oBAClB,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;oBAC1C,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;gBACxD,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;YAEH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAG,KAAe,CAAC,OAAO,CAAC,CAAC;gBACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;OAWG;IACK,eAAe,CAAC,OAAyB;QAC/C,IAAI,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,YAAY,CAAC,YAAY,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5G,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACrE,CAAC;QAED,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YACvG,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED;;;;;;;;;;;OAWG;IACK,KAAK,CAAC,eAAe,CAAC,OAAe;QAC3C,IAAI,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,aAAa,CAAA;QACjB,IAAI,CAAC;YACH,aAAa,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,kCAAmC,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,SAAS,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;YACjF,MAAM,IAAI,KAAK,CAAC,+EAA+E,CAAC,CAAC;QACnG,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACK,YAAY,CAAC,cAA+B,EAAE,MAAe;QACnE,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACxC,CAAC;CACF"}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import { GrokParserType, JsonLogProcessorType, LogAttributesRemapperType } from '../openapi/openApiTypes.js';
|
|
2
|
-
export declare const RAW_PRE_PARSER_FILTER = "pre_parser_filter";
|
|
3
|
-
export declare const RAW_PRE_WAREHOUSE_FILTER = "pre_data_warehouse_filter";
|
|
4
|
-
export declare const RAW_PRE_EXCEPTIONS_FILTER = "pre_exceptions_filter";
|
|
5
|
-
export declare const RAW_JSON_PROCESSOR = "json_log_processor";
|
|
6
|
-
export declare const RAW_ATTRIBUTES_REMAPPER = "log_attributes_remapper";
|
|
7
|
-
export declare const RAW_LOG_REDUCER = "log_reducer";
|
|
8
|
-
export declare const RAW_JSON_PROCESSOR_TYPE = JsonLogProcessorType.json_log_processor;
|
|
9
|
-
export declare const RAW_ATTRIBUTES_REMAPPER_TYPE = LogAttributesRemapperType.log_attributes_remapper;
|
|
10
|
-
export declare const RAW_GROK_PARSER_TYPE = GrokParserType.grok_parser;
|
|
11
|
-
export declare const RAW_PARSER_TYPES: ReadonlySet<string>;
|
|
12
|
-
//# sourceMappingURL=job-graph-log-pipeline-constants.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"job-graph-log-pipeline-constants.d.ts","sourceRoot":"","sources":["../../../src/main/typescript/lib/job-graph-log-pipeline-constants.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,oBAAoB,EACpB,yBAAyB,EAC1B,MAAM,wBAAwB,CAAC;AAEhC,eAAO,MAAM,qBAAqB,sBAAsB,CAAC;AACzD,eAAO,MAAM,wBAAwB,8BAA8B,CAAC;AACpE,eAAO,MAAM,yBAAyB,0BAA0B,CAAC;AACjE,eAAO,MAAM,kBAAkB,uBAAuB,CAAC;AACvD,eAAO,MAAM,uBAAuB,4BAA4B,CAAC;AACjE,eAAO,MAAM,eAAe,gBAAgB,CAAC;AAC7C,eAAO,MAAM,uBAAuB,0CAA0C,CAAC;AAC/E,eAAO,MAAM,4BAA4B,oDAAoD,CAAC;AAC9F,eAAO,MAAM,oBAAoB,6BAA6B,CAAC;AAE/D,eAAO,MAAM,gBAAgB,EAAE,WAAW,CAAC,MAAM,CAI/C,CAAC"}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { GrokParserType, JsonLogProcessorType, LogAttributesRemapperType, } from '../openapi/openApiTypes.js';
|
|
2
|
-
export const RAW_PRE_PARSER_FILTER = 'pre_parser_filter';
|
|
3
|
-
export const RAW_PRE_WAREHOUSE_FILTER = 'pre_data_warehouse_filter';
|
|
4
|
-
export const RAW_PRE_EXCEPTIONS_FILTER = 'pre_exceptions_filter';
|
|
5
|
-
export const RAW_JSON_PROCESSOR = 'json_log_processor';
|
|
6
|
-
export const RAW_ATTRIBUTES_REMAPPER = 'log_attributes_remapper';
|
|
7
|
-
export const RAW_LOG_REDUCER = 'log_reducer';
|
|
8
|
-
export const RAW_JSON_PROCESSOR_TYPE = JsonLogProcessorType.json_log_processor;
|
|
9
|
-
export const RAW_ATTRIBUTES_REMAPPER_TYPE = LogAttributesRemapperType.log_attributes_remapper;
|
|
10
|
-
export const RAW_GROK_PARSER_TYPE = GrokParserType.grok_parser;
|
|
11
|
-
export const RAW_PARSER_TYPES = new Set([
|
|
12
|
-
RAW_JSON_PROCESSOR_TYPE,
|
|
13
|
-
RAW_ATTRIBUTES_REMAPPER_TYPE,
|
|
14
|
-
RAW_GROK_PARSER_TYPE,
|
|
15
|
-
]);
|
|
16
|
-
//# sourceMappingURL=job-graph-log-pipeline-constants.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"job-graph-log-pipeline-constants.js","sourceRoot":"","sources":["../../../src/main/typescript/lib/job-graph-log-pipeline-constants.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,oBAAoB,EACpB,yBAAyB,GAC1B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,CAAC,MAAM,qBAAqB,GAAG,mBAAmB,CAAC;AACzD,MAAM,CAAC,MAAM,wBAAwB,GAAG,2BAA2B,CAAC;AACpE,MAAM,CAAC,MAAM,yBAAyB,GAAG,uBAAuB,CAAC;AACjE,MAAM,CAAC,MAAM,kBAAkB,GAAG,oBAAoB,CAAC;AACvD,MAAM,CAAC,MAAM,uBAAuB,GAAG,yBAAyB,CAAC;AACjE,MAAM,CAAC,MAAM,eAAe,GAAG,aAAa,CAAC;AAC7C,MAAM,CAAC,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,kBAAkB,CAAC;AAC/E,MAAM,CAAC,MAAM,4BAA4B,GAAG,yBAAyB,CAAC,uBAAuB,CAAC;AAC9F,MAAM,CAAC,MAAM,oBAAoB,GAAG,cAAc,CAAC,WAAW,CAAC;AAE/D,MAAM,CAAC,MAAM,gBAAgB,GAAwB,IAAI,GAAG,CAAC;IAC3D,uBAAuB;IACvB,4BAA4B;IAC5B,oBAAoB;CACrB,CAAC,CAAC"}
|
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Job graph transformation utilities for converting production jobs to test configurations.
|
|
3
|
-
*
|
|
4
|
-
* This module provides functions to transform a production job definition into a test job
|
|
5
|
-
* by replacing sources, sinks, and adding test-specific tagging. The transformations are
|
|
6
|
-
* purely functional - they do not make any API calls or create resources.
|
|
7
|
-
*
|
|
8
|
-
* Key transformations:
|
|
9
|
-
* - Source replacement: Replace vendor sources with dataset queries or keep originals with limits
|
|
10
|
-
* - Sink replacement: Replace vendor sinks with synchronous test sinks or test dataset sinks
|
|
11
|
-
* - Test tagging: Add test run ID and sink name tags to track test data
|
|
12
|
-
*
|
|
13
|
-
* The transformed job can then be executed using the standard job:create command.
|
|
14
|
-
*/
|
|
15
|
-
import { JobExecution, JobProcessing } from '../types.js';
|
|
16
|
-
import { SchemaCreateJob } from '../openapi/openApiTypes.js';
|
|
17
|
-
/**
|
|
18
|
-
* Options for transforming a job to a test configuration.
|
|
19
|
-
*/
|
|
20
|
-
export interface JobToTestOptions {
|
|
21
|
-
/** Target execution type (default: keep original) */
|
|
22
|
-
execution?: JobExecution;
|
|
23
|
-
/** Target processing type (default: keep original) */
|
|
24
|
-
processing?: JobProcessing;
|
|
25
|
-
/** Path to sample data file (not yet implemented) */
|
|
26
|
-
sampleDataFile?: string;
|
|
27
|
-
/** Dataset ID to use as data source (requires start and end) */
|
|
28
|
-
datasetId?: string;
|
|
29
|
-
/** Query to filter dataset source data (optional, requires datasetId) */
|
|
30
|
-
query?: string;
|
|
31
|
-
/** Start time for dataset query (ISO 8601 format) */
|
|
32
|
-
start?: string;
|
|
33
|
-
/** End time for dataset query (ISO 8601 format) */
|
|
34
|
-
end?: string;
|
|
35
|
-
/** Record limit for batch sources (default: 1000) */
|
|
36
|
-
limitRecords?: number;
|
|
37
|
-
/** Dataset ID for async test output (enables tagging) */
|
|
38
|
-
testDataset?: string;
|
|
39
|
-
/** Custom test run identifier (default: auto-generated) */
|
|
40
|
-
testTag?: string;
|
|
41
|
-
/** Custom test job name (default: {original}_test) */
|
|
42
|
-
testName?: string;
|
|
43
|
-
/**
|
|
44
|
-
* Tuning for the logs-event-sampler inserted after each live source in a
|
|
45
|
-
* source-preserving draft. Omitted fields fall back to the UI's draft
|
|
46
|
-
* defaults (see {@link DEFAULT_DRAFT_SAMPLER_BURST}).
|
|
47
|
-
*/
|
|
48
|
-
sampler?: DraftSamplerOptions;
|
|
49
|
-
}
|
|
50
|
-
/** Sampler knobs for a source-preserving draft, mirroring the UI's logs-event-sampler. */
|
|
51
|
-
export interface DraftSamplerOptions {
|
|
52
|
-
/** `maxAllowedRate` in messages/sec; omitted -> server default applies. */
|
|
53
|
-
maxAllowedRate?: number;
|
|
54
|
-
/** `maxBurstLimit` in messages; omitted -> {@link DEFAULT_DRAFT_SAMPLER_BURST}. */
|
|
55
|
-
maxBurstLimit?: number;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Default `maxBurstLimit` for a draft logs-event-sampler. Matches the UI's
|
|
59
|
-
* template-draft fallback (`logSampler?.maxBurstLimit ?? 1000`) so CLI raw-job
|
|
60
|
-
* drafts sample at the same rate as the UI's template drafts. `maxAllowedRate`
|
|
61
|
-
* has no CLI default: when unset we omit it and let the server default apply,
|
|
62
|
-
* exactly as the UI does.
|
|
63
|
-
*/
|
|
64
|
-
export declare const DEFAULT_DRAFT_SAMPLER_BURST = 1000;
|
|
65
|
-
/**
|
|
66
|
-
* Transforms a production job configuration into a test job configuration.
|
|
67
|
-
*
|
|
68
|
-
* This is the main entry point for job transformation. It applies the following changes:
|
|
69
|
-
* 1. Renames the job (appends "_test" or uses custom name)
|
|
70
|
-
* 2. Updates execution and processing types if specified
|
|
71
|
-
* 3. Adds test run ID tag to job tags
|
|
72
|
-
* 4. Transforms sources (replace with dataset query, sample data, or add limits)
|
|
73
|
-
* 5. Transforms sinks (replace with sync sink or test dataset sinks)
|
|
74
|
-
* 6. Adds test tagging operations before sinks
|
|
75
|
-
*
|
|
76
|
-
* The transformation is purely functional and does not make any API calls.
|
|
77
|
-
*
|
|
78
|
-
* @param originalJob - The production job to transform
|
|
79
|
-
* @param options - Transformation options
|
|
80
|
-
* @returns A new job configuration suitable for testing
|
|
81
|
-
* @throws Error if job graph is missing or invalid
|
|
82
|
-
*
|
|
83
|
-
* @example
|
|
84
|
-
* const testJob = transformJobToTest(prodJob, {
|
|
85
|
-
* execution: 'SYNCHRONOUS',
|
|
86
|
-
* processing: 'BATCH',
|
|
87
|
-
* datasetId: 'my_dataset',
|
|
88
|
-
* start: '2025-01-01T00:00:00Z',
|
|
89
|
-
* end: '2025-01-01T01:00:00Z',
|
|
90
|
-
* limitRecords: 100
|
|
91
|
-
* });
|
|
92
|
-
*/
|
|
93
|
-
export declare function transformJobToTest(originalJob: SchemaCreateJob, options: JobToTestOptions): SchemaCreateJob;
|
|
94
|
-
/**
|
|
95
|
-
* Build a source-preserving draft from a job-graph pipeline: keep the proposed
|
|
96
|
-
* source vertices streaming live (so add/remove-source edits are exercised),
|
|
97
|
-
* interpose a logs-event-sampler after each live source to bound per-source
|
|
98
|
-
* volume, strip production sinks, and fan output into a shared logs-sync-sink.
|
|
99
|
-
* Mirrors the UI's template draft mode for raw (non-templated) jobs. Verifies
|
|
100
|
-
* the graph runs, not external sink delivery.
|
|
101
|
-
*/
|
|
102
|
-
export declare function transformJobGraphToSourcePreservingDraft(originalJob: SchemaCreateJob, options?: JobToTestOptions): SchemaCreateJob;
|
|
103
|
-
/**
|
|
104
|
-
* Displays a summary of the transformations applied to a job.
|
|
105
|
-
*
|
|
106
|
-
* This shows a before/after comparison including:
|
|
107
|
-
* - Job name changes
|
|
108
|
-
* - Execution type changes
|
|
109
|
-
* - Processing type changes
|
|
110
|
-
* - operation and edge changes
|
|
111
|
-
* - Test run ID
|
|
112
|
-
*
|
|
113
|
-
* The output is formatted for console display.
|
|
114
|
-
*
|
|
115
|
-
* @param originalJob - The original production job
|
|
116
|
-
* @param transformedJob - The transformed test job
|
|
117
|
-
*/
|
|
118
|
-
export declare function showDiff(originalJob: SchemaCreateJob, transformedJob: SchemaCreateJob): void;
|
|
119
|
-
//# sourceMappingURL=job-graph-transformer.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"job-graph-transformer.d.ts","sourceRoot":"","sources":["../../../src/main/typescript/lib/job-graph-transformer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAiC,YAAY,EAAE,aAAa,EAAyB,MAAM,aAAa,CAAC;AAChH,OAAO,EAGL,eAAe,EAmBhB,MAAM,wBAAwB,CAAC;AAchC;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,qDAAqD;IACrD,SAAS,CAAC,EAAE,YAAY,CAAC;IAEzB,sDAAsD;IACtD,UAAU,CAAC,EAAE,aAAa,CAAC;IAE3B,qDAAqD;IACrD,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,yEAAyE;IACzE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mDAAmD;IACnD,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;OAIG;IACH,OAAO,CAAC,EAAE,mBAAmB,CAAC;CAC/B;AAED,0FAA0F;AAC1F,MAAM,WAAW,mBAAmB;IAClC,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mFAAmF;IACnF,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;GAMG;AACH,eAAO,MAAM,2BAA2B,OAAO,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,eAAe,EAC5B,OAAO,EAAE,gBAAgB,GACxB,eAAe,CAwDjB;AA+dD;;;;;;;GAOG;AACH,wBAAgB,wCAAwC,CACtD,WAAW,EAAE,eAAe,EAC5B,OAAO,GAAE,gBAAqB,GAC7B,eAAe,CAqFjB;AA8ID;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,QAAQ,CACtB,WAAW,EAAE,eAAe,EAC5B,cAAc,EAAE,eAAe,GAC9B,IAAI,CA4EN"}
|