@artemiskit/cli 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/dist/index.js +1800 -777
- package/dist/src/cli.d.ts.map +1 -1
- package/dist/src/commands/history.d.ts.map +1 -1
- package/dist/src/commands/redteam.d.ts.map +1 -1
- package/dist/src/commands/run.d.ts.map +1 -1
- package/dist/src/commands/stress.d.ts.map +1 -1
- package/dist/src/commands/validate.d.ts +6 -0
- package/dist/src/commands/validate.d.ts.map +1 -0
- package/package.json +6 -6
- package/src/cli.ts +2 -0
- package/src/commands/history.ts +58 -9
- package/src/commands/redteam.ts +28 -1
- package/src/commands/run.ts +121 -3
- package/src/commands/stress.ts +28 -0
- package/src/commands/validate.ts +254 -0
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Validate command - Validate scenarios without running them
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { readdirSync, statSync } from 'node:fs';
|
|
6
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
7
|
+
import { basename, join, resolve } from 'node:path';
|
|
8
|
+
import { ScenarioValidator, type ValidationResult, type ValidationSummary } from '@artemiskit/core';
|
|
9
|
+
import { generateValidationJUnitReport } from '@artemiskit/reports';
|
|
10
|
+
import { Glob } from 'bun';
|
|
11
|
+
import chalk from 'chalk';
|
|
12
|
+
import { Command } from 'commander';
|
|
13
|
+
import { icons } from '../ui/index.js';
|
|
14
|
+
|
|
15
|
+
interface ValidateOptions {
|
|
16
|
+
json?: boolean;
|
|
17
|
+
strict?: boolean;
|
|
18
|
+
quiet?: boolean;
|
|
19
|
+
export?: 'junit';
|
|
20
|
+
exportOutput?: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function validateCommand(): Command {
|
|
24
|
+
const cmd = new Command('validate');
|
|
25
|
+
|
|
26
|
+
cmd
|
|
27
|
+
.description('Validate scenario files without running them')
|
|
28
|
+
.argument('<path>', 'Path to scenario file, directory, or glob pattern')
|
|
29
|
+
.option('--json', 'Output as JSON')
|
|
30
|
+
.option('--strict', 'Treat warnings as errors')
|
|
31
|
+
.option('-q, --quiet', 'Only output errors (no success messages)')
|
|
32
|
+
.option('--export <format>', 'Export results to format (junit for CI integration)')
|
|
33
|
+
.option('--export-output <dir>', 'Output directory for exports (default: ./artemis-exports)')
|
|
34
|
+
.action(async (pathArg: string, options: ValidateOptions) => {
|
|
35
|
+
const validator = new ScenarioValidator({ strict: options.strict });
|
|
36
|
+
|
|
37
|
+
// Resolve files to validate
|
|
38
|
+
const files = resolveFiles(pathArg);
|
|
39
|
+
|
|
40
|
+
if (files.length === 0) {
|
|
41
|
+
if (options.json) {
|
|
42
|
+
console.log(
|
|
43
|
+
JSON.stringify(
|
|
44
|
+
{
|
|
45
|
+
valid: false,
|
|
46
|
+
error: `No scenario files found matching: ${pathArg}`,
|
|
47
|
+
results: [],
|
|
48
|
+
summary: { total: 0, passed: 0, failed: 0, withWarnings: 0 },
|
|
49
|
+
},
|
|
50
|
+
null,
|
|
51
|
+
2
|
|
52
|
+
)
|
|
53
|
+
);
|
|
54
|
+
} else {
|
|
55
|
+
console.log(chalk.red(`${icons.failed} No scenario files found matching: ${pathArg}`));
|
|
56
|
+
}
|
|
57
|
+
process.exit(2);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Validate all files
|
|
61
|
+
const results: ValidationResult[] = [];
|
|
62
|
+
|
|
63
|
+
if (!options.json && !options.quiet) {
|
|
64
|
+
console.log(chalk.bold('Validating scenarios...\n'));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
for (const file of files) {
|
|
68
|
+
const result = validator.validate(file);
|
|
69
|
+
results.push(result);
|
|
70
|
+
|
|
71
|
+
// In strict mode, warnings become errors
|
|
72
|
+
if (options.strict && result.warnings.length > 0) {
|
|
73
|
+
result.valid = false;
|
|
74
|
+
result.errors.push(
|
|
75
|
+
...result.warnings.map((w: ValidationResult['warnings'][0]) => ({
|
|
76
|
+
...w,
|
|
77
|
+
severity: 'error' as const,
|
|
78
|
+
}))
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (!options.json) {
|
|
83
|
+
printFileResult(result, options);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Calculate summary
|
|
88
|
+
const summary: ValidationSummary = {
|
|
89
|
+
total: results.length,
|
|
90
|
+
passed: results.filter((r) => r.valid && r.warnings.length === 0).length,
|
|
91
|
+
failed: results.filter((r) => !r.valid).length,
|
|
92
|
+
withWarnings: results.filter((r) => r.valid && r.warnings.length > 0).length,
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
// Output results
|
|
96
|
+
if (options.json) {
|
|
97
|
+
console.log(
|
|
98
|
+
JSON.stringify(
|
|
99
|
+
{
|
|
100
|
+
valid: summary.failed === 0,
|
|
101
|
+
results: results.map((r) => ({
|
|
102
|
+
file: r.file,
|
|
103
|
+
valid: r.valid,
|
|
104
|
+
errors: r.errors,
|
|
105
|
+
warnings: r.warnings,
|
|
106
|
+
})),
|
|
107
|
+
summary,
|
|
108
|
+
},
|
|
109
|
+
null,
|
|
110
|
+
2
|
|
111
|
+
)
|
|
112
|
+
);
|
|
113
|
+
} else if (!options.quiet) {
|
|
114
|
+
console.log();
|
|
115
|
+
printSummary(summary, options.strict);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Export to JUnit if requested
|
|
119
|
+
if (options.export === 'junit') {
|
|
120
|
+
const exportDir = options.exportOutput || './artemis-exports';
|
|
121
|
+
await mkdir(exportDir, { recursive: true });
|
|
122
|
+
const junit = generateValidationJUnitReport(results);
|
|
123
|
+
const junitPath = join(exportDir, `validation-${Date.now()}.xml`);
|
|
124
|
+
await writeFile(junitPath, junit);
|
|
125
|
+
if (!options.quiet) {
|
|
126
|
+
console.log(chalk.dim(`Exported: ${junitPath}`));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Exit with appropriate code
|
|
131
|
+
if (summary.failed > 0) {
|
|
132
|
+
process.exit(1);
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
return cmd;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Resolve files from path argument (file, directory, or glob)
|
|
141
|
+
*/
|
|
142
|
+
function resolveFiles(pathArg: string): string[] {
|
|
143
|
+
const resolved = resolve(pathArg);
|
|
144
|
+
|
|
145
|
+
try {
|
|
146
|
+
const stat = statSync(resolved);
|
|
147
|
+
|
|
148
|
+
if (stat.isFile()) {
|
|
149
|
+
// Single file
|
|
150
|
+
return [resolved];
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (stat.isDirectory()) {
|
|
154
|
+
// Directory - find all yaml files recursively
|
|
155
|
+
return findYamlFiles(resolved);
|
|
156
|
+
}
|
|
157
|
+
} catch {
|
|
158
|
+
// Path doesn't exist as file/directory - try as glob
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Try as glob pattern using Bun's Glob
|
|
162
|
+
const glob = new Glob(pathArg);
|
|
163
|
+
const matches: string[] = [];
|
|
164
|
+
for (const file of glob.scanSync({ absolute: true, onlyFiles: true })) {
|
|
165
|
+
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
|
166
|
+
matches.push(file);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return matches;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Find all YAML files in a directory recursively
|
|
175
|
+
*/
|
|
176
|
+
function findYamlFiles(dir: string): string[] {
|
|
177
|
+
const files: string[] = [];
|
|
178
|
+
|
|
179
|
+
const entries = readdirSync(dir, { withFileTypes: true });
|
|
180
|
+
|
|
181
|
+
for (const entry of entries) {
|
|
182
|
+
const fullPath = join(dir, entry.name);
|
|
183
|
+
|
|
184
|
+
if (entry.isDirectory()) {
|
|
185
|
+
files.push(...findYamlFiles(fullPath));
|
|
186
|
+
} else if (entry.isFile() && (entry.name.endsWith('.yaml') || entry.name.endsWith('.yml'))) {
|
|
187
|
+
files.push(fullPath);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
return files;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Print result for a single file
|
|
196
|
+
*/
|
|
197
|
+
function printFileResult(result: ValidationResult, options: ValidateOptions): void {
|
|
198
|
+
const fileName = basename(result.file);
|
|
199
|
+
|
|
200
|
+
if (result.valid && result.warnings.length === 0) {
|
|
201
|
+
if (!options.quiet) {
|
|
202
|
+
console.log(`${icons.passed} ${chalk.green(fileName)}`);
|
|
203
|
+
}
|
|
204
|
+
} else if (result.valid && result.warnings.length > 0) {
|
|
205
|
+
console.log(`${icons.warning} ${chalk.yellow(fileName)}`);
|
|
206
|
+
for (const warning of result.warnings) {
|
|
207
|
+
const location = warning.column
|
|
208
|
+
? `Line ${warning.line}:${warning.column}`
|
|
209
|
+
: `Line ${warning.line}`;
|
|
210
|
+
console.log(chalk.yellow(` ${location}: ${warning.message}`));
|
|
211
|
+
if (warning.suggestion) {
|
|
212
|
+
console.log(chalk.dim(` Suggestion: ${warning.suggestion}`));
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
} else {
|
|
216
|
+
console.log(`${icons.failed} ${chalk.red(fileName)}`);
|
|
217
|
+
for (const error of result.errors) {
|
|
218
|
+
const location = error.column ? `Line ${error.line}:${error.column}` : `Line ${error.line}`;
|
|
219
|
+
console.log(chalk.red(` ${location}: ${error.message}`));
|
|
220
|
+
if (error.suggestion) {
|
|
221
|
+
console.log(chalk.dim(` Suggestion: ${error.suggestion}`));
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
for (const warning of result.warnings) {
|
|
225
|
+
const location = warning.column
|
|
226
|
+
? `Line ${warning.line}:${warning.column}`
|
|
227
|
+
: `Line ${warning.line}`;
|
|
228
|
+
console.log(chalk.yellow(` ${location}: ${warning.message}`));
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Print validation summary
|
|
235
|
+
*/
|
|
236
|
+
function printSummary(summary: ValidationSummary, strict?: boolean): void {
|
|
237
|
+
const parts: string[] = [];
|
|
238
|
+
|
|
239
|
+
if (summary.passed > 0) {
|
|
240
|
+
parts.push(chalk.green(`${summary.passed} passed`));
|
|
241
|
+
}
|
|
242
|
+
if (summary.failed > 0) {
|
|
243
|
+
parts.push(chalk.red(`${summary.failed} failed`));
|
|
244
|
+
}
|
|
245
|
+
if (summary.withWarnings > 0 && !strict) {
|
|
246
|
+
parts.push(chalk.yellow(`${summary.withWarnings} with warnings`));
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const statusIcon = summary.failed > 0 ? icons.failed : icons.passed;
|
|
250
|
+
const statusColor = summary.failed > 0 ? chalk.red : chalk.green;
|
|
251
|
+
|
|
252
|
+
console.log(statusColor(`${statusIcon} ${parts.join(', ')}`));
|
|
253
|
+
console.log(chalk.dim(`${summary.total} scenario(s) validated`));
|
|
254
|
+
}
|