@artemiskit/cli 0.2.0 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -0
- package/dist/index.js +65256 -63756
- package/dist/src/cli.d.ts.map +1 -1
- package/dist/src/commands/baseline.d.ts +9 -0
- package/dist/src/commands/baseline.d.ts.map +1 -0
- package/dist/src/commands/history.d.ts.map +1 -1
- package/dist/src/commands/redteam.d.ts.map +1 -1
- package/dist/src/commands/run.d.ts.map +1 -1
- package/dist/src/commands/stress.d.ts.map +1 -1
- package/dist/src/config/schema.d.ts +8 -0
- package/dist/src/config/schema.d.ts.map +1 -1
- package/dist/src/utils/adapter.d.ts.map +1 -1
- package/package.json +6 -6
- package/src/cli.ts +2 -0
- package/src/commands/baseline.ts +473 -0
- package/src/commands/history.ts +58 -9
- package/src/commands/redteam.ts +19 -1
- package/src/commands/run.ts +479 -52
- package/src/commands/stress.ts +28 -0
- package/src/config/schema.ts +3 -0
- package/src/utils/adapter.ts +7 -0
package/src/commands/history.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* History command - View run history
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
+
import { formatCost } from '@artemiskit/core';
|
|
5
6
|
import chalk from 'chalk';
|
|
6
7
|
import { Command } from 'commander';
|
|
7
8
|
import { loadConfig } from '../config/loader.js';
|
|
@@ -13,6 +14,7 @@ interface HistoryOptions {
|
|
|
13
14
|
scenario?: string;
|
|
14
15
|
limit?: number;
|
|
15
16
|
config?: string;
|
|
17
|
+
showCost?: boolean;
|
|
16
18
|
}
|
|
17
19
|
|
|
18
20
|
function renderHistoryTable(
|
|
@@ -21,16 +23,20 @@ function renderHistoryTable(
|
|
|
21
23
|
scenario: string;
|
|
22
24
|
successRate: number;
|
|
23
25
|
createdAt: string;
|
|
24
|
-
|
|
26
|
+
estimatedCostUsd?: number;
|
|
27
|
+
}>,
|
|
28
|
+
showCost = false
|
|
25
29
|
): string {
|
|
26
30
|
// Column widths
|
|
27
31
|
const runIdWidth = 16;
|
|
28
|
-
const scenarioWidth = 30;
|
|
32
|
+
const scenarioWidth = showCost ? 25 : 30;
|
|
29
33
|
const rateWidth = 12;
|
|
30
34
|
const dateWidth = 20;
|
|
35
|
+
const costWidth = 10;
|
|
31
36
|
|
|
32
|
-
// Total width = borders(4) + columns + spacing
|
|
33
|
-
const
|
|
37
|
+
// Total width = borders(4) + columns + spacing
|
|
38
|
+
const baseWidth = 2 + runIdWidth + 1 + scenarioWidth + 1 + rateWidth + 1 + dateWidth + 2;
|
|
39
|
+
const width = showCost ? baseWidth + costWidth + 1 : baseWidth;
|
|
34
40
|
const border = '═'.repeat(width - 2);
|
|
35
41
|
|
|
36
42
|
const formatHeaderRow = () => {
|
|
@@ -38,6 +44,10 @@ function renderHistoryTable(
|
|
|
38
44
|
const scenarioPad = padText('Scenario', scenarioWidth);
|
|
39
45
|
const ratePad = padText('Success Rate', rateWidth, 'right');
|
|
40
46
|
const datePad = padText('Date', dateWidth, 'right');
|
|
47
|
+
if (showCost) {
|
|
48
|
+
const costPad = padText('Cost', costWidth, 'right');
|
|
49
|
+
return `║ ${runIdPad} ${scenarioPad} ${ratePad} ${costPad} ${datePad} ║`;
|
|
50
|
+
}
|
|
41
51
|
return `║ ${runIdPad} ${scenarioPad} ${ratePad} ${datePad} ║`;
|
|
42
52
|
};
|
|
43
53
|
|
|
@@ -49,6 +59,8 @@ function renderHistoryTable(
|
|
|
49
59
|
`╟${'─'.repeat(width - 2)}╢`,
|
|
50
60
|
];
|
|
51
61
|
|
|
62
|
+
let totalCost = 0;
|
|
63
|
+
|
|
52
64
|
for (const run of runs) {
|
|
53
65
|
const rateColor =
|
|
54
66
|
run.successRate >= 0.9 ? chalk.green : run.successRate >= 0.7 ? chalk.yellow : chalk.red;
|
|
@@ -70,7 +82,25 @@ function renderHistoryTable(
|
|
|
70
82
|
const dateStr = `${dateObj.toLocaleDateString()} ${dateObj.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })}`;
|
|
71
83
|
const datePad = padText(dateStr, dateWidth, 'right');
|
|
72
84
|
|
|
73
|
-
|
|
85
|
+
if (showCost) {
|
|
86
|
+
const costValue = run.estimatedCostUsd !== undefined ? formatCost(run.estimatedCostUsd) : '-';
|
|
87
|
+
const costPad = padText(costValue, costWidth, 'right');
|
|
88
|
+
if (run.estimatedCostUsd !== undefined) {
|
|
89
|
+
totalCost += run.estimatedCostUsd;
|
|
90
|
+
}
|
|
91
|
+
lines.push(`║ ${runIdPad} ${scenarioPad} ${rateColored} ${chalk.dim(costPad)} ${datePad} ║`);
|
|
92
|
+
} else {
|
|
93
|
+
lines.push(`║ ${runIdPad} ${scenarioPad} ${rateColored} ${datePad} ║`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Add total cost row if showing costs
|
|
98
|
+
if (showCost) {
|
|
99
|
+
lines.push(`╟${'─'.repeat(width - 2)}╢`);
|
|
100
|
+
const totalLabel = padText('Total', runIdWidth + 1 + scenarioWidth + 1 + rateWidth, 'right');
|
|
101
|
+
const totalCostStr = padText(formatCost(totalCost), costWidth, 'right');
|
|
102
|
+
const emptyDate = padText('', dateWidth, 'right');
|
|
103
|
+
lines.push(`║ ${totalLabel} ${chalk.bold(totalCostStr)} ${emptyDate} ║`);
|
|
74
104
|
}
|
|
75
105
|
|
|
76
106
|
lines.push(`╚${border}╝`);
|
|
@@ -84,14 +114,31 @@ function renderPlainHistory(
|
|
|
84
114
|
scenario: string;
|
|
85
115
|
successRate: number;
|
|
86
116
|
createdAt: string;
|
|
87
|
-
|
|
117
|
+
estimatedCostUsd?: number;
|
|
118
|
+
}>,
|
|
119
|
+
showCost = false
|
|
88
120
|
): string {
|
|
89
121
|
const lines = ['=== RUN HISTORY ===', ''];
|
|
90
122
|
|
|
123
|
+
let totalCost = 0;
|
|
124
|
+
|
|
91
125
|
for (const run of runs) {
|
|
92
126
|
const rate = `${(run.successRate * 100).toFixed(1)}%`;
|
|
93
127
|
const date = new Date(run.createdAt).toLocaleString();
|
|
94
|
-
|
|
128
|
+
if (showCost) {
|
|
129
|
+
const cost = run.estimatedCostUsd !== undefined ? formatCost(run.estimatedCostUsd) : '-';
|
|
130
|
+
if (run.estimatedCostUsd !== undefined) {
|
|
131
|
+
totalCost += run.estimatedCostUsd;
|
|
132
|
+
}
|
|
133
|
+
lines.push(`${run.runId} ${run.scenario} ${rate} ${cost} ${date}`);
|
|
134
|
+
} else {
|
|
135
|
+
lines.push(`${run.runId} ${run.scenario} ${rate} ${date}`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (showCost) {
|
|
140
|
+
lines.push('');
|
|
141
|
+
lines.push(`Total: ${formatCost(totalCost)}`);
|
|
95
142
|
}
|
|
96
143
|
|
|
97
144
|
return lines.join('\n');
|
|
@@ -106,6 +153,7 @@ export function historyCommand(): Command {
|
|
|
106
153
|
.option('-s, --scenario <scenario>', 'Filter by scenario')
|
|
107
154
|
.option('-l, --limit <number>', 'Limit number of results', '20')
|
|
108
155
|
.option('--config <path>', 'Path to config file')
|
|
156
|
+
.option('--show-cost', 'Show cost column and total')
|
|
109
157
|
.action(async (options: HistoryOptions) => {
|
|
110
158
|
const spinner = createSpinner('Loading history...');
|
|
111
159
|
spinner.start();
|
|
@@ -119,6 +167,7 @@ export function historyCommand(): Command {
|
|
|
119
167
|
project: options.project,
|
|
120
168
|
scenario: options.scenario,
|
|
121
169
|
limit,
|
|
170
|
+
includeCost: options.showCost,
|
|
122
171
|
});
|
|
123
172
|
|
|
124
173
|
spinner.succeed('Loaded history');
|
|
@@ -140,9 +189,9 @@ export function historyCommand(): Command {
|
|
|
140
189
|
|
|
141
190
|
// Show history table
|
|
142
191
|
if (isTTY) {
|
|
143
|
-
console.log(renderHistoryTable(runs));
|
|
192
|
+
console.log(renderHistoryTable(runs, options.showCost));
|
|
144
193
|
} else {
|
|
145
|
-
console.log(renderPlainHistory(runs));
|
|
194
|
+
console.log(renderPlainHistory(runs, options.showCost));
|
|
146
195
|
}
|
|
147
196
|
|
|
148
197
|
console.log();
|
package/src/commands/redteam.ts
CHANGED
|
@@ -32,7 +32,11 @@ import {
|
|
|
32
32
|
UnsafeResponseDetector,
|
|
33
33
|
loadCustomAttacks,
|
|
34
34
|
} from '@artemiskit/redteam';
|
|
35
|
-
import {
|
|
35
|
+
import {
|
|
36
|
+
generateJSONReport,
|
|
37
|
+
generateRedTeamHTMLReport,
|
|
38
|
+
generateRedTeamMarkdownReport,
|
|
39
|
+
} from '@artemiskit/reports';
|
|
36
40
|
import chalk from 'chalk';
|
|
37
41
|
import { Command } from 'commander';
|
|
38
42
|
import { nanoid } from 'nanoid';
|
|
@@ -66,6 +70,8 @@ interface RedteamOptions {
|
|
|
66
70
|
config?: string;
|
|
67
71
|
redact?: boolean;
|
|
68
72
|
redactPatterns?: string[];
|
|
73
|
+
export?: 'markdown';
|
|
74
|
+
exportOutput?: string;
|
|
69
75
|
}
|
|
70
76
|
|
|
71
77
|
export function redteamCommand(): Command {
|
|
@@ -91,6 +97,8 @@ export function redteamCommand(): Command {
|
|
|
91
97
|
'--redact-patterns <patterns...>',
|
|
92
98
|
'Custom redaction patterns (regex or built-in: email, phone, credit_card, ssn, api_key)'
|
|
93
99
|
)
|
|
100
|
+
.option('--export <format>', 'Export results to format (markdown)')
|
|
101
|
+
.option('--export-output <dir>', 'Output directory for exports (default: ./artemis-exports)')
|
|
94
102
|
.action(async (scenarioPath: string, options: RedteamOptions) => {
|
|
95
103
|
const spinner = createSpinner('Loading configuration...');
|
|
96
104
|
spinner.start();
|
|
@@ -495,6 +503,16 @@ export function redteamCommand(): Command {
|
|
|
495
503
|
console.log(chalk.dim(` JSON: ${jsonPath}`));
|
|
496
504
|
}
|
|
497
505
|
|
|
506
|
+
// Export to markdown if requested
|
|
507
|
+
if (options.export === 'markdown') {
|
|
508
|
+
const exportDir = options.exportOutput || './artemis-exports';
|
|
509
|
+
await mkdir(exportDir, { recursive: true });
|
|
510
|
+
const markdown = generateRedTeamMarkdownReport(manifest);
|
|
511
|
+
const mdPath = join(exportDir, `${runId}.md`);
|
|
512
|
+
await writeFile(mdPath, markdown);
|
|
513
|
+
console.log(chalk.dim(`Exported: ${mdPath}`));
|
|
514
|
+
}
|
|
515
|
+
|
|
498
516
|
// Exit with error if there were unsafe responses
|
|
499
517
|
if (metrics.unsafe_responses > 0) {
|
|
500
518
|
process.exit(1);
|