@artemiskit/cli 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/index.js +15 -17
- package/dist/src/__tests__/helpers/index.d.ts +6 -0
- package/dist/src/__tests__/helpers/index.d.ts.map +1 -0
- package/dist/src/__tests__/helpers/mock-adapter.d.ts +87 -0
- package/dist/src/__tests__/helpers/mock-adapter.d.ts.map +1 -0
- package/dist/src/__tests__/helpers/test-utils.d.ts +47 -0
- package/dist/src/__tests__/helpers/test-utils.d.ts.map +1 -0
- package/dist/src/commands/compare.d.ts.map +1 -1
- package/dist/src/commands/history.d.ts.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/redteam.d.ts.map +1 -1
- package/dist/src/commands/report.d.ts.map +1 -1
- package/dist/src/commands/run.d.ts.map +1 -1
- package/dist/src/ui/errors.d.ts.map +1 -1
- package/dist/src/ui/panels.d.ts.map +1 -1
- package/dist/src/ui/progress.d.ts.map +1 -1
- package/dist/src/ui/utils.d.ts.map +1 -1
- package/package.json +6 -6
- package/src/__tests__/helpers/mock-adapter.ts +22 -4
- package/src/__tests__/helpers/test-utils.ts +3 -3
- package/src/__tests__/integration/compare-command.test.ts +7 -7
- package/src/__tests__/integration/config.test.ts +2 -2
- package/src/__tests__/integration/history-command.test.ts +2 -2
- package/src/__tests__/integration/init-command.test.ts +3 -3
- package/src/__tests__/integration/report-command.test.ts +2 -2
- package/src/__tests__/integration/ui.test.ts +6 -6
- package/src/commands/compare.ts +2 -4
- package/src/commands/history.ts +2 -2
- package/src/commands/init.ts +2 -2
- package/src/commands/redteam.ts +6 -6
- package/src/commands/report.ts +3 -3
- package/src/commands/run.ts +4 -4
- package/src/commands/stress.ts +4 -4
- package/src/ui/errors.ts +1 -1
- package/src/ui/live-status.ts +1 -1
- package/src/ui/panels.ts +2 -2
- package/src/ui/progress.ts +1 -1
- package/src/ui/utils.ts +4 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# @artemiskit/cli
|
|
2
2
|
|
|
3
|
+
## 0.1.7
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- @artemiskit/cli\*\* (patch)
|
|
8
|
+
|
|
9
|
+
Enhanced CLI user experience and added comprehensive integration tests
|
|
10
|
+
|
|
11
|
+
**UI Enhancements:**
|
|
12
|
+
|
|
13
|
+
- Fixed table border alignment in compare and history commands (ANSI color codes no longer affect column widths)
|
|
14
|
+
- Added progress bars, error display panels, and summary boxes
|
|
15
|
+
- Added box-drawing tables with Unicode characters for structured output
|
|
16
|
+
- Added TTY detection for graceful fallback in non-TTY/CI environments
|
|
17
|
+
|
|
18
|
+
**Testing:**
|
|
19
|
+
|
|
20
|
+
- Added 60+ integration tests for CLI commands (init, history, compare, report)
|
|
21
|
+
- Added test helpers including mock LLM adapter and test utilities
|
|
22
|
+
- Achieved 80%+ source file test coverage (155 total tests passing)
|
|
23
|
+
|
|
24
|
+
**Documentation:**
|
|
25
|
+
|
|
26
|
+
- Updated ROADMAP.md to mark v0.1.x as complete
|
|
27
|
+
- Fixed docs to use correct YAML config format
|
|
28
|
+
|
|
3
29
|
## 0.1.6
|
|
4
30
|
|
|
5
31
|
### Patch Changes
|
package/dist/index.js
CHANGED
|
@@ -10424,7 +10424,7 @@ var {
|
|
|
10424
10424
|
Help
|
|
10425
10425
|
} = import__.default;
|
|
10426
10426
|
// package.json
|
|
10427
|
-
var version = "0.1.
|
|
10427
|
+
var version = "0.1.6";
|
|
10428
10428
|
|
|
10429
10429
|
// ../../node_modules/.bun/chalk@5.6.2/node_modules/chalk/source/vendor/ansi-styles/index.js
|
|
10430
10430
|
var ANSI_BACKGROUND_OFFSET = 10;
|
|
@@ -15093,7 +15093,8 @@ function padText(text, width, align = "left") {
|
|
|
15093
15093
|
const paddingNeeded = Math.max(0, width - visibleLength);
|
|
15094
15094
|
if (align === "center") {
|
|
15095
15095
|
return centerText(text, width);
|
|
15096
|
-
}
|
|
15096
|
+
}
|
|
15097
|
+
if (align === "right") {
|
|
15097
15098
|
return " ".repeat(paddingNeeded) + text;
|
|
15098
15099
|
}
|
|
15099
15100
|
return text + " ".repeat(paddingNeeded);
|
|
@@ -15105,7 +15106,7 @@ function truncate(text, maxWidth) {
|
|
|
15105
15106
|
const visibleLength = stripAnsi(text).length;
|
|
15106
15107
|
if (visibleLength <= maxWidth)
|
|
15107
15108
|
return text;
|
|
15108
|
-
return text.slice(0, maxWidth - 1)
|
|
15109
|
+
return `${text.slice(0, maxWidth - 1)}\u2026`;
|
|
15109
15110
|
}
|
|
15110
15111
|
function formatDuration(ms) {
|
|
15111
15112
|
if (ms < 1000)
|
|
@@ -15236,7 +15237,7 @@ function renderRedteamSummaryPanel(data) {
|
|
|
15236
15237
|
source_default.magenta("\u2551") + padText(` ${source_default.green("\u2713")} Safe: ${data.safeResponses} ${source_default.red("\u2717")} Unsafe: ${data.unsafeResponses}`, width - 2) + source_default.magenta("\u2551"),
|
|
15237
15238
|
source_default.magenta("\u2551") + padText(` ${source_default.cyan("\u2298")} Blocked: ${data.blockedResponses} ${source_default.yellow("!")} Errors: ${data.errorResponses}`, width - 2) + source_default.magenta("\u2551"),
|
|
15238
15239
|
source_default.magenta(`\u2560${border}\u2563`),
|
|
15239
|
-
source_default.magenta("\u2551") + padText(` Defense Rate: ${defenseColor(data.defenseRate.toFixed(1)
|
|
15240
|
+
source_default.magenta("\u2551") + padText(` Defense Rate: ${defenseColor(`${data.defenseRate.toFixed(1)}%`)}`, width - 2) + source_default.magenta("\u2551"),
|
|
15240
15241
|
source_default.magenta(`\u255A${border}\u255D`)
|
|
15241
15242
|
];
|
|
15242
15243
|
return lines.join(`
|
|
@@ -42051,7 +42052,7 @@ function compareCommand() {
|
|
|
42051
42052
|
const threshold = Number.parseFloat(String(options.threshold)) || 0.05;
|
|
42052
42053
|
const hasRegression = delta.successRate < -threshold;
|
|
42053
42054
|
if (hasRegression) {
|
|
42054
|
-
console.log(`${icons.failed} ${source_default.red("Regression detected!")}
|
|
42055
|
+
console.log(`${icons.failed} ${source_default.red("Regression detected!")} Success rate dropped by ${source_default.bold(`${Math.abs(delta.successRate * 100).toFixed(1)}%`)} ${source_default.dim(`(threshold: ${threshold * 100}%)`)}`);
|
|
42055
42056
|
process.exit(1);
|
|
42056
42057
|
} else {
|
|
42057
42058
|
console.log(`${icons.passed} ${source_default.green("No regression detected")}`);
|
|
@@ -42099,7 +42100,7 @@ function renderHistoryTable(runs) {
|
|
|
42099
42100
|
for (const run of runs) {
|
|
42100
42101
|
const rateColor = run.successRate >= 0.9 ? source_default.green : run.successRate >= 0.7 ? source_default.yellow : source_default.red;
|
|
42101
42102
|
const runIdPad = padText(run.runId, runIdWidth);
|
|
42102
|
-
const truncScenario = run.scenario.length > scenarioWidth - 2 ? run.scenario.slice(0, scenarioWidth - 3)
|
|
42103
|
+
const truncScenario = run.scenario.length > scenarioWidth - 2 ? `${run.scenario.slice(0, scenarioWidth - 3)}\u2026` : run.scenario;
|
|
42103
42104
|
const scenarioPad = padText(truncScenario, scenarioWidth);
|
|
42104
42105
|
const rateValue = `${(run.successRate * 100).toFixed(1)}%`;
|
|
42105
42106
|
const ratePad = padText(rateValue, rateWidth, "right");
|
|
@@ -42114,10 +42115,7 @@ function renderHistoryTable(runs) {
|
|
|
42114
42115
|
`);
|
|
42115
42116
|
}
|
|
42116
42117
|
function renderPlainHistory(runs) {
|
|
42117
|
-
const lines = [
|
|
42118
|
-
"=== RUN HISTORY ===",
|
|
42119
|
-
""
|
|
42120
|
-
];
|
|
42118
|
+
const lines = ["=== RUN HISTORY ===", ""];
|
|
42121
42119
|
for (const run of runs) {
|
|
42122
42120
|
const rate = `${(run.successRate * 100).toFixed(1)}%`;
|
|
42123
42121
|
const date = new Date(run.createdAt).toLocaleString();
|
|
@@ -42186,7 +42184,7 @@ function historyCommand() {
|
|
|
42186
42184
|
|
|
42187
42185
|
// src/commands/init.ts
|
|
42188
42186
|
import { existsSync as existsSync2 } from "fs";
|
|
42189
|
-
import { mkdir as mkdir2, readFile as readFile3, writeFile as writeFile2
|
|
42187
|
+
import { appendFile, mkdir as mkdir2, readFile as readFile3, writeFile as writeFile2 } from "fs/promises";
|
|
42190
42188
|
import { join as join3 } from "path";
|
|
42191
42189
|
var DEFAULT_CONFIG = `# ArtemisKit Configuration
|
|
42192
42190
|
project: my-project
|
|
@@ -42330,8 +42328,8 @@ async function appendEnvKeys(cwd) {
|
|
|
42330
42328
|
|
|
42331
42329
|
` : existingContent ? `
|
|
42332
42330
|
` : "";
|
|
42333
|
-
await appendFile(envPath, prefix + linesToAdd.join(`
|
|
42334
|
-
`)
|
|
42331
|
+
await appendFile(envPath, `${prefix + linesToAdd.join(`
|
|
42332
|
+
`)}
|
|
42335
42333
|
`);
|
|
42336
42334
|
}
|
|
42337
42335
|
return { added, skipped };
|
|
@@ -49604,7 +49602,7 @@ function redteamCommand() {
|
|
|
49604
49602
|
const detection = detector.detect(result.text);
|
|
49605
49603
|
const resultStatus = detection.unsafe ? "unsafe" : "safe";
|
|
49606
49604
|
if (isTTY) {
|
|
49607
|
-
process.stdout.write(
|
|
49605
|
+
process.stdout.write(`\r${" ".repeat(60)}\r`);
|
|
49608
49606
|
}
|
|
49609
49607
|
const statusIcon = detection.unsafe ? icons.failed : icons.passed;
|
|
49610
49608
|
const statusLabel = detection.unsafe ? source_default.red(`UNSAFE (${detection.severity})`) : source_default.green("SAFE");
|
|
@@ -49647,7 +49645,7 @@ function redteamCommand() {
|
|
|
49647
49645
|
const errorMessage = error.message;
|
|
49648
49646
|
const isContentFiltered = isProviderContentFilter(errorMessage);
|
|
49649
49647
|
if (isTTY) {
|
|
49650
|
-
process.stdout.write(
|
|
49648
|
+
process.stdout.write(`\r${" ".repeat(60)}\r`);
|
|
49651
49649
|
}
|
|
49652
49650
|
let errorPrompt = mutated.mutated;
|
|
49653
49651
|
let errorCaseRedaction;
|
|
@@ -49919,7 +49917,7 @@ function reportCommand() {
|
|
|
49919
49917
|
const htmlPath = join5(outputDir, `${runId}.html`);
|
|
49920
49918
|
await writeFile4(htmlPath, html);
|
|
49921
49919
|
generatedFiles.push(htmlPath);
|
|
49922
|
-
spinner.succeed(
|
|
49920
|
+
spinner.succeed("Generated HTML report");
|
|
49923
49921
|
}
|
|
49924
49922
|
if (format === "json" || format === "both") {
|
|
49925
49923
|
spinner.start("Generating JSON report...");
|
|
@@ -49927,7 +49925,7 @@ function reportCommand() {
|
|
|
49927
49925
|
const jsonPath = join5(outputDir, `${runId}.json`);
|
|
49928
49926
|
await writeFile4(jsonPath, json);
|
|
49929
49927
|
generatedFiles.push(jsonPath);
|
|
49930
|
-
spinner.succeed(
|
|
49928
|
+
spinner.succeed("Generated JSON report");
|
|
49931
49929
|
}
|
|
49932
49930
|
console.log();
|
|
49933
49931
|
console.log(renderInfoBox("Report Generated", [
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/__tests__/helpers/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mock adapter for CLI integration tests
|
|
3
|
+
*
|
|
4
|
+
* This module provides mock types and adapters for testing CLI commands
|
|
5
|
+
* without making actual LLM API calls.
|
|
6
|
+
*/
|
|
7
|
+
/** Message format for chat interactions */
|
|
8
|
+
export interface MockMessage {
|
|
9
|
+
role: 'system' | 'user' | 'assistant';
|
|
10
|
+
content: string;
|
|
11
|
+
}
|
|
12
|
+
/** Response from the mock adapter */
|
|
13
|
+
export interface MockLLMResponse {
|
|
14
|
+
content: string;
|
|
15
|
+
usage: {
|
|
16
|
+
input: number;
|
|
17
|
+
output: number;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
/** Mock adapter interface */
|
|
21
|
+
export interface MockLLMAdapter {
|
|
22
|
+
chat: (messages: MockMessage[]) => Promise<MockLLMResponse>;
|
|
23
|
+
}
|
|
24
|
+
export interface MockResponse {
|
|
25
|
+
content: string;
|
|
26
|
+
latencyMs?: number;
|
|
27
|
+
tokens?: {
|
|
28
|
+
input: number;
|
|
29
|
+
output: number;
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export interface MockAdapterOptions {
|
|
33
|
+
responses?: Map<string, MockResponse>;
|
|
34
|
+
defaultResponse?: MockResponse;
|
|
35
|
+
shouldFail?: boolean;
|
|
36
|
+
failureMessage?: string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Creates a mock LLM adapter for testing
|
|
40
|
+
*/
|
|
41
|
+
export declare function createMockAdapter(options?: MockAdapterOptions): MockLLMAdapter;
|
|
42
|
+
/**
|
|
43
|
+
* Preset responses for common test scenarios
|
|
44
|
+
*/
|
|
45
|
+
export declare const mockResponses: {
|
|
46
|
+
greeting: {
|
|
47
|
+
content: string;
|
|
48
|
+
latencyMs: number;
|
|
49
|
+
tokens: {
|
|
50
|
+
input: number;
|
|
51
|
+
output: number;
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
capitals: {
|
|
55
|
+
content: string;
|
|
56
|
+
latencyMs: number;
|
|
57
|
+
tokens: {
|
|
58
|
+
input: number;
|
|
59
|
+
output: number;
|
|
60
|
+
};
|
|
61
|
+
};
|
|
62
|
+
math: {
|
|
63
|
+
content: string;
|
|
64
|
+
latencyMs: number;
|
|
65
|
+
tokens: {
|
|
66
|
+
input: number;
|
|
67
|
+
output: number;
|
|
68
|
+
};
|
|
69
|
+
};
|
|
70
|
+
json: {
|
|
71
|
+
content: string;
|
|
72
|
+
latencyMs: number;
|
|
73
|
+
tokens: {
|
|
74
|
+
input: number;
|
|
75
|
+
output: number;
|
|
76
|
+
};
|
|
77
|
+
};
|
|
78
|
+
code: {
|
|
79
|
+
content: string;
|
|
80
|
+
latencyMs: number;
|
|
81
|
+
tokens: {
|
|
82
|
+
input: number;
|
|
83
|
+
output: number;
|
|
84
|
+
};
|
|
85
|
+
};
|
|
86
|
+
};
|
|
87
|
+
//# sourceMappingURL=mock-adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mock-adapter.d.ts","sourceRoot":"","sources":["../../../../src/__tests__/helpers/mock-adapter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,2CAA2C;AAC3C,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qCAAqC;AACrC,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C;AAED,6BAA6B;AAC7B,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;CAC7D;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACtC,eAAe,CAAC,EAAE,YAAY,CAAC;IAC/B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,GAAE,kBAAuB,GAAG,cAAc,CAoClF;AAED;;GAEG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0BzB,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test utilities for CLI integration tests
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Creates a temporary directory for test isolation
|
|
6
|
+
*/
|
|
7
|
+
export declare function createTestDir(prefix?: string): Promise<string>;
|
|
8
|
+
/**
|
|
9
|
+
* Cleans up a test directory
|
|
10
|
+
*/
|
|
11
|
+
export declare function cleanupTestDir(testDir: string): Promise<void>;
|
|
12
|
+
/**
|
|
13
|
+
* Creates a test scenario file
|
|
14
|
+
*/
|
|
15
|
+
export declare function createScenarioFile(dir: string, name: string, content: string): Promise<string>;
|
|
16
|
+
/**
|
|
17
|
+
* Creates a test config file
|
|
18
|
+
*/
|
|
19
|
+
export declare function createConfigFile(dir: string, config: Record<string, unknown>): Promise<string>;
|
|
20
|
+
/**
|
|
21
|
+
* Sample scenario templates for testing
|
|
22
|
+
*/
|
|
23
|
+
export declare const scenarioTemplates: {
|
|
24
|
+
simple: string;
|
|
25
|
+
multiCase: string;
|
|
26
|
+
withProvider: string;
|
|
27
|
+
exactMatch: string;
|
|
28
|
+
regexMatch: string;
|
|
29
|
+
failing: string;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Captures console output during test execution
|
|
33
|
+
*/
|
|
34
|
+
export declare class OutputCapture {
|
|
35
|
+
private originalLog;
|
|
36
|
+
private originalError;
|
|
37
|
+
private logs;
|
|
38
|
+
private errors;
|
|
39
|
+
start(): void;
|
|
40
|
+
stop(): {
|
|
41
|
+
logs: string[];
|
|
42
|
+
errors: string[];
|
|
43
|
+
};
|
|
44
|
+
getOutput(): string;
|
|
45
|
+
getErrors(): string;
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=test-utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-utils.d.ts","sourceRoot":"","sources":["../../../../src/__tests__/helpers/test-utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,wBAAsB,aAAa,CAAC,MAAM,SAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,CAI5E;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMnE;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAMjB;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B,OAAO,CAAC,MAAM,CAAC,CAejB;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB;;;;;;;CAgG7B,CAAC;AAEF;;GAEG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,WAAW,CAAmC;IACtD,OAAO,CAAC,aAAa,CAAuC;IAC5D,OAAO,CAAC,IAAI,CAAgB;IAC5B,OAAO,CAAC,MAAM,CAAgB;IAE9B,KAAK,IAAI,IAAI;IAeb,IAAI,IAAI;QAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE;IAM5C,SAAS,IAAI,MAAM;IAInB,SAAS,IAAI,MAAM;CAGpB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../../src/commands/compare.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../../src/commands/compare.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA2HpC,wBAAgB,cAAc,IAAI,OAAO,CAkFxC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../../src/commands/history.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../../src/commands/history.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA8FpC,wBAAgB,cAAc,IAAI,OAAO,CAiFxC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/commands/init.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/commands/init.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoLpC,wBAAgB,WAAW,IAAI,OAAO,CAkFrC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"AAAA;;GAEG;AA8BH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiCpC,wBAAgB,cAAc,IAAI,OAAO,
|
|
1
|
+
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"AAAA;;GAEG;AA8BH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiCpC,wBAAgB,cAAc,IAAI,OAAO,CAyZxC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/commands/report.ts"],"names":[],"mappings":"AAAA;;GAEG;AAWH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoDpC,wBAAgB,aAAa,IAAI,OAAO,
|
|
1
|
+
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/commands/report.ts"],"names":[],"mappings":"AAAA;;GAEG;AAWH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoDpC,wBAAgB,aAAa,IAAI,OAAO,CA2EvC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/commands/run.ts"],"names":[],"mappings":"AAAA;;GAEG;AASH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAmCpC,wBAAgB,UAAU,IAAI,OAAO,
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/commands/run.ts"],"names":[],"mappings":"AAAA;;GAEG;AASH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAmCpC,wBAAgB,UAAU,IAAI,OAAO,CA8MpC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../src/ui/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,MAAM,WAAW,YAAY;IAC3B,uBAAuB;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,iCAAiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,YAAY,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../src/ui/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,MAAM,WAAW,YAAY;IAC3B,uBAAuB;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,iCAAiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,YAAY,GAAG,MAAM,CAgFrD;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAsD5F;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,YAAY,CAqEpF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"panels.d.ts","sourceRoot":"","sources":["../../../src/ui/panels.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC5C;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,WAAW,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"panels.d.ts","sourceRoot":"","sources":["../../../src/ui/panels.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC5C;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAiC5D;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,iBAAiB,GAAG,MAAM,CA0DxE;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,CA4C1E;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAsBpE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"progress.d.ts","sourceRoot":"","sources":["../../../src/ui/progress.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,MAAM,WAAW,kBAAkB;IACjC,8CAA8C;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mCAAmC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,iCAAiC;IACjC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,mCAAmC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAWD;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,kBAAuB,GAC/B,MAAM,
|
|
1
|
+
{"version":3,"file":"progress.d.ts","sourceRoot":"","sources":["../../../src/ui/progress.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,MAAM,WAAW,kBAAkB;IACjC,8CAA8C;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mCAAmC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,iCAAiC;IACjC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,mCAAmC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAWD;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,kBAAuB,GAC/B,MAAM,CAyBR;AAED;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,UAAU,CAAM;gBAEZ,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB;IAK3D;;OAEG;IACH,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAK7B;;OAEG;IACH,SAAS,IAAI,IAAI;IAIjB;;OAEG;IACH,OAAO,CAAC,MAAM;IAYd;;OAEG;IACH,QAAQ,IAAI,IAAI;CAMjB;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAO3F"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/ui/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,eAAO,MAAM,KAAK,SAAgC,CAAC;AAEnD;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,MAAM,CAEzC;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAEhF;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAK9D;AAED;;GAEG;AACH,wBAAgB,OAAO,
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/ui/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,eAAO,MAAM,KAAK,SAAgC,CAAC;AAEnD;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,MAAM,CAEzC;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAEhF;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAK9D;AAED;;GAEG;AACH,wBAAgB,OAAO,CACrB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,MAAM,GAAG,OAAO,GAAG,QAAiB,GAC1C,MAAM,CAWR;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAG7C;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAI/D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAMjD;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEhD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"description": "Command-line interface for ArtemisKit LLM evaluation toolkit",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -45,11 +45,11 @@
|
|
|
45
45
|
"test": "bun test"
|
|
46
46
|
},
|
|
47
47
|
"dependencies": {
|
|
48
|
-
"@artemiskit/adapter-openai": "
|
|
49
|
-
"@artemiskit/adapter-vercel-ai": "
|
|
50
|
-
"@artemiskit/core": "
|
|
51
|
-
"@artemiskit/redteam": "
|
|
52
|
-
"@artemiskit/reports": "
|
|
48
|
+
"@artemiskit/adapter-openai": "0.1.5",
|
|
49
|
+
"@artemiskit/adapter-vercel-ai": "0.1.5",
|
|
50
|
+
"@artemiskit/core": "0.1.5",
|
|
51
|
+
"@artemiskit/redteam": "0.1.5",
|
|
52
|
+
"@artemiskit/reports": "0.1.5",
|
|
53
53
|
"chalk": "^5.3.0",
|
|
54
54
|
"cli-table3": "^0.6.3",
|
|
55
55
|
"commander": "^12.0.0",
|
|
@@ -1,8 +1,26 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Mock adapter for CLI integration tests
|
|
3
|
+
*
|
|
4
|
+
* This module provides mock types and adapters for testing CLI commands
|
|
5
|
+
* without making actual LLM API calls.
|
|
3
6
|
*/
|
|
4
7
|
|
|
5
|
-
|
|
8
|
+
/** Message format for chat interactions */
|
|
9
|
+
export interface MockMessage {
|
|
10
|
+
role: 'system' | 'user' | 'assistant';
|
|
11
|
+
content: string;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Response from the mock adapter */
|
|
15
|
+
export interface MockLLMResponse {
|
|
16
|
+
content: string;
|
|
17
|
+
usage: { input: number; output: number };
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Mock adapter interface */
|
|
21
|
+
export interface MockLLMAdapter {
|
|
22
|
+
chat: (messages: MockMessage[]) => Promise<MockLLMResponse>;
|
|
23
|
+
}
|
|
6
24
|
|
|
7
25
|
export interface MockResponse {
|
|
8
26
|
content: string;
|
|
@@ -20,7 +38,7 @@ export interface MockAdapterOptions {
|
|
|
20
38
|
/**
|
|
21
39
|
* Creates a mock LLM adapter for testing
|
|
22
40
|
*/
|
|
23
|
-
export function createMockAdapter(options: MockAdapterOptions = {}):
|
|
41
|
+
export function createMockAdapter(options: MockAdapterOptions = {}): MockLLMAdapter {
|
|
24
42
|
const {
|
|
25
43
|
responses = new Map(),
|
|
26
44
|
defaultResponse = {
|
|
@@ -33,7 +51,7 @@ export function createMockAdapter(options: MockAdapterOptions = {}): LLMAdapter
|
|
|
33
51
|
} = options;
|
|
34
52
|
|
|
35
53
|
return {
|
|
36
|
-
chat: async (messages): Promise<
|
|
54
|
+
chat: async (messages: MockMessage[]): Promise<MockLLMResponse> => {
|
|
37
55
|
if (shouldFail) {
|
|
38
56
|
throw new Error(failureMessage);
|
|
39
57
|
}
|
|
@@ -47,7 +65,7 @@ export function createMockAdapter(options: MockAdapterOptions = {}): LLMAdapter
|
|
|
47
65
|
|
|
48
66
|
// Simulate latency
|
|
49
67
|
if (mockResponse.latencyMs) {
|
|
50
|
-
await new Promise((resolve) => setTimeout(resolve, Math.min(mockResponse.latencyMs
|
|
68
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(mockResponse.latencyMs, 50)));
|
|
51
69
|
}
|
|
52
70
|
|
|
53
71
|
return {
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import { mkdir, rm, writeFile } from 'node:fs/promises';
|
|
6
|
-
import { join } from 'node:path';
|
|
7
6
|
import { tmpdir } from 'node:os';
|
|
7
|
+
import { join } from 'node:path';
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* Creates a temporary directory for test isolation
|
|
@@ -169,8 +169,8 @@ cases:
|
|
|
169
169
|
* Captures console output during test execution
|
|
170
170
|
*/
|
|
171
171
|
export class OutputCapture {
|
|
172
|
-
private originalLog: typeof console.log;
|
|
173
|
-
private originalError: typeof console.error;
|
|
172
|
+
private originalLog: typeof console.log = console.log;
|
|
173
|
+
private originalError: typeof console.error = console.error;
|
|
174
174
|
private logs: string[] = [];
|
|
175
175
|
private errors: string[] = [];
|
|
176
176
|
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
* Integration tests for compare command
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
6
6
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
7
7
|
import { join } from 'node:path';
|
|
8
|
-
import { createTestDir, cleanupTestDir } from '../helpers/test-utils.js';
|
|
9
8
|
import { createStorage } from '../../utils/storage.js';
|
|
9
|
+
import { cleanupTestDir, createTestDir } from '../helpers/test-utils.js';
|
|
10
10
|
|
|
11
11
|
describe('Compare Command', () => {
|
|
12
12
|
let testDir: string;
|
|
@@ -73,7 +73,7 @@ describe('Compare Command', () => {
|
|
|
73
73
|
},
|
|
74
74
|
});
|
|
75
75
|
|
|
76
|
-
const comparison = await storage.compare
|
|
76
|
+
const comparison = await storage.compare?.('baseline-001', 'current-001');
|
|
77
77
|
|
|
78
78
|
expect(comparison.baseline.metrics.success_rate).toBe(0.8);
|
|
79
79
|
expect(comparison.current.metrics.success_rate).toBe(1.0);
|
|
@@ -128,7 +128,7 @@ describe('Compare Command', () => {
|
|
|
128
128
|
},
|
|
129
129
|
});
|
|
130
130
|
|
|
131
|
-
const comparison = await storage.compare
|
|
131
|
+
const comparison = await storage.compare?.('baseline-002', 'current-002');
|
|
132
132
|
|
|
133
133
|
// Success rate dropped by 0.4 (40%)
|
|
134
134
|
expect(comparison.delta.successRate).toBeCloseTo(-0.4, 5);
|
|
@@ -166,7 +166,7 @@ describe('Compare Command', () => {
|
|
|
166
166
|
},
|
|
167
167
|
});
|
|
168
168
|
|
|
169
|
-
const comparison = await storage.compare
|
|
169
|
+
const comparison = await storage.compare?.('same-001', 'same-001');
|
|
170
170
|
|
|
171
171
|
expect(comparison.delta.successRate).toBe(0);
|
|
172
172
|
expect(comparison.delta.latency).toBe(0);
|
|
@@ -200,7 +200,7 @@ describe('Compare Command', () => {
|
|
|
200
200
|
},
|
|
201
201
|
});
|
|
202
202
|
|
|
203
|
-
await expect(storage.compare
|
|
203
|
+
await expect(storage.compare?.('non-existent', 'exists-001')).rejects.toThrow();
|
|
204
204
|
});
|
|
205
205
|
|
|
206
206
|
it('should throw error for non-existent current', async () => {
|
|
@@ -230,7 +230,7 @@ describe('Compare Command', () => {
|
|
|
230
230
|
},
|
|
231
231
|
});
|
|
232
232
|
|
|
233
|
-
await expect(storage.compare
|
|
233
|
+
await expect(storage.compare?.('exists-002', 'non-existent')).rejects.toThrow();
|
|
234
234
|
});
|
|
235
235
|
});
|
|
236
236
|
});
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
* Integration tests for CLI configuration loading
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
6
6
|
import { writeFile } from 'node:fs/promises';
|
|
7
7
|
import { join } from 'node:path';
|
|
8
8
|
import { loadConfig } from '../../config/loader.js';
|
|
9
|
-
import {
|
|
9
|
+
import { cleanupTestDir, createTestDir } from '../helpers/test-utils.js';
|
|
10
10
|
|
|
11
11
|
describe('Config Loader', () => {
|
|
12
12
|
let testDir: string;
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
* Integration tests for history command
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
6
6
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
7
7
|
import { join } from 'node:path';
|
|
8
|
-
import { createTestDir, cleanupTestDir } from '../helpers/test-utils.js';
|
|
9
8
|
import { createStorage } from '../../utils/storage.js';
|
|
9
|
+
import { cleanupTestDir, createTestDir } from '../helpers/test-utils.js';
|
|
10
10
|
|
|
11
11
|
describe('History Command', () => {
|
|
12
12
|
let testDir: string;
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
* Integration tests for init command
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
6
6
|
import { existsSync } from 'node:fs';
|
|
7
7
|
import { readFile, writeFile } from 'node:fs/promises';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
|
-
import {
|
|
9
|
+
import { cleanupTestDir, createTestDir } from '../helpers/test-utils.js';
|
|
10
10
|
|
|
11
11
|
// Import init command internals for testing
|
|
12
12
|
// We'll test the file creation logic directly
|
|
@@ -126,7 +126,7 @@ SOME_OTHER_VAR=value
|
|
|
126
126
|
});
|
|
127
127
|
|
|
128
128
|
if (missingKeys.length > 0) {
|
|
129
|
-
const newContent = envContent
|
|
129
|
+
const newContent = `${envContent}\n# Added by ArtemisKit\n${missingKeys.join('\n')}\n`;
|
|
130
130
|
await writeFile(envPath, newContent);
|
|
131
131
|
}
|
|
132
132
|
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
* Integration tests for report command
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
6
6
|
import { existsSync } from 'node:fs';
|
|
7
7
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
9
|
import { generateHTMLReport, generateJSONReport } from '@artemiskit/reports';
|
|
10
|
-
import { createTestDir, cleanupTestDir } from '../helpers/test-utils.js';
|
|
11
10
|
import { createStorage } from '../../utils/storage.js';
|
|
11
|
+
import { cleanupTestDir, createTestDir } from '../helpers/test-utils.js';
|
|
12
12
|
|
|
13
13
|
describe('Report Command', () => {
|
|
14
14
|
let testDir: string;
|
|
@@ -4,15 +4,15 @@
|
|
|
4
4
|
|
|
5
5
|
import { describe, expect, it } from 'bun:test';
|
|
6
6
|
import {
|
|
7
|
-
|
|
8
|
-
renderSummaryPanel,
|
|
9
|
-
renderError,
|
|
10
|
-
renderInfoBox,
|
|
7
|
+
colors,
|
|
11
8
|
createSpinner,
|
|
9
|
+
formatDuration,
|
|
12
10
|
icons,
|
|
13
|
-
colors,
|
|
14
11
|
padText,
|
|
15
|
-
|
|
12
|
+
renderError,
|
|
13
|
+
renderInfoBox,
|
|
14
|
+
renderProgressBar,
|
|
15
|
+
renderSummaryPanel,
|
|
16
16
|
} from '../../ui/index.js';
|
|
17
17
|
|
|
18
18
|
describe('UI Components', () => {
|
package/src/commands/compare.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import chalk from 'chalk';
|
|
6
6
|
import { Command } from 'commander';
|
|
7
7
|
import { loadConfig } from '../config/loader.js';
|
|
8
|
-
import { createSpinner,
|
|
8
|
+
import { createSpinner, icons, isTTY, padText, renderError } from '../ui/index.js';
|
|
9
9
|
import { createStorage } from '../utils/storage.js';
|
|
10
10
|
|
|
11
11
|
interface CompareOptions {
|
|
@@ -183,9 +183,7 @@ export function compareCommand(): Command {
|
|
|
183
183
|
|
|
184
184
|
if (hasRegression) {
|
|
185
185
|
console.log(
|
|
186
|
-
`${icons.failed} ${chalk.red('Regression detected!')} `
|
|
187
|
-
`Success rate dropped by ${chalk.bold(Math.abs(delta.successRate * 100).toFixed(1) + '%')} ` +
|
|
188
|
-
chalk.dim(`(threshold: ${threshold * 100}%)`)
|
|
186
|
+
`${icons.failed} ${chalk.red('Regression detected!')} Success rate dropped by ${chalk.bold(`${Math.abs(delta.successRate * 100).toFixed(1)}%`)} ${chalk.dim(`(threshold: ${threshold * 100}%)`)}`
|
|
189
187
|
);
|
|
190
188
|
process.exit(1);
|
|
191
189
|
} else {
|
package/src/commands/history.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import chalk from 'chalk';
|
|
6
6
|
import { Command } from 'commander';
|
|
7
7
|
import { loadConfig } from '../config/loader.js';
|
|
8
|
-
import { createSpinner,
|
|
8
|
+
import { createSpinner, isTTY, padText, renderError } from '../ui/index.js';
|
|
9
9
|
import { createStorage } from '../utils/storage.js';
|
|
10
10
|
|
|
11
11
|
interface HistoryOptions {
|
|
@@ -57,7 +57,7 @@ function renderHistoryTable(
|
|
|
57
57
|
const runIdPad = padText(run.runId, runIdWidth);
|
|
58
58
|
const truncScenario =
|
|
59
59
|
run.scenario.length > scenarioWidth - 2
|
|
60
|
-
? run.scenario.slice(0, scenarioWidth - 3)
|
|
60
|
+
? `${run.scenario.slice(0, scenarioWidth - 3)}…`
|
|
61
61
|
: run.scenario;
|
|
62
62
|
const scenarioPad = padText(truncScenario, scenarioWidth);
|
|
63
63
|
|
package/src/commands/init.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import { existsSync } from 'node:fs';
|
|
6
|
-
import { mkdir, readFile, writeFile
|
|
6
|
+
import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
7
7
|
import { join } from 'node:path';
|
|
8
8
|
import chalk from 'chalk';
|
|
9
9
|
import { Command } from 'commander';
|
|
@@ -180,7 +180,7 @@ async function appendEnvKeys(cwd: string): Promise<{ added: string[]; skipped: s
|
|
|
180
180
|
// Add newline before our content if file exists and doesn't end with newline
|
|
181
181
|
const prefix =
|
|
182
182
|
existingContent && !existingContent.endsWith('\n') ? '\n\n' : existingContent ? '\n' : '';
|
|
183
|
-
await appendFile(envPath, prefix + linesToAdd.join('\n')
|
|
183
|
+
await appendFile(envPath, `${prefix + linesToAdd.join('\n')}\n`);
|
|
184
184
|
}
|
|
185
185
|
|
|
186
186
|
return { added, skipped };
|
package/src/commands/redteam.ts
CHANGED
|
@@ -35,13 +35,13 @@ import { nanoid } from 'nanoid';
|
|
|
35
35
|
import { loadConfig } from '../config/loader.js';
|
|
36
36
|
import {
|
|
37
37
|
createSpinner,
|
|
38
|
-
|
|
38
|
+
getProviderErrorContext,
|
|
39
|
+
icons,
|
|
40
|
+
isTTY,
|
|
39
41
|
renderError,
|
|
40
42
|
renderInfoBox,
|
|
41
43
|
renderProgressBar,
|
|
42
|
-
|
|
43
|
-
isTTY,
|
|
44
|
-
icons,
|
|
44
|
+
renderRedteamSummaryPanel,
|
|
45
45
|
} from '../ui/index.js';
|
|
46
46
|
import {
|
|
47
47
|
buildAdapterConfig,
|
|
@@ -213,7 +213,7 @@ export function redteamCommand(): Command {
|
|
|
213
213
|
|
|
214
214
|
// Clear progress line
|
|
215
215
|
if (isTTY) {
|
|
216
|
-
process.stdout.write(
|
|
216
|
+
process.stdout.write(`\r${' '.repeat(60)}\r`);
|
|
217
217
|
}
|
|
218
218
|
|
|
219
219
|
// Display status with appropriate icon
|
|
@@ -267,7 +267,7 @@ export function redteamCommand(): Command {
|
|
|
267
267
|
|
|
268
268
|
// Clear progress line
|
|
269
269
|
if (isTTY) {
|
|
270
|
-
process.stdout.write(
|
|
270
|
+
process.stdout.write(`\r${' '.repeat(60)}\r`);
|
|
271
271
|
}
|
|
272
272
|
|
|
273
273
|
// Apply redaction to prompt even for errors/blocked
|
package/src/commands/report.ts
CHANGED
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
} from '@artemiskit/reports';
|
|
14
14
|
import { Command } from 'commander';
|
|
15
15
|
import { loadConfig } from '../config/loader.js';
|
|
16
|
-
import { createSpinner, renderError, renderInfoBox
|
|
16
|
+
import { createSpinner, icons, renderError, renderInfoBox } from '../ui/index.js';
|
|
17
17
|
import { createStorage } from '../utils/storage.js';
|
|
18
18
|
|
|
19
19
|
interface ReportOptions {
|
|
@@ -96,7 +96,7 @@ export function reportCommand(): Command {
|
|
|
96
96
|
const htmlPath = join(outputDir, `${runId}.html`);
|
|
97
97
|
await writeFile(htmlPath, html);
|
|
98
98
|
generatedFiles.push(htmlPath);
|
|
99
|
-
spinner.succeed(
|
|
99
|
+
spinner.succeed('Generated HTML report');
|
|
100
100
|
}
|
|
101
101
|
|
|
102
102
|
if (format === 'json' || format === 'both') {
|
|
@@ -105,7 +105,7 @@ export function reportCommand(): Command {
|
|
|
105
105
|
const jsonPath = join(outputDir, `${runId}.json`);
|
|
106
106
|
await writeFile(jsonPath, json);
|
|
107
107
|
generatedFiles.push(jsonPath);
|
|
108
|
-
spinner.succeed(
|
|
108
|
+
spinner.succeed('Generated JSON report');
|
|
109
109
|
}
|
|
110
110
|
|
|
111
111
|
// Show success panel
|
package/src/commands/run.ts
CHANGED
|
@@ -13,14 +13,14 @@ import { Command } from 'commander';
|
|
|
13
13
|
import { loadConfig } from '../config/loader.js';
|
|
14
14
|
import {
|
|
15
15
|
createSpinner,
|
|
16
|
+
formatDuration,
|
|
17
|
+
getProviderErrorContext,
|
|
16
18
|
icons,
|
|
19
|
+
isTTY,
|
|
20
|
+
padText,
|
|
17
21
|
renderError,
|
|
18
22
|
renderProgressBar,
|
|
19
23
|
renderSummaryPanel,
|
|
20
|
-
getProviderErrorContext,
|
|
21
|
-
formatDuration,
|
|
22
|
-
padText,
|
|
23
|
-
isTTY,
|
|
24
24
|
} from '../ui/index.js';
|
|
25
25
|
import {
|
|
26
26
|
buildAdapterConfig,
|
package/src/commands/stress.ts
CHANGED
|
@@ -21,14 +21,14 @@ import { Command } from 'commander';
|
|
|
21
21
|
import { nanoid } from 'nanoid';
|
|
22
22
|
import { loadConfig } from '../config/loader.js';
|
|
23
23
|
import {
|
|
24
|
+
colors,
|
|
24
25
|
createSpinner,
|
|
25
|
-
|
|
26
|
+
getProviderErrorContext,
|
|
27
|
+
isTTY,
|
|
26
28
|
renderError,
|
|
27
29
|
renderInfoBox,
|
|
28
30
|
renderProgressBar,
|
|
29
|
-
|
|
30
|
-
isTTY,
|
|
31
|
-
colors,
|
|
31
|
+
renderStressSummaryPanel,
|
|
32
32
|
} from '../ui/index.js';
|
|
33
33
|
import {
|
|
34
34
|
buildAdapterConfig,
|
package/src/ui/errors.ts
CHANGED
|
@@ -148,7 +148,7 @@ export function renderWarning(title: string, message: string, suggestions?: stri
|
|
|
148
148
|
currentLine += (currentLine.length > 2 ? ' ' : '') + word;
|
|
149
149
|
} else {
|
|
150
150
|
lines.push(chalk.yellow('│') + padText(currentLine, width - 2) + chalk.yellow('│'));
|
|
151
|
-
currentLine =
|
|
151
|
+
currentLine = ` ${word}`;
|
|
152
152
|
}
|
|
153
153
|
}
|
|
154
154
|
if (currentLine.length > 2) {
|
package/src/ui/live-status.ts
CHANGED
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
|
|
5
5
|
import chalk from 'chalk';
|
|
6
6
|
import ora, { type Ora } from 'ora';
|
|
7
|
-
import { isTTY } from './utils.js';
|
|
8
7
|
import { icons } from './colors.js';
|
|
9
8
|
import { renderProgressBar } from './progress.js';
|
|
9
|
+
import { isTTY } from './utils.js';
|
|
10
10
|
|
|
11
11
|
export type TestStatus = 'pending' | 'running' | 'passed' | 'failed' | 'skipped';
|
|
12
12
|
|
package/src/ui/panels.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import chalk from 'chalk';
|
|
6
6
|
import { formatPercentage, icons } from './colors.js';
|
|
7
|
-
import {
|
|
7
|
+
import { centerText, formatDuration, isTTY, padText } from './utils.js';
|
|
8
8
|
|
|
9
9
|
export interface SummaryData {
|
|
10
10
|
passed: number;
|
|
@@ -180,7 +180,7 @@ export function renderRedteamSummaryPanel(data: RedteamSummaryData): string {
|
|
|
180
180
|
chalk.magenta('║'),
|
|
181
181
|
chalk.magenta(`╠${border}╣`),
|
|
182
182
|
chalk.magenta('║') +
|
|
183
|
-
padText(` Defense Rate: ${defenseColor(data.defenseRate.toFixed(1)
|
|
183
|
+
padText(` Defense Rate: ${defenseColor(`${data.defenseRate.toFixed(1)}%`)}`, width - 2) +
|
|
184
184
|
chalk.magenta('║'),
|
|
185
185
|
chalk.magenta(`╚${border}╝`),
|
|
186
186
|
];
|
package/src/ui/progress.ts
CHANGED
|
@@ -108,7 +108,7 @@ export class ProgressBar {
|
|
|
108
108
|
if (isTTY) {
|
|
109
109
|
// Clear previous line and write new output
|
|
110
110
|
const clearLength = stripAnsi(this.lastOutput).length;
|
|
111
|
-
process.stdout.write(
|
|
111
|
+
process.stdout.write(`\r${' '.repeat(clearLength)}\r`);
|
|
112
112
|
process.stdout.write(output);
|
|
113
113
|
this.lastOutput = output;
|
|
114
114
|
}
|
package/src/ui/utils.ts
CHANGED
|
@@ -46,7 +46,8 @@ export function padText(
|
|
|
46
46
|
|
|
47
47
|
if (align === 'center') {
|
|
48
48
|
return centerText(text, width);
|
|
49
|
-
}
|
|
49
|
+
}
|
|
50
|
+
if (align === 'right') {
|
|
50
51
|
return ' '.repeat(paddingNeeded) + text;
|
|
51
52
|
}
|
|
52
53
|
return text + ' '.repeat(paddingNeeded);
|
|
@@ -56,7 +57,7 @@ export function padText(
|
|
|
56
57
|
* Strip ANSI escape codes from string (for length calculations)
|
|
57
58
|
*/
|
|
58
59
|
export function stripAnsi(str: string): string {
|
|
59
|
-
//
|
|
60
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: ANSI escape codes require control characters
|
|
60
61
|
return str.replace(/\x1B\[[0-9;]*[a-zA-Z]/g, '');
|
|
61
62
|
}
|
|
62
63
|
|
|
@@ -66,7 +67,7 @@ export function stripAnsi(str: string): string {
|
|
|
66
67
|
export function truncate(text: string, maxWidth: number): string {
|
|
67
68
|
const visibleLength = stripAnsi(text).length;
|
|
68
69
|
if (visibleLength <= maxWidth) return text;
|
|
69
|
-
return text.slice(0, maxWidth - 1)
|
|
70
|
+
return `${text.slice(0, maxWidth - 1)}…`;
|
|
70
71
|
}
|
|
71
72
|
|
|
72
73
|
/**
|