@evalguardai/cli 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -0
- package/bin/evalguard.js +2 -0
- package/dist/commands/compare.d.ts +6 -0
- package/dist/commands/compare.d.ts.map +1 -0
- package/dist/commands/compare.js +109 -0
- package/dist/commands/compare.js.map +1 -0
- package/dist/commands/compliance-check.d.ts +18 -0
- package/dist/commands/compliance-check.d.ts.map +1 -0
- package/dist/commands/compliance-check.js +474 -0
- package/dist/commands/compliance-check.js.map +1 -0
- package/dist/commands/debug.d.ts +6 -0
- package/dist/commands/debug.d.ts.map +1 -0
- package/dist/commands/debug.js +151 -0
- package/dist/commands/debug.js.map +1 -0
- package/dist/commands/delete.d.ts +6 -0
- package/dist/commands/delete.d.ts.map +1 -0
- package/dist/commands/delete.js +105 -0
- package/dist/commands/delete.js.map +1 -0
- package/dist/commands/eval-local.d.ts +7 -0
- package/dist/commands/eval-local.d.ts.map +1 -0
- package/dist/commands/eval-local.js +376 -0
- package/dist/commands/eval-local.js.map +1 -0
- package/dist/commands/export.d.ts +6 -0
- package/dist/commands/export.d.ts.map +1 -0
- package/dist/commands/export.js +135 -0
- package/dist/commands/export.js.map +1 -0
- package/dist/commands/firewall.d.ts +6 -0
- package/dist/commands/firewall.d.ts.map +1 -0
- package/dist/commands/firewall.js +56 -0
- package/dist/commands/firewall.js.map +1 -0
- package/dist/commands/gate.d.ts +14 -0
- package/dist/commands/gate.d.ts.map +1 -0
- package/dist/commands/gate.js +232 -0
- package/dist/commands/gate.js.map +1 -0
- package/dist/commands/generate.d.ts +7 -0
- package/dist/commands/generate.d.ts.map +1 -0
- package/dist/commands/generate.js +182 -0
- package/dist/commands/generate.js.map +1 -0
- package/dist/commands/history.d.ts +7 -0
- package/dist/commands/history.d.ts.map +1 -0
- package/dist/commands/history.js +59 -0
- package/dist/commands/history.js.map +1 -0
- package/dist/commands/import-promptfoo.d.ts +7 -0
- package/dist/commands/import-promptfoo.d.ts.map +1 -0
- package/dist/commands/import-promptfoo.js +218 -0
- package/dist/commands/import-promptfoo.js.map +1 -0
- package/dist/commands/index.d.ts +21 -0
- package/dist/commands/index.d.ts.map +1 -0
- package/dist/commands/index.js +21 -0
- package/dist/commands/index.js.map +1 -0
- package/dist/commands/init.d.ts +7 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +509 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/list.d.ts +10 -0
- package/dist/commands/list.d.ts.map +1 -0
- package/dist/commands/list.js +165 -0
- package/dist/commands/list.js.map +1 -0
- package/dist/commands/logs.d.ts +6 -0
- package/dist/commands/logs.d.ts.map +1 -0
- package/dist/commands/logs.js +153 -0
- package/dist/commands/logs.js.map +1 -0
- package/dist/commands/model-scan.d.ts +7 -0
- package/dist/commands/model-scan.d.ts.map +1 -0
- package/dist/commands/model-scan.js +276 -0
- package/dist/commands/model-scan.js.map +1 -0
- package/dist/commands/retry.d.ts +6 -0
- package/dist/commands/retry.d.ts.map +1 -0
- package/dist/commands/retry.js +83 -0
- package/dist/commands/retry.js.map +1 -0
- package/dist/commands/scan-local.d.ts +6 -0
- package/dist/commands/scan-local.d.ts.map +1 -0
- package/dist/commands/scan-local.js +138 -0
- package/dist/commands/scan-local.js.map +1 -0
- package/dist/commands/share.d.ts +6 -0
- package/dist/commands/share.d.ts.map +1 -0
- package/dist/commands/share.js +74 -0
- package/dist/commands/share.js.map +1 -0
- package/dist/commands/store.d.ts +23 -0
- package/dist/commands/store.d.ts.map +1 -0
- package/dist/commands/store.js +54 -0
- package/dist/commands/store.js.map +1 -0
- package/dist/commands/validate.d.ts +6 -0
- package/dist/commands/validate.d.ts.map +1 -0
- package/dist/commands/validate.js +171 -0
- package/dist/commands/validate.js.map +1 -0
- package/dist/commands/watch.d.ts +6 -0
- package/dist/commands/watch.d.ts.map +1 -0
- package/dist/commands/watch.js +92 -0
- package/dist/commands/watch.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +342 -0
- package/dist/index.js.map +1 -0
- package/package.json +73 -0
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import chalk from "chalk";
|
|
2
|
+
import ora from "ora";
|
|
3
|
+
import * as fs from "fs";
|
|
4
|
+
import * as path from "path";
|
|
5
|
+
export function registerGenerate(program) {
|
|
6
|
+
const gen = program
|
|
7
|
+
.command("generate")
|
|
8
|
+
.description("Generate synthetic test data");
|
|
9
|
+
// ─── generate tests ───
|
|
10
|
+
gen
|
|
11
|
+
.command("tests")
|
|
12
|
+
.description("Generate test cases from a description")
|
|
13
|
+
.argument("<description>", "What to test (e.g., 'customer support chatbot')")
|
|
14
|
+
.option("-n, --count <n>", "Number of test cases", "10")
|
|
15
|
+
.option("--model <model>", "LLM model for generation", "gpt-4o")
|
|
16
|
+
.option("--provider <provider>", "Provider name", "openai")
|
|
17
|
+
.option("--strategies <list>", "Evolution strategies (comma-separated)", "")
|
|
18
|
+
.option("--output <file>", "Output file path", "generated-tests.json")
|
|
19
|
+
.action(async (description, opts) => {
|
|
20
|
+
const { generateTestCases, createProvider } = await import("@evalguard/core");
|
|
21
|
+
const spinner = ora("Generating test cases...").start();
|
|
22
|
+
try {
|
|
23
|
+
const callLLM = await makeCallLLM(opts.provider, opts.model, createProvider);
|
|
24
|
+
const strategies = opts.strategies
|
|
25
|
+
? opts.strategies.split(",").map((s) => s.trim())
|
|
26
|
+
: [];
|
|
27
|
+
const result = await generateTestCases({
|
|
28
|
+
description,
|
|
29
|
+
count: parseInt(opts.count, 10),
|
|
30
|
+
callLLM,
|
|
31
|
+
strategies: strategies.length > 0 ? strategies : undefined,
|
|
32
|
+
});
|
|
33
|
+
spinner.stop();
|
|
34
|
+
console.log();
|
|
35
|
+
console.log(chalk.bold(` Generated ${result.testCases.length} test cases`));
|
|
36
|
+
console.log(chalk.dim(` Strategies: ${result.strategies.join(", ")}`));
|
|
37
|
+
console.log(chalk.dim(` Duration: ${result.duration}ms`));
|
|
38
|
+
if (result.totalFiltered > 0) {
|
|
39
|
+
console.log(chalk.dim(` Filtered: ${result.totalFiltered} duplicates removed`));
|
|
40
|
+
}
|
|
41
|
+
// Preview first 3
|
|
42
|
+
console.log();
|
|
43
|
+
for (const tc of result.testCases.slice(0, 3)) {
|
|
44
|
+
console.log(` ${chalk.cyan("Q:")} ${tc.input.slice(0, 80)}${tc.input.length > 80 ? "..." : ""}`);
|
|
45
|
+
console.log(` ${chalk.green("A:")} ${(tc.expected ?? "N/A").slice(0, 80)}`);
|
|
46
|
+
console.log();
|
|
47
|
+
}
|
|
48
|
+
if (result.testCases.length > 3) {
|
|
49
|
+
console.log(chalk.dim(` ... and ${result.testCases.length - 3} more`));
|
|
50
|
+
}
|
|
51
|
+
// Save output
|
|
52
|
+
const outputData = {
|
|
53
|
+
description,
|
|
54
|
+
model: opts.model,
|
|
55
|
+
generatedAt: new Date().toISOString(),
|
|
56
|
+
cases: result.testCases.map((tc) => ({
|
|
57
|
+
input: tc.input,
|
|
58
|
+
expectedOutput: tc.expected,
|
|
59
|
+
strategy: tc.strategy,
|
|
60
|
+
metadata: tc.metadata,
|
|
61
|
+
})),
|
|
62
|
+
};
|
|
63
|
+
fs.writeFileSync(opts.output, JSON.stringify(outputData, null, 2));
|
|
64
|
+
console.log(` ${chalk.green("✓")} Saved to ${chalk.cyan(opts.output)}`);
|
|
65
|
+
}
|
|
66
|
+
catch (err) {
|
|
67
|
+
spinner.fail(`Generation failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
68
|
+
process.exit(1);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
// ─── generate assertions ───
|
|
72
|
+
gen
|
|
73
|
+
.command("assertions")
|
|
74
|
+
.description("Auto-generate assertions for existing test cases")
|
|
75
|
+
.argument("<file>", "Path to test cases JSON file")
|
|
76
|
+
.option("--model <model>", "LLM model", "gpt-4o")
|
|
77
|
+
.option("--provider <provider>", "Provider name", "openai")
|
|
78
|
+
.option("--output <file>", "Output file path", "generated-assertions.json")
|
|
79
|
+
.action(async (file, opts) => {
|
|
80
|
+
const { generateAssertions, createProvider } = await import("@evalguard/core");
|
|
81
|
+
const spinner = ora("Generating assertions...").start();
|
|
82
|
+
try {
|
|
83
|
+
const filePath = path.resolve(file);
|
|
84
|
+
if (!fs.existsSync(filePath)) {
|
|
85
|
+
spinner.fail(`File not found: ${filePath}`);
|
|
86
|
+
process.exit(1);
|
|
87
|
+
}
|
|
88
|
+
const data = JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
|
89
|
+
const cases = (data.cases ?? data).map((c) => ({
|
|
90
|
+
input: c.input,
|
|
91
|
+
expected: c.expectedOutput ?? c.expected,
|
|
92
|
+
strategy: c.strategy ?? "base",
|
|
93
|
+
}));
|
|
94
|
+
const callLLM = await makeCallLLM(opts.provider, opts.model, createProvider);
|
|
95
|
+
const results = [];
|
|
96
|
+
for (let i = 0; i < cases.length; i++) {
|
|
97
|
+
spinner.text = `Generating assertions for case ${i + 1}/${cases.length}...`;
|
|
98
|
+
const assertions = await generateAssertions({ testCase: cases[i], callLLM });
|
|
99
|
+
results.push({ input: cases[i].input, assertions });
|
|
100
|
+
}
|
|
101
|
+
spinner.stop();
|
|
102
|
+
console.log();
|
|
103
|
+
console.log(chalk.bold(` Generated assertions for ${results.length} test cases`));
|
|
104
|
+
fs.writeFileSync(opts.output, JSON.stringify(results, null, 2));
|
|
105
|
+
console.log(` ${chalk.green("✓")} Saved to ${chalk.cyan(opts.output)}`);
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
spinner.fail(`Failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
109
|
+
process.exit(1);
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
// ─── generate rag ───
|
|
113
|
+
gen
|
|
114
|
+
.command("rag")
|
|
115
|
+
.description("Generate RAG test cases from documents")
|
|
116
|
+
.argument("<dir>", "Directory containing .txt/.md documents")
|
|
117
|
+
.option("-n, --count <n>", "Number of test cases", "10")
|
|
118
|
+
.option("--model <model>", "LLM model", "gpt-4o")
|
|
119
|
+
.option("--provider <provider>", "Provider name", "openai")
|
|
120
|
+
.option("--types <list>", "Question types (factual,inferential,comparative,multi-hop)", "factual,inferential,comparative")
|
|
121
|
+
.option("--output <file>", "Output file path", "rag-tests.json")
|
|
122
|
+
.action(async (dir, opts) => {
|
|
123
|
+
const { generateRAGTests, createProvider } = await import("@evalguard/core");
|
|
124
|
+
const spinner = ora("Loading documents...").start();
|
|
125
|
+
try {
|
|
126
|
+
const dirPath = path.resolve(dir);
|
|
127
|
+
if (!fs.existsSync(dirPath)) {
|
|
128
|
+
spinner.fail(`Directory not found: ${dirPath}`);
|
|
129
|
+
process.exit(1);
|
|
130
|
+
}
|
|
131
|
+
const files = fs.readdirSync(dirPath).filter((f) => f.endsWith(".txt") || f.endsWith(".md"));
|
|
132
|
+
if (files.length === 0) {
|
|
133
|
+
spinner.fail("No .txt or .md files found in directory.");
|
|
134
|
+
process.exit(1);
|
|
135
|
+
}
|
|
136
|
+
const documents = files.map((f) => fs.readFileSync(path.join(dirPath, f), "utf-8"));
|
|
137
|
+
spinner.text = `Generating RAG tests from ${files.length} documents...`;
|
|
138
|
+
const callLLM = await makeCallLLM(opts.provider, opts.model, createProvider);
|
|
139
|
+
const questionTypes = opts.types.split(",").map((t) => t.trim());
|
|
140
|
+
const tests = await generateRAGTests({
|
|
141
|
+
documents,
|
|
142
|
+
count: parseInt(opts.count, 10),
|
|
143
|
+
callLLM,
|
|
144
|
+
questionTypes,
|
|
145
|
+
});
|
|
146
|
+
spinner.stop();
|
|
147
|
+
console.log();
|
|
148
|
+
console.log(chalk.bold(` Generated ${tests.length} RAG test cases from ${files.length} documents`));
|
|
149
|
+
const outputData = {
|
|
150
|
+
sourceDir: dir,
|
|
151
|
+
sourceFiles: files,
|
|
152
|
+
generatedAt: new Date().toISOString(),
|
|
153
|
+
cases: tests.map((tc) => ({
|
|
154
|
+
input: tc.input,
|
|
155
|
+
expectedOutput: tc.expected,
|
|
156
|
+
strategy: tc.strategy,
|
|
157
|
+
metadata: tc.metadata,
|
|
158
|
+
})),
|
|
159
|
+
};
|
|
160
|
+
fs.writeFileSync(opts.output, JSON.stringify(outputData, null, 2));
|
|
161
|
+
console.log(` ${chalk.green("✓")} Saved to ${chalk.cyan(opts.output)}`);
|
|
162
|
+
}
|
|
163
|
+
catch (err) {
|
|
164
|
+
spinner.fail(`Failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
165
|
+
process.exit(1);
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
async function makeCallLLM(providerName, model, createProvider) {
|
|
170
|
+
const envMap = {
|
|
171
|
+
openai: "OPENAI_API_KEY", anthropic: "ANTHROPIC_API_KEY", gemini: "GEMINI_API_KEY",
|
|
172
|
+
mistral: "MISTRAL_API_KEY", groq: "GROQ_API_KEY", deepseek: "DEEPSEEK_API_KEY",
|
|
173
|
+
};
|
|
174
|
+
const envKey = envMap[providerName] ?? `${providerName.toUpperCase().replace(/-/g, "_")}_API_KEY`;
|
|
175
|
+
const apiKey = process.env[envKey] ?? process.env.EVALGUARD_PROVIDER_KEY ?? "";
|
|
176
|
+
const provider = createProvider(providerName, apiKey);
|
|
177
|
+
return async (prompt) => {
|
|
178
|
+
const response = await provider.chat([{ role: "user", content: prompt }], { model });
|
|
179
|
+
return response.content;
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
//# sourceMappingURL=generate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../src/commands/generate.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAK7B,MAAM,UAAU,gBAAgB,CAAC,OAAgB;IAC/C,MAAM,GAAG,GAAG,OAAO;SAChB,OAAO,CAAC,UAAU,CAAC;SACnB,WAAW,CAAC,8BAA8B,CAAC,CAAC;IAE/C,yBAAyB;IACzB,GAAG;SACA,OAAO,CAAC,OAAO,CAAC;SAChB,WAAW,CAAC,wCAAwC,CAAC;SACrD,QAAQ,CAAC,eAAe,EAAE,iDAAiD,CAAC;SAC5E,MAAM,CAAC,iBAAiB,EAAE,sBAAsB,EAAE,IAAI,CAAC;SACvD,MAAM,CAAC,iBAAiB,EAAE,0BAA0B,EAAE,QAAQ,CAAC;SAC/D,MAAM,CAAC,uBAAuB,EAAE,eAAe,EAAE,QAAQ,CAAC;SAC1D,MAAM,CAAC,qBAAqB,EAAE,wCAAwC,EAAE,EAAE,CAAC;SAC3E,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,sBAAsB,CAAC;SACrE,MAAM,CAAC,KAAK,EAAE,WAAmB,EAAE,IAA4F,EAAE,EAAE;QAClI,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAQ,CAAC;QAErF,MAAM,OAAO,GAAG,GAAG,CAAC,0BAA0B,CAAC,CAAC,KAAK,EAAE,CAAC;QAExD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;YAC7E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU;gBAChC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAwB;gBAChF,CAAC,CAAC,EAAE,CAAC;YAEP,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC;gBACrC,WAAW;gBACX,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;gBAC/B,OAAO;gBACP,UAAU,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;aAC3D,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,EAAE,CAAC;YAEf,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,SAAS,CAAC,MAAM,aAAa,CAAC,CAAC,CAAC;YAC7E,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACxE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;YAC3D,IAAI,MAAM,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,aAAa,qBAAqB,CAAC,CAAC,CAAC;YACnF,CAAC;YAED,kBAAkB;YAClB,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClG,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC7E,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,CAAC;YACD,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1E,CAAC;YAED,cAAc;YACd,MAAM,UAAU,GAAG;gBACjB,WAAW;gBACX,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,CAAC;oBACxC,KAAK,EAAE,EAAE,CAAC,KAAK;oBACf,cAAc,EAAE,EAAE,CAAC,QAAQ;oBAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ;oBACrB,QAAQ,EAAE,EAAE,CAAC,QAAQ;iBACtB,CAAC,CAAC;aACJ,CAAC;YAEF,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,sBAAsB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,8BAA8B;IAC9B,GAAG;SACA,OAAO,CAAC,YAAY,CAAC;SACrB,WAAW,CAAC,kDAAkD,CAAC;SAC/D,QAAQ,CAAC,QAAQ,EAAE,8BAA8B,CAAC;SAClD,MAAM,CAAC,iBAAiB,EAAE,WAAW,EAAE,QAAQ,CAAC;SAChD,MAAM,CAAC,uBAAuB,EAAE,eAAe,EAAE,QAAQ,CAAC;SAC1D,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,2BAA2B,CAAC;SAC1E,MAAM,CAAC,KAAK,EAAE,IAAY,EAAE,IAAyD,EAAE,EAAE;QACxF,MAAM,EAAE,kBAAkB,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAQ,CAAC;QAEtF,MAAM,OAAO,GAAG,GAAG,CAAC,0BAA0B,CAAC,CAAC,KAAK,EAAE,CAAC;QAExD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7B,OAAO,CAAC,IAAI,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;YAC5D,MAAM,KAAK,GAAwB,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;gBACvE,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,QAAQ,EAAE,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,QAAQ;gBACxC,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,MAAM;aAC/B,CAAC,CAAC,CAAC;YAEJ,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;YAC7E,MAAM,OAAO,GAAgD,EAAE,CAAC;YAEhE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,GAAG,kCAAkC,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAC5E,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC7E,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;YACtD,CAAC;YAED,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,8BAA8B,OAAO,CAAC,MAAM,aAAa,CAAC,CAAC,CAAC;YAEnF,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAChE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,WAAW,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,uBAAuB;IACvB,GAAG;SACA,OAAO,CAAC,KAAK,CAAC;SACd,WAAW,CAAC,wCAAwC,CAAC;SACrD,QAAQ,CAAC,OAAO,EAAE,yCAAyC,CAAC;SAC5D,MAAM,CAAC,iBAAiB,EAAE,sBAAsB,EAAE,IAAI,CAAC;SACvD,MAAM,CAAC,iBAAiB,EAAE,WAAW,EAAE,QAAQ,CAAC;SAChD,MAAM,CAAC,uBAAuB,EAAE,eAAe,EAAE,QAAQ,CAAC;SAC1D,MAAM,CAAC,gBAAgB,EAAE,4DAA4D,EAAE,iCAAiC,CAAC;SACzH,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,EAAE,gBAAgB,CAAC;SAC/D,MAAM,CAAC,KAAK,EAAE,GAAW,EAAE,IAAuF,EAAE,EAAE;QACrH,MAAM,EAAE,gBAAgB,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAQ,CAAC;QAEpF,MAAM,OAAO,GAAG,GAAG,CAAC,sBAAsB,CAAC,CAAC,KAAK,EAAE,CAAC;QAEpD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC,wBAAwB,OAAO,EAAE,CAAC,CAAC;gBAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YACrG,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YAC5F,OAAO,CAAC,IAAI,GAAG,6BAA6B,KAAK,CAAC,MAAM,eAAe,CAAC;YAExE,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;YAC7E,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAU,CAAC;YAElF,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC;gBACnC,SAAS;gBACT,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;gBAC/B,OAAO;gBACP,aAAa;aACd,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,MAAM,wBAAwB,KAAK,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC;YAErG,MAAM,UAAU,GAAG;gBACjB,SAAS,EAAE,GAAG;gBACd,WAAW,EAAE,KAAK;gBAClB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,CAAC;oBAC7B,KAAK,EAAE,EAAE,CAAC,KAAK;oBACf,cAAc,EAAE,EAAE,CAAC,QAAQ;oBAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ;oBACrB,QAAQ,EAAE,EAAE,CAAC,QAAQ;iBACtB,CAAC,CAAC;aACJ,CAAC;YAEF,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,WAAW,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,YAAoB,EAAE,KAAa,EAAE,cAAmB;IACjF,MAAM,MAAM,GAA2B;QACrC,MAAM,EAAE,gBAAgB,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,EAAE,gBAAgB;QAClF,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,kBAAkB;KAC/E,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,UAAU,CAAC;IAClG,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,EAAE,CAAC;IAE/E,MAAM,QAAQ,GAAG,cAAc,CAAC,YAAmB,EAAE,MAAM,CAAC,CAAC;IAC7D,OAAO,KAAK,EAAE,MAAc,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACrF,OAAO,QAAQ,CAAC,OAAO,CAAC;IAC1B,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `evalguard history` — View local eval/scan run history
|
|
3
|
+
* Stored in ~/.evalguard/results.json (SQLite-like local persistence)
|
|
4
|
+
*/
|
|
5
|
+
import { Command } from "commander";
|
|
6
|
+
export declare function registerHistory(program: Command): void;
|
|
7
|
+
//# sourceMappingURL=history.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../src/commands/history.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,wBAAgB,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CA2DtD"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import chalk from "chalk";
|
|
2
|
+
import { listRuns, getRun } from "./store.js";
|
|
3
|
+
export function registerHistory(program) {
|
|
4
|
+
program
|
|
5
|
+
.command("history")
|
|
6
|
+
.description("View local eval/scan run history (offline storage)")
|
|
7
|
+
.option("-t, --type <type>", "Filter by type: eval or scan")
|
|
8
|
+
.option("-n, --limit <number>", "Number of runs to show", "20")
|
|
9
|
+
.option("--id <id>", "Show details for a specific run")
|
|
10
|
+
.option("--json", "Output as JSON", false)
|
|
11
|
+
.action((opts) => {
|
|
12
|
+
if (opts.id) {
|
|
13
|
+
const run = getRun(opts.id);
|
|
14
|
+
if (!run) {
|
|
15
|
+
console.error(chalk.red(`Run not found: ${opts.id}`));
|
|
16
|
+
process.exit(1);
|
|
17
|
+
}
|
|
18
|
+
if (opts.json) {
|
|
19
|
+
console.log(JSON.stringify(run, null, 2));
|
|
20
|
+
}
|
|
21
|
+
else {
|
|
22
|
+
console.log();
|
|
23
|
+
console.log(chalk.bold(` ${run.name}`));
|
|
24
|
+
console.log(chalk.dim(` ID: ${run.id}`));
|
|
25
|
+
console.log(chalk.dim(` Type: ${run.type} | Model: ${run.model} | Provider: ${run.provider}`));
|
|
26
|
+
console.log(chalk.dim(` Time: ${run.timestamp}`));
|
|
27
|
+
console.log();
|
|
28
|
+
const color = run.passRate >= 0.8 ? chalk.green : run.passRate >= 0.5 ? chalk.yellow : chalk.red;
|
|
29
|
+
console.log(` Pass rate: ${color((run.passRate * 100).toFixed(1) + "%")}`);
|
|
30
|
+
console.log(` Results: ${chalk.green(run.passed + " passed")} / ${chalk.red(run.failed + " failed")} / ${run.total} total`);
|
|
31
|
+
console.log(` Score: ${run.score.toFixed(2)}/${run.maxScore}`);
|
|
32
|
+
console.log(` Latency: ${run.latencyMs}ms`);
|
|
33
|
+
console.log();
|
|
34
|
+
}
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
const type = opts.type;
|
|
38
|
+
const runs = listRuns(type, parseInt(opts.limit));
|
|
39
|
+
if (runs.length === 0) {
|
|
40
|
+
console.log(chalk.dim("\n No runs found. Run `evalguard eval:local` or `evalguard gate` first.\n"));
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
if (opts.json) {
|
|
44
|
+
console.log(JSON.stringify(runs, null, 2));
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
console.log();
|
|
48
|
+
console.log(chalk.bold(` Recent Runs (${runs.length})`));
|
|
49
|
+
console.log(chalk.dim(" " + "-".repeat(80)));
|
|
50
|
+
for (const run of runs) {
|
|
51
|
+
const color = run.passRate >= 0.8 ? chalk.green : run.passRate >= 0.5 ? chalk.yellow : chalk.red;
|
|
52
|
+
const icon = run.passRate >= 0.8 ? chalk.green("✓") : chalk.red("✗");
|
|
53
|
+
const date = new Date(run.timestamp).toLocaleString();
|
|
54
|
+
console.log(` ${icon} ${chalk.bold(run.name.slice(0, 30).padEnd(30))} ${color((run.passRate * 100).toFixed(0).padStart(4) + "%")} ${chalk.dim(`${run.passed}/${run.total}`)} ${chalk.dim(run.model.padEnd(15))} ${chalk.dim(date)}`);
|
|
55
|
+
}
|
|
56
|
+
console.log();
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=history.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"history.js","sourceRoot":"","sources":["../../src/commands/history.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,UAAU,eAAe,CAAC,OAAgB;IAC9C,OAAO;SACJ,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,oDAAoD,CAAC;SACjE,MAAM,CAAC,mBAAmB,EAAE,8BAA8B,CAAC;SAC3D,MAAM,CAAC,sBAAsB,EAAE,wBAAwB,EAAE,IAAI,CAAC;SAC9D,MAAM,CAAC,WAAW,EAAE,iCAAiC,CAAC;SACtD,MAAM,CAAC,QAAQ,EAAE,gBAAgB,EAAE,KAAK,CAAC;SACzC,MAAM,CAAC,CAAC,IAAkE,EAAE,EAAE;QAC7E,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5C,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,IAAI,aAAa,GAAG,CAAC,KAAK,gBAAgB,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAChG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,EAAE,CAAC;gBACd,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;gBACjG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC5E,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,KAAK,QAAQ,CAAC,CAAC;gBAC/H,OAAO,CAAC,GAAG,CAAC,gBAAgB,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACpE,OAAO,CAAC,GAAG,CAAC,gBAAgB,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC;gBAC/C,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAmC,CAAC;QACtD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAElD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,4EAA4E,CAAC,CAAC,CAAC;YACrG,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,OAAO;QACT,CAAC;QAED,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAE9C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;YACjG,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACrE,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,cAAc,EAAE,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxO,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `evalguard import:promptfoo <file>` — Import a Promptfoo config and convert
|
|
3
|
+
* it to an EvalGuard config.
|
|
4
|
+
*/
|
|
5
|
+
import { Command } from "commander";
|
|
6
|
+
export declare function registerImportPromptfoo(program: Command): void;
|
|
7
|
+
//# sourceMappingURL=import-promptfoo.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"import-promptfoo.d.ts","sourceRoot":"","sources":["../../src/commands/import-promptfoo.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAuIpC,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAqH9D"}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import chalk from "chalk";
|
|
2
|
+
import * as fs from "fs";
|
|
3
|
+
import * as path from "path";
|
|
4
|
+
// ─── Assertion type mapping ───
|
|
5
|
+
const ASSERTION_MAP = {
|
|
6
|
+
contains: "contains",
|
|
7
|
+
"not-contains": "not-contains",
|
|
8
|
+
icontains: "icontains",
|
|
9
|
+
"contains-any": "contains-any",
|
|
10
|
+
"contains-all": "contains-all",
|
|
11
|
+
equals: "equals",
|
|
12
|
+
"starts-with": "starts-with",
|
|
13
|
+
"ends-with": "ends-with",
|
|
14
|
+
regex: "regex",
|
|
15
|
+
"is-json": "json-valid",
|
|
16
|
+
"is-valid-openai-function-call": "function-call-valid",
|
|
17
|
+
"llm-rubric": "llm-grader",
|
|
18
|
+
"model-graded-closedqa": "llm-grader",
|
|
19
|
+
"model-graded-fact": "factuality",
|
|
20
|
+
factuality: "factuality",
|
|
21
|
+
"answer-relevance": "answer-relevance",
|
|
22
|
+
"context-faithfulness": "context-faithfulness",
|
|
23
|
+
"context-relevance": "context-relevance",
|
|
24
|
+
similar: "semantic-similarity",
|
|
25
|
+
cost: "cost",
|
|
26
|
+
latency: "latency",
|
|
27
|
+
toxicity: "toxicity",
|
|
28
|
+
bias: "bias",
|
|
29
|
+
"is-refusal": "is-refusal",
|
|
30
|
+
};
|
|
31
|
+
function mapProvider(raw) {
|
|
32
|
+
if (typeof raw === "string") {
|
|
33
|
+
// Format: "provider:model" e.g. "openai:gpt-4"
|
|
34
|
+
const parts = raw.split(":");
|
|
35
|
+
if (parts.length >= 2) {
|
|
36
|
+
const providerName = parts[0].replace("openai", "openai").replace("azureopenai", "azure");
|
|
37
|
+
const modelName = parts.slice(1).join(":");
|
|
38
|
+
return { provider: providerName, model: modelName };
|
|
39
|
+
}
|
|
40
|
+
return { provider: raw, model: raw };
|
|
41
|
+
}
|
|
42
|
+
// Object format: { id: "openai:gpt-4", config: { ... } }
|
|
43
|
+
const id = (raw.id ?? raw.label ?? "");
|
|
44
|
+
const mapped = mapProvider(id);
|
|
45
|
+
const config = raw.config;
|
|
46
|
+
if (config) {
|
|
47
|
+
// Strip apiKey from config (EvalGuard reads from env)
|
|
48
|
+
const { apiKey: _ak, ...rest } = config;
|
|
49
|
+
if (Object.keys(rest).length > 0) {
|
|
50
|
+
mapped.config = rest;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return mapped;
|
|
54
|
+
}
|
|
55
|
+
function mapAssertion(assertion) {
|
|
56
|
+
const type = assertion.type;
|
|
57
|
+
if (!type)
|
|
58
|
+
return null;
|
|
59
|
+
// Handle negation prefix
|
|
60
|
+
const isNegated = type.startsWith("not-") && !ASSERTION_MAP[type];
|
|
61
|
+
const baseType = isNegated ? type.replace(/^not-/, "") : type;
|
|
62
|
+
const mapped = ASSERTION_MAP[type] ?? ASSERTION_MAP[baseType];
|
|
63
|
+
if (!mapped) {
|
|
64
|
+
return { scorer: type, value: assertion.value, threshold: assertion.threshold };
|
|
65
|
+
}
|
|
66
|
+
const scorer = { scorer: isNegated ? `not-${mapped}` : mapped };
|
|
67
|
+
if (assertion.value !== undefined)
|
|
68
|
+
scorer.value = assertion.value;
|
|
69
|
+
if (assertion.threshold !== undefined)
|
|
70
|
+
scorer.threshold = assertion.threshold;
|
|
71
|
+
return scorer;
|
|
72
|
+
}
|
|
73
|
+
// ─── YAML parser (minimal — handles promptfoo config structure) ───
|
|
74
|
+
function parseYamlSimple(content) {
|
|
75
|
+
// Use dynamic import for yaml parsing
|
|
76
|
+
// For simplicity, we parse with JSON if possible, or use a basic YAML approach
|
|
77
|
+
try {
|
|
78
|
+
// Try JSON first
|
|
79
|
+
return JSON.parse(content);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// Basic YAML handling — real implementation would use js-yaml
|
|
83
|
+
// We re-export a function that loads yaml dynamically
|
|
84
|
+
throw new Error("YAML parsing requires the 'yaml' package. Install it with: npm install yaml\n" +
|
|
85
|
+
"Alternatively, convert your promptfooconfig.yaml to JSON first.");
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
async function loadYaml(filePath) {
|
|
89
|
+
const content = fs.readFileSync(filePath, "utf-8");
|
|
90
|
+
// Try JSON first
|
|
91
|
+
try {
|
|
92
|
+
return JSON.parse(content);
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
// Dynamically import yaml parser
|
|
96
|
+
try {
|
|
97
|
+
const yamlMod = await import("yaml");
|
|
98
|
+
const parse = yamlMod.parse ?? yamlMod.default?.parse;
|
|
99
|
+
if (parse)
|
|
100
|
+
return parse(content);
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
// Fallback
|
|
104
|
+
}
|
|
105
|
+
return parseYamlSimple(content);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// ─── Main command ───
|
|
109
|
+
export function registerImportPromptfoo(program) {
|
|
110
|
+
program
|
|
111
|
+
.command("import:promptfoo")
|
|
112
|
+
.description("Import a Promptfoo config and convert to EvalGuard format")
|
|
113
|
+
.argument("<file>", "Path to promptfooconfig.yaml or JSON file")
|
|
114
|
+
.option("-o, --output <path>", "Output file path", "evalguard.config.json")
|
|
115
|
+
.option("--dry-run", "Print the converted config without writing to disk")
|
|
116
|
+
.action(async (file, opts) => {
|
|
117
|
+
const filePath = path.resolve(file);
|
|
118
|
+
if (!fs.existsSync(filePath)) {
|
|
119
|
+
console.error(chalk.red(`File not found: ${filePath}`));
|
|
120
|
+
process.exit(1);
|
|
121
|
+
}
|
|
122
|
+
console.log();
|
|
123
|
+
console.log(chalk.bold.cyan(" EvalGuard") + chalk.dim(" — Promptfoo Import"));
|
|
124
|
+
console.log(chalk.dim(" ─────────────────────────────────────────────"));
|
|
125
|
+
console.log();
|
|
126
|
+
console.log(` Reading ${chalk.cyan(path.basename(filePath))}...`);
|
|
127
|
+
let config;
|
|
128
|
+
try {
|
|
129
|
+
config = await loadYaml(filePath);
|
|
130
|
+
}
|
|
131
|
+
catch (err) {
|
|
132
|
+
console.error(chalk.red(` Failed to parse: ${err.message}`));
|
|
133
|
+
process.exit(1);
|
|
134
|
+
}
|
|
135
|
+
// ── Convert providers ──
|
|
136
|
+
const rawProviders = (config.providers ?? []);
|
|
137
|
+
const providers = rawProviders.map(mapProvider);
|
|
138
|
+
// ── Convert prompts ──
|
|
139
|
+
const prompts = (config.prompts ?? []);
|
|
140
|
+
// ── Convert default assertions ──
|
|
141
|
+
const defaultTest = config.defaultTest;
|
|
142
|
+
const defaultScorers = [];
|
|
143
|
+
if (defaultTest?.assert) {
|
|
144
|
+
for (const a of defaultTest.assert) {
|
|
145
|
+
const mapped = mapAssertion(a);
|
|
146
|
+
if (mapped)
|
|
147
|
+
defaultScorers.push(mapped);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
// ── Convert test cases ──
|
|
151
|
+
const rawTests = (config.tests ?? []);
|
|
152
|
+
const cases = rawTests.map((test) => {
|
|
153
|
+
const testCase = {};
|
|
154
|
+
if (test.vars)
|
|
155
|
+
testCase.vars = test.vars;
|
|
156
|
+
if (test.description)
|
|
157
|
+
testCase.description = test.description;
|
|
158
|
+
const assertions = test.assert;
|
|
159
|
+
if (assertions) {
|
|
160
|
+
testCase.scorers = assertions
|
|
161
|
+
.map(mapAssertion)
|
|
162
|
+
.filter((s) => s !== null);
|
|
163
|
+
}
|
|
164
|
+
return testCase;
|
|
165
|
+
});
|
|
166
|
+
// ── Build output ──
|
|
167
|
+
const output = {
|
|
168
|
+
name: config.description ?? "Imported from Promptfoo",
|
|
169
|
+
providers,
|
|
170
|
+
prompts,
|
|
171
|
+
};
|
|
172
|
+
if (defaultScorers.length > 0)
|
|
173
|
+
output.defaultScorers = defaultScorers;
|
|
174
|
+
if (cases.length > 0)
|
|
175
|
+
output.cases = cases;
|
|
176
|
+
// ── Summary ──
|
|
177
|
+
let unmappedCount = 0;
|
|
178
|
+
for (const test of rawTests) {
|
|
179
|
+
const assertions = test.assert;
|
|
180
|
+
if (assertions) {
|
|
181
|
+
for (const a of assertions) {
|
|
182
|
+
if (!ASSERTION_MAP[a.type])
|
|
183
|
+
unmappedCount++;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
console.log();
|
|
188
|
+
console.log(chalk.bold(" Conversion Summary:"));
|
|
189
|
+
console.log(` ${chalk.green("\u2713")} ${providers.length} provider(s) mapped`);
|
|
190
|
+
console.log(` ${chalk.green("\u2713")} ${prompts.length} prompt(s) preserved`);
|
|
191
|
+
console.log(` ${chalk.green("\u2713")} ${cases.length} test case(s) converted`);
|
|
192
|
+
console.log(` ${chalk.green("\u2713")} ${defaultScorers.length} default scorer(s) mapped`);
|
|
193
|
+
if (unmappedCount > 0) {
|
|
194
|
+
console.log(` ${chalk.yellow("!")} ${unmappedCount} assertion(s) with no direct mapping (kept as-is)`);
|
|
195
|
+
}
|
|
196
|
+
if (opts.dryRun) {
|
|
197
|
+
console.log();
|
|
198
|
+
console.log(chalk.dim(" --- Dry Run Output ---"));
|
|
199
|
+
console.log(JSON.stringify(output, null, 2));
|
|
200
|
+
console.log();
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
// ── Write output ──
|
|
204
|
+
const outputPath = path.resolve(opts.output);
|
|
205
|
+
fs.writeFileSync(outputPath, JSON.stringify(output, null, 2), "utf-8");
|
|
206
|
+
console.log();
|
|
207
|
+
console.log(` ${chalk.green("\u2713")} Written to ${chalk.cyan(opts.output)}`);
|
|
208
|
+
console.log();
|
|
209
|
+
console.log(chalk.bold(" Next steps:"));
|
|
210
|
+
console.log(` 1. Review ${chalk.cyan(opts.output)}`);
|
|
211
|
+
console.log(` 2. Run: ${chalk.cyan("npx evalguard eval " + opts.output)}`);
|
|
212
|
+
console.log(` 3. Try: ${chalk.cyan("npx evalguard scan")} for 193 red team attacks`);
|
|
213
|
+
console.log();
|
|
214
|
+
console.log(chalk.dim(" Migration guide: https://evalguard.ai/docs/migrating-from-promptfoo"));
|
|
215
|
+
console.log();
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
//# sourceMappingURL=import-promptfoo.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"import-promptfoo.js","sourceRoot":"","sources":["../../src/commands/import-promptfoo.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,iCAAiC;AAEjC,MAAM,aAAa,GAA2B;IAC5C,QAAQ,EAAE,UAAU;IACpB,cAAc,EAAE,cAAc;IAC9B,SAAS,EAAE,WAAW;IACtB,cAAc,EAAE,cAAc;IAC9B,cAAc,EAAE,cAAc;IAC9B,MAAM,EAAE,QAAQ;IAChB,aAAa,EAAE,aAAa;IAC5B,WAAW,EAAE,WAAW;IACxB,KAAK,EAAE,OAAO;IACd,SAAS,EAAE,YAAY;IACvB,+BAA+B,EAAE,qBAAqB;IACtD,YAAY,EAAE,YAAY;IAC1B,uBAAuB,EAAE,YAAY;IACrC,mBAAmB,EAAE,YAAY;IACjC,UAAU,EAAE,YAAY;IACxB,kBAAkB,EAAE,kBAAkB;IACtC,sBAAsB,EAAE,sBAAsB;IAC9C,mBAAmB,EAAE,mBAAmB;IACxC,OAAO,EAAE,qBAAqB;IAC9B,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,SAAS;IAClB,QAAQ,EAAE,UAAU;IACpB,IAAI,EAAE,MAAM;IACZ,YAAY,EAAE,YAAY;CAC3B,CAAC;AAUF,SAAS,WAAW,CAAC,GAAqC;IACxD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,+CAA+C;QAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;YAC1F,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3C,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QACtD,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACvC,CAAC;IAED,yDAAyD;IACzD,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,CAAW,CAAC;IACjD,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,MAA6C,CAAC;IACjE,IAAI,MAAM,EAAE,CAAC;QACX,sDAAsD;QACtD,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;QACxC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAUD,SAAS,YAAY,CAAC,SAAkC;IACtD,MAAM,IAAI,GAAG,SAAS,CAAC,IAAc,CAAC;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,yBAAyB;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,QAAQ,CAAC,CAAC;IAE9D,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,SAA+B,EAAE,CAAC;IACxG,CAAC;IAED,MAAM,MAAM,GAAoB,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,OAAO,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACjF,IAAI,SAAS,CAAC,KAAK,KAAK,SAAS;QAAE,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;IAClE,IAAI,SAAS,CAAC,SAAS,KAAK,SAAS;QAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAmB,CAAC;IAExF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,qEAAqE;AAErE,SAAS,eAAe,CAAC,OAAe;IACtC,sCAAsC;IACtC,+EAA+E;IAC/E,IAAI,CAAC;QACH,iBAAiB;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;QAC9D,sDAAsD;QACtD,MAAM,IAAI,KAAK,CACb,+EAA+E;YAC/E,iEAAiE,CAClE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAgB;IACtC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEnD,iBAAiB;IACjB,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,iCAAiC;QACjC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC;YACtD,IAAI,KAAK;gBAAE,OAAO,KAAK,CAAC,OAAO,CAA4B,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACP,WAAW;QACb,CAAC;QACD,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;AACH,CAAC;AAED,uBAAuB;AAEvB,MAAM,UAAU,uBAAuB,CAAC,OAAgB;IACtD,OAAO;SACJ,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CAAC,2DAA2D,CAAC;SACxE,QAAQ,CAAC,QAAQ,EAAE,2CAA2C,CAAC;SAC/D,MAAM,CAAC,qBAAqB,EAAE,kBAAkB,EAAE,uBAAuB,CAAC;SAC1E,MAAM,CAAC,WAAW,EAAE,oDAAoD,CAAC;SACzE,MAAM,CAAC,KAAK,EAAE,IAAY,EAAE,IAA0C,EAAE,EAAE;QACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAEpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC/E,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC;QAEnE,IAAI,MAA+B,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAuB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,0BAA0B;QAC1B,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAyC,CAAC;QACtF,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAEhD,wBAAwB;QACxB,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAa,CAAC;QAEnD,mCAAmC;QACnC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAkD,CAAC;QAC9E,MAAM,cAAc,GAAsB,EAAE,CAAC;QAC7C,IAAI,WAAW,EAAE,MAAM,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,MAAmC,EAAE,CAAC;gBAChE,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,IAAI,MAAM;oBAAE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAA8B,CAAC;QACnE,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAClC,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,IAAI,IAAI,CAAC,IAAI;gBAAE,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YACzC,IAAI,IAAI,CAAC,WAAW;gBAAE,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;YAE9D,MAAM,UAAU,GAAG,IAAI,CAAC,MAA+C,CAAC;YACxE,IAAI,UAAU,EAAE,CAAC;gBACf,QAAQ,CAAC,OAAO,GAAG,UAAU;qBAC1B,GAAG,CAAC,YAAY,CAAC;qBACjB,MAAM,CAAC,CAAC,CAAC,EAAwB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;YACrD,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,qBAAqB;QACrB,MAAM,MAAM,GAA4B;YACtC,IAAI,EAAG,MAAM,CAAC,WAAsB,IAAI,yBAAyB;YACjE,SAAS;YACT,OAAO;SACR,CAAC;QAEF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,CAAC,cAAc,GAAG,cAAc,CAAC;QACtE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QAE3C,gBAAgB;QAChB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,MAA+C,CAAC;YACxE,IAAI,UAAU,EAAE,CAAC;gBACf,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;oBAC3B,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAc,CAAC;wBAAE,aAAa,EAAE,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,MAAM,qBAAqB,CAAC,CAAC;QACnF,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,MAAM,sBAAsB,CAAC,CAAC;QAClF,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,MAAM,yBAAyB,CAAC,CAAC;QACnF,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,cAAc,CAAC,MAAM,2BAA2B,CAAC,CAAC;QAC9F,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,aAAa,mDAAmD,CAAC,CAAC;QAC5G,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7C,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO;QACT,CAAC;QAED,qBAAqB;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7C,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAEvE,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,eAAe,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChF,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,CAAC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC9E,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,2BAA2B,CAAC,CAAC;QACxF,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,uEAAuE,CAAC,CAAC,CAAC;QAChG,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export { registerInit } from "./init.js";
|
|
2
|
+
export { registerEvalLocal } from "./eval-local.js";
|
|
3
|
+
export { registerScanLocal } from "./scan-local.js";
|
|
4
|
+
export { registerGenerate } from "./generate.js";
|
|
5
|
+
export { registerValidate } from "./validate.js";
|
|
6
|
+
export { registerCompare } from "./compare.js";
|
|
7
|
+
export { registerList } from "./list.js";
|
|
8
|
+
export { registerFirewall } from "./firewall.js";
|
|
9
|
+
export { registerWatch } from "./watch.js";
|
|
10
|
+
export { registerGate } from "./gate.js";
|
|
11
|
+
export { registerHistory } from "./history.js";
|
|
12
|
+
export { registerComplianceCheck } from "./compliance-check.js";
|
|
13
|
+
export { registerImportPromptfoo } from "./import-promptfoo.js";
|
|
14
|
+
export { registerShare } from "./share.js";
|
|
15
|
+
export { registerExport } from "./export.js";
|
|
16
|
+
export { registerRetry } from "./retry.js";
|
|
17
|
+
export { registerDebug } from "./debug.js";
|
|
18
|
+
export { registerLogs } from "./logs.js";
|
|
19
|
+
export { registerDelete } from "./delete.js";
|
|
20
|
+
export { registerModelScan } from "./model-scan.js";
|
|
21
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/commands/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export { registerInit } from "./init.js";
|
|
2
|
+
export { registerEvalLocal } from "./eval-local.js";
|
|
3
|
+
export { registerScanLocal } from "./scan-local.js";
|
|
4
|
+
export { registerGenerate } from "./generate.js";
|
|
5
|
+
export { registerValidate } from "./validate.js";
|
|
6
|
+
export { registerCompare } from "./compare.js";
|
|
7
|
+
export { registerList } from "./list.js";
|
|
8
|
+
export { registerFirewall } from "./firewall.js";
|
|
9
|
+
export { registerWatch } from "./watch.js";
|
|
10
|
+
export { registerGate } from "./gate.js";
|
|
11
|
+
export { registerHistory } from "./history.js";
|
|
12
|
+
export { registerComplianceCheck } from "./compliance-check.js";
|
|
13
|
+
export { registerImportPromptfoo } from "./import-promptfoo.js";
|
|
14
|
+
export { registerShare } from "./share.js";
|
|
15
|
+
export { registerExport } from "./export.js";
|
|
16
|
+
export { registerRetry } from "./retry.js";
|
|
17
|
+
export { registerDebug } from "./debug.js";
|
|
18
|
+
export { registerLogs } from "./logs.js";
|
|
19
|
+
export { registerDelete } from "./delete.js";
|
|
20
|
+
export { registerModelScan } from "./model-scan.js";
|
|
21
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/commands/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA4WpC,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAuFnD"}
|