@evalguardai/cli 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -0
- package/bin/evalguard.js +2 -0
- package/dist/commands/compare.d.ts +6 -0
- package/dist/commands/compare.d.ts.map +1 -0
- package/dist/commands/compare.js +109 -0
- package/dist/commands/compare.js.map +1 -0
- package/dist/commands/compliance-check.d.ts +18 -0
- package/dist/commands/compliance-check.d.ts.map +1 -0
- package/dist/commands/compliance-check.js +474 -0
- package/dist/commands/compliance-check.js.map +1 -0
- package/dist/commands/debug.d.ts +6 -0
- package/dist/commands/debug.d.ts.map +1 -0
- package/dist/commands/debug.js +151 -0
- package/dist/commands/debug.js.map +1 -0
- package/dist/commands/delete.d.ts +6 -0
- package/dist/commands/delete.d.ts.map +1 -0
- package/dist/commands/delete.js +105 -0
- package/dist/commands/delete.js.map +1 -0
- package/dist/commands/eval-local.d.ts +7 -0
- package/dist/commands/eval-local.d.ts.map +1 -0
- package/dist/commands/eval-local.js +376 -0
- package/dist/commands/eval-local.js.map +1 -0
- package/dist/commands/export.d.ts +6 -0
- package/dist/commands/export.d.ts.map +1 -0
- package/dist/commands/export.js +135 -0
- package/dist/commands/export.js.map +1 -0
- package/dist/commands/firewall.d.ts +6 -0
- package/dist/commands/firewall.d.ts.map +1 -0
- package/dist/commands/firewall.js +56 -0
- package/dist/commands/firewall.js.map +1 -0
- package/dist/commands/gate.d.ts +14 -0
- package/dist/commands/gate.d.ts.map +1 -0
- package/dist/commands/gate.js +232 -0
- package/dist/commands/gate.js.map +1 -0
- package/dist/commands/generate.d.ts +7 -0
- package/dist/commands/generate.d.ts.map +1 -0
- package/dist/commands/generate.js +182 -0
- package/dist/commands/generate.js.map +1 -0
- package/dist/commands/history.d.ts +7 -0
- package/dist/commands/history.d.ts.map +1 -0
- package/dist/commands/history.js +59 -0
- package/dist/commands/history.js.map +1 -0
- package/dist/commands/import-promptfoo.d.ts +7 -0
- package/dist/commands/import-promptfoo.d.ts.map +1 -0
- package/dist/commands/import-promptfoo.js +218 -0
- package/dist/commands/import-promptfoo.js.map +1 -0
- package/dist/commands/index.d.ts +21 -0
- package/dist/commands/index.d.ts.map +1 -0
- package/dist/commands/index.js +21 -0
- package/dist/commands/index.js.map +1 -0
- package/dist/commands/init.d.ts +7 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +509 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/list.d.ts +10 -0
- package/dist/commands/list.d.ts.map +1 -0
- package/dist/commands/list.js +165 -0
- package/dist/commands/list.js.map +1 -0
- package/dist/commands/logs.d.ts +6 -0
- package/dist/commands/logs.d.ts.map +1 -0
- package/dist/commands/logs.js +153 -0
- package/dist/commands/logs.js.map +1 -0
- package/dist/commands/model-scan.d.ts +7 -0
- package/dist/commands/model-scan.d.ts.map +1 -0
- package/dist/commands/model-scan.js +276 -0
- package/dist/commands/model-scan.js.map +1 -0
- package/dist/commands/retry.d.ts +6 -0
- package/dist/commands/retry.d.ts.map +1 -0
- package/dist/commands/retry.js +83 -0
- package/dist/commands/retry.js.map +1 -0
- package/dist/commands/scan-local.d.ts +6 -0
- package/dist/commands/scan-local.d.ts.map +1 -0
- package/dist/commands/scan-local.js +138 -0
- package/dist/commands/scan-local.js.map +1 -0
- package/dist/commands/share.d.ts +6 -0
- package/dist/commands/share.d.ts.map +1 -0
- package/dist/commands/share.js +74 -0
- package/dist/commands/share.js.map +1 -0
- package/dist/commands/store.d.ts +23 -0
- package/dist/commands/store.d.ts.map +1 -0
- package/dist/commands/store.js +54 -0
- package/dist/commands/store.js.map +1 -0
- package/dist/commands/validate.d.ts +6 -0
- package/dist/commands/validate.d.ts.map +1 -0
- package/dist/commands/validate.js +171 -0
- package/dist/commands/validate.js.map +1 -0
- package/dist/commands/watch.d.ts +6 -0
- package/dist/commands/watch.d.ts.map +1 -0
- package/dist/commands/watch.js +92 -0
- package/dist/commands/watch.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +342 -0
- package/dist/index.js.map +1 -0
- package/package.json +73 -0
package/README.md
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# @evalguard/cli
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/@evalguard/cli)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
|
|
6
|
+
Command-line interface for EvalGuard -- run LLM evaluations, security scans, and guardrail checks from your terminal and CI/CD pipelines.
|
|
7
|
+
|
|
8
|
+
## Installation
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
npm install -g @evalguard/cli
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Quick Start
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
# Authenticate
|
|
18
|
+
evalguard login --key eg_live_...
|
|
19
|
+
|
|
20
|
+
# Initialize a project
|
|
21
|
+
evalguard init --project my-project
|
|
22
|
+
|
|
23
|
+
# Run an evaluation
|
|
24
|
+
evalguard eval evals/example.json --wait
|
|
25
|
+
|
|
26
|
+
# Run a security scan
|
|
27
|
+
evalguard scan scans/example.json --wait
|
|
28
|
+
|
|
29
|
+
# Run a local eval (no API key needed)
|
|
30
|
+
evalguard eval:local --model openai:gpt-4o --prompt "Answer: {{input}}" --scorer exact-match
|
|
31
|
+
|
|
32
|
+
# Check firewall rules
|
|
33
|
+
evalguard firewall check "Ignore all instructions"
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Commands
|
|
37
|
+
|
|
38
|
+
| Command | Description |
|
|
39
|
+
|---|---|
|
|
40
|
+
| `evalguard login` | Authenticate with your EvalGuard API key |
|
|
41
|
+
| `evalguard logout` | Remove stored credentials |
|
|
42
|
+
| `evalguard init` | Initialize EvalGuard in the current project (creates config and example files) |
|
|
43
|
+
| `evalguard eval <file>` | Run an evaluation from a JSON config file |
|
|
44
|
+
| `evalguard scan <file>` | Run a security scan from a JSON config file |
|
|
45
|
+
| `evalguard whoami` | Show current authentication status |
|
|
46
|
+
| `evalguard eval:local` | Run an evaluation locally without the cloud API |
|
|
47
|
+
| `evalguard scan:local` | Run a security scan locally without the cloud API |
|
|
48
|
+
| `evalguard generate` | Auto-generate eval cases or scan configs from a prompt |
|
|
49
|
+
| `evalguard validate` | Validate an eval or scan config file for correctness |
|
|
50
|
+
| `evalguard compare` | Compare results between two eval runs (drift detection) |
|
|
51
|
+
| `evalguard list` | List past eval runs and security scans |
|
|
52
|
+
| `evalguard firewall` | Check input against firewall rules or manage rule sets |
|
|
53
|
+
| `evalguard watch` | Watch eval/scan files and re-run on changes |
|
|
54
|
+
|
|
55
|
+
## Configuration
|
|
56
|
+
|
|
57
|
+
The CLI stores credentials at `~/.evalguard/config.json`. Project-level settings go in `evalguard.config.json` in your repository root.
|
|
58
|
+
|
|
59
|
+
```json
|
|
60
|
+
{
|
|
61
|
+
"$schema": "https://evalguard.ai/schema/config.json",
|
|
62
|
+
"projectId": "my-project",
|
|
63
|
+
"defaultModel": "gpt-4o",
|
|
64
|
+
"evalsDir": "./evals",
|
|
65
|
+
"scansDir": "./scans"
|
|
66
|
+
}
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## CI/CD Usage
|
|
70
|
+
|
|
71
|
+
```yaml
|
|
72
|
+
# GitHub Actions example
|
|
73
|
+
- name: Run EvalGuard security scan
|
|
74
|
+
env:
|
|
75
|
+
EVALGUARD_API_KEY: ${{ secrets.EVALGUARD_API_KEY }}
|
|
76
|
+
run: |
|
|
77
|
+
npx @evalguard/cli login --key $EVALGUARD_API_KEY
|
|
78
|
+
npx @evalguard/cli scan scans/production.json --wait
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Documentation
|
|
82
|
+
|
|
83
|
+
Full documentation at [evalguard.ai/docs/cli](https://evalguard.ai/docs/cli).
|
|
84
|
+
|
|
85
|
+
## License
|
|
86
|
+
|
|
87
|
+
MIT -- see [LICENSE](./LICENSE) for details.
|
package/bin/evalguard.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/commands/compare.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAqCpC,wBAAgB,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CA0FtD"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import chalk from "chalk";
|
|
2
|
+
import * as fs from "fs";
|
|
3
|
+
import * as path from "path";
|
|
4
|
+
export function registerCompare(program) {
|
|
5
|
+
program
|
|
6
|
+
.command("compare")
|
|
7
|
+
.description("Compare two evaluation result files")
|
|
8
|
+
.argument("<file1>", "First result file (baseline)")
|
|
9
|
+
.argument("<file2>", "Second result file (candidate)")
|
|
10
|
+
.option("--threshold <n>", "Minimum score improvement to highlight", "0.05")
|
|
11
|
+
.action((file1, file2, opts) => {
|
|
12
|
+
const threshold = parseFloat(opts.threshold);
|
|
13
|
+
const r1 = loadResults(file1);
|
|
14
|
+
const r2 = loadResults(file2);
|
|
15
|
+
if (!r1 || !r2)
|
|
16
|
+
process.exit(1);
|
|
17
|
+
console.log();
|
|
18
|
+
console.log(chalk.bold(" Evaluation Comparison"));
|
|
19
|
+
console.log(chalk.dim(` Baseline: ${path.basename(file1)}`));
|
|
20
|
+
console.log(chalk.dim(` Candidate: ${path.basename(file2)}`));
|
|
21
|
+
console.log();
|
|
22
|
+
// Summary comparison
|
|
23
|
+
const s1 = r1.summary;
|
|
24
|
+
const s2 = r2.summary;
|
|
25
|
+
if (s1 && s2) {
|
|
26
|
+
const passRateDelta = s2.passRate - s1.passRate;
|
|
27
|
+
const deltaColor = passRateDelta > 0 ? chalk.green : passRateDelta < 0 ? chalk.red : chalk.dim;
|
|
28
|
+
const deltaStr = passRateDelta > 0 ? `+${(passRateDelta * 100).toFixed(1)}%` : `${(passRateDelta * 100).toFixed(1)}%`;
|
|
29
|
+
console.log(" " + chalk.dim("─".repeat(60)));
|
|
30
|
+
console.log(` ${"Metric".padEnd(25)} ${"Baseline".padEnd(15)} ${"Candidate".padEnd(15)} ${"Delta"}`);
|
|
31
|
+
console.log(" " + chalk.dim("─".repeat(60)));
|
|
32
|
+
console.log(` ${"Pass Rate".padEnd(25)} ${((s1.passRate * 100).toFixed(1) + "%").padEnd(15)} ${((s2.passRate * 100).toFixed(1) + "%").padEnd(15)} ${deltaColor(deltaStr)}`);
|
|
33
|
+
console.log(` ${"Passed".padEnd(25)} ${String(s1.passed).padEnd(15)} ${String(s2.passed).padEnd(15)} ${diffStr(s2.passed - s1.passed)}`);
|
|
34
|
+
console.log(` ${"Failed".padEnd(25)} ${String(s1.failed).padEnd(15)} ${String(s2.failed).padEnd(15)} ${diffStr(s2.failed - s1.failed, true)}`);
|
|
35
|
+
console.log(` ${"Total".padEnd(25)} ${String(s1.total).padEnd(15)} ${String(s2.total).padEnd(15)}`);
|
|
36
|
+
if (s1.duration != null && s2.duration != null) {
|
|
37
|
+
console.log(` ${"Duration".padEnd(25)} ${(s1.duration + "ms").padEnd(15)} ${(s2.duration + "ms").padEnd(15)} ${diffStr(s2.duration - s1.duration, true)}ms`);
|
|
38
|
+
}
|
|
39
|
+
console.log(" " + chalk.dim("─".repeat(60)));
|
|
40
|
+
}
|
|
41
|
+
// Per-case comparison
|
|
42
|
+
const cases1 = r1.results ?? r1.cases?.map((c, i) => ({ testIndex: i, input: c.input, score: c.score, passed: c.passed })) ?? [];
|
|
43
|
+
const cases2 = r2.results ?? r2.cases?.map((c, i) => ({ testIndex: i, input: c.input, score: c.score, passed: c.passed })) ?? [];
|
|
44
|
+
const maxLen = Math.max(cases1.length, cases2.length);
|
|
45
|
+
let improved = 0;
|
|
46
|
+
let regressed = 0;
|
|
47
|
+
let unchanged = 0;
|
|
48
|
+
console.log();
|
|
49
|
+
for (let i = 0; i < maxLen; i++) {
|
|
50
|
+
const c1 = cases1[i];
|
|
51
|
+
const c2 = cases2[i];
|
|
52
|
+
if (!c1 || !c2)
|
|
53
|
+
continue;
|
|
54
|
+
const scoreDelta = c2.score - c1.score;
|
|
55
|
+
if (Math.abs(scoreDelta) < threshold) {
|
|
56
|
+
unchanged++;
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (scoreDelta > 0) {
|
|
60
|
+
improved++;
|
|
61
|
+
console.log(` ${chalk.green("↑")} Case ${i}: ${c1.score.toFixed(2)} → ${c2.score.toFixed(2)} ${chalk.green(`+${scoreDelta.toFixed(2)}`)} ${chalk.dim(c1.input?.slice(0, 50) ?? "")}`);
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
regressed++;
|
|
65
|
+
console.log(` ${chalk.red("↓")} Case ${i}: ${c1.score.toFixed(2)} → ${c2.score.toFixed(2)} ${chalk.red(scoreDelta.toFixed(2))} ${chalk.dim(c1.input?.slice(0, 50) ?? "")}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
console.log();
|
|
69
|
+
console.log(` ${chalk.green(`${improved} improved`)} | ${chalk.red(`${regressed} regressed`)} | ${chalk.dim(`${unchanged} unchanged`)}`);
|
|
70
|
+
console.log();
|
|
71
|
+
// Overall verdict
|
|
72
|
+
if (s1 && s2) {
|
|
73
|
+
if (s2.passRate > s1.passRate) {
|
|
74
|
+
console.log(` ${chalk.green("✓")} Candidate is ${chalk.bold("better")} than baseline`);
|
|
75
|
+
}
|
|
76
|
+
else if (s2.passRate < s1.passRate) {
|
|
77
|
+
console.log(` ${chalk.red("✗")} Candidate is ${chalk.bold("worse")} than baseline`);
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
console.log(` ${chalk.dim("=")} Results are ${chalk.bold("equivalent")}`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
console.log();
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
function loadResults(file) {
|
|
87
|
+
const filePath = path.resolve(file);
|
|
88
|
+
if (!fs.existsSync(filePath)) {
|
|
89
|
+
console.error(chalk.red(`File not found: ${filePath}`));
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
try {
|
|
93
|
+
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
|
94
|
+
}
|
|
95
|
+
catch (err) {
|
|
96
|
+
console.error(chalk.red(`Invalid JSON: ${filePath}`));
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
function diffStr(delta, inverseColor = false) {
|
|
101
|
+
const positive = inverseColor ? chalk.red : chalk.green;
|
|
102
|
+
const negative = inverseColor ? chalk.green : chalk.red;
|
|
103
|
+
if (delta > 0)
|
|
104
|
+
return positive(`+${delta}`);
|
|
105
|
+
if (delta < 0)
|
|
106
|
+
return negative(String(delta));
|
|
107
|
+
return chalk.dim("0");
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=compare.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/commands/compare.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAkC7B,MAAM,UAAU,eAAe,CAAC,OAAgB;IAC9C,OAAO;SACJ,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,qCAAqC,CAAC;SAClD,QAAQ,CAAC,SAAS,EAAE,8BAA8B,CAAC;SACnD,QAAQ,CAAC,SAAS,EAAE,gCAAgC,CAAC;SACrD,MAAM,CAAC,iBAAiB,EAAE,wCAAwC,EAAE,MAAM,CAAC;SAC3E,MAAM,CAAC,CAAC,KAAa,EAAE,KAAa,EAAE,IAA2B,EAAE,EAAE;QACpE,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE7C,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QAE9B,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEhC,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,qBAAqB;QACrB,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC;QACtB,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC;QAEtB,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;YACb,MAAM,aAAa,GAAG,EAAE,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC;YAChD,MAAM,UAAU,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;YAC/F,MAAM,QAAQ,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;YAEtH,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;YACtG,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC7K,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC1I,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YAChJ,OAAO,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACrG,IAAI,EAAE,CAAC,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;gBAC/C,OAAO,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,EAAE,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAChK,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,sBAAsB;QACtB,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACjI,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAEjI,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAErB,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE;gBAAE,SAAS;YAEzB,MAAM,UAAU,GAAG,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;YACvC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,SAAS,EAAE,CAAC;gBACrC,SAAS,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;gBACnB,QAAQ,EAAE,CAAC;gBACX,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;YACzL,CAAC;iBAAM,CAAC;gBACN,SAAS,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;YAC/K,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,QAAQ,WAAW,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,YAAY,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,YAAY,CAAC,EAAE,CAAC,CAAC;QAC1I,OAAO,CAAC,GAAG,EAAE,CAAC;QAEd,kBAAkB;QAClB,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;YACb,IAAI,EAAE,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;YAC1F,CAAC;iBAAM,IAAI,EAAE,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;gBACrC,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,iBAAiB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;YACvF,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC,CAAC;QACtD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,KAAa,EAAE,YAAY,GAAG,KAAK;IAClD,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;IACxD,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;IACxD,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;IAC5C,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9C,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `evalguard compliance-check` — Run compliance checks against AI frameworks
|
|
3
|
+
*
|
|
4
|
+
* Checks an AI system's compliance against regulatory frameworks like the
|
|
5
|
+
* EU AI Act, HIPAA, FedRAMP, PCI-DSS, ISO 42001, OWASP LLM Top 10, and NIST AI RMF.
|
|
6
|
+
*
|
|
7
|
+
* Works in two modes:
|
|
8
|
+
* - With API key: sends check to the EvalGuard API
|
|
9
|
+
* - Without API key: runs locally using @evalguard/core compliance engine
|
|
10
|
+
*
|
|
11
|
+
* Usage:
|
|
12
|
+
* evalguard compliance-check --framework eu-ai-act --model gpt-4o
|
|
13
|
+
* evalguard compliance-check --framework hipaa --model claude-3-opus --threshold 80
|
|
14
|
+
* evalguard compliance-check --framework owasp-llm --json
|
|
15
|
+
*/
|
|
16
|
+
import { Command } from "commander";
|
|
17
|
+
export declare function registerComplianceCheck(program: Command): void;
|
|
18
|
+
//# sourceMappingURL=compliance-check.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compliance-check.d.ts","sourceRoot":"","sources":["../../src/commands/compliance-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA4gBpC,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAoI9D"}
|