llm-checker 3.2.1 → 3.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -7
- package/bin/enhanced_cli.js +447 -0
- package/package.json +7 -1
- package/src/index.js +84 -20
- package/src/models/deterministic-selector.js +406 -22
- package/src/models/intelligent-selector.js +89 -4
- package/src/policy/audit-reporter.js +420 -0
- package/src/policy/cli-policy.js +403 -0
- package/src/policy/policy-engine.js +497 -0
- package/src/policy/policy-manager.js +324 -0
- package/src/provenance/model-provenance.js +176 -0
package/README.md
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
<a href="https://www.npmjs.com/package/llm-checker"><img src="https://img.shields.io/npm/v/llm-checker?style=flat-square&color=0066FF" alt="npm version"></a>
|
|
18
18
|
<a href="https://www.npmjs.com/package/llm-checker"><img src="https://img.shields.io/npm/dm/llm-checker?style=flat-square&color=0066FF" alt="npm downloads"></a>
|
|
19
19
|
<a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-0066FF?style=flat-square" alt="License"></a>
|
|
20
|
+
<a href="https://discord.gg/mnmYrA7T"><img src="https://img.shields.io/discord/1457032977849520374?style=flat-square&color=0066FF&label=Discord" alt="Discord"></a>
|
|
20
21
|
<a href="https://nodejs.org/"><img src="https://img.shields.io/badge/node-%3E%3D16-0066FF?style=flat-square" alt="Node.js"></a>
|
|
21
22
|
</p>
|
|
22
23
|
|
|
@@ -26,7 +27,8 @@
|
|
|
26
27
|
<a href="#claude-code-mcp">Claude MCP</a> •
|
|
27
28
|
<a href="#commands">Commands</a> •
|
|
28
29
|
<a href="#scoring-system">Scoring</a> •
|
|
29
|
-
<a href="#supported-hardware">Hardware</a>
|
|
30
|
+
<a href="#supported-hardware">Hardware</a> •
|
|
31
|
+
<a href="https://discord.gg/mnmYrA7T"><img src="https://cdn.simpleicons.org/discord/0066FF" alt="Discord" width="14" height="14"> Discord</a>
|
|
30
32
|
</p>
|
|
31
33
|
|
|
32
34
|
---
|
|
@@ -43,7 +45,7 @@ Choosing the right LLM for your hardware is complex. With thousands of model var
|
|
|
43
45
|
|
|
44
46
|
| | Feature | Description |
|
|
45
47
|
|:---:|---|---|
|
|
46
|
-
| **
|
|
48
|
+
| **200+** | Dynamic Model Pool | Uses full scraped Ollama catalog/variants when available (with curated fallback) |
|
|
47
49
|
| **4D** | Scoring Engine | Quality, Speed, Fit, Context — weighted by use case |
|
|
48
50
|
| **Multi-GPU** | Hardware Detection | Apple Silicon, NVIDIA CUDA, AMD ROCm, Intel Arc, CPU |
|
|
49
51
|
| **Calibrated** | Memory Estimation | Bytes-per-parameter formula validated against real Ollama sizes |
|
|
@@ -87,6 +89,32 @@ npm install sql.js
|
|
|
87
89
|
|
|
88
90
|
---
|
|
89
91
|
|
|
92
|
+
## Distribution
|
|
93
|
+
|
|
94
|
+
LLM Checker is published in all primary channels:
|
|
95
|
+
|
|
96
|
+
- npm (latest): [`llm-checker@3.2.1`](https://www.npmjs.com/package/llm-checker)
|
|
97
|
+
- GitHub Release: [`v3.2.1` (2026-02-17)](https://github.com/Pavelevich/llm-checker/releases/tag/v3.2.1)
|
|
98
|
+
- GitHub Packages: [`@pavelevich/llm-checker`](https://github.com/users/Pavelevich/packages/npm/package/llm-checker)
|
|
99
|
+
|
|
100
|
+
### v3.2.1 Highlights
|
|
101
|
+
|
|
102
|
+
- Added vLLM/MLX runtime support and speculative decoding estimation.
|
|
103
|
+
- Improved GPU detection, added DGX Spark/GB10 support, strengthened Node runtime guards, and updated tooling comparison notes.
|
|
104
|
+
|
|
105
|
+
### Optional: Install from GitHub Packages
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
# 1) Configure registry + token (PAT with read:packages)
|
|
109
|
+
echo "@pavelevich:registry=https://npm.pkg.github.com" >> ~/.npmrc
|
|
110
|
+
echo "//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN}" >> ~/.npmrc
|
|
111
|
+
|
|
112
|
+
# 2) Install
|
|
113
|
+
npm install -g @pavelevich/llm-checker@3.2.1
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
90
118
|
## Quick Start
|
|
91
119
|
|
|
92
120
|
```bash
|
|
@@ -198,6 +226,58 @@ Claude will automatically call the right tools and give you actionable results.
|
|
|
198
226
|
| `search <query>` | Search models with filters and intelligent scoring |
|
|
199
227
|
| `smart-recommend` | Advanced recommendations using the full scoring engine |
|
|
200
228
|
|
|
229
|
+
### Enterprise Policy Commands
|
|
230
|
+
|
|
231
|
+
| Command | Description |
|
|
232
|
+
|---------|-------------|
|
|
233
|
+
| `policy init` | Generate a `policy.yaml` template for enterprise governance |
|
|
234
|
+
| `policy validate` | Validate a policy file and return non-zero on schema errors |
|
|
235
|
+
| `audit export` | Evaluate policy outcomes and export compliance reports (`json`, `csv`, `sarif`) |
|
|
236
|
+
|
|
237
|
+
### Policy Enforcement in `check` and `recommend`
|
|
238
|
+
|
|
239
|
+
Both `check` and `recommend` support `--policy <file>`.
|
|
240
|
+
|
|
241
|
+
- In `audit` mode, policy violations are reported but the command exits with `0`.
|
|
242
|
+
- In `enforce` mode, blocking violations return non-zero (default `1`).
|
|
243
|
+
- You can override the non-zero code with `enforcement.exit_code` in `policy.yaml`.
|
|
244
|
+
|
|
245
|
+
Examples:
|
|
246
|
+
|
|
247
|
+
```bash
|
|
248
|
+
llm-checker check --policy ./policy.yaml
|
|
249
|
+
llm-checker check --policy ./policy.yaml --use-case coding --runtime vllm
|
|
250
|
+
llm-checker recommend --policy ./policy.yaml --category coding
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
### Policy Audit Export
|
|
254
|
+
|
|
255
|
+
Use `audit export` when you need machine-readable compliance evidence for CI/CD gates, governance reviews, or security tooling.
|
|
256
|
+
|
|
257
|
+
```bash
|
|
258
|
+
# Single report format
|
|
259
|
+
llm-checker audit export --policy ./policy.yaml --command check --format json --out ./reports/check-policy.json
|
|
260
|
+
|
|
261
|
+
# Export all configured formats (json, csv, sarif)
|
|
262
|
+
llm-checker audit export --policy ./policy.yaml --command check --format all --out-dir ./reports
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
- `--command check|recommend` chooses the candidate source.
|
|
266
|
+
- `--format all` honors `reporting.formats` in your policy (falls back to `json,csv,sarif`).
|
|
267
|
+
- In `enforce` mode with blocking violations, reports are still written before non-zero exit.
|
|
268
|
+
|
|
269
|
+
### Provenance Fields in Reports
|
|
270
|
+
|
|
271
|
+
Each finding includes normalized model provenance fields:
|
|
272
|
+
|
|
273
|
+
- `source`
|
|
274
|
+
- `registry`
|
|
275
|
+
- `version`
|
|
276
|
+
- `license`
|
|
277
|
+
- `digest`
|
|
278
|
+
|
|
279
|
+
If a field is unavailable from model metadata, reports use `"unknown"` instead of omitting the field. This keeps downstream parsers deterministic.
|
|
280
|
+
|
|
201
281
|
### AI Commands
|
|
202
282
|
|
|
203
283
|
| Command | Description |
|
|
@@ -277,7 +357,9 @@ llm-checker search qwen --quant Q4_K_M --max-size 8
|
|
|
277
357
|
|
|
278
358
|
## Model Catalog
|
|
279
359
|
|
|
280
|
-
|
|
360
|
+
LLM Checker prioritizes the full scraped Ollama model cache (all families/sizes/variants) and falls back to a built-in curated catalog when cache is unavailable.
|
|
361
|
+
|
|
362
|
+
The curated fallback catalog includes 35+ models from the most popular Ollama families:
|
|
281
363
|
|
|
282
364
|
| Family | Models | Best For |
|
|
283
365
|
|--------|--------|----------|
|
|
@@ -291,7 +373,7 @@ The built-in catalog includes 35+ models from the most popular Ollama families:
|
|
|
291
373
|
| **LLaVA** | 7B, 13B | Vision |
|
|
292
374
|
| **Embeddings** | nomic-embed-text, mxbai-embed-large, bge-m3, all-minilm | RAG, search |
|
|
293
375
|
|
|
294
|
-
|
|
376
|
+
All available models are automatically combined with locally installed Ollama models for scoring.
|
|
295
377
|
|
|
296
378
|
---
|
|
297
379
|
|
|
@@ -418,7 +500,7 @@ The selector automatically picks the best quantization that fits your available
|
|
|
418
500
|
|
|
419
501
|
**Selector Pipeline:**
|
|
420
502
|
1. **Hardware profiling** — CPU, GPU, RAM, acceleration backend
|
|
421
|
-
2. **Model pool** — Merge
|
|
503
|
+
2. **Model pool** — Merge full Ollama scraped pool (or curated fallback) + installed models (deduped)
|
|
422
504
|
3. **Category filter** — Keep models relevant to the use case
|
|
423
505
|
4. **Quantization selection** — Best quant that fits in memory budget
|
|
424
506
|
5. **4D scoring** — Q, S, F, C with category-specific weights
|
|
@@ -477,7 +559,7 @@ src/
|
|
|
477
559
|
deterministic-selector.js # Primary selection algorithm
|
|
478
560
|
scoring-config.js # Centralized scoring weights
|
|
479
561
|
scoring-engine.js # Advanced scoring (smart-recommend)
|
|
480
|
-
catalog.json # Curated
|
|
562
|
+
catalog.json # Curated fallback catalog (35+ models)
|
|
481
563
|
ai/
|
|
482
564
|
multi-objective-selector.js # Multi-objective optimization
|
|
483
565
|
ai-check-selector.js # LLM-based evaluation
|
|
@@ -501,6 +583,9 @@ MIT License — see [LICENSE](LICENSE) for details.
|
|
|
501
583
|
|
|
502
584
|
<p align="center">
|
|
503
585
|
<a href="https://github.com/Pavelevich/llm-checker">GitHub</a> •
|
|
586
|
+
<a href="https://github.com/Pavelevich/llm-checker/releases">Releases</a> •
|
|
504
587
|
<a href="https://www.npmjs.com/package/llm-checker">npm</a> •
|
|
505
|
-
<a href="https://github.com/Pavelevich/llm-checker
|
|
588
|
+
<a href="https://github.com/users/Pavelevich/packages/npm/package/llm-checker">GitHub Packages</a> •
|
|
589
|
+
<a href="https://github.com/Pavelevich/llm-checker/issues">Issues</a> •
|
|
590
|
+
<a href="https://discord.gg/mnmYrA7T">Discord</a>
|
|
506
591
|
</p>
|
package/bin/enhanced_cli.js
CHANGED
|
@@ -24,6 +24,20 @@ const {
|
|
|
24
24
|
getRuntimeCommandSet
|
|
25
25
|
} = require('../src/runtime/runtime-support');
|
|
26
26
|
const SpeculativeDecodingEstimator = require('../src/models/speculative-decoding-estimator');
|
|
27
|
+
const PolicyManager = require('../src/policy/policy-manager');
|
|
28
|
+
const PolicyEngine = require('../src/policy/policy-engine');
|
|
29
|
+
const {
|
|
30
|
+
collectCandidatesFromAnalysis,
|
|
31
|
+
collectCandidatesFromRecommendationData,
|
|
32
|
+
buildPolicyRuntimeContext,
|
|
33
|
+
evaluatePolicyCandidates,
|
|
34
|
+
resolvePolicyEnforcement
|
|
35
|
+
} = require('../src/policy/cli-policy');
|
|
36
|
+
const {
|
|
37
|
+
buildComplianceReport,
|
|
38
|
+
serializeComplianceReport
|
|
39
|
+
} = require('../src/policy/audit-reporter');
|
|
40
|
+
const policyManager = new PolicyManager();
|
|
27
41
|
|
|
28
42
|
// ASCII Art for each command - Large text banners
|
|
29
43
|
const ASCII_ART = {
|
|
@@ -2060,6 +2074,365 @@ function extractModelName(command) {
|
|
|
2060
2074
|
return match ? match[1] : 'model';
|
|
2061
2075
|
}
|
|
2062
2076
|
|
|
2077
|
+
function loadPolicyConfiguration(policyFile) {
|
|
2078
|
+
const validation = policyManager.validatePolicyFile(policyFile);
|
|
2079
|
+
if (!validation.valid) {
|
|
2080
|
+
const details = validation.errors
|
|
2081
|
+
.map((entry) => `${entry.path}: ${entry.message}`)
|
|
2082
|
+
.join('; ');
|
|
2083
|
+
throw new Error(`Invalid policy file: ${details}`);
|
|
2084
|
+
}
|
|
2085
|
+
|
|
2086
|
+
return {
|
|
2087
|
+
policyPath: validation.path,
|
|
2088
|
+
policy: validation.policy,
|
|
2089
|
+
policyEngine: new PolicyEngine(validation.policy)
|
|
2090
|
+
};
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
function parseSizeFilterInput(sizeStr) {
|
|
2094
|
+
if (!sizeStr) return null;
|
|
2095
|
+
const match = String(sizeStr)
|
|
2096
|
+
.toUpperCase()
|
|
2097
|
+
.trim()
|
|
2098
|
+
.match(/^([0-9]+(?:\.[0-9]+)?)\s*(B|GB)?$/);
|
|
2099
|
+
if (!match) return null;
|
|
2100
|
+
|
|
2101
|
+
const value = Number.parseFloat(match[1]);
|
|
2102
|
+
const unit = match[2] || 'B';
|
|
2103
|
+
|
|
2104
|
+
// Convert to "B params" approximation used by existing check flow
|
|
2105
|
+
return unit === 'GB' ? value / 0.5 : value;
|
|
2106
|
+
}
|
|
2107
|
+
|
|
2108
|
+
function normalizeUseCaseInput(useCase = '') {
|
|
2109
|
+
const alias = String(useCase || '')
|
|
2110
|
+
.toLowerCase()
|
|
2111
|
+
.trim();
|
|
2112
|
+
|
|
2113
|
+
const useCaseMap = {
|
|
2114
|
+
embed: 'embeddings',
|
|
2115
|
+
embedding: 'embeddings',
|
|
2116
|
+
embeddings: 'embeddings',
|
|
2117
|
+
embedings: 'embeddings',
|
|
2118
|
+
talk: 'chat',
|
|
2119
|
+
talking: 'chat',
|
|
2120
|
+
conversation: 'chat',
|
|
2121
|
+
chat: 'chat'
|
|
2122
|
+
};
|
|
2123
|
+
|
|
2124
|
+
return useCaseMap[alias] || alias || 'general';
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
function resolveAuditFormats(formatOption, policy) {
|
|
2128
|
+
const requested = String(formatOption || 'json').trim().toLowerCase();
|
|
2129
|
+
const allowed = new Set(['json', 'csv', 'sarif']);
|
|
2130
|
+
|
|
2131
|
+
if (requested === 'all') {
|
|
2132
|
+
const configured = Array.isArray(policy?.reporting?.formats)
|
|
2133
|
+
? policy.reporting.formats
|
|
2134
|
+
.map((entry) => String(entry || '').trim().toLowerCase())
|
|
2135
|
+
.filter((entry) => allowed.has(entry))
|
|
2136
|
+
: [];
|
|
2137
|
+
|
|
2138
|
+
return configured.length > 0 ? configured : ['json', 'csv', 'sarif'];
|
|
2139
|
+
}
|
|
2140
|
+
|
|
2141
|
+
if (!allowed.has(requested)) {
|
|
2142
|
+
throw new Error('Invalid format. Use one of: json, csv, sarif, all');
|
|
2143
|
+
}
|
|
2144
|
+
|
|
2145
|
+
return [requested];
|
|
2146
|
+
}
|
|
2147
|
+
|
|
2148
|
+
function toAuditOutputPath({ outputPath, outputDir, commandName, format, timestamp }) {
|
|
2149
|
+
if (outputPath) {
|
|
2150
|
+
return path.resolve(outputPath);
|
|
2151
|
+
}
|
|
2152
|
+
|
|
2153
|
+
const safeTimestamp = timestamp.replace(/[:.]/g, '-');
|
|
2154
|
+
const extension = format === 'sarif' ? 'sarif.json' : format;
|
|
2155
|
+
const fileName = `${commandName}-policy-audit-${safeTimestamp}.${extension}`;
|
|
2156
|
+
return path.resolve(outputDir || 'audit-reports', fileName);
|
|
2157
|
+
}
|
|
2158
|
+
|
|
2159
|
+
function writeReportFile(filePath, content) {
|
|
2160
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
2161
|
+
fs.writeFileSync(filePath, content, 'utf8');
|
|
2162
|
+
}
|
|
2163
|
+
|
|
2164
|
+
function displayPolicySummary(commandName, policyConfig, evaluation, enforcement) {
|
|
2165
|
+
if (!policyConfig || !evaluation || !enforcement) return;
|
|
2166
|
+
|
|
2167
|
+
console.log('\n' + chalk.bgMagenta.white.bold(` POLICY SUMMARY (${commandName.toUpperCase()}) `));
|
|
2168
|
+
console.log(chalk.magenta('╭' + '─'.repeat(65)));
|
|
2169
|
+
console.log(chalk.magenta('│') + ` File: ${chalk.white(policyConfig.policyPath)}`);
|
|
2170
|
+
console.log(
|
|
2171
|
+
chalk.magenta('│') +
|
|
2172
|
+
` Mode: ${chalk.cyan(enforcement.mode)} | Action: ${chalk.cyan(enforcement.onViolation)}`
|
|
2173
|
+
);
|
|
2174
|
+
console.log(chalk.magenta('│') + ` Total checked: ${chalk.white.bold(evaluation.totalChecked)}`);
|
|
2175
|
+
console.log(chalk.magenta('│') + ` Pass: ${chalk.green.bold(evaluation.passCount)} | Fail: ${chalk.red.bold(evaluation.failCount)}`);
|
|
2176
|
+
console.log(
|
|
2177
|
+
chalk.magenta('│') +
|
|
2178
|
+
` Suppressed: ${chalk.yellow.bold(evaluation.suppressedViolationCount || 0)} | Exceptions: ${chalk.cyan.bold(
|
|
2179
|
+
evaluation.exceptionsAppliedCount || 0
|
|
2180
|
+
)}`
|
|
2181
|
+
);
|
|
2182
|
+
|
|
2183
|
+
if (evaluation.topViolations.length === 0) {
|
|
2184
|
+
console.log(chalk.magenta('│') + ` Top violations: ${chalk.green('none')}`);
|
|
2185
|
+
} else {
|
|
2186
|
+
console.log(chalk.magenta('│') + ` Top violations:`);
|
|
2187
|
+
evaluation.topViolations.slice(0, 3).forEach((violation) => {
|
|
2188
|
+
console.log(
|
|
2189
|
+
chalk.magenta('│') +
|
|
2190
|
+
` - ${chalk.yellow(violation.code)}: ${chalk.white(violation.count)}`
|
|
2191
|
+
);
|
|
2192
|
+
});
|
|
2193
|
+
}
|
|
2194
|
+
|
|
2195
|
+
if (enforcement.shouldBlock) {
|
|
2196
|
+
console.log(
|
|
2197
|
+
chalk.magenta('│') +
|
|
2198
|
+
chalk.red.bold(
|
|
2199
|
+
` Enforcement result: blocking violations detected (exit ${enforcement.exitCode})`
|
|
2200
|
+
)
|
|
2201
|
+
);
|
|
2202
|
+
} else if (enforcement.mode === 'audit' && enforcement.hasFailures) {
|
|
2203
|
+
console.log(
|
|
2204
|
+
chalk.magenta('│') +
|
|
2205
|
+
chalk.yellow(' Audit mode: violations reported, command exits with code 0')
|
|
2206
|
+
);
|
|
2207
|
+
} else if (enforcement.onViolation === 'warn' && enforcement.hasFailures) {
|
|
2208
|
+
console.log(
|
|
2209
|
+
chalk.magenta('│') +
|
|
2210
|
+
chalk.yellow(' Enforce+warn: violations reported, command exits with code 0')
|
|
2211
|
+
);
|
|
2212
|
+
} else {
|
|
2213
|
+
console.log(chalk.magenta('│') + chalk.green(' Policy check passed'));
|
|
2214
|
+
}
|
|
2215
|
+
|
|
2216
|
+
console.log(chalk.magenta('╰' + '─'.repeat(65)));
|
|
2217
|
+
}
|
|
2218
|
+
|
|
2219
|
+
const policyCommand = program
|
|
2220
|
+
.command('policy')
|
|
2221
|
+
.description('Manage enterprise policy files (policy.yaml)')
|
|
2222
|
+
.showHelpAfterError();
|
|
2223
|
+
|
|
2224
|
+
policyCommand
|
|
2225
|
+
.command('init')
|
|
2226
|
+
.description('Create a policy.yaml template')
|
|
2227
|
+
.option('-f, --file <path>', 'Policy file path', 'policy.yaml')
|
|
2228
|
+
.option('--force', 'Overwrite existing file if it already exists')
|
|
2229
|
+
.action((options) => {
|
|
2230
|
+
try {
|
|
2231
|
+
const result = policyManager.initPolicy(options.file, {
|
|
2232
|
+
force: Boolean(options.force)
|
|
2233
|
+
});
|
|
2234
|
+
|
|
2235
|
+
const status = result.overwritten ? 'overwritten' : 'created';
|
|
2236
|
+
console.log(chalk.green(`Policy file ${status}: ${result.path}`));
|
|
2237
|
+
} catch (error) {
|
|
2238
|
+
console.error(chalk.red(`Failed to initialize policy: ${error.message}`));
|
|
2239
|
+
process.exit(1);
|
|
2240
|
+
}
|
|
2241
|
+
});
|
|
2242
|
+
|
|
2243
|
+
policyCommand
|
|
2244
|
+
.command('validate')
|
|
2245
|
+
.description('Validate policy.yaml against the v1 schema')
|
|
2246
|
+
.option('-f, --file <path>', 'Policy file path', 'policy.yaml')
|
|
2247
|
+
.option('-j, --json', 'Output validation result as JSON')
|
|
2248
|
+
.action((options) => {
|
|
2249
|
+
try {
|
|
2250
|
+
const result = policyManager.validatePolicyFile(options.file);
|
|
2251
|
+
|
|
2252
|
+
if (options.json) {
|
|
2253
|
+
console.log(JSON.stringify({
|
|
2254
|
+
valid: result.valid,
|
|
2255
|
+
file: result.path,
|
|
2256
|
+
errorCount: result.errors.length,
|
|
2257
|
+
errors: result.errors
|
|
2258
|
+
}, null, 2));
|
|
2259
|
+
if (!result.valid) {
|
|
2260
|
+
process.exit(1);
|
|
2261
|
+
}
|
|
2262
|
+
} else if (result.valid) {
|
|
2263
|
+
const mode = result.policy?.mode || 'unknown';
|
|
2264
|
+
console.log(chalk.green(`Policy is valid (${mode} mode): ${result.path}`));
|
|
2265
|
+
} else {
|
|
2266
|
+
console.error(chalk.red(`Policy validation failed: ${result.path}`));
|
|
2267
|
+
result.errors.forEach((entry) => {
|
|
2268
|
+
console.error(chalk.red(` - ${entry.path}: ${entry.message}`));
|
|
2269
|
+
});
|
|
2270
|
+
process.exit(1);
|
|
2271
|
+
}
|
|
2272
|
+
} catch (error) {
|
|
2273
|
+
if (options.json) {
|
|
2274
|
+
console.log(JSON.stringify({
|
|
2275
|
+
valid: false,
|
|
2276
|
+
file: policyManager.resolvePolicyPath(options.file),
|
|
2277
|
+
errorCount: 1,
|
|
2278
|
+
errors: [{ path: 'file', message: error.message }]
|
|
2279
|
+
}, null, 2));
|
|
2280
|
+
} else {
|
|
2281
|
+
console.error(chalk.red(`Policy validation failed: ${error.message}`));
|
|
2282
|
+
}
|
|
2283
|
+
process.exit(1);
|
|
2284
|
+
}
|
|
2285
|
+
});
|
|
2286
|
+
|
|
2287
|
+
policyCommand.action(() => {
|
|
2288
|
+
policyCommand.outputHelp();
|
|
2289
|
+
});
|
|
2290
|
+
|
|
2291
|
+
const auditCommand = program
|
|
2292
|
+
.command('audit')
|
|
2293
|
+
.description('Run policy audits and export compliance reports')
|
|
2294
|
+
.showHelpAfterError();
|
|
2295
|
+
|
|
2296
|
+
auditCommand
|
|
2297
|
+
.command('export')
|
|
2298
|
+
.description('Evaluate policy compliance and export JSON/CSV/SARIF reports')
|
|
2299
|
+
.requiredOption('--policy <file>', 'Policy file path')
|
|
2300
|
+
.option('--command <name>', 'Evaluation source: check | recommend', 'check')
|
|
2301
|
+
.option('--format <format>', 'Report format: json | csv | sarif | all', 'json')
|
|
2302
|
+
.option('--out <path>', 'Output file path (single-format export only)')
|
|
2303
|
+
.option('--out-dir <path>', 'Output directory when --out is omitted', 'audit-reports')
|
|
2304
|
+
.option('-u, --use-case <case>', 'Use case when --command check is selected', 'general')
|
|
2305
|
+
.option('-c, --category <category>', 'Category hint when --command recommend is selected')
|
|
2306
|
+
.option('--runtime <runtime>', `Runtime for check mode (${SUPPORTED_RUNTIMES.join('|')})`, 'ollama')
|
|
2307
|
+
.option('--include-cloud', 'Include cloud models in check-mode analysis')
|
|
2308
|
+
.option('--max-size <size>', 'Maximum model size for check mode (e.g., "24B" or "12GB")')
|
|
2309
|
+
.option('--min-size <size>', 'Minimum model size for check mode (e.g., "3B" or "2GB")')
|
|
2310
|
+
.option('-l, --limit <number>', 'Model analysis limit for check mode', '25')
|
|
2311
|
+
.option('--no-verbose', 'Disable verbose progress while collecting audit inputs')
|
|
2312
|
+
.action(async (options) => {
|
|
2313
|
+
try {
|
|
2314
|
+
const policyConfig = loadPolicyConfiguration(options.policy);
|
|
2315
|
+
const selectedCommand = String(options.command || 'check')
|
|
2316
|
+
.toLowerCase()
|
|
2317
|
+
.trim();
|
|
2318
|
+
|
|
2319
|
+
if (!['check', 'recommend'].includes(selectedCommand)) {
|
|
2320
|
+
throw new Error('Invalid --command value. Use "check" or "recommend".');
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2323
|
+
const exportFormats = resolveAuditFormats(options.format, policyConfig.policy);
|
|
2324
|
+
if (options.out && exportFormats.length > 1) {
|
|
2325
|
+
throw new Error('--out can only be used with a single export format.');
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
const verboseEnabled = options.verbose !== false;
|
|
2329
|
+
const checker = new (getLLMChecker())({ verbose: verboseEnabled });
|
|
2330
|
+
const hardware = await checker.getSystemInfo();
|
|
2331
|
+
|
|
2332
|
+
let runtimeBackend = 'ollama';
|
|
2333
|
+
let policyCandidates = [];
|
|
2334
|
+
let analysisResult = null;
|
|
2335
|
+
let recommendationResult = null;
|
|
2336
|
+
|
|
2337
|
+
if (selectedCommand === 'check') {
|
|
2338
|
+
let selectedRuntime = normalizeRuntime(options.runtime);
|
|
2339
|
+
if (!runtimeSupportedOnHardware(selectedRuntime, hardware)) {
|
|
2340
|
+
selectedRuntime = 'ollama';
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
const maxSize = parseSizeFilterInput(options.maxSize);
|
|
2344
|
+
const minSize = parseSizeFilterInput(options.minSize);
|
|
2345
|
+
const normalizedUseCase = normalizeUseCaseInput(options.useCase);
|
|
2346
|
+
|
|
2347
|
+
analysisResult = await checker.analyze({
|
|
2348
|
+
useCase: normalizedUseCase,
|
|
2349
|
+
includeCloud: Boolean(options.includeCloud),
|
|
2350
|
+
limit: Number.parseInt(options.limit, 10) || 25,
|
|
2351
|
+
maxSize,
|
|
2352
|
+
minSize,
|
|
2353
|
+
runtime: selectedRuntime
|
|
2354
|
+
});
|
|
2355
|
+
|
|
2356
|
+
runtimeBackend = selectedRuntime;
|
|
2357
|
+
policyCandidates = collectCandidatesFromAnalysis(analysisResult);
|
|
2358
|
+
} else {
|
|
2359
|
+
recommendationResult = await checker.generateIntelligentRecommendations(hardware);
|
|
2360
|
+
if (!recommendationResult) {
|
|
2361
|
+
throw new Error('Unable to generate recommendation data for policy audit export.');
|
|
2362
|
+
}
|
|
2363
|
+
|
|
2364
|
+
runtimeBackend = normalizeRuntime(options.runtime || 'ollama');
|
|
2365
|
+
policyCandidates = collectCandidatesFromRecommendationData(recommendationResult);
|
|
2366
|
+
}
|
|
2367
|
+
|
|
2368
|
+
const policyContext = buildPolicyRuntimeContext({
|
|
2369
|
+
hardware,
|
|
2370
|
+
runtimeBackend
|
|
2371
|
+
});
|
|
2372
|
+
|
|
2373
|
+
const policyEvaluation = evaluatePolicyCandidates(
|
|
2374
|
+
policyConfig.policyEngine,
|
|
2375
|
+
policyCandidates,
|
|
2376
|
+
policyContext,
|
|
2377
|
+
policyConfig.policy
|
|
2378
|
+
);
|
|
2379
|
+
const policyEnforcement = resolvePolicyEnforcement(policyConfig.policy, policyEvaluation);
|
|
2380
|
+
|
|
2381
|
+
const report = buildComplianceReport({
|
|
2382
|
+
commandName: selectedCommand,
|
|
2383
|
+
policyPath: policyConfig.policyPath,
|
|
2384
|
+
policy: policyConfig.policy,
|
|
2385
|
+
evaluation: policyEvaluation,
|
|
2386
|
+
enforcement: policyEnforcement,
|
|
2387
|
+
runtimeContext: policyContext,
|
|
2388
|
+
options: {
|
|
2389
|
+
format: exportFormats,
|
|
2390
|
+
runtime: runtimeBackend,
|
|
2391
|
+
use_case: selectedCommand === 'check' ? normalizeUseCaseInput(options.useCase) : null,
|
|
2392
|
+
category: selectedCommand === 'recommend' ? options.category || null : null,
|
|
2393
|
+
include_cloud: Boolean(options.includeCloud)
|
|
2394
|
+
},
|
|
2395
|
+
hardware
|
|
2396
|
+
});
|
|
2397
|
+
|
|
2398
|
+
const generatedAt = report.generated_at || new Date().toISOString();
|
|
2399
|
+
const writtenFiles = [];
|
|
2400
|
+
exportFormats.forEach((format) => {
|
|
2401
|
+
const filePath = toAuditOutputPath({
|
|
2402
|
+
outputPath: options.out,
|
|
2403
|
+
outputDir: options.outDir,
|
|
2404
|
+
commandName: selectedCommand,
|
|
2405
|
+
format,
|
|
2406
|
+
timestamp: generatedAt
|
|
2407
|
+
});
|
|
2408
|
+
const content = serializeComplianceReport(report, format);
|
|
2409
|
+
writeReportFile(filePath, content);
|
|
2410
|
+
writtenFiles.push({ format, filePath });
|
|
2411
|
+
});
|
|
2412
|
+
|
|
2413
|
+
displayPolicySummary(`audit ${selectedCommand}`, policyConfig, policyEvaluation, policyEnforcement);
|
|
2414
|
+
|
|
2415
|
+
console.log('\n' + chalk.bgBlue.white.bold(' AUDIT EXPORT '));
|
|
2416
|
+
writtenFiles.forEach((entry) => {
|
|
2417
|
+
console.log(`${chalk.cyan(entry.format.toUpperCase())}: ${chalk.white(entry.filePath)}`);
|
|
2418
|
+
});
|
|
2419
|
+
|
|
2420
|
+
if (policyEnforcement.shouldBlock) {
|
|
2421
|
+
process.exit(policyEnforcement.exitCode);
|
|
2422
|
+
}
|
|
2423
|
+
} catch (error) {
|
|
2424
|
+
console.error(chalk.red(`Audit export failed: ${error.message}`));
|
|
2425
|
+
if (process.env.DEBUG) {
|
|
2426
|
+
console.error(error.stack);
|
|
2427
|
+
}
|
|
2428
|
+
process.exit(1);
|
|
2429
|
+
}
|
|
2430
|
+
});
|
|
2431
|
+
|
|
2432
|
+
auditCommand.action(() => {
|
|
2433
|
+
auditCommand.outputHelp();
|
|
2434
|
+
});
|
|
2435
|
+
|
|
2063
2436
|
program
|
|
2064
2437
|
.command('check')
|
|
2065
2438
|
.description('Analyze your system and show compatible LLM models')
|
|
@@ -2072,15 +2445,30 @@ program
|
|
|
2072
2445
|
.option('--include-cloud', 'Include cloud models in analysis')
|
|
2073
2446
|
.option('--ollama-only', 'Only show models available in Ollama')
|
|
2074
2447
|
.option('--runtime <runtime>', `Inference runtime (${SUPPORTED_RUNTIMES.join('|')})`, 'ollama')
|
|
2448
|
+
.option('--policy <file>', 'Evaluate candidate models against a policy file')
|
|
2075
2449
|
.option('--performance-test', 'Run performance benchmarks')
|
|
2076
2450
|
.option('--show-ollama-analysis', 'Show detailed Ollama model analysis')
|
|
2077
2451
|
.option('--no-verbose', 'Disable step-by-step progress display')
|
|
2452
|
+
.addHelpText(
|
|
2453
|
+
'after',
|
|
2454
|
+
`
|
|
2455
|
+
Enterprise policy examples:
|
|
2456
|
+
$ llm-checker check --policy ./policy.yaml
|
|
2457
|
+
$ llm-checker check --policy ./policy.yaml --use-case coding --runtime vllm
|
|
2458
|
+
$ llm-checker check --policy ./policy.yaml --include-cloud --max-size 24B
|
|
2459
|
+
|
|
2460
|
+
Policy scope:
|
|
2461
|
+
- Evaluates all compatible and marginal candidates discovered during analysis
|
|
2462
|
+
- Not limited to the top --limit results shown in output
|
|
2463
|
+
`
|
|
2464
|
+
)
|
|
2078
2465
|
.action(async (options) => {
|
|
2079
2466
|
showAsciiArt('check');
|
|
2080
2467
|
try {
|
|
2081
2468
|
// Use verbose progress unless explicitly disabled
|
|
2082
2469
|
const verboseEnabled = options.verbose !== false;
|
|
2083
2470
|
const checker = new (getLLMChecker())({ verbose: verboseEnabled });
|
|
2471
|
+
const policyConfig = options.policy ? loadPolicyConfiguration(options.policy) : null;
|
|
2084
2472
|
|
|
2085
2473
|
// If verbose is disabled, show simple loading message
|
|
2086
2474
|
if (!verboseEnabled) {
|
|
@@ -2147,6 +2535,23 @@ program
|
|
|
2147
2535
|
console.log(chalk.green(' done'));
|
|
2148
2536
|
}
|
|
2149
2537
|
|
|
2538
|
+
let policyEvaluation = null;
|
|
2539
|
+
let policyEnforcement = null;
|
|
2540
|
+
if (policyConfig) {
|
|
2541
|
+
const policyCandidates = collectCandidatesFromAnalysis(analysis);
|
|
2542
|
+
const policyContext = buildPolicyRuntimeContext({
|
|
2543
|
+
hardware,
|
|
2544
|
+
runtimeBackend: selectedRuntime
|
|
2545
|
+
});
|
|
2546
|
+
policyEvaluation = evaluatePolicyCandidates(
|
|
2547
|
+
policyConfig.policyEngine,
|
|
2548
|
+
policyCandidates,
|
|
2549
|
+
policyContext,
|
|
2550
|
+
policyConfig.policy
|
|
2551
|
+
);
|
|
2552
|
+
policyEnforcement = resolvePolicyEnforcement(policyConfig.policy, policyEvaluation);
|
|
2553
|
+
}
|
|
2554
|
+
|
|
2150
2555
|
// Simplified output - show only essential information
|
|
2151
2556
|
displaySimplifiedSystemInfo(hardware);
|
|
2152
2557
|
const recommendedModels = await displayModelRecommendations(
|
|
@@ -2158,6 +2563,13 @@ program
|
|
|
2158
2563
|
);
|
|
2159
2564
|
await displayQuickStartCommands(analysis, recommendedModels[0], recommendedModels, selectedRuntime);
|
|
2160
2565
|
|
|
2566
|
+
if (policyConfig && policyEvaluation && policyEnforcement) {
|
|
2567
|
+
displayPolicySummary('check', policyConfig, policyEvaluation, policyEnforcement);
|
|
2568
|
+
if (policyEnforcement.shouldBlock) {
|
|
2569
|
+
process.exit(policyEnforcement.exitCode);
|
|
2570
|
+
}
|
|
2571
|
+
}
|
|
2572
|
+
|
|
2161
2573
|
} catch (error) {
|
|
2162
2574
|
console.error(chalk.red('\nError:'), error.message);
|
|
2163
2575
|
if (process.env.DEBUG) {
|
|
@@ -2387,11 +2799,22 @@ program
|
|
|
2387
2799
|
.description('Get intelligent model recommendations for your hardware')
|
|
2388
2800
|
.option('-c, --category <category>', 'Get recommendations for specific category (coding, talking, reading, etc.)')
|
|
2389
2801
|
.option('--no-verbose', 'Disable step-by-step progress display')
|
|
2802
|
+
.option('--policy <file>', 'Evaluate recommendations against a policy file')
|
|
2803
|
+
.addHelpText(
|
|
2804
|
+
'after',
|
|
2805
|
+
`
|
|
2806
|
+
Enterprise policy examples:
|
|
2807
|
+
$ llm-checker recommend --policy ./policy.yaml
|
|
2808
|
+
$ llm-checker recommend --policy ./policy.yaml --category coding
|
|
2809
|
+
$ llm-checker recommend --policy ./policy.yaml --no-verbose
|
|
2810
|
+
`
|
|
2811
|
+
)
|
|
2390
2812
|
.action(async (options) => {
|
|
2391
2813
|
showAsciiArt('recommend');
|
|
2392
2814
|
try {
|
|
2393
2815
|
const verboseEnabled = options.verbose !== false;
|
|
2394
2816
|
const checker = new (getLLMChecker())({ verbose: verboseEnabled });
|
|
2817
|
+
const policyConfig = options.policy ? loadPolicyConfiguration(options.policy) : null;
|
|
2395
2818
|
|
|
2396
2819
|
if (!verboseEnabled) {
|
|
2397
2820
|
process.stdout.write(chalk.gray('Generating recommendations...'));
|
|
@@ -2409,12 +2832,36 @@ program
|
|
|
2409
2832
|
console.log(chalk.green(' done'));
|
|
2410
2833
|
}
|
|
2411
2834
|
|
|
2835
|
+
let policyEvaluation = null;
|
|
2836
|
+
let policyEnforcement = null;
|
|
2837
|
+
if (policyConfig) {
|
|
2838
|
+
const policyCandidates = collectCandidatesFromRecommendationData(intelligentRecommendations);
|
|
2839
|
+
const policyContext = buildPolicyRuntimeContext({
|
|
2840
|
+
hardware,
|
|
2841
|
+
runtimeBackend: 'ollama'
|
|
2842
|
+
});
|
|
2843
|
+
policyEvaluation = evaluatePolicyCandidates(
|
|
2844
|
+
policyConfig.policyEngine,
|
|
2845
|
+
policyCandidates,
|
|
2846
|
+
policyContext,
|
|
2847
|
+
policyConfig.policy
|
|
2848
|
+
);
|
|
2849
|
+
policyEnforcement = resolvePolicyEnforcement(policyConfig.policy, policyEvaluation);
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2412
2852
|
// Mostrar información del sistema
|
|
2413
2853
|
displaySystemInfo(hardware, { summary: { hardwareTier: intelligentRecommendations.summary.hardware_tier } });
|
|
2414
2854
|
|
|
2415
2855
|
// Mostrar recomendaciones
|
|
2416
2856
|
displayIntelligentRecommendations(intelligentRecommendations);
|
|
2417
2857
|
|
|
2858
|
+
if (policyConfig && policyEvaluation && policyEnforcement) {
|
|
2859
|
+
displayPolicySummary('recommend', policyConfig, policyEvaluation, policyEnforcement);
|
|
2860
|
+
if (policyEnforcement.shouldBlock) {
|
|
2861
|
+
process.exit(policyEnforcement.exitCode);
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
|
|
2418
2865
|
} catch (error) {
|
|
2419
2866
|
console.error(chalk.red('\nError:'), error.message);
|
|
2420
2867
|
if (process.env.DEBUG) {
|