llm-checker 3.2.0 → 3.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -7
- package/analyzer/compatibility.js +20 -0
- package/bin/cli.js +14 -0
- package/bin/enhanced_cli.js +580 -36
- package/package.json +11 -3
- package/src/ai/multi-objective-selector.js +28 -4
- package/src/hardware/backends/cuda-detector.js +32 -11
- package/src/hardware/detector.js +107 -5
- package/src/hardware/specs.js +8 -1
- package/src/index.js +161 -31
- package/src/models/deterministic-selector.js +406 -22
- package/src/models/expanded_database.js +8 -2
- package/src/models/intelligent-selector.js +89 -4
- package/src/models/scoring-engine.js +4 -0
- package/src/models/speculative-decoding-estimator.js +245 -0
- package/src/policy/audit-reporter.js +420 -0
- package/src/policy/cli-policy.js +403 -0
- package/src/policy/policy-engine.js +497 -0
- package/src/policy/policy-manager.js +324 -0
- package/src/provenance/model-provenance.js +176 -0
- package/src/runtime/runtime-support.js +174 -0
- package/bin/CLAUDE.md +0 -27
- package/src/CLAUDE.md +0 -18
- package/src/data/CLAUDE.md +0 -17
- package/src/hardware/CLAUDE.md +0 -18
- package/src/hardware/backends/CLAUDE.md +0 -17
- package/src/models/CLAUDE.md +0 -23
- package/src/ollama/CLAUDE.md +0 -30
- package/src/plugins/CLAUDE.md +0 -17
- package/src/utils/CLAUDE.md +0 -17
package/bin/enhanced_cli.js
CHANGED
|
@@ -16,6 +16,28 @@ function getLLMChecker() {
|
|
|
16
16
|
const { getLogger } = require('../src/utils/logger');
|
|
17
17
|
const fs = require('fs');
|
|
18
18
|
const path = require('path');
|
|
19
|
+
const {
|
|
20
|
+
SUPPORTED_RUNTIMES,
|
|
21
|
+
normalizeRuntime,
|
|
22
|
+
runtimeSupportedOnHardware,
|
|
23
|
+
getRuntimeDisplayName,
|
|
24
|
+
getRuntimeCommandSet
|
|
25
|
+
} = require('../src/runtime/runtime-support');
|
|
26
|
+
const SpeculativeDecodingEstimator = require('../src/models/speculative-decoding-estimator');
|
|
27
|
+
const PolicyManager = require('../src/policy/policy-manager');
|
|
28
|
+
const PolicyEngine = require('../src/policy/policy-engine');
|
|
29
|
+
const {
|
|
30
|
+
collectCandidatesFromAnalysis,
|
|
31
|
+
collectCandidatesFromRecommendationData,
|
|
32
|
+
buildPolicyRuntimeContext,
|
|
33
|
+
evaluatePolicyCandidates,
|
|
34
|
+
resolvePolicyEnforcement
|
|
35
|
+
} = require('../src/policy/cli-policy');
|
|
36
|
+
const {
|
|
37
|
+
buildComplianceReport,
|
|
38
|
+
serializeComplianceReport
|
|
39
|
+
} = require('../src/policy/audit-reporter');
|
|
40
|
+
const policyManager = new PolicyManager();
|
|
19
41
|
|
|
20
42
|
// ASCII Art for each command - Large text banners
|
|
21
43
|
const ASCII_ART = {
|
|
@@ -1406,10 +1428,18 @@ function displaySimplifiedSystemInfo(hardware) {
|
|
|
1406
1428
|
console.log(`Hardware Tier: ${tierColor.bold(tier)}`);
|
|
1407
1429
|
}
|
|
1408
1430
|
|
|
1409
|
-
async function displayModelRecommendations(analysis, hardware, useCase = 'general', limit = 1) {
|
|
1431
|
+
async function displayModelRecommendations(analysis, hardware, useCase = 'general', limit = 1, runtime = 'ollama') {
|
|
1410
1432
|
const title = limit === 1 ? 'RECOMMENDED MODEL' : `TOP ${limit} COMPATIBLE MODELS`;
|
|
1411
1433
|
console.log(chalk.green.bold(`\n${title}`));
|
|
1412
1434
|
console.log(chalk.gray('─'.repeat(50)));
|
|
1435
|
+
|
|
1436
|
+
const selectedRuntime = normalizeRuntime(runtime);
|
|
1437
|
+
const runtimeLabel = getRuntimeDisplayName(selectedRuntime);
|
|
1438
|
+
const speculativeEstimator = new SpeculativeDecodingEstimator();
|
|
1439
|
+
const speculativeCandidatePool = [
|
|
1440
|
+
...(analysis?.compatible || []),
|
|
1441
|
+
...(analysis?.marginal || [])
|
|
1442
|
+
];
|
|
1413
1443
|
|
|
1414
1444
|
// Find the best models from compatible models considering use case
|
|
1415
1445
|
let selectedModels = [];
|
|
@@ -1760,42 +1790,75 @@ async function displayModelRecommendations(analysis, hardware, useCase = 'genera
|
|
|
1760
1790
|
if (model.performanceEstimate) {
|
|
1761
1791
|
console.log(`Estimated Speed: ${chalk.yellow(model.performanceEstimate.estimatedTokensPerSecond || 'N/A')} tokens/sec`);
|
|
1762
1792
|
}
|
|
1763
|
-
|
|
1764
|
-
|
|
1793
|
+
|
|
1794
|
+
console.log(`Runtime: ${chalk.white(runtimeLabel)}`);
|
|
1795
|
+
const runtimeCommands = getRuntimeCommandSet(model, selectedRuntime);
|
|
1796
|
+
|
|
1797
|
+
// Check installation only when using Ollama runtime.
|
|
1765
1798
|
let isInstalled = false;
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1799
|
+
if (selectedRuntime === 'ollama') {
|
|
1800
|
+
try {
|
|
1801
|
+
isInstalled = await checkIfModelInstalled(model, analysis.ollamaInfo);
|
|
1802
|
+
if (isInstalled) {
|
|
1803
|
+
console.log(`Status: ${chalk.green('Already installed in Ollama')}`);
|
|
1804
|
+
} else if (analysis.ollamaInfo && analysis.ollamaInfo.available) {
|
|
1805
|
+
console.log(`Status: ${chalk.gray('Available for installation')}`);
|
|
1806
|
+
} else {
|
|
1807
|
+
console.log(`Status: ${chalk.yellow('Requires Ollama (not detected)')}`);
|
|
1808
|
+
}
|
|
1809
|
+
} catch (installCheckError) {
|
|
1810
|
+
if (analysis.ollamaInfo && analysis.ollamaInfo.available) {
|
|
1811
|
+
console.log(`Status: ${chalk.gray('Available for installation')}`);
|
|
1812
|
+
} else {
|
|
1813
|
+
console.log(`Status: ${chalk.yellow('Requires Ollama (not detected)')}`);
|
|
1814
|
+
}
|
|
1774
1815
|
}
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
if (
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1816
|
+
|
|
1817
|
+
const ollamaCommand = getOllamaInstallCommand(model);
|
|
1818
|
+
if (ollamaCommand) {
|
|
1819
|
+
const modelName = extractModelName(ollamaCommand);
|
|
1820
|
+
if (isInstalled) {
|
|
1821
|
+
console.log(`\nRun: ${chalk.cyan.bold(`ollama run ${modelName}`)}`);
|
|
1822
|
+
} else {
|
|
1823
|
+
console.log(`\nPull: ${chalk.cyan.bold(ollamaCommand)}`);
|
|
1824
|
+
}
|
|
1825
|
+
} else if (model.ollamaTag || model.ollamaId) {
|
|
1826
|
+
const tag = model.ollamaTag || model.ollamaId;
|
|
1827
|
+
if (isInstalled) {
|
|
1828
|
+
console.log(`\nRun: ${chalk.cyan.bold(`ollama run ${tag}`)}`);
|
|
1829
|
+
} else {
|
|
1830
|
+
console.log(`\nPull: ${chalk.cyan.bold(`ollama pull ${tag}`)}`);
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
} else {
|
|
1834
|
+
console.log(`Status: ${chalk.gray(`${runtimeLabel} runtime selected`)}`);
|
|
1835
|
+
console.log(`\nRun: ${chalk.cyan.bold(runtimeCommands.run)}`);
|
|
1836
|
+
if (index === 0) {
|
|
1837
|
+
console.log(`Install runtime: ${chalk.cyan.bold(runtimeCommands.install)}`);
|
|
1838
|
+
console.log(`Fetch model: ${chalk.cyan.bold(runtimeCommands.pull)}`);
|
|
1781
1839
|
}
|
|
1782
1840
|
}
|
|
1783
1841
|
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1842
|
+
const speculativeInfo =
|
|
1843
|
+
model.speculativeDecoding ||
|
|
1844
|
+
speculativeEstimator.estimate({
|
|
1845
|
+
model,
|
|
1846
|
+
candidates: speculativeCandidatePool,
|
|
1847
|
+
hardware,
|
|
1848
|
+
runtime: selectedRuntime
|
|
1849
|
+
});
|
|
1850
|
+
|
|
1851
|
+
if (speculativeInfo && speculativeInfo.runtime === selectedRuntime) {
|
|
1852
|
+
if (speculativeInfo.enabled) {
|
|
1853
|
+
console.log(
|
|
1854
|
+
`SpecDec: ${chalk.green(`+${speculativeInfo.estimatedThroughputGainPct}%`)} ` +
|
|
1855
|
+
`(${chalk.gray(`draft: ${speculativeInfo.draftModel}`)})`
|
|
1856
|
+
);
|
|
1857
|
+
} else if (speculativeInfo.estimatedSpeedup) {
|
|
1858
|
+
const suggested = speculativeInfo.suggestedDraftModel ? ` with ${speculativeInfo.suggestedDraftModel}` : '';
|
|
1859
|
+
console.log(
|
|
1860
|
+
`SpecDec estimate: ${chalk.yellow(`+${speculativeInfo.estimatedThroughputGainPct}%`)}${chalk.gray(suggested)}`
|
|
1861
|
+
);
|
|
1799
1862
|
}
|
|
1800
1863
|
}
|
|
1801
1864
|
}
|
|
@@ -1807,9 +1870,12 @@ async function displayModelRecommendations(analysis, hardware, useCase = 'genera
|
|
|
1807
1870
|
return selectedModels;
|
|
1808
1871
|
}
|
|
1809
1872
|
|
|
1810
|
-
async function displayQuickStartCommands(analysis, recommendedModel = null, allRecommended = null) {
|
|
1873
|
+
async function displayQuickStartCommands(analysis, recommendedModel = null, allRecommended = null, runtime = 'ollama') {
|
|
1811
1874
|
console.log(chalk.yellow.bold('\nQUICK START'));
|
|
1812
1875
|
console.log(chalk.gray('─'.repeat(50)));
|
|
1876
|
+
|
|
1877
|
+
const selectedRuntime = normalizeRuntime(runtime);
|
|
1878
|
+
const runtimeLabel = getRuntimeDisplayName(selectedRuntime);
|
|
1813
1879
|
|
|
1814
1880
|
// Use the first model from allRecommended if available, otherwise fallback to recommendedModel
|
|
1815
1881
|
let bestModel = (allRecommended && allRecommended.length > 0) ? allRecommended[0] : recommendedModel;
|
|
@@ -1824,6 +1890,33 @@ async function displayQuickStartCommands(analysis, recommendedModel = null, allR
|
|
|
1824
1890
|
}
|
|
1825
1891
|
}
|
|
1826
1892
|
|
|
1893
|
+
if (selectedRuntime !== 'ollama') {
|
|
1894
|
+
if (!bestModel) {
|
|
1895
|
+
console.log(`1. Try expanding search: ${chalk.cyan('llm-checker check --include-cloud')}`);
|
|
1896
|
+
return;
|
|
1897
|
+
}
|
|
1898
|
+
|
|
1899
|
+
const runtimeCommands = getRuntimeCommandSet(bestModel, selectedRuntime);
|
|
1900
|
+
console.log(`1. Install ${runtimeLabel}:`);
|
|
1901
|
+
console.log(` ${chalk.cyan.bold(runtimeCommands.install)}`);
|
|
1902
|
+
console.log(`2. Fetch model weights:`);
|
|
1903
|
+
console.log(` ${chalk.cyan.bold(runtimeCommands.pull)}`);
|
|
1904
|
+
console.log(`3. Run model:`);
|
|
1905
|
+
console.log(` ${chalk.cyan.bold(runtimeCommands.run)}`);
|
|
1906
|
+
|
|
1907
|
+
const speculative = bestModel.speculativeDecoding;
|
|
1908
|
+
if (speculative && speculative.enabled) {
|
|
1909
|
+
console.log(`4. SpecDec suggestion (${chalk.green(`+${speculative.estimatedThroughputGainPct}%`)}):`);
|
|
1910
|
+
if (selectedRuntime === 'vllm') {
|
|
1911
|
+
console.log(` ${chalk.cyan.bold(`${runtimeCommands.run} --speculative-model '${speculative.draftModelRef || speculative.draftModel}'`)}`);
|
|
1912
|
+
} else if (selectedRuntime === 'mlx') {
|
|
1913
|
+
console.log(` ${chalk.gray(`Use draft model ${speculative.draftModelRef || speculative.draftModel} when enabling speculative decoding in MLX-LM`)}`);
|
|
1914
|
+
}
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1917
|
+
return;
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1827
1920
|
if (analysis.ollamaInfo && !analysis.ollamaInfo.available) {
|
|
1828
1921
|
console.log(`1. Install Ollama: ${chalk.underline('https://ollama.ai')}`);
|
|
1829
1922
|
console.log(`2. Come back and run this command again`);
|
|
@@ -1981,6 +2074,365 @@ function extractModelName(command) {
|
|
|
1981
2074
|
return match ? match[1] : 'model';
|
|
1982
2075
|
}
|
|
1983
2076
|
|
|
2077
|
+
function loadPolicyConfiguration(policyFile) {
|
|
2078
|
+
const validation = policyManager.validatePolicyFile(policyFile);
|
|
2079
|
+
if (!validation.valid) {
|
|
2080
|
+
const details = validation.errors
|
|
2081
|
+
.map((entry) => `${entry.path}: ${entry.message}`)
|
|
2082
|
+
.join('; ');
|
|
2083
|
+
throw new Error(`Invalid policy file: ${details}`);
|
|
2084
|
+
}
|
|
2085
|
+
|
|
2086
|
+
return {
|
|
2087
|
+
policyPath: validation.path,
|
|
2088
|
+
policy: validation.policy,
|
|
2089
|
+
policyEngine: new PolicyEngine(validation.policy)
|
|
2090
|
+
};
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
function parseSizeFilterInput(sizeStr) {
|
|
2094
|
+
if (!sizeStr) return null;
|
|
2095
|
+
const match = String(sizeStr)
|
|
2096
|
+
.toUpperCase()
|
|
2097
|
+
.trim()
|
|
2098
|
+
.match(/^([0-9]+(?:\.[0-9]+)?)\s*(B|GB)?$/);
|
|
2099
|
+
if (!match) return null;
|
|
2100
|
+
|
|
2101
|
+
const value = Number.parseFloat(match[1]);
|
|
2102
|
+
const unit = match[2] || 'B';
|
|
2103
|
+
|
|
2104
|
+
// Convert to "B params" approximation used by existing check flow
|
|
2105
|
+
return unit === 'GB' ? value / 0.5 : value;
|
|
2106
|
+
}
|
|
2107
|
+
|
|
2108
|
+
function normalizeUseCaseInput(useCase = '') {
|
|
2109
|
+
const alias = String(useCase || '')
|
|
2110
|
+
.toLowerCase()
|
|
2111
|
+
.trim();
|
|
2112
|
+
|
|
2113
|
+
const useCaseMap = {
|
|
2114
|
+
embed: 'embeddings',
|
|
2115
|
+
embedding: 'embeddings',
|
|
2116
|
+
embeddings: 'embeddings',
|
|
2117
|
+
embedings: 'embeddings',
|
|
2118
|
+
talk: 'chat',
|
|
2119
|
+
talking: 'chat',
|
|
2120
|
+
conversation: 'chat',
|
|
2121
|
+
chat: 'chat'
|
|
2122
|
+
};
|
|
2123
|
+
|
|
2124
|
+
return useCaseMap[alias] || alias || 'general';
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
function resolveAuditFormats(formatOption, policy) {
|
|
2128
|
+
const requested = String(formatOption || 'json').trim().toLowerCase();
|
|
2129
|
+
const allowed = new Set(['json', 'csv', 'sarif']);
|
|
2130
|
+
|
|
2131
|
+
if (requested === 'all') {
|
|
2132
|
+
const configured = Array.isArray(policy?.reporting?.formats)
|
|
2133
|
+
? policy.reporting.formats
|
|
2134
|
+
.map((entry) => String(entry || '').trim().toLowerCase())
|
|
2135
|
+
.filter((entry) => allowed.has(entry))
|
|
2136
|
+
: [];
|
|
2137
|
+
|
|
2138
|
+
return configured.length > 0 ? configured : ['json', 'csv', 'sarif'];
|
|
2139
|
+
}
|
|
2140
|
+
|
|
2141
|
+
if (!allowed.has(requested)) {
|
|
2142
|
+
throw new Error('Invalid format. Use one of: json, csv, sarif, all');
|
|
2143
|
+
}
|
|
2144
|
+
|
|
2145
|
+
return [requested];
|
|
2146
|
+
}
|
|
2147
|
+
|
|
2148
|
+
function toAuditOutputPath({ outputPath, outputDir, commandName, format, timestamp }) {
|
|
2149
|
+
if (outputPath) {
|
|
2150
|
+
return path.resolve(outputPath);
|
|
2151
|
+
}
|
|
2152
|
+
|
|
2153
|
+
const safeTimestamp = timestamp.replace(/[:.]/g, '-');
|
|
2154
|
+
const extension = format === 'sarif' ? 'sarif.json' : format;
|
|
2155
|
+
const fileName = `${commandName}-policy-audit-${safeTimestamp}.${extension}`;
|
|
2156
|
+
return path.resolve(outputDir || 'audit-reports', fileName);
|
|
2157
|
+
}
|
|
2158
|
+
|
|
2159
|
+
function writeReportFile(filePath, content) {
|
|
2160
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
2161
|
+
fs.writeFileSync(filePath, content, 'utf8');
|
|
2162
|
+
}
|
|
2163
|
+
|
|
2164
|
+
function displayPolicySummary(commandName, policyConfig, evaluation, enforcement) {
|
|
2165
|
+
if (!policyConfig || !evaluation || !enforcement) return;
|
|
2166
|
+
|
|
2167
|
+
console.log('\n' + chalk.bgMagenta.white.bold(` POLICY SUMMARY (${commandName.toUpperCase()}) `));
|
|
2168
|
+
console.log(chalk.magenta('╭' + '─'.repeat(65)));
|
|
2169
|
+
console.log(chalk.magenta('│') + ` File: ${chalk.white(policyConfig.policyPath)}`);
|
|
2170
|
+
console.log(
|
|
2171
|
+
chalk.magenta('│') +
|
|
2172
|
+
` Mode: ${chalk.cyan(enforcement.mode)} | Action: ${chalk.cyan(enforcement.onViolation)}`
|
|
2173
|
+
);
|
|
2174
|
+
console.log(chalk.magenta('│') + ` Total checked: ${chalk.white.bold(evaluation.totalChecked)}`);
|
|
2175
|
+
console.log(chalk.magenta('│') + ` Pass: ${chalk.green.bold(evaluation.passCount)} | Fail: ${chalk.red.bold(evaluation.failCount)}`);
|
|
2176
|
+
console.log(
|
|
2177
|
+
chalk.magenta('│') +
|
|
2178
|
+
` Suppressed: ${chalk.yellow.bold(evaluation.suppressedViolationCount || 0)} | Exceptions: ${chalk.cyan.bold(
|
|
2179
|
+
evaluation.exceptionsAppliedCount || 0
|
|
2180
|
+
)}`
|
|
2181
|
+
);
|
|
2182
|
+
|
|
2183
|
+
if (evaluation.topViolations.length === 0) {
|
|
2184
|
+
console.log(chalk.magenta('│') + ` Top violations: ${chalk.green('none')}`);
|
|
2185
|
+
} else {
|
|
2186
|
+
console.log(chalk.magenta('│') + ` Top violations:`);
|
|
2187
|
+
evaluation.topViolations.slice(0, 3).forEach((violation) => {
|
|
2188
|
+
console.log(
|
|
2189
|
+
chalk.magenta('│') +
|
|
2190
|
+
` - ${chalk.yellow(violation.code)}: ${chalk.white(violation.count)}`
|
|
2191
|
+
);
|
|
2192
|
+
});
|
|
2193
|
+
}
|
|
2194
|
+
|
|
2195
|
+
if (enforcement.shouldBlock) {
|
|
2196
|
+
console.log(
|
|
2197
|
+
chalk.magenta('│') +
|
|
2198
|
+
chalk.red.bold(
|
|
2199
|
+
` Enforcement result: blocking violations detected (exit ${enforcement.exitCode})`
|
|
2200
|
+
)
|
|
2201
|
+
);
|
|
2202
|
+
} else if (enforcement.mode === 'audit' && enforcement.hasFailures) {
|
|
2203
|
+
console.log(
|
|
2204
|
+
chalk.magenta('│') +
|
|
2205
|
+
chalk.yellow(' Audit mode: violations reported, command exits with code 0')
|
|
2206
|
+
);
|
|
2207
|
+
} else if (enforcement.onViolation === 'warn' && enforcement.hasFailures) {
|
|
2208
|
+
console.log(
|
|
2209
|
+
chalk.magenta('│') +
|
|
2210
|
+
chalk.yellow(' Enforce+warn: violations reported, command exits with code 0')
|
|
2211
|
+
);
|
|
2212
|
+
} else {
|
|
2213
|
+
console.log(chalk.magenta('│') + chalk.green(' Policy check passed'));
|
|
2214
|
+
}
|
|
2215
|
+
|
|
2216
|
+
console.log(chalk.magenta('╰' + '─'.repeat(65)));
|
|
2217
|
+
}
|
|
2218
|
+
|
|
2219
|
+
const policyCommand = program
|
|
2220
|
+
.command('policy')
|
|
2221
|
+
.description('Manage enterprise policy files (policy.yaml)')
|
|
2222
|
+
.showHelpAfterError();
|
|
2223
|
+
|
|
2224
|
+
policyCommand
|
|
2225
|
+
.command('init')
|
|
2226
|
+
.description('Create a policy.yaml template')
|
|
2227
|
+
.option('-f, --file <path>', 'Policy file path', 'policy.yaml')
|
|
2228
|
+
.option('--force', 'Overwrite existing file if it already exists')
|
|
2229
|
+
.action((options) => {
|
|
2230
|
+
try {
|
|
2231
|
+
const result = policyManager.initPolicy(options.file, {
|
|
2232
|
+
force: Boolean(options.force)
|
|
2233
|
+
});
|
|
2234
|
+
|
|
2235
|
+
const status = result.overwritten ? 'overwritten' : 'created';
|
|
2236
|
+
console.log(chalk.green(`Policy file ${status}: ${result.path}`));
|
|
2237
|
+
} catch (error) {
|
|
2238
|
+
console.error(chalk.red(`Failed to initialize policy: ${error.message}`));
|
|
2239
|
+
process.exit(1);
|
|
2240
|
+
}
|
|
2241
|
+
});
|
|
2242
|
+
|
|
2243
|
+
policyCommand
|
|
2244
|
+
.command('validate')
|
|
2245
|
+
.description('Validate policy.yaml against the v1 schema')
|
|
2246
|
+
.option('-f, --file <path>', 'Policy file path', 'policy.yaml')
|
|
2247
|
+
.option('-j, --json', 'Output validation result as JSON')
|
|
2248
|
+
.action((options) => {
|
|
2249
|
+
try {
|
|
2250
|
+
const result = policyManager.validatePolicyFile(options.file);
|
|
2251
|
+
|
|
2252
|
+
if (options.json) {
|
|
2253
|
+
console.log(JSON.stringify({
|
|
2254
|
+
valid: result.valid,
|
|
2255
|
+
file: result.path,
|
|
2256
|
+
errorCount: result.errors.length,
|
|
2257
|
+
errors: result.errors
|
|
2258
|
+
}, null, 2));
|
|
2259
|
+
if (!result.valid) {
|
|
2260
|
+
process.exit(1);
|
|
2261
|
+
}
|
|
2262
|
+
} else if (result.valid) {
|
|
2263
|
+
const mode = result.policy?.mode || 'unknown';
|
|
2264
|
+
console.log(chalk.green(`Policy is valid (${mode} mode): ${result.path}`));
|
|
2265
|
+
} else {
|
|
2266
|
+
console.error(chalk.red(`Policy validation failed: ${result.path}`));
|
|
2267
|
+
result.errors.forEach((entry) => {
|
|
2268
|
+
console.error(chalk.red(` - ${entry.path}: ${entry.message}`));
|
|
2269
|
+
});
|
|
2270
|
+
process.exit(1);
|
|
2271
|
+
}
|
|
2272
|
+
} catch (error) {
|
|
2273
|
+
if (options.json) {
|
|
2274
|
+
console.log(JSON.stringify({
|
|
2275
|
+
valid: false,
|
|
2276
|
+
file: policyManager.resolvePolicyPath(options.file),
|
|
2277
|
+
errorCount: 1,
|
|
2278
|
+
errors: [{ path: 'file', message: error.message }]
|
|
2279
|
+
}, null, 2));
|
|
2280
|
+
} else {
|
|
2281
|
+
console.error(chalk.red(`Policy validation failed: ${error.message}`));
|
|
2282
|
+
}
|
|
2283
|
+
process.exit(1);
|
|
2284
|
+
}
|
|
2285
|
+
});
|
|
2286
|
+
|
|
2287
|
+
policyCommand.action(() => {
|
|
2288
|
+
policyCommand.outputHelp();
|
|
2289
|
+
});
|
|
2290
|
+
|
|
2291
|
+
const auditCommand = program
|
|
2292
|
+
.command('audit')
|
|
2293
|
+
.description('Run policy audits and export compliance reports')
|
|
2294
|
+
.showHelpAfterError();
|
|
2295
|
+
|
|
2296
|
+
auditCommand
|
|
2297
|
+
.command('export')
|
|
2298
|
+
.description('Evaluate policy compliance and export JSON/CSV/SARIF reports')
|
|
2299
|
+
.requiredOption('--policy <file>', 'Policy file path')
|
|
2300
|
+
.option('--command <name>', 'Evaluation source: check | recommend', 'check')
|
|
2301
|
+
.option('--format <format>', 'Report format: json | csv | sarif | all', 'json')
|
|
2302
|
+
.option('--out <path>', 'Output file path (single-format export only)')
|
|
2303
|
+
.option('--out-dir <path>', 'Output directory when --out is omitted', 'audit-reports')
|
|
2304
|
+
.option('-u, --use-case <case>', 'Use case when --command check is selected', 'general')
|
|
2305
|
+
.option('-c, --category <category>', 'Category hint when --command recommend is selected')
|
|
2306
|
+
.option('--runtime <runtime>', `Runtime for check mode (${SUPPORTED_RUNTIMES.join('|')})`, 'ollama')
|
|
2307
|
+
.option('--include-cloud', 'Include cloud models in check-mode analysis')
|
|
2308
|
+
.option('--max-size <size>', 'Maximum model size for check mode (e.g., "24B" or "12GB")')
|
|
2309
|
+
.option('--min-size <size>', 'Minimum model size for check mode (e.g., "3B" or "2GB")')
|
|
2310
|
+
.option('-l, --limit <number>', 'Model analysis limit for check mode', '25')
|
|
2311
|
+
.option('--no-verbose', 'Disable verbose progress while collecting audit inputs')
|
|
2312
|
+
.action(async (options) => {
|
|
2313
|
+
try {
|
|
2314
|
+
const policyConfig = loadPolicyConfiguration(options.policy);
|
|
2315
|
+
const selectedCommand = String(options.command || 'check')
|
|
2316
|
+
.toLowerCase()
|
|
2317
|
+
.trim();
|
|
2318
|
+
|
|
2319
|
+
if (!['check', 'recommend'].includes(selectedCommand)) {
|
|
2320
|
+
throw new Error('Invalid --command value. Use "check" or "recommend".');
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2323
|
+
const exportFormats = resolveAuditFormats(options.format, policyConfig.policy);
|
|
2324
|
+
if (options.out && exportFormats.length > 1) {
|
|
2325
|
+
throw new Error('--out can only be used with a single export format.');
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
const verboseEnabled = options.verbose !== false;
|
|
2329
|
+
const checker = new (getLLMChecker())({ verbose: verboseEnabled });
|
|
2330
|
+
const hardware = await checker.getSystemInfo();
|
|
2331
|
+
|
|
2332
|
+
let runtimeBackend = 'ollama';
|
|
2333
|
+
let policyCandidates = [];
|
|
2334
|
+
let analysisResult = null;
|
|
2335
|
+
let recommendationResult = null;
|
|
2336
|
+
|
|
2337
|
+
if (selectedCommand === 'check') {
|
|
2338
|
+
let selectedRuntime = normalizeRuntime(options.runtime);
|
|
2339
|
+
if (!runtimeSupportedOnHardware(selectedRuntime, hardware)) {
|
|
2340
|
+
selectedRuntime = 'ollama';
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
const maxSize = parseSizeFilterInput(options.maxSize);
|
|
2344
|
+
const minSize = parseSizeFilterInput(options.minSize);
|
|
2345
|
+
const normalizedUseCase = normalizeUseCaseInput(options.useCase);
|
|
2346
|
+
|
|
2347
|
+
analysisResult = await checker.analyze({
|
|
2348
|
+
useCase: normalizedUseCase,
|
|
2349
|
+
includeCloud: Boolean(options.includeCloud),
|
|
2350
|
+
limit: Number.parseInt(options.limit, 10) || 25,
|
|
2351
|
+
maxSize,
|
|
2352
|
+
minSize,
|
|
2353
|
+
runtime: selectedRuntime
|
|
2354
|
+
});
|
|
2355
|
+
|
|
2356
|
+
runtimeBackend = selectedRuntime;
|
|
2357
|
+
policyCandidates = collectCandidatesFromAnalysis(analysisResult);
|
|
2358
|
+
} else {
|
|
2359
|
+
recommendationResult = await checker.generateIntelligentRecommendations(hardware);
|
|
2360
|
+
if (!recommendationResult) {
|
|
2361
|
+
throw new Error('Unable to generate recommendation data for policy audit export.');
|
|
2362
|
+
}
|
|
2363
|
+
|
|
2364
|
+
runtimeBackend = normalizeRuntime(options.runtime || 'ollama');
|
|
2365
|
+
policyCandidates = collectCandidatesFromRecommendationData(recommendationResult);
|
|
2366
|
+
}
|
|
2367
|
+
|
|
2368
|
+
const policyContext = buildPolicyRuntimeContext({
|
|
2369
|
+
hardware,
|
|
2370
|
+
runtimeBackend
|
|
2371
|
+
});
|
|
2372
|
+
|
|
2373
|
+
const policyEvaluation = evaluatePolicyCandidates(
|
|
2374
|
+
policyConfig.policyEngine,
|
|
2375
|
+
policyCandidates,
|
|
2376
|
+
policyContext,
|
|
2377
|
+
policyConfig.policy
|
|
2378
|
+
);
|
|
2379
|
+
const policyEnforcement = resolvePolicyEnforcement(policyConfig.policy, policyEvaluation);
|
|
2380
|
+
|
|
2381
|
+
const report = buildComplianceReport({
|
|
2382
|
+
commandName: selectedCommand,
|
|
2383
|
+
policyPath: policyConfig.policyPath,
|
|
2384
|
+
policy: policyConfig.policy,
|
|
2385
|
+
evaluation: policyEvaluation,
|
|
2386
|
+
enforcement: policyEnforcement,
|
|
2387
|
+
runtimeContext: policyContext,
|
|
2388
|
+
options: {
|
|
2389
|
+
format: exportFormats,
|
|
2390
|
+
runtime: runtimeBackend,
|
|
2391
|
+
use_case: selectedCommand === 'check' ? normalizeUseCaseInput(options.useCase) : null,
|
|
2392
|
+
category: selectedCommand === 'recommend' ? options.category || null : null,
|
|
2393
|
+
include_cloud: Boolean(options.includeCloud)
|
|
2394
|
+
},
|
|
2395
|
+
hardware
|
|
2396
|
+
});
|
|
2397
|
+
|
|
2398
|
+
const generatedAt = report.generated_at || new Date().toISOString();
|
|
2399
|
+
const writtenFiles = [];
|
|
2400
|
+
exportFormats.forEach((format) => {
|
|
2401
|
+
const filePath = toAuditOutputPath({
|
|
2402
|
+
outputPath: options.out,
|
|
2403
|
+
outputDir: options.outDir,
|
|
2404
|
+
commandName: selectedCommand,
|
|
2405
|
+
format,
|
|
2406
|
+
timestamp: generatedAt
|
|
2407
|
+
});
|
|
2408
|
+
const content = serializeComplianceReport(report, format);
|
|
2409
|
+
writeReportFile(filePath, content);
|
|
2410
|
+
writtenFiles.push({ format, filePath });
|
|
2411
|
+
});
|
|
2412
|
+
|
|
2413
|
+
displayPolicySummary(`audit ${selectedCommand}`, policyConfig, policyEvaluation, policyEnforcement);
|
|
2414
|
+
|
|
2415
|
+
console.log('\n' + chalk.bgBlue.white.bold(' AUDIT EXPORT '));
|
|
2416
|
+
writtenFiles.forEach((entry) => {
|
|
2417
|
+
console.log(`${chalk.cyan(entry.format.toUpperCase())}: ${chalk.white(entry.filePath)}`);
|
|
2418
|
+
});
|
|
2419
|
+
|
|
2420
|
+
if (policyEnforcement.shouldBlock) {
|
|
2421
|
+
process.exit(policyEnforcement.exitCode);
|
|
2422
|
+
}
|
|
2423
|
+
} catch (error) {
|
|
2424
|
+
console.error(chalk.red(`Audit export failed: ${error.message}`));
|
|
2425
|
+
if (process.env.DEBUG) {
|
|
2426
|
+
console.error(error.stack);
|
|
2427
|
+
}
|
|
2428
|
+
process.exit(1);
|
|
2429
|
+
}
|
|
2430
|
+
});
|
|
2431
|
+
|
|
2432
|
+
auditCommand.action(() => {
|
|
2433
|
+
auditCommand.outputHelp();
|
|
2434
|
+
});
|
|
2435
|
+
|
|
1984
2436
|
program
|
|
1985
2437
|
.command('check')
|
|
1986
2438
|
.description('Analyze your system and show compatible LLM models')
|
|
@@ -1992,15 +2444,31 @@ program
|
|
|
1992
2444
|
.option('--min-size <size>', 'Minimum model size to consider (e.g., "7B" or "7GB")')
|
|
1993
2445
|
.option('--include-cloud', 'Include cloud models in analysis')
|
|
1994
2446
|
.option('--ollama-only', 'Only show models available in Ollama')
|
|
2447
|
+
.option('--runtime <runtime>', `Inference runtime (${SUPPORTED_RUNTIMES.join('|')})`, 'ollama')
|
|
2448
|
+
.option('--policy <file>', 'Evaluate candidate models against a policy file')
|
|
1995
2449
|
.option('--performance-test', 'Run performance benchmarks')
|
|
1996
2450
|
.option('--show-ollama-analysis', 'Show detailed Ollama model analysis')
|
|
1997
2451
|
.option('--no-verbose', 'Disable step-by-step progress display')
|
|
2452
|
+
.addHelpText(
|
|
2453
|
+
'after',
|
|
2454
|
+
`
|
|
2455
|
+
Enterprise policy examples:
|
|
2456
|
+
$ llm-checker check --policy ./policy.yaml
|
|
2457
|
+
$ llm-checker check --policy ./policy.yaml --use-case coding --runtime vllm
|
|
2458
|
+
$ llm-checker check --policy ./policy.yaml --include-cloud --max-size 24B
|
|
2459
|
+
|
|
2460
|
+
Policy scope:
|
|
2461
|
+
- Evaluates all compatible and marginal candidates discovered during analysis
|
|
2462
|
+
- Not limited to the top --limit results shown in output
|
|
2463
|
+
`
|
|
2464
|
+
)
|
|
1998
2465
|
.action(async (options) => {
|
|
1999
2466
|
showAsciiArt('check');
|
|
2000
2467
|
try {
|
|
2001
2468
|
// Use verbose progress unless explicitly disabled
|
|
2002
2469
|
const verboseEnabled = options.verbose !== false;
|
|
2003
2470
|
const checker = new (getLLMChecker())({ verbose: verboseEnabled });
|
|
2471
|
+
const policyConfig = options.policy ? loadPolicyConfiguration(options.policy) : null;
|
|
2004
2472
|
|
|
2005
2473
|
// If verbose is disabled, show simple loading message
|
|
2006
2474
|
if (!verboseEnabled) {
|
|
@@ -2008,6 +2476,16 @@ program
|
|
|
2008
2476
|
}
|
|
2009
2477
|
|
|
2010
2478
|
const hardware = await checker.getSystemInfo();
|
|
2479
|
+
let selectedRuntime = normalizeRuntime(options.runtime);
|
|
2480
|
+
if (!runtimeSupportedOnHardware(selectedRuntime, hardware)) {
|
|
2481
|
+
const runtimeLabel = getRuntimeDisplayName(selectedRuntime);
|
|
2482
|
+
console.log(
|
|
2483
|
+
chalk.yellow(
|
|
2484
|
+
`\nWarning: ${runtimeLabel} is not supported on this hardware. Falling back to Ollama.`
|
|
2485
|
+
)
|
|
2486
|
+
);
|
|
2487
|
+
selectedRuntime = 'ollama';
|
|
2488
|
+
}
|
|
2011
2489
|
|
|
2012
2490
|
// Normalize and fix use-case typos
|
|
2013
2491
|
const normalizeUseCase = (useCase = '') => {
|
|
@@ -2049,17 +2527,48 @@ program
|
|
|
2049
2527
|
performanceTest: options.performanceTest,
|
|
2050
2528
|
limit: parseInt(options.limit) || 10,
|
|
2051
2529
|
maxSize: maxSize,
|
|
2052
|
-
minSize: minSize
|
|
2530
|
+
minSize: minSize,
|
|
2531
|
+
runtime: selectedRuntime
|
|
2053
2532
|
});
|
|
2054
2533
|
|
|
2055
2534
|
if (!verboseEnabled) {
|
|
2056
2535
|
console.log(chalk.green(' done'));
|
|
2057
2536
|
}
|
|
2058
2537
|
|
|
2538
|
+
let policyEvaluation = null;
|
|
2539
|
+
let policyEnforcement = null;
|
|
2540
|
+
if (policyConfig) {
|
|
2541
|
+
const policyCandidates = collectCandidatesFromAnalysis(analysis);
|
|
2542
|
+
const policyContext = buildPolicyRuntimeContext({
|
|
2543
|
+
hardware,
|
|
2544
|
+
runtimeBackend: selectedRuntime
|
|
2545
|
+
});
|
|
2546
|
+
policyEvaluation = evaluatePolicyCandidates(
|
|
2547
|
+
policyConfig.policyEngine,
|
|
2548
|
+
policyCandidates,
|
|
2549
|
+
policyContext,
|
|
2550
|
+
policyConfig.policy
|
|
2551
|
+
);
|
|
2552
|
+
policyEnforcement = resolvePolicyEnforcement(policyConfig.policy, policyEvaluation);
|
|
2553
|
+
}
|
|
2554
|
+
|
|
2059
2555
|
// Simplified output - show only essential information
|
|
2060
2556
|
displaySimplifiedSystemInfo(hardware);
|
|
2061
|
-
const recommendedModels = await displayModelRecommendations(
|
|
2062
|
-
|
|
2557
|
+
const recommendedModels = await displayModelRecommendations(
|
|
2558
|
+
analysis,
|
|
2559
|
+
hardware,
|
|
2560
|
+
normalizedUseCase,
|
|
2561
|
+
parseInt(options.limit) || 1,
|
|
2562
|
+
selectedRuntime
|
|
2563
|
+
);
|
|
2564
|
+
await displayQuickStartCommands(analysis, recommendedModels[0], recommendedModels, selectedRuntime);
|
|
2565
|
+
|
|
2566
|
+
if (policyConfig && policyEvaluation && policyEnforcement) {
|
|
2567
|
+
displayPolicySummary('check', policyConfig, policyEvaluation, policyEnforcement);
|
|
2568
|
+
if (policyEnforcement.shouldBlock) {
|
|
2569
|
+
process.exit(policyEnforcement.exitCode);
|
|
2570
|
+
}
|
|
2571
|
+
}
|
|
2063
2572
|
|
|
2064
2573
|
} catch (error) {
|
|
2065
2574
|
console.error(chalk.red('\nError:'), error.message);
|
|
@@ -2290,11 +2799,22 @@ program
|
|
|
2290
2799
|
.description('Get intelligent model recommendations for your hardware')
|
|
2291
2800
|
.option('-c, --category <category>', 'Get recommendations for specific category (coding, talking, reading, etc.)')
|
|
2292
2801
|
.option('--no-verbose', 'Disable step-by-step progress display')
|
|
2802
|
+
.option('--policy <file>', 'Evaluate recommendations against a policy file')
|
|
2803
|
+
.addHelpText(
|
|
2804
|
+
'after',
|
|
2805
|
+
`
|
|
2806
|
+
Enterprise policy examples:
|
|
2807
|
+
$ llm-checker recommend --policy ./policy.yaml
|
|
2808
|
+
$ llm-checker recommend --policy ./policy.yaml --category coding
|
|
2809
|
+
$ llm-checker recommend --policy ./policy.yaml --no-verbose
|
|
2810
|
+
`
|
|
2811
|
+
)
|
|
2293
2812
|
.action(async (options) => {
|
|
2294
2813
|
showAsciiArt('recommend');
|
|
2295
2814
|
try {
|
|
2296
2815
|
const verboseEnabled = options.verbose !== false;
|
|
2297
2816
|
const checker = new (getLLMChecker())({ verbose: verboseEnabled });
|
|
2817
|
+
const policyConfig = options.policy ? loadPolicyConfiguration(options.policy) : null;
|
|
2298
2818
|
|
|
2299
2819
|
if (!verboseEnabled) {
|
|
2300
2820
|
process.stdout.write(chalk.gray('Generating recommendations...'));
|
|
@@ -2312,12 +2832,36 @@ program
|
|
|
2312
2832
|
console.log(chalk.green(' done'));
|
|
2313
2833
|
}
|
|
2314
2834
|
|
|
2835
|
+
let policyEvaluation = null;
|
|
2836
|
+
let policyEnforcement = null;
|
|
2837
|
+
if (policyConfig) {
|
|
2838
|
+
const policyCandidates = collectCandidatesFromRecommendationData(intelligentRecommendations);
|
|
2839
|
+
const policyContext = buildPolicyRuntimeContext({
|
|
2840
|
+
hardware,
|
|
2841
|
+
runtimeBackend: 'ollama'
|
|
2842
|
+
});
|
|
2843
|
+
policyEvaluation = evaluatePolicyCandidates(
|
|
2844
|
+
policyConfig.policyEngine,
|
|
2845
|
+
policyCandidates,
|
|
2846
|
+
policyContext,
|
|
2847
|
+
policyConfig.policy
|
|
2848
|
+
);
|
|
2849
|
+
policyEnforcement = resolvePolicyEnforcement(policyConfig.policy, policyEvaluation);
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2315
2852
|
// Mostrar información del sistema
|
|
2316
2853
|
displaySystemInfo(hardware, { summary: { hardwareTier: intelligentRecommendations.summary.hardware_tier } });
|
|
2317
2854
|
|
|
2318
2855
|
// Mostrar recomendaciones
|
|
2319
2856
|
displayIntelligentRecommendations(intelligentRecommendations);
|
|
2320
2857
|
|
|
2858
|
+
if (policyConfig && policyEvaluation && policyEnforcement) {
|
|
2859
|
+
displayPolicySummary('recommend', policyConfig, policyEvaluation, policyEnforcement);
|
|
2860
|
+
if (policyEnforcement.shouldBlock) {
|
|
2861
|
+
process.exit(policyEnforcement.exitCode);
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
|
|
2321
2865
|
} catch (error) {
|
|
2322
2866
|
console.error(chalk.red('\nError:'), error.message);
|
|
2323
2867
|
if (process.env.DEBUG) {
|