llm-checker 3.2.0 → 3.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,6 +16,28 @@ function getLLMChecker() {
16
16
  const { getLogger } = require('../src/utils/logger');
17
17
  const fs = require('fs');
18
18
  const path = require('path');
19
+ const {
20
+ SUPPORTED_RUNTIMES,
21
+ normalizeRuntime,
22
+ runtimeSupportedOnHardware,
23
+ getRuntimeDisplayName,
24
+ getRuntimeCommandSet
25
+ } = require('../src/runtime/runtime-support');
26
+ const SpeculativeDecodingEstimator = require('../src/models/speculative-decoding-estimator');
27
+ const PolicyManager = require('../src/policy/policy-manager');
28
+ const PolicyEngine = require('../src/policy/policy-engine');
29
+ const {
30
+ collectCandidatesFromAnalysis,
31
+ collectCandidatesFromRecommendationData,
32
+ buildPolicyRuntimeContext,
33
+ evaluatePolicyCandidates,
34
+ resolvePolicyEnforcement
35
+ } = require('../src/policy/cli-policy');
36
+ const {
37
+ buildComplianceReport,
38
+ serializeComplianceReport
39
+ } = require('../src/policy/audit-reporter');
40
+ const policyManager = new PolicyManager();
19
41
 
20
42
  // ASCII Art for each command - Large text banners
21
43
  const ASCII_ART = {
@@ -1406,10 +1428,18 @@ function displaySimplifiedSystemInfo(hardware) {
1406
1428
  console.log(`Hardware Tier: ${tierColor.bold(tier)}`);
1407
1429
  }
1408
1430
 
1409
- async function displayModelRecommendations(analysis, hardware, useCase = 'general', limit = 1) {
1431
+ async function displayModelRecommendations(analysis, hardware, useCase = 'general', limit = 1, runtime = 'ollama') {
1410
1432
  const title = limit === 1 ? 'RECOMMENDED MODEL' : `TOP ${limit} COMPATIBLE MODELS`;
1411
1433
  console.log(chalk.green.bold(`\n${title}`));
1412
1434
  console.log(chalk.gray('─'.repeat(50)));
1435
+
1436
+ const selectedRuntime = normalizeRuntime(runtime);
1437
+ const runtimeLabel = getRuntimeDisplayName(selectedRuntime);
1438
+ const speculativeEstimator = new SpeculativeDecodingEstimator();
1439
+ const speculativeCandidatePool = [
1440
+ ...(analysis?.compatible || []),
1441
+ ...(analysis?.marginal || [])
1442
+ ];
1413
1443
 
1414
1444
  // Find the best models from compatible models considering use case
1415
1445
  let selectedModels = [];
@@ -1760,42 +1790,75 @@ async function displayModelRecommendations(analysis, hardware, useCase = 'genera
1760
1790
  if (model.performanceEstimate) {
1761
1791
  console.log(`Estimated Speed: ${chalk.yellow(model.performanceEstimate.estimatedTokensPerSecond || 'N/A')} tokens/sec`);
1762
1792
  }
1763
-
1764
- // Check if it's already installed by comparing with Ollama integration
1793
+
1794
+ console.log(`Runtime: ${chalk.white(runtimeLabel)}`);
1795
+ const runtimeCommands = getRuntimeCommandSet(model, selectedRuntime);
1796
+
1797
+ // Check installation only when using Ollama runtime.
1765
1798
  let isInstalled = false;
1766
- try {
1767
- isInstalled = await checkIfModelInstalled(model, analysis.ollamaInfo);
1768
- if (isInstalled) {
1769
- console.log(`Status: ${chalk.green('Already installed in Ollama')}`);
1770
- } else if (analysis.ollamaInfo && analysis.ollamaInfo.available) {
1771
- console.log(`Status: ${chalk.gray('Available for installation')}`);
1772
- } else {
1773
- console.log(`Status: ${chalk.yellow('Requires Ollama (not detected)')}`);
1799
+ if (selectedRuntime === 'ollama') {
1800
+ try {
1801
+ isInstalled = await checkIfModelInstalled(model, analysis.ollamaInfo);
1802
+ if (isInstalled) {
1803
+ console.log(`Status: ${chalk.green('Already installed in Ollama')}`);
1804
+ } else if (analysis.ollamaInfo && analysis.ollamaInfo.available) {
1805
+ console.log(`Status: ${chalk.gray('Available for installation')}`);
1806
+ } else {
1807
+ console.log(`Status: ${chalk.yellow('Requires Ollama (not detected)')}`);
1808
+ }
1809
+ } catch (installCheckError) {
1810
+ if (analysis.ollamaInfo && analysis.ollamaInfo.available) {
1811
+ console.log(`Status: ${chalk.gray('Available for installation')}`);
1812
+ } else {
1813
+ console.log(`Status: ${chalk.yellow('Requires Ollama (not detected)')}`);
1814
+ }
1774
1815
  }
1775
- } catch (installCheckError) {
1776
- // If checking installation status fails, show based on Ollama availability
1777
- if (analysis.ollamaInfo && analysis.ollamaInfo.available) {
1778
- console.log(`Status: ${chalk.gray('Available for installation')}`);
1779
- } else {
1780
- console.log(`Status: ${chalk.yellow('Requires Ollama (not detected)')}`);
1816
+
1817
+ const ollamaCommand = getOllamaInstallCommand(model);
1818
+ if (ollamaCommand) {
1819
+ const modelName = extractModelName(ollamaCommand);
1820
+ if (isInstalled) {
1821
+ console.log(`\nRun: ${chalk.cyan.bold(`ollama run ${modelName}`)}`);
1822
+ } else {
1823
+ console.log(`\nPull: ${chalk.cyan.bold(ollamaCommand)}`);
1824
+ }
1825
+ } else if (model.ollamaTag || model.ollamaId) {
1826
+ const tag = model.ollamaTag || model.ollamaId;
1827
+ if (isInstalled) {
1828
+ console.log(`\nRun: ${chalk.cyan.bold(`ollama run ${tag}`)}`);
1829
+ } else {
1830
+ console.log(`\nPull: ${chalk.cyan.bold(`ollama pull ${tag}`)}`);
1831
+ }
1832
+ }
1833
+ } else {
1834
+ console.log(`Status: ${chalk.gray(`${runtimeLabel} runtime selected`)}`);
1835
+ console.log(`\nRun: ${chalk.cyan.bold(runtimeCommands.run)}`);
1836
+ if (index === 0) {
1837
+ console.log(`Install runtime: ${chalk.cyan.bold(runtimeCommands.install)}`);
1838
+ console.log(`Fetch model: ${chalk.cyan.bold(runtimeCommands.pull)}`);
1781
1839
  }
1782
1840
  }
1783
1841
 
1784
- // Show pull/run command directly in each model block (Issue #3)
1785
- const ollamaCommand = getOllamaInstallCommand(model);
1786
- if (ollamaCommand) {
1787
- const modelName = extractModelName(ollamaCommand);
1788
- if (isInstalled) {
1789
- console.log(`\nCommand: ${chalk.cyan.bold(`ollama run ${modelName}`)}`);
1790
- } else {
1791
- console.log(`\nCommand: ${chalk.cyan.bold(ollamaCommand)}`);
1792
- }
1793
- } else if (model.ollamaTag || model.ollamaId) {
1794
- const tag = model.ollamaTag || model.ollamaId;
1795
- if (isInstalled) {
1796
- console.log(`\nCommand: ${chalk.cyan.bold(`ollama run ${tag}`)}`);
1797
- } else {
1798
- console.log(`\nCommand: ${chalk.cyan.bold(`ollama pull ${tag}`)}`);
1842
+ const speculativeInfo =
1843
+ model.speculativeDecoding ||
1844
+ speculativeEstimator.estimate({
1845
+ model,
1846
+ candidates: speculativeCandidatePool,
1847
+ hardware,
1848
+ runtime: selectedRuntime
1849
+ });
1850
+
1851
+ if (speculativeInfo && speculativeInfo.runtime === selectedRuntime) {
1852
+ if (speculativeInfo.enabled) {
1853
+ console.log(
1854
+ `SpecDec: ${chalk.green(`+${speculativeInfo.estimatedThroughputGainPct}%`)} ` +
1855
+ `(${chalk.gray(`draft: ${speculativeInfo.draftModel}`)})`
1856
+ );
1857
+ } else if (speculativeInfo.estimatedSpeedup) {
1858
+ const suggested = speculativeInfo.suggestedDraftModel ? ` with ${speculativeInfo.suggestedDraftModel}` : '';
1859
+ console.log(
1860
+ `SpecDec estimate: ${chalk.yellow(`+${speculativeInfo.estimatedThroughputGainPct}%`)}${chalk.gray(suggested)}`
1861
+ );
1799
1862
  }
1800
1863
  }
1801
1864
  }
@@ -1807,9 +1870,12 @@ async function displayModelRecommendations(analysis, hardware, useCase = 'genera
1807
1870
  return selectedModels;
1808
1871
  }
1809
1872
 
1810
- async function displayQuickStartCommands(analysis, recommendedModel = null, allRecommended = null) {
1873
+ async function displayQuickStartCommands(analysis, recommendedModel = null, allRecommended = null, runtime = 'ollama') {
1811
1874
  console.log(chalk.yellow.bold('\nQUICK START'));
1812
1875
  console.log(chalk.gray('─'.repeat(50)));
1876
+
1877
+ const selectedRuntime = normalizeRuntime(runtime);
1878
+ const runtimeLabel = getRuntimeDisplayName(selectedRuntime);
1813
1879
 
1814
1880
  // Use the first model from allRecommended if available, otherwise fallback to recommendedModel
1815
1881
  let bestModel = (allRecommended && allRecommended.length > 0) ? allRecommended[0] : recommendedModel;
@@ -1824,6 +1890,33 @@ async function displayQuickStartCommands(analysis, recommendedModel = null, allR
1824
1890
  }
1825
1891
  }
1826
1892
 
1893
+ if (selectedRuntime !== 'ollama') {
1894
+ if (!bestModel) {
1895
+ console.log(`1. Try expanding search: ${chalk.cyan('llm-checker check --include-cloud')}`);
1896
+ return;
1897
+ }
1898
+
1899
+ const runtimeCommands = getRuntimeCommandSet(bestModel, selectedRuntime);
1900
+ console.log(`1. Install ${runtimeLabel}:`);
1901
+ console.log(` ${chalk.cyan.bold(runtimeCommands.install)}`);
1902
+ console.log(`2. Fetch model weights:`);
1903
+ console.log(` ${chalk.cyan.bold(runtimeCommands.pull)}`);
1904
+ console.log(`3. Run model:`);
1905
+ console.log(` ${chalk.cyan.bold(runtimeCommands.run)}`);
1906
+
1907
+ const speculative = bestModel.speculativeDecoding;
1908
+ if (speculative && speculative.enabled) {
1909
+ console.log(`4. SpecDec suggestion (${chalk.green(`+${speculative.estimatedThroughputGainPct}%`)}):`);
1910
+ if (selectedRuntime === 'vllm') {
1911
+ console.log(` ${chalk.cyan.bold(`${runtimeCommands.run} --speculative-model '${speculative.draftModelRef || speculative.draftModel}'`)}`);
1912
+ } else if (selectedRuntime === 'mlx') {
1913
+ console.log(` ${chalk.gray(`Use draft model ${speculative.draftModelRef || speculative.draftModel} when enabling speculative decoding in MLX-LM`)}`);
1914
+ }
1915
+ }
1916
+
1917
+ return;
1918
+ }
1919
+
1827
1920
  if (analysis.ollamaInfo && !analysis.ollamaInfo.available) {
1828
1921
  console.log(`1. Install Ollama: ${chalk.underline('https://ollama.ai')}`);
1829
1922
  console.log(`2. Come back and run this command again`);
@@ -1981,6 +2074,365 @@ function extractModelName(command) {
1981
2074
  return match ? match[1] : 'model';
1982
2075
  }
1983
2076
 
2077
+ function loadPolicyConfiguration(policyFile) {
2078
+ const validation = policyManager.validatePolicyFile(policyFile);
2079
+ if (!validation.valid) {
2080
+ const details = validation.errors
2081
+ .map((entry) => `${entry.path}: ${entry.message}`)
2082
+ .join('; ');
2083
+ throw new Error(`Invalid policy file: ${details}`);
2084
+ }
2085
+
2086
+ return {
2087
+ policyPath: validation.path,
2088
+ policy: validation.policy,
2089
+ policyEngine: new PolicyEngine(validation.policy)
2090
+ };
2091
+ }
2092
+
2093
+ function parseSizeFilterInput(sizeStr) {
2094
+ if (!sizeStr) return null;
2095
+ const match = String(sizeStr)
2096
+ .toUpperCase()
2097
+ .trim()
2098
+ .match(/^([0-9]+(?:\.[0-9]+)?)\s*(B|GB)?$/);
2099
+ if (!match) return null;
2100
+
2101
+ const value = Number.parseFloat(match[1]);
2102
+ const unit = match[2] || 'B';
2103
+
2104
+ // Convert to "B params" approximation used by existing check flow
2105
+ return unit === 'GB' ? value / 0.5 : value;
2106
+ }
2107
+
2108
+ function normalizeUseCaseInput(useCase = '') {
2109
+ const alias = String(useCase || '')
2110
+ .toLowerCase()
2111
+ .trim();
2112
+
2113
+ const useCaseMap = {
2114
+ embed: 'embeddings',
2115
+ embedding: 'embeddings',
2116
+ embeddings: 'embeddings',
2117
+ embedings: 'embeddings',
2118
+ talk: 'chat',
2119
+ talking: 'chat',
2120
+ conversation: 'chat',
2121
+ chat: 'chat'
2122
+ };
2123
+
2124
+ return useCaseMap[alias] || alias || 'general';
2125
+ }
2126
+
2127
+ function resolveAuditFormats(formatOption, policy) {
2128
+ const requested = String(formatOption || 'json').trim().toLowerCase();
2129
+ const allowed = new Set(['json', 'csv', 'sarif']);
2130
+
2131
+ if (requested === 'all') {
2132
+ const configured = Array.isArray(policy?.reporting?.formats)
2133
+ ? policy.reporting.formats
2134
+ .map((entry) => String(entry || '').trim().toLowerCase())
2135
+ .filter((entry) => allowed.has(entry))
2136
+ : [];
2137
+
2138
+ return configured.length > 0 ? configured : ['json', 'csv', 'sarif'];
2139
+ }
2140
+
2141
+ if (!allowed.has(requested)) {
2142
+ throw new Error('Invalid format. Use one of: json, csv, sarif, all');
2143
+ }
2144
+
2145
+ return [requested];
2146
+ }
2147
+
2148
+ function toAuditOutputPath({ outputPath, outputDir, commandName, format, timestamp }) {
2149
+ if (outputPath) {
2150
+ return path.resolve(outputPath);
2151
+ }
2152
+
2153
+ const safeTimestamp = timestamp.replace(/[:.]/g, '-');
2154
+ const extension = format === 'sarif' ? 'sarif.json' : format;
2155
+ const fileName = `${commandName}-policy-audit-${safeTimestamp}.${extension}`;
2156
+ return path.resolve(outputDir || 'audit-reports', fileName);
2157
+ }
2158
+
2159
+ function writeReportFile(filePath, content) {
2160
+ fs.mkdirSync(path.dirname(filePath), { recursive: true });
2161
+ fs.writeFileSync(filePath, content, 'utf8');
2162
+ }
2163
+
2164
+ function displayPolicySummary(commandName, policyConfig, evaluation, enforcement) {
2165
+ if (!policyConfig || !evaluation || !enforcement) return;
2166
+
2167
+ console.log('\n' + chalk.bgMagenta.white.bold(` POLICY SUMMARY (${commandName.toUpperCase()}) `));
2168
+ console.log(chalk.magenta('╭' + '─'.repeat(65)));
2169
+ console.log(chalk.magenta('│') + ` File: ${chalk.white(policyConfig.policyPath)}`);
2170
+ console.log(
2171
+ chalk.magenta('│') +
2172
+ ` Mode: ${chalk.cyan(enforcement.mode)} | Action: ${chalk.cyan(enforcement.onViolation)}`
2173
+ );
2174
+ console.log(chalk.magenta('│') + ` Total checked: ${chalk.white.bold(evaluation.totalChecked)}`);
2175
+ console.log(chalk.magenta('│') + ` Pass: ${chalk.green.bold(evaluation.passCount)} | Fail: ${chalk.red.bold(evaluation.failCount)}`);
2176
+ console.log(
2177
+ chalk.magenta('│') +
2178
+ ` Suppressed: ${chalk.yellow.bold(evaluation.suppressedViolationCount || 0)} | Exceptions: ${chalk.cyan.bold(
2179
+ evaluation.exceptionsAppliedCount || 0
2180
+ )}`
2181
+ );
2182
+
2183
+ if (evaluation.topViolations.length === 0) {
2184
+ console.log(chalk.magenta('│') + ` Top violations: ${chalk.green('none')}`);
2185
+ } else {
2186
+ console.log(chalk.magenta('│') + ` Top violations:`);
2187
+ evaluation.topViolations.slice(0, 3).forEach((violation) => {
2188
+ console.log(
2189
+ chalk.magenta('│') +
2190
+ ` - ${chalk.yellow(violation.code)}: ${chalk.white(violation.count)}`
2191
+ );
2192
+ });
2193
+ }
2194
+
2195
+ if (enforcement.shouldBlock) {
2196
+ console.log(
2197
+ chalk.magenta('│') +
2198
+ chalk.red.bold(
2199
+ ` Enforcement result: blocking violations detected (exit ${enforcement.exitCode})`
2200
+ )
2201
+ );
2202
+ } else if (enforcement.mode === 'audit' && enforcement.hasFailures) {
2203
+ console.log(
2204
+ chalk.magenta('│') +
2205
+ chalk.yellow(' Audit mode: violations reported, command exits with code 0')
2206
+ );
2207
+ } else if (enforcement.onViolation === 'warn' && enforcement.hasFailures) {
2208
+ console.log(
2209
+ chalk.magenta('│') +
2210
+ chalk.yellow(' Enforce+warn: violations reported, command exits with code 0')
2211
+ );
2212
+ } else {
2213
+ console.log(chalk.magenta('│') + chalk.green(' Policy check passed'));
2214
+ }
2215
+
2216
+ console.log(chalk.magenta('╰' + '─'.repeat(65)));
2217
+ }
2218
+
2219
+ const policyCommand = program
2220
+ .command('policy')
2221
+ .description('Manage enterprise policy files (policy.yaml)')
2222
+ .showHelpAfterError();
2223
+
2224
+ policyCommand
2225
+ .command('init')
2226
+ .description('Create a policy.yaml template')
2227
+ .option('-f, --file <path>', 'Policy file path', 'policy.yaml')
2228
+ .option('--force', 'Overwrite existing file if it already exists')
2229
+ .action((options) => {
2230
+ try {
2231
+ const result = policyManager.initPolicy(options.file, {
2232
+ force: Boolean(options.force)
2233
+ });
2234
+
2235
+ const status = result.overwritten ? 'overwritten' : 'created';
2236
+ console.log(chalk.green(`Policy file ${status}: ${result.path}`));
2237
+ } catch (error) {
2238
+ console.error(chalk.red(`Failed to initialize policy: ${error.message}`));
2239
+ process.exit(1);
2240
+ }
2241
+ });
2242
+
2243
+ policyCommand
2244
+ .command('validate')
2245
+ .description('Validate policy.yaml against the v1 schema')
2246
+ .option('-f, --file <path>', 'Policy file path', 'policy.yaml')
2247
+ .option('-j, --json', 'Output validation result as JSON')
2248
+ .action((options) => {
2249
+ try {
2250
+ const result = policyManager.validatePolicyFile(options.file);
2251
+
2252
+ if (options.json) {
2253
+ console.log(JSON.stringify({
2254
+ valid: result.valid,
2255
+ file: result.path,
2256
+ errorCount: result.errors.length,
2257
+ errors: result.errors
2258
+ }, null, 2));
2259
+ if (!result.valid) {
2260
+ process.exit(1);
2261
+ }
2262
+ } else if (result.valid) {
2263
+ const mode = result.policy?.mode || 'unknown';
2264
+ console.log(chalk.green(`Policy is valid (${mode} mode): ${result.path}`));
2265
+ } else {
2266
+ console.error(chalk.red(`Policy validation failed: ${result.path}`));
2267
+ result.errors.forEach((entry) => {
2268
+ console.error(chalk.red(` - ${entry.path}: ${entry.message}`));
2269
+ });
2270
+ process.exit(1);
2271
+ }
2272
+ } catch (error) {
2273
+ if (options.json) {
2274
+ console.log(JSON.stringify({
2275
+ valid: false,
2276
+ file: policyManager.resolvePolicyPath(options.file),
2277
+ errorCount: 1,
2278
+ errors: [{ path: 'file', message: error.message }]
2279
+ }, null, 2));
2280
+ } else {
2281
+ console.error(chalk.red(`Policy validation failed: ${error.message}`));
2282
+ }
2283
+ process.exit(1);
2284
+ }
2285
+ });
2286
+
2287
+ policyCommand.action(() => {
2288
+ policyCommand.outputHelp();
2289
+ });
2290
+
2291
+ const auditCommand = program
2292
+ .command('audit')
2293
+ .description('Run policy audits and export compliance reports')
2294
+ .showHelpAfterError();
2295
+
2296
+ auditCommand
2297
+ .command('export')
2298
+ .description('Evaluate policy compliance and export JSON/CSV/SARIF reports')
2299
+ .requiredOption('--policy <file>', 'Policy file path')
2300
+ .option('--command <name>', 'Evaluation source: check | recommend', 'check')
2301
+ .option('--format <format>', 'Report format: json | csv | sarif | all', 'json')
2302
+ .option('--out <path>', 'Output file path (single-format export only)')
2303
+ .option('--out-dir <path>', 'Output directory when --out is omitted', 'audit-reports')
2304
+ .option('-u, --use-case <case>', 'Use case when --command check is selected', 'general')
2305
+ .option('-c, --category <category>', 'Category hint when --command recommend is selected')
2306
+ .option('--runtime <runtime>', `Runtime for check mode (${SUPPORTED_RUNTIMES.join('|')})`, 'ollama')
2307
+ .option('--include-cloud', 'Include cloud models in check-mode analysis')
2308
+ .option('--max-size <size>', 'Maximum model size for check mode (e.g., "24B" or "12GB")')
2309
+ .option('--min-size <size>', 'Minimum model size for check mode (e.g., "3B" or "2GB")')
2310
+ .option('-l, --limit <number>', 'Model analysis limit for check mode', '25')
2311
+ .option('--no-verbose', 'Disable verbose progress while collecting audit inputs')
2312
+ .action(async (options) => {
2313
+ try {
2314
+ const policyConfig = loadPolicyConfiguration(options.policy);
2315
+ const selectedCommand = String(options.command || 'check')
2316
+ .toLowerCase()
2317
+ .trim();
2318
+
2319
+ if (!['check', 'recommend'].includes(selectedCommand)) {
2320
+ throw new Error('Invalid --command value. Use "check" or "recommend".');
2321
+ }
2322
+
2323
+ const exportFormats = resolveAuditFormats(options.format, policyConfig.policy);
2324
+ if (options.out && exportFormats.length > 1) {
2325
+ throw new Error('--out can only be used with a single export format.');
2326
+ }
2327
+
2328
+ const verboseEnabled = options.verbose !== false;
2329
+ const checker = new (getLLMChecker())({ verbose: verboseEnabled });
2330
+ const hardware = await checker.getSystemInfo();
2331
+
2332
+ let runtimeBackend = 'ollama';
2333
+ let policyCandidates = [];
2334
+ let analysisResult = null;
2335
+ let recommendationResult = null;
2336
+
2337
+ if (selectedCommand === 'check') {
2338
+ let selectedRuntime = normalizeRuntime(options.runtime);
2339
+ if (!runtimeSupportedOnHardware(selectedRuntime, hardware)) {
2340
+ selectedRuntime = 'ollama';
2341
+ }
2342
+
2343
+ const maxSize = parseSizeFilterInput(options.maxSize);
2344
+ const minSize = parseSizeFilterInput(options.minSize);
2345
+ const normalizedUseCase = normalizeUseCaseInput(options.useCase);
2346
+
2347
+ analysisResult = await checker.analyze({
2348
+ useCase: normalizedUseCase,
2349
+ includeCloud: Boolean(options.includeCloud),
2350
+ limit: Number.parseInt(options.limit, 10) || 25,
2351
+ maxSize,
2352
+ minSize,
2353
+ runtime: selectedRuntime
2354
+ });
2355
+
2356
+ runtimeBackend = selectedRuntime;
2357
+ policyCandidates = collectCandidatesFromAnalysis(analysisResult);
2358
+ } else {
2359
+ recommendationResult = await checker.generateIntelligentRecommendations(hardware);
2360
+ if (!recommendationResult) {
2361
+ throw new Error('Unable to generate recommendation data for policy audit export.');
2362
+ }
2363
+
2364
+ runtimeBackend = normalizeRuntime(options.runtime || 'ollama');
2365
+ policyCandidates = collectCandidatesFromRecommendationData(recommendationResult);
2366
+ }
2367
+
2368
+ const policyContext = buildPolicyRuntimeContext({
2369
+ hardware,
2370
+ runtimeBackend
2371
+ });
2372
+
2373
+ const policyEvaluation = evaluatePolicyCandidates(
2374
+ policyConfig.policyEngine,
2375
+ policyCandidates,
2376
+ policyContext,
2377
+ policyConfig.policy
2378
+ );
2379
+ const policyEnforcement = resolvePolicyEnforcement(policyConfig.policy, policyEvaluation);
2380
+
2381
+ const report = buildComplianceReport({
2382
+ commandName: selectedCommand,
2383
+ policyPath: policyConfig.policyPath,
2384
+ policy: policyConfig.policy,
2385
+ evaluation: policyEvaluation,
2386
+ enforcement: policyEnforcement,
2387
+ runtimeContext: policyContext,
2388
+ options: {
2389
+ format: exportFormats,
2390
+ runtime: runtimeBackend,
2391
+ use_case: selectedCommand === 'check' ? normalizeUseCaseInput(options.useCase) : null,
2392
+ category: selectedCommand === 'recommend' ? options.category || null : null,
2393
+ include_cloud: Boolean(options.includeCloud)
2394
+ },
2395
+ hardware
2396
+ });
2397
+
2398
+ const generatedAt = report.generated_at || new Date().toISOString();
2399
+ const writtenFiles = [];
2400
+ exportFormats.forEach((format) => {
2401
+ const filePath = toAuditOutputPath({
2402
+ outputPath: options.out,
2403
+ outputDir: options.outDir,
2404
+ commandName: selectedCommand,
2405
+ format,
2406
+ timestamp: generatedAt
2407
+ });
2408
+ const content = serializeComplianceReport(report, format);
2409
+ writeReportFile(filePath, content);
2410
+ writtenFiles.push({ format, filePath });
2411
+ });
2412
+
2413
+ displayPolicySummary(`audit ${selectedCommand}`, policyConfig, policyEvaluation, policyEnforcement);
2414
+
2415
+ console.log('\n' + chalk.bgBlue.white.bold(' AUDIT EXPORT '));
2416
+ writtenFiles.forEach((entry) => {
2417
+ console.log(`${chalk.cyan(entry.format.toUpperCase())}: ${chalk.white(entry.filePath)}`);
2418
+ });
2419
+
2420
+ if (policyEnforcement.shouldBlock) {
2421
+ process.exit(policyEnforcement.exitCode);
2422
+ }
2423
+ } catch (error) {
2424
+ console.error(chalk.red(`Audit export failed: ${error.message}`));
2425
+ if (process.env.DEBUG) {
2426
+ console.error(error.stack);
2427
+ }
2428
+ process.exit(1);
2429
+ }
2430
+ });
2431
+
2432
+ auditCommand.action(() => {
2433
+ auditCommand.outputHelp();
2434
+ });
2435
+
1984
2436
  program
1985
2437
  .command('check')
1986
2438
  .description('Analyze your system and show compatible LLM models')
@@ -1992,15 +2444,31 @@ program
1992
2444
  .option('--min-size <size>', 'Minimum model size to consider (e.g., "7B" or "7GB")')
1993
2445
  .option('--include-cloud', 'Include cloud models in analysis')
1994
2446
  .option('--ollama-only', 'Only show models available in Ollama')
2447
+ .option('--runtime <runtime>', `Inference runtime (${SUPPORTED_RUNTIMES.join('|')})`, 'ollama')
2448
+ .option('--policy <file>', 'Evaluate candidate models against a policy file')
1995
2449
  .option('--performance-test', 'Run performance benchmarks')
1996
2450
  .option('--show-ollama-analysis', 'Show detailed Ollama model analysis')
1997
2451
  .option('--no-verbose', 'Disable step-by-step progress display')
2452
+ .addHelpText(
2453
+ 'after',
2454
+ `
2455
+ Enterprise policy examples:
2456
+ $ llm-checker check --policy ./policy.yaml
2457
+ $ llm-checker check --policy ./policy.yaml --use-case coding --runtime vllm
2458
+ $ llm-checker check --policy ./policy.yaml --include-cloud --max-size 24B
2459
+
2460
+ Policy scope:
2461
+ - Evaluates all compatible and marginal candidates discovered during analysis
2462
+ - Not limited to the top --limit results shown in output
2463
+ `
2464
+ )
1998
2465
  .action(async (options) => {
1999
2466
  showAsciiArt('check');
2000
2467
  try {
2001
2468
  // Use verbose progress unless explicitly disabled
2002
2469
  const verboseEnabled = options.verbose !== false;
2003
2470
  const checker = new (getLLMChecker())({ verbose: verboseEnabled });
2471
+ const policyConfig = options.policy ? loadPolicyConfiguration(options.policy) : null;
2004
2472
 
2005
2473
  // If verbose is disabled, show simple loading message
2006
2474
  if (!verboseEnabled) {
@@ -2008,6 +2476,16 @@ program
2008
2476
  }
2009
2477
 
2010
2478
  const hardware = await checker.getSystemInfo();
2479
+ let selectedRuntime = normalizeRuntime(options.runtime);
2480
+ if (!runtimeSupportedOnHardware(selectedRuntime, hardware)) {
2481
+ const runtimeLabel = getRuntimeDisplayName(selectedRuntime);
2482
+ console.log(
2483
+ chalk.yellow(
2484
+ `\nWarning: ${runtimeLabel} is not supported on this hardware. Falling back to Ollama.`
2485
+ )
2486
+ );
2487
+ selectedRuntime = 'ollama';
2488
+ }
2011
2489
 
2012
2490
  // Normalize and fix use-case typos
2013
2491
  const normalizeUseCase = (useCase = '') => {
@@ -2049,17 +2527,48 @@ program
2049
2527
  performanceTest: options.performanceTest,
2050
2528
  limit: parseInt(options.limit) || 10,
2051
2529
  maxSize: maxSize,
2052
- minSize: minSize
2530
+ minSize: minSize,
2531
+ runtime: selectedRuntime
2053
2532
  });
2054
2533
 
2055
2534
  if (!verboseEnabled) {
2056
2535
  console.log(chalk.green(' done'));
2057
2536
  }
2058
2537
 
2538
+ let policyEvaluation = null;
2539
+ let policyEnforcement = null;
2540
+ if (policyConfig) {
2541
+ const policyCandidates = collectCandidatesFromAnalysis(analysis);
2542
+ const policyContext = buildPolicyRuntimeContext({
2543
+ hardware,
2544
+ runtimeBackend: selectedRuntime
2545
+ });
2546
+ policyEvaluation = evaluatePolicyCandidates(
2547
+ policyConfig.policyEngine,
2548
+ policyCandidates,
2549
+ policyContext,
2550
+ policyConfig.policy
2551
+ );
2552
+ policyEnforcement = resolvePolicyEnforcement(policyConfig.policy, policyEvaluation);
2553
+ }
2554
+
2059
2555
  // Simplified output - show only essential information
2060
2556
  displaySimplifiedSystemInfo(hardware);
2061
- const recommendedModels = await displayModelRecommendations(analysis, hardware, normalizedUseCase, parseInt(options.limit) || 1);
2062
- await displayQuickStartCommands(analysis, recommendedModels[0], recommendedModels);
2557
+ const recommendedModels = await displayModelRecommendations(
2558
+ analysis,
2559
+ hardware,
2560
+ normalizedUseCase,
2561
+ parseInt(options.limit) || 1,
2562
+ selectedRuntime
2563
+ );
2564
+ await displayQuickStartCommands(analysis, recommendedModels[0], recommendedModels, selectedRuntime);
2565
+
2566
+ if (policyConfig && policyEvaluation && policyEnforcement) {
2567
+ displayPolicySummary('check', policyConfig, policyEvaluation, policyEnforcement);
2568
+ if (policyEnforcement.shouldBlock) {
2569
+ process.exit(policyEnforcement.exitCode);
2570
+ }
2571
+ }
2063
2572
 
2064
2573
  } catch (error) {
2065
2574
  console.error(chalk.red('\nError:'), error.message);
@@ -2290,11 +2799,22 @@ program
2290
2799
  .description('Get intelligent model recommendations for your hardware')
2291
2800
  .option('-c, --category <category>', 'Get recommendations for specific category (coding, talking, reading, etc.)')
2292
2801
  .option('--no-verbose', 'Disable step-by-step progress display')
2802
+ .option('--policy <file>', 'Evaluate recommendations against a policy file')
2803
+ .addHelpText(
2804
+ 'after',
2805
+ `
2806
+ Enterprise policy examples:
2807
+ $ llm-checker recommend --policy ./policy.yaml
2808
+ $ llm-checker recommend --policy ./policy.yaml --category coding
2809
+ $ llm-checker recommend --policy ./policy.yaml --no-verbose
2810
+ `
2811
+ )
2293
2812
  .action(async (options) => {
2294
2813
  showAsciiArt('recommend');
2295
2814
  try {
2296
2815
  const verboseEnabled = options.verbose !== false;
2297
2816
  const checker = new (getLLMChecker())({ verbose: verboseEnabled });
2817
+ const policyConfig = options.policy ? loadPolicyConfiguration(options.policy) : null;
2298
2818
 
2299
2819
  if (!verboseEnabled) {
2300
2820
  process.stdout.write(chalk.gray('Generating recommendations...'));
@@ -2312,12 +2832,36 @@ program
2312
2832
  console.log(chalk.green(' done'));
2313
2833
  }
2314
2834
 
2835
+ let policyEvaluation = null;
2836
+ let policyEnforcement = null;
2837
+ if (policyConfig) {
2838
+ const policyCandidates = collectCandidatesFromRecommendationData(intelligentRecommendations);
2839
+ const policyContext = buildPolicyRuntimeContext({
2840
+ hardware,
2841
+ runtimeBackend: 'ollama'
2842
+ });
2843
+ policyEvaluation = evaluatePolicyCandidates(
2844
+ policyConfig.policyEngine,
2845
+ policyCandidates,
2846
+ policyContext,
2847
+ policyConfig.policy
2848
+ );
2849
+ policyEnforcement = resolvePolicyEnforcement(policyConfig.policy, policyEvaluation);
2850
+ }
2851
+
2315
2852
  // Mostrar información del sistema
2316
2853
  displaySystemInfo(hardware, { summary: { hardwareTier: intelligentRecommendations.summary.hardware_tier } });
2317
2854
 
2318
2855
  // Mostrar recomendaciones
2319
2856
  displayIntelligentRecommendations(intelligentRecommendations);
2320
2857
 
2858
+ if (policyConfig && policyEvaluation && policyEnforcement) {
2859
+ displayPolicySummary('recommend', policyConfig, policyEvaluation, policyEnforcement);
2860
+ if (policyEnforcement.shouldBlock) {
2861
+ process.exit(policyEnforcement.exitCode);
2862
+ }
2863
+ }
2864
+
2321
2865
  } catch (error) {
2322
2866
  console.error(chalk.red('\nError:'), error.message);
2323
2867
  if (process.env.DEBUG) {