llm-checker 3.2.0 → 3.2.2

package/src/policy/policy-manager.js

@@ -0,0 +1,324 @@
+const fs = require('fs');
+const path = require('path');
+const YAML = require('yaml');
+
+const ALLOWED_POLICY_MODES = ['audit', 'enforce'];
+const ALLOWED_ENFORCEMENT_BEHAVIOR = ['warn', 'error'];
+const ALLOWED_REPORT_FORMATS = ['json', 'csv', 'sarif'];
+
+function isPlainObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isPositiveNumber(value) {
+    return typeof value === 'number' && Number.isFinite(value) && value > 0;
+}
+
+function isNonEmptyString(value) {
+    return typeof value === 'string' && value.trim().length > 0;
+}
+
+class PolicyManager {
+    getTemplate() {
+        return `version: 1
+org: your-org
+mode: enforce # audit | enforce
+
+rules:
+  models:
+    allow:
+      - "qwen2.5-coder:*"
+      - "llama3.1:*"
+    deny:
+      - "*uncensored*"
+    max_size_gb: 24
+    max_params_b: 32
+    allowed_quantizations: ["Q4_K_M", "Q5_K_M", "Q8_0"]
+
+  runtime:
+    required_backends: ["metal", "cuda"]
+    min_ram_gb: 32
+    local_only: true
+
+  compliance:
+    approved_licenses: ["mit", "apache-2.0", "llama"]
+
+enforcement:
+  on_violation: error # warn | error
+  exit_code: 3
+  allow_exceptions: true
+
+exceptions:
+  - model: "deepseek-r1:32b"
+    reason: "Approved PoC"
+    approver: "security@example.com"
+    expires_at: "2026-06-30"
+
+reporting:
+  formats: ["json", "csv", "sarif"]
+`;
+    }
+
+    resolvePolicyPath(policyFile = 'policy.yaml', cwd = process.cwd()) {
+        return path.isAbsolute(policyFile) ? policyFile : path.resolve(cwd, policyFile);
+    }
+
+    initPolicy(policyFile = 'policy.yaml', options = {}) {
+        const { force = false, cwd = process.cwd() } = options;
+        const targetPath = this.resolvePolicyPath(policyFile, cwd);
+
+        const exists = fs.existsSync(targetPath);
+        if (exists && !force) {
+            throw new Error(`Policy file already exists at ${targetPath}. Use --force to overwrite.`);
+        }
+
+        fs.mkdirSync(path.dirname(targetPath), { recursive: true });
+        fs.writeFileSync(targetPath, this.getTemplate(), 'utf8');
+
+        return {
+            path: targetPath,
+            overwritten: exists && force
+        };
+    }
+
+    loadPolicy(policyFile = 'policy.yaml', options = {}) {
+        const { cwd = process.cwd() } = options;
+        const policyPath = this.resolvePolicyPath(policyFile, cwd);
+
+        if (!fs.existsSync(policyPath)) {
+            throw new Error(`Policy file not found: ${policyPath}`);
+        }
+
+        const source = fs.readFileSync(policyPath, 'utf8');
+        const doc = YAML.parseDocument(source, { prettyErrors: true });
+
+        if (doc.errors && doc.errors.length > 0) {
+            const first = doc.errors[0];
+            throw new Error(`Invalid YAML in ${policyPath}: ${String(first.message || first)}`);
+        }
+
+        const policy = doc.toJSON();
+        if (!isPlainObject(policy)) {
+            throw new Error(`Invalid YAML in ${policyPath}: root must be an object`);
+        }
+
+        return { path: policyPath, policy };
+    }
+
+    validatePolicyFile(policyFile = 'policy.yaml', options = {}) {
+        const loaded = this.loadPolicy(policyFile, options);
+        const validation = this.validatePolicyObject(loaded.policy);
+        return {
+            ...validation,
+            path: loaded.path,
+            policy: loaded.policy
+        };
+    }
+
+    validatePolicyObject(policy) {
+        const errors = [];
+        const addError = (fieldPath, message) => {
+            errors.push({ path: fieldPath, message });
+        };
+
+        if (!isPlainObject(policy)) {
+            addError('root', 'Policy must be an object.');
+            return { valid: false, errors };
+        }
+
+        if (!Number.isInteger(policy.version) || policy.version < 1) {
+            addError('version', 'Version must be an integer >= 1.');
+        }
+
+        if (!isNonEmptyString(policy.org)) {
+            addError('org', 'Organization must be a non-empty string.');
+        }
+
+        if (!isNonEmptyString(policy.mode) || !ALLOWED_POLICY_MODES.includes(policy.mode)) {
+            addError('mode', `Mode must be one of: ${ALLOWED_POLICY_MODES.join(', ')}.`);
+        }
+
+        if (!isPlainObject(policy.rules)) {
+            addError('rules', 'Rules section is required and must be an object.');
+        } else {
+            this.validateModelsRules(policy.rules.models, addError);
+            this.validateRuntimeRules(policy.rules.runtime, addError);
+            this.validateComplianceRules(policy.rules.compliance, addError);
+        }
+
+        this.validateEnforcement(policy.enforcement, addError);
+        this.validateExceptions(policy.exceptions, addError);
+        this.validateReporting(policy.reporting, addError);
+
+        return {
+            valid: errors.length === 0,
+            errors
+        };
+    }
+
+    validateModelsRules(models, addError) {
+        if (models === undefined) return;
+
+        if (!isPlainObject(models)) {
+            addError('rules.models', 'Must be an object.');
+            return;
+        }
+
+        this.validateStringArray(models.allow, 'rules.models.allow', addError);
+        this.validateStringArray(models.deny, 'rules.models.deny', addError);
+        this.validateStringArray(
+            models.allowed_quantizations,
+            'rules.models.allowed_quantizations',
+            addError
+        );
+
+        if (models.max_size_gb !== undefined && !isPositiveNumber(models.max_size_gb)) {
+            addError('rules.models.max_size_gb', 'Must be a positive number.');
+        }
+
+        if (models.max_params_b !== undefined && !isPositiveNumber(models.max_params_b)) {
+            addError('rules.models.max_params_b', 'Must be a positive number.');
+        }
+    }
+
+    validateRuntimeRules(runtime, addError) {
+        if (runtime === undefined) return;
+
+        if (!isPlainObject(runtime)) {
+            addError('rules.runtime', 'Must be an object.');
+            return;
+        }
+
+        this.validateStringArray(runtime.required_backends, 'rules.runtime.required_backends', addError);
+
+        if (runtime.min_ram_gb !== undefined && !isPositiveNumber(runtime.min_ram_gb)) {
+            addError('rules.runtime.min_ram_gb', 'Must be a positive number.');
+        }
+
+        if (runtime.local_only !== undefined && typeof runtime.local_only !== 'boolean') {
+            addError('rules.runtime.local_only', 'Must be a boolean.');
+        }
+    }
+
+    validateComplianceRules(compliance, addError) {
+        if (compliance === undefined) return;
+
+        if (!isPlainObject(compliance)) {
+            addError('rules.compliance', 'Must be an object.');
+            return;
+        }
+
+        this.validateStringArray(
+            compliance.approved_licenses,
+            'rules.compliance.approved_licenses',
+            addError
+        );
+    }
+
+    validateEnforcement(enforcement, addError) {
+        if (enforcement === undefined) return;
+
+        if (!isPlainObject(enforcement)) {
+            addError('enforcement', 'Must be an object.');
+            return;
+        }
+
+        if (
+            enforcement.on_violation !== undefined &&
+            !ALLOWED_ENFORCEMENT_BEHAVIOR.includes(enforcement.on_violation)
+        ) {
+            addError(
+                'enforcement.on_violation',
+                `Must be one of: ${ALLOWED_ENFORCEMENT_BEHAVIOR.join(', ')}.`
+            );
+        }
+
+        if (enforcement.exit_code !== undefined) {
+            if (!Number.isInteger(enforcement.exit_code) || enforcement.exit_code < 1 || enforcement.exit_code > 255) {
+                addError('enforcement.exit_code', 'Must be an integer between 1 and 255.');
+            }
+        }
+
+        if (enforcement.allow_exceptions !== undefined && typeof enforcement.allow_exceptions !== 'boolean') {
+            addError('enforcement.allow_exceptions', 'Must be a boolean.');
+        }
+    }
+
+    validateExceptions(exceptions, addError) {
+        if (exceptions === undefined) return;
+
+        if (!Array.isArray(exceptions)) {
+            addError('exceptions', 'Must be an array.');
+            return;
+        }
+
+        exceptions.forEach((entry, index) => {
+            const basePath = `exceptions[${index}]`;
+            if (!isPlainObject(entry)) {
+                addError(basePath, 'Each exception must be an object.');
+                return;
+            }
+
+            if (!isNonEmptyString(entry.model)) {
+                addError(`${basePath}.model`, 'Model must be a non-empty string.');
+            }
+
+            if (entry.reason !== undefined && !isNonEmptyString(entry.reason)) {
+                addError(`${basePath}.reason`, 'Reason must be a non-empty string.');
+            }
+
+            if (entry.approver !== undefined && !isNonEmptyString(entry.approver)) {
+                addError(`${basePath}.approver`, 'Approver must be a non-empty string.');
+            }
+
+            if (entry.expires_at !== undefined && !isNonEmptyString(entry.expires_at)) {
+                addError(`${basePath}.expires_at`, 'expires_at must be a non-empty string.');
+            }
+        });
+    }
+
+    validateReporting(reporting, addError) {
+        if (reporting === undefined) return;
+
+        if (!isPlainObject(reporting)) {
+            addError('reporting', 'Must be an object.');
+            return;
+        }
+
+        if (reporting.formats !== undefined) {
+            if (!Array.isArray(reporting.formats)) {
+                addError('reporting.formats', 'Must be an array.');
+            } else {
+                reporting.formats.forEach((format, index) => {
+                    if (!isNonEmptyString(format)) {
+                        addError(`reporting.formats[${index}]`, 'Format must be a non-empty string.');
+                        return;
+                    }
+
+                    if (!ALLOWED_REPORT_FORMATS.includes(format)) {
+                        addError(
+                            `reporting.formats[${index}]`,
+                            `Unsupported format "${format}". Allowed: ${ALLOWED_REPORT_FORMATS.join(', ')}.`
+                        );
+                    }
+                });
+            }
+        }
+    }
+
+    validateStringArray(value, fieldPath, addError) {
+        if (value === undefined) return;
+        if (!Array.isArray(value)) {
+            addError(fieldPath, 'Must be an array of strings.');
+            return;
+        }
+
+        value.forEach((entry, index) => {
+            if (!isNonEmptyString(entry)) {
+                addError(`${fieldPath}[${index}]`, 'Must be a non-empty string.');
+            }
+        });
+    }
+}
+
+module.exports = PolicyManager;

package/src/provenance/model-provenance.js

@@ -0,0 +1,176 @@
+const UNKNOWN_VALUE = 'unknown';
+
+const UNKNOWN_MARKERS = new Set(['', 'unknown', 'n/a', 'na', 'none', 'unspecified', 'not-provided']);
+
+const LICENSE_ALIASES = {
+    mitlicense: 'mit',
+    mit: 'mit',
+    apache2: 'apache-2.0',
+    'apache2.0': 'apache-2.0',
+    'apache-2': 'apache-2.0',
+    'apache-2.0': 'apache-2.0',
+    'apache-2.0-license': 'apache-2.0',
+    'llama2': 'llama',
+    'llama3': 'llama',
+    'llama3.1': 'llama',
+    'llama3.2': 'llama',
+    'meta-llama': 'llama'
+};
+
+function asString(value) {
+    if (value === undefined || value === null) return '';
+    return String(value).trim();
+}
+
+function sanitizeValue(value) {
+    const text = asString(value);
+    if (!text) return UNKNOWN_VALUE;
+
+    const normalized = text.toLowerCase();
+    return UNKNOWN_MARKERS.has(normalized) ? UNKNOWN_VALUE : text;
+}
+
+function normalizeSource(value) {
+    const normalized = sanitizeValue(value).toLowerCase();
+    if (normalized === UNKNOWN_VALUE) return UNKNOWN_VALUE;
+
+    const source = normalized.replace(/\s+/g, '_').replace(/-/g, '_');
+
+    if (source === 'ollama_local') return 'ollama_local';
+    if (source === 'ollama_database') return 'ollama_database';
+    if (source === 'enhanced_with_ollama') return 'enhanced_with_ollama';
+    if (source === 'static_database') return 'static_database';
+
+    return source;
+}
+
+function normalizeRegistry(value, source = UNKNOWN_VALUE) {
+    const registry = sanitizeValue(value);
+    if (registry !== UNKNOWN_VALUE) return registry;
+
+    if (source.includes('ollama')) {
+        return 'ollama.com';
+    }
+
+    return UNKNOWN_VALUE;
+}
+
+function normalizeVersion(value) {
+    return sanitizeValue(value);
+}
+
+function normalizeDigest(value) {
+    return sanitizeValue(value);
+}
+
+function normalizeLicense(value) {
+    const raw = sanitizeValue(value);
+    if (raw === UNKNOWN_VALUE) return UNKNOWN_VALUE;
+
+    const lowered = raw.toLowerCase();
+    const canonicalKey = lowered.replace(/\s+/g, '').replace(/_/g, '-');
+    if (LICENSE_ALIASES[canonicalKey]) return LICENSE_ALIASES[canonicalKey];
+
+    const hyphenated = lowered.replace(/\s+/g, '-').replace(/_/g, '-');
+    const aliasByHyphen = LICENSE_ALIASES[hyphenated];
+    if (aliasByHyphen) return aliasByHyphen;
+
+    return hyphenated;
+}
+
+function extractVersionFromIdentifier(identifier) {
+    const text = asString(identifier);
+    if (!text) return UNKNOWN_VALUE;
+
+    if (text.includes(':')) {
+        const [, tag] = text.split(':');
+        return sanitizeValue(tag);
+    }
+
+    return UNKNOWN_VALUE;
+}
+
+function extractProvenance(model = {}, defaults = {}) {
+    const source = normalizeSource(
+        model?.provenance?.source ||
+            model?.source ||
+            defaults?.source
+    );
+
+    const registry = normalizeRegistry(
+        model?.provenance?.registry ||
+            model?.registry ||
+            model?.source_registry ||
+            defaults?.registry,
+        source
+    );
+
+    const version = normalizeVersion(
+        model?.provenance?.version ||
+            model?.version ||
+            model?.tag ||
+            model?.model_tag ||
+            defaults?.version ||
+            extractVersionFromIdentifier(
+                model?.model_identifier ||
+                    model?.modelIdentifier ||
+                    model?.identifier ||
+                    model?.model_id ||
+                    model?.modelId
+            )
+    );
+
+    const license = normalizeLicense(
+        model?.provenance?.license ||
+            model?.license ||
+            model?.license_id ||
+            model?.licenseId ||
+            defaults?.license
+    );
+
+    const digest = normalizeDigest(
+        model?.provenance?.digest ||
+            model?.digest ||
+            model?.hash ||
+            model?.sha256 ||
+            defaults?.digest
+    );
+
+    return {
+        source,
+        registry,
+        version,
+        license,
+        digest
+    };
+}
+
+function attachModelProvenance(model, defaults = {}) {
+    if (!model || typeof model !== 'object' || Array.isArray(model)) {
+        return model;
+    }
+
+    const provenance = extractProvenance(model, defaults);
+
+    return {
+        ...model,
+        source: provenance.source,
+        license: provenance.license,
+        version: provenance.version,
+        digest: provenance.digest,
+        provenance
+    };
+}
+
+function attachProvenanceToCollection(models, defaults = {}) {
+    if (!Array.isArray(models)) return [];
+    return models.map((model) => attachModelProvenance(model, defaults));
+}
+
+module.exports = {
+    UNKNOWN_VALUE,
+    normalizeLicense,
+    extractProvenance,
+    attachModelProvenance,
+    attachProvenanceToCollection
+};

package/src/runtime/runtime-support.js

@@ -0,0 +1,174 @@
+const SUPPORTED_RUNTIMES = ['ollama', 'vllm', 'mlx'];
+
+function normalizeRuntime(runtime = 'ollama') {
+    const normalized = String(runtime || 'ollama').trim().toLowerCase();
+    return SUPPORTED_RUNTIMES.includes(normalized) ? normalized : 'ollama';
+}
+
+function getRuntimeDisplayName(runtime = 'ollama') {
+    const normalized = normalizeRuntime(runtime);
+    if (normalized === 'vllm') return 'vLLM';
+    if (normalized === 'mlx') return 'MLX-LM';
+    return 'Ollama';
+}
+
+function isAppleSiliconHardware(hardware = {}) {
+    const osPlatform = String(hardware?.os?.platform || '').toLowerCase();
+    const arch = String(
+        hardware?.cpu?.architecture ||
+        hardware?.summary?.architecture ||
+        ''
+    ).toLowerCase();
+    const cpuBrand = String(hardware?.cpu?.brand || '').toLowerCase();
+    const gpuModel = String(hardware?.gpu?.model || '').toLowerCase();
+    const isDarwin = osPlatform === 'darwin' || osPlatform === 'macos';
+    const hasAppleChipSignal =
+        arch.includes('apple silicon') ||
+        cpuBrand.includes('apple') ||
+        gpuModel.includes('apple');
+
+    // Prefer explicit Apple signals and avoid treating generic Linux ARM64 as Apple Silicon.
+    if (isDarwin) {
+        return arch === 'arm64' || hasAppleChipSignal;
+    }
+
+    // Fallback for partial hardware payloads that still expose Apple-specific identifiers.
+    return hasAppleChipSignal;
+}
+
+function runtimeSupportedOnHardware(runtime = 'ollama', hardware = {}) {
+    const normalized = normalizeRuntime(runtime);
+    if (normalized === 'mlx') {
+        return isAppleSiliconHardware(hardware);
+    }
+    return true;
+}
+
+function runtimeSupportsSpeculativeDecoding(runtime = 'ollama') {
+    const normalized = normalizeRuntime(runtime);
+    return normalized === 'vllm' || normalized === 'mlx';
+}
+
+function shellEscapeArg(value = '') {
+    const text = String(value || '');
+    if (!text) return "''";
+    return `'${text.replace(/'/g, `'\\''`)}'`;
+}
+
+function extractFromInstallCommand(command = '') {
+    const match = String(command).match(/ollama\s+pull\s+(.+)$/i);
+    return match ? match[1].trim() : '';
+}
+
+function slugifyModelName(text = '') {
+    return String(text)
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-+|-+$/g, '') || 'model';
+}
+
+function getRuntimeModelRef(model = {}, runtime = 'ollama') {
+    const normalized = normalizeRuntime(runtime);
+
+    const candidates = [
+        model.hfModel,
+        model.hfId,
+        model.huggingfaceId,
+        model.model_identifier,
+        model.identifier,
+        model.cloudData?.identifier,
+        model.ollamaTag,
+        extractFromInstallCommand(model.installation?.ollama),
+        model.ollamaId,
+        model.name
+    ].filter(Boolean);
+
+    const raw = String(candidates[0] || '').trim();
+    if (!raw) return 'model';
+
+    if (normalized === 'ollama') {
+        return raw;
+    }
+
+    if (raw.includes('/')) {
+        return raw;
+    }
+
+    // Remove Ollama-style tag suffix for non-Ollama runtimes.
+    const base = raw.split(':')[0].trim();
+    if (base) {
+        return /\s/.test(base) ? slugifyModelName(base) : base;
+    }
+
+    return slugifyModelName(raw);
+}
+
+function getRuntimeInstallCommand(runtime = 'ollama') {
+    const normalized = normalizeRuntime(runtime);
+
+    if (normalized === 'vllm') {
+        return 'pip install -U "vllm>=0.6.0"';
+    }
+
+    if (normalized === 'mlx') {
+        return 'pip install -U mlx-lm';
+    }
+
+    return 'ollama --version || (brew install ollama)';
+}
+
+function getRuntimePullCommand(model = {}, runtime = 'ollama') {
+    const normalized = normalizeRuntime(runtime);
+    const modelRef = getRuntimeModelRef(model, runtime);
+
+    if (normalized === 'vllm') {
+        return `huggingface-cli download ${shellEscapeArg(modelRef)}`;
+    }
+
+    if (normalized === 'mlx') {
+        const localName = slugifyModelName(modelRef);
+        return `python -m mlx_lm.convert --hf-path ${shellEscapeArg(modelRef)} --mlx-path ./models/${localName}`;
+    }
+
+    return `ollama pull ${modelRef}`;
+}
+
+function getRuntimeRunCommand(model = {}, runtime = 'ollama') {
+    const normalized = normalizeRuntime(runtime);
+    const modelRef = getRuntimeModelRef(model, runtime);
+
+    if (normalized === 'vllm') {
+        return `python -m vllm.entrypoints.openai.api_server --model ${shellEscapeArg(modelRef)} --host 0.0.0.0 --port 8000`;
+    }
+
+    if (normalized === 'mlx') {
+        return `python -m mlx_lm.generate --model ${shellEscapeArg(modelRef)} --prompt "Hello"`;
+    }
+
+    return `ollama run ${modelRef}`;
+}
+
+function getRuntimeCommandSet(model = {}, runtime = 'ollama') {
+    const normalized = normalizeRuntime(runtime);
+    return {
+        runtime: normalized,
+        displayName: getRuntimeDisplayName(normalized),
+        modelRef: getRuntimeModelRef(model, normalized),
+        install: getRuntimeInstallCommand(normalized),
+        pull: getRuntimePullCommand(model, normalized),
+        run: getRuntimeRunCommand(model, normalized)
+    };
+}
+
+module.exports = {
+    SUPPORTED_RUNTIMES,
+    normalizeRuntime,
+    getRuntimeDisplayName,
+    runtimeSupportedOnHardware,
+    runtimeSupportsSpeculativeDecoding,
+    getRuntimeModelRef,
+    getRuntimeInstallCommand,
+    getRuntimePullCommand,
+    getRuntimeRunCommand,
+    getRuntimeCommandSet
+};

package/bin/CLAUDE.md

@@ -1,27 +0,0 @@
-<claude-mem-context>
-# Recent Activity
-
-<!-- This section is auto-generated by claude-mem. Edit content outside the tags. -->
-
-### Feb 12, 2026
-
-| ID | Time | T | Title | Read |
-|----|------|---|-------|------|
-| #3492 | 10:24 PM | 🔵 | Enhanced CLI Structure - Lazy Loading with ASCII Art Branding | ~456 |
-| #3436 | 9:57 PM | 🔵 | Enhanced CLI Implementation - Command-Line Interface with ASCII Art and Ollama Integration | ~575 |
-
-### Feb 14, 2026
-
-| ID | Time | T | Title | Read |
-|----|------|---|-------|------|
-| #4369 | 7:19 PM | 🟣 | Added 6 advanced MCP-exclusive tools for benchmarking, optimization, and intelligent model management | ~703 |
-| #4368 | " | 🟣 | MCP server implementation added to llm-checker with benchmark tool | ~387 |
-| #4363 | 7:18 PM | 🟣 | MCP tools test suite successfully validates all 7 llm-checker tools | ~551 |
-| #4361 | 7:17 PM | 🟣 | MCP server test suite created for validating Ollama integration tools | ~524 |
-| #4356 | " | 🟣 | Six advanced MCP tools added to llm-checker server beyond CLI wrapping | ~750 |
-| #4349 | 7:15 PM | 🟣 | MCP server implementation exposes 9 tools for LLM hardware analysis and Ollama model management | ~539 |
-| #4341 | 6:49 PM | 🟣 | MCP server implementation committed to llm-checker repository | ~431 |
-| #4339 | " | 🟣 | MCP server implementation and documentation added to llm-checker repository | ~457 |
-| #4338 | " | ✅ | MCP server validated in llm-checker repository structure | ~245 |
-| #4333 | 6:48 PM | 🟣 | MCP server integrated into llm-checker package distribution | ~409 |
-</claude-mem-context>