llm-checker 3.5.14 → 3.6.1

package/src/models/intelligent-selector.js

@@ -10,6 +10,7 @@ const ScoringEngine = require('./scoring-engine');
 const UnifiedDetector = require('../hardware/unified-detector');
 const PolicyManager = require('../policy/policy-manager');
 const PolicyEngine = require('../policy/policy-engine');
+const { rankModels } = require('./scoring-core');
 
 function isPlainObject(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value);
@@ -66,7 +67,9 @@ class IntelligentSelector {
         // Apply filters
         const filtered = this.applyFilters(variants, opts, hardware);
 
-        // Score all filtered variants
+        // Score all filtered variants. ScoringEngine still produces the
+        // per-variant `score` objects (final/components/meta) consumed by the
+        // smart-recommend display, but the RANKING is unified below.
         const scored = this.scoring.filterAndScore(filtered, hardware, {
             useCase: opts.useCase,
             targetContext: opts.targetContext,
@@ -75,6 +78,12 @@ class IntelligentSelector {
             headroom: opts.headroom || 2
         });
 
+        // Unify ranking with the canonical scoring core (issue #88): re-order
+        // the scored list and rewrite each item's final score using the shared
+        // DeterministicModelSelector so smart-recommend agrees with
+        // `check`/`recommend` and inherits the PR #89 high-capacity floor.
+        await this.applyUnifiedRanking(scored, hardware, opts);
+
         const policyEngine = this.resolvePolicyEngine(opts);
         const scoredWithPolicy = policyEngine.evaluateScoredVariants(
             scored,
@@ -114,6 +123,83 @@ class IntelligentSelector {
         };
     }
 
+    /**
+     * Re-rank the ScoringEngine-scored variants using the canonical scoring
+     * core so smart-recommend's ordering and headline scores match
+     * `check`/`recommend` and inherit the high-capacity right-sizing floor.
+     *
+     * Mutates `scored` in place: it is sorted by the unified score and each
+     * item's `score.final` is overwritten with the canonical 0-100 score.
+     * Component/meta sub-scores are left intact so the existing display (which
+     * shows Q/S/F and estimated TPS) keeps working. If the core cannot rank a
+     * variant (or throws), that item keeps its original ScoringEngine score and
+     * sorts after the unified ones, preserving a sensible fallback ordering.
+     */
+    async applyUnifiedRanking(scored, hardware, opts = {}) {
+        if (!Array.isArray(scored) || scored.length === 0) return scored;
+
+        let ranking;
+        try {
+            ranking = await rankModels(
+                scored.map((item) => item.variant),
+                hardware,
+                {
+                    category: opts.useCase || 'general',
+                    optimizeFor: opts.optimizeFor || opts.optimize || 'balanced',
+                    runtime: opts.runtime || 'ollama',
+                    topN: scored.length
+                }
+            );
+        } catch (error) {
+            return scored; // Defensive: keep original ScoringEngine ordering.
+        }
+
+        if (!ranking || !Array.isArray(ranking.candidates)) return scored;
+
+        // Map each source variant -> its unified score + ordering index.
+        const unifiedByVariant = new Map();
+        ranking.candidates.forEach((candidate, index) => {
+            const source = candidate?.meta?.__source;
+            if (!source) return;
+            unifiedByVariant.set(source, {
+                unifiedScore: Math.round(candidate.score * 10) / 10,
+                rank: index,
+                quant: candidate.quant,
+                estimatedTPS: candidate.estTPS
+            });
+        });
+
+        for (const item of scored) {
+            const unified = unifiedByVariant.get(item.variant);
+            if (!unified) {
+                // Not ranked by the core (e.g. filtered out): sort last and tag
+                // so any downstream tie-breaks are deterministic.
+                item.__unifiedRank = Number.MAX_SAFE_INTEGER;
+                continue;
+            }
+            item.__unifiedRank = unified.rank;
+            if (item.score) {
+                item.score.final = Math.min(100, Math.max(0, Math.round(unified.unifiedScore)));
+                if (item.score.meta) {
+                    item.score.meta.unifiedScore = unified.unifiedScore;
+                }
+            }
+        }
+
+        scored.sort((a, b) => {
+            const ra = Number.isFinite(a.__unifiedRank) ? a.__unifiedRank : Number.MAX_SAFE_INTEGER;
+            const rb = Number.isFinite(b.__unifiedRank) ? b.__unifiedRank : Number.MAX_SAFE_INTEGER;
+            if (ra !== rb) return ra - rb;
+            return (b.score?.final || 0) - (a.score?.final || 0);
+        });
+
+        for (const item of scored) {
+            delete item.__unifiedRank;
+        }
+
+        return scored;
+    }
+
     /**
      * Resolve policy engine from explicit options, in-memory policy, or policy file.
      */

package/src/models/requirements.js

@@ -134,17 +134,22 @@ class RequirementsCalculator {
     }
 
     parseModelSize(sizeString) {
-        const normalized = sizeString.toLowerCase().replace(/[^0-9.kmb]/g, '');
-
-        if (normalized.includes('k')) {
-            return parseFloat(normalized.replace('k', '')) / 1000;
-        } else if (normalized.includes('m')) {
-            return parseFloat(normalized.replace('m', '')) / 1000;
-        } else if (normalized.includes('b')) {
-            return parseFloat(normalized.replace('b', ''));
-        } else {
-            return parseFloat(normalized);
-        }
+        // Anchor the number to its unit instead of globally stripping every char
+        // that isn't 0-9.kmb: the old approach kept stray k/m/b from model words, so
+        // "Llama 3.2 3B" normalized to "m3.23b" and parsed as 0.003B, and unit-only
+        // inputs produced NaN. Prefer a number that carries a B/M/K unit (the real
+        // size token, "3B") over a bare number (a version like "3.2").
+        const text = String(sizeString || '');
+        const match = text.match(/(\d+(?:\.\d+)?)\s*([kmb])\b/i) || text.match(/(\d+(?:\.\d+)?)/);
+        if (!match) return 1;
+
+        const value = parseFloat(match[1]);
+        if (!Number.isFinite(value)) return 1;
+
+        const unit = (match[2] || 'b').toLowerCase();
+        if (unit === 'k') return value / 1_000_000; // thousands of params -> billions
+        if (unit === 'm') return value / 1000;       // millions of params -> billions
+        return value;                                // billions
     }
 
     getContextMultiplier(contextLength) {

package/src/models/scoring-core.js

@@ -0,0 +1,341 @@
+/**
+ * Unified Scoring Core (GitHub issue #88)
+ * =======================================
+ *
+ * Historically the three user-facing recommendation surfaces each ran a
+ * DIFFERENT scoring engine, so they could disagree about the best model and
+ * only one of them ("recommend") received the PR #89 high-capacity
+ * right-sizing fix:
+ *
+ *   - `check`           -> MultiObjectiveSelector  (src/ai/multi-objective-selector.js)
+ *   - `recommend`       -> DeterministicModelSelector (src/models/deterministic-selector.js)
+ *   - `smart-recommend` -> ScoringEngine            (src/models/scoring-engine.js)
+ *
+ * This module establishes the DeterministicModelSelector as the SINGLE
+ * canonical ranking core (it already carries PR #89's
+ * `calculateHighCapacitySizeAdjustment` / `getHighCapacitySizeTarget`). The
+ * other commands keep their own model SOURCE and DISPLAY shape, but route the
+ * actual ranking/selection through `rankModels()` below so that, for identical
+ * (model, hardware) inputs, every command produces identical scores and the
+ * high-capacity floor applies everywhere.
+ *
+ * The hard part is that each command feeds the engine a different model shape:
+ *   - `check` uses ExpandedModelsDatabase rows ({ name: "Llama 3.1 8B", size: "8B", ... })
+ *   - `smart-recommend` uses sql.js variant rows ({ model_id, params_b, size_gb, quant, ... })
+ *   - `recommend` uses the Ollama scrape/catalog (already deterministic-friendly)
+ *
+ * `normalizeToDeterministic()` converts any of those into the
+ * already-normalized deterministic model shape and keeps a back-reference
+ * (`__source`) so callers can recover their original object after ranking.
+ */
+
+const DeterministicModelSelector = require('./deterministic-selector');
+
+// One shared selector instance: it is stateless w.r.t. the model pool (the
+// pool is always passed in via options), so sharing it is safe and avoids
+// re-allocating the large prior tables on every call.
+const canonicalSelector = new DeterministicModelSelector();
+
+const PARAM_HINT_REGEX = /(\d+(?:\.\d+)?)\s*([bm])\b/i;
+
+function toFiniteNumber(value) {
+    if (typeof value === 'number' && Number.isFinite(value)) return value;
+    if (typeof value === 'string' && value.trim() !== '' && Number.isFinite(Number(value))) {
+        return Number(value);
+    }
+    return null;
+}
+
+/**
+ * Best-effort parameter-count extraction (in billions) from a model's various
+ * shapes. Mirrors the heuristics the individual engines used so the unified
+ * ranking sees the same parameter counts they would have.
+ */
+function deriveParamsB(model = {}) {
+    const direct =
+        toFiniteNumber(model.paramsB) ??
+        toFiniteNumber(model.params_b) ??
+        toFiniteNumber(model.parameter_count_b);
+    if (Number.isFinite(direct) && direct > 0) return direct;
+
+    // Parse from explicit "size" / name strings (e.g. "8B", "Qwen 0.5B", "405b").
+    const textCandidates = [model.size, model.parameter_size, model.name, model.model_name, model.model_id, model.model_identifier, model.tag];
+    for (const candidate of textCandidates) {
+        if (typeof candidate !== 'string') continue;
+        const match = candidate.match(PARAM_HINT_REGEX);
+        if (match) {
+            const value = parseFloat(match[1]);
+            if (Number.isFinite(value) && value > 0) {
+                return match[2].toLowerCase() === 'm' ? value / 1000 : value;
+            }
+        }
+    }
+
+    // Fall back to inferring from an artifact size in GB (Q4 ~ 0.6GB/B).
+    const sizeGB = deriveSizeGB(model, null);
+    if (Number.isFinite(sizeGB) && sizeGB > 0) {
+        return Math.max(0.5, Math.round((sizeGB / 0.6) * 2) / 2);
+    }
+
+    return 7; // Same neutral fallback the engines used.
+}
+
+function deriveSizeGB(model = {}, paramsB = null) {
+    const direct =
+        toFiniteNumber(model.sizeGB) ??
+        toFiniteNumber(model.size_gb) ??
+        toFiniteNumber(model.real_size_gb) ??
+        toFiniteNumber(model.estimated_size_gb);
+    if (Number.isFinite(direct) && direct > 0) return direct;
+
+    // "size" may be a file size like "4.4GB" (vs a parameter count like "8B").
+    if (typeof model.size === 'string') {
+        const gbMatch = model.size.match(/(\d+(?:\.\d+)?)\s*g(?:i)?b\b/i);
+        if (gbMatch) return parseFloat(gbMatch[1]);
+    }
+    if (typeof model.installedSize === 'string') {
+        const gbMatch = model.installedSize.match(/(\d+(?:\.\d+)?)\s*g(?:i)?b\b/i);
+        if (gbMatch) return parseFloat(gbMatch[1]);
+    }
+
+    if (Number.isFinite(paramsB) && paramsB > 0) {
+        return Math.max(0.4, Math.round((paramsB * 0.6 + 0.4) * 10) / 10);
+    }
+    return null;
+}
+
+function deriveContext(model = {}) {
+    const candidates = [
+        model.ctxMax,
+        model.context_length,
+        model.contextLength,
+        model.context,
+        model.performance && model.performance.context_length
+    ];
+    for (const candidate of candidates) {
+        if (typeof candidate === 'number' && Number.isFinite(candidate) && candidate > 0) {
+            return Math.round(candidate);
+        }
+        if (typeof candidate === 'string') {
+            const parsed = canonicalSelector.parseContextLength(candidate);
+            if (Number.isFinite(parsed) && parsed > 0) return parsed;
+        }
+    }
+    return 4096;
+}
+
+function deriveQuant(model = {}) {
+    const candidates = [model.quant, model.quantization];
+    for (const candidate of candidates) {
+        if (typeof candidate === 'string' && candidate.trim()) {
+            return canonicalSelector.normalizeQuantization(candidate);
+        }
+        if (Array.isArray(candidate) && candidate.length > 0) {
+            // ExpandedModelsDatabase stores quantization as an array of options;
+            // prefer the highest-quality option that the hierarchy knows about.
+            const normalized = candidate.map((q) => canonicalSelector.normalizeQuantization(q));
+            for (const q of canonicalSelector.quantHierarchy) {
+                if (normalized.includes(q)) return q;
+            }
+            return normalized[0];
+        }
+    }
+    return 'Q4_K_M';
+}
+
+function deriveModalities(model = {}) {
+    if (Array.isArray(model.modalities) && model.modalities.length > 0) {
+        return model.modalities.map((m) => String(m).toLowerCase());
+    }
+    const text = [
+        model.name,
+        model.model_name,
+        model.model_id,
+        model.model_identifier,
+        model.specialization,
+        model.category,
+        ...(Array.isArray(model.capabilities) ? model.capabilities : [String(model.capabilities || '')]),
+        ...(Array.isArray(model.input_types) ? model.input_types : [])
+    ]
+        .filter(Boolean)
+        .join(' ')
+        .toLowerCase();
+    const hasVision = /\b(vision|vl\b|llava|bakllava|moondream|multimodal|image)\b/.test(text);
+    return hasVision ? ['text', 'vision'] : ['text'];
+}
+
+function deriveTags(model = {}, modalities = ['text']) {
+    const explicit = Array.isArray(model.tags) ? model.tags.map((t) => String(t).toLowerCase()) : [];
+    const tags = new Set(explicit);
+
+    const text = [
+        model.name,
+        model.model_name,
+        model.model_id,
+        model.model_identifier,
+        model.specialization,
+        model.category,
+        model.tag,
+        ...(Array.isArray(model.capabilities)
+            ? model.capabilities
+            : [String(model.capabilities || '')]),
+        ...(Array.isArray(model.use_cases) ? model.use_cases : [])
+    ]
+        .filter(Boolean)
+        .join(' ')
+        .toLowerCase();
+
+    if (/\b(coder|code|programming|deepseek-coder|starcoder|codellama|codegemma)\b/.test(text)) tags.add('coder');
+    if (/\binstruct\b/.test(text)) tags.add('instruct');
+    if (/\b(chat|assistant|conversation)\b/.test(text)) tags.add('chat');
+    if (/\bembed/.test(text)) tags.add('embedding');
+    if (/\b(reason|math|logic)\b/.test(text)) tags.add('reasoning');
+    if (modalities.includes('vision')) tags.add('vision');
+
+    // Most general/instruct models should compete in the general category even
+    // when no explicit tag exists, matching the permissive engines' behavior.
+    if (tags.size === 0) tags.add('chat');
+
+    return [...tags];
+}
+
+function deriveIdentifier(model = {}) {
+    return (
+        model.model_identifier ||
+        model.tag ||
+        model.model_id ||
+        model.model_name ||
+        model.name ||
+        'unknown-model'
+    );
+}
+
+/**
+ * Convert an arbitrary model object (any of the three command shapes) into the
+ * already-normalized deterministic model shape. The original object is kept on
+ * `__source` so callers can map ranking output back to their display shape.
+ */
+function normalizeToDeterministic(model = {}) {
+    const paramsB = deriveParamsB(model);
+    const sizeGB = deriveSizeGB(model, paramsB) ?? Math.max(0.4, paramsB * 0.6);
+    const modalities = deriveModalities(model);
+    const tags = deriveTags(model, modalities);
+    const quant = deriveQuant(model);
+    const identifier = deriveIdentifier(model);
+
+    const moeFlag = Boolean(model.isMoE || model.is_moe);
+    const totalParamsB =
+        toFiniteNumber(model.totalParamsB) ?? toFiniteNumber(model.total_params_b) ?? null;
+    const activeParamsB =
+        toFiniteNumber(model.activeParamsB) ?? toFiniteNumber(model.active_params_b) ?? null;
+
+    return {
+        // Canonical deterministic fields (pass `alreadyNormalized` branch).
+        name: identifier,
+        model_identifier: identifier,
+        family: model.family || canonicalSelector.extractFamily(String(identifier)),
+        paramsB,
+        ctxMax: deriveContext(model),
+        quant,
+        sizeGB,
+        modalities,
+        tags,
+        installed: Boolean(model.installed || model.isOllamaInstalled),
+        pulls: toFiniteNumber(model.pulls) ?? toFiniteNumber(model.actual_pulls) ?? 0,
+        availableQuantizations: [quant],
+        sizeByQuant: { [quant]: sizeGB },
+        // MoE passthrough so memory/speed estimation stays accurate.
+        isMoE: moeFlag,
+        is_moe: moeFlag,
+        totalParamsB,
+        activeParamsB,
+        expertCount: toFiniteNumber(model.expertCount) ?? toFiniteNumber(model.expert_count) ?? null,
+        expertsActivePerToken:
+            toFiniteNumber(model.expertsActivePerToken) ??
+            toFiniteNumber(model.experts_active_per_token) ??
+            null,
+        // Freshness defaults: neutral so we never penalize callers that lack
+        // timestamps (the per-command sources already carry these when known).
+        freshnessScore: toFiniteNumber(model.freshnessScore) ?? 55,
+        isStale: Boolean(model.isStale),
+        isDeprecated: Boolean(model.isDeprecated || model.deprecated),
+        source: model.source || 'unified_core',
+        // Back-reference for result mapping.
+        __source: model
+    };
+}
+
+/**
+ * Canonical ranking entrypoint. Returns the deterministic selector's ranked
+ * candidates (each carries `.score`, `.components {Q,S,F,C,H}`, `.meta`, etc.)
+ * for the supplied models/hardware. Each candidate's `meta.__source` points
+ * back to the caller's original model object.
+ *
+ * @param {Array<Object>} models   Arbitrary model objects (any command shape).
+ * @param {Object} hardware        Hardware profile (any detector shape).
+ * @param {Object} options
+ * @param {string} [options.category='general']  Use-case / category.
+ * @param {string} [options.optimizeFor='balanced']
+ * @param {string} [options.runtime='ollama']
+ * @param {number} [options.topN]  Limit (default: all candidates).
+ * @returns {Promise<{category, candidates, total_evaluated, hardware}>}
+ */
+async function rankModels(models, hardware, options = {}) {
+    const category = normalizeCategory(options.category || options.useCase || 'general');
+    const pool = (Array.isArray(models) ? models : [])
+        .filter(Boolean)
+        .map(normalizeToDeterministic);
+
+    const result = await canonicalSelector.selectModels(category, {
+        topN: typeof options.topN === 'number' ? options.topN : pool.length || 1,
+        enableProbe: false,
+        silent: true,
+        optimizeFor: options.optimizeFor || options.optimize || options.objective || 'balanced',
+        runtime: options.runtime || 'ollama',
+        hardware: hardware || undefined,
+        installedModels: [],
+        modelPool: pool
+    });
+
+    return result;
+}
+
+/**
+ * Map deterministic-engine categories from the broader vocabulary the commands
+ * use (e.g. `check`/`smart-recommend` accept `chat`, `creative`, `vision`,
+ * `talking`) onto the categories the deterministic core scores.
+ */
+function normalizeCategory(category) {
+    const normalized = String(category || 'general').toLowerCase().trim();
+    const map = {
+        general: 'general',
+        chat: 'general',
+        talking: 'general',
+        assistant: 'general',
+        coding: 'coding',
+        code: 'coding',
+        programming: 'coding',
+        reasoning: 'reasoning',
+        math: 'reasoning',
+        multimodal: 'multimodal',
+        vision: 'multimodal',
+        embeddings: 'embeddings',
+        embedding: 'embeddings',
+        summarization: 'summarization',
+        reading: 'reading',
+        creative: 'general',
+        fast: 'general',
+        quality: 'general',
+        longctx: 'reading'
+    };
+    return map[normalized] || 'general';
+}
+
+module.exports = {
+    canonicalSelector,
+    rankModels,
+    normalizeToDeterministic,
+    normalizeCategory,
+    deriveParamsB,
+    deriveSizeGB
+};

package/src/models/scoring-engine.js

@@ -25,6 +25,7 @@ class ScoringEngine {
         // Model family quality rankings (0-100 base score)
         this.familyQuality = {
             // Frontier models
+            'qwen3': 95,
             'qwen2.5': 95,
             'qwen2': 90,
             'llama3.3': 95,
@@ -35,12 +36,15 @@ class ScoringEngine {
             'deepseek-v2.5': 94,
             'deepseek-coder-v2': 92,
             'deepseek-r1': 96,
+            'gemma3': 90,
             'gemma2': 90,
             'gemma': 82,
+            'phi4': 92,
             'phi-4': 92,
             'phi-3.5': 88,
             'phi-3': 85,
             'phi-2': 75,
+            'granite3': 80,
             'mistral-large': 94,
             'mistral': 85,
             'mixtral': 88,
@@ -443,8 +447,11 @@ class ScoringEngine {
             // Tight fit
             return 70 + (1.0 - usage) / 0.15 * 20;
         } else if (usage <= 1.2) {
-            // May work with swapping (especially on Mac)
-            return 50 - (usage - 1.0) * 100;
+            // May work with swapping (especially on Mac). Decay continuously from
+            // 70 at usage=1.0 down to 0 at usage=1.2 — the old `50 - (usage-1)*100`
+            // dropped to ~50 the instant usage crossed 1.0, a 20-point cliff that
+            // could flip the ranking of two near-identical-size variants.
+            return 70 - (usage - 1.0) / 0.2 * 70;
         } else {
             // Won't fit
             return 0;

package/src/ollama/capacity-planner.js

@@ -307,10 +307,21 @@ class OllamaCapacityPlanner {
         const fallbackState = this.computeLoadState(modelPool, fallbackCtx, 1, budgetGB);
         const fallbackTotalGB = fallbackState.baseTotalGB + fallbackState.kvAtContextGB;
 
+        // Even the reduced "recommended" / "fallback" settings can exceed the budget
+        // when a single model's base memory alone is larger than the budget. Surface
+        // that explicitly so callers don't blindly apply env vars that will OOM.
+        const recommendedFits = recommendedTotalGB <= budgetGB;
+        const fallbackFits = fallbackTotalGB <= budgetGB;
+
         const notes = [];
         if (!requestedFits) {
             notes.push('Requested settings exceed available memory budget; reduced settings are recommended.');
         }
+        if (!fallbackFits) {
+            notes.push('No safe configuration fits the memory budget for this model selection; choose a smaller or more-quantized model.');
+        } else if (!recommendedFits) {
+            notes.push('Recommended settings still exceed the budget; apply the fallback settings instead.');
+        }
         if (recommendedCtx < requestedCtx) {
             notes.push(`Context reduced from ${requestedCtx} to ${recommendedCtx} to avoid memory pressure.`);
         }
@@ -366,7 +377,8 @@ class OllamaCapacityPlanner {
                 max_loaded_models: recommendedLoaded,
                 max_queue: maxQueue,
                 keep_alive: profile.keepAlive,
-                flash_attention: flashAttention
+                flash_attention: flashAttention,
+                fits: recommendedFits
             },
             memory: {
                 budgetGB: Math.round(budgetGB * 100) / 100,
@@ -379,7 +391,8 @@ class OllamaCapacityPlanner {
                 num_ctx: fallbackCtx,
                 num_parallel: 1,
                 max_loaded_models: 1,
-                estimated_memory_gb: Math.round(fallbackTotalGB * 100) / 100
+                estimated_memory_gb: Math.round(fallbackTotalGB * 100) / 100,
+                fits: fallbackFits
             },
             shell: {
                 env: {