@mlbottleneck/engine 0.4.1 → 0.5.1

package/mlbottleneck-engine.d.ts

@@ -95,6 +95,27 @@ export interface DeviceSummary {
   } | null;
 }
 
+export interface MeasuredRun {
+  tokensPerSecond: number;
+  origin: 'community' | 'lab';
+  stack: 'stock' | 'lab-baseline' | 'tuned';
+  model: string;
+  hardware: string;
+  deviceCount: number;
+  runtime: string;
+  quantization: string;
+  depthTokens: number;
+  /** Concurrent requests in the measured run (always equals the request's batch size). */
+  concurrency: number;
+  /** tokensPerSecond is per request; this is the combined rate across the batch. */
+  aggregateTokensPerSecond: number;
+  speculation: { method: string; tokens: number | null } | null;
+  /** Same runtime, quantization family, and device count as the request. */
+  sameSetup: boolean;
+  url: string | null;
+  note: string | null;
+}
+
 export interface Prediction {
   fits: boolean;
   strategy: { key: Strategy; label: string; reasoning: string | null; auto: boolean };
@@ -118,6 +139,13 @@ export interface Prediction {
     verifiedPeers: number;
   } | null;
   memory: { modelSizeGB: number | null; residentWeightsGB: number | null; kvCacheGB: number | null; availableGB: number | null };
+  /** Measured runs on the same model and hardware template (null when none exist). */
+  measured: {
+    /** Closest stock measurement: a community gold run or a lab stock/baseline row. */
+    nearest: MeasuredRun | null;
+    /** Closest tuned neural.download lab result: what a tuned stack reached, never a stock reference. */
+    labTuned: MeasuredRun | null;
+  };
   bottleneck: string | null;
   power: { watts: number | null; tdpWatts: number | null; costPerDay: number | null; costPer1KTokens: number | null } | null;
   devices: DeviceSummary[];
@@ -142,7 +170,12 @@ export interface HardwareListing {
 
 export interface EvidenceSnapshot {
   generatedAt?: string;
+  /** Community gold rows: calibrate the engine (peer correction, optimized target). */
   goldCases: any[];
+  /** neural.download lab rows (stock / lab-baseline / tuned): measured references only, never calibration. */
+  labCases?: any[];
+  labSource?: string;
+  labUpdated?: string;
 }
 
 export interface Engine {

package/mlbottleneck-engine.mjs

@@ -1,4 +1,4 @@
-/*! ML Bottleneck engine v0.4.1 | https://mlbottleneck.com | MIT */
+/*! ML Bottleneck engine v0.5.1 | https://mlbottleneck.com | MIT */
 // Generated by scripts/build-sdk.mjs from engine.js and sdk/api.js. Do not edit.
 
 function createEngine(options = {}) {
@@ -23,13 +23,22 @@ function createEngine(options = {}) {
         }
     }
 
-    // Benchmark evidence (Localmaxxing gold rows) used for peer calibration.
-    const ENGINE_EVIDENCE = { goldCases: [] };
+    // Benchmark evidence. Localmaxxing gold rows calibrate the engine (peer
+    // correction, optimized target). Lab rows (data/lab-evidence.json, the
+    // author's neural.download Intel lab) never calibrate it: they are shown
+    // beside a plan as measured references — stock / lab-baseline rows can be
+    // the nearest measured run, tuned rows show what a tuned stack reached.
+    const ENGINE_EVIDENCE = { goldCases: [], labCases: [] };
     let goldValidationCache = null;
+    let labEvidenceCache = null;
+    let labValidationCache = null;
 
     function setEngineEvidence(snapshot) {
         ENGINE_EVIDENCE.goldCases = Array.isArray(snapshot?.goldCases) ? snapshot.goldCases : [];
+        ENGINE_EVIDENCE.labCases = Array.isArray(snapshot?.labCases) ? snapshot.labCases : [];
         goldValidationCache = null;
+        labEvidenceCache = null;
+        labValidationCache = null;
     }
 
     // Nominal storage bytes per weight for each quantization family. Real
@@ -7357,6 +7366,244 @@ function createEngine(options = {}) {
     }
 
 
+    // ---- Lab evidence (neural.download) ------------------------------------
+    const LAB_EVIDENCE_ORIGIN = 'neural.download';
+    const TENSOR_CAPABLE_RUNTIMES = ['vllm', 'sglang', 'tensorrt_llm'];
+
+    // One measured lab point in the shape the gold helpers understand. Depth
+    // sweeps and speculation ladders expand into one row per point so a plan
+    // at 32K depth finds the 32K measurement, not the 128-token one.
+    function normalizeLabEvidenceRow(row, overrides = {}) {
+        const preset = MODEL_PRESETS[row.presetKey];
+        const template = DEVICE_TEMPLATES[row.hardwareTemplate];
+        if (!preset || !template) return null;
+        const format = getQuantFormat(row.quantization);
+        const quantKey = format ? format.family : String(row.quantization || 'q4').toLowerCase();
+        const promptTokens = Number.isFinite(overrides.promptTokens) ? overrides.promptTokens : (Number.isFinite(row.promptTokens) ? row.promptTokens : 128);
+        const outputTokens = Number.isFinite(row.outputTokens) && row.outputTokens > 0 ? row.outputTokens : 128;
+        const speculation = overrides.speculation !== undefined ? overrides.speculation : (row.speculation || null);
+        const observedTokS = Number.isFinite(overrides.observedTokS) ? overrides.observedTokS : row.observedTokS;
+        if (!Number.isFinite(observedTokS) || observedTokS <= 0) return null;
+        const batchSize = Math.max(1, Number(overrides.batchSize) || 1);
+        const deviceCount = Math.max(1, Number(row.deviceCount) || 1);
+        const runtime = FRAMEWORK_PROFILES[row.runtimeKey];
+        const strategy = row.strategy || (deviceCount > 1 ? (TENSOR_CAPABLE_RUNTIMES.includes(row.runtimeKey) ? 'tensor' : 'pipeline') : 'pipeline');
+        return {
+            id: overrides.id || row.id,
+            origin: LAB_EVIDENCE_ORIGIN,
+            isLab: true,
+            stack: row.stack || 'stock',
+            source: row.url || '',
+            url: row.url || '',
+            note: row.note || '',
+            model: preset.label || row.presetKey,
+            hfId: preset.hfId || '',
+            presetKey: row.presetKey,
+            hardware: template.name || row.hardwareTemplate,
+            hardwareTemplate: row.hardwareTemplate,
+            deviceCount,
+            runtimeKey: row.runtimeKey,
+            engine: runtime ? runtime.label : row.runtimeKey,
+            quantization: row.quantization || quantKey,
+            quantKey,
+            strategy,
+            speculation,
+            promptTokens,
+            outputTokens,
+            contextLength: promptTokens + outputTokens,
+            decodeContextTokens: promptTokens + outputTokens / 2,
+            // Per-request decode rate (one sequence's tokens per second); the
+            // aggregate across a batched run is kept separately.
+            observedTokS,
+            batchSize,
+            aggregateTokS: Number.isFinite(overrides.aggregateTokS) ? overrides.aggregateTokS : observedTokS * batchSize,
+            prefillTokS: Number.isFinite(overrides.prefillTokS) ? overrides.prefillTokS : (Number.isFinite(row.prefillTokS) ? row.prefillTokS : null),
+            reproducibility: 1
+        };
+    }
+
+    function getLabEvidenceRows() {
+        if (labEvidenceCache) return labEvidenceCache;
+        const rows = [];
+        for (const row of ENGINE_EVIDENCE.labCases || []) {
+            if (!row || typeof row !== 'object' || !row.id) continue;
+            const base = normalizeLabEvidenceRow(row);
+            if (base) rows.push(base);
+            for (const point of Array.isArray(row.depthSweep) ? row.depthSweep : []) {
+                const swept = normalizeLabEvidenceRow(row, {
+                    id: `${row.id}@${point.promptTokens}`,
+                    promptTokens: point.promptTokens,
+                    observedTokS: point.decodeTokS,
+                    prefillTokS: point.prefillTokS
+                });
+                if (swept) rows.push(swept);
+            }
+            for (const rung of Array.isArray(row.speculationLadder) ? row.speculationLadder : []) {
+                const stepped = normalizeLabEvidenceRow(row, {
+                    id: `${row.id}@${rung.method}${rung.tokens}`,
+                    speculation: { method: rung.method, tokens: rung.tokens },
+                    observedTokS: rung.observedTokS
+                });
+                if (stepped) rows.push(stepped);
+            }
+            // Concurrency sweeps: { users, perUserTokS, aggregateTokS } per level
+            // (llama-batched-bench -npl / vllm bench serve --max-concurrency).
+            for (const level of Array.isArray(row.concurrencySweep) ? row.concurrencySweep : []) {
+                const users = Math.max(1, Number(level.users) || 1);
+                const perUser = Number.isFinite(level.perUserTokS) ? level.perUserTokS : (Number.isFinite(level.aggregateTokS) ? level.aggregateTokS / users : NaN);
+                const batched = normalizeLabEvidenceRow(row, {
+                    id: `${row.id}@u${users}`,
+                    batchSize: users,
+                    observedTokS: perUser,
+                    aggregateTokS: Number.isFinite(level.aggregateTokS) ? level.aggregateTokS : perUser * users
+                });
+                if (batched) rows.push(batched);
+            }
+        }
+        labEvidenceCache = rows;
+        return rows;
+    }
+
+    // Rows that can stand in as the plan's "nearest measured" run: community
+    // gold runs plus the lab's stock / baseline rows without speculation. A
+    // tuned stack is not a stock reference.
+    function getMeasuredReferenceRows() {
+        return getGoldValidationRows().concat(getLabEvidenceRows().filter(row => row.stack !== 'tuned' && !row.speculation));
+    }
+
+    function describeSameSetup(target, row) {
+        return row.runtimeKey === target.runtimeKey && row.quantKey === target.quantKey &&
+            (row.deviceCount || 1) === (target.deviceCount || 1);
+    }
+
+    // A reference run must serve the same number of concurrent requests as
+    // the plan: per-request decode at 16 users says nothing about one user.
+    function sameBatch(target, row) {
+        return (row.batchSize || 1) === (target.batchSize || 1);
+    }
+
+    function depthDistance(target, row) {
+        const targetDepth = Math.max(64, Number.isFinite(target.decodeContextTokens) ? target.decodeContextTokens : (target.contextLength || 64));
+        const rowDepth = Math.max(64, Number.isFinite(row.decodeContextTokens) ? row.decodeContextTokens : (row.contextLength || 64));
+        return Math.abs(Math.log(targetDepth / rowDepth));
+    }
+
+    // Closest measured run for the ladder: same preset and hardware template
+    // are required; runtime, quantization, device count, and decode depth
+    // refine the choice. Community runs and lab stock rows compete equally.
+    function findNearestMeasuredRun(target) {
+        if (!target?.presetKey || !target.hardwareTemplate) return null;
+        const candidates = getMeasuredReferenceRows()
+            .filter(row => row.presetKey === target.presetKey && row.hardwareTemplate === target.hardwareTemplate && sameBatch(target, row))
+            .map(row => ({ row, score: getGoldSimilarity(target, row) }))
+            .sort((a, b) => b.score - a.score ||
+                depthDistance(target, a.row) - depthDistance(target, b.row) ||
+                (b.row.reproducibility || 0) - (a.row.reproducibility || 0));
+        const best = candidates[0];
+        if (!best) return null;
+        return { ...best.row, sameSetup: describeSameSetup(target, best.row), similarity: best.score };
+    }
+
+    // The tuned lab result closest to this plan on the same machine (same
+    // hardware template and device count — a different card count is a
+    // different roofline); runtime, quantization, and depth refine the
+    // choice and ties go to the faster run. Labeled as tuned, never used as
+    // a stock reference.
+    function findLabTunedRun(target) {
+        if (!target?.presetKey || !target.hardwareTemplate) return null;
+        const candidates = getLabEvidenceRows()
+            .filter(row => row.stack === 'tuned' && row.presetKey === target.presetKey &&
+                row.hardwareTemplate === target.hardwareTemplate && (row.deviceCount || 1) === (target.deviceCount || 1) && sameBatch(target, row))
+            .map(row => ({ row, score: getGoldSimilarity(target, row) }))
+            .sort((a, b) => b.score - a.score ||
+                depthDistance(target, a.row) - depthDistance(target, b.row) ||
+                b.row.observedTokS - a.row.observedTokS);
+        const best = candidates[0];
+        if (!best) return null;
+        return { ...best.row, sameSetup: describeSameSetup(target, best.row), similarity: best.score };
+    }
+
+    function buildEvidenceTarget(modelConfig, devicesArray, metrics = null) {
+        return {
+            hfId: modelConfig.hfId,
+            presetKey: modelConfig.modelPreset,
+            hardwareTemplate: devicesArray[0]?.template || '',
+            runtimeKey: getFrameworkProfile(modelConfig, devicesArray).key,
+            quantKey: modelConfig.quantizationType,
+            quantization: modelConfig.quantFormat || '',
+            deviceCount: devicesArray.length,
+            contextLength: modelConfig.seqLength,
+            decodeContextTokens: getDecodeContextTokens(modelConfig),
+            hasOverflow: Array.isArray(metrics) ? metrics.some(metric => metric.hasOverflow) : false,
+            batchSize: modelConfig.batchSize || 1
+        };
+    }
+
+    // The engine's view of one lab row in the row's own setup (speculation
+    // included when the row used it): stock projection, optimized target, and
+    // physical roofline. Stock rows show how the engine tracks the lab's
+    // hardware; tuned rows show how much of the optimized target a tuned
+    // stack actually reached.
+    function calculateLabCaseProjection(row) {
+        const preset = MODEL_PRESETS[row.presetKey];
+        const template = DEVICE_TEMPLATES[row.hardwareTemplate];
+        if (!preset || !template || !DTYPE_SIZES[row.quantKey]) return null;
+        const devices = Array.from({ length: row.deviceCount }, (_, index) => ({
+            id: index + 1,
+            template: row.hardwareTemplate,
+            name: `${template.name || row.hardwareTemplate}${row.deviceCount > 1 ? ` #${index + 1}` : ''}`,
+            ...JSON.parse(JSON.stringify(template))
+        }));
+        const strategy = row.strategy;
+        const spec = row.speculation;
+        const modelConfig = normalizeModelConfig({
+            ...preset,
+            modelPreset: row.presetKey,
+            quantizationType: row.quantKey,
+            quantFormat: getQuantFormat(row.quantization) ? row.quantization : '',
+            runtimeFramework: row.runtimeKey,
+            parallelismStrategy: strategy,
+            optimizationMode: spec ? 'speculative' : 'none',
+            specMethod: spec ? spec.method : 'mtp',
+            specTokens: spec ? spec.tokens : null,
+            specAcceptance: null,
+            specDraftRatio: spec && Number.isFinite(spec.draftRatio) ? spec.draftRatio : null,
+            kvCacheCompression: 'none',
+            batchSize: row.batchSize || 1,
+            promptTokens: row.promptTokens,
+            outputTokens: row.outputTokens,
+            seqLength: row.promptTokens + row.outputTokens
+        });
+        const metrics = calculateMetricsForConfig(modelConfig, devices);
+        const genericTokS = calculateSystemRateFromDeviceRates(metrics.map(metric => metric.decodeTokensPerSecond), strategy, row.batchSize || 1, devices, getSystemRateOptions(modelConfig));
+        const prefillProjectedTokS = getSystemPrefillRateForMetrics(modelConfig, metrics, devices, strategy);
+        const calibration = calculateCurrentCalibration(modelConfig, metrics, genericTokS, strategy, devices);
+        if (!calibration || !Number.isFinite(genericTokS) || genericTokS <= 0) return null;
+        return {
+            ...row,
+            genericTokS,
+            prefillProjectedTokS,
+            expectedTokS: calibration.expectedTokS,
+            optimizedTokS: calibration.optimizedTokS,
+            physicalTokS: calibration.physicalTokS,
+            latencyBoundTokS: calibration.latencyBoundTokS,
+            calibrationPeers: calibration.peers,
+            confidence: calibration.confidence,
+            observedToGeneric: row.observedTokS / genericTokS,
+            observedToExpected: row.observedTokS / calibration.expectedTokS,
+            observedToOptimized: row.observedTokS / calibration.optimizedTokS,
+            observedToPhysical: row.observedTokS / calibration.physicalTokS,
+            hasOverflow: metrics.some(metric => metric.hasOverflow)
+        };
+    }
+
+    function getLabValidationRows() {
+        if (labValidationCache) return labValidationCache;
+        labValidationCache = getLabEvidenceRows().map(calculateLabCaseProjection).filter(Boolean);
+        return labValidationCache;
+    }
+
+
     function calculateCurrentCalibration(modelConfig, metrics, genericSystemRate, strategy, devicesArray = null) {
         const devices = devicesArray || defaultDevices();
         if (!metrics?.length || !Number.isFinite(genericSystemRate)) return null;
@@ -7669,6 +7916,7 @@ function createEngine(options = {}) {
         }
         const power = calculatePowerAndCost(devices, aggregateDecode, metrics, request.usage || {});
         const speculationActive = Boolean(primary?.speculation && primary.speculationMultiplier > 1);
+        const evidenceTarget = buildEvidenceTarget(config, devices, metrics);
         return {
             fits,
             strategy: { key: strategy, label: strategy, reasoning: strategyInfo?.reasoning || null, auto: Boolean(strategyInfo) },
@@ -7702,6 +7950,10 @@ function createEngine(options = {}) {
                 kvCacheGB: sdkRound(metrics.reduce((sum, metric) => sum + (metric.residentKvCacheGB || 0), 0), 2),
                 availableGB: sdkRound(devices.reduce((sum, device) => sum + (parseFloat(device.memoryGB) || 0), 0), 1)
             },
+            measured: {
+                nearest: sdkSummarizeMeasuredRun(findNearestMeasuredRun(evidenceTarget)),
+                labTuned: sdkSummarizeMeasuredRun(findLabTunedRun(evidenceTarget))
+            },
             bottleneck: primary?.decodeTimeBreakdown?.dominant || null,
             power: power ? { watts: sdkRound(power.actualPowerWatts, 0), tdpWatts: sdkRound(power.totalTDP, 0), costPerDay: sdkRound(power.dailyCost, 3), costPer1KTokens: sdkRound(power.costPer1KTokens, 5) } : null,
             devices: metrics.map((metric, index) => sdkSummarizeDevice(metric, devices[index])),
@@ -7731,6 +7983,29 @@ function createEngine(options = {}) {
         };
     }
 
+    // A measured reference run (community gold row or neural.download lab row)
+    // in a stable shape; null when nothing on the same model + hardware exists.
+    function sdkSummarizeMeasuredRun(row) {
+        if (!row) return null;
+        return {
+            tokensPerSecond: sdkRound(row.observedTokS, 2),
+            origin: row.isLab ? 'lab' : 'community',
+            stack: row.isLab ? row.stack : 'stock',
+            model: row.model,
+            hardware: row.hardware,
+            deviceCount: row.deviceCount || 1,
+            runtime: row.runtimeKey,
+            quantization: row.quantization || row.quantKey,
+            depthTokens: Math.round(row.decodeContextTokens || row.contextLength || 0),
+            concurrency: row.batchSize || 1,
+            aggregateTokensPerSecond: sdkRound(row.aggregateTokS || row.observedTokS * (row.batchSize || 1), 2),
+            speculation: row.speculation ? { method: row.speculation.method, tokens: row.speculation.tokens ?? null } : null,
+            sameSetup: Boolean(row.sameSetup),
+            url: row.source || row.url || null,
+            note: row.note || null
+        };
+    }
+
     function listModels() {
         return Object.entries(MODEL_PRESETS).map(([key, preset]) => ({
             key,
@@ -7792,11 +8067,11 @@ function createEngine(options = {}) {
 
     if (options.snapshot) setEngineEvidence(options.snapshot);
     const api = createApi();
-    api.version = "0.4.1";
+    api.version = "0.5.1";
     api.evidenceGeneratedAt = options.snapshot?.generatedAt || null;
     return api;
 }
 
-const version = "0.4.1";
+const version = "0.5.1";
 export { createEngine, version };
 export default createEngine;