@mlbottleneck/engine 0.5.0 → 0.5.1

package/README.md

@@ -80,8 +80,8 @@ const { ceiling } = engine.predict({ model: 'qwen3.6_35b_a3b', hardware: 'AMD St
   memory:  { modelSizeGB, residentWeightsGB, kvCacheGB, availableGB },
   measured: { nearest,                     // closest stock measurement on this model + hardware template (community gold run or lab stock row), or null
               labTuned },                  // closest tuned neural.download lab run on the same machine (same template and device count), or null
-           // each: { tokensPerSecond, origin: 'community'|'lab', stack, model, hardware, deviceCount, runtime, quantization,
-           //         depthTokens, speculation, sameSetup, url, note }
+           // each: { tokensPerSecond (per request), aggregateTokensPerSecond, concurrency, origin: 'community'|'lab', stack,
+           //         model, hardware, deviceCount, runtime, quantization, depthTokens, speculation, sameSetup, url, note }
   bottleneck: 'memory' | 'compute' | 'runtime' | 'coordination' | ...,   // devices[].coreBinding adds 'attention' for deep contexts
   power:   { watts, tdpWatts, costPerDay, costPer1KTokens },
   devices: [{ name, template, residentWeightGB, kvCacheGB, hasOverflow, overflowMode,

package/mlbottleneck-engine.d.ts

@@ -105,6 +105,10 @@ export interface MeasuredRun {
   runtime: string;
   quantization: string;
   depthTokens: number;
+  /** Concurrent requests in the measured run (always equals the request's batch size). */
+  concurrency: number;
+  /** tokensPerSecond is per request; this is the combined rate across the batch. */
+  aggregateTokensPerSecond: number;
   speculation: { method: string; tokens: number | null } | null;
   /** Same runtime, quantization family, and device count as the request. */
   sameSetup: boolean;

package/mlbottleneck-engine.mjs

@@ -1,4 +1,4 @@
-/*! ML Bottleneck engine v0.5.0 | https://mlbottleneck.com | MIT */
+/*! ML Bottleneck engine v0.5.1 | https://mlbottleneck.com | MIT */
 // Generated by scripts/build-sdk.mjs from engine.js and sdk/api.js. Do not edit.
 
 function createEngine(options = {}) {
@@ -7384,6 +7384,7 @@ function createEngine(options = {}) {
         const speculation = overrides.speculation !== undefined ? overrides.speculation : (row.speculation || null);
         const observedTokS = Number.isFinite(overrides.observedTokS) ? overrides.observedTokS : row.observedTokS;
         if (!Number.isFinite(observedTokS) || observedTokS <= 0) return null;
+        const batchSize = Math.max(1, Number(overrides.batchSize) || 1);
         const deviceCount = Math.max(1, Number(row.deviceCount) || 1);
         const runtime = FRAMEWORK_PROFILES[row.runtimeKey];
         const strategy = row.strategy || (deviceCount > 1 ? (TENSOR_CAPABLE_RUNTIMES.includes(row.runtimeKey) ? 'tensor' : 'pipeline') : 'pipeline');
@@ -7411,7 +7412,11 @@ function createEngine(options = {}) {
             outputTokens,
             contextLength: promptTokens + outputTokens,
             decodeContextTokens: promptTokens + outputTokens / 2,
+            // Per-request decode rate (one sequence's tokens per second); the
+            // aggregate across a batched run is kept separately.
             observedTokS,
+            batchSize,
+            aggregateTokS: Number.isFinite(overrides.aggregateTokS) ? overrides.aggregateTokS : observedTokS * batchSize,
             prefillTokS: Number.isFinite(overrides.prefillTokS) ? overrides.prefillTokS : (Number.isFinite(row.prefillTokS) ? row.prefillTokS : null),
             reproducibility: 1
         };
@@ -7441,6 +7446,19 @@ function createEngine(options = {}) {
                 });
                 if (stepped) rows.push(stepped);
             }
+            // Concurrency sweeps: { users, perUserTokS, aggregateTokS } per level
+            // (llama-batched-bench -npl / vllm bench serve --max-concurrency).
+            for (const level of Array.isArray(row.concurrencySweep) ? row.concurrencySweep : []) {
+                const users = Math.max(1, Number(level.users) || 1);
+                const perUser = Number.isFinite(level.perUserTokS) ? level.perUserTokS : (Number.isFinite(level.aggregateTokS) ? level.aggregateTokS / users : NaN);
+                const batched = normalizeLabEvidenceRow(row, {
+                    id: `${row.id}@u${users}`,
+                    batchSize: users,
+                    observedTokS: perUser,
+                    aggregateTokS: Number.isFinite(level.aggregateTokS) ? level.aggregateTokS : perUser * users
+                });
+                if (batched) rows.push(batched);
+            }
         }
         labEvidenceCache = rows;
         return rows;
@@ -7458,6 +7476,12 @@ function createEngine(options = {}) {
             (row.deviceCount || 1) === (target.deviceCount || 1);
     }
 
+    // A reference run must serve the same number of concurrent requests as
+    // the plan: per-request decode at 16 users says nothing about one user.
+    function sameBatch(target, row) {
+        return (row.batchSize || 1) === (target.batchSize || 1);
+    }
+
     function depthDistance(target, row) {
         const targetDepth = Math.max(64, Number.isFinite(target.decodeContextTokens) ? target.decodeContextTokens : (target.contextLength || 64));
         const rowDepth = Math.max(64, Number.isFinite(row.decodeContextTokens) ? row.decodeContextTokens : (row.contextLength || 64));
@@ -7470,7 +7494,7 @@ function createEngine(options = {}) {
     function findNearestMeasuredRun(target) {
         if (!target?.presetKey || !target.hardwareTemplate) return null;
         const candidates = getMeasuredReferenceRows()
-            .filter(row => row.presetKey === target.presetKey && row.hardwareTemplate === target.hardwareTemplate)
+            .filter(row => row.presetKey === target.presetKey && row.hardwareTemplate === target.hardwareTemplate && sameBatch(target, row))
             .map(row => ({ row, score: getGoldSimilarity(target, row) }))
             .sort((a, b) => b.score - a.score ||
                 depthDistance(target, a.row) - depthDistance(target, b.row) ||
@@ -7489,7 +7513,7 @@ function createEngine(options = {}) {
         if (!target?.presetKey || !target.hardwareTemplate) return null;
         const candidates = getLabEvidenceRows()
             .filter(row => row.stack === 'tuned' && row.presetKey === target.presetKey &&
-                row.hardwareTemplate === target.hardwareTemplate && (row.deviceCount || 1) === (target.deviceCount || 1))
+                row.hardwareTemplate === target.hardwareTemplate && (row.deviceCount || 1) === (target.deviceCount || 1) && sameBatch(target, row))
             .map(row => ({ row, score: getGoldSimilarity(target, row) }))
             .sort((a, b) => b.score - a.score ||
                 depthDistance(target, a.row) - depthDistance(target, b.row) ||
@@ -7545,13 +7569,13 @@ function createEngine(options = {}) {
             specAcceptance: null,
             specDraftRatio: spec && Number.isFinite(spec.draftRatio) ? spec.draftRatio : null,
             kvCacheCompression: 'none',
-            batchSize: 1,
+            batchSize: row.batchSize || 1,
             promptTokens: row.promptTokens,
             outputTokens: row.outputTokens,
             seqLength: row.promptTokens + row.outputTokens
         });
         const metrics = calculateMetricsForConfig(modelConfig, devices);
-        const genericTokS = calculateSystemRateFromDeviceRates(metrics.map(metric => metric.decodeTokensPerSecond), strategy, 1, devices, getSystemRateOptions(modelConfig));
+        const genericTokS = calculateSystemRateFromDeviceRates(metrics.map(metric => metric.decodeTokensPerSecond), strategy, row.batchSize || 1, devices, getSystemRateOptions(modelConfig));
         const prefillProjectedTokS = getSystemPrefillRateForMetrics(modelConfig, metrics, devices, strategy);
         const calibration = calculateCurrentCalibration(modelConfig, metrics, genericTokS, strategy, devices);
         if (!calibration || !Number.isFinite(genericTokS) || genericTokS <= 0) return null;
@@ -7973,6 +7997,8 @@ function createEngine(options = {}) {
             runtime: row.runtimeKey,
             quantization: row.quantization || row.quantKey,
             depthTokens: Math.round(row.decodeContextTokens || row.contextLength || 0),
+            concurrency: row.batchSize || 1,
+            aggregateTokensPerSecond: sdkRound(row.aggregateTokS || row.observedTokS * (row.batchSize || 1), 2),
             speculation: row.speculation ? { method: row.speculation.method, tokens: row.speculation.tokens ?? null } : null,
             sameSetup: Boolean(row.sameSetup),
             url: row.source || row.url || null,
@@ -8041,11 +8067,11 @@ function createEngine(options = {}) {
 
     if (options.snapshot) setEngineEvidence(options.snapshot);
     const api = createApi();
-    api.version = "0.5.0";
+    api.version = "0.5.1";
     api.evidenceGeneratedAt = options.snapshot?.generatedAt || null;
     return api;
 }
 
-const version = "0.5.0";
+const version = "0.5.1";
 export { createEngine, version };
 export default createEngine;

package/mlbottleneck-engine.umd.js

@@ -1,4 +1,4 @@
-/*! ML Bottleneck engine v0.5.0 | https://mlbottleneck.com | MIT */
+/*! ML Bottleneck engine v0.5.1 | https://mlbottleneck.com | MIT */
 // Generated by scripts/build-sdk.mjs from engine.js and sdk/api.js. Do not edit.
 (function (root, factory) {
     if (typeof define === 'function' && define.amd) {
@@ -7393,6 +7393,7 @@
             const speculation = overrides.speculation !== undefined ? overrides.speculation : (row.speculation || null);
             const observedTokS = Number.isFinite(overrides.observedTokS) ? overrides.observedTokS : row.observedTokS;
             if (!Number.isFinite(observedTokS) || observedTokS <= 0) return null;
+            const batchSize = Math.max(1, Number(overrides.batchSize) || 1);
             const deviceCount = Math.max(1, Number(row.deviceCount) || 1);
             const runtime = FRAMEWORK_PROFILES[row.runtimeKey];
             const strategy = row.strategy || (deviceCount > 1 ? (TENSOR_CAPABLE_RUNTIMES.includes(row.runtimeKey) ? 'tensor' : 'pipeline') : 'pipeline');
@@ -7420,7 +7421,11 @@
                 outputTokens,
                 contextLength: promptTokens + outputTokens,
                 decodeContextTokens: promptTokens + outputTokens / 2,
+                // Per-request decode rate (one sequence's tokens per second); the
+                // aggregate across a batched run is kept separately.
                 observedTokS,
+                batchSize,
+                aggregateTokS: Number.isFinite(overrides.aggregateTokS) ? overrides.aggregateTokS : observedTokS * batchSize,
                 prefillTokS: Number.isFinite(overrides.prefillTokS) ? overrides.prefillTokS : (Number.isFinite(row.prefillTokS) ? row.prefillTokS : null),
                 reproducibility: 1
             };
@@ -7450,6 +7455,19 @@
                     });
                     if (stepped) rows.push(stepped);
                 }
+                // Concurrency sweeps: { users, perUserTokS, aggregateTokS } per level
+                // (llama-batched-bench -npl / vllm bench serve --max-concurrency).
+                for (const level of Array.isArray(row.concurrencySweep) ? row.concurrencySweep : []) {
+                    const users = Math.max(1, Number(level.users) || 1);
+                    const perUser = Number.isFinite(level.perUserTokS) ? level.perUserTokS : (Number.isFinite(level.aggregateTokS) ? level.aggregateTokS / users : NaN);
+                    const batched = normalizeLabEvidenceRow(row, {
+                        id: `${row.id}@u${users}`,
+                        batchSize: users,
+                        observedTokS: perUser,
+                        aggregateTokS: Number.isFinite(level.aggregateTokS) ? level.aggregateTokS : perUser * users
+                    });
+                    if (batched) rows.push(batched);
+                }
             }
             labEvidenceCache = rows;
             return rows;
@@ -7467,6 +7485,12 @@
                 (row.deviceCount || 1) === (target.deviceCount || 1);
         }
 
+        // A reference run must serve the same number of concurrent requests as
+        // the plan: per-request decode at 16 users says nothing about one user.
+        function sameBatch(target, row) {
+            return (row.batchSize || 1) === (target.batchSize || 1);
+        }
+
         function depthDistance(target, row) {
             const targetDepth = Math.max(64, Number.isFinite(target.decodeContextTokens) ? target.decodeContextTokens : (target.contextLength || 64));
             const rowDepth = Math.max(64, Number.isFinite(row.decodeContextTokens) ? row.decodeContextTokens : (row.contextLength || 64));
@@ -7479,7 +7503,7 @@
         function findNearestMeasuredRun(target) {
             if (!target?.presetKey || !target.hardwareTemplate) return null;
             const candidates = getMeasuredReferenceRows()
-                .filter(row => row.presetKey === target.presetKey && row.hardwareTemplate === target.hardwareTemplate)
+                .filter(row => row.presetKey === target.presetKey && row.hardwareTemplate === target.hardwareTemplate && sameBatch(target, row))
                 .map(row => ({ row, score: getGoldSimilarity(target, row) }))
                 .sort((a, b) => b.score - a.score ||
                     depthDistance(target, a.row) - depthDistance(target, b.row) ||
@@ -7498,7 +7522,7 @@
             if (!target?.presetKey || !target.hardwareTemplate) return null;
             const candidates = getLabEvidenceRows()
                 .filter(row => row.stack === 'tuned' && row.presetKey === target.presetKey &&
-                    row.hardwareTemplate === target.hardwareTemplate && (row.deviceCount || 1) === (target.deviceCount || 1))
+                    row.hardwareTemplate === target.hardwareTemplate && (row.deviceCount || 1) === (target.deviceCount || 1) && sameBatch(target, row))
                 .map(row => ({ row, score: getGoldSimilarity(target, row) }))
                 .sort((a, b) => b.score - a.score ||
                     depthDistance(target, a.row) - depthDistance(target, b.row) ||
@@ -7554,13 +7578,13 @@
                 specAcceptance: null,
                 specDraftRatio: spec && Number.isFinite(spec.draftRatio) ? spec.draftRatio : null,
                 kvCacheCompression: 'none',
-                batchSize: 1,
+                batchSize: row.batchSize || 1,
                 promptTokens: row.promptTokens,
                 outputTokens: row.outputTokens,
                 seqLength: row.promptTokens + row.outputTokens
             });
             const metrics = calculateMetricsForConfig(modelConfig, devices);
-            const genericTokS = calculateSystemRateFromDeviceRates(metrics.map(metric => metric.decodeTokensPerSecond), strategy, 1, devices, getSystemRateOptions(modelConfig));
+            const genericTokS = calculateSystemRateFromDeviceRates(metrics.map(metric => metric.decodeTokensPerSecond), strategy, row.batchSize || 1, devices, getSystemRateOptions(modelConfig));
             const prefillProjectedTokS = getSystemPrefillRateForMetrics(modelConfig, metrics, devices, strategy);
             const calibration = calculateCurrentCalibration(modelConfig, metrics, genericTokS, strategy, devices);
             if (!calibration || !Number.isFinite(genericTokS) || genericTokS <= 0) return null;
@@ -7982,6 +8006,8 @@
                 runtime: row.runtimeKey,
                 quantization: row.quantization || row.quantKey,
                 depthTokens: Math.round(row.decodeContextTokens || row.contextLength || 0),
+                concurrency: row.batchSize || 1,
+                aggregateTokensPerSecond: sdkRound(row.aggregateTokS || row.observedTokS * (row.batchSize || 1), 2),
                 speculation: row.speculation ? { method: row.speculation.method, tokens: row.speculation.tokens ?? null } : null,
                 sameSetup: Boolean(row.sameSetup),
                 url: row.source || row.url || null,
@@ -8050,10 +8076,10 @@
 
         if (options.snapshot) setEngineEvidence(options.snapshot);
         const api = createApi();
-        api.version = "0.5.0";
+        api.version = "0.5.1";
         api.evidenceGeneratedAt = options.snapshot?.generatedAt || null;
         return api;
     }
 
-    return { createEngine, version: "0.5.0" };
+    return { createEngine, version: "0.5.1" };
 }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mlbottleneck/engine",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Physics-based LLM inference planner: decode/prefill tokens per second, memory fit, multi-GPU strategy, and speculative-decoding gains for any model on any hardware, calibrated on community benchmarks.",
   "keywords": [
     "llm",