@aria-framework/ai 0.10.0 → 0.11.0

package/benchmark.js

@@ -0,0 +1,172 @@
+/**
+ * A fixed-workload speed test for one endpoint.
+ *
+ * WHY NOT JUST TIME A CALL. Timing an ordinary request measures the model's verbosity as much as
+ * its speed: ask two models the same question and the wordier one looks slower. The workload here
+ * cannot be finished early — counting to a thousand always exceeds the reply budget — so every run
+ * ends at `finish_reason: length` and what is measured is generation rate, not how talkative the
+ * model felt.
+ *
+ * A FIRST ATTEMPT ASKED IT TO "continue the sequence" AND THE MODEL STOPPED AFTER 21 TOKENS. An
+ * open-ended instruction is an invitation to stop; a destination it cannot reach is not.
+ *
+ * WHY A DURATION BUDGET RATHER THAN A TOKEN COUNT. A fixed 256 tokens took 33 seconds on a local 9B
+ * — four of those would block a button press for over two minutes and flirt with the per-call
+ * timeout. Fixing the token count was meant to make runs comparable, but a RATE is already
+ * normalised: measuring 39 tokens over five seconds and 512 tokens over five seconds both yield
+ * tokens per second. So each run is given roughly TARGET_MS of work, bounded at both ends, and the
+ * budget actually used is reported. Measured on the same endpoint, the two approaches agreed to
+ * within 1% (7.6 tok/s over 21 tokens, 7.7 over 256).
+ *
+ * WHY THE WARM-UP IS REPORTED RATHER THAN DISCARDED. LM Studio unloads the previous model when a new
+ * one loads and Ollama unloads on idle, so the first call can pay a multi-second load — 28 seconds
+ * was observed on a 22 GB model. That number decides whether an endpoint can serve anything
+ * interactive, and it is invisible in a warm average. It is the point of the first call, not a
+ * sample to throw away. It doubles as the rate estimate that sizes the runs that follow.
+ *
+ * WHY THREE RUNS. One sample on a GPU shared with anything else is noise. The median of three
+ * survives a neighbour's job starting mid-test.
+ *
+ * REASONING IS COUNTED, NOT EXCLUDED. A thinking model spends completion tokens before it writes
+ * anything, and those are real work at a real rate — but they are not answer, so a tok/s that
+ * silently includes them is not comparable with a model that does not think. Both facts are
+ * reported and the caller is told which is which.
+ */
+
+'use strict';
+
+/**
+ * A workload with a destination the model cannot reach inside the budget.
+ *
+ * Counting is the least ambiguous instruction across model families — no judgement, no refusal
+ * risk, nothing to be clever about — so what is measured is generation rate rather than how a
+ * particular model feels about the prompt.
+ */
+const WORKLOAD = {
+  system: 'Count upward. Output one number per line and nothing else.',
+  user: 'Count from 1 to 1000.'
+};
+
+/** Roughly how long each measured run should generate for. Long enough to average out jitter,
+ *  short enough that pressing a button does not feel like it has hung. */
+const TARGET_MS = 4000;
+
+/** The warm-up's budget: enough to pay the model load and estimate a rate, cheap if it fails. */
+const PROBE_TOKENS = 64;
+
+/** Bounds on the per-run budget, so a very fast endpoint cannot run up a bill and a very slow one
+ *  still produces a usable sample. */
+const MIN_TOKENS = 48;
+const MAX_TOKENS = 320;
+
+/** Middle value, or the mean of the two middle ones. Not the average: one stall would drag it. */
+function median(values) {
+  if (!values.length) return null;
+  const v = values.slice().sort((a, b) => a - b);
+  const mid = Math.floor(v.length / 2);
+  return v.length % 2 ? v[mid] : Math.round(((v[mid - 1] + v[mid]) / 2) * 10) / 10;
+}
+
+const rate = (tokens, ms) => (ms > 0 ? Math.round((tokens / (ms / 1000)) * 10) / 10 : null);
+
+/**
+ * Run one measured call.
+ *
+ * NEVER THROWS. A model that spends its whole reply budget on internal reasoning makes complete()
+ * throw with no answer to return — but it still generated tokens, at a rate worth knowing. When the
+ * error carries usage that is a measurement; when it does not, it is a failed run and says so.
+ */
+async function once(complete, cfg, maxTokens) {
+  const started = Date.now();
+  try {
+    const r = await complete({
+      system: WORKLOAD.system,
+      messages: [{ role: 'user', content: WORKLOAD.user }],
+      maxTokens,
+      temperature: 0
+    }, cfg);
+    const ms = r.ms || (Date.now() - started);
+    const tokens = (r.usage && r.usage.completion) || 0;
+    return {
+      ms, tokens, maxTokens, tokensPerSec: rate(tokens, ms),
+      reasonedChars: r.reasonedFor || 0, reasoningOnly: false, error: null
+    };
+  } catch (err) {
+    const ms = Date.now() - started;
+    const tokens = (err.usage && (err.usage.completion_tokens || err.usage.completion)) || 0;
+    if (!tokens) {
+      return { ms, tokens: 0, maxTokens, tokensPerSec: null, reasonedChars: 0, reasoningOnly: false, error: err.message };
+    }
+    // Tokens were produced; they simply were not an answer. That is still throughput.
+    return {
+      ms, tokens, maxTokens, tokensPerSec: rate(tokens, ms),
+      reasonedChars: 0, reasoningOnly: true, error: null
+    };
+  }
+}
+
+/**
+ * Measure one endpoint.
+ *
+ * @param {Function} complete  the client's complete(opts, cfg)
+ * @param {object}   cfg       a resolved endpoint config
+ * @param {{runs?: number, targetMs?: number}} [opts]
+ */
+async function benchmark(complete, cfg, opts = {}) {
+  const runs = Math.max(1, Math.min(10, Number(opts.runs) || 3));
+  const targetMs = Math.max(500, Math.min(30000, Number(opts.targetMs) || TARGET_MS));
+
+  if (!cfg || !cfg.enabled) {
+    return { ok: false, error: 'This endpoint is not enabled, so it was not asked to do anything.' };
+  }
+
+  // THE FIRST CALL IS THE COLD ONE. Its duration answers "can this serve a person who is waiting",
+  // which no later run can give once the model is resident.
+  const cold = await once(complete, cfg, PROBE_TOKENS);
+  if (cold.error) {
+    return {
+      ok: false, model: cfg.model || null, coldMs: cold.ms, runs: [], medianTokensPerSec: null,
+      reasoning: false, reasoningOnly: false, totalTokens: cold.tokens, runTokens: null,
+      error: cold.error
+    };
+  }
+
+  // A SECOND SMALL CALL, NOW WARM, PURELY TO SIZE THE RUNS.
+  //
+  // Sizing from the COLD call starves the measurement on exactly the endpoints that most need a
+  // decent sample: a 35B model that spent 26 seconds loading and then produced 64 tokens looks like
+  // 2.4 tok/s, so its runs were budgeted at the floor — when warm it actually managed 66.8, and
+  // four seconds of its work is closer to 270 tokens. The load is not the rate, and using it as one
+  // makes the fast endpoints look like the slow ones.
+  const pace = await once(complete, cfg, PROBE_TOKENS);
+  const estimated = pace.tokensPerSec || cold.tokensPerSec || 20;
+  const runTokens = Math.max(MIN_TOKENS, Math.min(MAX_TOKENS, Math.round((estimated * targetMs) / 1000)));
+
+  const measured = [];
+  for (let i = 0; i < runs; i += 1) {
+    measured.push(await once(complete, cfg, runTokens));
+  }
+
+  const good = measured.filter((r) => r.tokensPerSec != null);
+  const failed = measured.filter((r) => r.error);
+
+  return {
+    ok: good.length > 0,
+    model: cfg.model || null,
+    // Reported separately and never folded into the median: mixing a cold call into a warm average
+    // is how an endpoint that takes half a minute to wake looks merely mediocre.
+    coldMs: cold.ms,
+    runs: measured,
+    runTokens,
+    medianTokensPerSec: median(good.map((r) => r.tokensPerSec)),
+    // TRUE when any run thought before answering. The rate above still counts those tokens — they
+    // were generated — but it is not comparable with a model that does not think, and the caller
+    // needs to be able to say so rather than presenting one number as though it were the same.
+    reasoning: measured.some((r) => r.reasonedChars > 0 || r.reasoningOnly),
+    reasoningOnly: good.length > 0 && good.every((r) => r.reasoningOnly),
+    totalTokens: cold.tokens + pace.tokens + measured.reduce((n, r) => n + r.tokens, 0),
+    error: good.length ? null : (failed[0] && failed[0].error) || 'No run produced any tokens.'
+  };
+}
+
+module.exports = { benchmark, WORKLOAD, TARGET_MS, MIN_TOKENS, MAX_TOKENS };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@aria-framework/ai",
   "description": "Aria App Framework — AI module. A dependency-injected model seam (createAiClient) over several providers (LM Studio / OpenAI-compatible / Anthropic), with a fact-preservation guard, generic Polish and Generate writing engines, and a browser polish widget. Prompts and config stay in the consuming app.",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "license": "UNLICENSED",
   "private": false,
   "publishConfig": {
@@ -16,7 +16,8 @@
     "generate.js",
     "providers/openai-compatible.js",
     "providers/anthropic.js",
-    "browser/ai-polish.js", "usageStore.js", "providerStore.js", "health.js", "views/"
+    "browser/ai-polish.js", "usageStore.js", "providerStore.js", "health.js",
+    "benchmark.js", "views/"
   ],
   "peerDependencies": {
     "@aria-framework/db-worker": ">=0.7.0"
@@ -25,6 +26,6 @@
     "@aria-framework/db-worker": { "optional": true }
   },
   "scripts": {
-    "test": "node test/smoke.js && node test/usageStore.js && node test/providerStore.js && node test/health.js && node test/listModels.js && node test/benchmark.js && node test/views.js"
+    "test": "node test/smoke.js && node test/usageStore.js && node test/providerStore.js && node test/health.js && node test/listModels.js && node test/benchmark.js && node test/packaging.js && node test/views.js"
   }
 }

package/providerStore.js

@@ -34,10 +34,18 @@
 /** Columns the caller may set. `id` is separate — it is the key and is never updated in place. */
 const FIELDS = [
   'label', 'kind', 'base_url', 'model', 'embedding_model',
-  'context_tokens', 'max_tokens', 'timeout_ms', 'daily_token_cap', 'enabled', 'sort_order'
+  'context_tokens', 'max_tokens', 'timeout_ms', 'daily_token_cap', 'enabled', 'sort_order',
+  // WHAT THIS ENDPOINT IS FOR, expressed as a floor. A registry describes tiers — a triage endpoint
+  // has to be fast and a deep-dive one does not — and without a number recorded per endpoint, every
+  // speed test means recalling what the figure used to be. 0 means "no expectation".
+  //
+  // An app that has not added the column is unaffected: clean() skips anything undefined, so the
+  // field is only ever written by a caller that knows about it.
+  'min_tokens_per_sec'
 ];
 
-const NUMERIC = new Set(['context_tokens', 'max_tokens', 'timeout_ms', 'daily_token_cap', 'enabled', 'sort_order']);
+const NUMERIC = new Set(['context_tokens', 'max_tokens', 'timeout_ms', 'daily_token_cap', 'enabled',
+  'sort_order', 'min_tokens_per_sec']);
 
 /** An id an operator typed, constrained so it can appear in a URL and a log line unescaped. */
 function assertId(id) {
@@ -163,7 +171,10 @@ function createProviderStore(opts = {}) {
         timeoutMs: Number(row.timeout_ms) || 60000,
         maxTokens: Number(row.max_tokens) || 1024,
         contextTokens: Number(row.context_tokens) || 8192,
-        dailyTokenCap: Number(row.daily_token_cap) || 0
+        dailyTokenCap: Number(row.daily_token_cap) || 0,
+        // Carried on the resolved config so a speed test can judge a result without a second read.
+        // 0 means no expectation was recorded, which is different from "expected to be slow".
+        minTokensPerSec: Number(row.min_tokens_per_sec) || 0
       };
     }
   };
@@ -193,6 +204,10 @@ function schemaFor(dialect) {
   daily_token_cap   INTEGER NOT NULL DEFAULT 0,
   enabled           INTEGER NOT NULL DEFAULT 1,
   sort_order        INTEGER NOT NULL DEFAULT 0,
+  -- The floor this endpoint is expected to clear, in generated tokens per second, so a speed test
+  -- can report a verdict rather than a number to be remembered. 0 means no expectation was set,
+  -- which is not the same as expecting it to be slow.
+  min_tokens_per_sec INTEGER NOT NULL DEFAULT 0,
   created_at        TEXT    NOT NULL DEFAULT (${t.now()})
 `;
 }