ai-lcr 0.3.0 → 0.4.0

package/CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to `ai-lcr` are documented here. The format follows
 [Keep a Changelog](https://keepachangelog.com/), and the project adheres to
 [Semantic Versioning](https://semver.org/).
 
+## [0.4.0] — 2026-06-02
+
+All additions are optional and backward compatible.
+
+### Added
+
+- **`CallRecord.ttftMs` — time to first token.** Streaming calls now report TTFT,
+  the industry-standard responsiveness metric: ms from the winning provider's
+  stream attempt start to its first content token (`text-delta` /
+  `reasoning-delta`). Measured against the *winner's* attempt, so failover
+  overhead (already in `latencyMs`) doesn't distort it. `undefined` for
+  `doGenerate` (no streaming → no "first token") and for calls that failed before
+  producing content. `formatCallRecord` shows it inline next to total latency when
+  present (`412ms (ttft 88ms)`). With `latencyMs` and `outputTokens` on the same
+  record, output throughput is derivable: `outputTokens / ((latencyMs − ttftMs) /
+  1000)` tokens/sec.
+
 ## [0.3.0] — 2026-06-02
 
 Integration-feedback pass from wiring ai-lcr into a real agentic product

package/README.md

@@ -184,6 +184,7 @@ interface CallRecord {
   ok: boolean;
   failedOver: boolean;       // more than one provider was tried
   latencyMs: number;
+  ttftMs?: number;           // streaming only: time to first token (winner's first content delta) — industry-standard responsiveness metric
   inputTokens: number;
   outputTokens: number;
   cachedInputTokens?: number; // prompt-cache hits the winner read (when reported)
@@ -196,6 +197,8 @@ interface CallRecord {
 
 **Savings, not just spend.** Whenever at least one provider in a chain carries a `cost`, `baselineUsd` is what the same call would have cost on the most expensive priced leg (typically your safety-net fallback). `baselineUsd − costUsd` is the money routing saved on that call — the number a cost dashboard exists to show.
 
+**Responsiveness, not just total time.** On streaming calls (`streamText`, `streamObject`, streaming agents), `ttftMs` is the **time to first token** — measured from the winning provider's attempt start to its first content delta. It's the metric most LLM dashboards lead with, because it's what a user feels as "how fast did it start replying". Total `latencyMs` covers the whole stream including any failover; `ttftMs` isolates the serving model's responsiveness. It's `undefined` for `generateText`/`generateObject` (no streaming → no "first" token) and for calls that failed before any content. Output throughput (tokens/sec) is then `outputTokens / ((latencyMs − ttftMs) / 1000)`.
+
 **Cache-aware cost.** Add `cacheRead` (USD per 1M cached input tokens) to a provider's `cost` and ai-lcr bills prompt-cache hits at that rate when the call reports `usage.inputTokens.cacheRead`. Omit it and cached tokens fall back to the full `input` rate (unchanged from before). For cache-heavy traffic (e.g. Anthropic, where a cache read is ~0.1×) this keeps `costUsd` honest — and `cachedInputTokens` lets a dashboard audit it:
 
 ```ts
@@ -312,11 +315,11 @@ API_KEY=$KUNAVO_API_KEY BASE=https://api.kunavo.com \
   REF_API_KEY=$OPENROUTER_API_KEY REF_BASE=https://openrouter.ai/api \
   bash scripts/check-provider.sh
 
-# TokenMart uses vendor-prefixed model IDs
-API_KEY=$TOKENMART_API_KEY BASE=https://api.tokenmart.ai \
-  MODEL_1=google/gemini-3-flash    REF_1=google/gemini-3-flash-preview \
-  MODEL_2=anthropic/claude-sonnet-4-6 REF_2=anthropic/claude-sonnet-4.6 \
-  CACHE_MODEL=anthropic/claude-sonnet-4-6 \
+# TokenMart (Inference AI) uses bare, un-prefixed model IDs
+API_KEY=$INFERENCE_API_KEY BASE=https://model.service-inference.ai \
+  MODEL_1=gemini-3-flash-preview      REF_1=google/gemini-3-flash-preview \
+  MODEL_2=claude-sonnet-4-6           REF_2=anthropic/claude-sonnet-4.6 \
+  CACHE_MODEL=claude-sonnet-4-6 \
   REF_API_KEY=$OPENROUTER_API_KEY REF_BASE=https://openrouter.ai/api \
   bash scripts/check-provider.sh
 ```

package/README.zh-CN.md

@@ -232,11 +232,11 @@ API_KEY=$KUNAVO_API_KEY BASE=https://api.kunavo.com \
   REF_API_KEY=$OPENROUTER_API_KEY REF_BASE=https://openrouter.ai/api \
   bash scripts/check-provider.sh
 
-# TokenMart 使用 vendor 前缀的模型 ID
-API_KEY=$TOKENMART_API_KEY BASE=https://api.tokenmart.ai \
-  MODEL_1=google/gemini-3-flash    REF_1=google/gemini-3-flash-preview \
-  MODEL_2=anthropic/claude-sonnet-4-6 REF_2=anthropic/claude-sonnet-4.6 \
-  CACHE_MODEL=anthropic/claude-sonnet-4-6 \
+# TokenMart（Inference AI）使用不带 vendor 前缀的裸模型 ID
+API_KEY=$INFERENCE_API_KEY BASE=https://model.service-inference.ai \
+  MODEL_1=gemini-3-flash-preview      REF_1=google/gemini-3-flash-preview \
+  MODEL_2=claude-sonnet-4-6           REF_2=anthropic/claude-sonnet-4.6 \
+  CACHE_MODEL=claude-sonnet-4-6 \
   REF_API_KEY=$OPENROUTER_API_KEY REF_BASE=https://openrouter.ai/api \
   bash scripts/check-provider.sh
 ```

package/dist/index.cjs

@@ -312,7 +312,7 @@ var LcrFallbackModel = class {
     return max;
   }
   /** Winner settled: record the attempt, fire `onCost` (compat) + `onCall`. */
-  finalizeOk(ctx, provider, attemptStart, usage) {
+  finalizeOk(ctx, provider, attemptStart, usage, ttftMs) {
     ctx.attempts.push({ provider: provider.label, ok: true, latencyMs: Date.now() - attemptStart });
     const inputTokens = usage?.inputTokens?.total ?? 0;
     const outputTokens = usage?.outputTokens?.total ?? 0;
@@ -334,6 +334,7 @@ var LcrFallbackModel = class {
       ok: true,
       failedOver: ctx.attempts.length > 1,
       latencyMs: Date.now() - ctx.startedAt,
+      ...ttftMs !== void 0 ? { ttftMs } : {},
       inputTokens,
       outputTokens,
       ...cacheReadTokens > 0 ? { cachedInputTokens: cacheReadTokens } : {},
@@ -433,6 +434,7 @@ var LcrFallbackModel = class {
     const triedBeforeServing = tried;
     let usage;
     let streamedAny = false;
+    let ttftMs;
     const stream = new ReadableStream({
       async start(controller) {
         let reader = null;
@@ -446,11 +448,14 @@ var LcrFallbackModel = class {
             }
             if (done) break;
             if (value.type === "finish") usage = value.usage;
+            if (ttftMs === void 0 && (value.type === "text-delta" || value.type === "reasoning-delta")) {
+              ttftMs = Date.now() - servingAttemptStart;
+            }
             controller.enqueue(value);
             if (value.type !== "stream-start") streamedAny = true;
           }
           self.settleSticky(servingIdx);
-          self.finalizeOk(ctx, servingProvider, servingAttemptStart, usage);
+          self.finalizeOk(ctx, servingProvider, servingAttemptStart, usage, ttftMs);
           controller.close();
         } catch (error) {
           self.emitError(error, servingProvider.label);
@@ -512,7 +517,8 @@ function formatCallRecord(record, opts = {}) {
   const glyph = !record.ok ? "\u2717" : record.failedOver ? "\u26A0" : "\u2713";
   const chain = record.attempts.map((a) => a.provider).join("\u2192") || record.winner || "\u2014";
   const status = formatCost(record);
-  let line = `${glyph} ${record.model}  ${chain}  ${record.latencyMs}ms  ${status}`;
+  const timing = record.ttftMs !== void 0 ? `${record.latencyMs}ms (ttft ${record.ttftMs}ms)` : `${record.latencyMs}ms`;
+  let line = `${glyph} ${record.model}  ${chain}  ${timing}  ${status}`;
   if (record.ok && record.baselineUsd !== void 0 && record.baselineUsd > record.costUsd) {
     line += `  (saved $${(record.baselineUsd - record.costUsd).toFixed(4)})`;
   }

package/dist/index.d.cts

@@ -85,6 +85,18 @@ interface CallRecord {
     failedOver: boolean;
     /** Total wall time across all attempts, ms. */
     latencyMs: number;
+    /**
+     * Time to first token (TTFT), ms — the industry-standard responsiveness
+     * metric. Measured from the *winning* provider's stream attempt start to its
+     * first content token (`text-delta` / `reasoning-delta`), so it captures how
+     * fast the model that actually served started replying, not failover overhead
+     * (that's already in `latencyMs`). Streaming only: **undefined** for
+     * `doGenerate` (the whole response lands at once, so there's no "first token")
+     * and for calls that failed before producing any content. With `latencyMs` and
+     * `outputTokens`, output throughput is derivable: `outputTokens / ((latencyMs −
+     * ttftMs) / 1000)` tokens/sec.
+     */
+    ttftMs?: number;
     inputTokens: number;
     outputTokens: number;
     /**
@@ -141,6 +153,7 @@ declare function classifyErrorKind(error: unknown): ErrorKind;
  * latency, cost, and — when anything failed — the reason for each failed hop.
  *
  *   ✓ text  tokenmart                      412ms  $0.0003
+ *   ✓ text  tokenmart                      412ms (ttft 88ms)  $0.0003   ← streaming: TTFT shown when known
  *   ⚠ text  tokenmart→openrouter           910ms  $0.0004   ⤷ tokenmart 502
  *   ✗ text  deepseek→tokenmart→openrouter  1240ms FAILED    ⤷ deepseek 401, tokenmart 502, openrouter 429
  *

package/dist/index.d.ts

@@ -85,6 +85,18 @@ interface CallRecord {
     failedOver: boolean;
     /** Total wall time across all attempts, ms. */
     latencyMs: number;
+    /**
+     * Time to first token (TTFT), ms — the industry-standard responsiveness
+     * metric. Measured from the *winning* provider's stream attempt start to its
+     * first content token (`text-delta` / `reasoning-delta`), so it captures how
+     * fast the model that actually served started replying, not failover overhead
+     * (that's already in `latencyMs`). Streaming only: **undefined** for
+     * `doGenerate` (the whole response lands at once, so there's no "first token")
+     * and for calls that failed before producing any content. With `latencyMs` and
+     * `outputTokens`, output throughput is derivable: `outputTokens / ((latencyMs −
+     * ttftMs) / 1000)` tokens/sec.
+     */
+    ttftMs?: number;
     inputTokens: number;
     outputTokens: number;
     /**
@@ -141,6 +153,7 @@ declare function classifyErrorKind(error: unknown): ErrorKind;
  * latency, cost, and — when anything failed — the reason for each failed hop.
  *
  *   ✓ text  tokenmart                      412ms  $0.0003
+ *   ✓ text  tokenmart                      412ms (ttft 88ms)  $0.0003   ← streaming: TTFT shown when known
  *   ⚠ text  tokenmart→openrouter           910ms  $0.0004   ⤷ tokenmart 502
  *   ✗ text  deepseek→tokenmart→openrouter  1240ms FAILED    ⤷ deepseek 401, tokenmart 502, openrouter 429
  *

package/dist/index.js

@@ -271,7 +271,7 @@ var LcrFallbackModel = class {
     return max;
   }
   /** Winner settled: record the attempt, fire `onCost` (compat) + `onCall`. */
-  finalizeOk(ctx, provider, attemptStart, usage) {
+  finalizeOk(ctx, provider, attemptStart, usage, ttftMs) {
     ctx.attempts.push({ provider: provider.label, ok: true, latencyMs: Date.now() - attemptStart });
     const inputTokens = usage?.inputTokens?.total ?? 0;
     const outputTokens = usage?.outputTokens?.total ?? 0;
@@ -293,6 +293,7 @@ var LcrFallbackModel = class {
       ok: true,
       failedOver: ctx.attempts.length > 1,
       latencyMs: Date.now() - ctx.startedAt,
+      ...ttftMs !== void 0 ? { ttftMs } : {},
       inputTokens,
       outputTokens,
       ...cacheReadTokens > 0 ? { cachedInputTokens: cacheReadTokens } : {},
@@ -392,6 +393,7 @@ var LcrFallbackModel = class {
     const triedBeforeServing = tried;
     let usage;
     let streamedAny = false;
+    let ttftMs;
     const stream = new ReadableStream({
       async start(controller) {
         let reader = null;
@@ -405,11 +407,14 @@ var LcrFallbackModel = class {
             }
             if (done) break;
             if (value.type === "finish") usage = value.usage;
+            if (ttftMs === void 0 && (value.type === "text-delta" || value.type === "reasoning-delta")) {
+              ttftMs = Date.now() - servingAttemptStart;
+            }
             controller.enqueue(value);
             if (value.type !== "stream-start") streamedAny = true;
           }
           self.settleSticky(servingIdx);
-          self.finalizeOk(ctx, servingProvider, servingAttemptStart, usage);
+          self.finalizeOk(ctx, servingProvider, servingAttemptStart, usage, ttftMs);
           controller.close();
         } catch (error) {
           self.emitError(error, servingProvider.label);
@@ -471,7 +476,8 @@ function formatCallRecord(record, opts = {}) {
   const glyph = !record.ok ? "\u2717" : record.failedOver ? "\u26A0" : "\u2713";
   const chain = record.attempts.map((a) => a.provider).join("\u2192") || record.winner || "\u2014";
   const status = formatCost(record);
-  let line = `${glyph} ${record.model}  ${chain}  ${record.latencyMs}ms  ${status}`;
+  const timing = record.ttftMs !== void 0 ? `${record.latencyMs}ms (ttft ${record.ttftMs}ms)` : `${record.latencyMs}ms`;
+  let line = `${glyph} ${record.model}  ${chain}  ${timing}  ${status}`;
   if (record.ok && record.baselineUsd !== void 0 && record.baselineUsd > record.costUsd) {
     line += `  (saved $${(record.baselineUsd - record.costUsd).toFixed(4)})`;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-lcr",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Least Cost Routing for LLMs — route every model call to the cheapest available provider, fall back automatically, and track real cost. Built for the Vercel AI SDK.",
   "keywords": [
     "ai",