@outputai/llm 0.12.1-next.39bbc03.0 → 0.12.1-next.4abbd65.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/llm",
-  "version": "0.12.1-next.39bbc03.0",
+  "version": "0.12.1-next.4abbd65.0",
   "description": "Framework abstraction to interact with LLM models",
   "type": "module",
   "main": "src/index.js",
@@ -13,7 +13,8 @@
     "./src/index.d.ts",
     "./src/prompt/!(*.spec).js",
     "./src/prompt/markup/!(*.spec).js",
-    "./src/utils/!(*.spec).js"
+    "./src/utils/!(*.spec).js",
+    "./src/utils/*.json"
   ],
   "dependencies": {
     "decimal.js": "10.6.0",
@@ -21,7 +22,7 @@
     "gray-matter": "4.0.3",
     "liquidjs": "10.27.2",
     "undici": "8.9.0",
-    "@outputai/core": "0.12.1-next.39bbc03.0"
+    "@outputai/core": "0.12.1-next.4abbd65.0"
   },
   "devDependencies": {
     "@ai-sdk/amazon-bedrock": "5.0.57",
@@ -46,6 +47,7 @@
     "access": "public"
   },
   "scripts": {
-    "build": "pnpm exec tsc -p ./tsconfig.typecheck.json"
+    "build": "pnpm exec tsc -p ./tsconfig.typecheck.json",
+    "take-models-pricing-snapshot": "node ./scripts/take_models_pricing_snapshot.js"
   }
 }

package/src/index.d.ts

@@ -383,6 +383,7 @@ export interface LLMUsageEvent {
 
 export type LLMGenerationCostItemStatus = 'ok' | 'fallback' | 'missing';
 export type LLMGenerationCostStatus = 'precise' | 'imprecise' | 'incomplete';
+export type LLMGenerationCostPricingFreshness = 'live' | 'cached' | 'stale' | 'snapshot';
 
 /** Cost calculated for one normalized LLM usage item. */
 export interface LLMGenerationCostItem {
@@ -404,6 +405,8 @@ export interface LLMGenerationCost extends BaseAttribute {
   request: number | null;
   total: number | null;
   status: LLMGenerationCostStatus;
+  /** How current the rate table was. Absent on costs recorded before this field existed. */
+  pricingFreshness?: LLMGenerationCostPricingFreshness | null;
   items: LLMGenerationCostItem[];
 }
 

package/src/utils/cost.js

@@ -1,4 +1,4 @@
-import { fetchModelsPricing } from './models_pricing.js';
+import { fetchModelsPricing, Freshness } from './models_pricing.js';
 import { Tracing } from '@outputai/core/sdk/runtime';
 import { LLMGenerationUsage, LLMGenerationUsageItem } from './usage.js';
 import { GroundingPpmMap } from './grounding.js';
@@ -45,19 +45,21 @@ export class LLMGenerationCost extends Tracing.Attribute.BaseAttribute {
   request = null;
   total = null;
   status = LLMGenerationCost.Status.INCOMPLETE;
+  pricingFreshness = null;
   items = [];
 
-  constructor( modelId, providerId, items, usageStatus ) {
+  constructor( modelId, providerId, items, usageStatus, pricingFreshness ) {
     super( LLMGenerationCost.TYPE );
     this.modelId = modelId;
     this.providerId = providerId;
     this.items = items;
+    this.pricingFreshness = pricingFreshness;
 
     const meaningfulItems = items.filter( v => v.amount > 0 );
 
     if ( meaningfulItems.some( v => v.status === LLMGenerationCostItem.Status.MISSING ) || usageStatus === LLMGenerationUsage.Status.INCOMPLETE ) {
       this.status = LLMGenerationCost.Status.INCOMPLETE;
-    } else if ( meaningfulItems.some( v => v.status === LLMGenerationCostItem.Status.FALLBACK ) ) {
+    } else if ( meaningfulItems.some( v => v.status === LLMGenerationCostItem.Status.FALLBACK ) || pricingFreshness === Freshness.SNAPSHOT ) {
       this.status = LLMGenerationCost.Status.IMPRECISE;
     } else {
       this.status = LLMGenerationCost.Status.PRECISE;
@@ -118,7 +120,7 @@ const resolvePrice = ( { group, label, pricing } ) => {
  * @returns {Promise<LLMGenerationCost | null>} LLM generation cost with input, output, total and breakdown
  */
 export const calculateCosts = async usage => {
-  const models = await fetchModelsPricing();
+  const { models, freshness: pricingFreshness } = await fetchModelsPricing();
 
   if ( !models ) {
     Logger.warn( 'Failed to fetch models pricing', { namespace: 'LLM' } );
@@ -144,5 +146,5 @@ export const calculateCosts = async usage => {
     Logger.warn( 'Grounded call with no grounding rate for model', { namespace: 'LLM', modelId, providerId } );
   }
 
-  return new LLMGenerationCost( modelId, providerId, items, usage.status );
+  return new LLMGenerationCost( modelId, providerId, items, usage.status, pricingFreshness );
 };

package/src/utils/models_pricing.js

@@ -1,9 +1,11 @@
 import { Logger } from '@outputai/core';
 import { EnvHttpProxyAgent, fetch } from 'undici';
+import modelsPricingSnapshot from './models_pricing_snapshot.json' with { type: 'json' };
 
 const logger = Logger.createLogger( 'LLM' );
 const costTableUrl = 'https://models.dev/api.json';
 const cacheTTL = 1000 * 60 * 60 * 24; // 1 day
+const cooldownTTL = 1000 * 60 * 10; // 10 minutes
 
 /* Ignore HTTP/2. Check: https://github.com/growthxai/output/issues/299 */
 const dispatcher = new EnvHttpProxyAgent( { allowH2: false } );
@@ -13,23 +15,42 @@ export const cache = {
   expiresAt: 0
 };
 
+export const Freshness = {
+  LIVE: 'live',
+  CACHED: 'cached',
+  STALE: 'stale',
+  SNAPSHOT: 'snapshot'
+};
+
+export const state = {
+  ignoreLiveRequestsUntil: 0
+};
+
 const parseData = data => {
   const map = new Map();
   try {
-    for ( const provider of Object.values( data ) ) {
+    for ( const [ providerId, provider ] of Object.entries( data ) ) {
+      if ( providerId === '_meta' ) {
+        continue;
+      }
       for ( const [ modelName, { cost } ] of Object.entries( provider.models ?? {} ) ) {
         if ( cost ) { // some models don't have cost
-          map.set( `${provider.id}/${modelName}`, cost );
+          map.set( `${providerId}/${modelName}`, cost );
         }
       }
     }
+    if ( map.size === 0 ) {
+      throw new Error( 'Empty response' );
+    }
     return map;
   } catch ( error ) {
-    logger.error( `Models pricing: Data parsing failure "${error.name}".` );
+    logger.error( `Models pricing: Data parsing failure "${error.message}".` );
     return null;
   }
 };
 
+const fallbackTable = parseData( modelsPricingSnapshot );
+
 const fetchData = async () => {
   try {
     const res = await fetch( costTableUrl, { dispatcher } );
@@ -47,22 +68,26 @@ const fetchData = async () => {
 
 export const fetchModelsPricing = async () => {
   if ( cache.content && cache.expiresAt > Date.now() ) {
-    return cache.content;
+    return { models: cache.content, freshness: Freshness.CACHED };
   }
 
-  const table = await fetchData();
-  const content = table ? parseData( table ) : null;
+  if ( state.ignoreLiveRequestsUntil < Date.now() ) {
+    const table = await fetchData();
+    const models = table ? parseData( table ) : null;
 
-  if ( content ) {
-    cache.content = content;
-    cache.expiresAt = Date.now() + cacheTTL;
-    return content;
+    if ( models ) {
+      cache.content = models;
+      cache.expiresAt = Date.now() + cacheTTL;
+      return { models, freshness: Freshness.LIVE };
+    } else {
+      state.ignoreLiveRequestsUntil = Date.now() + cooldownTTL;
+    }
   }
 
   if ( cache.content ) {
     logger.warn( 'Models pricing: using stale cache.' );
-    return cache.content;
+    return { models: cache.content, freshness: Freshness.STALE };
   }
-
-  return null;
+  logger.warn( 'Models pricing: using built-in fallback.' );
+  return { models: fallbackTable, freshness: Freshness.SNAPSHOT };
 };