lumencode 0.4.3

package/lib/pricing-loader.js

@@ -0,0 +1,287 @@
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const DATA_DIR = join(__dirname, '..', 'data');
+const PRICING_FILE = join(DATA_DIR, 'pricing.json');
+const CACHE_FILE = join(DATA_DIR, 'pricing-cache.json');
+
+const PROVIDER_PREFIXES = ['anthropic--', 'bedrock--', 'vertex--'];
+
+// Portkey API 端点
+const PORTKEY_SINGLE_MODEL_URL = 'https://api.portkey.ai/model-configs/pricing';
+
+// 内存中的合并定价表
+let pricingTable = new Map();
+// 记录已尝试过 API 查询但失败的模型，避免重复请求
+const apiFailedModels = new Set();
+
+// 模型家族关键词 → 用于 fuzzy match 的子串匹配规则
+// 顺序：更具体的关键词放前面（如 gpt-4.1-mini 在 gpt-4.1 之前），避免被通用关键词截胡
+const FUZZY_KEYWORDS = [
+  // Claude
+  'opus', 'sonnet', 'haiku',
+  // OpenAI
+  'gpt-4.1-mini', 'gpt-4.1-nano', 'gpt-4.1', 'gpt-5', 'gpt-4o', 'gpt-4',
+  'o4-mini', 'o3-pro', 'o3', 'o1',
+  // Google
+  'gemini',
+  // 中国厂商
+  'glm', 'kimi', 'minimax', 'qwen', 'deepseek', 'doubao', 'baichuan',
+  'yi-', 'grok', 'llama',
+  // Mistral
+  'mistral', 'codestral',
+];
+
+// ── 内部工具 ──
+
+function ensureDataDir() {
+  if (!existsSync(DATA_DIR)) {
+    mkdirSync(DATA_DIR, { recursive: true });
+  }
+}
+
+function loadJsonFile(filePath) {
+  if (!existsSync(filePath)) return null;
+  try {
+    const raw = readFileSync(filePath, 'utf-8');
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+
+function saveJsonFile(filePath, data) {
+  ensureDataDir();
+  writeFileSync(filePath, JSON.stringify(data, null, 2) + '\n', 'utf-8');
+}
+
+function mergeIntoTable(source) {
+  if (!source || typeof source !== 'object') return;
+  for (const [key, value] of Object.entries(source)) {
+    if (key.startsWith('_')) continue;
+    // 允许两种形式：{ input, output, ... } 或 { aliasOf: "target-name" }
+    if (value && typeof value === 'object' && !value.unknown) {
+      pricingTable.set(key, value);
+    }
+  }
+}
+
+// 解析 aliasOf 链：若 entry 是 { aliasOf: "..." }，递归查找最终定价
+function resolveAlias(entry, depth = 0) {
+  if (!entry || depth >= 5) return entry;
+  if (entry.aliasOf && typeof entry.aliasOf === 'string') {
+    const target = pricingTable.get(entry.aliasOf);
+    if (target) return resolveAlias(target, depth + 1);
+    // 别名指向的目标不存在：尝试 fuzzy 查找一次（覆盖 Portkey 同步键）
+    const fuzzy = fuzzyLookup(entry.aliasOf.toLowerCase());
+    if (fuzzy) return resolveAlias(fuzzy, depth + 1);
+    return { unknown: true };
+  }
+  return entry;
+}
+
+// 在表中按 fuzzy 关键词查找匹配项
+function fuzzyLookup(modelLower) {
+  let matchedKeyword = null;
+  for (const kw of FUZZY_KEYWORDS) {
+    if (modelLower.includes(kw)) {
+      matchedKeyword = kw;
+      break;
+    }
+  }
+  if (!matchedKeyword) return null;
+
+  for (const [key, pricing] of pricingTable) {
+    const keyLower = key.toLowerCase();
+    if (!keyLower.includes(matchedKeyword)) continue;
+    // o3 不应匹配 o3-pro
+    if (matchedKeyword === 'o3' && keyLower.includes('pro')) continue;
+    // gpt-4.1 不应匹配 gpt-4.1-mini/nano
+    if (matchedKeyword === 'gpt-4.1' && (keyLower.includes('mini') || keyLower.includes('nano'))) continue;
+    return pricing;
+  }
+  return null;
+}
+
+function inferProvider(modelName) {
+  const lower = (modelName || '').toLowerCase();
+  if (lower.includes('claude')) return 'anthropic';
+  if (lower.includes('gemini') || lower.includes('palm')) return 'google';
+  if (lower.includes('gpt') || lower.includes('o1') || lower.includes('o3') || lower.includes('o4')) return 'openai';
+  if (lower.includes('deepseek')) return 'deepseek';
+  if (lower.includes('mistral')) return 'mistral-ai';
+  if (lower.includes('grok')) return 'xai';
+  return 'openai';
+}
+
+// Portkey 格式 (cents/token) → 内部格式 ($/1M tokens)
+function convertPortkeyPricing(portkeyData) {
+  if (!portkeyData || !portkeyData.pay_as_you_go) return null;
+
+  const payg = portkeyData.pay_as_you_go;
+  const getPrice = (field) => {
+    const val = payg[field];
+    if (val && typeof val.price === 'number') {
+      return val.price * 10000;
+    }
+    return 0;
+  };
+
+  const result = {
+    input: getPrice('request_token'),
+    output: getPrice('response_token'),
+    cacheRead: getPrice('cache_read_input_token'),
+    cacheCreate: getPrice('cache_write_input_token'),
+  };
+
+  if (result.input === 0 && result.output === 0) return null;
+  return result;
+}
+
+// ── 公共接口 ──
+
+/**
+ * 同步初始化定价模块。
+ * 加载顺序（后加载的覆盖先加载的）：
+ *   1. data/pricing.json 中的 models（Portkey 批量同步的数据）
+ *   2. data/pricing-cache.json（运行时 API 查询缓存）
+ *   3. data/pricing.json 中的 overrides（权威覆盖 + 别名映射）
+ */
+export function initPricing() {
+  pricingTable.clear();
+  apiFailedModels.clear();
+
+  const pricingData = loadJsonFile(PRICING_FILE);
+  if (pricingData && pricingData.models) {
+    mergeIntoTable(pricingData.models);
+  }
+
+  const cacheData = loadJsonFile(CACHE_FILE);
+  if (cacheData && cacheData.models) {
+    mergeIntoTable(cacheData.models);
+  }
+
+  // overrides 最后加载，优先级最高
+  if (pricingData && pricingData.overrides) {
+    mergeIntoTable(pricingData.overrides);
+  }
+}
+
+/**
+ * 解析模型定价。
+ * 四层回退：精确匹配 → 去 provider 前缀 → 家族关键词 fuzzy match → unknown。
+ * 支持 aliasOf 别名映射（在 overrides 中可用 { aliasOf: "target" } 形式）。
+ */
+export function resolveModelPricing(model) {
+  if (!model) return { unknown: true };
+
+  if (pricingTable.size === 0) {
+    initPricing();
+  }
+
+  // Tier 1: 精确匹配
+  if (pricingTable.has(model)) {
+    return resolveAlias(pricingTable.get(model));
+  }
+
+  // Tier 2: 去掉 provider 前缀
+  let stripped = model;
+  for (const prefix of PROVIDER_PREFIXES) {
+    if (model.startsWith(prefix)) {
+      stripped = model.slice(prefix.length);
+      break;
+    }
+  }
+  if (pricingTable.has(stripped)) {
+    return resolveAlias(pricingTable.get(stripped));
+  }
+
+  // Tier 3: 按家族关键词 fuzzy 匹配
+  const lower = stripped.toLowerCase();
+  const fuzzy = fuzzyLookup(lower);
+  if (fuzzy) return resolveAlias(fuzzy);
+
+  return { unknown: true };
+}
+
+function getModel(record) {
+  return record.model || record.metadata?.model || '';
+}
+
+/**
+ * 异步预加载所有未知模型的定价。
+ * 扫描 records，对本地没有的模型调用 Portkey API。
+ * - 成功：写入 pricing-cache.json，下次启动自动加载
+ * - 失败：标记为已失败，本次会话不再重试；该模型定价为 unknown（不计算费用）
+ */
+export async function preloadUnknownPricing(records) {
+  if (!records || records.length === 0) return;
+
+  const unknownModels = new Set();
+  for (const r of records) {
+    const model = getModel(r);
+    if (!model || apiFailedModels.has(model)) continue;
+    const pricing = resolveModelPricing(model);
+    if (pricing.unknown) {
+      unknownModels.add(model);
+    }
+  }
+
+  if (unknownModels.size === 0) return;
+
+  const CONCURRENCY = 5;
+  const models = [...unknownModels];
+  const newCache = {};
+
+  for (let i = 0; i < models.length; i += CONCURRENCY) {
+    const batch = models.slice(i, i + CONCURRENCY);
+    const results = await Promise.all(
+      batch.map(async (model) => {
+        try {
+          const provider = inferProvider(model);
+          const url = `${PORTKEY_SINGLE_MODEL_URL}/${provider}/${encodeURIComponent(model)}`;
+          const controller = new AbortController();
+          const timeout = setTimeout(() => controller.abort(), 5000);
+
+          const res = await fetch(url, { signal: controller.signal });
+          clearTimeout(timeout);
+
+          if (!res.ok) {
+            apiFailedModels.add(model);
+            return null;
+          }
+
+          const data = await res.json();
+          const pricing = convertPortkeyPricing(data);
+          if (pricing) {
+            return { model, pricing };
+          }
+          apiFailedModels.add(model);
+        } catch {
+          apiFailedModels.add(model);
+        }
+        return null;
+      })
+    );
+
+    for (const result of results) {
+      if (result) {
+        pricingTable.set(result.model, result.pricing);
+        newCache[result.model] = result.pricing;
+      }
+    }
+  }
+
+  if (Object.keys(newCache).length > 0) {
+    const existing = loadJsonFile(CACHE_FILE) || { _meta: { source: 'portkey-api-cache' }, models: {} };
+    existing.models = { ...existing.models, ...newCache };
+    existing._meta.updatedAt = new Date().toISOString();
+    saveJsonFile(CACHE_FILE, existing);
+  }
+}
+
+export function getLoadedModelCount() {
+  return pricingTable.size;
+}

package/lib/record-utils.js

@@ -0,0 +1,35 @@
+// UsageRecord 兼容辅助函数
+// 统一处理新格式（inputTokens/outputTokens）和旧格式（tokens.input/tokens.output）
+
+export function getInputTokens(r) {
+  if (r.inputTokens !== undefined) return r.inputTokens;
+  return r.tokens?.input || 0;
+}
+
+export function getOutputTokens(r) {
+  if (r.outputTokens !== undefined) return r.outputTokens;
+  return r.tokens?.output || 0;
+}
+
+export function getCacheRead(r) {
+  if (r.cacheReadTokens !== undefined) return r.cacheReadTokens;
+  return r.tokens?.cacheRead || 0;
+}
+
+export function getCacheCreate(r) {
+  if (r.cacheWriteTokens !== undefined) return r.cacheWriteTokens;
+  return r.tokens?.cacheCreate || 0;
+}
+
+export function getModel(r) {
+  return r.model || '';
+}
+
+export function isAssistantRecord(r) {
+  if (r.metadata?.type === 'assistant') return true;
+  if (r.metadata?.type === 'user') return false;
+  if (r.tool === 'codex') return true;
+  if (r.tool === 'opencode' && r.metadata?.role !== 'user') return true;
+  if (r.type === 'assistant' && !r.tool) return true;
+  return false;
+}