@tokscale/cli 1.0.17 → 1.0.19

package/src/pricing.ts

@@ -1,309 +0,0 @@
-/**
- * Pricing data fetcher using LiteLLM as source
- * Features disk caching with 1-hour TTL
- */
-
-import * as fs from "node:fs";
-import * as path from "node:path";
-import * as os from "node:os";
-
-function escapeRegex(str: string): string {
-  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-
-export function normalizeModelName(modelId: string): string | null {
-  const lower = modelId.toLowerCase();
-
-  if (lower.includes("opus")) {
-    if (lower.includes("4.5") || lower.includes("4-5")) {
-      return "opus-4-5";
-    } else if (lower.includes("4")) {
-      return "opus-4";
-    }
-  }
-  if (lower.includes("sonnet")) {
-    if (lower.includes("4.5") || lower.includes("4-5")) {
-      return "sonnet-4-5";
-    } else if (lower.includes("4")) {
-      return "sonnet-4";
-    } else if (lower.includes("3.7") || lower.includes("3-7")) {
-      return "sonnet-3-7";
-    } else if (lower.includes("3.5") || lower.includes("3-5")) {
-      return "sonnet-3-5";
-    }
-  }
-  if (lower.includes("haiku") && (lower.includes("4.5") || lower.includes("4-5"))) {
-    return "haiku-4-5";
-  }
-
-  if (lower === "o3") {
-    return "o3";
-  }
-  if (lower.startsWith("gpt-4o") || lower === "gpt-4o") {
-    return "gpt-4o";
-  }
-  if (lower.startsWith("gpt-4.1") || lower.includes("gpt-4.1")) {
-    return "gpt-4.1";
-  }
-
-  if (lower.includes("gemini-2.5-pro")) {
-    return "gemini-2.5-pro";
-  }
-  if (lower.includes("gemini-2.5-flash")) {
-    return "gemini-2.5-flash";
-  }
-
-  return null;
-}
-
-export function isWordBoundaryMatch(haystack: string, needle: string): boolean {
-  const pos = haystack.indexOf(needle);
-  if (pos === -1) return false;
-
-  const beforeOk = pos === 0 || !/[a-zA-Z0-9]/.test(haystack[pos - 1]);
-  const afterOk =
-    pos + needle.length === haystack.length ||
-    !/[a-zA-Z0-9]/.test(haystack[pos + needle.length]);
-
-  return beforeOk && afterOk;
-}
-
-const LITELLM_PRICING_URL =
-  "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
-
-const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
-
-export interface LiteLLMModelPricing {
-  input_cost_per_token?: number;
-  output_cost_per_token?: number;
-  cache_creation_input_token_cost?: number;
-  cache_read_input_token_cost?: number;
-  input_cost_per_token_above_200k_tokens?: number;
-  output_cost_per_token_above_200k_tokens?: number;
-  cache_creation_input_token_cost_above_200k_tokens?: number;
-  cache_read_input_token_cost_above_200k_tokens?: number;
-}
-
-export type PricingDataset = Record<string, LiteLLMModelPricing>;
-
-interface CachedPricing {
-  timestamp: number;
-  data: PricingDataset;
-}
-
-/**
- * Format for passing pricing to Rust native module
- * Note: napi-rs expects undefined (not null) for Rust Option<T> fields
- */
-export interface PricingEntry {
-  modelId: string;
-  pricing: {
-    inputCostPerToken: number;
-    outputCostPerToken: number;
-    cacheReadInputTokenCost?: number;
-    cacheCreationInputTokenCost?: number;
-  };
-}
-
-function getCacheDir(): string {
-  const cacheHome = process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache");
-  return path.join(cacheHome, "tokscale");
-}
-
-function getCachePath(): string {
-  return path.join(getCacheDir(), "pricing.json");
-}
-
-function loadCachedPricing(): CachedPricing | null {
-  try {
-    const cachePath = getCachePath();
-    if (!fs.existsSync(cachePath)) {
-      return null;
-    }
-
-    const content = fs.readFileSync(cachePath, "utf-8");
-    const cached = JSON.parse(content) as CachedPricing;
-
-    // Check TTL
-    const age = Date.now() - cached.timestamp;
-    if (age > CACHE_TTL_MS) {
-      return null; // Cache expired
-    }
-
-    return cached;
-  } catch {
-    return null;
-  }
-}
-
-function saveCachedPricing(data: PricingDataset): void {
-  try {
-    const cacheDir = getCacheDir();
-    if (!fs.existsSync(cacheDir)) {
-      fs.mkdirSync(cacheDir, { recursive: true });
-    }
-
-    const cached: CachedPricing = {
-      timestamp: Date.now(),
-      data,
-    };
-
-    fs.writeFileSync(getCachePath(), JSON.stringify(cached), "utf-8");
-  } catch {
-    // Ignore cache write errors
-  }
-}
-
-export class PricingFetcher {
-  private pricingData: PricingDataset | null = null;
-
-  /**
-   * Fetch pricing data (with disk cache, 1-hour TTL)
-   */
-  async fetchPricing(): Promise<PricingDataset> {
-    if (this.pricingData) return this.pricingData;
-
-    // Try to load from cache first
-    const cached = loadCachedPricing();
-    if (cached) {
-      this.pricingData = cached.data;
-      return this.pricingData;
-    }
-
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), 15000);
-    
-    let response: Response;
-    try {
-      response = await fetch(LITELLM_PRICING_URL, { signal: controller.signal });
-    } finally {
-      clearTimeout(timeoutId);
-    }
-    
-    if (!response.ok) {
-      throw new Error(`Failed to fetch pricing: ${response.status}`);
-    }
-
-    this.pricingData = (await response.json()) as PricingDataset;
-
-    // Save to cache
-    saveCachedPricing(this.pricingData);
-
-    return this.pricingData;
-  }
-
-  /**
-   * Get raw pricing dataset
-   */
-  getPricingData(): PricingDataset | null {
-    return this.pricingData;
-  }
-
-  /**
-   * Convert pricing data to format expected by Rust native module
-   */
-  toPricingEntries(): PricingEntry[] {
-    if (!this.pricingData) return [];
-
-    return Object.entries(this.pricingData).map(([modelId, pricing]) => ({
-      modelId,
-      pricing: {
-        inputCostPerToken: pricing.input_cost_per_token ?? 0,
-        outputCostPerToken: pricing.output_cost_per_token ?? 0,
-        // napi-rs expects undefined (not null) for Option<T> fields
-        cacheReadInputTokenCost: pricing.cache_read_input_token_cost,
-        cacheCreationInputTokenCost: pricing.cache_creation_input_token_cost,
-      },
-    }));
-  }
-
-  getModelPricing(modelID: string): LiteLLMModelPricing | null {
-    if (!this.pricingData) return null;
-
-    // Direct lookup
-    if (this.pricingData[modelID]) {
-      return this.pricingData[modelID];
-    }
-
-    // Try with provider prefix
-    const prefixes = ["anthropic/", "openai/", "google/", "bedrock/"];
-    for (const prefix of prefixes) {
-      if (this.pricingData[prefix + modelID]) {
-        return this.pricingData[prefix + modelID];
-      }
-    }
-
-    const normalized = normalizeModelName(modelID);
-    if (normalized) {
-      if (this.pricingData[normalized]) {
-        return this.pricingData[normalized];
-      }
-      for (const prefix of prefixes) {
-        if (this.pricingData[prefix + normalized]) {
-          return this.pricingData[prefix + normalized];
-        }
-      }
-    }
-
-    const lowerModelID = modelID.toLowerCase();
-    const lowerNormalized = normalized?.toLowerCase();
-    const sortedKeys = Object.keys(this.pricingData).sort();
-
-    for (const key of sortedKeys) {
-      const lowerKey = key.toLowerCase();
-      if (isWordBoundaryMatch(lowerKey, lowerModelID)) {
-        return this.pricingData[key];
-      }
-      if (lowerNormalized && isWordBoundaryMatch(lowerKey, lowerNormalized)) {
-        return this.pricingData[key];
-      }
-    }
-
-    for (const key of sortedKeys) {
-      const lowerKey = key.toLowerCase();
-      if (isWordBoundaryMatch(lowerModelID, lowerKey)) {
-        return this.pricingData[key];
-      }
-      if (lowerNormalized && isWordBoundaryMatch(lowerNormalized, lowerKey)) {
-        return this.pricingData[key];
-      }
-    }
-
-    return null;
-  }
-
-  calculateCost(
-    tokens: {
-      input: number;
-      output: number;
-      reasoning?: number;
-      cacheRead: number;
-      cacheWrite: number;
-    },
-    pricing: LiteLLMModelPricing
-  ): number {
-    const inputCost = tokens.input * (pricing.input_cost_per_token ?? 0);
-    const outputCost =
-      (tokens.output + (tokens.reasoning ?? 0)) * (pricing.output_cost_per_token ?? 0);
-    const cacheWriteCost =
-      tokens.cacheWrite * (pricing.cache_creation_input_token_cost ?? 0);
-    const cacheReadCost =
-      tokens.cacheRead * (pricing.cache_read_input_token_cost ?? 0);
-
-    return inputCost + outputCost + cacheWriteCost + cacheReadCost;
-  }
-}
-
-/**
- * Clear pricing cache (for testing or forced refresh)
- */
-export function clearPricingCache(): void {
-  try {
-    const cachePath = getCachePath();
-    if (fs.existsSync(cachePath)) {
-      fs.unlinkSync(cachePath);
-    }
-  } catch {
-    // Ignore errors
-  }
-}

package/src/sessions/claudecode.ts

@@ -1,119 +0,0 @@
-/**
- * Claude Code (Anthropic official) session parser
- * Reads from ~/.claude/projects/
- */
-
-import * as fs from "node:fs";
-import * as path from "node:path";
-import * as os from "node:os";
-import { createUnifiedMessage, type UnifiedMessage, type TokenBreakdown } from "./types.js";
-
-interface ClaudeCodeEntry {
-  type: string;
-  timestamp?: string;
-  message?: {
-    model?: string;
-    usage?: {
-      input_tokens?: number;
-      output_tokens?: number;
-      cache_read_input_tokens?: number;
-      cache_creation_input_tokens?: number;
-    };
-  };
-}
-
-export function getClaudeCodeProjectsPath(): string {
-  return path.join(os.homedir(), ".claude", "projects");
-}
-
-function findJsonlFiles(dir: string): string[] {
-  const files: string[] = [];
-
-  function walk(currentDir: string) {
-    try {
-      const entries = fs.readdirSync(currentDir, { withFileTypes: true });
-      for (const entry of entries) {
-        const fullPath = path.join(currentDir, entry.name);
-        if (entry.isDirectory()) {
-          walk(fullPath);
-        } else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
-          files.push(fullPath);
-        }
-      }
-    } catch {
-      // Skip inaccessible directories
-    }
-  }
-
-  walk(dir);
-  return files;
-}
-
-export function parseClaudeCodeMessages(): UnifiedMessage[] {
-  const projectsPath = getClaudeCodeProjectsPath();
-
-  if (!fs.existsSync(projectsPath)) {
-    return [];
-  }
-
-  const messages: UnifiedMessage[] = [];
-  const files = findJsonlFiles(projectsPath);
-
-  for (const file of files) {
-    // Use file path as session ID
-    const sessionId = path.relative(projectsPath, file).replace(/\.jsonl$/, "");
-
-    try {
-      const content = fs.readFileSync(file, "utf-8");
-      const lines = content.split(/\r?\n/);
-
-      for (const line of lines) {
-        const trimmed = line.trim();
-        if (!trimmed) continue;
-
-        try {
-          const entry = JSON.parse(trimmed) as ClaudeCodeEntry;
-
-          // Process assistant messages with usage data and timestamp
-          if (
-            entry.type === "assistant" &&
-            entry.message?.usage &&
-            entry.timestamp
-          ) {
-            const model = entry.message.model || "unknown";
-            const usage = entry.message.usage;
-            const timestamp = new Date(entry.timestamp).getTime();
-
-            // Skip invalid timestamps
-            if (isNaN(timestamp)) continue;
-
-            const tokens: TokenBreakdown = {
-              input: usage.input_tokens || 0,
-              output: usage.output_tokens || 0,
-              cacheRead: usage.cache_read_input_tokens || 0,
-              cacheWrite: usage.cache_creation_input_tokens || 0,
-              reasoning: 0,
-            };
-
-            messages.push(
-              createUnifiedMessage(
-                "claude",
-                model,
-                "anthropic",
-                sessionId,
-                timestamp,
-                tokens
-              )
-            );
-          }
-        } catch {
-          // Skip malformed lines
-        }
-      }
-    } catch {
-      // Skip unreadable files
-    }
-  }
-
-  return messages;
-}

package/src/sessions/codex.ts

@@ -1,227 +0,0 @@
-/**
- * Codex CLI (OpenAI) session parser
- * Reads from ~/.codex/sessions/
- */
-
-import * as fs from "node:fs";
-import * as path from "node:path";
-import * as os from "node:os";
-import { createUnifiedMessage, type UnifiedMessage, type TokenBreakdown } from "./types.js";
-
-interface CodexTokenCountInfo {
-  last_token_usage?: {
-    input_tokens?: number;
-    cached_input_tokens?: number;
-    cache_read_input_tokens?: number;
-    output_tokens?: number;
-    reasoning_output_tokens?: number;
-    total_tokens?: number;
-  };
-  total_token_usage?: {
-    input_tokens?: number;
-    cached_input_tokens?: number;
-    cache_read_input_tokens?: number;
-    output_tokens?: number;
-    reasoning_output_tokens?: number;
-    total_tokens?: number;
-  };
-  model?: string;
-  model_name?: string;
-}
-
-interface CodexEntry {
-  type: string;
-  timestamp?: string;
-  payload?: {
-    type?: string;
-    timestamp?: string;
-    model?: string;
-    model_name?: string;
-    info?: CodexTokenCountInfo;
-  };
-}
-
-export function getCodexSessionsPath(): string {
-  const codexHome = process.env.CODEX_HOME || path.join(os.homedir(), ".codex");
-  return path.join(codexHome, "sessions");
-}
-
-function findJsonlFiles(dir: string): string[] {
-  const files: string[] = [];
-
-  function walk(currentDir: string) {
-    try {
-      const entries = fs.readdirSync(currentDir, { withFileTypes: true });
-      for (const entry of entries) {
-        const fullPath = path.join(currentDir, entry.name);
-        if (entry.isDirectory()) {
-          walk(fullPath);
-        } else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
-          files.push(fullPath);
-        }
-      }
-    } catch {
-      // Skip inaccessible directories
-    }
-  }
-
-  walk(dir);
-  return files;
-}
-
-function extractModel(payload: Record<string, unknown>): string | undefined {
-  // Direct model field
-  if (typeof payload.model === "string" && payload.model.trim()) {
-    return payload.model.trim();
-  }
-
-  // model_name field
-  if (typeof payload.model_name === "string" && payload.model_name.trim()) {
-    return payload.model_name.trim();
-  }
-
-  // Nested in info
-  const info = payload.info as Record<string, unknown> | undefined;
-  if (info) {
-    if (typeof info.model === "string" && info.model.trim()) {
-      return info.model.trim();
-    }
-    if (typeof info.model_name === "string" && info.model_name.trim()) {
-      return info.model_name.trim();
-    }
-  }
-
-  return undefined;
-}
-
-export function parseCodexMessages(): UnifiedMessage[] {
-  const sessionsPath = getCodexSessionsPath();
-
-  if (!fs.existsSync(sessionsPath)) {
-    return [];
-  }
-
-  const messages: UnifiedMessage[] = [];
-  const files = findJsonlFiles(sessionsPath);
-
-  for (const file of files) {
-    // Use file path as session ID
-    const sessionId = path.relative(sessionsPath, file).replace(/\.jsonl$/, "");
-
-    try {
-      const content = fs.readFileSync(file, "utf-8");
-      const lines = content.split(/\r?\n/);
-
-      let currentModel: string | undefined;
-      let previousTotals: {
-        input: number;
-        cached: number;
-        output: number;
-      } | null = null;
-
-      for (const line of lines) {
-        const trimmed = line.trim();
-        if (!trimmed) continue;
-
-        try {
-          const entry = JSON.parse(trimmed) as CodexEntry;
-
-          // Extract model from turn_context
-          if (entry.type === "turn_context" && entry.payload) {
-            const model = extractModel(entry.payload as Record<string, unknown>);
-            if (model) currentModel = model;
-            continue;
-          }
-
-          // Process token_count events with timestamp
-          if (entry.type === "event_msg" && entry.payload?.type === "token_count") {
-            const info = entry.payload.info;
-            if (!info) continue;
-
-            // Extract model from payload
-            const payloadModel = extractModel(entry.payload as Record<string, unknown>);
-            if (payloadModel) currentModel = payloadModel;
-
-            const model = currentModel || "unknown";
-
-            // Extract timestamp
-            const timestamp = entry.timestamp
-              ? new Date(entry.timestamp).getTime()
-              : entry.payload?.timestamp
-                ? new Date(entry.payload.timestamp).getTime()
-                : Date.now();
-
-            // Skip invalid timestamps
-            if (isNaN(timestamp)) continue;
-
-            // Get usage data
-            const lastUsage = info.last_token_usage;
-            const totalUsage = info.total_token_usage;
-
-            let delta = {
-              input: 0,
-              cached: 0,
-              output: 0,
-            };
-
-            if (lastUsage) {
-              delta = {
-                input: lastUsage.input_tokens || 0,
-                cached: lastUsage.cached_input_tokens || lastUsage.cache_read_input_tokens || 0,
-                output: lastUsage.output_tokens || 0,
-              };
-            } else if (totalUsage && previousTotals) {
-              delta = {
-                input: Math.max((totalUsage.input_tokens || 0) - previousTotals.input, 0),
-                cached: Math.max(
-                  (totalUsage.cached_input_tokens || totalUsage.cache_read_input_tokens || 0) -
-                    previousTotals.cached,
-                  0
-                ),
-                output: Math.max((totalUsage.output_tokens || 0) - previousTotals.output, 0),
-              };
-            }
-
-            if (totalUsage) {
-              previousTotals = {
-                input: totalUsage.input_tokens || 0,
-                cached: totalUsage.cached_input_tokens || totalUsage.cache_read_input_tokens || 0,
-                output: totalUsage.output_tokens || 0,
-              };
-            }
-
-            // Skip empty deltas
-            if (delta.input === 0 && delta.cached === 0 && delta.output === 0) {
-              continue;
-            }
-
-            const tokens: TokenBreakdown = {
-              input: delta.input,
-              output: delta.output,
-              cacheRead: delta.cached,
-              cacheWrite: 0, // Codex doesn't track cache write
-              reasoning: 0,
-            };
-
-            messages.push(
-              createUnifiedMessage(
-                "codex",
-                model,
-                "openai",
-                sessionId,
-                timestamp,
-                tokens
-              )
-            );
-          }
-        } catch {
-          // Skip malformed lines
-        }
-      }
-    } catch {
-      // Skip unreadable files
-    }
-  }
-
-  return messages;
-}