@od-oneapp/ai-platform 0.1.0

package/src/observability/analytics/index.ts

@@ -0,0 +1,593 @@
+/**
+ * Analytics Module
+ *
+ * AI usage analytics and monitoring:
+ * - Token usage tracking
+ * - Cost estimation
+ * - Performance metrics
+ * - Usage reporting
+ *
+ * @module analytics
+ */
+
+import { generateId } from '../../shared';
+
+// -----------------------------------------------------------------------------
+// Types
+// -----------------------------------------------------------------------------
+
+/** Model pricing configuration for analytics */
+export interface AnalyticsPricing {
+  readonly inputPer1kTokens: number;
+  readonly outputPer1kTokens: number;
+  readonly currency?: string;
+}
+
+/** Usage event */
+export interface UsageEvent {
+  readonly id: string;
+  readonly timestamp: number;
+  readonly model: string;
+  readonly operation: 'generate' | 'stream' | 'embed' | 'tool';
+  readonly inputTokens: number;
+  readonly outputTokens: number;
+  readonly totalTokens: number;
+  readonly cost: number;
+  readonly latencyMs: number;
+  readonly success: boolean;
+  readonly error?: string;
+  readonly metadata?: Record<string, unknown>;
+}
+
+/** Usage summary */
+export interface UsageSummary {
+  readonly period: { start: number; end: number };
+  readonly totalRequests: number;
+  readonly successfulRequests: number;
+  readonly failedRequests: number;
+  readonly totalInputTokens: number;
+  readonly totalOutputTokens: number;
+  readonly totalTokens: number;
+  readonly totalCost: number;
+  readonly avgLatencyMs: number;
+  readonly p95LatencyMs: number;
+  readonly byModel: Record<string, ModelUsage>;
+  readonly byOperation: Record<string, OperationUsage>;
+}
+
+/** Model-specific usage */
+export interface ModelUsage {
+  readonly model: string;
+  readonly requests: number;
+  readonly inputTokens: number;
+  readonly outputTokens: number;
+  readonly cost: number;
+  readonly avgLatencyMs: number;
+}
+
+/** Operation-specific usage */
+export interface OperationUsage {
+  readonly operation: string;
+  readonly requests: number;
+  readonly successRate: number;
+  readonly avgLatencyMs: number;
+}
+
+/** Budget alert */
+export interface BudgetAlert {
+  readonly type: 'cost' | 'tokens' | 'requests';
+  readonly threshold: number;
+  readonly current: number;
+  readonly period: 'hourly' | 'daily' | 'weekly' | 'monthly';
+}
+
+// -----------------------------------------------------------------------------
+// Default Pricing
+// -----------------------------------------------------------------------------
+
+/**
+ * Default model pricing rates.
+ * Last updated: January 2026
+ *
+ * WARNING: These prices may become stale. For production use, consider:
+ * - Providing explicit `customPricing` configuration
+ * - Implementing a mechanism to fetch live pricing from providers
+ */
+const DEFAULT_PRICING: Record<string, AnalyticsPricing> = {
+  // OpenAI
+  'gpt-4o': { inputPer1kTokens: 0.005, outputPer1kTokens: 0.015 },
+  'gpt-4o-mini': { inputPer1kTokens: 0.00015, outputPer1kTokens: 0.0006 },
+  'gpt-4-turbo': { inputPer1kTokens: 0.01, outputPer1kTokens: 0.03 },
+  'gpt-4': { inputPer1kTokens: 0.03, outputPer1kTokens: 0.06 },
+  'gpt-3.5-turbo': { inputPer1kTokens: 0.0005, outputPer1kTokens: 0.0015 },
+
+  // Anthropic
+  'claude-3-5-sonnet': { inputPer1kTokens: 0.003, outputPer1kTokens: 0.015 },
+  'claude-3-opus': { inputPer1kTokens: 0.015, outputPer1kTokens: 0.075 },
+  'claude-3-sonnet': { inputPer1kTokens: 0.003, outputPer1kTokens: 0.015 },
+  'claude-3-haiku': { inputPer1kTokens: 0.00025, outputPer1kTokens: 0.00125 },
+
+  // Embeddings
+  'text-embedding-3-small': { inputPer1kTokens: 0.00002, outputPer1kTokens: 0 },
+  'text-embedding-3-large': { inputPer1kTokens: 0.00013, outputPer1kTokens: 0 },
+};
+
+// -----------------------------------------------------------------------------
+// Analytics Collector
+// -----------------------------------------------------------------------------
+
+/**
+ * Creates an analytics collector for tracking AI usage.
+ *
+ * @example
+ * ```typescript
+ * const analytics = createAnalyticsCollector({
+ *   onAlert: (alert) => console.warn('Budget alert:', alert),
+ *   budgets: {
+ *     daily: { cost: 100, tokens: 1_000_000 }
+ *   }
+ * });
+ *
+ * analytics.record({
+ *   model: 'gpt-4o',
+ *   operation: 'generate',
+ *   inputTokens: 100,
+ *   outputTokens: 50,
+ *   latencyMs: 1200,
+ *   success: true
+ * });
+ *
+ * const summary = analytics.getSummary('daily');
+ * ```
+ */
+export function createAnalyticsCollector(options?: {
+  customPricing?: Record<string, AnalyticsPricing>;
+  budgets?: {
+    hourly?: { cost?: number; tokens?: number; requests?: number };
+    daily?: { cost?: number; tokens?: number; requests?: number };
+  };
+  onAlert?: (alert: BudgetAlert) => void;
+  maxHistory?: number;
+}): AnalyticsCollector {
+  const { customPricing = {}, budgets, onAlert, maxHistory = 10000 } = options ?? {};
+  const pricing = { ...DEFAULT_PRICING, ...customPricing };
+  const events: UsageEvent[] = [];
+  let recordsSinceLastBudgetCheck = 0;
+  const BUDGET_CHECK_INTERVAL = 100;
+
+  function calculateCost(model: string, inputTokens: number, outputTokens: number): number {
+    const modelPricing = pricing[model] ?? { inputPer1kTokens: 0, outputPer1kTokens: 0 };
+    return (
+      (inputTokens / 1000) * modelPricing.inputPer1kTokens +
+      (outputTokens / 1000) * modelPricing.outputPer1kTokens
+    );
+  }
+
+  function record(input: {
+    model: string;
+    operation: UsageEvent['operation'];
+    inputTokens: number;
+    outputTokens: number;
+    latencyMs: number;
+    success: boolean;
+    error?: string;
+    metadata?: Record<string, unknown>;
+  }): UsageEvent {
+    const cost = calculateCost(input.model, input.inputTokens, input.outputTokens);
+
+    const event: UsageEvent = {
+      id: generateId(),
+      timestamp: Date.now(),
+      model: input.model,
+      operation: input.operation,
+      inputTokens: input.inputTokens,
+      outputTokens: input.outputTokens,
+      totalTokens: input.inputTokens + input.outputTokens,
+      cost,
+      latencyMs: input.latencyMs,
+      success: input.success,
+      error: input.error,
+      metadata: input.metadata,
+    };
+
+    events.push(event);
+
+    // Trim history efficiently (single splice instead of repeated shift)
+    if (events.length > maxHistory) {
+      events.splice(0, events.length - maxHistory);
+    }
+
+    // Check budgets periodically to avoid O(n) per record
+    recordsSinceLastBudgetCheck++;
+    if (recordsSinceLastBudgetCheck >= BUDGET_CHECK_INTERVAL) {
+      checkBudgets();
+      recordsSinceLastBudgetCheck = 0;
+    }
+
+    return event;
+  }
+
+  function checkBudgets(): void {
+    if (!budgets || !onAlert) return;
+
+    const now = Date.now();
+
+    if (budgets.hourly) {
+      const hourStart = now - 60 * 60 * 1000;
+      const hourEvents = events.filter(e => e.timestamp >= hourStart);
+      checkBudgetThresholds(hourEvents, budgets.hourly, 'hourly');
+    }
+
+    if (budgets.daily) {
+      const dayStart = now - 24 * 60 * 60 * 1000;
+      const dayEvents = events.filter(e => e.timestamp >= dayStart);
+      checkBudgetThresholds(dayEvents, budgets.daily, 'daily');
+    }
+  }
+
+  function checkBudgetThresholds(
+    periodEvents: UsageEvent[],
+    budget: { cost?: number; tokens?: number; requests?: number },
+    period: 'hourly' | 'daily' | 'weekly' | 'monthly',
+  ): void {
+    if (!onAlert) return;
+
+    const totalCost = periodEvents.reduce((sum, e) => sum + e.cost, 0);
+    const totalTokens = periodEvents.reduce((sum, e) => sum + e.totalTokens, 0);
+    const totalRequests = periodEvents.length;
+
+    if (budget.cost && totalCost >= budget.cost * 0.9) {
+      onAlert({ type: 'cost', threshold: budget.cost, current: totalCost, period });
+    }
+    if (budget.tokens && totalTokens >= budget.tokens * 0.9) {
+      onAlert({ type: 'tokens', threshold: budget.tokens, current: totalTokens, period });
+    }
+    if (budget.requests && totalRequests >= budget.requests * 0.9) {
+      onAlert({ type: 'requests', threshold: budget.requests, current: totalRequests, period });
+    }
+  }
+
+  function getSummary(period: 'hourly' | 'daily' | 'weekly' | 'monthly' | 'all'): UsageSummary {
+    const now = Date.now();
+    let start: number;
+
+    switch (period) {
+      case 'hourly':
+        start = now - 60 * 60 * 1000;
+        break;
+      case 'daily':
+        start = now - 24 * 60 * 60 * 1000;
+        break;
+      case 'weekly':
+        start = now - 7 * 24 * 60 * 60 * 1000;
+        break;
+      case 'monthly':
+        start = now - 30 * 24 * 60 * 60 * 1000;
+        break;
+      case 'all':
+        start = 0;
+        break;
+      default:
+        start = 0;
+    }
+
+    const periodEvents = events.filter(e => e.timestamp >= start);
+    return computeSummary(periodEvents, start, now);
+  }
+
+  function computeSummary(periodEvents: UsageEvent[], start: number, end: number): UsageSummary {
+    const byModel: Record<string, ModelUsage & { totalLatencyMs: number }> = {};
+    const byOperation: Record<string, OperationUsage & { totalLatencyMs: number }> = {};
+
+    let totalInputTokens = 0;
+    let totalOutputTokens = 0;
+    let totalCost = 0;
+    let totalLatency = 0;
+    let successCount = 0;
+    const latencies: number[] = [];
+
+    for (const event of periodEvents) {
+      totalInputTokens += event.inputTokens;
+      totalOutputTokens += event.outputTokens;
+      totalCost += event.cost;
+      totalLatency += event.latencyMs;
+      latencies.push(event.latencyMs);
+      if (event.success) successCount++;
+
+      // By model - track totals instead of running average
+      const modelUsage = byModel[event.model];
+      if (modelUsage) {
+        byModel[event.model] = {
+          ...modelUsage,
+          requests: modelUsage.requests + 1,
+          inputTokens: modelUsage.inputTokens + event.inputTokens,
+          outputTokens: modelUsage.outputTokens + event.outputTokens,
+          cost: modelUsage.cost + event.cost,
+          totalLatencyMs: modelUsage.totalLatencyMs + event.latencyMs,
+        };
+      } else {
+        byModel[event.model] = {
+          model: event.model,
+          requests: 1,
+          inputTokens: event.inputTokens,
+          outputTokens: event.outputTokens,
+          cost: event.cost,
+          avgLatencyMs: event.latencyMs,
+          totalLatencyMs: event.latencyMs,
+        };
+      }
+
+      // By operation - track totals instead of running average
+      const opUsage = byOperation[event.operation];
+      if (opUsage) {
+        const newRequests = opUsage.requests + 1;
+        const newSuccessCount = opUsage.successRate * opUsage.requests + (event.success ? 1 : 0);
+        byOperation[event.operation] = {
+          ...opUsage,
+          requests: newRequests,
+          successRate: newSuccessCount / newRequests,
+          totalLatencyMs: opUsage.totalLatencyMs + event.latencyMs,
+        };
+      } else {
+        byOperation[event.operation] = {
+          operation: event.operation,
+          requests: 1,
+          successRate: event.success ? 1 : 0,
+          avgLatencyMs: event.latencyMs,
+          totalLatencyMs: event.latencyMs,
+        };
+      }
+    }
+
+    // Compute final averages from totals
+    for (const model of Object.keys(byModel)) {
+      const modelUsage = byModel[model];
+      if (!modelUsage) continue;
+      const divisor = modelUsage.requests || 1;
+      byModel[model] = {
+        ...modelUsage,
+        model,
+        avgLatencyMs: modelUsage.totalLatencyMs / divisor,
+      };
+      delete (byModel[model] as { totalLatencyMs?: number }).totalLatencyMs;
+    }
+
+    for (const operation of Object.keys(byOperation)) {
+      const opUsage = byOperation[operation];
+      if (!opUsage) continue;
+      const divisor = opUsage.requests || 1;
+      byOperation[operation] = {
+        ...opUsage,
+        operation,
+        avgLatencyMs: opUsage.totalLatencyMs / divisor,
+      };
+      delete (byOperation[operation] as { totalLatencyMs?: number }).totalLatencyMs;
+    }
+
+    // Calculate p95 latency
+    latencies.sort((a, b) => a - b);
+    const p95Index = Math.floor(latencies.length * 0.95);
+    const p95Latency = latencies[p95Index] ?? 0;
+
+    return {
+      period: { start, end },
+      totalRequests: periodEvents.length,
+      successfulRequests: successCount,
+      failedRequests: periodEvents.length - successCount,
+      totalInputTokens,
+      totalOutputTokens,
+      totalTokens: totalInputTokens + totalOutputTokens,
+      totalCost,
+      avgLatencyMs: periodEvents.length > 0 ? totalLatency / periodEvents.length : 0,
+      p95LatencyMs: p95Latency,
+      byModel,
+      byOperation,
+    };
+  }
+
+  function getEvents(options?: {
+    since?: number;
+    until?: number;
+    model?: string;
+    operation?: UsageEvent['operation'];
+    limit?: number;
+  }): UsageEvent[] {
+    let filtered = [...events];
+    const { since, until, model, operation, limit } = options ?? {};
+
+    if (since !== undefined) {
+      filtered = filtered.filter(e => e.timestamp >= since);
+    }
+    if (until !== undefined) {
+      filtered = filtered.filter(e => e.timestamp <= until);
+    }
+    if (model) {
+      filtered = filtered.filter(e => e.model === model);
+    }
+    if (operation) {
+      filtered = filtered.filter(e => e.operation === operation);
+    }
+    if (limit !== undefined) {
+      if (limit <= 0) return [];
+      filtered = filtered.slice(-limit);
+    }
+
+    return filtered;
+  }
+
+  function clear(): void {
+    events.length = 0;
+  }
+
+  function exportData(): UsageEvent[] {
+    return [...events];
+  }
+
+  function importData(data: UsageEvent[]): void {
+    // Validate imported events have required fields
+    const allowedOps = new Set(['generate', 'stream', 'embed', 'tool']);
+    for (const event of data) {
+      if (typeof event.id !== 'string' || typeof event.timestamp !== 'number') {
+        throw new Error(
+          `Invalid event data: missing required id or timestamp (event id: ${event.id ?? 'unknown'})`,
+        );
+      }
+      if (
+        typeof event.timestamp !== 'number' ||
+        !Number.isInteger(event.timestamp) ||
+        event.timestamp <= 0
+      ) {
+        throw new Error(
+          `Invalid event data: timestamp must be a positive integer (event id: ${event.id ?? 'unknown'})`,
+        );
+      }
+      if (typeof event.model !== 'string' || event.model.length === 0) {
+        throw new Error(
+          `Invalid event data: model must be a non-empty string (event id: ${event.id ?? 'unknown'})`,
+        );
+      }
+      if (typeof event.operation !== 'string') {
+        throw new Error(
+          `Invalid event data: missing required operation (event id: ${event.id ?? 'unknown'})`,
+        );
+      }
+      if (!allowedOps.has(event.operation)) {
+        throw new Error(
+          `Invalid event data: unsupported operation ${event.operation} (event id: ${event.id ?? 'unknown'})`,
+        );
+      }
+      const numericFields = [
+        event.inputTokens,
+        event.outputTokens,
+        event.totalTokens,
+        event.cost,
+        event.latencyMs,
+      ];
+      if (numericFields.some(v => !Number.isFinite(v) || v < 0)) {
+        throw new Error(
+          `Invalid event data: numeric fields must be finite and >= 0 (event id: ${event.id ?? 'unknown'})`,
+        );
+      }
+    }
+    events.push(...data);
+    // Trim efficiently with single splice instead of repeated shift
+    if (events.length > maxHistory) {
+      events.splice(0, events.length - maxHistory);
+    }
+  }
+
+  return {
+    record,
+    getSummary,
+    getEvents,
+    calculateCost,
+    clear,
+    exportData,
+    importData,
+  };
+}
+
+export interface AnalyticsCollector {
+  record: (input: {
+    model: string;
+    operation: UsageEvent['operation'];
+    inputTokens: number;
+    outputTokens: number;
+    latencyMs: number;
+    success: boolean;
+    error?: string;
+    metadata?: Record<string, unknown>;
+  }) => UsageEvent;
+  getSummary: (period: 'hourly' | 'daily' | 'weekly' | 'monthly' | 'all') => UsageSummary;
+  getEvents: (options?: {
+    since?: number;
+    until?: number;
+    model?: string;
+    operation?: UsageEvent['operation'];
+    limit?: number;
+  }) => UsageEvent[];
+  calculateCost: (model: string, inputTokens: number, outputTokens: number) => number;
+  clear: () => void;
+  exportData: () => UsageEvent[];
+  importData: (data: UsageEvent[]) => void;
+}
+
+// -----------------------------------------------------------------------------
+// Reporting Helpers
+// -----------------------------------------------------------------------------
+
+/**
+ * Formats cost as currency string.
+ */
+export function formatCost(cost: number, currency = 'USD'): string {
+  return new Intl.NumberFormat('en-US', {
+    style: 'currency',
+    currency,
+    minimumFractionDigits: 4,
+    maximumFractionDigits: 4,
+  }).format(cost);
+}
+
+/**
+ * Formats token count with SI prefixes.
+ */
+export function formatTokens(tokens: number): string {
+  if (tokens >= 1_000_000) {
+    return `${(tokens / 1_000_000).toFixed(2)}M`;
+  }
+  if (tokens >= 1_000) {
+    return `${(tokens / 1_000).toFixed(2)}K`;
+  }
+  return tokens.toString();
+}
+
+/**
+ * Formats latency in human-readable form.
+ */
+export function formatLatency(ms: number): string {
+  if (ms >= 1000) {
+    return `${(ms / 1000).toFixed(2)}s`;
+  }
+  return `${ms.toFixed(0)}ms`;
+}
+
+/**
+ * Generates a usage report as markdown.
+ */
+export function generateReport(summary: UsageSummary): string {
+  const lines: string[] = [];
+
+  const periodStr = `${new Date(summary.period.start).toISOString()} - ${new Date(summary.period.end).toISOString()}`;
+
+  lines.push('# AI Usage Report');
+  lines.push(`\nPeriod: ${periodStr}\n`);
+
+  lines.push('## Summary');
+  lines.push(`- Total Requests: ${summary.totalRequests}`);
+  lines.push(
+    `- Success Rate: ${summary.totalRequests > 0 ? ((summary.successfulRequests / summary.totalRequests) * 100).toFixed(1) : '0.0'}%`,
+  );
+  lines.push(`- Total Tokens: ${formatTokens(summary.totalTokens)}`);
+  lines.push(`- Total Cost: ${formatCost(summary.totalCost)}`);
+  lines.push(`- Avg Latency: ${formatLatency(summary.avgLatencyMs)}`);
+  lines.push(`- P95 Latency: ${formatLatency(summary.p95LatencyMs)}`);
+
+  lines.push('\n## By Model');
+  for (const [model, usage] of Object.entries(summary.byModel)) {
+    lines.push(`\n### ${model}`);
+    lines.push(`- Requests: ${usage.requests}`);
+    lines.push(`- Tokens: ${formatTokens(usage.inputTokens + usage.outputTokens)}`);
+    lines.push(`- Cost: ${formatCost(usage.cost)}`);
+    lines.push(`- Avg Latency: ${formatLatency(usage.avgLatencyMs)}`);
+  }
+
+  lines.push('\n## By Operation');
+  for (const [op, usage] of Object.entries(summary.byOperation)) {
+    lines.push(`\n### ${op}`);
+    lines.push(`- Requests: ${usage.requests}`);
+    lines.push(`- Success Rate: ${(usage.successRate * 100).toFixed(1)}%`);
+    lines.push(`- Avg Latency: ${formatLatency(usage.avgLatencyMs)}`);
+  }
+
+  return lines.join('\n');
+}

package/src/observability/cost/index.ts

@@ -0,0 +1,16 @@
+/**
+ * Cost tracking and attribution module
+ *
+ * Provides middleware for tracking AI usage costs, enforcing budgets,
+ * and attributing costs to users, teams, and features.
+ *
+ * @module @od-oneapp/ai-platform/cost
+ */
+
+// Types
+export type { UsageEvent, CostMetadata, BudgetConfig, UsageSummary, UsageCallback } from './types';
+export { BudgetExceededError } from './types';
+
+// Tracking
+export { createUsageTracker, calculateCost, extractMetadata } from './tracking/usage-tracker';
+export { createBudgetGuard, type GetCurrentUsage } from './tracking/budget-guard';