@operor/analytics 0.1.0

package/src/SQLiteAnalyticsStore.ts

@@ -0,0 +1,367 @@
+import Database from 'better-sqlite3';
+import type {
+  AnalyticsStore,
+  AnalyticsEvent,
+  AnalyticsSummary,
+  MessageVolumeData,
+  AgentStatsData,
+  CustomerStatsData,
+  KbStatsData,
+  ConversationSession,
+  TimeRange,
+} from './types.js';
+
+const SESSION_GAP_MS = 30 * 60 * 1000; // 30 minutes
+
+export class SQLiteAnalyticsStore implements AnalyticsStore {
+  private db: Database.Database;
+
+  constructor(dbPath: string = './analytics.db') {
+    this.db = new Database(dbPath);
+    this.db.pragma('journal_mode = WAL');
+    this.db.pragma('foreign_keys = ON');
+  }
+
+  async initialize(): Promise<void> {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS message_events (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        timestamp INTEGER NOT NULL,
+        channel TEXT NOT NULL,
+        customer_phone TEXT NOT NULL,
+        customer_id TEXT NOT NULL,
+        agent_name TEXT NOT NULL,
+        intent TEXT NOT NULL DEFAULT '',
+        intent_confidence REAL NOT NULL DEFAULT 0,
+        response_time_ms INTEGER NOT NULL DEFAULT 0,
+        llm_cost REAL NOT NULL DEFAULT 0,
+        kb_top_score REAL NOT NULL DEFAULT 0,
+        kb_is_faq_match INTEGER NOT NULL DEFAULT 0,
+        kb_result_count INTEGER NOT NULL DEFAULT 0,
+        tool_calls_count INTEGER NOT NULL DEFAULT 0,
+        tool_calls_json TEXT,
+        message_length INTEGER NOT NULL DEFAULT 0,
+        response_length INTEGER NOT NULL DEFAULT 0,
+        is_new_customer INTEGER NOT NULL DEFAULT 0,
+        was_escalated INTEGER NOT NULL DEFAULT 0,
+        conversation_id TEXT,
+        prompt_tokens INTEGER NOT NULL DEFAULT 0,
+        completion_tokens INTEGER NOT NULL DEFAULT 0
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_events_timestamp ON message_events(timestamp);
+      CREATE INDEX IF NOT EXISTS idx_events_agent ON message_events(agent_name);
+      CREATE INDEX IF NOT EXISTS idx_events_customer ON message_events(customer_phone);
+      CREATE INDEX IF NOT EXISTS idx_events_conversation ON message_events(conversation_id);
+
+      CREATE TABLE IF NOT EXISTS conversations (
+        id TEXT PRIMARY KEY,
+        customer_id TEXT NOT NULL,
+        customer_phone TEXT NOT NULL,
+        channel TEXT NOT NULL,
+        agent_id TEXT NOT NULL,
+        started_at INTEGER NOT NULL,
+        ended_at INTEGER NOT NULL,
+        message_count INTEGER NOT NULL DEFAULT 0,
+        tool_call_count INTEGER NOT NULL DEFAULT 0,
+        total_duration_ms INTEGER NOT NULL DEFAULT 0,
+        total_cost REAL NOT NULL DEFAULT 0,
+        resolved INTEGER NOT NULL DEFAULT 0,
+        resolution_type TEXT
+      );
+
+      CREATE TABLE IF NOT EXISTS daily_metrics (
+        date TEXT NOT NULL,
+        metric TEXT NOT NULL,
+        channel TEXT NOT NULL DEFAULT '',
+        agent_id TEXT NOT NULL DEFAULT '',
+        value REAL NOT NULL DEFAULT 0,
+        PRIMARY KEY (date, metric, channel, agent_id)
+      );
+
+      CREATE TABLE IF NOT EXISTS analytics_meta (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+    `);
+  }
+
+  async close(): Promise<void> {
+    this.db.close();
+  }
+
+  async recordMessageEvent(event: AnalyticsEvent): Promise<void> {
+    this.db.prepare(`
+      INSERT INTO message_events
+        (timestamp, channel, customer_phone, customer_id, agent_name, intent,
+         intent_confidence, response_time_ms, llm_cost, kb_top_score, kb_is_faq_match,
+         kb_result_count, tool_calls_count, tool_calls_json, message_length,
+         response_length, is_new_customer, was_escalated, conversation_id,
+         prompt_tokens, completion_tokens)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      event.timestamp,
+      event.channel,
+      event.customerPhone,
+      event.customerId,
+      event.agentName,
+      event.intent,
+      event.intentConfidence,
+      event.responseTimeMs,
+      event.llmCost,
+      event.kbTopScore,
+      event.kbIsFaqMatch ? 1 : 0,
+      event.kbResultCount,
+      event.toolCallsCount,
+      event.toolCallsJson,
+      event.messageLength,
+      event.responseLength,
+      event.isNewCustomer ? 1 : 0,
+      event.wasEscalated ? 1 : 0,
+      event.conversationId,
+      event.promptTokens,
+      event.completionTokens,
+    );
+  }
+
+  async getOrCreateConversation(
+    customerPhone: string,
+    channel: string,
+    agentName: string,
+  ): Promise<string> {
+    const now = Date.now();
+    const cutoff = now - SESSION_GAP_MS;
+
+    // Find an active conversation within the session gap
+    const existing = this.db.prepare(`
+      SELECT id FROM conversations
+      WHERE customer_phone = ? AND channel = ? AND agent_id = ? AND ended_at > ?
+      ORDER BY ended_at DESC LIMIT 1
+    `).get(customerPhone, channel, agentName, cutoff) as any;
+
+    if (existing) {
+      return existing.id as string;
+    }
+
+    // Create new conversation
+    const id = crypto.randomUUID();
+    this.db.prepare(`
+      INSERT INTO conversations (id, customer_id, customer_phone, channel, agent_id, started_at, ended_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `).run(id, '', customerPhone, channel, agentName, now, now);
+
+    return id;
+  }
+
+  async updateConversationStats(
+    conversationId: string,
+    event: AnalyticsEvent,
+  ): Promise<void> {
+    this.db.prepare(`
+      UPDATE conversations SET
+        customer_id = CASE WHEN customer_id = '' THEN ? ELSE customer_id END,
+        ended_at = ?,
+        message_count = message_count + 1,
+        tool_call_count = tool_call_count + ?,
+        total_duration_ms = ? - started_at,
+        total_cost = total_cost + ?
+      WHERE id = ?
+    `).run(
+      event.customerId,
+      event.timestamp,
+      event.toolCallsCount,
+      event.timestamp,
+      event.llmCost,
+      conversationId,
+    );
+  }
+
+  async getSummary(range?: TimeRange): Promise<AnalyticsSummary> {
+    const where = range ? 'WHERE timestamp >= ? AND timestamp <= ?' : '';
+    const params = range ? [range.from, range.to] : [];
+
+    const row = this.db.prepare(`
+      SELECT
+        COUNT(*) as total,
+        AVG(response_time_ms) as avg_response_time,
+        COUNT(DISTINCT customer_phone) as unique_customers,
+        SUM(CASE WHEN is_new_customer = 1 THEN 1 ELSE 0 END) as new_customers,
+        SUM(CASE WHEN kb_is_faq_match = 1 THEN 1 ELSE 0 END) as faq_hits,
+        SUM(CASE WHEN kb_top_score >= 0.7 THEN 1 ELSE 0 END) as kb_answered,
+        SUM(CASE WHEN kb_top_score < 0.3 THEN 1 ELSE 0 END) as no_answer
+      FROM message_events ${where}
+    `).get(...params) as any;
+
+    const total = row.total || 0;
+
+    // Peak day
+    const peakRow = this.db.prepare(`
+      SELECT date(timestamp / 1000, 'unixepoch') as day, COUNT(*) as cnt
+      FROM message_events ${where}
+      GROUP BY day ORDER BY cnt DESC LIMIT 1
+    `).get(...params) as any;
+
+    // Agent breakdown
+    const agentRows = this.db.prepare(`
+      SELECT agent_name, COUNT(*) as cnt
+      FROM message_events ${where}
+      GROUP BY agent_name
+    `).all(...params) as any[];
+
+    const agentBreakdown: Record<string, number> = {};
+    for (const r of agentRows) {
+      agentBreakdown[r.agent_name] = r.cnt;
+    }
+
+    // Days in range for avg calculation
+    const daysRow = this.db.prepare(`
+      SELECT COUNT(DISTINCT date(timestamp / 1000, 'unixepoch')) as days
+      FROM message_events ${where}
+    `).get(...params) as any;
+    const days = daysRow.days || 1;
+
+    return {
+      totalMessages: total,
+      avgPerDay: total / days,
+      peakDay: peakRow?.day ?? null,
+      kbAnsweredPct: total > 0 ? (row.kb_answered / total) * 100 : 0,
+      llmFallbackPct: total > 0 ? ((total - row.kb_answered) / total) * 100 : 0,
+      noAnswerPct: total > 0 ? (row.no_answer / total) * 100 : 0,
+      avgResponseTime: row.avg_response_time || 0,
+      faqHitRate: total > 0 ? (row.faq_hits / total) * 100 : 0,
+      uniqueCustomers: row.unique_customers || 0,
+      newCustomers: row.new_customers || 0,
+      returningCustomers: (row.unique_customers || 0) - (row.new_customers || 0),
+      agentBreakdown,
+      pendingReviews: 0, // populated externally from copilot
+    };
+  }
+
+  async getMessageVolume(range?: TimeRange, groupBy: 'day' | 'hour' = 'day'): Promise<MessageVolumeData[]> {
+    const where = range ? 'WHERE timestamp >= ? AND timestamp <= ?' : '';
+    const params = range ? [range.from, range.to] : [];
+
+    const dateExpr = groupBy === 'hour'
+      ? "strftime('%Y-%m-%d %H:00', timestamp / 1000, 'unixepoch')"
+      : "date(timestamp / 1000, 'unixepoch')";
+
+    const rows = this.db.prepare(`
+      SELECT ${dateExpr} as date, COUNT(*) as count
+      FROM message_events ${where}
+      GROUP BY date ORDER BY date
+    `).all(...params) as any[];
+
+    return rows.map(r => ({ date: r.date, count: r.count }));
+  }
+
+  async getAgentStats(range?: TimeRange): Promise<AgentStatsData[]> {
+    const where = range ? 'WHERE timestamp >= ? AND timestamp <= ?' : '';
+    const params = range ? [range.from, range.to] : [];
+
+    const rows = this.db.prepare(`
+      SELECT
+        agent_name,
+        COUNT(*) as message_count,
+        AVG(response_time_ms) as avg_response_time,
+        AVG(intent_confidence) as avg_confidence,
+        SUM(CASE WHEN was_escalated = 1 THEN 1 ELSE 0 END) as escalation_count,
+        SUM(tool_calls_count) as tool_call_count
+      FROM message_events ${where}
+      GROUP BY agent_name
+    `).all(...params) as any[];
+
+    return rows.map(r => ({
+      agentName: r.agent_name,
+      messageCount: r.message_count,
+      avgResponseTimeMs: r.avg_response_time || 0,
+      avgConfidence: r.avg_confidence || 0,
+      escalationCount: r.escalation_count || 0,
+      toolCallCount: r.tool_call_count || 0,
+    }));
+  }
+
+  async getCustomerStats(range?: TimeRange, limit = 50): Promise<CustomerStatsData[]> {
+    const where = range ? 'WHERE timestamp >= ? AND timestamp <= ?' : '';
+    const params: any[] = range ? [range.from, range.to, limit] : [limit];
+
+    const rows = this.db.prepare(`
+      SELECT
+        customer_id,
+        customer_phone,
+        COUNT(*) as message_count,
+        MIN(timestamp) as first_seen,
+        MAX(timestamp) as last_seen,
+        MAX(is_new_customer) as is_new
+      FROM message_events ${where}
+      GROUP BY customer_phone
+      ORDER BY message_count DESC
+      LIMIT ?
+    `).all(...params) as any[];
+
+    return rows.map(r => ({
+      customerId: r.customer_id,
+      customerPhone: r.customer_phone,
+      messageCount: r.message_count,
+      firstSeen: r.first_seen,
+      lastSeen: r.last_seen,
+      isNew: !!r.is_new,
+    }));
+  }
+
+  async getKbStats(range?: TimeRange): Promise<KbStatsData> {
+    const where = range ? 'WHERE timestamp >= ? AND timestamp <= ?' : '';
+    const params = range ? [range.from, range.to] : [];
+
+    const row = this.db.prepare(`
+      SELECT
+        COUNT(*) as total,
+        SUM(CASE WHEN kb_is_faq_match = 1 THEN 1 ELSE 0 END) as faq_hits,
+        AVG(kb_top_score) as avg_score
+      FROM message_events ${where}
+    `).get(...params) as any;
+
+    const total = row.total || 0;
+
+    const topHits = await this.getTopFaqHits(10, range);
+    const topMisses = await this.getTopMisses(10, range);
+
+    return {
+      totalQueries: total,
+      faqHits: row.faq_hits || 0,
+      faqHitRate: total > 0 ? ((row.faq_hits || 0) / total) * 100 : 0,
+      avgTopScore: row.avg_score || 0,
+      topFaqHits: topHits,
+      topMisses: topMisses,
+    };
+  }
+
+  async getTopFaqHits(limit = 10, range?: TimeRange): Promise<Array<{ intent: string; count: number }>> {
+    const where = range
+      ? 'WHERE kb_is_faq_match = 1 AND timestamp >= ? AND timestamp <= ?'
+      : 'WHERE kb_is_faq_match = 1';
+    const params: any[] = range ? [range.from, range.to, limit] : [limit];
+
+    const rows = this.db.prepare(`
+      SELECT intent, COUNT(*) as count
+      FROM message_events ${where}
+      GROUP BY intent ORDER BY count DESC LIMIT ?
+    `).all(...params) as any[];
+
+    return rows.map(r => ({ intent: r.intent, count: r.count }));
+  }
+
+  async getTopMisses(limit = 10, range?: TimeRange): Promise<Array<{ intent: string; count: number; avgScore: number }>> {
+    const where = range
+      ? 'WHERE kb_top_score < 0.5 AND kb_top_score > 0 AND timestamp >= ? AND timestamp <= ?'
+      : 'WHERE kb_top_score < 0.5 AND kb_top_score > 0';
+    const params: any[] = range ? [range.from, range.to, limit] : [limit];
+
+    const rows = this.db.prepare(`
+      SELECT intent, COUNT(*) as count, AVG(kb_top_score) as avg_score
+      FROM message_events ${where}
+      GROUP BY intent ORDER BY count DESC LIMIT ?
+    `).all(...params) as any[];
+
+    return rows.map(r => ({ intent: r.intent, count: r.count, avgScore: r.avg_score }));
+  }
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from './types.js';
+export { SQLiteAnalyticsStore } from './SQLiteAnalyticsStore.js';
+export { InMemoryAnalyticsStore } from './InMemoryAnalyticsStore.js';
+export { AnalyticsCollector } from './AnalyticsCollector.js';

package/src/types.ts

@@ -0,0 +1,149 @@
+/**
+ * Analytics types for Operor
+ */
+
+/** A single message event recorded by the analytics collector */
+export interface AnalyticsEvent {
+  id?: number;
+  timestamp: number;
+  channel: string;
+  customerPhone: string;
+  customerId: string;
+  agentName: string;
+  intent: string;
+  intentConfidence: number;
+  responseTimeMs: number;
+  llmCost: number;
+  kbTopScore: number;
+  kbIsFaqMatch: boolean;
+  kbResultCount: number;
+  toolCallsCount: number;
+  toolCallsJson: string | null;
+  messageLength: number;
+  responseLength: number;
+  isNewCustomer: boolean;
+  wasEscalated: boolean;
+  conversationId: string | null;
+  promptTokens: number;
+  completionTokens: number;
+}
+
+/** Aggregated summary for a time range */
+export interface AnalyticsSummary {
+  totalMessages: number;
+  avgPerDay: number;
+  peakDay: string | null;
+  kbAnsweredPct: number;
+  llmFallbackPct: number;
+  noAnswerPct: number;
+  avgResponseTime: number;
+  faqHitRate: number;
+  uniqueCustomers: number;
+  newCustomers: number;
+  returningCustomers: number;
+  agentBreakdown: Record<string, number>;
+  pendingReviews: number;
+}
+
+/** Message volume data point (for charts) */
+export interface MessageVolumeData {
+  date: string;
+  count: number;
+  channel?: string;
+}
+
+/** Per-agent stats */
+export interface AgentStatsData {
+  agentName: string;
+  messageCount: number;
+  avgResponseTimeMs: number;
+  avgConfidence: number;
+  escalationCount: number;
+  toolCallCount: number;
+}
+
+/** Per-customer stats */
+export interface CustomerStatsData {
+  customerId: string;
+  customerPhone: string;
+  messageCount: number;
+  firstSeen: number;
+  lastSeen: number;
+  isNew: boolean;
+}
+
+/** Knowledge Base stats from analytics */
+export interface KbStatsData {
+  totalQueries: number;
+  faqHits: number;
+  faqHitRate: number;
+  avgTopScore: number;
+  topFaqHits: Array<{ intent: string; count: number }>;
+  topMisses: Array<{ intent: string; count: number; avgScore: number }>;
+}
+
+/** A conversation session (gap-based) */
+export interface ConversationSession {
+  id: string;
+  customerId: string;
+  customerPhone: string;
+  channel: string;
+  agentId: string;
+  startedAt: number;
+  endedAt: number;
+  messageCount: number;
+  toolCallCount: number;
+  totalDurationMs: number;
+  totalCost: number;
+  resolved: boolean;
+  resolutionType: string | null;
+}
+
+/** Time range filter for queries */
+export interface TimeRange {
+  from: number;
+  to: number;
+}
+
+/** Store interface for analytics persistence */
+export interface AnalyticsStore {
+  initialize(): Promise<void>;
+  close(): Promise<void>;
+
+  /** Record a message event */
+  recordMessageEvent(event: AnalyticsEvent): Promise<void>;
+
+  /** Get or create a conversation session */
+  getOrCreateConversation(
+    customerPhone: string,
+    channel: string,
+    agentName: string,
+  ): Promise<string>;
+
+  /** Update conversation stats after a message */
+  updateConversationStats(
+    conversationId: string,
+    event: AnalyticsEvent,
+  ): Promise<void>;
+
+  /** Get aggregated summary for a time range */
+  getSummary(range?: TimeRange): Promise<AnalyticsSummary>;
+
+  /** Get message volume over time */
+  getMessageVolume(range?: TimeRange, groupBy?: 'day' | 'hour'): Promise<MessageVolumeData[]>;
+
+  /** Get per-agent stats */
+  getAgentStats(range?: TimeRange): Promise<AgentStatsData[]>;
+
+  /** Get per-customer stats */
+  getCustomerStats(range?: TimeRange, limit?: number): Promise<CustomerStatsData[]>;
+
+  /** Get KB-related stats */
+  getKbStats(range?: TimeRange): Promise<KbStatsData>;
+
+  /** Get top FAQ hits */
+  getTopFaqHits(limit?: number, range?: TimeRange): Promise<Array<{ intent: string; count: number }>>;
+
+  /** Get top misses (low KB score) */
+  getTopMisses(limit?: number, range?: TimeRange): Promise<Array<{ intent: string; count: number; avgScore: number }>>;
+}

package/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/tsdown.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'tsdown';
+
+export default defineConfig({
+  entry: ['src/index.ts'],
+  format: ['esm'],
+  dts: true,
+  clean: true,
+  sourcemap: true,
+  outExtensions: () => ({ js: '.js', dts: '.d.ts' }),
+});