@operor/analytics 0.1.0

package/src/AnalyticsCollector.test.ts

@@ -0,0 +1,344 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { AnalyticsCollector } from './AnalyticsCollector.js';
+import { InMemoryAnalyticsStore } from './InMemoryAnalyticsStore.js';
+
+/** Minimal emitter mock matching the Emitter interface */
+function createEmitter() {
+  const handlers = new Map<string, Set<Function>>();
+  return {
+    on(event: string, handler: Function) {
+      if (!handlers.has(event)) handlers.set(event, new Set());
+      handlers.get(event)!.add(handler);
+    },
+    off(event: string, handler: Function) {
+      handlers.get(event)?.delete(handler);
+    },
+    emit(event: string, ...args: any[]) {
+      for (const h of handlers.get(event) ?? []) h(...args);
+    },
+    listenerCount(event: string) {
+      return handlers.get(event)?.size ?? 0;
+    },
+  };
+}
+
+function makePayload(overrides: Record<string, any> = {}) {
+  return {
+    message: {
+      id: 'msg_1',
+      from: '+1234567890',
+      text: 'Where is my order?',
+      timestamp: Date.now(),
+      channel: 'whatsapp',
+      provider: 'whatsapp',
+    },
+    response: {
+      text: 'Your order is on the way!',
+      duration: 350,
+      cost: 0.003,
+      toolCalls: [],
+      metadata: {
+        kbTopScore: 0.88,
+        kbIsFaqMatch: true,
+        kbResultCount: 2,
+        promptTokens: 150,
+        completionTokens: 60,
+      },
+    },
+    agent: 'support',
+    intent: { intent: 'order_status', confidence: 0.95 },
+    customer: { id: 'cust_1', phone: '+1234567890' },
+    duration: 350,
+    cost: 0.003,
+    ...overrides,
+  };
+}
+
+describe('AnalyticsCollector', () => {
+  let store: InMemoryAnalyticsStore;
+  let collector: AnalyticsCollector;
+  let emitter: ReturnType<typeof createEmitter>;
+
+  beforeEach(async () => {
+    store = new InMemoryAnalyticsStore();
+    await store.initialize();
+    collector = new AnalyticsCollector(store);
+    emitter = createEmitter();
+  });
+
+  // --- attach / detach ---
+
+  describe('attach and detach', () => {
+    it('registers message:processed and error handlers', () => {
+      collector.attach(emitter);
+      expect(emitter.listenerCount('message:processed')).toBe(1);
+      expect(emitter.listenerCount('error')).toBe(1);
+    });
+
+    it('removes handlers on detach', () => {
+      collector.attach(emitter);
+      collector.detach(emitter);
+      expect(emitter.listenerCount('message:processed')).toBe(0);
+      expect(emitter.listenerCount('error')).toBe(0);
+    });
+
+    it('can attach to multiple emitters', () => {
+      const emitter2 = createEmitter();
+      collector.attach(emitter);
+      // Attaching again replaces internal handler refs, but both emitters have listeners
+      collector.attach(emitter2);
+      expect(emitter2.listenerCount('message:processed')).toBe(1);
+    });
+  });
+
+  // --- Event recording ---
+
+  describe('event recording', () => {
+    it('records a message:processed event', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload());
+      // Wait for async handler
+      await new Promise(r => setTimeout(r, 50));
+
+      const summary = await store.getSummary();
+      expect(summary.totalMessages).toBe(1);
+    });
+
+    it('extracts agent name correctly', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({ agent: 'billing' }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const stats = await store.getAgentStats();
+      expect(stats[0].agentName).toBe('billing');
+    });
+
+    it('extracts intent correctly', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        intent: { intent: 'refund_request', confidence: 0.8 },
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const kb = await store.getKbStats();
+      expect(kb.totalQueries).toBe(1);
+    });
+
+    it('extracts KB metadata from response.metadata', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        response: {
+          text: 'Answer',
+          duration: 200,
+          metadata: { kbTopScore: 0.92, kbIsFaqMatch: true, kbResultCount: 5 },
+        },
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const kb = await store.getKbStats();
+      expect(kb.faqHits).toBe(1);
+    });
+
+    it('handles missing metadata gracefully', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        response: { text: 'Answer', duration: 200 },
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const summary = await store.getSummary();
+      expect(summary.totalMessages).toBe(1);
+    });
+
+    it('extracts tool calls from response.toolCalls', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        response: {
+          text: 'Done',
+          duration: 500,
+          toolCalls: [
+            { name: 'get_order', params: { id: '123' } },
+            { name: 'track_shipment', params: { id: '456' } },
+          ],
+          metadata: {},
+        },
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const stats = await store.getAgentStats();
+      expect(stats[0].toolCallCount).toBe(2);
+    });
+
+    it('falls back to payload.toolCalls when response.toolCalls is missing', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        response: { text: 'Done', duration: 500, metadata: {} },
+        toolCalls: [{ name: 'get_order', params: {} }],
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const stats = await store.getAgentStats();
+      expect(stats[0].toolCallCount).toBe(1);
+    });
+
+    it('extracts message and response lengths', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        message: { id: '1', from: '+1', text: 'Hello', timestamp: Date.now(), channel: 'whatsapp', provider: 'whatsapp' },
+        response: { text: 'Hi there, how can I help?', duration: 100, metadata: {} },
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const summary = await store.getSummary();
+      expect(summary.totalMessages).toBe(1);
+    });
+
+    it('creates conversation session for each event', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload());
+      emitter.emit('message:processed', makePayload());
+      await new Promise(r => setTimeout(r, 50));
+
+      const summary = await store.getSummary();
+      expect(summary.totalMessages).toBe(2);
+    });
+
+    it('detects new customer when firstInteraction is very recent', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        customer: { id: 'new_cust', phone: '+999', firstInteraction: new Date() },
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const summary = await store.getSummary();
+      expect(summary.newCustomers).toBe(1);
+    });
+
+    it('does not mark as new customer when firstInteraction is old', async () => {
+      collector.attach(emitter);
+      const oldDate = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); // 7 days ago
+      emitter.emit('message:processed', makePayload({
+        customer: { id: 'old_cust', phone: '+888', firstInteraction: oldDate },
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const summary = await store.getSummary();
+      expect(summary.newCustomers).toBe(0);
+    });
+
+    it('uses response.duration, falls back to payload.duration', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        response: { text: 'Hi', duration: 0, metadata: {} },
+        duration: 999,
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      // Should not crash — duration extracted from one of the sources
+      const summary = await store.getSummary();
+      expect(summary.totalMessages).toBe(1);
+    });
+
+    it('uses response.cost, falls back to payload.cost', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        response: { text: 'Hi', duration: 100, cost: undefined, metadata: {} },
+        cost: 0.005,
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const summary = await store.getSummary();
+      expect(summary.totalMessages).toBe(1);
+    });
+  });
+
+  // --- Error resilience ---
+
+  describe('error resilience', () => {
+    it('does not throw when store.recordMessageEvent fails', async () => {
+      const failStore = new InMemoryAnalyticsStore();
+      await failStore.initialize();
+      vi.spyOn(failStore, 'recordMessageEvent').mockRejectedValue(new Error('DB write failed'));
+
+      const failCollector = new AnalyticsCollector(failStore, { debug: true });
+      failCollector.attach(emitter);
+
+      // Should not throw
+      emitter.emit('message:processed', makePayload());
+      await new Promise(r => setTimeout(r, 50));
+    });
+
+    it('does not throw when store.getOrCreateConversation fails', async () => {
+      const failStore = new InMemoryAnalyticsStore();
+      await failStore.initialize();
+      vi.spyOn(failStore, 'getOrCreateConversation').mockRejectedValue(new Error('DB read failed'));
+
+      const failCollector = new AnalyticsCollector(failStore, { debug: true });
+      failCollector.attach(emitter);
+
+      emitter.emit('message:processed', makePayload());
+      await new Promise(r => setTimeout(r, 50));
+      // No crash
+    });
+
+    it('handles error events without crashing', async () => {
+      collector.attach(emitter);
+      emitter.emit('error', { error: new Error('Something broke'), message: { text: 'test' } });
+      await new Promise(r => setTimeout(r, 50));
+      // No crash, no events recorded
+      const summary = await store.getSummary();
+      expect(summary.totalMessages).toBe(0);
+    });
+
+    it('handles malformed payload gracefully (missing message.text)', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        message: { id: '1', from: '+1', text: undefined, timestamp: Date.now(), channel: 'whatsapp', provider: 'whatsapp' },
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const summary = await store.getSummary();
+      expect(summary.totalMessages).toBe(1);
+    });
+
+    it('handles missing customer.firstInteraction', async () => {
+      collector.attach(emitter);
+      emitter.emit('message:processed', makePayload({
+        customer: { id: 'cust_x', phone: '+1' },
+      }));
+      await new Promise(r => setTimeout(r, 50));
+
+      const summary = await store.getSummary();
+      expect(summary.totalMessages).toBe(1);
+      expect(summary.newCustomers).toBe(0);
+    });
+  });
+
+  // --- Debug mode ---
+
+  describe('debug mode', () => {
+    it('logs when debug is enabled', async () => {
+      const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+      const debugCollector = new AnalyticsCollector(store, { debug: true });
+      debugCollector.attach(emitter);
+
+      emitter.emit('message:processed', makePayload());
+      await new Promise(r => setTimeout(r, 50));
+
+      expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('[Analytics]'));
+      logSpy.mockRestore();
+    });
+
+    it('does not log when debug is disabled', async () => {
+      const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+      collector.attach(emitter);
+
+      emitter.emit('message:processed', makePayload());
+      await new Promise(r => setTimeout(r, 50));
+
+      const analyticsCalls = logSpy.mock.calls.filter(c => String(c[0]).includes('[Analytics]'));
+      expect(analyticsCalls).toHaveLength(0);
+      logSpy.mockRestore();
+    });
+  });
+});

package/src/AnalyticsCollector.ts

@@ -0,0 +1,148 @@
+import type { AnalyticsStore, AnalyticsEvent } from './types.js';
+
+/**
+ * Event payload from Operor 'message:processed' event.
+ * Kept loose to avoid a hard dependency on @operor/core.
+ */
+interface MessageProcessedPayload {
+  message: {
+    id?: string;
+    from: string;
+    text: string;
+    timestamp: number;
+    channel: string;
+    provider: string;
+  };
+  response: {
+    text: string;
+    duration: number;
+    cost?: number;
+    toolCalls?: Array<{ name: string; params: any; result?: any; success?: boolean }>;
+    metadata?: Record<string, any>;
+  };
+  agent: string;
+  intent: {
+    intent: string;
+    confidence: number;
+  };
+  customer: {
+    id: string;
+    phone?: string;
+    firstInteraction?: Date;
+  };
+  toolCalls?: Array<{ name: string; params: any }>;
+  duration: number;
+  cost?: number;
+}
+
+/** Minimal EventEmitter interface — matches eventemitter3 used by Operor */
+interface Emitter {
+  on(event: string, handler: (...args: any[]) => void): void;
+  off(event: string, handler: (...args: any[]) => void): void;
+}
+
+/**
+ * Listens to Operor events and persists analytics data.
+ */
+export class AnalyticsCollector {
+  private store: AnalyticsStore;
+  private debug: boolean;
+  private boundHandlers = new Map<string, (...args: any[]) => void>();
+
+  constructor(store: AnalyticsStore, options?: { debug?: boolean }) {
+    this.store = store;
+    this.debug = options?.debug ?? false;
+  }
+
+  /**
+   * Attach to an Operor instance (or any EventEmitter with the same events).
+   */
+  attach(os: Emitter): void {
+    const onProcessed = (payload: MessageProcessedPayload) => {
+      this.handleProcessed(payload).catch(err => {
+        if (this.debug) console.warn('[Analytics] Error recording event:', err?.message);
+      });
+    };
+
+    const onError = (payload: { error: any; message?: any }) => {
+      this.handleError(payload).catch(err => {
+        if (this.debug) console.warn('[Analytics] Error recording error event:', err?.message);
+      });
+    };
+
+    os.on('message:processed', onProcessed);
+    os.on('error', onError);
+
+    this.boundHandlers.set('message:processed', onProcessed);
+    this.boundHandlers.set('error', onError);
+  }
+
+  /**
+   * Detach from an Operor instance.
+   */
+  detach(os: Emitter): void {
+    for (const [event, handler] of this.boundHandlers) {
+      os.off(event, handler);
+    }
+    this.boundHandlers.clear();
+  }
+
+  private async handleProcessed(payload: MessageProcessedPayload): Promise<void> {
+    const { message, response, agent, intent, customer } = payload;
+
+    const toolCalls = response.toolCalls || payload.toolCalls || [];
+    const meta = response.metadata || {};
+
+    // Detect if this is a new customer (first interaction is very recent)
+    const isNew = customer.firstInteraction
+      ? (Date.now() - customer.firstInteraction.getTime()) < 60_000
+      : false;
+
+    // Get or create conversation session
+    const conversationId = await this.store.getOrCreateConversation(
+      message.from,
+      message.channel || message.provider,
+      agent,
+    );
+
+    const event: AnalyticsEvent = {
+      timestamp: message.timestamp || Date.now(),
+      channel: message.channel || message.provider,
+      customerPhone: message.from,
+      customerId: customer.id,
+      agentName: agent,
+      intent: intent.intent,
+      intentConfidence: intent.confidence,
+      responseTimeMs: response.duration || payload.duration || 0,
+      llmCost: response.cost ?? payload.cost ?? 0,
+      kbTopScore: meta.kbTopScore ?? 0,
+      kbIsFaqMatch: meta.kbIsFaqMatch ?? false,
+      kbResultCount: meta.kbResultCount ?? 0,
+      toolCallsCount: toolCalls.length,
+      toolCallsJson: toolCalls.length > 0 ? JSON.stringify(toolCalls) : null,
+      messageLength: message.text?.length ?? 0,
+      responseLength: response.text?.length ?? 0,
+      isNewCustomer: isNew,
+      wasEscalated: false, // set externally if escalation detected
+      conversationId,
+      promptTokens: meta.promptTokens ?? 0,
+      completionTokens: meta.completionTokens ?? 0,
+    };
+
+    await this.store.recordMessageEvent(event);
+    await this.store.updateConversationStats(conversationId, event);
+
+    if (this.debug) {
+      this.log(`Recorded event: agent=${agent} customer=${message.from} intent=${intent.intent}`);
+    }
+  }
+
+  private async handleError(_payload: { error: any; message?: any }): Promise<void> {
+    // Error events don't have enough structured data to record as full events.
+    // Future: could track error counts in daily_metrics.
+  }
+
+  private log(msg: string): void {
+    console.log(`[Analytics] ${msg}`);
+  }
+}