@snap-agent/analytics 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,625 @@
+import { PerformanceTimings, TokenMetrics, RAGMetrics, AnalyticsPlugin, RequestTrackingData, ResponseTrackingData, ErrorTrackingData } from '@snap-agent/core';
+export { ErrorTrackingData, PerformanceTimings, RAGMetrics, RequestTrackingData, ResponseTrackingData, TokenMetrics } from '@snap-agent/core';
+
+interface StoredRequest {
+    id: string;
+    agentId: string;
+    threadId?: string;
+    userId?: string;
+    timestamp: Date;
+    messageLength: number;
+    model?: string;
+    provider?: string;
+}
+interface StoredResponse {
+    id: string;
+    requestId: string;
+    agentId: string;
+    threadId?: string;
+    userId?: string;
+    timestamp: Date;
+    responseLength: number;
+    timings: PerformanceTimings;
+    tokens: TokenMetrics;
+    rag?: RAGMetrics;
+    success: boolean;
+    errorType?: string;
+    model?: string;
+    provider?: string;
+}
+interface StoredError {
+    id: string;
+    agentId: string;
+    threadId?: string;
+    timestamp: Date;
+    errorType: string;
+    errorMessage: string;
+    component?: string;
+}
+interface AnalyticsQueryOptions {
+    agentId?: string;
+    threadId?: string;
+    userId?: string;
+    startDate?: Date;
+    endDate?: Date;
+    limit?: number;
+    offset?: number;
+}
+interface AggregationOptions extends AnalyticsQueryOptions {
+    groupBy?: 'hour' | 'day' | 'week' | 'month';
+}
+/**
+ * Storage adapter interface for analytics data persistence.
+ * Implementations handle storage and retrieval of analytics metrics.
+ */
+interface AnalyticsStorage {
+    /**
+     * Save request records
+     */
+    saveRequests(requests: StoredRequest[]): Promise<void>;
+    /**
+     * Save response records
+     */
+    saveResponses(responses: StoredResponse[]): Promise<void>;
+    /**
+     * Save error records
+     */
+    saveErrors(errors: StoredError[]): Promise<void>;
+    /**
+     * Get requests with optional filtering
+     */
+    getRequests(options?: AnalyticsQueryOptions): Promise<StoredRequest[]>;
+    /**
+     * Get responses with optional filtering
+     */
+    getResponses(options?: AnalyticsQueryOptions): Promise<StoredResponse[]>;
+    /**
+     * Get errors with optional filtering
+     */
+    getErrors(options?: AnalyticsQueryOptions): Promise<StoredError[]>;
+    /**
+     * Get aggregated request count by time period
+     */
+    getRequestCount(options?: AggregationOptions): Promise<number>;
+    /**
+     * Get aggregated response count by time period
+     */
+    getResponseCount(options?: AggregationOptions): Promise<number>;
+    /**
+     * Get aggregated error count by time period
+     */
+    getErrorCount(options?: AggregationOptions): Promise<number>;
+    /**
+     * Delete records older than the specified date
+     */
+    deleteOlderThan(date: Date): Promise<{
+        requests: number;
+        responses: number;
+        errors: number;
+    }>;
+    /**
+     * Clear all analytics data
+     */
+    clear(): Promise<void>;
+    /**
+     * Check if storage is connected and ready
+     */
+    isReady(): Promise<boolean>;
+    /**
+     * Close the storage connection (if applicable)
+     */
+    close(): Promise<void>;
+}
+
+/**
+ * In-memory analytics storage implementation.
+ * Suitable for development, testing, or short-lived processes.
+ * Data is lost when the process terminates.
+ */
+declare class MemoryAnalyticsStorage implements AnalyticsStorage {
+    private requests;
+    private responses;
+    private errors;
+    saveRequests(requests: StoredRequest[]): Promise<void>;
+    saveResponses(responses: StoredResponse[]): Promise<void>;
+    saveErrors(errors: StoredError[]): Promise<void>;
+    getRequests(options?: AnalyticsQueryOptions): Promise<StoredRequest[]>;
+    getResponses(options?: AnalyticsQueryOptions): Promise<StoredResponse[]>;
+    getErrors(options?: AnalyticsQueryOptions): Promise<StoredError[]>;
+    getRequestCount(options?: AggregationOptions): Promise<number>;
+    getResponseCount(options?: AggregationOptions): Promise<number>;
+    getErrorCount(options?: AggregationOptions): Promise<number>;
+    deleteOlderThan(date: Date): Promise<{
+        requests: number;
+        responses: number;
+        errors: number;
+    }>;
+    clear(): Promise<void>;
+    isReady(): Promise<boolean>;
+    close(): Promise<void>;
+    /**
+     * Filter records based on query options
+     */
+    private filterRecords;
+    /**
+     * Get current storage stats (for debugging)
+     */
+    getStats(): {
+        requests: number;
+        responses: number;
+        errors: number;
+    };
+}
+
+interface MongoAnalyticsStorageConfig {
+    /**
+     * MongoDB connection URI
+     */
+    uri: string;
+    /**
+     * Database name
+     * @default 'snap-agent-analytics'
+     */
+    database?: string;
+    /**
+     * Collection prefix
+     * @default 'analytics'
+     */
+    collectionPrefix?: string;
+    /**
+     * Create indexes on first connection
+     * @default true
+     */
+    createIndexes?: boolean;
+}
+/**
+ * MongoDB analytics storage implementation.
+ * Provides persistent storage with efficient querying and aggregation.
+ */
+declare class MongoAnalyticsStorage implements AnalyticsStorage {
+    private client;
+    private db;
+    private requestsCollection;
+    private responsesCollection;
+    private errorsCollection;
+    private readonly config;
+    private connectionPromise;
+    constructor(config: MongoAnalyticsStorageConfig);
+    /**
+     * Ensure connection to MongoDB
+     */
+    private ensureConnection;
+    private connect;
+    private createIndexes;
+    saveRequests(requests: StoredRequest[]): Promise<void>;
+    saveResponses(responses: StoredResponse[]): Promise<void>;
+    saveErrors(errors: StoredError[]): Promise<void>;
+    getRequests(options?: AnalyticsQueryOptions): Promise<StoredRequest[]>;
+    getResponses(options?: AnalyticsQueryOptions): Promise<StoredResponse[]>;
+    getErrors(options?: AnalyticsQueryOptions): Promise<StoredError[]>;
+    getRequestCount(options?: AggregationOptions): Promise<number>;
+    getResponseCount(options?: AggregationOptions): Promise<number>;
+    getErrorCount(options?: AggregationOptions): Promise<number>;
+    deleteOlderThan(date: Date): Promise<{
+        requests: number;
+        responses: number;
+        errors: number;
+    }>;
+    clear(): Promise<void>;
+    isReady(): Promise<boolean>;
+    close(): Promise<void>;
+    /**
+     * Build MongoDB query from options
+     */
+    private buildQuery;
+    /**
+     * Get aggregated metrics with grouping by time period
+     */
+    getAggregatedMetrics(options?: AggregationOptions): Promise<{
+        totalRequests: number;
+        totalResponses: number;
+        totalErrors: number;
+        avgLatency: number;
+        totalTokens: number;
+        errorRate: number;
+    }>;
+}
+
+interface AnalyticsConfig {
+    /**
+     * Enable performance metrics
+     * @default true
+     */
+    enablePerformance?: boolean;
+    /**
+     * Enable RAG metrics
+     * @default true
+     */
+    enableRAG?: boolean;
+    /**
+     * Enable cost tracking
+     * @default true
+     */
+    enableCost?: boolean;
+    /**
+     * Enable conversation metrics
+     * @default true
+     */
+    enableConversation?: boolean;
+    /**
+     * Enable error tracking
+     * @default true
+     */
+    enableErrors?: boolean;
+    /**
+     * Cost per 1K tokens by model (for cost calculation)
+     * Format: { 'gpt-4o': { input: 0.005, output: 0.015 } }
+     */
+    modelCosts?: Record<string, {
+        input: number;
+        output: number;
+    }>;
+    /**
+     * Cost per 1K embedding tokens
+     * @default 0.0001
+     */
+    embeddingCost?: number;
+    /**
+     * Data retention in days (0 = forever)
+     * @default 30
+     */
+    retentionDays?: number;
+    /**
+     * Flush interval in ms (for batched writes)
+     * @default 5000
+     */
+    flushInterval?: number;
+    /**
+     * Custom event handler for real-time metrics
+     */
+    onMetric?: (metric: MetricEvent) => void;
+    /**
+     * Analytics storage adapter for persistent storage.
+     * If not provided, uses in-memory storage (data lost on restart).
+     *
+     * @example
+     * // Use MongoDB storage
+     * import { MongoAnalyticsStorage } from '@snap-agent/analytics/storage';
+     *
+     * const analytics = new SnapAgentAnalytics({
+     *   storage: new MongoAnalyticsStorage({ uri: process.env.MONGODB_URI! }),
+     * });
+     */
+    storage?: AnalyticsStorage;
+}
+interface MetricEvent {
+    type: 'request' | 'response' | 'error';
+    timestamp: Date;
+    data: RequestTrackingData | ResponseTrackingData | ErrorTrackingData;
+}
+/**
+ * Data passed to onFlush callback
+ */
+interface FlushData {
+    requests: StoredRequest[];
+    responses: StoredResponse[];
+    errors: StoredError[];
+    timestamp: Date;
+}
+/**
+ * Performance metrics aggregation
+ */
+interface PerformanceMetrics {
+    totalRequests: number;
+    avgLatency: number;
+    p50Latency: number;
+    p95Latency: number;
+    p99Latency: number;
+    minLatency: number;
+    maxLatency: number;
+    avgLLMTime: number;
+    avgRAGTime: number;
+    avgPluginTime: number;
+    avgDbTime: number;
+    avgTimeToFirstToken: number;
+    avgTimeToLastToken: number;
+    latencyDistribution: {
+        under100ms: number;
+        under500ms: number;
+        under1s: number;
+        under5s: number;
+        over5s: number;
+    };
+}
+/**
+ * RAG metrics aggregation
+ */
+interface RAGAnalyticsMetrics {
+    totalQueries: number;
+    avgDocumentsRetrieved: number;
+    avgVectorSearchTime: number;
+    avgEmbeddingTime: number;
+    cacheHitRate: number;
+    cacheMissRate: number;
+    avgSimilarityScore: number;
+    avgRerankTime: number;
+    avgContextLength: number;
+    avgContextTokens: number;
+    avgSourcesCount: number;
+    retrievalSuccessRate: number;
+}
+/**
+ * Cost metrics aggregation
+ */
+interface CostMetrics {
+    totalCost: number;
+    totalTokens: number;
+    totalPromptTokens: number;
+    totalCompletionTokens: number;
+    avgTokensPerRequest: number;
+    avgCostPerRequest: number;
+    tokenEfficiency: number;
+    costByModel: Record<string, number>;
+    costByAgent: Record<string, number>;
+    tokensByModel: Record<string, number>;
+    totalEmbeddingTokens: number;
+    totalEmbeddingCost: number;
+    dailyCosts: Record<string, number>;
+}
+/**
+ * Conversation quality metrics
+ */
+interface ConversationMetrics {
+    totalThreads: number;
+    totalMessages: number;
+    avgMessagesPerThread: number;
+    avgThreadDuration: number;
+    avgSessionLength: number;
+    userReturnRate: number;
+    threadAbandonmentRate: number;
+    avgInputLength: number;
+    avgOutputLength: number;
+    inputLengthDistribution: {
+        short: number;
+        medium: number;
+        long: number;
+        veryLong: number;
+    };
+}
+/**
+ * Error metrics aggregation
+ */
+interface ErrorMetrics {
+    totalErrors: number;
+    errorRate: number;
+    errorsByType: Record<string, number>;
+    llmErrors: number;
+    ragErrors: number;
+    pluginErrors: number;
+    dbErrors: number;
+    networkErrors: number;
+    timeoutErrors: number;
+    rateLimitHits: number;
+    successRate: number;
+    retryCount: number;
+    fallbackUsage: number;
+    recentErrors: Array<{
+        timestamp: Date;
+        type: string;
+        message: string;
+        agentId: string;
+    }>;
+}
+/**
+ * All metrics aggregated
+ */
+interface AggregatedMetrics {
+    period: {
+        start: Date;
+        end: Date;
+    };
+    performance: PerformanceMetrics;
+    rag: RAGAnalyticsMetrics;
+    cost: CostMetrics;
+    conversation: ConversationMetrics;
+    errors: ErrorMetrics;
+}
+/**
+ * Time series data point
+ */
+interface TimeSeriesMetric {
+    timestamp: Date;
+    value: number;
+    metadata?: Record<string, any>;
+}
+/**
+ * Percentile metrics helper
+ */
+interface PercentileMetrics {
+    p50: number;
+    p75: number;
+    p90: number;
+    p95: number;
+    p99: number;
+}
+/**
+ * SnapAgent Analytics Plugin
+ *
+ * Comprehensive analytics tracking for AI agents with 5 metric categories:
+ * 1. Performance Metrics - Latency, timing breakdown, percentiles
+ * 2. RAG Metrics - Retrieval stats, cache rates, similarity scores
+ * 3. Cost & Token Metrics - Usage tracking, cost calculation
+ * 4. Conversation Metrics - Engagement, session quality
+ * 5. Error Metrics - Error rates, reliability stats
+ */
+declare class SnapAgentAnalytics implements AnalyticsPlugin {
+    name: string;
+    type: "analytics";
+    private config;
+    private storage;
+    private requests;
+    private responses;
+    private errors;
+    private threadStats;
+    private userSessions;
+    private idCounter;
+    private flushTimer?;
+    private pendingRequests;
+    private pendingResponses;
+    private pendingErrors;
+    private isFlushing;
+    constructor(config?: AnalyticsConfig);
+    /**
+     * Track incoming request (basic)
+     */
+    trackRequest(data: {
+        agentId: string;
+        threadId?: string;
+        message: string;
+        timestamp: Date;
+    }): Promise<void>;
+    /**
+     * Track response (basic)
+     */
+    trackResponse(data: {
+        agentId: string;
+        threadId?: string;
+        response: string;
+        latency: number;
+        tokensUsed?: number;
+        timestamp: Date;
+    }): Promise<void>;
+    /**
+     * Track request with full metadata
+     */
+    trackRequestExtended(data: RequestTrackingData): Promise<void>;
+    /**
+     * Track response with full metrics
+     */
+    trackResponseExtended(data: ResponseTrackingData): Promise<void>;
+    /**
+     * Track errors
+     */
+    trackError(data: ErrorTrackingData): Promise<void>;
+    /**
+     * Get aggregated metrics
+     */
+    getMetrics(options?: {
+        agentId?: string;
+        startDate?: Date;
+        endDate?: Date;
+        groupBy?: 'hour' | 'day' | 'week' | 'month';
+    }): Promise<AggregatedMetrics>;
+    /**
+     * Get performance metrics
+     */
+    getPerformanceMetrics(options?: {
+        agentId?: string;
+        startDate?: Date;
+        endDate?: Date;
+    }): PerformanceMetrics;
+    /**
+     * Get RAG metrics
+     */
+    getRAGMetrics(options?: {
+        agentId?: string;
+        startDate?: Date;
+        endDate?: Date;
+    }): RAGAnalyticsMetrics;
+    /**
+     * Get cost metrics
+     */
+    getCostMetrics(options?: {
+        agentId?: string;
+        startDate?: Date;
+        endDate?: Date;
+    }): CostMetrics;
+    /**
+     * Get conversation metrics
+     */
+    getConversationMetrics(options?: {
+        agentId?: string;
+        startDate?: Date;
+        endDate?: Date;
+    }): ConversationMetrics;
+    /**
+     * Get error metrics
+     */
+    getErrorMetrics(options?: {
+        agentId?: string;
+        startDate?: Date;
+        endDate?: Date;
+    }): ErrorMetrics;
+    /**
+     * Get time series data
+     */
+    getTimeSeries(metric: 'latency' | 'tokens' | 'cost' | 'errors' | 'requests', options?: {
+        agentId?: string;
+        startDate?: Date;
+        endDate?: Date;
+        groupBy?: 'hour' | 'day' | 'week';
+    }): TimeSeriesMetric[];
+    private calculatePerformanceMetrics;
+    private calculateRAGMetrics;
+    private calculateCostMetrics;
+    private calculateConversationMetrics;
+    private calculateErrorMetrics;
+    private calculateCost;
+    private updateThreadStats;
+    private trackUserSession;
+    private filterResponses;
+    private filterRequests;
+    private filterErrors;
+    private getTimeKey;
+    private avg;
+    private percentile;
+    private cleanup;
+    private emptyPerformanceMetrics;
+    private emptyRAGMetrics;
+    private emptyCostMetrics;
+    private emptyConversationMetrics;
+    /**
+     * Get raw data for export
+     */
+    exportData(): {
+        requests: StoredRequest[];
+        responses: StoredResponse[];
+        errors: StoredError[];
+    };
+    /**
+     * Clear all analytics data
+     */
+    clear(): void;
+    /**
+     * Get summary statistics
+     */
+    getSummary(): Record<string, any>;
+    /**
+     * Start the flush timer
+     */
+    private startFlushTimer;
+    /**
+     * Stop the flush timer
+     */
+    private stopFlushTimer;
+    /**
+     * Flush pending data to external storage via onFlush callback
+     * @returns Promise that resolves when flush is complete
+     */
+    flush(): Promise<void>;
+    /**
+     * Stop the analytics plugin and flush remaining data
+     * Call this before shutting down to ensure all data is persisted
+     */
+    stop(): Promise<void>;
+    /**
+     * Get count of pending (unflushed) items
+     */
+    getPendingCount(): {
+        requests: number;
+        responses: number;
+        errors: number;
+    };
+}
+
+export { type AggregatedMetrics, type AggregationOptions, type AnalyticsConfig, type AnalyticsQueryOptions, type AnalyticsStorage, type ConversationMetrics, type CostMetrics, type ErrorMetrics, type FlushData, MemoryAnalyticsStorage, type MetricEvent, MongoAnalyticsStorage, type MongoAnalyticsStorageConfig, type PercentileMetrics, type PerformanceMetrics, type RAGAnalyticsMetrics, SnapAgentAnalytics, type StoredError, type StoredRequest, type StoredResponse, type TimeSeriesMetric };