@elevasis/core 0.1.0

package/src/operations/observability/__tests__/schemas.test.ts

@@ -0,0 +1,151 @@
+import { describe, it, expect } from 'vitest'
+import {
+  ErrorRecordSchema,
+  ErrorDetailFullSchema,
+  ErrorDistributionSchema,
+  ErrorTrendSchema
+} from '../schemas'
+
+describe('Error Tracking Schemas', () => {
+  describe('ErrorRecordSchema', () => {
+    it('validates valid error record', () => {
+      const validRecord = {
+        id: '123e4567-e89b-12d3-a456-426614174000',
+        timestamp: '2024-01-15T10:30:00Z',
+        errorType: 'ValidationError',
+        message: 'Invalid input provided',
+        executionId: '223e4567-e89b-12d3-a456-426614174000',
+        resourceId: 'workflow-123',
+        resourceName: 'Test Workflow',
+        severity: 'warning' as const
+      }
+
+      const result = ErrorRecordSchema.safeParse(validRecord)
+      expect(result.success).toBe(true)
+    })
+
+    it('rejects invalid severity', () => {
+      const invalidRecord = {
+        id: '123e4567-e89b-12d3-a456-426614174000',
+        timestamp: '2024-01-15T10:30:00Z',
+        errorType: 'ValidationError',
+        message: 'Invalid input provided',
+        executionId: '223e4567-e89b-12d3-a456-426614174000',
+        resourceId: 'workflow-123',
+        resourceName: 'Test Workflow',
+        severity: 'invalid' // Invalid severity
+      }
+
+      const result = ErrorRecordSchema.safeParse(invalidRecord)
+      expect(result.success).toBe(false)
+    })
+
+    it('rejects non-UUID id', () => {
+      const invalidRecord = {
+        id: 'not-a-uuid',
+        timestamp: '2024-01-15T10:30:00Z',
+        errorType: 'ValidationError',
+        message: 'Invalid input provided',
+        executionId: '223e4567-e89b-12d3-a456-426614174000',
+        resourceId: 'workflow-123',
+        resourceName: 'Test Workflow',
+        severity: 'warning' as const
+      }
+
+      const result = ErrorRecordSchema.safeParse(invalidRecord)
+      expect(result.success).toBe(false)
+    })
+  })
+
+  describe('ErrorDetailFullSchema', () => {
+    it('validates error detail with all optional fields', () => {
+      const validDetail = {
+        id: '123e4567-e89b-12d3-a456-426614174000',
+        timestamp: '2024-01-15T10:30:00Z',
+        errorType: 'ValidationError',
+        message: 'Invalid input provided',
+        executionId: '223e4567-e89b-12d3-a456-426614174000',
+        resourceId: 'workflow-123',
+        resourceName: 'Test Workflow',
+        severity: 'critical' as const,
+        stackTrace: 'Error: Test\n  at line 1',
+        retryAttempt: 2,
+        stepName: 'ValidateInput',
+        stepSequence: 1,
+        errorContext: { field: 'email', value: 'invalid' },
+        executionContext: { userId: '123', orgId: '456' }
+      }
+
+      const result = ErrorDetailFullSchema.safeParse(validDetail)
+      expect(result.success).toBe(true)
+    })
+
+    it('validates error detail without optional fields', () => {
+      const minimalDetail = {
+        id: '123e4567-e89b-12d3-a456-426614174000',
+        timestamp: '2024-01-15T10:30:00Z',
+        errorType: 'ValidationError',
+        message: 'Invalid input provided',
+        executionId: '223e4567-e89b-12d3-a456-426614174000',
+        resourceId: 'workflow-123',
+        resourceName: 'Test Workflow',
+        severity: 'info' as const
+      }
+
+      const result = ErrorDetailFullSchema.safeParse(minimalDetail)
+      expect(result.success).toBe(true)
+    })
+  })
+
+  describe('ErrorDistributionSchema', () => {
+    it('validates error distribution data', () => {
+      const validDistribution = {
+        label: 'ValidationError',
+        count: 42,
+        percentage: 35.5
+      }
+
+      const result = ErrorDistributionSchema.safeParse(validDistribution)
+      expect(result.success).toBe(true)
+    })
+
+    it('rejects missing fields', () => {
+      const invalidDistribution = {
+        label: 'ValidationError',
+        count: 42
+        // Missing percentage
+      }
+
+      const result = ErrorDistributionSchema.safeParse(invalidDistribution)
+      expect(result.success).toBe(false)
+    })
+  })
+
+  describe('ErrorTrendSchema', () => {
+    it('validates error trend data', () => {
+      const validTrend = {
+        time: '2024-01-15T10:00:00Z',
+        errorCount: 150,
+        criticalCount: 10,
+        warningCount: 50,
+        infoCount: 90
+      }
+
+      const result = ErrorTrendSchema.safeParse(validTrend)
+      expect(result.success).toBe(true)
+    })
+
+    it('rejects non-number counts', () => {
+      const invalidTrend = {
+        time: '2024-01-15T10:00:00Z',
+        errorCount: '150', // Should be number
+        criticalCount: 10,
+        warningCount: 50,
+        infoCount: 90
+      }
+
+      const result = ErrorTrendSchema.safeParse(invalidTrend)
+      expect(result.success).toBe(false)
+    })
+  })
+})

package/src/operations/observability/__tests__/types.test.ts

@@ -0,0 +1,109 @@
+import { describe, it, expect } from 'vitest'
+import type {
+  ErrorRecord,
+  ErrorDetailFull,
+  ErrorDetailResponse,
+  ErrorDistribution,
+  ErrorTrend,
+  FailingResource
+} from '../types'
+
+describe('Error Tracking Types', () => {
+  it('ErrorRecord type is correctly defined', () => {
+    const record: ErrorRecord = {
+      id: '123e4567-e89b-12d3-a456-426614174000',
+      timestamp: '2024-01-15T10:30:00Z',
+      errorType: 'ValidationError',
+      message: 'Invalid input',
+      executionId: '223e4567-e89b-12d3-a456-426614174000',
+      resourceId: 'workflow-123',
+      resourceName: 'Test Workflow',
+      severity: 'warning'
+    }
+
+    expect(record).toBeDefined()
+    expect(record.severity).toBe('warning')
+  })
+
+  it('ErrorDetailFull extends ErrorRecord with optional fields', () => {
+    const detail: ErrorDetailFull = {
+      id: '123e4567-e89b-12d3-a456-426614174000',
+      timestamp: '2024-01-15T10:30:00Z',
+      errorType: 'ValidationError',
+      message: 'Invalid input',
+      executionId: '223e4567-e89b-12d3-a456-426614174000',
+      resourceId: 'workflow-123',
+      resourceName: 'Test Workflow',
+      severity: 'critical',
+      stackTrace: 'Error stack...',
+      retryAttempt: 2,
+      stepName: 'ValidateInput',
+      stepSequence: 1,
+      errorContext: { field: 'email' },
+      executionContext: { userId: '123' }
+    }
+
+    expect(detail).toBeDefined()
+    expect(detail.stackTrace).toBe('Error stack...')
+  })
+
+  it('ErrorDetailResponse contains paginated error records', () => {
+    const response: ErrorDetailResponse = {
+      errors: [],
+      total: 100,
+      page: 1,
+      limit: 20
+    }
+
+    expect(response).toBeDefined()
+    expect(response.total).toBe(100)
+  })
+
+  it('ErrorDistribution contains analytics data', () => {
+    const distribution: ErrorDistribution = {
+      label: 'ValidationError',
+      count: 42,
+      percentage: 35.5
+    }
+
+    expect(distribution).toBeDefined()
+    expect(distribution.percentage).toBe(35.5)
+  })
+
+  it('ErrorTrend contains time-series data', () => {
+    const trend: ErrorTrend = {
+      time: '2024-01-15T10:00:00Z',
+      errorCount: 150,
+      criticalCount: 10,
+      warningCount: 50,
+      infoCount: 90
+    }
+
+    expect(trend).toBeDefined()
+    expect(trend.errorCount).toBe(150)
+  })
+
+  it('FailingResource contains resource health data', () => {
+    const resource: FailingResource = {
+      resourceId: 'workflow-123',
+      resourceName: 'Test Workflow',
+      errorCount: 25,
+      criticalCount: 5,
+      warningCount: 15,
+      mostCommonError: 'ValidationError'
+    }
+
+    expect(resource).toBeDefined()
+    expect(resource.mostCommonError).toBe('ValidationError')
+  })
+
+  it('severity type is correctly constrained', () => {
+    const validSeverities: Array<ErrorRecord['severity']> = [
+      'critical',
+      'warning',
+      'info'
+    ]
+
+    expect(validSeverities).toHaveLength(3)
+  })
+})

package/src/operations/observability/__tests__/utils.test.ts

@@ -0,0 +1,54 @@
+import { describe, it, expect } from 'vitest'
+import { formatBucketTime, getTimeRangeLabel, getTimeRangeDates } from '../utils'
+
+describe('formatBucketTime', () => {
+  it('formats hour granularity as HH:MM (24h)', () => {
+    // 2024-01-15T14:30:00Z => should produce a time string like "14:30" (locale-dependent)
+    const result = formatBucketTime('2024-01-15T14:30:00Z', 'hour')
+    // Verify it contains a colon (time format) and does not contain a month name
+    expect(result).toMatch(/\d{1,2}:\d{2}/)
+  })
+
+  it('formats day granularity as "Mon DD"', () => {
+    const result = formatBucketTime('2024-01-15T14:30:00Z', 'day')
+    // Should contain a short month and day number, e.g. "Jan 15"
+    expect(result).toMatch(/[A-Z][a-z]{2}\s+\d{1,2}/)
+  })
+
+  it('handles midnight timestamps for hour granularity', () => {
+    const result = formatBucketTime('2024-06-01T00:00:00Z', 'hour')
+    expect(result).toMatch(/\d{1,2}:\d{2}/)
+  })
+
+  it('handles different months for day granularity', () => {
+    const jan = formatBucketTime('2024-01-05T12:00:00Z', 'day')
+    const dec = formatBucketTime('2024-12-25T12:00:00Z', 'day')
+    // Both should have month abbreviations
+    expect(jan).toMatch(/[A-Z][a-z]{2}/)
+    expect(dec).toMatch(/[A-Z][a-z]{2}/)
+    // They should be different
+    expect(jan).not.toBe(dec)
+  })
+})
+
+describe('getTimeRangeLabel', () => {
+  it('returns correct labels for all time ranges', () => {
+    expect(getTimeRangeLabel('1h')).toBe('Last 1 hour')
+    expect(getTimeRangeLabel('24h')).toBe('Last 24 hours')
+    expect(getTimeRangeLabel('7d')).toBe('Last 7 days')
+    expect(getTimeRangeLabel('30d')).toBe('Last 30 days')
+  })
+})
+
+describe('getTimeRangeDates', () => {
+  it('returns ISO date strings', () => {
+    const result = getTimeRangeDates('24h')
+    expect(result.startDate).toMatch(/\d{4}-\d{2}-\d{2}T/)
+    expect(result.endDate).toMatch(/\d{4}-\d{2}-\d{2}T/)
+  })
+
+  it('returns start before end', () => {
+    const result = getTimeRangeDates('7d')
+    expect(new Date(result.startDate).getTime()).toBeLessThan(new Date(result.endDate).getTime())
+  })
+})

package/src/operations/observability/ai-usage-collector.ts

@@ -0,0 +1,64 @@
+import type { AIUsageSummary, AICallRecord, AICallContext, BaseAICall, LLMUsageData } from './types'
+import { calculateCost, LLMModel } from '../../execution/engine/llm/model-info'
+
+/**
+ * AIUsageCollector
+ * Centralized token tracking that aggregates usage across all LLM calls in an execution
+ */
+export class AIUsageCollector {
+  private model: LLMModel = 'gpt-5' // Default, will be overwritten on first record()
+  private calls: AICallRecord[] = []
+  private callSequence: number = 0
+
+  /**
+   * Record a single AI call with usage metrics
+   *
+   * @param usage - Token usage and latency data from LLM adapter
+   * @param callType - Type discriminator (agent-reasoning, tool, etc.)
+   * @param context - Optional typed context specific to callType
+   */
+  record(usage: LLMUsageData, callType: BaseAICall['callType'] = 'other', context?: AICallContext): void {
+    this.callSequence++
+    this.model = usage.model
+
+    // Use actual cost from provider (e.g., OpenRouter) if available, otherwise calculate
+    const costUsd = usage.cost ?? calculateCost(usage.model, usage.inputTokens, usage.outputTokens)
+
+    this.calls.push({
+      callSequence: this.callSequence,
+      callType,
+      model: usage.model,
+      inputTokens: usage.inputTokens,
+      outputTokens: usage.outputTokens,
+      costUsd,
+      latencyMs: usage.latencyMs,
+      context
+    })
+  }
+
+  /**
+   * Get aggregated summary of all AI calls
+   */
+  getSummary(): AIUsageSummary {
+    const totalInputTokens = this.calls.reduce((sum, c) => sum + c.inputTokens, 0)
+    const totalOutputTokens = this.calls.reduce((sum, c) => sum + c.outputTokens, 0)
+    const totalCostUsd = this.calls.reduce((sum, c) => sum + c.costUsd, 0)
+
+    return {
+      model: this.model,
+      totalInputTokens,
+      totalOutputTokens,
+      totalTokens: totalInputTokens + totalOutputTokens,
+      totalCostUsd,
+      callCount: this.calls.length,
+      calls: this.calls
+    }
+  }
+
+  /**
+   * Check if any usage has been recorded
+   */
+  hasUsage(): boolean {
+    return this.calls.length > 0
+  }
+}

package/src/operations/observability/index.ts

@@ -0,0 +1,13 @@
+/**
+ * Observability module exports
+ * Types, schemas, and collector classes for execution monitoring and metrics
+ */
+
+// Export all types
+export * from './types'
+
+// Export validation schemas
+export * from './schemas'
+
+// Export utility functions
+export * from './utils'

package/src/operations/observability/metrics-collector.ts

@@ -0,0 +1,49 @@
+import type { ExecutionMetricsSummary, ResourceMetricsConfig } from './types'
+
+/**
+ * MetricsCollector
+ * Tracks execution timing and ROI metrics
+ */
+export class MetricsCollector {
+  private timings: Map<string, number> = new Map()
+  private durationMs?: number
+
+  /**
+   * Start a timer with a label
+   */
+  startTimer(label: string): void {
+    this.timings.set(label, Date.now())
+  }
+
+  /**
+   * End a timer and calculate duration
+   * If label is 'execution', stores duration for metrics summary
+   */
+  endTimer(label: string): number | null {
+    const start = this.timings.get(label)
+    if (!start) return null
+
+    const duration = Date.now() - start
+
+    if (label === 'execution') {
+      this.durationMs = duration
+    }
+
+    this.timings.delete(label)
+    return duration
+  }
+
+  /**
+   * Build execution metrics summary with optional ROI calculation
+   */
+  buildExecutionMetrics(metricsConfig?: ResourceMetricsConfig): ExecutionMetricsSummary {
+    const automationSavingsUsd = metricsConfig
+      ? (metricsConfig.estimatedManualMinutes / 60) * metricsConfig.hourlyLaborRateUsd
+      : undefined
+
+    return {
+      durationMs: this.durationMs,
+      automationSavingsUsd
+    }
+  }
+}

package/src/operations/observability/schemas.ts

@@ -0,0 +1,39 @@
+import { z } from 'zod'
+
+// ============================================================================
+// Error Tracking Validation Schemas
+// ============================================================================
+
+export const ErrorRecordSchema = z.object({
+  id: z.string().uuid(),
+  timestamp: z.string(),
+  errorType: z.string(),
+  message: z.string(),
+  executionId: z.string().uuid(),
+  resourceId: z.string(),
+  resourceName: z.string(),
+  severity: z.enum(['critical', 'warning', 'info'])
+})
+
+export const ErrorDetailFullSchema = ErrorRecordSchema.extend({
+  stackTrace: z.string().optional(),
+  retryAttempt: z.number().optional(),
+  stepName: z.string().optional(),
+  stepSequence: z.number().optional(),
+  errorContext: z.record(z.string(), z.unknown()).optional(),
+  executionContext: z.record(z.string(), z.unknown()).optional()
+})
+
+export const ErrorDistributionSchema = z.object({
+  label: z.string(),
+  count: z.number(),
+  percentage: z.number()
+})
+
+export const ErrorTrendSchema = z.object({
+  time: z.string(),
+  errorCount: z.number(),
+  criticalCount: z.number(),
+  warningCount: z.number(),
+  infoCount: z.number()
+})