@dotsetlabs/bellwether 0.10.0

package/dist/metrics/prometheus.js

@@ -0,0 +1,241 @@
+/**
+ * Prometheus metrics exporter.
+ *
+ * Exports metrics in Prometheus text format for scraping.
+ */
+/**
+ * Default histogram buckets for latency metrics (in ms).
+ */
+const DEFAULT_LATENCY_BUCKETS = [10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000];
+/**
+ * Metric definitions for Prometheus export.
+ */
+const METRIC_DEFINITIONS = [
+    // Token usage metrics
+    {
+        name: 'bellwether_llm_tokens_total',
+        type: 'counter',
+        help: 'Total number of tokens used',
+        labels: ['provider', 'model', 'direction'],
+    },
+    {
+        name: 'bellwether_llm_calls_total',
+        type: 'counter',
+        help: 'Total number of LLM API calls',
+        labels: ['provider', 'model'],
+    },
+    // Cost metrics
+    {
+        name: 'bellwether_cost_usd_total',
+        type: 'counter',
+        help: 'Total cost in USD',
+        labels: ['provider'],
+    },
+    // Operation timing metrics
+    {
+        name: 'bellwether_operation_duration_seconds',
+        type: 'histogram',
+        help: 'Duration of operations in seconds',
+        labels: ['operation', 'status'],
+    },
+    {
+        name: 'bellwether_operation_total',
+        type: 'counter',
+        help: 'Total number of operations',
+        labels: ['operation', 'status'],
+    },
+    // Error metrics
+    {
+        name: 'bellwether_errors_total',
+        type: 'counter',
+        help: 'Total number of errors',
+        labels: ['category', 'retryable'],
+    },
+    // Interview metrics
+    {
+        name: 'bellwether_interview_tools_discovered',
+        type: 'gauge',
+        help: 'Number of tools discovered in current interview',
+    },
+    {
+        name: 'bellwether_interview_questions_generated',
+        type: 'gauge',
+        help: 'Number of questions generated in current interview',
+    },
+    {
+        name: 'bellwether_interview_tool_calls_total',
+        type: 'counter',
+        help: 'Total tool calls in current interview',
+        labels: ['status'],
+    },
+    {
+        name: 'bellwether_interview_duration_seconds',
+        type: 'gauge',
+        help: 'Duration of current/last interview in seconds',
+    },
+    {
+        name: 'bellwether_interview_progress_ratio',
+        type: 'gauge',
+        help: 'Progress of current interview (0-1)',
+    },
+];
+/**
+ * Export metrics in Prometheus text format.
+ */
+export function exportPrometheusMetrics(collector) {
+    const lines = [];
+    const aggregated = collector.getAggregatedMetrics();
+    const interview = collector.getInterviewMetrics();
+    // Helper to add metric header
+    const addHeader = (metric) => {
+        lines.push(`# HELP ${metric.name} ${metric.help}`);
+        lines.push(`# TYPE ${metric.name} ${metric.type}`);
+    };
+    // Helper to format labels
+    const formatLabels = (labels) => {
+        if (!labels || Object.keys(labels).length === 0)
+            return '';
+        const pairs = Object.entries(labels).map(([k, v]) => `${k}="${escapeLabel(v)}"`);
+        return `{${pairs.join(',')}}`;
+    };
+    // Helper to add metric value
+    const addValue = (name, value, labels) => {
+        lines.push(`${name}${formatLabels(labels)} ${value}`);
+    };
+    // Token usage metrics
+    const tokenMetric = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_llm_tokens_total');
+    addHeader(tokenMetric);
+    for (const usage of aggregated.tokenUsage) {
+        addValue(tokenMetric.name, usage.totalInputTokens, {
+            provider: usage.provider,
+            model: usage.model,
+            direction: 'input',
+        });
+        addValue(tokenMetric.name, usage.totalOutputTokens, {
+            provider: usage.provider,
+            model: usage.model,
+            direction: 'output',
+        });
+    }
+    // LLM calls metric
+    const callsMetric = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_llm_calls_total');
+    addHeader(callsMetric);
+    for (const usage of aggregated.tokenUsage) {
+        addValue(callsMetric.name, usage.callCount, {
+            provider: usage.provider,
+            model: usage.model,
+        });
+    }
+    // Cost metrics
+    const costMetric = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_cost_usd_total');
+    addHeader(costMetric);
+    for (const cost of aggregated.costByProvider) {
+        addValue(costMetric.name, cost.costUSD, { provider: cost.provider });
+    }
+    // Operation duration histogram
+    const durationMetric = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_operation_duration_seconds');
+    addHeader(durationMetric);
+    for (const stat of aggregated.operationStats) {
+        // Add histogram buckets
+        const buckets = calculateHistogramBuckets(collector.getRawRecords().timing
+            .filter(r => r.operation === stat.operation)
+            .map(r => r.durationMs / 1000), // Convert to seconds
+        DEFAULT_LATENCY_BUCKETS.map(b => b / 1000));
+        for (let i = 0; i < buckets.le.length; i++) {
+            addValue(`${durationMetric.name}_bucket`, buckets.counts[i], {
+                operation: stat.operation,
+                status: 'all',
+                le: buckets.le[i] === Infinity ? '+Inf' : String(buckets.le[i]),
+            });
+        }
+        addValue(`${durationMetric.name}_sum`, buckets.sum, { operation: stat.operation, status: 'all' });
+        addValue(`${durationMetric.name}_count`, buckets.count, { operation: stat.operation, status: 'all' });
+    }
+    // Operation total counter
+    const opTotalMetric = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_operation_total');
+    addHeader(opTotalMetric);
+    for (const stat of aggregated.operationStats) {
+        addValue(opTotalMetric.name, stat.successCount, { operation: stat.operation, status: 'success' });
+        addValue(opTotalMetric.name, stat.failureCount, { operation: stat.operation, status: 'failure' });
+    }
+    // Error metrics
+    const errorMetric = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_errors_total');
+    addHeader(errorMetric);
+    for (const stat of aggregated.errorStats) {
+        addValue(errorMetric.name, stat.retryableCount, { category: stat.category, retryable: 'true' });
+        addValue(errorMetric.name, stat.terminalCount, { category: stat.category, retryable: 'false' });
+    }
+    // Interview-specific metrics
+    if (interview) {
+        const toolsMetric = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_interview_tools_discovered');
+        addHeader(toolsMetric);
+        addValue(toolsMetric.name, interview.toolsDiscovered);
+        const questionsMetric = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_interview_questions_generated');
+        addHeader(questionsMetric);
+        addValue(questionsMetric.name, interview.questionsGenerated);
+        const toolCallsMetric = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_interview_tool_calls_total');
+        addHeader(toolCallsMetric);
+        addValue(toolCallsMetric.name, interview.toolCallsSucceeded, { status: 'success' });
+        addValue(toolCallsMetric.name, interview.toolCallsFailed, { status: 'failure' });
+        const durationGauge = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_interview_duration_seconds');
+        addHeader(durationGauge);
+        const durationSec = interview.totalDurationMs
+            ? interview.totalDurationMs / 1000
+            : (Date.now() - interview.startedAt.getTime()) / 1000;
+        addValue(durationGauge.name, durationSec);
+        const progressMetric = METRIC_DEFINITIONS.find(m => m.name === 'bellwether_interview_progress_ratio');
+        addHeader(progressMetric);
+        const expectedCalls = interview.toolsDiscovered * 5;
+        const progress = expectedCalls > 0 ? Math.min(1, interview.toolCallsMade / expectedCalls) : 0;
+        addValue(progressMetric.name, progress);
+    }
+    return lines.join('\n');
+}
+/**
+ * Calculate histogram buckets from raw values.
+ */
+function calculateHistogramBuckets(values, boundaries) {
+    const bucketBoundaries = [...boundaries, Infinity];
+    const counts = new Array(bucketBoundaries.length).fill(0);
+    let sum = 0;
+    for (const value of values) {
+        sum += value;
+        for (let i = 0; i < bucketBoundaries.length; i++) {
+            if (value <= bucketBoundaries[i]) {
+                counts[i]++;
+            }
+        }
+    }
+    // Make counts cumulative
+    for (let i = 1; i < counts.length; i++) {
+        counts[i] += counts[i - 1];
+    }
+    return {
+        le: bucketBoundaries,
+        counts,
+        sum,
+        count: values.length,
+    };
+}
+/**
+ * Escape label value for Prometheus format.
+ */
+function escapeLabel(value) {
+    return value
+        .replace(/\\/g, '\\\\')
+        .replace(/"/g, '\\"')
+        .replace(/\n/g, '\\n');
+}
+/**
+ * Export metrics as JSON for dashboard consumption.
+ */
+export function exportMetricsJSON(collector) {
+    return JSON.stringify(collector.getDashboardMetrics(), null, 2);
+}
+/**
+ * Get metric definitions for documentation.
+ */
+export function getMetricDefinitions() {
+    return [...METRIC_DEFINITIONS];
+}
+//# sourceMappingURL=prometheus.js.map

package/dist/metrics/types.d.ts

@@ -0,0 +1,209 @@
+/**
+ * Metrics collection types for observability.
+ */
+/**
+ * Error category for metrics tracking.
+ */
+export type ErrorCategory = 'transport' | 'llm_auth' | 'llm_rate_limit' | 'llm_quota' | 'llm_refusal' | 'llm_parse' | 'llm_connection' | 'llm_timeout' | 'interview' | 'workflow' | 'config' | 'unknown';
+/**
+ * Operation type for timing metrics.
+ */
+export type OperationType = 'llm_call' | 'llm_stream' | 'tool_call' | 'discovery' | 'question_generation' | 'response_analysis' | 'profile_synthesis' | 'summary_generation' | 'interview_total' | 'workflow_execution' | 'assertion_evaluation' | 'baseline_comparison';
+/**
+ * Token usage record.
+ */
+export interface TokenUsageRecord {
+    provider: string;
+    model: string;
+    inputTokens: number;
+    outputTokens: number;
+    timestamp: Date;
+    operation?: string;
+}
+/**
+ * Operation timing record.
+ */
+export interface TimingRecord {
+    operation: OperationType;
+    durationMs: number;
+    success: boolean;
+    timestamp: Date;
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Error record for metrics.
+ */
+export interface ErrorRecord {
+    category: ErrorCategory;
+    code: string;
+    message: string;
+    timestamp: Date;
+    retryable: boolean;
+    operation?: string;
+}
+/**
+ * Cost record.
+ */
+export interface CostRecord {
+    provider: string;
+    model: string;
+    inputCost: number;
+    outputCost: number;
+    totalCost: number;
+    timestamp: Date;
+}
+/**
+ * Interview metrics summary.
+ */
+export interface InterviewMetrics {
+    /** Start time */
+    startedAt: Date;
+    /** End time */
+    endedAt?: Date;
+    /** Total duration in ms */
+    totalDurationMs?: number;
+    /** Tools discovered */
+    toolsDiscovered: number;
+    /** Questions generated */
+    questionsGenerated: number;
+    /** Tool calls made */
+    toolCallsMade: number;
+    /** Tool calls succeeded */
+    toolCallsSucceeded: number;
+    /** Tool calls failed */
+    toolCallsFailed: number;
+    /** Personas used */
+    personasUsed: number;
+    /** LLM calls made */
+    llmCallsMade: number;
+    /** Total input tokens */
+    totalInputTokens: number;
+    /** Total output tokens */
+    totalOutputTokens: number;
+    /** Total cost USD */
+    totalCostUSD: number;
+    /** Errors by category */
+    errorsByCategory: Record<ErrorCategory, number>;
+}
+/**
+ * Aggregated metrics for a time period.
+ */
+export interface AggregatedMetrics {
+    /** Time period start */
+    periodStart: Date;
+    /** Time period end */
+    periodEnd: Date;
+    /** Token usage by provider/model */
+    tokenUsage: {
+        provider: string;
+        model: string;
+        totalInputTokens: number;
+        totalOutputTokens: number;
+        callCount: number;
+    }[];
+    /** Operation timing statistics */
+    operationStats: {
+        operation: OperationType;
+        count: number;
+        successCount: number;
+        failureCount: number;
+        avgDurationMs: number;
+        minDurationMs: number;
+        maxDurationMs: number;
+        p50DurationMs: number;
+        p95DurationMs: number;
+        p99DurationMs: number;
+    }[];
+    /** Error statistics */
+    errorStats: {
+        category: ErrorCategory;
+        count: number;
+        retryableCount: number;
+        terminalCount: number;
+    }[];
+    /** Total cost */
+    totalCostUSD: number;
+    costByProvider: {
+        provider: string;
+        costUSD: number;
+    }[];
+}
+/**
+ * Prometheus metric types.
+ */
+export type PrometheusMetricType = 'counter' | 'gauge' | 'histogram' | 'summary';
+/**
+ * Prometheus metric definition.
+ */
+export interface PrometheusMetric {
+    name: string;
+    type: PrometheusMetricType;
+    help: string;
+    labels?: string[];
+}
+/**
+ * Prometheus metric value.
+ */
+export interface PrometheusMetricValue {
+    name: string;
+    value: number;
+    labels?: Record<string, string>;
+    timestamp?: number;
+}
+/**
+ * Histogram bucket configuration.
+ */
+export interface HistogramBuckets {
+    /** Bucket boundaries */
+    le: number[];
+    /** Count per bucket */
+    counts: number[];
+    /** Total sum */
+    sum: number;
+    /** Total count */
+    count: number;
+}
+/**
+ * Dashboard-compatible metrics output.
+ */
+export interface DashboardMetrics {
+    /** Timestamp of metrics snapshot */
+    timestamp: string;
+    /** Interview summary */
+    interview: {
+        status: 'running' | 'completed' | 'failed';
+        progress: number;
+        toolsTotal: number;
+        toolsCompleted: number;
+        questionsTotal: number;
+        questionsCompleted: number;
+    };
+    /** Token usage */
+    tokens: {
+        input: number;
+        output: number;
+        total: number;
+    };
+    /** Cost */
+    cost: {
+        current: number;
+        projected: number;
+        currency: string;
+    };
+    /** Performance */
+    performance: {
+        avgLLMLatencyMs: number;
+        avgToolLatencyMs: number;
+        errorsTotal: number;
+        successRate: number;
+    };
+    /** Provider breakdown */
+    providers: {
+        name: string;
+        calls: number;
+        tokens: number;
+        errors: number;
+        avgLatencyMs: number;
+    }[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/metrics/types.js

@@ -0,0 +1,5 @@
+/**
+ * Metrics collection types for observability.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/persona/builtins.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Built-in persona definitions.
+ */
+import type { Persona, BuiltInPersonaId } from './types.js';
+/**
+ * Technical Writer persona - balanced documentation focus.
+ */
+export declare const technicalWriterPersona: Persona;
+/**
+ * Security Tester persona - vulnerability-focused.
+ *
+ * Note: Prompts are designed to avoid triggering LLM safety filters while still
+ * enabling meaningful security testing. We describe test categories rather than
+ * including specific payloads, letting the LLM generate appropriate test inputs.
+ */
+export declare const securityTesterPersona: Persona;
+/**
+ * QA Engineer persona - edge case and error focus.
+ */
+export declare const qaEngineerPersona: Persona;
+/**
+ * Novice User persona - usability and error message focus.
+ */
+export declare const noviceUserPersona: Persona;
+/**
+ * Map of built-in persona IDs to definitions.
+ */
+export declare const BUILTIN_PERSONAS: Record<BuiltInPersonaId, Persona>;
+/**
+ * Get a built-in persona by ID.
+ */
+export declare function getBuiltinPersona(id: BuiltInPersonaId): Persona;
+/**
+ * Check if a persona ID is a built-in.
+ */
+export declare function isBuiltinPersona(id: string): id is BuiltInPersonaId;
+/**
+ * Get all built-in persona IDs.
+ */
+export declare function getBuiltinPersonaIds(): BuiltInPersonaId[];
+/**
+ * Default persona for interviews.
+ */
+export declare const DEFAULT_PERSONA: Persona;
+/**
+ * Parse persona list from string array of persona IDs.
+ * Returns the DEFAULT_PERSONA if the list is empty or contains no valid personas.
+ *
+ * @param personaList - Array of persona ID strings
+ * @param warnOnUnknown - Optional callback for unknown persona warnings
+ * @returns Array of resolved Persona objects
+ */
+export declare function parsePersonas(personaList: string[], warnOnUnknown?: (unknownName: string, validNames: string[]) => void): Persona[];
+//# sourceMappingURL=builtins.d.ts.map