@lov3kaizen/agentsea-evaluate 0.5.1

package/dist/index-Cq5LwG_3.d.mts

@@ -0,0 +1,322 @@
+import { EventEmitter } from 'eventemitter3';
+
+interface ContinuousEvalConfig {
+    pipeline: EvaluationPipelineRef;
+    sampleRate: number;
+    schedule?: string;
+    storage?: 'memory' | 'sqlite' | 'postgres';
+    storagePath?: string;
+    retentionDays?: number;
+}
+interface EvaluationPipelineRef {
+    evaluate(input: EvalInput): Promise<EvalOutput>;
+}
+interface EvalInput {
+    input: string;
+    output: string;
+    context?: string[];
+    metadata?: Record<string, unknown>;
+}
+interface EvalOutput {
+    scores: Record<string, number>;
+    passed: boolean;
+    durationMs: number;
+}
+type MonitoringStatus = 'stopped' | 'running' | 'paused' | 'error';
+interface ContinuousEvalStats {
+    status: MonitoringStatus;
+    startedAt?: number;
+    lastEvalAt?: number;
+    totalEvaluations: number;
+    passRate: number;
+    avgScores: Record<string, number>;
+    alertsTriggered: number;
+}
+type AlertChannelType = 'slack' | 'email' | 'webhook' | 'pagerduty';
+interface AlertChannelConfig {
+    type: AlertChannelType;
+    webhook?: string;
+    to?: string[];
+    apiKey?: string;
+    channel?: string;
+}
+interface AlertRule {
+    metric: string;
+    threshold: number;
+    direction: 'above' | 'below';
+    window?: number;
+    minSamples?: number;
+    severity?: 'info' | 'warning' | 'critical';
+}
+interface AlertManagerConfig {
+    channels: AlertChannelConfig[];
+    rules?: Record<string, AlertRule>;
+    cooldownMs?: number;
+    groupingWindow?: number;
+}
+interface Alert {
+    id: string;
+    rule: AlertRule;
+    metric: string;
+    currentValue: number;
+    threshold: number;
+    severity: 'info' | 'warning' | 'critical';
+    message: string;
+    triggeredAt: number;
+    resolvedAt?: number;
+    acknowledged?: boolean;
+    metadata?: Record<string, unknown>;
+}
+interface AlertNotification {
+    alertId: string;
+    channel: AlertChannelType;
+    sentAt: number;
+    success: boolean;
+    error?: string;
+}
+interface RegressionDetectorConfig {
+    baseline: BaselineMetrics;
+    sensitivity: 'low' | 'medium' | 'high';
+    windowSize: number;
+    minSamples?: number;
+    pValueThreshold?: number;
+}
+interface BaselineMetrics {
+    metrics: Record<string, MetricBaseline>;
+    sampleCount: number;
+    timestamp: number;
+}
+interface MetricBaseline {
+    mean: number;
+    std: number;
+    min: number;
+    max: number;
+    p50: number;
+    p90: number;
+    p95: number;
+}
+interface RegressionResult {
+    detected: boolean;
+    regressions: MetricRegression[];
+    improvements: MetricImprovement[];
+    unchanged: string[];
+}
+interface MetricRegression {
+    metric: string;
+    baselineValue: number;
+    currentValue: number;
+    changePercent: number;
+    pValue: number;
+    severity: 'minor' | 'moderate' | 'severe';
+}
+interface MetricImprovement {
+    metric: string;
+    baselineValue: number;
+    currentValue: number;
+    changePercent: number;
+    pValue: number;
+}
+interface ABTestConfig {
+    name: string;
+    description?: string;
+    variants: ABTestVariants;
+    trafficSplit: number;
+    metrics: string[];
+    minSamples: number;
+    significanceLevel?: number;
+    maxDuration?: number;
+}
+interface ABTestVariants {
+    control: VariantConfig;
+    treatment: VariantConfig;
+}
+interface VariantConfig {
+    name?: string;
+    model?: string;
+    prompt?: string;
+    parameters?: Record<string, unknown>;
+}
+type ABTestStatus = 'draft' | 'running' | 'paused' | 'completed' | 'cancelled';
+interface ABTest {
+    id: string;
+    name: string;
+    config: ABTestConfig;
+    status: ABTestStatus;
+    startedAt?: number;
+    completedAt?: number;
+    results?: ABTestResults;
+}
+interface ABTestResults {
+    controlSamples: number;
+    treatmentSamples: number;
+    metrics: Record<string, ABMetricResult>;
+    winner: 'control' | 'treatment' | 'none';
+    isSignificant: boolean;
+    confidence: number;
+    recommendation: string;
+}
+interface ABMetricResult {
+    control: MetricSummary;
+    treatment: MetricSummary;
+    difference: number;
+    differencePercent: number;
+    pValue: number;
+    isSignificant: boolean;
+    winner: 'control' | 'treatment' | 'none';
+}
+interface MetricSummary {
+    mean: number;
+    std: number;
+    sampleCount: number;
+    confidenceInterval: [number, number];
+}
+interface SampleAssignment {
+    variant: 'control' | 'treatment';
+    testId: string;
+    assignedAt: number;
+}
+interface TimeSeriesPoint {
+    timestamp: number;
+    value: number;
+    metadata?: Record<string, unknown>;
+}
+interface TimeSeries {
+    metric: string;
+    points: TimeSeriesPoint[];
+    aggregation?: 'sum' | 'avg' | 'min' | 'max' | 'count';
+    interval?: number;
+}
+interface DashboardUpdate {
+    timestamp: number;
+    status: MonitoringStatus;
+    metrics: Record<string, number>;
+    recentAlerts: Alert[];
+    activeTests: ABTest[];
+    regressions?: RegressionResult;
+}
+interface HistoricalQueryOptions {
+    metric: string;
+    startTime: number;
+    endTime: number;
+    interval?: 'minute' | 'hour' | 'day' | 'week';
+    aggregation?: 'avg' | 'min' | 'max' | 'sum' | 'count';
+}
+interface ScheduleConfig {
+    cron?: string;
+    interval?: number;
+    timezone?: string;
+    immediate?: boolean;
+}
+type ContinuousEvalEventType = 'eval:started' | 'eval:completed' | 'eval:error' | 'alert:triggered' | 'alert:resolved' | 'regression:detected' | 'test:started' | 'test:completed' | 'test:significant';
+interface ContinuousEvalEvent {
+    type: ContinuousEvalEventType;
+    data: Record<string, unknown>;
+    timestamp: number;
+}
+
+interface AlertManagerEvents {
+    'alert:triggered': (alert: Alert) => void;
+    'alert:resolved': (alert: Alert) => void;
+    'notification:sent': (notification: AlertNotification) => void;
+}
+declare class AlertManager extends EventEmitter<AlertManagerEvents> {
+    private channels;
+    private rules;
+    private activeAlerts;
+    private cooldownMs;
+    private lastAlertTime;
+    private alertCount;
+    constructor(config: AlertManagerConfig);
+    addRule(rule: AlertRule): void;
+    removeRule(metric: string): boolean;
+    check(metric: string, value: number): Alert | null;
+    private triggerAlert;
+    resolveAlert(metric: string): void;
+    acknowledgeAlert(alertId: string): boolean;
+    private sendNotifications;
+    private sendToChannel;
+    private formatMessage;
+    getActiveAlerts(): Alert[];
+    getAlertCount(): number;
+    getRules(): AlertRule[];
+}
+declare function createAlertManager(config: AlertManagerConfig): AlertManager;
+
+interface ABTestEvents {
+    'test:started': () => void;
+    'test:completed': (results: ABTestResults) => void;
+    'test:significant': (metric: string, winner: 'control' | 'treatment') => void;
+    'sample:assigned': (assignment: SampleAssignment) => void;
+}
+declare class ABTestRunner extends EventEmitter<ABTestEvents> {
+    readonly id: string;
+    readonly name: string;
+    private config;
+    private status;
+    private startedAt?;
+    private completedAt?;
+    private controlSamples;
+    private treatmentSamples;
+    private sampleCount;
+    constructor(config: ABTestConfig);
+    start(): Promise<void>;
+    stop(): ABTestResults;
+    pause(): void;
+    resume(): void;
+    assignVariant(): 'control' | 'treatment';
+    recordSample(variant: 'control' | 'treatment', sampleId: string, scores: Record<string, number>): void;
+    getResults(): ABTestResults;
+    private calculateMetricResult;
+    private getScoresForMetric;
+    private calculateSummary;
+    private calculatePValue;
+    private normalCDF;
+    private checkSignificance;
+    private generateRecommendation;
+    getStatus(): ABTestStatus;
+    getConfig(): ABTestConfig;
+    getTimestamps(): {
+        startedAt?: number;
+        completedAt?: number;
+    };
+}
+declare function createABTestRunner(config: ABTestConfig): ABTestRunner;
+
+interface ContinuousEvalEvents {
+    'eval:started': () => void;
+    'eval:completed': (result: EvalOutput) => void;
+    'eval:error': (error: Error) => void;
+    'status:changed': (status: MonitoringStatus) => void;
+}
+declare class ContinuousEval extends EventEmitter<ContinuousEvalEvents> {
+    private pipeline;
+    private sampleRate;
+    private status;
+    private startedAt?;
+    private lastEvalAt?;
+    private totalEvaluations;
+    private passedCount;
+    private scoreHistory;
+    private alertManager?;
+    private abTests;
+    private intervalId?;
+    constructor(config: ContinuousEvalConfig);
+    setAlerts(alertManager: AlertManager, rules: Record<string, {
+        threshold: number;
+        direction: 'above' | 'below';
+    }>): void;
+    start(): void;
+    stop(): void;
+    pause(): void;
+    resume(): void;
+    evaluate(input: EvalInput): Promise<EvalOutput | null>;
+    getStats(): ContinuousEvalStats;
+    createABTest(config: ABTestConfig): ABTestRunner;
+    getABTest(id: string): ABTestRunner | undefined;
+    getABTests(): ABTestRunner[];
+    getScoreHistory(metric: string): number[];
+    reset(): void;
+}
+declare function createContinuousEval(config: ContinuousEvalConfig): ContinuousEval;
+
+export { type AlertChannelType as A, type BaselineMetrics as B, type ContinuousEvalConfig as C, type DashboardUpdate as D, type EvaluationPipelineRef as E, createAlertManager as F, ABTestRunner as G, type HistoricalQueryOptions as H, createABTestRunner as I, type MonitoringStatus as M, type RegressionDetectorConfig as R, type SampleAssignment as S, type TimeSeriesPoint as T, type VariantConfig as V, type EvalInput as a, type EvalOutput as b, type ContinuousEvalStats as c, type AlertChannelConfig as d, type AlertRule as e, type AlertManagerConfig as f, type Alert as g, type AlertNotification as h, type MetricBaseline as i, type RegressionResult as j, type MetricRegression as k, type MetricImprovement as l, type ABTestConfig as m, type ABTestVariants as n, type ABTestStatus as o, type ABTest as p, type ABTestResults as q, type ABMetricResult as r, type MetricSummary as s, type TimeSeries as t, type ScheduleConfig as u, type ContinuousEvalEventType as v, type ContinuousEvalEvent as w, ContinuousEval as x, createContinuousEval as y, AlertManager as z };

package/dist/index-Cq5LwG_3.d.ts

@@ -0,0 +1,322 @@
+import { EventEmitter } from 'eventemitter3';
+
+interface ContinuousEvalConfig {
+    pipeline: EvaluationPipelineRef;
+    sampleRate: number;
+    schedule?: string;
+    storage?: 'memory' | 'sqlite' | 'postgres';
+    storagePath?: string;
+    retentionDays?: number;
+}
+interface EvaluationPipelineRef {
+    evaluate(input: EvalInput): Promise<EvalOutput>;
+}
+interface EvalInput {
+    input: string;
+    output: string;
+    context?: string[];
+    metadata?: Record<string, unknown>;
+}
+interface EvalOutput {
+    scores: Record<string, number>;
+    passed: boolean;
+    durationMs: number;
+}
+type MonitoringStatus = 'stopped' | 'running' | 'paused' | 'error';
+interface ContinuousEvalStats {
+    status: MonitoringStatus;
+    startedAt?: number;
+    lastEvalAt?: number;
+    totalEvaluations: number;
+    passRate: number;
+    avgScores: Record<string, number>;
+    alertsTriggered: number;
+}
+type AlertChannelType = 'slack' | 'email' | 'webhook' | 'pagerduty';
+interface AlertChannelConfig {
+    type: AlertChannelType;
+    webhook?: string;
+    to?: string[];
+    apiKey?: string;
+    channel?: string;
+}
+interface AlertRule {
+    metric: string;
+    threshold: number;
+    direction: 'above' | 'below';
+    window?: number;
+    minSamples?: number;
+    severity?: 'info' | 'warning' | 'critical';
+}
+interface AlertManagerConfig {
+    channels: AlertChannelConfig[];
+    rules?: Record<string, AlertRule>;
+    cooldownMs?: number;
+    groupingWindow?: number;
+}
+interface Alert {
+    id: string;
+    rule: AlertRule;
+    metric: string;
+    currentValue: number;
+    threshold: number;
+    severity: 'info' | 'warning' | 'critical';
+    message: string;
+    triggeredAt: number;
+    resolvedAt?: number;
+    acknowledged?: boolean;
+    metadata?: Record<string, unknown>;
+}
+interface AlertNotification {
+    alertId: string;
+    channel: AlertChannelType;
+    sentAt: number;
+    success: boolean;
+    error?: string;
+}
+interface RegressionDetectorConfig {
+    baseline: BaselineMetrics;
+    sensitivity: 'low' | 'medium' | 'high';
+    windowSize: number;
+    minSamples?: number;
+    pValueThreshold?: number;
+}
+interface BaselineMetrics {
+    metrics: Record<string, MetricBaseline>;
+    sampleCount: number;
+    timestamp: number;
+}
+interface MetricBaseline {
+    mean: number;
+    std: number;
+    min: number;
+    max: number;
+    p50: number;
+    p90: number;
+    p95: number;
+}
+interface RegressionResult {
+    detected: boolean;
+    regressions: MetricRegression[];
+    improvements: MetricImprovement[];
+    unchanged: string[];
+}
+interface MetricRegression {
+    metric: string;
+    baselineValue: number;
+    currentValue: number;
+    changePercent: number;
+    pValue: number;
+    severity: 'minor' | 'moderate' | 'severe';
+}
+interface MetricImprovement {
+    metric: string;
+    baselineValue: number;
+    currentValue: number;
+    changePercent: number;
+    pValue: number;
+}
+interface ABTestConfig {
+    name: string;
+    description?: string;
+    variants: ABTestVariants;
+    trafficSplit: number;
+    metrics: string[];
+    minSamples: number;
+    significanceLevel?: number;
+    maxDuration?: number;
+}
+interface ABTestVariants {
+    control: VariantConfig;
+    treatment: VariantConfig;
+}
+interface VariantConfig {
+    name?: string;
+    model?: string;
+    prompt?: string;
+    parameters?: Record<string, unknown>;
+}
+type ABTestStatus = 'draft' | 'running' | 'paused' | 'completed' | 'cancelled';
+interface ABTest {
+    id: string;
+    name: string;
+    config: ABTestConfig;
+    status: ABTestStatus;
+    startedAt?: number;
+    completedAt?: number;
+    results?: ABTestResults;
+}
+interface ABTestResults {
+    controlSamples: number;
+    treatmentSamples: number;
+    metrics: Record<string, ABMetricResult>;
+    winner: 'control' | 'treatment' | 'none';
+    isSignificant: boolean;
+    confidence: number;
+    recommendation: string;
+}
+interface ABMetricResult {
+    control: MetricSummary;
+    treatment: MetricSummary;
+    difference: number;
+    differencePercent: number;
+    pValue: number;
+    isSignificant: boolean;
+    winner: 'control' | 'treatment' | 'none';
+}
+interface MetricSummary {
+    mean: number;
+    std: number;
+    sampleCount: number;
+    confidenceInterval: [number, number];
+}
+interface SampleAssignment {
+    variant: 'control' | 'treatment';
+    testId: string;
+    assignedAt: number;
+}
+interface TimeSeriesPoint {
+    timestamp: number;
+    value: number;
+    metadata?: Record<string, unknown>;
+}
+interface TimeSeries {
+    metric: string;
+    points: TimeSeriesPoint[];
+    aggregation?: 'sum' | 'avg' | 'min' | 'max' | 'count';
+    interval?: number;
+}
+interface DashboardUpdate {
+    timestamp: number;
+    status: MonitoringStatus;
+    metrics: Record<string, number>;
+    recentAlerts: Alert[];
+    activeTests: ABTest[];
+    regressions?: RegressionResult;
+}
+interface HistoricalQueryOptions {
+    metric: string;
+    startTime: number;
+    endTime: number;
+    interval?: 'minute' | 'hour' | 'day' | 'week';
+    aggregation?: 'avg' | 'min' | 'max' | 'sum' | 'count';
+}
+interface ScheduleConfig {
+    cron?: string;
+    interval?: number;
+    timezone?: string;
+    immediate?: boolean;
+}
+type ContinuousEvalEventType = 'eval:started' | 'eval:completed' | 'eval:error' | 'alert:triggered' | 'alert:resolved' | 'regression:detected' | 'test:started' | 'test:completed' | 'test:significant';
+interface ContinuousEvalEvent {
+    type: ContinuousEvalEventType;
+    data: Record<string, unknown>;
+    timestamp: number;
+}
+
+interface AlertManagerEvents {
+    'alert:triggered': (alert: Alert) => void;
+    'alert:resolved': (alert: Alert) => void;
+    'notification:sent': (notification: AlertNotification) => void;
+}
+declare class AlertManager extends EventEmitter<AlertManagerEvents> {
+    private channels;
+    private rules;
+    private activeAlerts;
+    private cooldownMs;
+    private lastAlertTime;
+    private alertCount;
+    constructor(config: AlertManagerConfig);
+    addRule(rule: AlertRule): void;
+    removeRule(metric: string): boolean;
+    check(metric: string, value: number): Alert | null;
+    private triggerAlert;
+    resolveAlert(metric: string): void;
+    acknowledgeAlert(alertId: string): boolean;
+    private sendNotifications;
+    private sendToChannel;
+    private formatMessage;
+    getActiveAlerts(): Alert[];
+    getAlertCount(): number;
+    getRules(): AlertRule[];
+}
+declare function createAlertManager(config: AlertManagerConfig): AlertManager;
+
+interface ABTestEvents {
+    'test:started': () => void;
+    'test:completed': (results: ABTestResults) => void;
+    'test:significant': (metric: string, winner: 'control' | 'treatment') => void;
+    'sample:assigned': (assignment: SampleAssignment) => void;
+}
+declare class ABTestRunner extends EventEmitter<ABTestEvents> {
+    readonly id: string;
+    readonly name: string;
+    private config;
+    private status;
+    private startedAt?;
+    private completedAt?;
+    private controlSamples;
+    private treatmentSamples;
+    private sampleCount;
+    constructor(config: ABTestConfig);
+    start(): Promise<void>;
+    stop(): ABTestResults;
+    pause(): void;
+    resume(): void;
+    assignVariant(): 'control' | 'treatment';
+    recordSample(variant: 'control' | 'treatment', sampleId: string, scores: Record<string, number>): void;
+    getResults(): ABTestResults;
+    private calculateMetricResult;
+    private getScoresForMetric;
+    private calculateSummary;
+    private calculatePValue;
+    private normalCDF;
+    private checkSignificance;
+    private generateRecommendation;
+    getStatus(): ABTestStatus;
+    getConfig(): ABTestConfig;
+    getTimestamps(): {
+        startedAt?: number;
+        completedAt?: number;
+    };
+}
+declare function createABTestRunner(config: ABTestConfig): ABTestRunner;
+
+interface ContinuousEvalEvents {
+    'eval:started': () => void;
+    'eval:completed': (result: EvalOutput) => void;
+    'eval:error': (error: Error) => void;
+    'status:changed': (status: MonitoringStatus) => void;
+}
+declare class ContinuousEval extends EventEmitter<ContinuousEvalEvents> {
+    private pipeline;
+    private sampleRate;
+    private status;
+    private startedAt?;
+    private lastEvalAt?;
+    private totalEvaluations;
+    private passedCount;
+    private scoreHistory;
+    private alertManager?;
+    private abTests;
+    private intervalId?;
+    constructor(config: ContinuousEvalConfig);
+    setAlerts(alertManager: AlertManager, rules: Record<string, {
+        threshold: number;
+        direction: 'above' | 'below';
+    }>): void;
+    start(): void;
+    stop(): void;
+    pause(): void;
+    resume(): void;
+    evaluate(input: EvalInput): Promise<EvalOutput | null>;
+    getStats(): ContinuousEvalStats;
+    createABTest(config: ABTestConfig): ABTestRunner;
+    getABTest(id: string): ABTestRunner | undefined;
+    getABTests(): ABTestRunner[];
+    getScoreHistory(metric: string): number[];
+    reset(): void;
+}
+declare function createContinuousEval(config: ContinuousEvalConfig): ContinuousEval;
+
+export { type AlertChannelType as A, type BaselineMetrics as B, type ContinuousEvalConfig as C, type DashboardUpdate as D, type EvaluationPipelineRef as E, createAlertManager as F, ABTestRunner as G, type HistoricalQueryOptions as H, createABTestRunner as I, type MonitoringStatus as M, type RegressionDetectorConfig as R, type SampleAssignment as S, type TimeSeriesPoint as T, type VariantConfig as V, type EvalInput as a, type EvalOutput as b, type ContinuousEvalStats as c, type AlertChannelConfig as d, type AlertRule as e, type AlertManagerConfig as f, type Alert as g, type AlertNotification as h, type MetricBaseline as i, type RegressionResult as j, type MetricRegression as k, type MetricImprovement as l, type ABTestConfig as m, type ABTestVariants as n, type ABTestStatus as o, type ABTest as p, type ABTestResults as q, type ABMetricResult as r, type MetricSummary as s, type TimeSeries as t, type ScheduleConfig as u, type ContinuousEvalEventType as v, type ContinuousEvalEvent as w, ContinuousEval as x, createContinuousEval as y, AlertManager as z };