dino-spec 16.2.0 → 17.2.0

package/dist/core/experiments/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/experiments/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,EACL,aAAa,EACb,sBAAsB,EACtB,sBAAsB,EACtB,oBAAoB,EACpB,sBAAsB,EACtB,kBAAkB,EAClB,aAAa,EACb,gBAAgB,GACjB,MAAM,wBAAwB,CAAA;AAU/B,OAAO,EACL,mBAAmB,EACnB,0BAA0B,EAC1B,mBAAmB,EACnB,mBAAmB,EACnB,yBAAyB,EACzB,oBAAoB,EACpB,gBAAgB,EAChB,aAAa,EACb,sBAAsB,EACtB,qBAAqB,EACrB,eAAe,EACf,GAAG,EACH,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,wBAAwB,CAAA;AAM/B,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,oBAAoB,EACpB,mBAAmB,EACnB,eAAe,EACf,sBAAsB,GACvB,MAAM,uBAAuB,CAAA;AAS9B,OAAO,EACL,uBAAuB,EACvB,oBAAoB,EACpB,wBAAwB,EACxB,oCAAoC,GACrC,MAAM,wBAAwB,CAAA"}

package/dist/core/experiments/metrics-segmenter.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Metrics Segmenter - v17.0.0
+ *
+ * Queries and segments metrics by A/B testing experiment arms.
+ * Integrates with the metrics-collector to provide experiment-specific analysis.
+ *
+ * Features:
+ * - Query metrics by experiment ID
+ * - Segment results by arm (control vs treatment)
+ * - Calculate per-arm aggregates
+ * - Support backward compatibility (metrics without experiment data)
+ */
+import type { MetricEntry, OperationType } from '../metrics/metrics-schema.js';
+/**
+ * Metrics segmented by experiment arm
+ */
+export interface SegmentedMetrics {
+    /** Experiment ID */
+    experimentId: string;
+    /** Control arm metrics */
+    control: ArmMetrics;
+    /** Treatment arm metrics */
+    treatment: ArmMetrics;
+    /** Metrics without arm assignment (backward compatibility) */
+    unassigned: ArmMetrics;
+}
+/**
+ * Aggregated metrics for a single arm
+ */
+export interface ArmMetrics {
+    /** Total operation count */
+    count: number;
+    /** Success count */
+    successCount: number;
+    /** Failure count */
+    failureCount: number;
+    /** Skipped count */
+    skippedCount: number;
+    /** Success rate (0-1) */
+    successRate: number;
+    /** Total tokens saved */
+    tokensSaved: number;
+    /** Total tokens freed */
+    tokensFreed: number;
+    /** Average duration in ms */
+    avgDurationMs: number;
+    /** Average budget percentage */
+    avgBudgetPercentage: number;
+    /** Raw entries */
+    entries: MetricEntry[];
+}
+/**
+ * Summary of experiment metrics for display
+ */
+export interface ExperimentSummary {
+    /** Experiment ID */
+    experimentId: string;
+    /** Control arm summary */
+    control: {
+        sampleSize: number;
+        successRate: number;
+        avgTokensSaved: number;
+        avgTokensFreed: number;
+        avgDurationMs: number;
+    };
+    /** Treatment arm summary */
+    treatment: {
+        sampleSize: number;
+        successRate: number;
+        avgTokensSaved: number;
+        avgTokensFreed: number;
+        avgDurationMs: number;
+    };
+    /** Comparison */
+    comparison: {
+        sampleSizeDiff: number;
+        successRateDiff: number;
+        tokensSavedDiff: number;
+        tokensFreedDiff: number;
+        durationDiff: number;
+    };
+}
+/**
+ * Query metrics by experiment ID, segmented by arm
+ */
+export declare function queryMetricsByExperiment(projectDir: string, experimentId: string): Promise<SegmentedMetrics>;
+/**
+ * Get experiment summary for analysis
+ */
+export declare function getExperimentSummary(projectDir: string, experimentId: string): Promise<ExperimentSummary>;
+/**
+ * Query metrics by experiment and operation type
+ */
+export declare function queryMetricsByExperimentAndOperation(projectDir: string, experimentId: string, operationType: OperationType): Promise<SegmentedMetrics>;
+/**
+ * Format experiment summary for display
+ */
+export declare function formatExperimentSummary(summary: ExperimentSummary): string;
+//# sourceMappingURL=metrics-segmenter.d.ts.map

package/dist/core/experiments/metrics-segmenter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics-segmenter.d.ts","sourceRoot":"","sources":["../../../src/core/experiments/metrics-segmenter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAA;AAM9E;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAA;IACnB,4BAA4B;IAC5B,SAAS,EAAE,UAAU,CAAA;IACrB,8DAA8D;IAC9D,UAAU,EAAE,UAAU,CAAA;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAA;IACnB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAA;IACnB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAA;IACnB,6BAA6B;IAC7B,aAAa,EAAE,MAAM,CAAA;IACrB,gCAAgC;IAChC,mBAAmB,EAAE,MAAM,CAAA;IAC3B,kBAAkB;IAClB,OAAO,EAAE,WAAW,EAAE,CAAA;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,0BAA0B;IAC1B,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,MAAM,CAAA;QACnB,cAAc,EAAE,MAAM,CAAA;QACtB,cAAc,EAAE,MAAM,CAAA;QACtB,aAAa,EAAE,MAAM,CAAA;KACtB,CAAA;IACD,4BAA4B;IAC5B,SAAS,EAAE;QACT,UAAU,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,MAAM,CAAA;QACnB,cAAc,EAAE,MAAM,CAAA;QACtB,cAAc,EAAE,MAAM,CAAA;QACtB,aAAa,EAAE,MAAM,CAAA;KACtB,CAAA;IACD,iBAAiB;IACjB,UAAU,EAAE;QACV,cAAc,EAAE,MAAM,CAAA;QACtB,eAAe,EAAE,MAAM,CAAA;QACvB,eAAe,EAAE,MAAM,CAAA;QACvB,eAAe,EAAE,MAAM,CAAA;QACvB,YAAY,EAAE,MAAM,CAAA;KACrB,CAAA;CACF;AAMD;;GAEG;AACH,wBAAsB,wBAAwB,CAC5C,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,gBAAgB,CAAC,CAiB3B;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,iBAAiB,CAAC,CAqC5B;AAED;;GAEG;AACH,wBAAsB,oCAAoC,CACxD,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,EACpB,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,gBAAgB,CAAC,CAU3B;AAkED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAmC1E"}

package/dist/core/experiments/metrics-segmenter.js

@@ -0,0 +1,171 @@
+/**
+ * Metrics Segmenter - v17.0.0
+ *
+ * Queries and segments metrics by A/B testing experiment arms.
+ * Integrates with the metrics-collector to provide experiment-specific analysis.
+ *
+ * Features:
+ * - Query metrics by experiment ID
+ * - Segment results by arm (control vs treatment)
+ * - Calculate per-arm aggregates
+ * - Support backward compatibility (metrics without experiment data)
+ */
+import { getMetricsState } from '../metrics/metrics-collector.js';
+// =============================================================================
+// Query Functions
+// =============================================================================
+/**
+ * Query metrics by experiment ID, segmented by arm
+ */
+export async function queryMetricsByExperiment(projectDir, experimentId) {
+    const state = await getMetricsState(projectDir);
+    // Filter entries by experiment ID
+    const experimentEntries = state.entries.filter((e) => e.experimentId === experimentId);
+    // Segment by arm
+    const controlEntries = experimentEntries.filter((e) => e.experimentArm === 'control');
+    const treatmentEntries = experimentEntries.filter((e) => e.experimentArm === 'treatment');
+    const unassignedEntries = experimentEntries.filter((e) => !e.experimentArm);
+    return {
+        experimentId,
+        control: calculateArmMetrics(controlEntries),
+        treatment: calculateArmMetrics(treatmentEntries),
+        unassigned: calculateArmMetrics(unassignedEntries),
+    };
+}
+/**
+ * Get experiment summary for analysis
+ */
+export async function getExperimentSummary(projectDir, experimentId) {
+    const segmented = await queryMetricsByExperiment(projectDir, experimentId);
+    const controlAvgTokensSaved = segmented.control.count > 0 ? segmented.control.tokensSaved / segmented.control.count : 0;
+    const treatmentAvgTokensSaved = segmented.treatment.count > 0 ? segmented.treatment.tokensSaved / segmented.treatment.count : 0;
+    const controlAvgTokensFreed = segmented.control.count > 0 ? segmented.control.tokensFreed / segmented.control.count : 0;
+    const treatmentAvgTokensFreed = segmented.treatment.count > 0 ? segmented.treatment.tokensFreed / segmented.treatment.count : 0;
+    return {
+        experimentId,
+        control: {
+            sampleSize: segmented.control.count,
+            successRate: segmented.control.successRate,
+            avgTokensSaved: controlAvgTokensSaved,
+            avgTokensFreed: controlAvgTokensFreed,
+            avgDurationMs: segmented.control.avgDurationMs,
+        },
+        treatment: {
+            sampleSize: segmented.treatment.count,
+            successRate: segmented.treatment.successRate,
+            avgTokensSaved: treatmentAvgTokensSaved,
+            avgTokensFreed: treatmentAvgTokensFreed,
+            avgDurationMs: segmented.treatment.avgDurationMs,
+        },
+        comparison: {
+            sampleSizeDiff: segmented.treatment.count - segmented.control.count,
+            successRateDiff: segmented.treatment.successRate - segmented.control.successRate,
+            tokensSavedDiff: treatmentAvgTokensSaved - controlAvgTokensSaved,
+            tokensFreedDiff: treatmentAvgTokensFreed - controlAvgTokensFreed,
+            durationDiff: segmented.treatment.avgDurationMs - segmented.control.avgDurationMs,
+        },
+    };
+}
+/**
+ * Query metrics by experiment and operation type
+ */
+export async function queryMetricsByExperimentAndOperation(projectDir, experimentId, operationType) {
+    const segmented = await queryMetricsByExperiment(projectDir, experimentId);
+    // Filter each arm by operation type
+    return {
+        experimentId,
+        control: filterArmByOperation(segmented.control, operationType),
+        treatment: filterArmByOperation(segmented.treatment, operationType),
+        unassigned: filterArmByOperation(segmented.unassigned, operationType),
+    };
+}
+// =============================================================================
+// Helper Functions
+// =============================================================================
+/**
+ * Calculate aggregated metrics for an arm
+ */
+function calculateArmMetrics(entries) {
+    if (entries.length === 0) {
+        return {
+            count: 0,
+            successCount: 0,
+            failureCount: 0,
+            skippedCount: 0,
+            successRate: 0,
+            tokensSaved: 0,
+            tokensFreed: 0,
+            avgDurationMs: 0,
+            avgBudgetPercentage: 0,
+            entries: [],
+        };
+    }
+    let successCount = 0;
+    let failureCount = 0;
+    let skippedCount = 0;
+    let tokensSaved = 0;
+    let tokensFreed = 0;
+    let totalDuration = 0;
+    let totalBudget = 0;
+    for (const entry of entries) {
+        if (entry.outcome === 'success')
+            successCount++;
+        if (entry.outcome === 'failure')
+            failureCount++;
+        if (entry.outcome === 'skipped')
+            skippedCount++;
+        tokensSaved += entry.tokens.saved;
+        tokensFreed += entry.tokens.freed;
+        totalDuration += entry.durationMs;
+        totalBudget += entry.context.budgetPercentage;
+    }
+    return {
+        count: entries.length,
+        successCount,
+        failureCount,
+        skippedCount,
+        successRate: entries.length > 0 ? successCount / entries.length : 0,
+        tokensSaved,
+        tokensFreed,
+        avgDurationMs: totalDuration / entries.length,
+        avgBudgetPercentage: totalBudget / entries.length,
+        entries,
+    };
+}
+/**
+ * Filter arm metrics by operation type
+ */
+function filterArmByOperation(arm, operationType) {
+    const filtered = arm.entries.filter((e) => e.operationType === operationType);
+    return calculateArmMetrics(filtered);
+}
+/**
+ * Format experiment summary for display
+ */
+export function formatExperimentSummary(summary) {
+    const lines = [];
+    lines.push(`## Experiment Summary: ${summary.experimentId}`);
+    lines.push('');
+    lines.push('### Control Arm');
+    lines.push(`- Sample Size: ${summary.control.sampleSize}`);
+    lines.push(`- Success Rate: ${(summary.control.successRate * 100).toFixed(1)}%`);
+    lines.push(`- Avg Tokens Saved: ${summary.control.avgTokensSaved.toFixed(0)}`);
+    lines.push(`- Avg Duration: ${summary.control.avgDurationMs.toFixed(0)}ms`);
+    lines.push('');
+    lines.push('### Treatment Arm');
+    lines.push(`- Sample Size: ${summary.treatment.sampleSize}`);
+    lines.push(`- Success Rate: ${(summary.treatment.successRate * 100).toFixed(1)}%`);
+    lines.push(`- Avg Tokens Saved: ${summary.treatment.avgTokensSaved.toFixed(0)}`);
+    lines.push(`- Avg Duration: ${summary.treatment.avgDurationMs.toFixed(0)}ms`);
+    lines.push('');
+    lines.push('### Comparison (Treatment - Control)');
+    const successDiffSign = summary.comparison.successRateDiff >= 0 ? '+' : '';
+    const tokensDiffSign = summary.comparison.tokensSavedDiff >= 0 ? '+' : '';
+    const durationDiffSign = summary.comparison.durationDiff >= 0 ? '+' : '';
+    lines.push(`- Sample Size Diff: ${summary.comparison.sampleSizeDiff}`);
+    lines.push(`- Success Rate Diff: ${successDiffSign}${(summary.comparison.successRateDiff * 100).toFixed(1)}%`);
+    lines.push(`- Tokens Saved Diff: ${tokensDiffSign}${summary.comparison.tokensSavedDiff.toFixed(0)}`);
+    lines.push(`- Duration Diff: ${durationDiffSign}${summary.comparison.durationDiff.toFixed(0)}ms`);
+    return lines.join('\n');
+}
+//# sourceMappingURL=metrics-segmenter.js.map

package/dist/core/experiments/metrics-segmenter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics-segmenter.js","sourceRoot":"","sources":["../../../src/core/experiments/metrics-segmenter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAA;AA+EjE,gFAAgF;AAChF,kBAAkB;AAClB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,UAAkB,EAClB,YAAoB;IAEpB,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,UAAU,CAAC,CAAA;IAE/C,kCAAkC;IAClC,MAAM,iBAAiB,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,YAAY,CAAC,CAAA;IAEtF,iBAAiB;IACjB,MAAM,cAAc,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,SAAS,CAAC,CAAA;IACrF,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,WAAW,CAAC,CAAA;IACzF,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;IAE3E,OAAO;QACL,YAAY;QACZ,OAAO,EAAE,mBAAmB,CAAC,cAAc,CAAC;QAC5C,SAAS,EAAE,mBAAmB,CAAC,gBAAgB,CAAC;QAChD,UAAU,EAAE,mBAAmB,CAAC,iBAAiB,CAAC;KACnD,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,UAAkB,EAClB,YAAoB;IAEpB,MAAM,SAAS,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;IAE1E,MAAM,qBAAqB,GACzB,SAAS,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3F,MAAM,uBAAuB,GAC3B,SAAS,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IAEjG,MAAM,qBAAqB,GACzB,SAAS,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3F,MAAM,uBAAuB,GAC3B,SAAS,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IAEjG,OAAO;QACL,YAAY;QACZ,OAAO,EAAE;YACP,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,KAAK;YACnC,WAAW,EAAE,SAAS,CAAC,OAAO,CAAC,WAAW;YAC1C,cAAc,EAAE,qBAAqB;YACrC,cAAc,EAAE,qBAAqB;YACrC,aAAa,EAAE,SAAS,CAAC,OAAO,CAAC,aAAa;SAC/C;QACD,SAAS,EAAE;YACT,UAAU,EAAE,SAAS,CAAC,SAAS,CAAC,KAAK;YACrC,WAAW,EAAE,SAAS,CAAC,SAAS,CAAC,WAAW;YAC5C,cAAc,EAAE,uBAAuB;YACvC,cAAc,EAAE,uBAAuB;YACvC,aAAa,EAAE,SAAS,CAAC,SAAS,CAAC,aAAa;SACjD;QACD,UAAU,EAAE;YACV,cAAc,EAAE,SAAS,CAAC,SAAS,CAAC,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK;YACnE,eAAe,EAAE,SAAS,CAAC,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,WAAW;YAChF,eAAe,EAAE,uBAAuB,GAAG,qBAAqB;YAChE,eAAe,EAAE,uBAAuB,GAAG,qBAAqB;YAChE,YAAY,EAAE,SAAS,CAAC,SAAS,CAAC,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,aAAa;SAClF;KACF,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oCAAoC,CACxD,UAAkB,EAClB,YAAoB,EACpB,aAA4B;IAE5B,MAAM,SAAS,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;IAE1E,oCAAoC;IACpC,OAAO;QACL,YAAY;QACZ,OAAO,EAAE,oBAAoB,CAAC,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC;QAC/D,SAAS,EAAE,oBAAoB,CAAC,SAAS,CAAC,SAAS,EAAE,aAAa,CAAC;QACnE,UAAU,EAAE,oBAAoB,CAAC,SAAS,CAAC,UAAU,EAAE,aAAa,CAAC;KACtE,CAAA;AACH,CAAC;AAED,gFAAgF;AAChF,mBAAmB;AACnB,gFAAgF;AAEhF;;GAEG;AACH,SAAS,mBAAmB,CAAC,OAAsB;IACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,KAAK,EAAE,CAAC;YACR,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,CAAC;YAChB,mBAAmB,EAAE,CAAC;YACtB,OAAO,EAAE,EAAE;SACZ,CAAA;IACH,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,IAAI,WAAW,GAAG,CAAC,CAAA;IAEnB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAAE,YAAY,EAAE,CAAA;QAC/C,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAAE,YAAY,EAAE,CAAA;QAC/C,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAAE,YAAY,EAAE,CAAA;QAE/C,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAA;QACjC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAA;QACjC,aAAa,IAAI,KAAK,CAAC,UAAU,CAAA;QACjC,WAAW,IAAI,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAA;IAC/C,CAAC;IAED,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,YAAY;QACZ,YAAY;QACZ,YAAY;QACZ,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACnE,WAAW;QACX,WAAW;QACX,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM;QAC7C,mBAAmB,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM;QACjD,OAAO;KACR,CAAA;AACH,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,GAAe,EAAE,aAA4B;IACzE,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,aAAa,CAAC,CAAA;IAC7E,OAAO,mBAAmB,CAAC,QAAQ,CAAC,CAAA;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAA0B;IAChE,MAAM,KAAK,GAAa,EAAE,CAAA;IAE1B,KAAK,CAAC,IAAI,CAAC,0BAA0B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAC5D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEd,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAC7B,KAAK,CAAC,IAAI,CAAC,kBAAkB,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;IAC1D,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IAChF,KAAK,CAAC,IAAI,CAAC,uBAAuB,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC9E,KAAK,CAAC,IAAI,CAAC,mBAAmB,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAC3E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEd,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;IAC/B,KAAK,CAAC,IAAI,CAAC,kBAAkB,OAAO,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,CAAA;IAC5D,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IAClF,KAAK,CAAC,IAAI,CAAC,uBAAuB,OAAO,CAAC,SAAS,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAChF,KAAK,CAAC,IAAI,CAAC,mBAAmB,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAC7E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEd,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAA;IAClD,MAAM,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;IAC1E,MAAM,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;IACzE,MAAM,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;IAExE,KAAK,CAAC,IAAI,CAAC,uBAAuB,OAAO,CAAC,UAAU,CAAC,cAAc,EAAE,CAAC,CAAA;IACtE,KAAK,CAAC,IAAI,CACR,wBAAwB,eAAe,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACnG,CAAA;IACD,KAAK,CAAC,IAAI,CACR,wBAAwB,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACzF,CAAA;IACD,KAAK,CAAC,IAAI,CAAC,oBAAoB,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAEjG,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC"}

package/dist/core/experiments/statistical-analyzer.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Statistical Analyzer - v17.0.0
+ *
+ * Performs statistical significance testing for A/B testing experiments.
+ * Implements chi-square and Welch's t-test without external dependencies.
+ *
+ * Features:
+ * - Chi-square test for success rate comparison
+ * - Welch's t-test for continuous metrics
+ * - Auto-conclusion logic based on sample size and p-value
+ * - Integration with experiment store and metrics segmenter
+ */
+import type { ExperimentWinner } from './experiment-schema.js';
+import type { ExperimentSummary } from './metrics-segmenter.js';
+/**
+ * Statistical test result
+ */
+export interface TestResult {
+    /** Test statistic value */
+    statistic: number;
+    /** P-value (probability) */
+    pValue: number;
+    /** Degrees of freedom */
+    degreesOfFreedom?: number;
+    /** Is result significant at 0.05 level? */
+    isSignificant: boolean;
+}
+/**
+ * Complete experiment analysis
+ */
+export interface ExperimentAnalysis {
+    /** Experiment ID */
+    experimentId: string;
+    /** Experiment summary stats */
+    summary: ExperimentSummary;
+    /** Success rate test result */
+    successRateTest: TestResult;
+    /** Token savings test result (continuous metric) */
+    tokensSavedTest: TestResult;
+    /** Tokens freed test result (continuous metric) */
+    tokensFreedTest: TestResult;
+    /** Duration test result (continuous metric) */
+    durationTest: TestResult;
+    /** Recommended winner based on analysis */
+    recommendedWinner: ExperimentWinner;
+    /** Can auto-conclude? */
+    canAutoConclude: boolean;
+    /** Reasons preventing auto-conclusion */
+    blockingReasons: string[];
+}
+/**
+ * Analyze an experiment and return full statistical analysis
+ */
+export declare function analyzeExperiment(projectDir: string, experimentId: string): Promise<ExperimentAnalysis>;
+/**
+ * Auto-conclude experiment if conditions are met
+ *
+ * @returns true if experiment was concluded, false otherwise
+ */
+export declare function autoConcludeExperiment(projectDir: string, experimentId: string): Promise<boolean>;
+/**
+ * Calculate significance for two groups (success counts)
+ *
+ * Convenience function for external use.
+ */
+export declare function calculateSignificance(controlSuccess: number, controlTotal: number, treatmentSuccess: number, treatmentTotal: number): TestResult;
+/**
+ * Format analysis for display
+ */
+export declare function formatAnalysis(analysis: ExperimentAnalysis): string;
+//# sourceMappingURL=statistical-analyzer.d.ts.map

package/dist/core/experiments/statistical-analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistical-analyzer.d.ts","sourceRoot":"","sources":["../../../src/core/experiments/statistical-analyzer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAc,gBAAgB,EAAE,MAAM,wBAAwB,CAAA;AAE1E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAO/D;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,yBAAyB;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,2CAA2C;IAC3C,aAAa,EAAE,OAAO,CAAA;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,+BAA+B;IAC/B,OAAO,EAAE,iBAAiB,CAAA;IAC1B,+BAA+B;IAC/B,eAAe,EAAE,UAAU,CAAA;IAC3B,oDAAoD;IACpD,eAAe,EAAE,UAAU,CAAA;IAC3B,mDAAmD;IACnD,eAAe,EAAE,UAAU,CAAA;IAC3B,+CAA+C;IAC/C,YAAY,EAAE,UAAU,CAAA;IACxB,2CAA2C;IAC3C,iBAAiB,EAAE,gBAAgB,CAAA;IACnC,yBAAyB;IACzB,eAAe,EAAE,OAAO,CAAA;IACxB,yCAAyC;IACzC,eAAe,EAAE,MAAM,EAAE,CAAA;CAC1B;AAgLD;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,kBAAkB,CAAC,CA8D7B;AAsHD;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,OAAO,CAAC,CAWlB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,MAAM,EACxB,cAAc,EAAE,MAAM,GACrB,UAAU,CAKZ;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,kBAAkB,GAAG,MAAM,CA8CnE"}