agentic-qe 3.15.8 → 3.15.9

package/dist/domains/learning-optimization/interfaces.d.ts

@@ -23,6 +23,11 @@ export interface LearnedPattern {
     readonly template: PatternTemplate;
     readonly createdAt: Date;
     readonly lastUsedAt: Date;
+    /**
+     * #1109: 'assigned' when the pattern's rates were learned from assigned
+     * rewards. Untagged patterns are read by `patternRewardSourceOf`.
+     */
+    readonly rewardSource?: 'measured' | 'assigned';
 }
 export type PatternType = 'test-pattern' | 'fix-pattern' | 'optimization-pattern' | 'detection-pattern' | 'workflow-pattern' | 'failure-pattern';
 export interface PatternContext {
@@ -63,6 +68,13 @@ export interface Experience {
      * when real matched-pattern data exists, never a placeholder.
      */
     readonly appliedPatternIds?: readonly string[];
+    /**
+     * #1109: whether `result.success` and `reward` come from an observed
+     * outcome ('measured') or were assigned by the recorder before any outcome
+     * existed ('assigned'). Assigned experiences are kept but excluded from
+     * every reported rate. Untagged rows are read by `rewardSourceOf`.
+     */
+    readonly rewardSource?: 'measured' | 'assigned';
 }
 export interface StateSnapshot {
     readonly context: Record<string, unknown>;
@@ -110,9 +122,25 @@ export interface OptimizedStrategy {
     readonly objective: OptimizationObjective;
     readonly currentStrategy: Strategy;
     readonly optimizedStrategy: Strategy;
+    /** Relative improvement; an estimate, see `evaluation.estimated` (#1099). */
     readonly improvement: number;
+    /** Share of held-out experiences matching the fitted expectation. */
     readonly confidence: number;
     readonly validationResults: ValidationResult[];
+    readonly evaluation?: OptimizationEvaluation;
+}
+/**
+ * How an optimization's figures were derived (#1099). The optimizer fits on
+ * the older experiences and validates on the most recent ones it never saw.
+ */
+export interface OptimizationEvaluation {
+    readonly method: 'held-out';
+    readonly trainingSamples: number;
+    readonly heldOutSamples: number;
+    /** Figures that were modelled, not observed: the optimized strategy never ran. */
+    readonly estimated: string[];
+    /** Figures that could not be computed from the experiences given. */
+    readonly unmeasured?: string[];
 }
 export interface OptimizationObjective {
     readonly metric: string;
@@ -205,6 +233,8 @@ export interface PatternStats {
     readonly avgConfidence: number;
     readonly avgSuccessRate: number;
     readonly topPatterns: LearnedPattern[];
+    /** #1109: patterns left out of every figure because they were learned from assigned rewards. */
+    readonly assignedPatternsExcluded?: number;
 }
 /**
  * Experience Mining Service
@@ -233,9 +263,16 @@ export interface IExperienceMiningService {
     clusterExperiences(experiences: Experience[]): Promise<Result<ExperienceCluster[]>>;
 }
 export interface MinedInsights {
+    /** Every experience recorded in the range, measured or assigned. */
     readonly experienceCount: number;
-    readonly successRate: number;
-    readonly avgReward: number;
+    /** #1109: experiences with a measured outcome; the rates below use only these. */
+    readonly measuredExperienceCount?: number;
+    /** #1109: experiences left out of the rates because their reward was assigned. */
+    readonly assignedRewardsExcluded?: number;
+    /** Set when no measured experience exists; the rates are then null. */
+    readonly evidenceStatus?: 'insufficient-evidence';
+    readonly successRate: number | null;
+    readonly avgReward: number | null;
     readonly patterns: LearnedPattern[];
     readonly anomalies: ExperienceAnomaly[];
     readonly recommendations: string[];
@@ -245,7 +282,8 @@ export interface ExperienceCluster {
     readonly centroid: StateSnapshot;
     readonly experiences: Experience[];
     readonly commonActions: string[];
-    readonly avgReward: number;
+    /** Over members with a measured outcome; null when every member's reward was assigned. */
+    readonly avgReward: number | null;
 }
 export interface ExperienceAnomaly {
     readonly experienceId: string;
@@ -283,9 +321,15 @@ export interface ABTestConfig {
 }
 export interface ABTestResult {
     readonly winner: 'A' | 'B' | 'inconclusive';
+    /** Set when either strategy has fewer recorded samples than `minSamples`. */
+    readonly evidenceStatus?: 'insufficient-evidence';
+    readonly unmeasured?: string[];
     readonly strategyAMetrics: Record<string, number>;
     readonly strategyBMetrics: Record<string, number>;
-    readonly pValue: number;
+    /** Absent when there was not enough recorded evidence to test. */
+    readonly pValue?: number;
+    /** Why pValue is absent although both arms had enough samples. */
+    readonly pValueUnavailableReason?: 'zero-variance';
     readonly sampleSizeA: number;
     readonly sampleSizeB: number;
 }
@@ -295,6 +339,9 @@ export interface StrategyEvaluation {
     readonly strengths: string[];
     readonly weaknesses: string[];
     readonly improvementAreas: string[];
+    /** #1109: no measured experience was available, so no metric is reported. */
+    readonly evidenceStatus?: 'insufficient-evidence';
+    readonly assignedRewardsExcluded?: number;
 }
 /**
  * Knowledge Synthesis Service
@@ -419,15 +466,20 @@ export interface Improvement {
     readonly before: number;
     readonly after: number;
     readonly percentChange: number;
+    /** True when `after` and `percentChange` were modelled, not observed (#1099). */
+    readonly estimated?: boolean;
 }
 export interface OptimizationReport {
     readonly domainsOptimized: number;
     readonly totalStrategies: number;
+    /** Mean of the optimizers' estimated improvements (#1099). */
     readonly avgImprovement: number;
+    readonly improvementEstimated?: boolean;
     readonly byDomain: Record<DomainName, DomainOptimizationResult>;
 }
 export interface DomainOptimizationResult {
     readonly strategiesOptimized: number;
+    /** Estimated, see OptimizedStrategy.evaluation (#1099). */
     readonly avgImprovement: number;
     readonly bestStrategy: Strategy;
 }

package/dist/domains/learning-optimization/plugin.js

@@ -397,6 +397,8 @@ export class LearningOptimizationPlugin extends BaseDomainPlugin {
                 duration: 0,
             },
             reward: 0.8,
+            // #1109: generation is recorded before any run, so no outcome exists yet.
+            rewardSource: 'assigned',
             // ADR-110: real matched-pattern IDs from test-generation/coordinator.ts's
             // GeneratedTests.patternsUsed. This event always represents a successful
             // generation, so nulls won't fire from here in practice — threaded for
@@ -435,6 +437,7 @@ export class LearningOptimizationPlugin extends BaseDomainPlugin {
                 duration: payload.duration,
             },
             reward: successRate,
+            rewardSource: 'measured',
         });
         // Record production metric
         await this.productionIntel.recordMetric('test_success_rate', successRate, 'ratio', 'test-execution', ['automated']);
@@ -462,6 +465,8 @@ export class LearningOptimizationPlugin extends BaseDomainPlugin {
                 duration: 0,
             },
             reward: 1 - payload.riskScore,
+            // #1109: finding a gap is not an observed success of the detection.
+            rewardSource: 'assigned',
         });
     }
     async handleQualityGate(event) {
@@ -484,6 +489,7 @@ export class LearningOptimizationPlugin extends BaseDomainPlugin {
                 duration: 0,
             },
             reward: payload.passed ? 1 : 0,
+            rewardSource: 'measured',
         });
     }
     async handleDefectPredicted(event) {
@@ -505,7 +511,9 @@ export class LearningOptimizationPlugin extends BaseDomainPlugin {
                 outcome: { probability: payload.probability },
                 duration: 0,
             },
-            reward: 0.7, // Predictions are valuable
+            reward: 0.7,
+            // #1109: the prediction has not been confirmed or refuted yet.
+            rewardSource: 'assigned',
         });
     }
     async handleImpactAnalysis(event) {
@@ -534,6 +542,8 @@ export class LearningOptimizationPlugin extends BaseDomainPlugin {
                 duration: 0,
             },
             reward: 0.8,
+            // #1109: the analysis has no observed outcome to score.
+            rewardSource: 'assigned',
         });
     }
     // ============================================================================

package/dist/domains/learning-optimization/reward-source.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Agentic QE v3 - Experience reward provenance (#1109)
+ *
+ * Some event handlers record an experience before any outcome has been
+ * observed: a generated test, a defect prediction, an impact analysis or a
+ * detected coverage gap is stored with a fixed `success: true` and a reward
+ * the handler chose. Those experiences are kept, but they are tagged
+ * `rewardSource: 'assigned'` and excluded from every reported success rate,
+ * confidence, ranking, milestone, average reward and anomaly baseline.
+ *
+ * Reading rule for rows written before the tag existed:
+ * - `rewardSource` present: it is authoritative.
+ * - absent, and the row was written by one of the event handlers that always
+ *   assigned its reward (matched on agent + action below): `assigned`.
+ * - absent otherwise: `measured`. Every other writer derived success and
+ *   reward from an observed outcome (test-run pass rate, quality-gate verdict,
+ *   captured task quality).
+ *
+ * Patterns learned before the tag existed are read the same way: a pattern
+ * whose template lists only actions from those handlers, in the handler's own
+ * domain, was mined from assigned rewards (its 100% rate came from them).
+ */
+import type { Experience, LearnedPattern } from './interfaces.js';
+export type RewardSource = 'measured' | 'assigned';
+type ProvenanceFields = Pick<Experience, 'agentId' | 'action' | 'rewardSource'>;
+export declare function rewardSourceOf(experience: ProvenanceFields): RewardSource;
+export declare function isMeasuredExperience(experience: ProvenanceFields): boolean;
+/** Split experiences into those with a measured outcome and a count of the rest. */
+export declare function partitionByRewardSource<T extends ProvenanceFields>(experiences: readonly T[]): {
+    measured: T[];
+    assignedExcluded: number;
+};
+type PatternProvenanceFields = Pick<LearnedPattern, 'domain' | 'template' | 'rewardSource'>;
+export declare function patternRewardSourceOf(pattern: PatternProvenanceFields): RewardSource;
+export declare function isMeasuredPattern(pattern: PatternProvenanceFields): boolean;
+export {};
+//# sourceMappingURL=reward-source.d.ts.map

package/dist/domains/learning-optimization/reward-source.js

@@ -0,0 +1,58 @@
+/**
+ * Agentic QE v3 - Experience reward provenance (#1109)
+ *
+ * Some event handlers record an experience before any outcome has been
+ * observed: a generated test, a defect prediction, an impact analysis or a
+ * detected coverage gap is stored with a fixed `success: true` and a reward
+ * the handler chose. Those experiences are kept, but they are tagged
+ * `rewardSource: 'assigned'` and excluded from every reported success rate,
+ * confidence, ranking, milestone, average reward and anomaly baseline.
+ *
+ * Reading rule for rows written before the tag existed:
+ * - `rewardSource` present: it is authoritative.
+ * - absent, and the row was written by one of the event handlers that always
+ *   assigned its reward (matched on agent + action below): `assigned`.
+ * - absent otherwise: `measured`. Every other writer derived success and
+ *   reward from an observed outcome (test-run pass rate, quality-gate verdict,
+ *   captured task quality).
+ *
+ * Patterns learned before the tag existed are read the same way: a pattern
+ * whose template lists only actions from those handlers, in the handler's own
+ * domain, was mined from assigned rewards (its 100% rate came from them).
+ */
+/** `agentId.value:action` pairs whose untagged rows carry an assigned reward. */
+const LEGACY_ASSIGNED = new Set([
+    'test-generation:test-generated',
+    'coverage-analysis:gap-detection',
+    'defect-intelligence:defect-prediction',
+    'code-intelligence:impact-analysis',
+]);
+export function rewardSourceOf(experience) {
+    if (experience.rewardSource)
+        return experience.rewardSource;
+    return LEGACY_ASSIGNED.has(`${experience.agentId?.value}:${experience.action}`)
+        ? 'assigned'
+        : 'measured';
+}
+export function isMeasuredExperience(experience) {
+    return rewardSourceOf(experience) === 'measured';
+}
+/** Split experiences into those with a measured outcome and a count of the rest. */
+export function partitionByRewardSource(experiences) {
+    const measured = experiences.filter(isMeasuredExperience);
+    return { measured, assignedExcluded: experiences.length - measured.length };
+}
+const COMMON_ACTIONS_LINE = /^\/\/ Common actions: (.*)$/m;
+export function patternRewardSourceOf(pattern) {
+    if (pattern.rewardSource)
+        return pattern.rewardSource;
+    const line = COMMON_ACTIONS_LINE.exec(pattern.template?.content ?? '');
+    const actions = line ? line[1].split(',').map((a) => a.trim()).filter(Boolean) : [];
+    const allAssigned = actions.length > 0
+        && actions.every((action) => LEGACY_ASSIGNED.has(`${pattern.domain}:${action}`));
+    return allAssigned ? 'assigned' : 'measured';
+}
+export function isMeasuredPattern(pattern) {
+    return patternRewardSourceOf(pattern) === 'measured';
+}
+//# sourceMappingURL=reward-source.js.map

package/dist/domains/learning-optimization/services/learning-coordinator.d.ts

@@ -120,7 +120,7 @@ export declare class LearningCoordinatorService implements IPatternLearningServi
     /**
      * Learn a pattern from a set of experiences
      */
-    learnPattern(experiences: Experience[]): Promise<Result<LearnedPattern>>;
+    learnPattern(input: Experience[]): Promise<Result<LearnedPattern>>;
     /**
      * Find patterns matching the given context
      */
@@ -153,9 +153,9 @@ export declare class LearningCoordinatorService implements IPatternLearningServi
      */
     private getNullStore;
     private recordPatternNulls;
-    /**
-     * Mine experiences for insights
-     */
+    /** Read recorded input without the pattern-mining side effects. */
+    getExperiences(domain: DomainName, timeRange: TimeRange): Promise<Result<Experience[]>>;
+    /** Mine experiences for insights. */
     mineExperiences(domain: DomainName, timeRange: TimeRange): Promise<Result<MinedInsights>>;
     /**
      * Calculate reward for an experience result

package/dist/domains/learning-optimization/services/learning-coordinator.js

@@ -8,6 +8,7 @@ import { ok, err } from '../../../shared/types/index.js';
 import { rehydrateDates } from '../../../shared/utils/kv-date-rehydrate.js';
 import { toError } from '../../../shared/error-utils.js';
 import { readActivePatterns } from '../active-patterns.js';
+import { isMeasuredPattern, partitionByRewardSource } from '../reward-source.js';
 import { createQEFlashAttention, } from '../../../integrations/ruvector/wrappers.js';
 import { getClaudeModelForTier } from '../../../shared/llm/model-registry.js';
 import { createSafeRecord } from '../../../shared/utils/safe-record.js';
@@ -39,10 +40,7 @@ const LEARNING_NS = { namespace: 'learning-optimization' };
  * provenance and archived patterns written by this service.
  */
 function readActiveLearnedPatterns(memory) {
-    return readActivePatterns(memory, {
-        retrieve: LEARNING_NS,
-        onTruncated: (scanned) => logger.warn('Active pattern key scan hit its cap; results may be incomplete', { scanned }),
-    });
+    return readActivePatterns(memory, { retrieve: LEARNING_NS });
 }
 export class LearningCoordinatorService {
     config;
@@ -216,10 +214,15 @@ Provide:
     /**
      * Learn a pattern from a set of experiences
      */
-    async learnPattern(experiences) {
+    async learnPattern(input) {
         try {
+            // #1109: an assigned reward is not evidence that the action succeeded.
+            const { measured: experiences, assignedExcluded } = partitionByRewardSource(input);
             if (experiences.length < this.config.minExperiencesForPattern) {
-                return err(new Error(`Need at least ${this.config.minExperiencesForPattern} experiences to learn a pattern`));
+                const excluded = assignedExcluded > 0
+                    ? ` (${assignedExcluded} with an assigned reward excluded)`
+                    : '';
+                return err(new Error(`Need at least ${this.config.minExperiencesForPattern} measured experiences to learn a pattern${excluded}`));
             }
             // Analyze experiences to extract common patterns
             const commonActions = this.extractCommonActions(experiences);
@@ -246,6 +249,7 @@ Provide:
                 template,
                 createdAt: new Date(),
                 lastUsedAt: new Date(),
+                rewardSource: 'measured',
             };
             // Store pattern in memory
             await this.storePattern(pattern);
@@ -262,7 +266,8 @@ Provide:
      */
     async findMatchingPatterns(context, limit = 10) {
         try {
-            const patterns = (await readActiveLearnedPatterns(this.memory)).filter((pattern) => this.matchesContext(pattern, context));
+            // #1109: a pattern learned from assigned rewards has no observed rate to rank by.
+            const patterns = (await readActiveLearnedPatterns(this.memory)).filter((pattern) => isMeasuredPattern(pattern) && this.matchesContext(pattern, context));
             // Sort by confidence and usage
             patterns.sort((a, b) => {
                 const scoreA = a.confidence * 0.6 + (a.successRate * 0.4);
@@ -370,6 +375,7 @@ Provide:
                 template: bestPattern.template,
                 createdAt: new Date(),
                 lastUsedAt: new Date(),
+                rewardSource: patterns.every(isMeasuredPattern) ? 'measured' : 'assigned',
             };
             // Store consolidated pattern
             await this.storePattern(consolidatedPattern);
@@ -388,7 +394,10 @@ Provide:
      */
     async getPatternStats(domain) {
         try {
-            const patterns = (await readActiveLearnedPatterns(this.memory)).filter((pattern) => !domain || pattern.domain === domain);
+            const inDomain = (await readActiveLearnedPatterns(this.memory)).filter((pattern) => !domain || pattern.domain === domain);
+            // #1109: every figure below describes patterns with measured rates only.
+            const patterns = inDomain.filter(isMeasuredPattern);
+            const assignedPatternsExcluded = inDomain.length - patterns.length;
             const byType = {
                 'test-pattern': 0,
                 'fix-pattern': 0,
@@ -431,6 +440,7 @@ Provide:
                 avgConfidence: patterns.length > 0 ? totalConfidence / patterns.length : 0,
                 avgSuccessRate: patterns.length > 0 ? totalSuccessRate / patterns.length : 0,
                 topPatterns,
+                ...(assignedPatternsExcluded > 0 ? { assignedPatternsExcluded } : {}),
             });
         }
         catch (error) {
@@ -518,20 +528,39 @@ Provide:
             }
         }
     }
-    /**
-     * Mine experiences for insights
-     */
+    /** Read recorded input without the pattern-mining side effects. */
+    async getExperiences(domain, timeRange) {
+        try {
+            return ok(await this.getExperiencesByDomainAndTime(domain, timeRange));
+        }
+        catch (error) {
+            return err(toError(error));
+        }
+    }
+    /** Mine experiences for insights. */
     async mineExperiences(domain, timeRange) {
         try {
-            const experiences = await this.getExperiencesByDomainAndTime(domain, timeRange);
+            const recorded = await this.getExperiencesByDomainAndTime(domain, timeRange);
+            // #1109: rates, rewards, patterns and anomalies use measured outcomes only.
+            const { measured: experiences, assignedExcluded } = partitionByRewardSource(recorded);
+            const counts = {
+                experienceCount: recorded.length,
+                measuredExperienceCount: experiences.length,
+                assignedRewardsExcluded: assignedExcluded,
+            };
             if (experiences.length === 0) {
                 return ok({
-                    experienceCount: 0,
-                    successRate: 0,
-                    avgReward: 0,
+                    ...counts,
+                    evidenceStatus: 'insufficient-evidence',
+                    successRate: null,
+                    avgReward: null,
                     patterns: [],
                     anomalies: [],
-                    recommendations: ['No experiences found in the given time range'],
+                    recommendations: [
+                        recorded.length === 0
+                            ? 'No experiences found in the given time range'
+                            : `No measured outcomes in the given time range; ${assignedExcluded} experience(s) with an assigned reward were not scored`,
+                    ],
                 });
             }
             const successRate = this.calculateSuccessRate(experiences);
@@ -540,7 +569,7 @@ Provide:
             const anomalies = this.detectAnomalies(experiences);
             const recommendations = this.generateRecommendations(experiences, successRate, anomalies);
             return ok({
-                experienceCount: experiences.length,
+                ...counts,
                 successRate,
                 avgReward,
                 patterns,
@@ -586,7 +615,10 @@ Provide:
         try {
             const keys = await this.memory.search(`learning:experience:index:agent:${agentId.value}:*`, limit, LEARNING_NS);
             const experiences = [];
-            for (const key of keys) {
+            for (const found of keys) {
+                // In-memory search includes its namespace; SQLite returns a bare key.
+                const prefix = 'learning-optimization:';
+                const key = found.startsWith(prefix) ? found.slice(prefix.length) : found;
                 const experienceId = await this.memory.get(key, LEARNING_NS);
                 if (experienceId) {
                     // #493: rehydrate `timestamp` at the kv-read seam — the sort
@@ -637,7 +669,8 @@ Provide:
                 if (clusterExperiences.length >= 2) {
                     const centroid = this.calculateCentroid(clusterExperiences);
                     const commonActions = this.extractCommonActions(clusterExperiences);
-                    const avgReward = this.calculateAverageReward(clusterExperiences);
+                    const { measured } = partitionByRewardSource(clusterExperiences);
+                    const avgReward = measured.length > 0 ? this.calculateAverageReward(measured) : null;
                     result.push({
                         id: `cluster-${clusterId++}`,
                         centroid,
@@ -704,7 +737,10 @@ Provide:
     async getExperiencesByDomainAndTime(domain, timeRange) {
         const keys = await this.memory.search(`learning:experience:index:domain:${domain}:*`, 1000, LEARNING_NS);
         const experiences = [];
-        for (const key of keys) {
+        for (const found of keys) {
+            // In-memory search includes its namespace; SQLite returns a bare key.
+            const prefix = 'learning-optimization:';
+            const key = found.startsWith(prefix) ? found.slice(prefix.length) : found;
             const experienceId = await this.memory.get(key, LEARNING_NS);
             if (experienceId) {
                 const experience = await this.memory.get(`learning:experience:${experienceId}`, LEARNING_NS);
@@ -728,10 +764,10 @@ Provide:
         return experiences;
     }
     async extractPatternsFromExperiences(experiences) {
-        // Group successful experiences by action type
-        const successfulExperiences = experiences.filter((e) => e.result.success);
+        // Group by action type. Failures stay in their group so a pattern's
+        // success rate is the observed rate, not 100% by selection (#1109).
         const actionGroups = new Map();
-        for (const exp of successfulExperiences) {
+        for (const exp of experiences) {
             const existing = actionGroups.get(exp.action) || [];
             existing.push(exp);
             actionGroups.set(exp.action, existing);

package/dist/domains/learning-optimization/services/metrics-optimizer.d.ts

@@ -35,7 +35,7 @@ export declare class MetricsOptimizerService implements IStrategyOptimizerServic
     /**
      * Optimize a strategy for a given objective
      */
-    optimizeStrategy(currentStrategy: Strategy, objective: OptimizationObjective, experiences: Experience[]): Promise<Result<OptimizedStrategy>>;
+    optimizeStrategy(currentStrategy: Strategy, objective: OptimizationObjective, input: Experience[]): Promise<Result<OptimizedStrategy>>;
     /**
      * Run A/B test between two strategies
      */
@@ -47,7 +47,7 @@ export declare class MetricsOptimizerService implements IStrategyOptimizerServic
     /**
      * Evaluate strategy performance
      */
-    evaluateStrategy(strategy: Strategy, experiences: Experience[]): Promise<Result<StrategyEvaluation>>;
+    evaluateStrategy(strategy: Strategy, input: Experience[]): Promise<Result<StrategyEvaluation>>;
     /**
      * Track metrics for a strategy
      */
@@ -60,18 +60,28 @@ export declare class MetricsOptimizerService implements IStrategyOptimizerServic
      * Calculate optimal parameters using grid search
      */
     gridSearchOptimize(parameterRanges: Record<string, number[]>, objective: OptimizationObjective, experiences: Experience[]): Promise<Result<Record<string, unknown>>>;
-    private evaluateStrategyPerformance;
+    /**
+     * Keys matching `pattern` under LEARNING_NS, ready for a namespaced get().
+     * In-memory search includes its namespace; SQLite returns a bare key.
+     */
+    private searchLearningKeys;
     private optimizeParameters;
     private scoreParameters;
     private calculateParamSimilarity;
     private predictOutcome;
     private validateOptimization;
+    /** Relative change; undefined when there is no baseline to compare with. */
     private calculateImprovement;
-    private calculateOptimizationConfidence;
     private inferDomainFromExperiences;
+    /**
+     * Values of `metric` recorded for this strategy through trackMetrics(),
+     * whose strategy id is the strategy name. Only recorded observations are
+     * returned; the caller decides whether there are enough (#1099).
+     */
     private collectStrategyMetrics;
     private calculateMean;
     private calculateStdDev;
+    /** Undefined when neither group varies: the test statistic is undefined. */
     private calculatePValue;
     private normalCDF;
     private scoreContextMatch;