@hmharness/evolution 0.14.15 → 0.14.18

package/dist/external.d.ts

@@ -0,0 +1,80 @@
+/**
+ * @hmharness/evolution - External Task Injection (P0-06)
+ *
+ * The audit's anti-self-validation requirement: "如果 Agent 自己生成任务、
+ * 自己执行、自己评测，就很容易得到自证式 improvement"
+ *
+ * This module provides:
+ * 1. External task intake: real user/external project tasks enter the system
+ * 2. Holdout marking: external tasks go to a SEPARATE holdout set
+ * 3. Contamination prevention: external holdout tasks NEVER enter training
+ * 4. Independence verification: holdout results are measured separately
+ */
+export interface ExternalTask {
+    /** unique id */
+    id: string;
+    /** the task prompt (from a real user, not self-generated) */
+    prompt: string;
+    /** source identifier (e.g. "github-issue", "user-request", "external-benchmark") */
+    source: string;
+    /** when this task was submitted */
+    submittedAt: string;
+    /** expected outcome (if known) for verification */
+    expectedOutcome?: string;
+    /** the agent's actual result (filled after execution) */
+    actualResult?: string;
+    /** whether this task passed */
+    passed?: boolean;
+    /** metadata */
+    metadata?: Record<string, unknown>;
+}
+export interface HoldoutReport {
+    totalExternal: number;
+    totalEvaluated: number;
+    passRate: number;
+    bySource: Record<string, {
+        total: number;
+        passed: number;
+    }>;
+    /** contamination check: any external tasks found in training data? */
+    contaminationDetected: boolean;
+    contaminationCount: number;
+}
+/**
+ * Submit an external task into the system.
+ * External tasks are marked separately and NEVER enter the training set.
+ */
+export declare function submitExternalTask(home: string, task: {
+    prompt: string;
+    source: string;
+    expectedOutcome?: string;
+    metadata?: Record<string, unknown>;
+}): Promise<ExternalTask>;
+/**
+ * Read all external tasks.
+ */
+export declare function readExternalTasks(home: string): Promise<ExternalTask[]>;
+/**
+ * CRITICAL: Check that no external holdout tasks have leaked into
+ * the training data (DPO pairs). This is the audit's anti-cheating mechanism.
+ *
+ * Pure - testable.
+ */
+export declare function checkContamination(externalTasks: ExternalTask[], trainingPairPrompts: string[]): {
+    contaminated: boolean;
+    count: number;
+    details: string[];
+};
+/**
+ * Build a holdout report from external task results.
+ * Pure - testable.
+ */
+export declare function buildHoldoutReport(tasks: ExternalTask[], contaminationResult: {
+    contaminated: boolean;
+    count: number;
+}): HoldoutReport;
+/**
+ * Generate a summary of external task status.
+ * Pure - testable.
+ */
+export declare function externalTaskSummary(tasks: ExternalTask[]): string;

package/dist/external.js

@@ -0,0 +1,102 @@
+/**
+ * @hmharness/evolution - External Task Injection (P0-06)
+ *
+ * The audit's anti-self-validation requirement: "如果 Agent 自己生成任务、
+ * 自己执行、自己评测，就很容易得到自证式 improvement"
+ *
+ * This module provides:
+ * 1. External task intake: real user/external project tasks enter the system
+ * 2. Holdout marking: external tasks go to a SEPARATE holdout set
+ * 3. Contamination prevention: external holdout tasks NEVER enter training
+ * 4. Independence verification: holdout results are measured separately
+ */
+import { readFile, appendFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+const externalFile = (home) => join(home, 'evolution', 'external-tasks.jsonl');
+/**
+ * Submit an external task into the system.
+ * External tasks are marked separately and NEVER enter the training set.
+ */
+export async function submitExternalTask(home, task) {
+    const dir = join(home, 'evolution');
+    await mkdir(dir, { recursive: true });
+    const ext = {
+        id: `ext-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`,
+        prompt: task.prompt,
+        source: task.source,
+        submittedAt: new Date().toISOString(),
+        expectedOutcome: task.expectedOutcome,
+        metadata: task.metadata,
+    };
+    await appendFile(externalFile(home), JSON.stringify(ext) + '\n', 'utf8');
+    return ext;
+}
+/**
+ * Read all external tasks.
+ */
+export async function readExternalTasks(home) {
+    try {
+        const text = await readFile(externalFile(home), 'utf8');
+        return text.split('\n').filter(Boolean).map(l => JSON.parse(l));
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * CRITICAL: Check that no external holdout tasks have leaked into
+ * the training data (DPO pairs). This is the audit's anti-cheating mechanism.
+ *
+ * Pure - testable.
+ */
+export function checkContamination(externalTasks, trainingPairPrompts) {
+    const externalPrompts = new Set(externalTasks.map(t => t.prompt.trim().toLowerCase()));
+    const details = [];
+    let count = 0;
+    for (const prompt of trainingPairPrompts) {
+        const normalized = prompt.trim().toLowerCase();
+        // exact match or high overlap (first 100 chars)
+        const prefix = normalized.slice(0, 100);
+        if (externalPrompts.has(normalized) || [...externalPrompts].some(ep => ep.startsWith(prefix))) {
+            count++;
+            details.push(`training prompt matches external task: "${prompt.slice(0, 60)}..."`);
+        }
+    }
+    return { contaminated: count > 0, count, details };
+}
+/**
+ * Build a holdout report from external task results.
+ * Pure - testable.
+ */
+export function buildHoldoutReport(tasks, contaminationResult) {
+    const evaluated = tasks.filter(t => t.passed !== undefined);
+    const passed = evaluated.filter(t => t.passed).length;
+    const bySource = {};
+    for (const t of evaluated) {
+        const src = t.source || 'unknown';
+        if (!bySource[src])
+            bySource[src] = { total: 0, passed: 0 };
+        bySource[src].total++;
+        if (t.passed)
+            bySource[src].passed++;
+    }
+    return {
+        totalExternal: tasks.length,
+        totalEvaluated: evaluated.length,
+        passRate: evaluated.length > 0 ? passed / evaluated.length : 0,
+        bySource,
+        contaminationDetected: contaminationResult.contaminated,
+        contaminationCount: contaminationResult.count,
+    };
+}
+/**
+ * Generate a summary of external task status.
+ * Pure - testable.
+ */
+export function externalTaskSummary(tasks) {
+    const pending = tasks.filter(t => t.passed === undefined).length;
+    const evaluated = tasks.filter(t => t.passed !== undefined).length;
+    const passed = tasks.filter(t => t.passed === true).length;
+    const sources = [...new Set(tasks.map(t => t.source))];
+    return `external tasks: ${tasks.length} total, ${pending} pending, ${evaluated} evaluated (${passed} passed), sources: ${sources.join(', ') || 'none'}`;
+}

package/dist/index.d.ts

@@ -7,6 +7,7 @@ export * from './readiness.ts';
 export * from './labels.ts';
 export * from './reward-model.ts';
 export * from './judge.ts';
+export * from './sequential.ts';
 export * from './insights.ts';
 export * from './skills.ts';
 export * from './bench.ts';
@@ -16,3 +17,4 @@ export * from './impact.ts';
 export * from './workflows.ts';
 export * from './knowledge.ts';
 export * from './radar.ts';
+export * from './external.ts';

package/dist/index.js

@@ -7,6 +7,7 @@ export * from "./readiness.js";
 export * from "./labels.js";
 export * from "./reward-model.js";
 export * from "./judge.js";
+export * from "./sequential.js";
 export * from "./insights.js";
 export * from "./skills.js";
 export * from "./bench.js";
@@ -16,3 +17,4 @@ export * from "./impact.js";
 export * from "./workflows.js";
 export * from "./knowledge.js";
 export * from "./radar.js";
+export * from "./external.js";

package/dist/sequential.d.ts

@@ -0,0 +1,81 @@
+/**
+ * @hmharness/evolution - Sequential Experimentation (P0-05, 2026-09-24 audit)
+ *
+ * Replaces the fixed "10 percentage points difference" threshold with
+ * Sequential Probability Ratio Test (SPRT) for binomial outcomes.
+ *
+ * The audit said: "应升级为 Sequential Probability Ratio Test / Bayesian
+ * decision / anytime-valid inference 中的一个正式方案, 并同时加入多重比较、
+ * 实验停止规则、护栏指标和效应量置信区间"
+ *
+ * SPRT advantage over fixed-threshold:
+ * - Can stop EARLY when evidence is conclusive (saves API calls)
+ * - Has provable error rates (alpha/beta are hard bounds)
+ * - Anytime-valid: the error guarantee holds at ANY stopping point
+ */
+export interface SPRTParams {
+    /** H0: p_control (null hypothesis baseline success rate) */
+    p0: number;
+    /** H1: p_treatment (alternative hypothesis success rate) */
+    p1: number;
+    /** Type I error rate (false positive): reject H0 when H0 is true */
+    alpha: number;
+    /** Type II error rate (false negative): fail to reject H0 when H1 is true */
+    beta: number;
+}
+export interface SPRTResult {
+    /** decision at current sample sizes */
+    decision: 'continue' | 'accept-h0' | 'accept-h1' | 'insufficient';
+    /** log-likelihood ratio */
+    logLR: number;
+    /** lower boundary for accepting H0 */
+    lowerBound: number;
+    /** upper boundary for accepting H1 */
+    upperBound: number;
+    /** total observations so far */
+    nControl: number;
+    nTreatment: number;
+    /** estimated effect (treatment - control success rate) */
+    effectEstimate: number;
+    /** 95% CI for the effect */
+    effectCI: [number, number];
+    /** minimum observations per arm before any decision */
+    minN: number;
+}
+/**
+ * Wald's Sequential Probability Ratio Test for comparing two binomial rates.
+ *
+ * At each observation, compute the log-likelihood ratio:
+ *   Λ = Σ log(L(p1)/L(p0))
+ *
+ * Decision boundaries:
+ *   Λ ≤ log(β/(1-α))  → accept H0 (no difference / treatment not better)
+ *   Λ ≥ log((1-β)/α)  → accept H1 (treatment IS better)
+ *   otherwise          → continue collecting data
+ */
+export declare function sprt(controlSuccesses: number, controlTotal: number, treatmentSuccesses: number, treatmentTotal: number, params: SPRTParams, minN?: number): SPRTResult;
+/**
+ * Convenience: standard SPRT for evolution experiments.
+ * H0: treatment is NOT better than control (same rate)
+ * H1: treatment is at least 5pp better
+ */
+export declare function evolutionSPRT(controlSuccesses: number, controlTotal: number, treatmentSuccesses: number, treatmentTotal: number): SPRTResult;
+/**
+ * Multiple comparison correction (Bonferroni) for running multiple
+ * experiments simultaneously - the audit's "多重比较" requirement.
+ */
+export declare function bonferroniAlpha(numExperiments: number, familyAlpha?: number): number;
+/**
+ * Guardrail check: a treatment that improves the target metric but
+ * degrades a guardrail metric beyond tolerance should be rejected.
+ * The audit's "护栏指标" requirement.
+ */
+export declare function checkGuardrails(target: SPRTResult, guardrails: Array<{
+    name: string;
+    controlValue: number;
+    treatmentValue: number;
+    maxRegression: number;
+}>): {
+    pass: boolean;
+    violations: string[];
+};

package/dist/sequential.js

@@ -0,0 +1,123 @@
+/**
+ * @hmharness/evolution - Sequential Experimentation (P0-05, 2026-09-24 audit)
+ *
+ * Replaces the fixed "10 percentage points difference" threshold with
+ * Sequential Probability Ratio Test (SPRT) for binomial outcomes.
+ *
+ * The audit said: "应升级为 Sequential Probability Ratio Test / Bayesian
+ * decision / anytime-valid inference 中的一个正式方案, 并同时加入多重比较、
+ * 实验停止规则、护栏指标和效应量置信区间"
+ *
+ * SPRT advantage over fixed-threshold:
+ * - Can stop EARLY when evidence is conclusive (saves API calls)
+ * - Has provable error rates (alpha/beta are hard bounds)
+ * - Anytime-valid: the error guarantee holds at ANY stopping point
+ */
+/**
+ * Wald's Sequential Probability Ratio Test for comparing two binomial rates.
+ *
+ * At each observation, compute the log-likelihood ratio:
+ *   Λ = Σ log(L(p1)/L(p0))
+ *
+ * Decision boundaries:
+ *   Λ ≤ log(β/(1-α))  → accept H0 (no difference / treatment not better)
+ *   Λ ≥ log((1-β)/α)  → accept H1 (treatment IS better)
+ *   otherwise          → continue collecting data
+ */
+export function sprt(controlSuccesses, controlTotal, treatmentSuccesses, treatmentTotal, params, minN = 30) {
+    const { p0, p1, alpha, beta } = params;
+    const nC = controlTotal;
+    const nT = treatmentTotal;
+    const xC = controlSuccesses;
+    const xT = treatmentSuccesses;
+    // boundaries
+    const lowerBound = Math.log(beta / (1 - alpha));
+    const upperBound = Math.log((1 - beta) / alpha);
+    // log-likelihood ratio for binomial
+    // Under H0: both arms have success rate p0
+    // Under H1: treatment has p1, control has p0
+    let logLR = 0;
+    // treatment arm observations
+    if (nT > 0) {
+        const logL1 = xT * Math.log(p1) + (nT - xT) * Math.log(1 - p1);
+        const logL0 = xT * Math.log(p0) + (nT - xT) * Math.log(1 - p0);
+        logLR += logL1 - logL0;
+    }
+    // minimum sample check
+    if (nC < minN || nT < minN) {
+        return {
+            decision: 'insufficient',
+            logLR, lowerBound, upperBound,
+            nControl: nC, nTreatment: nT,
+            effectEstimate: nT > 0 && nC > 0 ? (xT / nT) - (xC / nC) : 0,
+            effectCI: [-1, 1],
+            minN,
+        };
+    }
+    // decision
+    let decision;
+    if (logLR <= lowerBound) {
+        decision = 'accept-h0';
+    }
+    else if (logLR >= upperBound) {
+        decision = 'accept-h1';
+    }
+    else {
+        decision = 'continue';
+    }
+    // effect estimate and CI (Wald normal approximation)
+    const pControl = nC > 0 ? xC / nC : 0;
+    const pTreatment = nT > 0 ? xT / nT : 0;
+    const effect = pTreatment - pControl;
+    const seC = nC > 0 ? Math.sqrt(pControl * (1 - pControl) / nC) : 0;
+    const seT = nT > 0 ? Math.sqrt(pTreatment * (1 - pTreatment) / nT) : 0;
+    const se = Math.sqrt(seC * seC + seT * seT);
+    const z = 1.96; // 95% CI
+    const ci = [
+        Math.max(-1, effect - z * se),
+        Math.min(1, effect + z * se),
+    ];
+    return {
+        decision, logLR, lowerBound, upperBound,
+        nControl: nC, nTreatment: nT,
+        effectEstimate: effect,
+        effectCI: ci,
+        minN,
+    };
+}
+/**
+ * Convenience: standard SPRT for evolution experiments.
+ * H0: treatment is NOT better than control (same rate)
+ * H1: treatment is at least 5pp better
+ */
+export function evolutionSPRT(controlSuccesses, controlTotal, treatmentSuccesses, treatmentTotal) {
+    // estimate baseline from control
+    const p0 = controlTotal > 0 ? controlSuccesses / controlTotal : 0.5;
+    // H1: 5pp improvement (minimum meaningful effect)
+    const p1 = Math.min(0.99, p0 + 0.05);
+    return sprt(controlSuccesses, controlTotal, treatmentSuccesses, treatmentTotal, {
+        p0, p1, alpha: 0.05, beta: 0.10,
+    }, 30);
+}
+/**
+ * Multiple comparison correction (Bonferroni) for running multiple
+ * experiments simultaneously - the audit's "多重比较" requirement.
+ */
+export function bonferroniAlpha(numExperiments, familyAlpha = 0.05) {
+    return familyAlpha / Math.max(1, numExperiments);
+}
+/**
+ * Guardrail check: a treatment that improves the target metric but
+ * degrades a guardrail metric beyond tolerance should be rejected.
+ * The audit's "护栏指标" requirement.
+ */
+export function checkGuardrails(target, guardrails) {
+    const violations = [];
+    for (const g of guardrails) {
+        const regression = g.treatmentValue - g.controlValue;
+        if (regression > g.maxRegression) {
+            violations.push(`${g.name}: regression ${regression.toFixed(3)} > tolerance ${g.maxRegression}`);
+        }
+    }
+    return { pass: violations.length === 0, violations };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evolution",
-  "version": "0.14.15",
+  "version": "0.14.18",
   "description": "hmharness evolution subsystem: persistent memory, insight capture, skill library, and the bench that gives evolution its fitness signal. First-class kernel citizen, not a plugin.",
   "type": "module",
   "main": "dist/index.js",
@@ -15,7 +15,7 @@
     "build": "tsc -p tsconfig.build.json"
   },
   "dependencies": {
-    "@hmharness/kernel": "0.9.2"
+    "@hmharness/kernel": "0.9.4"
   },
   "files": [
     "dist"