@hmharness/evolution 0.14.16 → 0.14.18

package/dist/external.d.ts

@@ -0,0 +1,80 @@
+/**
+ * @hmharness/evolution - External Task Injection (P0-06)
+ *
+ * The audit's anti-self-validation requirement: "如果 Agent 自己生成任务、
+ * 自己执行、自己评测，就很容易得到自证式 improvement"
+ *
+ * This module provides:
+ * 1. External task intake: real user/external project tasks enter the system
+ * 2. Holdout marking: external tasks go to a SEPARATE holdout set
+ * 3. Contamination prevention: external holdout tasks NEVER enter training
+ * 4. Independence verification: holdout results are measured separately
+ */
+export interface ExternalTask {
+    /** unique id */
+    id: string;
+    /** the task prompt (from a real user, not self-generated) */
+    prompt: string;
+    /** source identifier (e.g. "github-issue", "user-request", "external-benchmark") */
+    source: string;
+    /** when this task was submitted */
+    submittedAt: string;
+    /** expected outcome (if known) for verification */
+    expectedOutcome?: string;
+    /** the agent's actual result (filled after execution) */
+    actualResult?: string;
+    /** whether this task passed */
+    passed?: boolean;
+    /** metadata */
+    metadata?: Record<string, unknown>;
+}
+export interface HoldoutReport {
+    totalExternal: number;
+    totalEvaluated: number;
+    passRate: number;
+    bySource: Record<string, {
+        total: number;
+        passed: number;
+    }>;
+    /** contamination check: any external tasks found in training data? */
+    contaminationDetected: boolean;
+    contaminationCount: number;
+}
+/**
+ * Submit an external task into the system.
+ * External tasks are marked separately and NEVER enter the training set.
+ */
+export declare function submitExternalTask(home: string, task: {
+    prompt: string;
+    source: string;
+    expectedOutcome?: string;
+    metadata?: Record<string, unknown>;
+}): Promise<ExternalTask>;
+/**
+ * Read all external tasks.
+ */
+export declare function readExternalTasks(home: string): Promise<ExternalTask[]>;
+/**
+ * CRITICAL: Check that no external holdout tasks have leaked into
+ * the training data (DPO pairs). This is the audit's anti-cheating mechanism.
+ *
+ * Pure - testable.
+ */
+export declare function checkContamination(externalTasks: ExternalTask[], trainingPairPrompts: string[]): {
+    contaminated: boolean;
+    count: number;
+    details: string[];
+};
+/**
+ * Build a holdout report from external task results.
+ * Pure - testable.
+ */
+export declare function buildHoldoutReport(tasks: ExternalTask[], contaminationResult: {
+    contaminated: boolean;
+    count: number;
+}): HoldoutReport;
+/**
+ * Generate a summary of external task status.
+ * Pure - testable.
+ */
+export declare function externalTaskSummary(tasks: ExternalTask[]): string;

package/dist/external.js

@@ -0,0 +1,102 @@
+/**
+ * @hmharness/evolution - External Task Injection (P0-06)
+ *
+ * The audit's anti-self-validation requirement: "如果 Agent 自己生成任务、
+ * 自己执行、自己评测，就很容易得到自证式 improvement"
+ *
+ * This module provides:
+ * 1. External task intake: real user/external project tasks enter the system
+ * 2. Holdout marking: external tasks go to a SEPARATE holdout set
+ * 3. Contamination prevention: external holdout tasks NEVER enter training
+ * 4. Independence verification: holdout results are measured separately
+ */
+import { readFile, appendFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+const externalFile = (home) => join(home, 'evolution', 'external-tasks.jsonl');
+/**
+ * Submit an external task into the system.
+ * External tasks are marked separately and NEVER enter the training set.
+ */
+export async function submitExternalTask(home, task) {
+    const dir = join(home, 'evolution');
+    await mkdir(dir, { recursive: true });
+    const ext = {
+        id: `ext-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`,
+        prompt: task.prompt,
+        source: task.source,
+        submittedAt: new Date().toISOString(),
+        expectedOutcome: task.expectedOutcome,
+        metadata: task.metadata,
+    };
+    await appendFile(externalFile(home), JSON.stringify(ext) + '\n', 'utf8');
+    return ext;
+}
+/**
+ * Read all external tasks.
+ */
+export async function readExternalTasks(home) {
+    try {
+        const text = await readFile(externalFile(home), 'utf8');
+        return text.split('\n').filter(Boolean).map(l => JSON.parse(l));
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * CRITICAL: Check that no external holdout tasks have leaked into
+ * the training data (DPO pairs). This is the audit's anti-cheating mechanism.
+ *
+ * Pure - testable.
+ */
+export function checkContamination(externalTasks, trainingPairPrompts) {
+    const externalPrompts = new Set(externalTasks.map(t => t.prompt.trim().toLowerCase()));
+    const details = [];
+    let count = 0;
+    for (const prompt of trainingPairPrompts) {
+        const normalized = prompt.trim().toLowerCase();
+        // exact match or high overlap (first 100 chars)
+        const prefix = normalized.slice(0, 100);
+        if (externalPrompts.has(normalized) || [...externalPrompts].some(ep => ep.startsWith(prefix))) {
+            count++;
+            details.push(`training prompt matches external task: "${prompt.slice(0, 60)}..."`);
+        }
+    }
+    return { contaminated: count > 0, count, details };
+}
+/**
+ * Build a holdout report from external task results.
+ * Pure - testable.
+ */
+export function buildHoldoutReport(tasks, contaminationResult) {
+    const evaluated = tasks.filter(t => t.passed !== undefined);
+    const passed = evaluated.filter(t => t.passed).length;
+    const bySource = {};
+    for (const t of evaluated) {
+        const src = t.source || 'unknown';
+        if (!bySource[src])
+            bySource[src] = { total: 0, passed: 0 };
+        bySource[src].total++;
+        if (t.passed)
+            bySource[src].passed++;
+    }
+    return {
+        totalExternal: tasks.length,
+        totalEvaluated: evaluated.length,
+        passRate: evaluated.length > 0 ? passed / evaluated.length : 0,
+        bySource,
+        contaminationDetected: contaminationResult.contaminated,
+        contaminationCount: contaminationResult.count,
+    };
+}
+/**
+ * Generate a summary of external task status.
+ * Pure - testable.
+ */
+export function externalTaskSummary(tasks) {
+    const pending = tasks.filter(t => t.passed === undefined).length;
+    const evaluated = tasks.filter(t => t.passed !== undefined).length;
+    const passed = tasks.filter(t => t.passed === true).length;
+    const sources = [...new Set(tasks.map(t => t.source))];
+    return `external tasks: ${tasks.length} total, ${pending} pending, ${evaluated} evaluated (${passed} passed), sources: ${sources.join(', ') || 'none'}`;
+}

package/dist/index.d.ts

@@ -17,3 +17,4 @@ export * from './impact.ts';
 export * from './workflows.ts';
 export * from './knowledge.ts';
 export * from './radar.ts';
+export * from './external.ts';

package/dist/index.js

@@ -17,3 +17,4 @@ export * from "./impact.js";
 export * from "./workflows.js";
 export * from "./knowledge.js";
 export * from "./radar.js";
+export * from "./external.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evolution",
-  "version": "0.14.16",
+  "version": "0.14.18",
   "description": "hmharness evolution subsystem: persistent memory, insight capture, skill library, and the bench that gives evolution its fitness signal. First-class kernel citizen, not a plugin.",
   "type": "module",
   "main": "dist/index.js",
@@ -15,7 +15,7 @@
     "build": "tsc -p tsconfig.build.json"
   },
   "dependencies": {
-    "@hmharness/kernel": "0.9.3"
+    "@hmharness/kernel": "0.9.4"
   },
   "files": [
     "dist"