@hmharness/evaluation 0.8.3 → 0.8.5

package/dist/index.d.ts

@@ -3,3 +3,5 @@ export * from './evaluators.ts';
 export * from './runner.ts';
 export * from './harmonybench.ts';
 export * from './score.ts';
+export * from './sdk.ts';
+export * from './marketplace.ts';

package/dist/index.js

@@ -3,3 +3,5 @@ export * from "./evaluators.js";
 export * from "./runner.js";
 export * from "./harmonybench.js";
 export * from "./score.js";
+export * from "./sdk.js";
+export * from "./marketplace.js";

package/dist/marketplace.d.ts

@@ -0,0 +1,82 @@
+/**
+ * @hmharness/evaluation - Benchmark Marketplace (P2-07)
+ *
+ * The audit called for: "允许外部贡献任务，但生产 gate 只读取签名/审核后的 fixture"
+ *
+ * Provides governance for external benchmark contributions:
+ * 1. Contribution submission with metadata
+ * 2. Review/approval workflow
+ * 3. Signature verification (simulated - real crypto in production)
+ * 4. Registry of approved vs pending vs rejected fixtures
+ */
+export type FixtureStatus = 'pending' | 'approved' | 'rejected' | 'deprecated';
+export interface BenchmarkFixture {
+    id: string;
+    name: string;
+    description: string;
+    category: string;
+    difficulty: 1 | 2 | 3;
+    prompt: string;
+    expectedOutput: string;
+    assertionType: 'exact' | 'contains' | 'regex';
+    assertionValue: string;
+    /** contributor metadata */
+    contributor: string;
+    submittedAt: string;
+    /** governance */
+    status: FixtureStatus;
+    reviewedBy?: string;
+    reviewedAt?: string;
+    reviewNotes?: string;
+    /** content hash for integrity verification */
+    contentHash: string;
+    /** signature (simulated - real HMAC in production) */
+    signature?: string;
+}
+export interface MarketplaceStats {
+    total: number;
+    approved: number;
+    pending: number;
+    rejected: number;
+    byCategory: Record<string, number>;
+}
+/**
+ * Compute a content hash for a fixture (simple but deterministic).
+ * Pure - testable.
+ */
+export declare function hashFixture(fixture: Omit<BenchmarkFixture, 'contentHash' | 'status' | 'submittedAt'>): string;
+/**
+ * Sign a fixture (simulated HMAC).
+ * Pure - testable.
+ */
+export declare function signFixture(fixture: BenchmarkFixture, secret: string): string;
+/**
+ * Verify a fixture signature.
+ * Pure - testable.
+ */
+export declare function verifySignature(fixture: BenchmarkFixture, secret: string): boolean;
+/**
+ * Validate a contributed fixture for completeness.
+ * Pure - testable.
+ */
+export declare function validateFixture(fixture: unknown): {
+    valid: boolean;
+    errors: string[];
+};
+/**
+ * The fixture registry - manages the marketplace lifecycle.
+ */
+export declare class FixtureRegistry {
+    private fixtures;
+    submit(fixture: BenchmarkFixture): {
+        ok: boolean;
+        reason?: string;
+    };
+    review(id: string, approved: boolean, reviewer: string, notes?: string): boolean;
+    deprecate(id: string): boolean;
+    /** Only approved fixtures are usable in production gates */
+    getApproved(): BenchmarkFixture[];
+    getPending(): BenchmarkFixture[];
+    get(id: string): BenchmarkFixture | undefined;
+    stats(): MarketplaceStats;
+}

package/dist/marketplace.js

@@ -0,0 +1,125 @@
+/**
+ * @hmharness/evaluation - Benchmark Marketplace (P2-07)
+ *
+ * The audit called for: "允许外部贡献任务，但生产 gate 只读取签名/审核后的 fixture"
+ *
+ * Provides governance for external benchmark contributions:
+ * 1. Contribution submission with metadata
+ * 2. Review/approval workflow
+ * 3. Signature verification (simulated - real crypto in production)
+ * 4. Registry of approved vs pending vs rejected fixtures
+ */
+/**
+ * Compute a content hash for a fixture (simple but deterministic).
+ * Pure - testable.
+ */
+export function hashFixture(fixture) {
+    const content = JSON.stringify({
+        name: fixture.name, prompt: fixture.prompt,
+        expected: fixture.expectedOutput, assertion: fixture.assertionValue,
+    });
+    let hash = 0;
+    for (let i = 0; i < content.length; i++) {
+        hash = ((hash << 5) - hash + content.charCodeAt(i)) | 0;
+    }
+    return `fx-${Math.abs(hash).toString(16).padStart(8, '0')}`;
+}
+/**
+ * Sign a fixture (simulated HMAC).
+ * Pure - testable.
+ */
+export function signFixture(fixture, secret) {
+    const payload = `${fixture.id}:${fixture.contentHash}:${secret}`;
+    let hash = 0;
+    for (let i = 0; i < payload.length; i++) {
+        hash = ((hash << 5) - hash + payload.charCodeAt(i)) | 0;
+    }
+    return `sig-${Math.abs(hash).toString(16)}`;
+}
+/**
+ * Verify a fixture signature.
+ * Pure - testable.
+ */
+export function verifySignature(fixture, secret) {
+    if (!fixture.signature)
+        return false;
+    return fixture.signature === signFixture(fixture, secret);
+}
+/**
+ * Validate a contributed fixture for completeness.
+ * Pure - testable.
+ */
+export function validateFixture(fixture) {
+    const errors = [];
+    const f = fixture;
+    if (!f.name)
+        errors.push('name is required');
+    if (!f.prompt)
+        errors.push('prompt is required');
+    if (!f.contributor)
+        errors.push('contributor is required');
+    if (!f.category)
+        errors.push('category is required');
+    if (f.difficulty === undefined || ![1, 2, 3].includes(f.difficulty))
+        errors.push('difficulty must be 1, 2, or 3');
+    if (!f.assertionType || !['exact', 'contains', 'regex'].includes(f.assertionType))
+        errors.push('assertionType must be exact/contains/regex');
+    if (!f.assertionValue)
+        errors.push('assertionValue is required');
+    return { valid: errors.length === 0, errors };
+}
+/**
+ * The fixture registry - manages the marketplace lifecycle.
+ */
+export class FixtureRegistry {
+    fixtures = new Map();
+    submit(fixture) {
+        const v = validateFixture(fixture);
+        if (!v.valid)
+            return { ok: false, reason: v.errors.join('; ') };
+        if (this.fixtures.has(fixture.id))
+            return { ok: false, reason: `fixture ${fixture.id} already exists` };
+        this.fixtures.set(fixture.id, fixture);
+        return { ok: true };
+    }
+    review(id, approved, reviewer, notes) {
+        const f = this.fixtures.get(id);
+        if (!f)
+            return false;
+        f.status = approved ? 'approved' : 'rejected';
+        f.reviewedBy = reviewer;
+        f.reviewedAt = new Date().toISOString();
+        f.reviewNotes = notes;
+        return true;
+    }
+    deprecate(id) {
+        const f = this.fixtures.get(id);
+        if (!f)
+            return false;
+        f.status = 'deprecated';
+        return true;
+    }
+    /** Only approved fixtures are usable in production gates */
+    getApproved() {
+        return [...this.fixtures.values()].filter(f => f.status === 'approved');
+    }
+    getPending() {
+        return [...this.fixtures.values()].filter(f => f.status === 'pending');
+    }
+    get(id) {
+        return this.fixtures.get(id);
+    }
+    stats() {
+        const all = [...this.fixtures.values()];
+        const byCategory = {};
+        for (const f of all)
+            byCategory[f.category] = (byCategory[f.category] ?? 0) + 1;
+        return {
+            total: all.length,
+            approved: all.filter(f => f.status === 'approved').length,
+            pending: all.filter(f => f.status === 'pending').length,
+            rejected: all.filter(f => f.status === 'rejected').length,
+            byCategory,
+        };
+    }
+}

package/dist/sdk.d.ts

@@ -0,0 +1,83 @@
+/**
+ * @hmharness/evaluation - Evaluator SDK (P1-02)
+ *
+ * The audit called for: "统一 evaluator plugin、evidence references、metric registry"
+ *
+ * This module provides the plugin interface that ALL evaluators implement,
+ * evidence references that link scores to concrete proof, and a metric
+ * registry for named, versioned scoring functions.
+ */
+/** Evidence tier per the existing evidence ladder (M2) */
+export type EvidenceTier = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8;
+/** Reference to concrete evidence backing an evaluation */
+export interface EvidenceRef {
+    /** what kind of evidence */
+    kind: 'build-log' | 'test-result' | 'command-output' | 'file-content' | 'judge-score' | 'device-log' | 'metric';
+    /** where to find it (path, URL, or inline) */
+    source: string;
+    /** relevant excerpt (for audit trail) */
+    excerpt?: string;
+    /** evidence tier (1=build, 7=llm-judge, 8=self-report) */
+    tier: EvidenceTier;
+}
+/** The result of running an evaluator */
+export interface SdkEvaluationResult {
+    /** evaluator that produced this result */
+    evaluatorId: string;
+    /** pass/fail/indeterminate */
+    outcome: 'pass' | 'fail' | 'indeterminate';
+    /** numeric score [0,1] if applicable */
+    score?: number;
+    /** human-readable detail */
+    detail: string;
+    /** evidence backing this result */
+    evidence: EvidenceRef[];
+    /** when this evaluation ran */
+    evaluatedAt: string;
+    /** duration in ms */
+    durationMs: number;
+}
+/** The evaluator plugin interface - ALL evaluators implement this */
+export interface SdkEvaluator {
+    /** unique evaluator id */
+    readonly id: string;
+    /** what this evaluator checks */
+    readonly description: string;
+    /** evidence tier this evaluator operates at */
+    readonly tier: EvidenceTier;
+    /** evaluate and return a result */
+    evaluate(input: unknown): Promise<SdkEvaluationResult>;
+}
+/** A named metric in the registry */
+export interface MetricDef {
+    name: string;
+    description: string;
+    /** units (e.g. 'percent', 'count', 'ms') */
+    unit: string;
+    /** higher is better? (for display) */
+    higherIsBetter: boolean;
+    /** compute from evaluation results */
+    compute: (results: SdkEvaluationResult[]) => number;
+}
+/**
+ * The metric registry - named, versioned scoring functions.
+ * Metrics are registered once and referenced by name everywhere.
+ */
+export declare class MetricRegistry {
+    private metrics;
+    register(def: MetricDef): {
+        ok: boolean;
+        reason?: string;
+    };
+    get(name: string): MetricDef | undefined;
+    list(): MetricDef[];
+    /** Compute a metric from evaluation results */
+    compute(name: string, results: SdkEvaluationResult[]): number | undefined;
+}
+/** Create a standard metric registry with common metrics pre-registered */
+export declare function createDefaultRegistry(): MetricRegistry;
+/**
+ * Create a simple pass/fail evaluator from a predicate function.
+ * Convenience factory for the most common evaluator pattern.
+ */
+export declare function createPredicateEvaluator(id: string, description: string, tier: EvidenceTier, predicate: (input: unknown) => boolean, evidenceKind?: EvidenceRef['kind']): SdkEvaluator;

package/dist/sdk.js

@@ -0,0 +1,85 @@
+/**
+ * @hmharness/evaluation - Evaluator SDK (P1-02)
+ *
+ * The audit called for: "统一 evaluator plugin、evidence references、metric registry"
+ *
+ * This module provides the plugin interface that ALL evaluators implement,
+ * evidence references that link scores to concrete proof, and a metric
+ * registry for named, versioned scoring functions.
+ */
+/**
+ * The metric registry - named, versioned scoring functions.
+ * Metrics are registered once and referenced by name everywhere.
+ */
+export class MetricRegistry {
+    metrics = new Map();
+    register(def) {
+        if (this.metrics.has(def.name)) {
+            return { ok: false, reason: `metric ${def.name} already registered` };
+        }
+        this.metrics.set(def.name, def);
+        return { ok: true };
+    }
+    get(name) {
+        return this.metrics.get(name);
+    }
+    list() {
+        return [...this.metrics.values()];
+    }
+    /** Compute a metric from evaluation results */
+    compute(name, results) {
+        const m = this.metrics.get(name);
+        if (!m)
+            return undefined;
+        return m.compute(results);
+    }
+}
+/** Create a standard metric registry with common metrics pre-registered */
+export function createDefaultRegistry() {
+    const reg = new MetricRegistry();
+    reg.register({
+        name: 'pass_rate', description: 'fraction of evaluations that passed',
+        unit: 'fraction', higherIsBetter: true,
+        compute: (rs) => rs.length > 0 ? rs.filter(r => r.outcome === 'pass').length / rs.length : 0,
+    });
+    reg.register({
+        name: 'avg_score', description: 'average numeric score',
+        unit: 'fraction', higherIsBetter: true,
+        compute: (rs) => {
+            const scored = rs.filter(r => r.score !== undefined);
+            return scored.length > 0 ? scored.reduce((a, b) => a + (b.score ?? 0), 0) / scored.length : 0;
+        },
+    });
+    reg.register({
+        name: 'avg_duration', description: 'average evaluation duration',
+        unit: 'ms', higherIsBetter: false,
+        compute: (rs) => rs.length > 0 ? rs.reduce((a, b) => a + b.durationMs, 0) / rs.length : 0,
+    });
+    reg.register({
+        name: 'evidence_coverage', description: 'fraction of results with evidence',
+        unit: 'fraction', higherIsBetter: true,
+        compute: (rs) => rs.length > 0 ? rs.filter(r => r.evidence.length > 0).length / rs.length : 0,
+    });
+    return reg;
+}
+/**
+ * Create a simple pass/fail evaluator from a predicate function.
+ * Convenience factory for the most common evaluator pattern.
+ */
+export function createPredicateEvaluator(id, description, tier, predicate, evidenceKind = 'command-output') {
+    return {
+        id, description, tier,
+        async evaluate(input) {
+            const start = Date.now();
+            const pass = predicate(input);
+            return {
+                evaluatorId: id,
+                outcome: pass ? 'pass' : 'fail',
+                detail: pass ? 'predicate satisfied' : 'predicate not satisfied',
+                evidence: [{ kind: evidenceKind, source: 'inline', tier, excerpt: String(input).slice(0, 200) }],
+                evaluatedAt: new Date().toISOString(),
+                durationMs: Date.now() - start,
+            };
+        },
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evaluation",
-  "version": "0.8.3",
+  "version": "0.8.5",
   "description": "hmharness evaluation: the Evaluator/Judge contract (V2 blueprint M2). Hard evidence outranks LLM judgment - build results, exit codes, exact/regex assertions first; the LLM judge is a last resort and is labeled as such. Evaluations attach to trajectories (judge.completed events).",
   "type": "module",
   "main": "dist/index.js",