@hmharness/evaluation 0.8.4 → 0.8.5

package/dist/index.d.ts

@@ -4,3 +4,4 @@ export * from './runner.ts';
 export * from './harmonybench.ts';
 export * from './score.ts';
 export * from './sdk.ts';
+export * from './marketplace.ts';

package/dist/index.js

@@ -4,3 +4,4 @@ export * from "./runner.js";
 export * from "./harmonybench.js";
 export * from "./score.js";
 export * from "./sdk.js";
+export * from "./marketplace.js";

package/dist/marketplace.d.ts

@@ -0,0 +1,82 @@
+/**
+ * @hmharness/evaluation - Benchmark Marketplace (P2-07)
+ *
+ * The audit called for: "允许外部贡献任务，但生产 gate 只读取签名/审核后的 fixture"
+ *
+ * Provides governance for external benchmark contributions:
+ * 1. Contribution submission with metadata
+ * 2. Review/approval workflow
+ * 3. Signature verification (simulated - real crypto in production)
+ * 4. Registry of approved vs pending vs rejected fixtures
+ */
+export type FixtureStatus = 'pending' | 'approved' | 'rejected' | 'deprecated';
+export interface BenchmarkFixture {
+    id: string;
+    name: string;
+    description: string;
+    category: string;
+    difficulty: 1 | 2 | 3;
+    prompt: string;
+    expectedOutput: string;
+    assertionType: 'exact' | 'contains' | 'regex';
+    assertionValue: string;
+    /** contributor metadata */
+    contributor: string;
+    submittedAt: string;
+    /** governance */
+    status: FixtureStatus;
+    reviewedBy?: string;
+    reviewedAt?: string;
+    reviewNotes?: string;
+    /** content hash for integrity verification */
+    contentHash: string;
+    /** signature (simulated - real HMAC in production) */
+    signature?: string;
+}
+export interface MarketplaceStats {
+    total: number;
+    approved: number;
+    pending: number;
+    rejected: number;
+    byCategory: Record<string, number>;
+}
+/**
+ * Compute a content hash for a fixture (simple but deterministic).
+ * Pure - testable.
+ */
+export declare function hashFixture(fixture: Omit<BenchmarkFixture, 'contentHash' | 'status' | 'submittedAt'>): string;
+/**
+ * Sign a fixture (simulated HMAC).
+ * Pure - testable.
+ */
+export declare function signFixture(fixture: BenchmarkFixture, secret: string): string;
+/**
+ * Verify a fixture signature.
+ * Pure - testable.
+ */
+export declare function verifySignature(fixture: BenchmarkFixture, secret: string): boolean;
+/**
+ * Validate a contributed fixture for completeness.
+ * Pure - testable.
+ */
+export declare function validateFixture(fixture: unknown): {
+    valid: boolean;
+    errors: string[];
+};
+/**
+ * The fixture registry - manages the marketplace lifecycle.
+ */
+export declare class FixtureRegistry {
+    private fixtures;
+    submit(fixture: BenchmarkFixture): {
+        ok: boolean;
+        reason?: string;
+    };
+    review(id: string, approved: boolean, reviewer: string, notes?: string): boolean;
+    deprecate(id: string): boolean;
+    /** Only approved fixtures are usable in production gates */
+    getApproved(): BenchmarkFixture[];
+    getPending(): BenchmarkFixture[];
+    get(id: string): BenchmarkFixture | undefined;
+    stats(): MarketplaceStats;
+}

package/dist/marketplace.js

@@ -0,0 +1,125 @@
+/**
+ * @hmharness/evaluation - Benchmark Marketplace (P2-07)
+ *
+ * The audit called for: "允许外部贡献任务，但生产 gate 只读取签名/审核后的 fixture"
+ *
+ * Provides governance for external benchmark contributions:
+ * 1. Contribution submission with metadata
+ * 2. Review/approval workflow
+ * 3. Signature verification (simulated - real crypto in production)
+ * 4. Registry of approved vs pending vs rejected fixtures
+ */
+/**
+ * Compute a content hash for a fixture (simple but deterministic).
+ * Pure - testable.
+ */
+export function hashFixture(fixture) {
+    const content = JSON.stringify({
+        name: fixture.name, prompt: fixture.prompt,
+        expected: fixture.expectedOutput, assertion: fixture.assertionValue,
+    });
+    let hash = 0;
+    for (let i = 0; i < content.length; i++) {
+        hash = ((hash << 5) - hash + content.charCodeAt(i)) | 0;
+    }
+    return `fx-${Math.abs(hash).toString(16).padStart(8, '0')}`;
+}
+/**
+ * Sign a fixture (simulated HMAC).
+ * Pure - testable.
+ */
+export function signFixture(fixture, secret) {
+    const payload = `${fixture.id}:${fixture.contentHash}:${secret}`;
+    let hash = 0;
+    for (let i = 0; i < payload.length; i++) {
+        hash = ((hash << 5) - hash + payload.charCodeAt(i)) | 0;
+    }
+    return `sig-${Math.abs(hash).toString(16)}`;
+}
+/**
+ * Verify a fixture signature.
+ * Pure - testable.
+ */
+export function verifySignature(fixture, secret) {
+    if (!fixture.signature)
+        return false;
+    return fixture.signature === signFixture(fixture, secret);
+}
+/**
+ * Validate a contributed fixture for completeness.
+ * Pure - testable.
+ */
+export function validateFixture(fixture) {
+    const errors = [];
+    const f = fixture;
+    if (!f.name)
+        errors.push('name is required');
+    if (!f.prompt)
+        errors.push('prompt is required');
+    if (!f.contributor)
+        errors.push('contributor is required');
+    if (!f.category)
+        errors.push('category is required');
+    if (f.difficulty === undefined || ![1, 2, 3].includes(f.difficulty))
+        errors.push('difficulty must be 1, 2, or 3');
+    if (!f.assertionType || !['exact', 'contains', 'regex'].includes(f.assertionType))
+        errors.push('assertionType must be exact/contains/regex');
+    if (!f.assertionValue)
+        errors.push('assertionValue is required');
+    return { valid: errors.length === 0, errors };
+}
+/**
+ * The fixture registry - manages the marketplace lifecycle.
+ */
+export class FixtureRegistry {
+    fixtures = new Map();
+    submit(fixture) {
+        const v = validateFixture(fixture);
+        if (!v.valid)
+            return { ok: false, reason: v.errors.join('; ') };
+        if (this.fixtures.has(fixture.id))
+            return { ok: false, reason: `fixture ${fixture.id} already exists` };
+        this.fixtures.set(fixture.id, fixture);
+        return { ok: true };
+    }
+    review(id, approved, reviewer, notes) {
+        const f = this.fixtures.get(id);
+        if (!f)
+            return false;
+        f.status = approved ? 'approved' : 'rejected';
+        f.reviewedBy = reviewer;
+        f.reviewedAt = new Date().toISOString();
+        f.reviewNotes = notes;
+        return true;
+    }
+    deprecate(id) {
+        const f = this.fixtures.get(id);
+        if (!f)
+            return false;
+        f.status = 'deprecated';
+        return true;
+    }
+    /** Only approved fixtures are usable in production gates */
+    getApproved() {
+        return [...this.fixtures.values()].filter(f => f.status === 'approved');
+    }
+    getPending() {
+        return [...this.fixtures.values()].filter(f => f.status === 'pending');
+    }
+    get(id) {
+        return this.fixtures.get(id);
+    }
+    stats() {
+        const all = [...this.fixtures.values()];
+        const byCategory = {};
+        for (const f of all)
+            byCategory[f.category] = (byCategory[f.category] ?? 0) + 1;
+        return {
+            total: all.length,
+            approved: all.filter(f => f.status === 'approved').length,
+            pending: all.filter(f => f.status === 'pending').length,
+            rejected: all.filter(f => f.status === 'rejected').length,
+            byCategory,
+        };
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evaluation",
-  "version": "0.8.4",
+  "version": "0.8.5",
   "description": "hmharness evaluation: the Evaluator/Judge contract (V2 blueprint M2). Hard evidence outranks LLM judgment - build results, exit codes, exact/regex assertions first; the LLM judge is a last resort and is labeled as such. Evaluations attach to trajectories (judge.completed events).",
   "type": "module",
   "main": "dist/index.js",