@hmharness/evaluation 0.8.2 → 0.8.4

package/dist/index.d.ts

@@ -2,3 +2,5 @@ export * from './types.ts';
 export * from './evaluators.ts';
 export * from './runner.ts';
 export * from './harmonybench.ts';
+export * from './score.ts';
+export * from './sdk.ts';

package/dist/index.js

@@ -2,3 +2,5 @@ export * from "./types.js";
 export * from "./evaluators.js";
 export * from "./runner.js";
 export * from "./harmonybench.js";
+export * from "./score.js";
+export * from "./sdk.js";

package/dist/score.d.ts

@@ -0,0 +1,82 @@
+/**
+ * @hmharness/evaluation - HMH Score 1.0 (P1-03)
+ *
+ * The audit called for: "统一 0-100 多维总分体系" with dimensions:
+ * correctness, test, reliability, repair, security, cost, latency,
+ * human, maintainability.
+ *
+ * HMH Score is the single number that tracks whether the harness is
+ * actually getting better. Every dimension maps to [0,100]; the composite
+ * is a weighted mean. Weights are versioned (changing weights = version bump).
+ */
+export declare const HMH_SCORE_VERSION = "1.0.0";
+export type ScoreDimension = 'correctness' | 'test' | 'reliability' | 'repair' | 'security' | 'cost' | 'latency' | 'human' | 'maintainability';
+/** All dimensions in canonical order */
+export declare const ALL_DIMENSIONS: ScoreDimension[];
+/** Default weights (sum to 1.0) - versioned, changing requires bump */
+export declare const DEFAULT_WEIGHTS: Record<ScoreDimension, number>;
+export interface DimensionScore {
+    dimension: ScoreDimension;
+    /** raw score [0,100] */
+    score: number;
+    /** how this score was computed (for audit) */
+    method: string;
+    /** sample size used to compute this score */
+    sampleSize: number;
+}
+export interface HMHScore {
+    version: string;
+    /** composite score [0,100] */
+    composite: number;
+    /** per-dimension breakdown */
+    dimensions: DimensionScore[];
+    /** when this score was computed */
+    computedAt: string;
+    /** any warnings about data quality */
+    warnings: string[];
+}
+/**
+ * Compute the composite HMH Score from dimension scores.
+ * Pure - testable.
+ */
+export declare function computeHMHScore(dimensions: DimensionScore[], weights?: Record<ScoreDimension, number>): HMHScore;
+/**
+ * Compute correctness from pass rate.
+ * Pure - testable.
+ */
+export declare function correctnessFromPassRate(passRate: number, sampleSize: number): DimensionScore;
+/**
+ * Compute test coverage score.
+ * Pure - testable.
+ */
+export declare function testFromCoverage(coveragePercent: number): DimensionScore;
+/**
+ * Compute reliability from success rate.
+ * Pure - testable.
+ */
+export declare function reliabilityFromSuccessRate(successRate: number, sessions: number): DimensionScore;
+/**
+ * Compute security score from red-team results.
+ * Pure - testable.
+ */
+export declare function securityFromRedTeam(blocked: number, total: number): DimensionScore;
+/**
+ * Compute cost efficiency (inverse of cost, normalized).
+ * Pure - testable.
+ */
+export declare function costFromTokensPerTask(avgTokens: number, budgetTokens: number): DimensionScore;
+/**
+ * Compute latency score (inverse of duration, normalized).
+ * Pure - testable.
+ */
+export declare function latencyFromDuration(avgMs: number, budgetMs: number): DimensionScore;
+/**
+ * Compute human satisfaction from judge scores.
+ * Pure - testable.
+ */
+export declare function humanFromJudgeScores(scores: number[]): DimensionScore;
+/**
+ * Format an HMH Score as a human-readable string.
+ * Pure - testable.
+ */
+export declare function formatHMHScore(score: HMHScore): string;

package/dist/score.js

@@ -0,0 +1,168 @@
+/**
+ * @hmharness/evaluation - HMH Score 1.0 (P1-03)
+ *
+ * The audit called for: "统一 0-100 多维总分体系" with dimensions:
+ * correctness, test, reliability, repair, security, cost, latency,
+ * human, maintainability.
+ *
+ * HMH Score is the single number that tracks whether the harness is
+ * actually getting better. Every dimension maps to [0,100]; the composite
+ * is a weighted mean. Weights are versioned (changing weights = version bump).
+ */
+export const HMH_SCORE_VERSION = '1.0.0';
+/** All dimensions in canonical order */
+export const ALL_DIMENSIONS = [
+    'correctness', 'test', 'reliability', 'repair', 'security',
+    'cost', 'latency', 'human', 'maintainability',
+];
+/** Default weights (sum to 1.0) - versioned, changing requires bump */
+export const DEFAULT_WEIGHTS = {
+    correctness: 0.25,
+    test: 0.15,
+    reliability: 0.12,
+    repair: 0.08,
+    security: 0.10,
+    cost: 0.08,
+    latency: 0.07,
+    human: 0.10,
+    maintainability: 0.05,
+};
+/**
+ * Compute the composite HMH Score from dimension scores.
+ * Pure - testable.
+ */
+export function computeHMHScore(dimensions, weights = DEFAULT_WEIGHTS) {
+    const warnings = [];
+    const dimMap = new Map(dimensions.map(d => [d.dimension, d]));
+    const missing = ALL_DIMENSIONS.filter(d => !dimMap.has(d));
+    if (missing.length > 0)
+        warnings.push(`missing dimensions: ${missing.join(', ')} (treated as 0)`);
+    let composite = 0;
+    let totalWeight = 0;
+    for (const dim of ALL_DIMENSIONS) {
+        const ds = dimMap.get(dim);
+        const w = weights[dim] ?? 0;
+        if (ds) {
+            composite += Math.max(0, Math.min(100, ds.score)) * w;
+            totalWeight += w;
+        }
+    }
+    if (totalWeight > 0)
+        composite = composite / totalWeight;
+    return {
+        version: HMH_SCORE_VERSION,
+        composite: Math.round(composite * 100) / 100,
+        dimensions,
+        computedAt: new Date().toISOString(),
+        warnings,
+    };
+}
+/**
+ * Compute correctness from pass rate.
+ * Pure - testable.
+ */
+export function correctnessFromPassRate(passRate, sampleSize) {
+    return {
+        dimension: 'correctness',
+        score: Math.round(passRate * 100),
+        method: `pass rate × 100 (${sampleSize} samples)`,
+        sampleSize,
+    };
+}
+/**
+ * Compute test coverage score.
+ * Pure - testable.
+ */
+export function testFromCoverage(coveragePercent) {
+    return {
+        dimension: 'test',
+        score: Math.round(Math.min(100, coveragePercent)),
+        method: `test coverage % (${coveragePercent.toFixed(1)}%)`,
+        sampleSize: 1,
+    };
+}
+/**
+ * Compute reliability from success rate.
+ * Pure - testable.
+ */
+export function reliabilityFromSuccessRate(successRate, sessions) {
+    return {
+        dimension: 'reliability',
+        score: Math.round(successRate * 100),
+        method: `session success rate × 100 (${sessions} sessions)`,
+        sampleSize: sessions,
+    };
+}
+/**
+ * Compute security score from red-team results.
+ * Pure - testable.
+ */
+export function securityFromRedTeam(blocked, total) {
+    const rate = total > 0 ? blocked / total : 0;
+    return {
+        dimension: 'security',
+        score: Math.round(rate * 100),
+        method: `red-team block rate (${blocked}/${total} attacks blocked)`,
+        sampleSize: total,
+    };
+}
+/**
+ * Compute cost efficiency (inverse of cost, normalized).
+ * Pure - testable.
+ */
+export function costFromTokensPerTask(avgTokens, budgetTokens) {
+    const ratio = budgetTokens > 0 ? avgTokens / budgetTokens : 1;
+    const score = Math.round(Math.max(0, Math.min(100, (1 - ratio) * 100)));
+    return {
+        dimension: 'cost',
+        score,
+        method: `token efficiency (avg ${avgTokens} / budget ${budgetTokens})`,
+        sampleSize: 1,
+    };
+}
+/**
+ * Compute latency score (inverse of duration, normalized).
+ * Pure - testable.
+ */
+export function latencyFromDuration(avgMs, budgetMs) {
+    const ratio = budgetMs > 0 ? avgMs / budgetMs : 1;
+    const score = Math.round(Math.max(0, Math.min(100, (1 - ratio) * 100)));
+    return {
+        dimension: 'latency',
+        score,
+        method: `latency efficiency (avg ${avgMs}ms / budget ${budgetMs}ms)`,
+        sampleSize: 1,
+    };
+}
+/**
+ * Compute human satisfaction from judge scores.
+ * Pure - testable.
+ */
+export function humanFromJudgeScores(scores) {
+    if (scores.length === 0) {
+        return { dimension: 'human', score: 0, method: 'no judge scores', sampleSize: 0 };
+    }
+    const avg = scores.reduce((a, b) => a + b, 0) / scores.length;
+    return {
+        dimension: 'human',
+        score: Math.round((avg / 5) * 100), // 5-point scale → 0-100
+        method: `avg judge score (${avg.toFixed(2)}/5 across ${scores.length} sessions)`,
+        sampleSize: scores.length,
+    };
+}
+/**
+ * Format an HMH Score as a human-readable string.
+ * Pure - testable.
+ */
+export function formatHMHScore(score) {
+    const lines = [`HMH Score v${score.version}: ${score.composite}/100`];
+    const sorted = [...score.dimensions].sort((a, b) => b.score - a.score);
+    for (const d of sorted) {
+        const bar = '█'.repeat(Math.round(d.score / 10)) + '░'.repeat(10 - Math.round(d.score / 10));
+        lines.push(`  ${d.dimension.padEnd(16)} ${bar} ${String(d.score).padStart(3)}  (${d.method})`);
+    }
+    if (score.warnings.length > 0) {
+        lines.push(`  ⚠ ${score.warnings.join('; ')}`);
+    }
+    return lines.join('\n');
+}

package/dist/sdk.d.ts

@@ -0,0 +1,83 @@
+/**
+ * @hmharness/evaluation - Evaluator SDK (P1-02)
+ *
+ * The audit called for: "统一 evaluator plugin、evidence references、metric registry"
+ *
+ * This module provides the plugin interface that ALL evaluators implement,
+ * evidence references that link scores to concrete proof, and a metric
+ * registry for named, versioned scoring functions.
+ */
+/** Evidence tier per the existing evidence ladder (M2) */
+export type EvidenceTier = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8;
+/** Reference to concrete evidence backing an evaluation */
+export interface EvidenceRef {
+    /** what kind of evidence */
+    kind: 'build-log' | 'test-result' | 'command-output' | 'file-content' | 'judge-score' | 'device-log' | 'metric';
+    /** where to find it (path, URL, or inline) */
+    source: string;
+    /** relevant excerpt (for audit trail) */
+    excerpt?: string;
+    /** evidence tier (1=build, 7=llm-judge, 8=self-report) */
+    tier: EvidenceTier;
+}
+/** The result of running an evaluator */
+export interface SdkEvaluationResult {
+    /** evaluator that produced this result */
+    evaluatorId: string;
+    /** pass/fail/indeterminate */
+    outcome: 'pass' | 'fail' | 'indeterminate';
+    /** numeric score [0,1] if applicable */
+    score?: number;
+    /** human-readable detail */
+    detail: string;
+    /** evidence backing this result */
+    evidence: EvidenceRef[];
+    /** when this evaluation ran */
+    evaluatedAt: string;
+    /** duration in ms */
+    durationMs: number;
+}
+/** The evaluator plugin interface - ALL evaluators implement this */
+export interface SdkEvaluator {
+    /** unique evaluator id */
+    readonly id: string;
+    /** what this evaluator checks */
+    readonly description: string;
+    /** evidence tier this evaluator operates at */
+    readonly tier: EvidenceTier;
+    /** evaluate and return a result */
+    evaluate(input: unknown): Promise<SdkEvaluationResult>;
+}
+/** A named metric in the registry */
+export interface MetricDef {
+    name: string;
+    description: string;
+    /** units (e.g. 'percent', 'count', 'ms') */
+    unit: string;
+    /** higher is better? (for display) */
+    higherIsBetter: boolean;
+    /** compute from evaluation results */
+    compute: (results: SdkEvaluationResult[]) => number;
+}
+/**
+ * The metric registry - named, versioned scoring functions.
+ * Metrics are registered once and referenced by name everywhere.
+ */
+export declare class MetricRegistry {
+    private metrics;
+    register(def: MetricDef): {
+        ok: boolean;
+        reason?: string;
+    };
+    get(name: string): MetricDef | undefined;
+    list(): MetricDef[];
+    /** Compute a metric from evaluation results */
+    compute(name: string, results: SdkEvaluationResult[]): number | undefined;
+}
+/** Create a standard metric registry with common metrics pre-registered */
+export declare function createDefaultRegistry(): MetricRegistry;
+/**
+ * Create a simple pass/fail evaluator from a predicate function.
+ * Convenience factory for the most common evaluator pattern.
+ */
+export declare function createPredicateEvaluator(id: string, description: string, tier: EvidenceTier, predicate: (input: unknown) => boolean, evidenceKind?: EvidenceRef['kind']): SdkEvaluator;

package/dist/sdk.js

@@ -0,0 +1,85 @@
+/**
+ * @hmharness/evaluation - Evaluator SDK (P1-02)
+ *
+ * The audit called for: "统一 evaluator plugin、evidence references、metric registry"
+ *
+ * This module provides the plugin interface that ALL evaluators implement,
+ * evidence references that link scores to concrete proof, and a metric
+ * registry for named, versioned scoring functions.
+ */
+/**
+ * The metric registry - named, versioned scoring functions.
+ * Metrics are registered once and referenced by name everywhere.
+ */
+export class MetricRegistry {
+    metrics = new Map();
+    register(def) {
+        if (this.metrics.has(def.name)) {
+            return { ok: false, reason: `metric ${def.name} already registered` };
+        }
+        this.metrics.set(def.name, def);
+        return { ok: true };
+    }
+    get(name) {
+        return this.metrics.get(name);
+    }
+    list() {
+        return [...this.metrics.values()];
+    }
+    /** Compute a metric from evaluation results */
+    compute(name, results) {
+        const m = this.metrics.get(name);
+        if (!m)
+            return undefined;
+        return m.compute(results);
+    }
+}
+/** Create a standard metric registry with common metrics pre-registered */
+export function createDefaultRegistry() {
+    const reg = new MetricRegistry();
+    reg.register({
+        name: 'pass_rate', description: 'fraction of evaluations that passed',
+        unit: 'fraction', higherIsBetter: true,
+        compute: (rs) => rs.length > 0 ? rs.filter(r => r.outcome === 'pass').length / rs.length : 0,
+    });
+    reg.register({
+        name: 'avg_score', description: 'average numeric score',
+        unit: 'fraction', higherIsBetter: true,
+        compute: (rs) => {
+            const scored = rs.filter(r => r.score !== undefined);
+            return scored.length > 0 ? scored.reduce((a, b) => a + (b.score ?? 0), 0) / scored.length : 0;
+        },
+    });
+    reg.register({
+        name: 'avg_duration', description: 'average evaluation duration',
+        unit: 'ms', higherIsBetter: false,
+        compute: (rs) => rs.length > 0 ? rs.reduce((a, b) => a + b.durationMs, 0) / rs.length : 0,
+    });
+    reg.register({
+        name: 'evidence_coverage', description: 'fraction of results with evidence',
+        unit: 'fraction', higherIsBetter: true,
+        compute: (rs) => rs.length > 0 ? rs.filter(r => r.evidence.length > 0).length / rs.length : 0,
+    });
+    return reg;
+}
+/**
+ * Create a simple pass/fail evaluator from a predicate function.
+ * Convenience factory for the most common evaluator pattern.
+ */
+export function createPredicateEvaluator(id, description, tier, predicate, evidenceKind = 'command-output') {
+    return {
+        id, description, tier,
+        async evaluate(input) {
+            const start = Date.now();
+            const pass = predicate(input);
+            return {
+                evaluatorId: id,
+                outcome: pass ? 'pass' : 'fail',
+                detail: pass ? 'predicate satisfied' : 'predicate not satisfied',
+                evidence: [{ kind: evidenceKind, source: 'inline', tier, excerpt: String(input).slice(0, 200) }],
+                evaluatedAt: new Date().toISOString(),
+                durationMs: Date.now() - start,
+            };
+        },
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evaluation",
-  "version": "0.8.2",
+  "version": "0.8.4",
   "description": "hmharness evaluation: the Evaluator/Judge contract (V2 blueprint M2). Hard evidence outranks LLM judgment - build results, exit codes, exact/regex assertions first; the LLM judge is a last resort and is labeled as such. Evaluations attach to trajectories (judge.completed events).",
   "type": "module",
   "main": "dist/index.js",