@hmharness/evaluation 0.8.1 → 0.8.3

package/dist/index.d.ts

@@ -2,3 +2,4 @@ export * from './types.ts';
 export * from './evaluators.ts';
 export * from './runner.ts';
 export * from './harmonybench.ts';
+export * from './score.ts';

package/dist/index.js

@@ -2,3 +2,4 @@ export * from "./types.js";
 export * from "./evaluators.js";
 export * from "./runner.js";
 export * from "./harmonybench.js";
+export * from "./score.js";

package/dist/score.d.ts

@@ -0,0 +1,82 @@
+/**
+ * @hmharness/evaluation - HMH Score 1.0 (P1-03)
+ *
+ * The audit called for: "统一 0-100 多维总分体系" with dimensions:
+ * correctness, test, reliability, repair, security, cost, latency,
+ * human, maintainability.
+ *
+ * HMH Score is the single number that tracks whether the harness is
+ * actually getting better. Every dimension maps to [0,100]; the composite
+ * is a weighted mean. Weights are versioned (changing weights = version bump).
+ */
+export declare const HMH_SCORE_VERSION = "1.0.0";
+export type ScoreDimension = 'correctness' | 'test' | 'reliability' | 'repair' | 'security' | 'cost' | 'latency' | 'human' | 'maintainability';
+/** All dimensions in canonical order */
+export declare const ALL_DIMENSIONS: ScoreDimension[];
+/** Default weights (sum to 1.0) - versioned, changing requires bump */
+export declare const DEFAULT_WEIGHTS: Record<ScoreDimension, number>;
+export interface DimensionScore {
+    dimension: ScoreDimension;
+    /** raw score [0,100] */
+    score: number;
+    /** how this score was computed (for audit) */
+    method: string;
+    /** sample size used to compute this score */
+    sampleSize: number;
+}
+export interface HMHScore {
+    version: string;
+    /** composite score [0,100] */
+    composite: number;
+    /** per-dimension breakdown */
+    dimensions: DimensionScore[];
+    /** when this score was computed */
+    computedAt: string;
+    /** any warnings about data quality */
+    warnings: string[];
+}
+/**
+ * Compute the composite HMH Score from dimension scores.
+ * Pure - testable.
+ */
+export declare function computeHMHScore(dimensions: DimensionScore[], weights?: Record<ScoreDimension, number>): HMHScore;
+/**
+ * Compute correctness from pass rate.
+ * Pure - testable.
+ */
+export declare function correctnessFromPassRate(passRate: number, sampleSize: number): DimensionScore;
+/**
+ * Compute test coverage score.
+ * Pure - testable.
+ */
+export declare function testFromCoverage(coveragePercent: number): DimensionScore;
+/**
+ * Compute reliability from success rate.
+ * Pure - testable.
+ */
+export declare function reliabilityFromSuccessRate(successRate: number, sessions: number): DimensionScore;
+/**
+ * Compute security score from red-team results.
+ * Pure - testable.
+ */
+export declare function securityFromRedTeam(blocked: number, total: number): DimensionScore;
+/**
+ * Compute cost efficiency (inverse of cost, normalized).
+ * Pure - testable.
+ */
+export declare function costFromTokensPerTask(avgTokens: number, budgetTokens: number): DimensionScore;
+/**
+ * Compute latency score (inverse of duration, normalized).
+ * Pure - testable.
+ */
+export declare function latencyFromDuration(avgMs: number, budgetMs: number): DimensionScore;
+/**
+ * Compute human satisfaction from judge scores.
+ * Pure - testable.
+ */
+export declare function humanFromJudgeScores(scores: number[]): DimensionScore;
+/**
+ * Format an HMH Score as a human-readable string.
+ * Pure - testable.
+ */
+export declare function formatHMHScore(score: HMHScore): string;

package/dist/score.js

@@ -0,0 +1,168 @@
+/**
+ * @hmharness/evaluation - HMH Score 1.0 (P1-03)
+ *
+ * The audit called for: "统一 0-100 多维总分体系" with dimensions:
+ * correctness, test, reliability, repair, security, cost, latency,
+ * human, maintainability.
+ *
+ * HMH Score is the single number that tracks whether the harness is
+ * actually getting better. Every dimension maps to [0,100]; the composite
+ * is a weighted mean. Weights are versioned (changing weights = version bump).
+ */
+export const HMH_SCORE_VERSION = '1.0.0';
+/** All dimensions in canonical order */
+export const ALL_DIMENSIONS = [
+    'correctness', 'test', 'reliability', 'repair', 'security',
+    'cost', 'latency', 'human', 'maintainability',
+];
+/** Default weights (sum to 1.0) - versioned, changing requires bump */
+export const DEFAULT_WEIGHTS = {
+    correctness: 0.25,
+    test: 0.15,
+    reliability: 0.12,
+    repair: 0.08,
+    security: 0.10,
+    cost: 0.08,
+    latency: 0.07,
+    human: 0.10,
+    maintainability: 0.05,
+};
+/**
+ * Compute the composite HMH Score from dimension scores.
+ * Pure - testable.
+ */
+export function computeHMHScore(dimensions, weights = DEFAULT_WEIGHTS) {
+    const warnings = [];
+    const dimMap = new Map(dimensions.map(d => [d.dimension, d]));
+    const missing = ALL_DIMENSIONS.filter(d => !dimMap.has(d));
+    if (missing.length > 0)
+        warnings.push(`missing dimensions: ${missing.join(', ')} (treated as 0)`);
+    let composite = 0;
+    let totalWeight = 0;
+    for (const dim of ALL_DIMENSIONS) {
+        const ds = dimMap.get(dim);
+        const w = weights[dim] ?? 0;
+        if (ds) {
+            composite += Math.max(0, Math.min(100, ds.score)) * w;
+            totalWeight += w;
+        }
+    }
+    if (totalWeight > 0)
+        composite = composite / totalWeight;
+    return {
+        version: HMH_SCORE_VERSION,
+        composite: Math.round(composite * 100) / 100,
+        dimensions,
+        computedAt: new Date().toISOString(),
+        warnings,
+    };
+}
+/**
+ * Compute correctness from pass rate.
+ * Pure - testable.
+ */
+export function correctnessFromPassRate(passRate, sampleSize) {
+    return {
+        dimension: 'correctness',
+        score: Math.round(passRate * 100),
+        method: `pass rate × 100 (${sampleSize} samples)`,
+        sampleSize,
+    };
+}
+/**
+ * Compute test coverage score.
+ * Pure - testable.
+ */
+export function testFromCoverage(coveragePercent) {
+    return {
+        dimension: 'test',
+        score: Math.round(Math.min(100, coveragePercent)),
+        method: `test coverage % (${coveragePercent.toFixed(1)}%)`,
+        sampleSize: 1,
+    };
+}
+/**
+ * Compute reliability from success rate.
+ * Pure - testable.
+ */
+export function reliabilityFromSuccessRate(successRate, sessions) {
+    return {
+        dimension: 'reliability',
+        score: Math.round(successRate * 100),
+        method: `session success rate × 100 (${sessions} sessions)`,
+        sampleSize: sessions,
+    };
+}
+/**
+ * Compute security score from red-team results.
+ * Pure - testable.
+ */
+export function securityFromRedTeam(blocked, total) {
+    const rate = total > 0 ? blocked / total : 0;
+    return {
+        dimension: 'security',
+        score: Math.round(rate * 100),
+        method: `red-team block rate (${blocked}/${total} attacks blocked)`,
+        sampleSize: total,
+    };
+}
+/**
+ * Compute cost efficiency (inverse of cost, normalized).
+ * Pure - testable.
+ */
+export function costFromTokensPerTask(avgTokens, budgetTokens) {
+    const ratio = budgetTokens > 0 ? avgTokens / budgetTokens : 1;
+    const score = Math.round(Math.max(0, Math.min(100, (1 - ratio) * 100)));
+    return {
+        dimension: 'cost',
+        score,
+        method: `token efficiency (avg ${avgTokens} / budget ${budgetTokens})`,
+        sampleSize: 1,
+    };
+}
+/**
+ * Compute latency score (inverse of duration, normalized).
+ * Pure - testable.
+ */
+export function latencyFromDuration(avgMs, budgetMs) {
+    const ratio = budgetMs > 0 ? avgMs / budgetMs : 1;
+    const score = Math.round(Math.max(0, Math.min(100, (1 - ratio) * 100)));
+    return {
+        dimension: 'latency',
+        score,
+        method: `latency efficiency (avg ${avgMs}ms / budget ${budgetMs}ms)`,
+        sampleSize: 1,
+    };
+}
+/**
+ * Compute human satisfaction from judge scores.
+ * Pure - testable.
+ */
+export function humanFromJudgeScores(scores) {
+    if (scores.length === 0) {
+        return { dimension: 'human', score: 0, method: 'no judge scores', sampleSize: 0 };
+    }
+    const avg = scores.reduce((a, b) => a + b, 0) / scores.length;
+    return {
+        dimension: 'human',
+        score: Math.round((avg / 5) * 100), // 5-point scale → 0-100
+        method: `avg judge score (${avg.toFixed(2)}/5 across ${scores.length} sessions)`,
+        sampleSize: scores.length,
+    };
+}
+/**
+ * Format an HMH Score as a human-readable string.
+ * Pure - testable.
+ */
+export function formatHMHScore(score) {
+    const lines = [`HMH Score v${score.version}: ${score.composite}/100`];
+    const sorted = [...score.dimensions].sort((a, b) => b.score - a.score);
+    for (const d of sorted) {
+        const bar = '█'.repeat(Math.round(d.score / 10)) + '░'.repeat(10 - Math.round(d.score / 10));
+        lines.push(`  ${d.dimension.padEnd(16)} ${bar} ${String(d.score).padStart(3)}  (${d.method})`);
+    }
+    if (score.warnings.length > 0) {
+        lines.push(`  ⚠ ${score.warnings.join('; ')}`);
+    }
+    return lines.join('\n');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evaluation",
-  "version": "0.8.1",
+  "version": "0.8.3",
   "description": "hmharness evaluation: the Evaluator/Judge contract (V2 blueprint M2). Hard evidence outranks LLM judgment - build results, exit codes, exact/regex assertions first; the LLM judge is a last resort and is labeled as such. Evaluations attach to trajectories (judge.completed events).",
   "type": "module",
   "main": "dist/index.js",