dsh-eval-pack 0.1.0-alpha.7 → 0.1.0-alpha.9

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [0.1.0-alpha.9] — 2026-11-01
+
+### Added
+
+- `eval.pass_at_k`: unbiased pass@k estimator `1 - C(n-c,k)/C(n,k)` with exact binomials, plus the per-task and mean values. Verified by `node --test test/*.test.mjs` (12/12).
+
+## [Unreleased]
+
+### Added
+
+- `eval.summary`: aggregate pass count, pass rate and failing task ids for a graded run, using the same grader as `eval.grade`.
+- Canonical-JSON grading: object key order no longer changes the verdict; array order still matters.
+
+Verified by `node --test test/*.test.mjs` (7/7).
+
 ## [0.1.0-alpha.7] — 2026-11-01
 
 ### Changed

package/README-zh.md

@@ -132,7 +132,6 @@ pnpm build && pnpm lint && pnpm typecheck && pnpm test
 - 本地运行；不联网、不调用模型、不上传。
 - 只使用公开/合成 fixture；不含客户数据或客户证据。
 - alpha/partial；not counted，不作 GA 或生产可用声明。
-- 冻结包仍可安装；不再加新功能，只修真实故障。
 
 ## 相关项目
 

package/README.md

@@ -132,7 +132,6 @@ pnpm build && pnpm lint && pnpm typecheck && pnpm test
 - Runs locally; no network, no model call, no upload.
 - Public/synthetic fixtures only; no customer data, no customer evidence.
 - Alpha/partial; not counted, no GA or production-readiness claim.
-- Frozen packages stay installable; no new features, only genuine breakage fixes.
 
 ## Related work
 

package/lib/core/harness.d.ts

@@ -8,4 +8,41 @@ export interface EvalPack {
     tasks: EvalTask[];
 }
 export declare function lintPack(input: unknown): string[];
+/** Stable serialisation: object keys sorted at every level, arrays kept in order. */
+export declare function canonical(value: unknown): string;
 export declare function grade(actual: unknown, expected: unknown): boolean;
+export interface EvalResult {
+    id: string;
+    actual: unknown;
+    expected: unknown;
+}
+export interface EvalSummary {
+    total: number;
+    passed: number;
+    failed: number;
+    pass_rate: number;
+    failures: string[];
+}
+export declare function summarize(results: EvalResult[]): EvalSummary;
+export interface PassAtKInput {
+    task_id: string;
+    samples: number;
+    correct: number;
+}
+export interface PassAtKResult {
+    task_id: string;
+    k: number;
+    n: number;
+    c: number;
+    pass_at_k: number;
+}
+/**
+ * Unbiased pass@k estimator: 1 - C(n-c, k) / C(n, k).
+ * Clean-room from the published estimator definition; exact binomials via BigInt.
+ */
+export declare function passAtK(task: PassAtKInput, k: number): PassAtKResult;
+export declare function passAtKMean(tasks: PassAtKInput[], k: number): {
+    k: number;
+    tasks: PassAtKResult[];
+    mean: number;
+};

package/lib/core/harness.js

@@ -23,6 +23,63 @@ export function lintPack(input) {
     }
     return errors;
 }
+/** Stable serialisation: object keys sorted at every level, arrays kept in order. */
+export function canonical(value) {
+    const walk = (input) => {
+        if (Array.isArray(input))
+            return input.map(walk);
+        if (input && typeof input === 'object') {
+            const out = {};
+            for (const key of Object.keys(input).sort())
+                out[key] = walk(input[key]);
+            return out;
+        }
+        return input;
+    };
+    return JSON.stringify(walk(value));
+}
 export function grade(actual, expected) {
-    return JSON.stringify(actual) === JSON.stringify(expected);
+    return canonical(actual) === canonical(expected);
+}
+export function summarize(results) {
+    const failures = [];
+    let passed = 0;
+    for (const r of results) {
+        if (grade(r.actual, r.expected))
+            passed += 1;
+        else
+            failures.push(r.id);
+    }
+    const total = results.length;
+    return { total, passed, failed: total - passed, pass_rate: total ? Math.round((passed / total) * 1e6) / 1e6 : 0, failures };
+}
+function comb(n, k) {
+    if (k < 0 || k > n)
+        return 0n;
+    let result = 1n;
+    for (let i = 1n; i <= BigInt(k); i += 1n)
+        result = (result * (BigInt(n) - BigInt(k) + i)) / i;
+    return result;
+}
+/**
+ * Unbiased pass@k estimator: 1 - C(n-c, k) / C(n, k).
+ * Clean-room from the published estimator definition; exact binomials via BigInt.
+ */
+export function passAtK(task, k) {
+    const { samples, correct } = task;
+    if (!Number.isInteger(samples) || samples < 1)
+        throw new Error('E_EVAL_SAMPLES: samples must be a positive integer');
+    if (!Number.isInteger(correct) || correct < 0 || correct > samples)
+        throw new Error('E_EVAL_CORRECT: correct must be between 0 and samples');
+    if (!Number.isInteger(k) || k < 1 || k > samples)
+        throw new Error('E_EVAL_K: k must be between 1 and samples');
+    const denominator = comb(samples, k);
+    const numerator = comb(samples - correct, k);
+    const value = 1 - Number(numerator) / Number(denominator);
+    return { task_id: task.task_id, k, n: samples, c: correct, pass_at_k: Math.round(value * 1e6) / 1e6 };
+}
+export function passAtKMean(tasks, k) {
+    const results = tasks.map((t) => passAtK(t, k));
+    const mean = results.length ? Math.round((results.reduce((sum, r) => sum + r.pass_at_k, 0) / results.length) * 1e6) / 1e6 : 0;
+    return { k, tasks: results, mean };
 }

package/lib/plugin/tools.js

@@ -1,5 +1,5 @@
 import { defineTool } from '@deepseek-ai/dsh-tools';
-import { grade, lintPack } from '../core/harness.js';
+import { grade, lintPack, passAtKMean, summarize } from '../core/harness.js';
 const text = (t) => [{ type: 'text', text: t }];
 export function createTools() {
     const lint = defineTool({
@@ -16,5 +16,19 @@ export function createTools() {
         output: { schema: { type: 'object', additionalProperties: false, properties: { pass: { type: 'boolean', required: true } } }, render: (_a, v) => text(`grade ${v.pass ? 'pass' : 'fail'}`) },
         async execute(args) { return { pass: grade(args.actual, args.expected) }; },
     });
-    return [lint, gradeTool];
+    const summary = defineTool({
+        name: 'eval.summary',
+        description: 'Aggregate graded results for a public/synthetic run: pass count, pass rate and the failing task ids, using the same canonical-JSON grader. Read-only; no runner.',
+        parameters: { results: { type: 'array', required: true, items: { type: 'object', additionalProperties: false, properties: { id: { type: 'string', required: true }, actual: { type: 'json', required: true }, expected: { type: 'json', required: true } } } } },
+        output: { schema: { type: 'object', additionalProperties: false, properties: { total: { type: 'number', required: true }, passed: { type: 'number', required: true }, failed: { type: 'number', required: true }, pass_rate: { type: 'number', required: true }, failures: { type: 'array', items: { type: 'string' }, required: true } } }, render: (_a, v) => text(`summary ${v.passed}/${v.total}`) },
+        async execute(args) { return summarize(args.results); },
+    });
+    const passAtKTool = defineTool({
+        name: 'eval.pass_at_k',
+        description: 'Unbiased pass@k for sampled tasks: 1 - C(n-c, k)/C(n, k), exact binomials. Read-only; no runner.',
+        parameters: { k: { type: 'number', required: true }, tasks: { type: 'array', required: true, items: { type: 'object', additionalProperties: false, properties: { task_id: { type: 'string', required: true }, samples: { type: 'number', required: true }, correct: { type: 'number', required: true } } } } },
+        output: { schema: { type: 'object', additionalProperties: false, properties: { k: { type: 'number', required: true }, mean: { type: 'number', required: true }, tasks: { type: 'array', required: true, items: { type: 'object', additionalProperties: false, properties: { task_id: { type: 'string', required: true }, pass_at_k: { type: 'number', required: true } } } } } }, render: (_a, v) => text(`pass@${v.k} mean ${v.mean}`) },
+        async execute(args) { const r = passAtKMean(args.tasks, args.k); return { k: r.k, mean: r.mean, tasks: r.tasks.map((t) => ({ task_id: t.task_id, pass_at_k: t.pass_at_k })) }; },
+    });
+    return [lint, gradeTool, summary, passAtKTool];
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-eval-pack",
-  "version": "0.1.0-alpha.7",
+  "version": "0.1.0-alpha.9",
   "description": "Public/synthetic eval-pack lint and deterministic JSON grader; not a validated benchmark.",
   "type": "module",
   "main": "./lib/plugin/index.js",

package/screenshots.json

@@ -10,5 +10,17 @@
     "kind": "demo",
     "alt": "Animated dsh-eval-pack terminal session",
     "caption": "The same run, assembled line by line."
+  },
+  {
+    "path": "docs/assets/dsh-eval-pack-capability.png",
+    "kind": "capability",
+    "alt": "dsh-eval-pack real test run",
+    "caption": "Real command: node --test test/summary.test.mjs — 4 tests passed, 0 failed (canonical grading and run summary). Run 2026-11-01; the image is rendered from the captured terminal output."
+  },
+  {
+    "path": "docs/assets/dsh-eval-pack-capability2.png",
+    "kind": "capability",
+    "alt": "dsh-eval-pack second capability run",
+    "caption": "Real command: node --test test/pass-at-k.test.mjs — 5 tests passed, 0 failed (Unbiased pass@k estimator with exact binomials.) Run 2026-11-01; rendered from the captured terminal output."
   }
 ]