dsh-eval-pack 0.1.0-alpha.8 → 0.1.0-alpha.9

package/CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 
-## [0.1.0-alpha.8] — 2026-11-01
+## [0.1.0-alpha.9] — 2026-11-01
+
+### Added
+
+- `eval.pass_at_k`: unbiased pass@k estimator `1 - C(n-c,k)/C(n,k)` with exact binomials, plus the per-task and mean values. Verified by `node --test test/*.test.mjs` (12/12).
+
+## [Unreleased]
 
 ### Added
 

package/lib/core/harness.d.ts

@@ -24,3 +24,25 @@ export interface EvalSummary {
     failures: string[];
 }
 export declare function summarize(results: EvalResult[]): EvalSummary;
+export interface PassAtKInput {
+    task_id: string;
+    samples: number;
+    correct: number;
+}
+export interface PassAtKResult {
+    task_id: string;
+    k: number;
+    n: number;
+    c: number;
+    pass_at_k: number;
+}
+/**
+ * Unbiased pass@k estimator: 1 - C(n-c, k) / C(n, k).
+ * Clean-room from the published estimator definition; exact binomials via BigInt.
+ */
+export declare function passAtK(task: PassAtKInput, k: number): PassAtKResult;
+export declare function passAtKMean(tasks: PassAtKInput[], k: number): {
+    k: number;
+    tasks: PassAtKResult[];
+    mean: number;
+};

package/lib/core/harness.js

@@ -53,3 +53,33 @@ export function summarize(results) {
     const total = results.length;
     return { total, passed, failed: total - passed, pass_rate: total ? Math.round((passed / total) * 1e6) / 1e6 : 0, failures };
 }
+function comb(n, k) {
+    if (k < 0 || k > n)
+        return 0n;
+    let result = 1n;
+    for (let i = 1n; i <= BigInt(k); i += 1n)
+        result = (result * (BigInt(n) - BigInt(k) + i)) / i;
+    return result;
+}
+/**
+ * Unbiased pass@k estimator: 1 - C(n-c, k) / C(n, k).
+ * Clean-room from the published estimator definition; exact binomials via BigInt.
+ */
+export function passAtK(task, k) {
+    const { samples, correct } = task;
+    if (!Number.isInteger(samples) || samples < 1)
+        throw new Error('E_EVAL_SAMPLES: samples must be a positive integer');
+    if (!Number.isInteger(correct) || correct < 0 || correct > samples)
+        throw new Error('E_EVAL_CORRECT: correct must be between 0 and samples');
+    if (!Number.isInteger(k) || k < 1 || k > samples)
+        throw new Error('E_EVAL_K: k must be between 1 and samples');
+    const denominator = comb(samples, k);
+    const numerator = comb(samples - correct, k);
+    const value = 1 - Number(numerator) / Number(denominator);
+    return { task_id: task.task_id, k, n: samples, c: correct, pass_at_k: Math.round(value * 1e6) / 1e6 };
+}
+export function passAtKMean(tasks, k) {
+    const results = tasks.map((t) => passAtK(t, k));
+    const mean = results.length ? Math.round((results.reduce((sum, r) => sum + r.pass_at_k, 0) / results.length) * 1e6) / 1e6 : 0;
+    return { k, tasks: results, mean };
+}

package/lib/plugin/tools.js

@@ -1,5 +1,5 @@
 import { defineTool } from '@deepseek-ai/dsh-tools';
-import { grade, lintPack, summarize } from '../core/harness.js';
+import { grade, lintPack, passAtKMean, summarize } from '../core/harness.js';
 const text = (t) => [{ type: 'text', text: t }];
 export function createTools() {
     const lint = defineTool({
@@ -23,5 +23,12 @@ export function createTools() {
         output: { schema: { type: 'object', additionalProperties: false, properties: { total: { type: 'number', required: true }, passed: { type: 'number', required: true }, failed: { type: 'number', required: true }, pass_rate: { type: 'number', required: true }, failures: { type: 'array', items: { type: 'string' }, required: true } } }, render: (_a, v) => text(`summary ${v.passed}/${v.total}`) },
         async execute(args) { return summarize(args.results); },
     });
-    return [lint, gradeTool, summary];
+    const passAtKTool = defineTool({
+        name: 'eval.pass_at_k',
+        description: 'Unbiased pass@k for sampled tasks: 1 - C(n-c, k)/C(n, k), exact binomials. Read-only; no runner.',
+        parameters: { k: { type: 'number', required: true }, tasks: { type: 'array', required: true, items: { type: 'object', additionalProperties: false, properties: { task_id: { type: 'string', required: true }, samples: { type: 'number', required: true }, correct: { type: 'number', required: true } } } } },
+        output: { schema: { type: 'object', additionalProperties: false, properties: { k: { type: 'number', required: true }, mean: { type: 'number', required: true }, tasks: { type: 'array', required: true, items: { type: 'object', additionalProperties: false, properties: { task_id: { type: 'string', required: true }, pass_at_k: { type: 'number', required: true } } } } } }, render: (_a, v) => text(`pass@${v.k} mean ${v.mean}`) },
+        async execute(args) { const r = passAtKMean(args.tasks, args.k); return { k: r.k, mean: r.mean, tasks: r.tasks.map((t) => ({ task_id: t.task_id, pass_at_k: t.pass_at_k })) }; },
+    });
+    return [lint, gradeTool, summary, passAtKTool];
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-eval-pack",
-  "version": "0.1.0-alpha.8",
+  "version": "0.1.0-alpha.9",
   "description": "Public/synthetic eval-pack lint and deterministic JSON grader; not a validated benchmark.",
   "type": "module",
   "main": "./lib/plugin/index.js",

package/screenshots.json

@@ -16,5 +16,11 @@
     "kind": "capability",
     "alt": "dsh-eval-pack real test run",
     "caption": "Real command: node --test test/summary.test.mjs — 4 tests passed, 0 failed (canonical grading and run summary). Run 2026-11-01; the image is rendered from the captured terminal output."
+  },
+  {
+    "path": "docs/assets/dsh-eval-pack-capability2.png",
+    "kind": "capability",
+    "alt": "dsh-eval-pack second capability run",
+    "caption": "Real command: node --test test/pass-at-k.test.mjs — 5 tests passed, 0 failed (Unbiased pass@k estimator with exact binomials.) Run 2026-11-01; rendered from the captured terminal output."
   }
 ]