dsh-eval-pack 0.1.0-alpha.7 → 0.1.0-alpha.8

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [0.1.0-alpha.8] — 2026-11-01
+
+### Added
+
+- `eval.summary`: aggregate pass count, pass rate and failing task ids for a graded run, using the same grader as `eval.grade`.
+- Canonical-JSON grading: object key order no longer changes the verdict; array order still matters.
+
+Verified by `node --test test/*.test.mjs` (7/7).
+
 ## [0.1.0-alpha.7] — 2026-11-01
 
 ### Changed

package/README-zh.md

@@ -132,7 +132,6 @@ pnpm build && pnpm lint && pnpm typecheck && pnpm test
 - 本地运行；不联网、不调用模型、不上传。
 - 只使用公开/合成 fixture；不含客户数据或客户证据。
 - alpha/partial；not counted，不作 GA 或生产可用声明。
-- 冻结包仍可安装；不再加新功能，只修真实故障。
 
 ## 相关项目
 

package/README.md

@@ -132,7 +132,6 @@ pnpm build && pnpm lint && pnpm typecheck && pnpm test
 - Runs locally; no network, no model call, no upload.
 - Public/synthetic fixtures only; no customer data, no customer evidence.
 - Alpha/partial; not counted, no GA or production-readiness claim.
-- Frozen packages stay installable; no new features, only genuine breakage fixes.
 
 ## Related work
 

package/lib/core/harness.d.ts

@@ -8,4 +8,19 @@ export interface EvalPack {
     tasks: EvalTask[];
 }
 export declare function lintPack(input: unknown): string[];
+/** Stable serialisation: object keys sorted at every level, arrays kept in order. */
+export declare function canonical(value: unknown): string;
 export declare function grade(actual: unknown, expected: unknown): boolean;
+export interface EvalResult {
+    id: string;
+    actual: unknown;
+    expected: unknown;
+}
+export interface EvalSummary {
+    total: number;
+    passed: number;
+    failed: number;
+    pass_rate: number;
+    failures: string[];
+}
+export declare function summarize(results: EvalResult[]): EvalSummary;

package/lib/core/harness.js

@@ -23,6 +23,33 @@ export function lintPack(input) {
     }
     return errors;
 }
+/** Stable serialisation: object keys sorted at every level, arrays kept in order. */
+export function canonical(value) {
+    const walk = (input) => {
+        if (Array.isArray(input))
+            return input.map(walk);
+        if (input && typeof input === 'object') {
+            const out = {};
+            for (const key of Object.keys(input).sort())
+                out[key] = walk(input[key]);
+            return out;
+        }
+        return input;
+    };
+    return JSON.stringify(walk(value));
+}
 export function grade(actual, expected) {
-    return JSON.stringify(actual) === JSON.stringify(expected);
+    return canonical(actual) === canonical(expected);
+}
+export function summarize(results) {
+    const failures = [];
+    let passed = 0;
+    for (const r of results) {
+        if (grade(r.actual, r.expected))
+            passed += 1;
+        else
+            failures.push(r.id);
+    }
+    const total = results.length;
+    return { total, passed, failed: total - passed, pass_rate: total ? Math.round((passed / total) * 1e6) / 1e6 : 0, failures };
 }

package/lib/plugin/tools.js

@@ -1,5 +1,5 @@
 import { defineTool } from '@deepseek-ai/dsh-tools';
-import { grade, lintPack } from '../core/harness.js';
+import { grade, lintPack, summarize } from '../core/harness.js';
 const text = (t) => [{ type: 'text', text: t }];
 export function createTools() {
     const lint = defineTool({
@@ -16,5 +16,12 @@ export function createTools() {
         output: { schema: { type: 'object', additionalProperties: false, properties: { pass: { type: 'boolean', required: true } } }, render: (_a, v) => text(`grade ${v.pass ? 'pass' : 'fail'}`) },
         async execute(args) { return { pass: grade(args.actual, args.expected) }; },
     });
-    return [lint, gradeTool];
+    const summary = defineTool({
+        name: 'eval.summary',
+        description: 'Aggregate graded results for a public/synthetic run: pass count, pass rate and the failing task ids, using the same canonical-JSON grader. Read-only; no runner.',
+        parameters: { results: { type: 'array', required: true, items: { type: 'object', additionalProperties: false, properties: { id: { type: 'string', required: true }, actual: { type: 'json', required: true }, expected: { type: 'json', required: true } } } } },
+        output: { schema: { type: 'object', additionalProperties: false, properties: { total: { type: 'number', required: true }, passed: { type: 'number', required: true }, failed: { type: 'number', required: true }, pass_rate: { type: 'number', required: true }, failures: { type: 'array', items: { type: 'string' }, required: true } } }, render: (_a, v) => text(`summary ${v.passed}/${v.total}`) },
+        async execute(args) { return summarize(args.results); },
+    });
+    return [lint, gradeTool, summary];
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-eval-pack",
-  "version": "0.1.0-alpha.7",
+  "version": "0.1.0-alpha.8",
   "description": "Public/synthetic eval-pack lint and deterministic JSON grader; not a validated benchmark.",
   "type": "module",
   "main": "./lib/plugin/index.js",

package/screenshots.json

@@ -10,5 +10,11 @@
     "kind": "demo",
     "alt": "Animated dsh-eval-pack terminal session",
     "caption": "The same run, assembled line by line."
+  },
+  {
+    "path": "docs/assets/dsh-eval-pack-capability.png",
+    "kind": "capability",
+    "alt": "dsh-eval-pack real test run",
+    "caption": "Real command: node --test test/summary.test.mjs — 4 tests passed, 0 failed (canonical grading and run summary). Run 2026-11-01; the image is rendered from the captured terminal output."
   }
 ]