dsh-session-eval 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -2
- package/README.md +2 -0
- package/README.zh.md +2 -0
- package/lib/index.d.ts +3 -2
- package/lib/index.js +47 -20
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
|
+
## 0.2.4 — 2026-10-11
|
|
2
|
+
|
|
3
|
+
Feature release: eval-history report files.
|
|
4
|
+
|
|
5
|
+
- New: `/eval-history --out PATH[.md|.json]` writes the history report to a Markdown or JSON file instead of only echoing it (contributed by @varshith84 in #4, merged as 117f918 after a maintainer rebase).
|
|
6
|
+
|
|
7
|
+
# Changelog
|
|
8
|
+
|
|
9
|
+
|
|
1
10
|
## 0.2.3 — 2026-10-03
|
|
2
11
|
|
|
3
12
|
Docs polish round (no runtime changes).
|
|
@@ -7,8 +16,6 @@ Docs polish round (no runtime changes).
|
|
|
7
16
|
- Community section; bilingual issue templates (`.github/ISSUE_TEMPLATE/`).
|
|
8
17
|
- English/Chinese README sections kept in sync.
|
|
9
18
|
|
|
10
|
-
# Changelog
|
|
11
|
-
|
|
12
19
|
## 0.2.2 — 2026-10-02
|
|
13
20
|
|
|
14
21
|
Contributor-first docs release.
|
package/README.md
CHANGED
|
@@ -91,6 +91,8 @@ Both are useful; they don't compete. Benchmarks test what you *predicted*; retro
|
|
|
91
91
|
| `/eval-history [N]` | Grade the last N sessions in this workspace and show the trend |
|
|
92
92
|
| `/eval-history … --json` | Machine-readable history report |
|
|
93
93
|
|
|
94
|
+
Use `/eval-history --out report.md` to save the terminal card as Markdown, or add `--json` to save the JSON payload. Relative paths resolve against the current session workspace (the process working directory when unavailable). Parent directories must already exist; file errors are reported without creating directories. Existing output files are overwritten.
|
|
95
|
+
|
|
94
96
|
`/eval-history` finds the sessions for you via the same `sessionQuery` seam `/archive` uses (scoped to the current workspace's `cwd`), grades each, and diffs the first against the last. `/eval-diff` remains the tool when you already know the two ids that matter.
|
|
95
97
|
|
|
96
98
|
Typical use: grade the ten sessions before and after a plugin or prompt change; a persistent `Reliability` regression is an early warning that the change hurt real work, not just benchmarks.
|
package/README.zh.md
CHANGED
|
@@ -91,6 +91,8 @@ Notes:
|
|
|
91
91
|
| `/eval-history [N]` | 本工作区最近 N 个会话成绩单 + 首末趋势 |
|
|
92
92
|
| `/eval-history … --json` | 机器可读历史报告 |
|
|
93
93
|
|
|
94
|
+
使用 `/eval-history --out report.md` 将终端卡片保存为 Markdown,添加 `--json` 则保存 JSON。相对路径基于当前会话工作目录(缺失时使用进程工作目录)。父目录必须已存在;写入失败会返回错误,不会创建目录。已存在的输出文件会被覆盖。
|
|
95
|
+
|
|
94
96
|
`/eval-history` 通过与 `/archive` 相同的 `sessionQuery` 接缝自动找到会话(按当前工作区 `cwd` 过滤),逐个评级,并把首尾两个会话做趋势对比——不用再手动挑 id。已经知道两个关键 id 时用 `/eval-diff`。
|
|
95
97
|
|
|
96
98
|
典型用法:换插件或 prompt 前后各评十个会话;`Reliability` 持续回归 = 改动伤害了真实工作的早期预警。
|
package/lib/index.d.ts
CHANGED
|
@@ -133,11 +133,12 @@ declare function executeEval(ctx: Context, invocation: CommandInvocation, seam?:
|
|
|
133
133
|
declare function executeEvalDiff(ctx: Context, invocation: CommandInvocation, seam?: QuerySeam): Promise<CommandResult>;
|
|
134
134
|
//#endregion
|
|
135
135
|
//#region src/historyCommand.d.ts
|
|
136
|
-
declare const EVAL_HISTORY_USAGE = "Usage: /eval-history [N] [--json] (N = last N sessions, 1-20, default 5)";
|
|
136
|
+
declare const EVAL_HISTORY_USAGE = "Usage: /eval-history [N] [--json] [--out PATH] (N = last N sessions, 1-20, default 5)";
|
|
137
137
|
/** Args for /eval-history. */
|
|
138
138
|
interface EvalHistoryArgs {
|
|
139
139
|
readonly count: number;
|
|
140
140
|
readonly json: boolean;
|
|
141
|
+
readonly out?: string;
|
|
141
142
|
}
|
|
142
143
|
/** Parse /eval-history input; returns args or a usage-error string. */
|
|
143
144
|
declare function parseEvalHistoryArgs(rawInput: string): EvalHistoryArgs | string;
|
|
@@ -177,7 +178,7 @@ declare function executeEvalHistory(ctx: Context, invocation: CommandInvocation,
|
|
|
177
178
|
*
|
|
178
179
|
* @module dsh-session-eval/version
|
|
179
180
|
*/
|
|
180
|
-
declare const VERSION = "0.2.
|
|
181
|
+
declare const VERSION = "0.2.4";
|
|
181
182
|
//#endregion
|
|
182
183
|
//#region src/index.d.ts
|
|
183
184
|
declare const name = "session-eval";
|
package/lib/index.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { SessionId } from "@deepseek-ai/dsh-session";
|
|
2
|
+
import { writeFile } from "node:fs/promises";
|
|
3
|
+
import { resolve } from "node:path";
|
|
2
4
|
//#region src/metrics.ts
|
|
3
5
|
/** Count a correction signal for every extra user message inside a consecutive-user run. */
|
|
4
6
|
function countCorrections(events) {
|
|
@@ -227,7 +229,7 @@ function compareSessions(before, after) {
|
|
|
227
229
|
*
|
|
228
230
|
* @module dsh-session-eval/version
|
|
229
231
|
*/
|
|
230
|
-
const VERSION = "0.2.
|
|
232
|
+
const VERSION = "0.2.4";
|
|
231
233
|
//#endregion
|
|
232
234
|
//#region src/evalCommand.ts
|
|
233
235
|
const EVAL_USAGE = "Usage: /eval [--id <sessionId>] [--json]";
|
|
@@ -327,14 +329,14 @@ async function executeEval(ctx, invocation, seam = ctx.sessionQuery) {
|
|
|
327
329
|
if (parsed.json === true) return {
|
|
328
330
|
kind: "success",
|
|
329
331
|
text: JSON.stringify({
|
|
330
|
-
generator: "dsh-session-eval v0.2.
|
|
332
|
+
generator: "dsh-session-eval v0.2.4",
|
|
331
333
|
metrics,
|
|
332
334
|
card
|
|
333
335
|
}, null, 2)
|
|
334
336
|
};
|
|
335
337
|
return {
|
|
336
338
|
kind: "success",
|
|
337
|
-
text: renderCard(card, "dsh-session-eval v0.2.
|
|
339
|
+
text: renderCard(card, "dsh-session-eval v0.2.4")
|
|
338
340
|
};
|
|
339
341
|
}
|
|
340
342
|
/** Execute /eval-diff against the session-query seam. */
|
|
@@ -358,25 +360,44 @@ async function executeEvalDiff(ctx, invocation, seam = ctx.sessionQuery) {
|
|
|
358
360
|
if (parsed.json === true) return {
|
|
359
361
|
kind: "success",
|
|
360
362
|
text: JSON.stringify({
|
|
361
|
-
generator: "dsh-session-eval v0.2.
|
|
363
|
+
generator: "dsh-session-eval v0.2.4",
|
|
362
364
|
report
|
|
363
365
|
}, null, 2)
|
|
364
366
|
};
|
|
365
367
|
return {
|
|
366
368
|
kind: "success",
|
|
367
|
-
text: renderTrend(report, "dsh-session-eval v0.2.
|
|
369
|
+
text: renderTrend(report, "dsh-session-eval v0.2.4")
|
|
368
370
|
};
|
|
369
371
|
}
|
|
370
372
|
//#endregion
|
|
371
373
|
//#region src/historyCommand.ts
|
|
372
|
-
|
|
374
|
+
/**
|
|
375
|
+
* `/eval-history` — grade the last N sessions in this workspace and show the trend.
|
|
376
|
+
*
|
|
377
|
+
* Completes the regression-tracking story: `/eval-diff` compares two sessions
|
|
378
|
+
* you already know about; this command finds them for you (via
|
|
379
|
+
* `ctx.sessionQuery.listSessions()`, same seam as `/archive` in
|
|
380
|
+
* dsh-session-export) and answers "is the trend improving?" out of the box.
|
|
381
|
+
*
|
|
382
|
+
* @module dsh-session-eval/historyCommand
|
|
383
|
+
*/
|
|
384
|
+
const EVAL_HISTORY_USAGE = "Usage: /eval-history [N] [--json] [--out PATH] (N = last N sessions, 1-20, default 5)";
|
|
373
385
|
/** Parse /eval-history input; returns args or a usage-error string. */
|
|
374
386
|
function parseEvalHistoryArgs(rawInput) {
|
|
375
387
|
const tokens = rawInput.trim().split(/\s+/).filter((t) => t.length > 0);
|
|
376
388
|
let count = 5;
|
|
377
389
|
let json = false;
|
|
378
390
|
let sawCount = false;
|
|
379
|
-
|
|
391
|
+
let out;
|
|
392
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
393
|
+
const token = tokens[i];
|
|
394
|
+
if (token === "--out") {
|
|
395
|
+
if (out !== void 0) return `--out may be given only once.\n${EVAL_HISTORY_USAGE}`;
|
|
396
|
+
const path = tokens[++i];
|
|
397
|
+
if (!path || path.startsWith("--")) return `--out requires a path.\n${EVAL_HISTORY_USAGE}`;
|
|
398
|
+
out = path;
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
380
401
|
if (token === "--json") {
|
|
381
402
|
if (json) return `--json may be given only once.\n${EVAL_HISTORY_USAGE}`;
|
|
382
403
|
json = true;
|
|
@@ -392,7 +413,8 @@ function parseEvalHistoryArgs(rawInput) {
|
|
|
392
413
|
}
|
|
393
414
|
return {
|
|
394
415
|
count,
|
|
395
|
-
json
|
|
416
|
+
json,
|
|
417
|
+
...out !== void 0 ? { out } : {}
|
|
396
418
|
};
|
|
397
419
|
}
|
|
398
420
|
function toMillis(createdAt) {
|
|
@@ -441,10 +463,6 @@ async function executeEvalHistory(ctx, invocation, seam = ctx.sessionQuery) {
|
|
|
441
463
|
id: String(header.id),
|
|
442
464
|
createdAt: toMillis(header.createdAt)
|
|
443
465
|
})).sort((a, b) => (a.createdAt ?? 0) - (b.createdAt ?? 0)).slice(-parsed.count);
|
|
444
|
-
if (recent.length === 0) return {
|
|
445
|
-
kind: "success",
|
|
446
|
-
text: "No sessions found in this workspace; nothing to grade."
|
|
447
|
-
};
|
|
448
466
|
const entries = [];
|
|
449
467
|
for (const header of recent) {
|
|
450
468
|
let events;
|
|
@@ -468,16 +486,25 @@ async function executeEvalHistory(ctx, invocation, seam = ctx.sessionQuery) {
|
|
|
468
486
|
entries,
|
|
469
487
|
trend: entries.length >= 2 ? compareSessions(entries[0].metrics, entries[entries.length - 1].metrics) : null
|
|
470
488
|
};
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
489
|
+
const text = parsed.json ? JSON.stringify({
|
|
490
|
+
generator: `dsh-session-eval v${VERSION}`,
|
|
491
|
+
report
|
|
492
|
+
}, null, 2) : renderHistory(report);
|
|
493
|
+
if (parsed.out !== void 0) {
|
|
494
|
+
const path = resolve(typeof ownCwd === "string" ? ownCwd : process.cwd(), parsed.out);
|
|
495
|
+
try {
|
|
496
|
+
const output = parsed.json ? text : `\`\`\`text\n${text}\n\`\`\``;
|
|
497
|
+
await writeFile(path, `${output}\n`, "utf8");
|
|
498
|
+
} catch (error) {
|
|
499
|
+
return {
|
|
500
|
+
kind: "error",
|
|
501
|
+
text: `Could not write history report to ${path}: ${error instanceof Error ? error.message : String(error)}`
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
}
|
|
478
505
|
return {
|
|
479
506
|
kind: "success",
|
|
480
|
-
text
|
|
507
|
+
text
|
|
481
508
|
};
|
|
482
509
|
}
|
|
483
510
|
//#endregion
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-session-eval",
|
|
3
3
|
"description": "Retrospective agent-session evaluation for DeepSeek Harness: deterministic grade cards from persisted session logs, cross-session regression diffs, era comparison — no benchmark runs, no LLM judges.",
|
|
4
|
-
"version": "0.2.
|
|
4
|
+
"version": "0.2.4",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|