oh-my-knowledge 0.34.0 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/README.zh.md +3 -0
- package/dist/assets/agent-skills/omk/SKILL.md +2 -2
- package/dist/assets/agent-skills/omk/references/commands.md +45 -0
- package/dist/authoring/evolver.js +13 -4
- package/dist/cli/commands/doctor.js +2 -1
- package/dist/cli/commands/eval/index.d.ts +1 -0
- package/dist/cli/commands/eval/index.js +40 -5
- package/dist/cli/commands/install.d.ts +2 -0
- package/dist/cli/commands/install.js +45 -20
- package/dist/cli/commands/list.d.ts +46 -0
- package/dist/cli/commands/list.js +243 -0
- package/dist/cli/commands/sample.d.ts +1 -1
- package/dist/cli/commands/sample.js +25 -12
- package/dist/cli/lib/cmd-flags.d.ts +1 -0
- package/dist/cli/lib/i18n-dict/install.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/install.js +20 -0
- package/dist/cli/lib/i18n-dict/list.d.ts +3 -0
- package/dist/cli/lib/i18n-dict/list.js +32 -0
- package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/run.js +8 -0
- package/dist/cli/lib/i18n-dict.d.ts +2 -1
- package/dist/cli/lib/i18n-dict.js +2 -0
- package/dist/cli/lib/parse-run-config/variant-resolution.js +7 -0
- package/dist/cli/lib/run-tally.js +1 -1
- package/dist/cli/lib/shared.js +1 -1
- package/dist/doctor/index.js +6 -6
- package/dist/eval-core/cache.d.ts +11 -3
- package/dist/eval-core/cache.js +13 -5
- package/dist/eval-core/dependency-checker.js +5 -3
- package/dist/eval-core/evaluation-execution.js +4 -1
- package/dist/eval-core/evaluation-reporting.d.ts +1 -0
- package/dist/eval-core/evaluation-reporting.js +20 -9
- package/dist/eval-core/execution-strategy.js +6 -4
- package/dist/eval-core/task-planner.js +2 -1
- package/dist/eval-workflows/batch-evaluation-workflow.js +3 -2
- package/dist/eval-workflows/evaluation-preparation.js +3 -1
- package/dist/eval-workflows/run-evaluation.js +2 -2
- package/dist/inputs/content-hash.d.ts +28 -0
- package/dist/inputs/content-hash.js +106 -0
- package/dist/inputs/eval-config.js +44 -11
- package/dist/inputs/materialize-copy.d.ts +37 -0
- package/dist/inputs/materialize-copy.js +193 -0
- package/dist/inputs/skill-loader.d.ts +65 -2
- package/dist/inputs/skill-loader.js +314 -16
- package/dist/inputs/source-resolver.d.ts +16 -9
- package/dist/inputs/source-resolver.js +57 -77
- package/dist/managed/evidence.d.ts +22 -0
- package/dist/managed/evidence.js +143 -0
- package/dist/managed/index.d.ts +2 -0
- package/dist/managed/index.js +2 -0
- package/dist/managed/list-view.d.ts +55 -0
- package/dist/managed/list-view.js +66 -0
- package/dist/managed/store.d.ts +16 -20
- package/dist/managed/store.js +85 -84
- package/dist/observability/experience.d.ts +2 -0
- package/dist/observability/experience.js +31 -2
- package/dist/observability/inbox.js +31 -5
- package/dist/observability/review-state.js +22 -11
- package/dist/observability/soft-standards/llm-extractor.js +3 -3
- package/dist/observability/soft-standards/skill-standards-store.d.ts +1 -0
- package/dist/observability/soft-standards/skill-standards-store.js +77 -14
- package/dist/observability/soft-standards/types.d.ts +2 -2
- package/dist/renderer/html-renderer.js +3 -3
- package/dist/renderer/layout.js +2 -2
- package/dist/server/report-server.js +1 -1
- package/dist/server/report-store.d.ts +1 -1
- package/dist/server/report-store.js +16 -15
- package/dist/server/skill-index.js +5 -4
- package/dist/types/doctor.d.ts +2 -2
- package/dist/types/doctor.js +1 -1
- package/dist/types/eval.d.ts +12 -1
- package/dist/types/managed.d.ts +29 -4
- package/dist/types/observability.d.ts +6 -6
- package/dist/types/report.d.ts +24 -5
- package/package.json +3 -3
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { existsSync, lstatSync, readdirSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute, join, resolve } from 'node:path';
|
|
3
|
+
import { Flags } from '@oclif/core';
|
|
4
|
+
import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
|
|
5
|
+
import { BaseCommand } from '../oclif/base-command.js';
|
|
6
|
+
import { tCli } from '../lib/i18n.js';
|
|
7
|
+
import { resolveInstallSource } from '../../inputs/source-resolver.js';
|
|
8
|
+
import { buildManagedListRows, globalManagedDir, hashArtifactSource, isDistributablePath, loadAllManagedRecords, managedDir, resolveManagedDir, } from '../../managed/index.js';
|
|
9
|
+
// 本地源读取成本上限。skill 是小体量 markdown(+少量 references 资产),远超这些边界的源必是手改 /
|
|
10
|
+
// 投毒,拒读避免 `omk list`(只读命令,读盘上可能随仓库分发的 locator)被 /dev/zero / 超大文件 / 巨型
|
|
11
|
+
// 目录递归拖垮(DoS)。
|
|
12
|
+
const MAX_FILE_SOURCE_BYTES = 8 * 1024 * 1024; // 单文件-skill 上限
|
|
13
|
+
const MAX_DIR_SOURCE_BYTES = 64 * 1024 * 1024; // 目录-skill 整树累计上限
|
|
14
|
+
const MAX_DIR_SOURCE_FILES = 4000; // 目录-skill 文件数上限
|
|
15
|
+
const MAX_DIR_SOURCE_DEPTH = 64; // 目录递归深度上限
|
|
16
|
+
/**
|
|
17
|
+
* 目录-skill 源的**有边界**整树哈:先恢复 resolveFileSource 的形态校验(SKILL.md 存在、是常规文件、非
|
|
18
|
+
* 软链 —— 否则 locator 可指向任意可读目录让 list 递归读整棵树),再 stat-walk(只 stat 不读)按与
|
|
19
|
+
* hashArtifactSource 同一 `isDistributablePath` 过滤累计 文件数 / 字节 / 深度,超界即返回 null(unreachable),
|
|
20
|
+
* 把单文件 DoS 不再换成目录递归 DoS。通过后才真正 hashArtifactSource。返回 null = 拒读 / 非法 skill 目录。
|
|
21
|
+
*/
|
|
22
|
+
function boundedDirSkillHash(abs) {
|
|
23
|
+
const skillMd = join(abs, 'SKILL.md');
|
|
24
|
+
let md;
|
|
25
|
+
try {
|
|
26
|
+
md = lstatSync(skillMd);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return null; // 无 SKILL.md → 不是 skill 目录,拒
|
|
30
|
+
}
|
|
31
|
+
if (md.isSymbolicLink() || !md.isFile() || md.nlink !== 1)
|
|
32
|
+
return null; // SKILL.md 必须是常规、非软链、非硬链
|
|
33
|
+
let files = 0;
|
|
34
|
+
let bytes = 0;
|
|
35
|
+
const within = (dir, segs, depth) => {
|
|
36
|
+
if (depth > MAX_DIR_SOURCE_DEPTH)
|
|
37
|
+
return false;
|
|
38
|
+
let entries;
|
|
39
|
+
try {
|
|
40
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
for (const e of entries) {
|
|
46
|
+
const ns = [...segs, e.name];
|
|
47
|
+
if (!isDistributablePath(ns))
|
|
48
|
+
continue; // 与 hashArtifactSource 读取范围一致
|
|
49
|
+
if (e.isDirectory()) {
|
|
50
|
+
if (!within(join(dir, e.name), ns, depth + 1))
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
else if (e.isFile()) { // 软链 Dirent 既非 isFile 也非 isDirectory → 天然跳过
|
|
54
|
+
if (++files > MAX_DIR_SOURCE_FILES)
|
|
55
|
+
return false;
|
|
56
|
+
try {
|
|
57
|
+
const fst = lstatSync(join(dir, e.name));
|
|
58
|
+
if (fst.nlink !== 1)
|
|
59
|
+
return false; // 硬链(nlink>1)可在树内别名树外敏感 inode → 拒读整树
|
|
60
|
+
bytes += fst.size;
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
if (bytes > MAX_DIR_SOURCE_BYTES)
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return true;
|
|
70
|
+
};
|
|
71
|
+
if (!within(abs, [], 0))
|
|
72
|
+
return null;
|
|
73
|
+
return hashArtifactSource(abs, true);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* 探测一条受管记录**当前源**的状态(三态,喂 buildManagedListRow 判 drift / 生命周期)。
|
|
77
|
+
* - 远端 git(带 url):源身份钉不可变 SHA(install 时 pin),内容恒定 → 直接取 record.contentHash,
|
|
78
|
+
* 不联网(list 是快读命令,不为 drift 检查发网络请求)。reachable。
|
|
79
|
+
* - 本地 git:locator `git:<ref>:<spec>` 复用 resolveInstallSource 在**仓库对象库**内重物化重哈
|
|
80
|
+
* (读取受 git 边界约束,无任意文件读 DoS)。解析不到(常因 `omk list` 的 cwd 与 install 时不同、
|
|
81
|
+
* spec 随 cwd 漂)→ 抛错 → **reachable:false(未核,不当 stale)**,避免对未改动的 skill 误报漂移。
|
|
82
|
+
* - 本地 file:locator 是绝对路径,但受管 JSON **用户可手改 / 随仓库分发**(无 install、无 opt-in 即被
|
|
83
|
+
* loadAllManagedRecords 读到)。只读命令绝不盲读任意路径:**拒软链、只读常规文件 / 真目录、单文件
|
|
84
|
+
* 设 size cap**(挡 `evil.md → /dev/zero` 这类 readFileSync 无界 DoS 与项目外任意读)。守卫不过 →
|
|
85
|
+
* reachable:false。目录-skill 整树哈本就跳软链(hashArtifactSource 用 isFile() 过滤)。
|
|
86
|
+
*/
|
|
87
|
+
export function probeSourceState(record) {
|
|
88
|
+
const s = record.source;
|
|
89
|
+
if (s.sourceKind === 'git' && s.url)
|
|
90
|
+
return { reachable: true, hash: record.contentHash };
|
|
91
|
+
try {
|
|
92
|
+
if (s.sourceKind === 'git') {
|
|
93
|
+
const resolved = resolveInstallSource(s.locator);
|
|
94
|
+
try {
|
|
95
|
+
return { reachable: true, hash: hashArtifactSource(resolved.localRoot, resolved.isDirectorySkill) };
|
|
96
|
+
}
|
|
97
|
+
finally {
|
|
98
|
+
resolved.cleanup();
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// 本地 file 源:守卫后再读。locator 必须是 install 实际写出的形态(绝对路径) —— 受管 JSON 随仓库
|
|
102
|
+
// 分发、无 opt-in 即被读到,相对 locator 不是 install 产物,拒,避免按 cwd 解析到项目外。
|
|
103
|
+
if (!isAbsolute(s.locator))
|
|
104
|
+
return { reachable: false };
|
|
105
|
+
const abs = resolve(s.locator);
|
|
106
|
+
if (!existsSync(abs))
|
|
107
|
+
return { reachable: false };
|
|
108
|
+
const st = lstatSync(abs); // lstat:不跟随软链 —— 软链直接拒(防 evil.md → /dev/zero)
|
|
109
|
+
if (st.isSymbolicLink())
|
|
110
|
+
return { reachable: false };
|
|
111
|
+
if (s.isDirectorySkill) {
|
|
112
|
+
if (!st.isDirectory())
|
|
113
|
+
return { reachable: false };
|
|
114
|
+
const hash = boundedDirSkillHash(abs); // 形态校验 + 成本边界(防任意目录递归读 / 目录 DoS)
|
|
115
|
+
return hash === null ? { reachable: false } : { reachable: true, hash };
|
|
116
|
+
}
|
|
117
|
+
// 单文件-skill:恢复 install(resolveFileSource)的形态约束 —— 必须是 `.md` 常规文件、非硬链、≤ size cap。
|
|
118
|
+
// 否则攻击者可写 locator:`/etc/passwd` / `~/.ssh/id_rsa`(非 .md 直接拒),或用 `.md` 命名的**硬链**别名
|
|
119
|
+
// 树外敏感文件绕过扩展名 / 软链守卫(lstat 分不出硬链)→ 诱 list 把任意本地文件读进进程参与 hash。install
|
|
120
|
+
// 写出的是 nlink=1 的全新副本,拒 nlink>1 不误伤合法源。非 install 形态一律 reachable:false。
|
|
121
|
+
if (!/\.md$/i.test(abs) || !st.isFile() || st.nlink !== 1 || st.size > MAX_FILE_SOURCE_BYTES)
|
|
122
|
+
return { reachable: false };
|
|
123
|
+
return { reachable: true, hash: hashArtifactSource(abs, false) };
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return { reachable: false };
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
/** CJK 全角字符按 2 列计宽,使含中文表头的列也能对齐。 */
|
|
130
|
+
export function dispWidth(s) {
|
|
131
|
+
let w = 0;
|
|
132
|
+
for (const ch of s)
|
|
133
|
+
w += /[ᄀ-ᅟ⺀-가-힣豈-︰-﹏-⦆¢-₩]/.test(ch) ? 2 : 1;
|
|
134
|
+
return w;
|
|
135
|
+
}
|
|
136
|
+
function pad(s, width) {
|
|
137
|
+
return s + ' '.repeat(Math.max(0, width - dispWidth(s)));
|
|
138
|
+
}
|
|
139
|
+
/** 按**显示宽度**截断(不是 code unit):逐码点累加 dispWidth,绝不切断 surrogate 对、CJK 也不溢出列。 */
|
|
140
|
+
export function truncate(s, max) {
|
|
141
|
+
if (dispWidth(s) <= max)
|
|
142
|
+
return s;
|
|
143
|
+
let w = 0;
|
|
144
|
+
let out = '';
|
|
145
|
+
for (const ch of s) {
|
|
146
|
+
const cw = dispWidth(ch);
|
|
147
|
+
if (w + cw > max - 1)
|
|
148
|
+
break; // 留 1 列给省略号
|
|
149
|
+
out += ch;
|
|
150
|
+
w += cw;
|
|
151
|
+
}
|
|
152
|
+
return `${out}…`;
|
|
153
|
+
}
|
|
154
|
+
/** 洗不可信显示字符:managed JSON 可随仓库分发,name / sourceLabel / verdict 等字段塞进表格前必须先洗,
|
|
155
|
+
* 否则会破坏表格甚至伪造终端输出。一律映射到可见 U+FFFD(--json 路径保留原值给脚本)。
|
|
156
|
+
* 用 Unicode **属性类**而非手列码点 —— 手列清单天然有缺口(BiDi、U+2028 / 2029、Tags 块都曾漏一轮补一轮),
|
|
157
|
+
* 属性类一次覆盖整类、新码点自动纳入:
|
|
158
|
+
* - `\p{Cc}` 控制符(C0 / C1 / DEL,含 ESC / 换行 / 回车 / TAB)→ 杀 ANSI / OSC 转义与终端控制;
|
|
159
|
+
* - `\p{Cf}` 格式符(BiDi 重排 / 隔离、零宽、joiners、BOM、Tags、interlinear)→ 防 Trojan-Source 视觉伪造与零宽隐藏 / 分割;
|
|
160
|
+
* - `\p{Zl}` / `\p{Zp}` 行 / 段分隔(U+2028 / 2029)→ LF 的 Unicode 孪生,防换行伪造表格行;
|
|
161
|
+
* - `\p{Mn}` / `\p{Me}` 非间距 / 封闭组合附加符 → 变可见,避免零前进宽度令 dispWidth 与终端列错位
|
|
162
|
+
* (间距组合符 `\p{Mc}` 合法占 1 列,保留);
|
|
163
|
+
* - Hangul filler(U+115F / U+1160 / U+3164,属 Lo 不在上述任何类)→ 零宽显示诡计,补列。 */
|
|
164
|
+
export function sanitizeCell(s) {
|
|
165
|
+
return s.replace(/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}\p{Mn}\p{Me}ᅟᅠㅤ]/gu, '�');
|
|
166
|
+
}
|
|
167
|
+
export function renderTable(rows, lang) {
|
|
168
|
+
const headers = [
|
|
169
|
+
tCli('cli.list.col_name', lang),
|
|
170
|
+
tCli('cli.list.col_kind', lang),
|
|
171
|
+
tCli('cli.list.col_state', lang),
|
|
172
|
+
tCli('cli.list.col_verdict', lang),
|
|
173
|
+
tCli('cli.list.col_evidence', lang),
|
|
174
|
+
tCli('cli.list.col_source', lang),
|
|
175
|
+
];
|
|
176
|
+
const cells = rows.map((r) => [
|
|
177
|
+
truncate(sanitizeCell(r.name), 40), // name 与 source 同为用户可控、可超长 → 同样按显示宽度截断,防撑爆表宽
|
|
178
|
+
r.kind, // ArtifactKind 枚举(validator 已收窄),无需洗
|
|
179
|
+
// 不可达 → 标「?」(drift 未核),绝不冒充 stale;reachable 且漂移才 stale ⚠️。
|
|
180
|
+
!r.reachable ? `${r.state} ?` : r.state === 'stale' ? 'stale ⚠️' : r.state,
|
|
181
|
+
r.latestVerdict ? sanitizeCell(r.latestVerdict) : '—',
|
|
182
|
+
`${r.currentEvidenceCount}/${r.totalEvidenceCount}`,
|
|
183
|
+
truncate(sanitizeCell(r.sourceLabel), 48),
|
|
184
|
+
]);
|
|
185
|
+
const widths = headers.map((h, i) => Math.max(dispWidth(h), ...cells.map((c) => dispWidth(c[i]))));
|
|
186
|
+
const line = (c) => c.map((v, i) => pad(v, widths[i])).join(' ').trimEnd();
|
|
187
|
+
return [line(headers), ...cells.map(line)].join('\n');
|
|
188
|
+
}
|
|
189
|
+
export default class List extends BaseCommand {
|
|
190
|
+
static description = bilingual({
|
|
191
|
+
zh: '列出受管 skill 及其证据状态:生命周期(installed / measurable / stale)、最新 verdict、证据数、源。',
|
|
192
|
+
en: 'List managed skills with evidence status: lifecycle (installed / measurable / stale), latest verdict, evidence count, source.',
|
|
193
|
+
});
|
|
194
|
+
static examples = [
|
|
195
|
+
{ description: bilingual({ zh: '列出当前项目的受管 skill', en: 'List managed skills in the current project' }), command: '<%= config.bin %> list' },
|
|
196
|
+
{ description: bilingual({ zh: '列出全局受管 skill', en: 'List globally managed skills' }), command: '<%= config.bin %> list --global' },
|
|
197
|
+
{ description: bilingual({ zh: '机器可读 JSON 输出', en: 'Machine-readable JSON output' }), command: '<%= config.bin %> list --json' },
|
|
198
|
+
];
|
|
199
|
+
static flags = {
|
|
200
|
+
lang: LANG_FLAG,
|
|
201
|
+
global: Flags.boolean({
|
|
202
|
+
description: bilingual({ zh: '看全局受管目录(~/.oh-my-knowledge/managed)而非项目 .omk/managed', en: 'Show the global managed dir (~/.oh-my-knowledge/managed) instead of project .omk/managed' }),
|
|
203
|
+
}),
|
|
204
|
+
json: Flags.boolean({
|
|
205
|
+
description: bilingual({ zh: '输出 JSON(含完整可比性 marker),供脚本消费', en: 'Output JSON (with full comparability markers) for scripts' }),
|
|
206
|
+
}),
|
|
207
|
+
};
|
|
208
|
+
async run() {
|
|
209
|
+
const { flags } = await this.parse(List);
|
|
210
|
+
const lang = this.lang;
|
|
211
|
+
await this.runWithCliExit(async () => {
|
|
212
|
+
const dir = flags.global ? globalManagedDir() : resolveManagedDir(managedDir());
|
|
213
|
+
const records = loadAllManagedRecords(dir);
|
|
214
|
+
const rows = buildManagedListRows(records, probeSourceState);
|
|
215
|
+
if (flags.json) {
|
|
216
|
+
// 版本化信封 —— 与 eval / doctor / diagnosis 等机读出口一致(都带 schemaVersion),让未来字段增删 /
|
|
217
|
+
// 改名是可检测的版本 bump,而非脚本静默 break。rows 的逐行形态见 ManagedListRow。
|
|
218
|
+
this.log(JSON.stringify({ schemaVersion: 1, rows }, null, 2));
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
if (rows.length === 0) {
|
|
222
|
+
process.stderr.write(tCli('cli.list.empty', lang));
|
|
223
|
+
process.stderr.write(tCli('cli.list.empty_hint', lang));
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
const scope = dir === globalManagedDir()
|
|
227
|
+
? (lang === 'zh' ? '全局' : 'global')
|
|
228
|
+
: (lang === 'zh' ? '项目' : 'project');
|
|
229
|
+
process.stderr.write(tCli('cli.list.header', lang, { scope, count: rows.length }));
|
|
230
|
+
this.log(renderTable(rows, lang));
|
|
231
|
+
// 注脚块前留一行空白(无论先触发哪条 note);空行挂在块上而非 drift 分支,避免只 unreachable 时丢分隔。
|
|
232
|
+
const hasDrift = rows.some((r) => r.drifted);
|
|
233
|
+
const hasUnreachable = rows.some((r) => !r.reachable);
|
|
234
|
+
if (hasDrift || hasUnreachable)
|
|
235
|
+
process.stderr.write('\n');
|
|
236
|
+
if (hasDrift)
|
|
237
|
+
process.stderr.write(tCli('cli.list.drift_note', lang));
|
|
238
|
+
if (hasUnreachable)
|
|
239
|
+
process.stderr.write(tCli('cli.list.unreachable_note', lang));
|
|
240
|
+
process.stderr.write(tCli('cli.list.legend', lang));
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
}
|
|
@@ -6,7 +6,7 @@ export declare function collectSampleDesignFailureIds(report: Pick<Report, 'resu
|
|
|
6
6
|
export declare function assertFixReportMatchesCurrentInputs(params: {
|
|
7
7
|
report: Pick<Report, 'meta'>;
|
|
8
8
|
treatmentName: string;
|
|
9
|
-
|
|
9
|
+
currentContentHash: string;
|
|
10
10
|
samples: SampleType[];
|
|
11
11
|
sampleIds: Set<string>;
|
|
12
12
|
lang?: CliLang;
|
|
@@ -9,7 +9,8 @@ import { CliExit } from '../lib/cli-exit.js';
|
|
|
9
9
|
import { tCli } from '../lib/i18n.js';
|
|
10
10
|
import { DEFAULT_REPORTS_DIR } from '../lib/parse-run-config.js';
|
|
11
11
|
import { loadSamples, parseYaml } from '../../inputs/load-samples.js';
|
|
12
|
-
import { hashSample
|
|
12
|
+
import { hashSample } from '../../eval-core/evaluation-reporting.js';
|
|
13
|
+
import { hashArtifactSource } from '../../inputs/content-hash.js';
|
|
13
14
|
function isRecord(value) {
|
|
14
15
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
15
16
|
}
|
|
@@ -48,20 +49,29 @@ export function collectSampleDesignFailureIds(report, treatmentName) {
|
|
|
48
49
|
return ids;
|
|
49
50
|
}
|
|
50
51
|
export function assertFixReportMatchesCurrentInputs(params) {
|
|
51
|
-
const { report, treatmentName,
|
|
52
|
+
const { report, treatmentName, currentContentHash, samples, sampleIds } = params;
|
|
52
53
|
const lang = params.lang ?? 'zh';
|
|
53
54
|
const issues = [];
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
// schemaVersion < 2 的报告:artifactHashes 是旧「仅 SKILL.md 正文文本」哈,与当前「整棵可分发树」哈
|
|
56
|
+
// 不同空间,直接比对会必然误报不一致。识别后给可见提示(归入 issues → 触发「请先重跑 eval」),
|
|
57
|
+
// 不拿旧文本哈与当前树哈错配比对。sample 指纹口径未变,下面照常校。
|
|
58
|
+
if ((report.meta.schemaVersion ?? 0) < 2) {
|
|
57
59
|
issues.push(lang === 'zh'
|
|
58
|
-
?
|
|
59
|
-
: `Report
|
|
60
|
+
? `报告早于树哈纪元(skill 指纹口径已从「仅 SKILL.md 文本」改为「整棵可分发树」),无法与当前指纹比对。`
|
|
61
|
+
: `Report predates the tree-hash era (skill fingerprint changed from SKILL.md-body-text to whole-tree); cannot compare against the current fingerprint.`);
|
|
60
62
|
}
|
|
61
|
-
else
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
else {
|
|
64
|
+
const expectedSkillHash = report.meta.artifactHashes?.[treatmentName];
|
|
65
|
+
if (!expectedSkillHash) {
|
|
66
|
+
issues.push(lang === 'zh'
|
|
67
|
+
? `报告缺少 ${treatmentName} 的 skill 指纹,无法确认诊断对应当前 SKILL.md。`
|
|
68
|
+
: `Report is missing the skill hash for ${treatmentName}; cannot verify it matches the current SKILL.md.`);
|
|
69
|
+
}
|
|
70
|
+
else if (expectedSkillHash !== currentContentHash) {
|
|
71
|
+
issues.push(lang === 'zh'
|
|
72
|
+
? `skill 指纹不一致:报告 ${expectedSkillHash},当前 ${currentContentHash}。`
|
|
73
|
+
: `Skill hash mismatch: report ${expectedSkillHash}, current ${currentContentHash}.`);
|
|
74
|
+
}
|
|
65
75
|
}
|
|
66
76
|
const reportSampleHashes = report.meta.sampleHashes;
|
|
67
77
|
if (!reportSampleHashes) {
|
|
@@ -196,11 +206,14 @@ async function runSampleFix(args, flags, lang) {
|
|
|
196
206
|
process.stderr.write(lang === 'zh' ? '✅ 没有 sample_design 类型的失败,无需修复\n' : '✅ No sample_design failures found, nothing to fix\n');
|
|
197
207
|
return;
|
|
198
208
|
}
|
|
209
|
+
// 当前内容指纹走整树哈,与 eval 报告口径一致:dir-skill(用户传 .../SKILL.md)哈整棵 skill 目录、
|
|
210
|
+
// 单文件 .md 哈单文件字节。
|
|
211
|
+
const currentContentHash = hashArtifactSource(isDir ? skillDir : resolvedSkillPath, isDir);
|
|
199
212
|
try {
|
|
200
213
|
assertFixReportMatchesCurrentInputs({
|
|
201
214
|
report,
|
|
202
215
|
treatmentName,
|
|
203
|
-
|
|
216
|
+
currentContentHash,
|
|
204
217
|
samples,
|
|
205
218
|
sampleIds: sampleDesignIds,
|
|
206
219
|
lang,
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CliMessage } from './types.js';
|
|
2
|
-
export type InstallMessageKey = 'cli.install.asset_missing' | 'cli.install.unknown_input' | 'cli.install.unknown_target' | 'cli.install.invalid_target_combo' | 'cli.install.no_detected_targets' | 'cli.install.target_exists' | 'cli.install.plan' | 'cli.install.plan_skill' | 'cli.install.installed' | 'cli.install.kind_unsupported' | 'cli.install.copied' | 'cli.install.adopted' | 'cli.install.registered' | 'cli.install.target_overlaps_source' | 'cli.install.path_not_found' | 'cli.install.skillmd_missing' | 'cli.install.skillmd_is_symlink' | 'cli.install.not_a_skill' | 'cli.install.not_a_git_repo' | 'cli.install.git_skill_not_found' | 'cli.install.git_unsafe_path' | 'cli.install.next_hint';
|
|
2
|
+
export type InstallMessageKey = 'cli.install.asset_missing' | 'cli.install.unknown_input' | 'cli.install.unknown_target' | 'cli.install.invalid_target_combo' | 'cli.install.no_detected_targets' | 'cli.install.target_exists' | 'cli.install.plan' | 'cli.install.plan_skill' | 'cli.install.installed' | 'cli.install.kind_unsupported' | 'cli.install.copied' | 'cli.install.adopted' | 'cli.install.registered' | 'cli.install.target_overlaps_source' | 'cli.install.path_not_found' | 'cli.install.skillmd_missing' | 'cli.install.skillmd_is_symlink' | 'cli.install.not_a_skill' | 'cli.install.not_a_git_repo' | 'cli.install.git_skill_not_found' | 'cli.install.git_unsafe_path' | 'cli.install.invalid_remote_url' | 'cli.install.remote_fetch_failed' | 'cli.install.remote_ref_not_found' | 'cli.install.remote_skill_not_found' | 'cli.install.git_ref_needs_url' | 'cli.install.next_hint';
|
|
3
3
|
export declare const installDict: Record<InstallMessageKey, CliMessage>;
|
|
@@ -83,6 +83,26 @@ export const installDict = {
|
|
|
83
83
|
zh: 'git tree 含越界路径 {path}(.. / 绝对路径 / 空段),拒绝物化以防写出临时目录。该 skill 的 git tree 可能被手工构造,请核查来源。',
|
|
84
84
|
en: 'git tree contains an out-of-bounds path {path} (.. / absolute / empty segment); refusing to materialize to avoid escaping the temp dir. The skill git tree may be hand-crafted; verify the source.',
|
|
85
85
|
},
|
|
86
|
+
'cli.install.invalid_remote_url': {
|
|
87
|
+
zh: '不是合法的 git 远端 URL:{url}。支持 https:// / ssh:// / git:// / file:// 或 scp 形式 git@host:path。',
|
|
88
|
+
en: 'Not a valid git remote URL: {url}. Supported: https:// / ssh:// / git:// / file:// or scp-form git@host:path.',
|
|
89
|
+
},
|
|
90
|
+
'cli.install.remote_fetch_failed': {
|
|
91
|
+
zh: '从远端 {url} 拉取 ref {ref} 失败。请检查 URL、ref 是否存在、以及本机 git 凭证(SSH key / credential helper)。',
|
|
92
|
+
en: 'Failed to fetch ref {ref} from remote {url}. Check the URL, whether the ref exists, and your local git credentials (SSH key / credential helper).',
|
|
93
|
+
},
|
|
94
|
+
'cli.install.remote_ref_not_found': {
|
|
95
|
+
zh: '远端 {url} 上解析不到 ref {ref}。',
|
|
96
|
+
en: 'Could not resolve ref {ref} on remote {url}.',
|
|
97
|
+
},
|
|
98
|
+
'cli.install.remote_skill_not_found': {
|
|
99
|
+
zh: '在远端 {url} 的 ref {ref} 下找不到 skill {name}(既无 {name}/SKILL.md 也无 {name}.md)。',
|
|
100
|
+
en: 'Skill {name} not found at ref {ref} of remote {url} (neither {name}/SKILL.md nor {name}.md).',
|
|
101
|
+
},
|
|
102
|
+
'cli.install.git_ref_needs_url': {
|
|
103
|
+
zh: '--git-ref 需要配 --git-url 一起用(远端 git 源 = URL + ref + 仓库内路径)。',
|
|
104
|
+
en: '--git-ref requires --git-url (a remote git source = URL + ref + in-repo path).',
|
|
105
|
+
},
|
|
86
106
|
'cli.install.next_hint': {
|
|
87
107
|
zh: '现在可以在 coding agent 中说「用 omk 评测这个 skill」。',
|
|
88
108
|
en: 'You can now ask your coding agent: "use omk to evaluate this skill".',
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { CliMessage } from './types.js';
|
|
2
|
+
export type ListMessageKey = 'cli.list.header' | 'cli.list.empty' | 'cli.list.empty_hint' | 'cli.list.col_name' | 'cli.list.col_kind' | 'cli.list.col_state' | 'cli.list.col_verdict' | 'cli.list.col_evidence' | 'cli.list.col_source' | 'cli.list.drift_note' | 'cli.list.unreachable_note' | 'cli.list.legend';
|
|
3
|
+
export declare const listDict: Record<ListMessageKey, CliMessage>;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export const listDict = {
|
|
2
|
+
'cli.list.header': {
|
|
3
|
+
zh: '受管 skill({scope},{count} 条)\n',
|
|
4
|
+
en: 'Managed skills ({scope}, {count})\n',
|
|
5
|
+
},
|
|
6
|
+
'cli.list.empty': {
|
|
7
|
+
zh: '没有受管记录。\n',
|
|
8
|
+
en: 'No managed records.\n',
|
|
9
|
+
},
|
|
10
|
+
'cli.list.empty_hint': {
|
|
11
|
+
zh: '用 omk install <skill> 登记并分发一个 skill 后,它会出现在这里。\n',
|
|
12
|
+
en: 'Run omk install <skill> to register and distribute a skill; it will show up here.\n',
|
|
13
|
+
},
|
|
14
|
+
'cli.list.col_name': { zh: '名称', en: 'NAME' },
|
|
15
|
+
'cli.list.col_kind': { zh: '类型', en: 'KIND' },
|
|
16
|
+
'cli.list.col_state': { zh: '状态', en: 'STATE' },
|
|
17
|
+
'cli.list.col_verdict': { zh: 'VERDICT', en: 'VERDICT' },
|
|
18
|
+
'cli.list.col_evidence': { zh: '证据', en: 'EVIDENCE' },
|
|
19
|
+
'cli.list.col_source': { zh: '源', en: 'SOURCE' },
|
|
20
|
+
'cli.list.drift_note': {
|
|
21
|
+
zh: '⚠️ = 源内容已漂移、脱离证据(stale);重跑 omk eval 重新取证。\n',
|
|
22
|
+
en: '⚠️ = source content drifted off its evidence (stale); re-run omk eval to re-measure.\n',
|
|
23
|
+
},
|
|
24
|
+
'cli.list.unreachable_note': {
|
|
25
|
+
zh: '? = 源此处不可达 / 拒读,drift 未核(非 stale);本地 git 源请在原仓库根目录跑 omk list。\n',
|
|
26
|
+
en: '? = source unreachable / refused here, drift unchecked (not stale); for a local git source run omk list from the original repo root.\n',
|
|
27
|
+
},
|
|
28
|
+
'cli.list.legend': {
|
|
29
|
+
zh: '证据列 = 当前有效 / 全部(历史含旧内容证据,供回滚)。\n',
|
|
30
|
+
en: 'EVIDENCE column = current / total (history keeps old-content evidence for rollback).\n',
|
|
31
|
+
},
|
|
32
|
+
};
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CliMessage } from './types.js';
|
|
2
|
-
export type RunMessageKey = 'cli.progress.preflight_starting' | 'cli.progress.sample_retry' | 'cli.progress.sample_error' | 'cli.progress.sample_executing' | 'cli.progress.sample_exec_done' | 'cli.progress.output_preview' | 'cli.progress.judging' | 'cli.progress.judged' | 'cli.progress.skipped' | 'cli.progress.sample_done' | 'cli.progress.sample_failed_done' | 'cli.run.invalid_repeat' | 'cli.run.invalid_judge_repeat' | 'cli.run.no_debias_length_active' | 'cli.run.invalid_bootstrap_samples' | 'cli.run.bootstrap_samples_too_large' | 'cli.run.dry_run_no_scores' | 'cli.run.skill_section' | 'cli.run.run_section' | 'cli.run.batch_complete' | 'cli.run.batch_verdict_header' | 'cli.run.batch_child_report_missing' | 'cli.run.eval_complete' | 'cli.run.tally' | 'cli.run.report_saved' | 'cli.run.report_only_gate_skipped' | 'cli.run.report_server_running' | 'cli.run.report_server_view' | 'cli.run.report_server_stop' | 'cli.run.no_serve_in_non_tty' | 'cli.run.no_serve_view_hint' | 'cli.run.gold_load_failed' | 'cli.run.gold_load_issue' | 'cli.run.contamination_warning' | 'cli.run.skip_connectivity_warning';
|
|
2
|
+
export type RunMessageKey = 'cli.progress.preflight_starting' | 'cli.progress.sample_retry' | 'cli.progress.sample_error' | 'cli.progress.sample_executing' | 'cli.progress.sample_exec_done' | 'cli.progress.output_preview' | 'cli.progress.judging' | 'cli.progress.judged' | 'cli.progress.skipped' | 'cli.progress.sample_done' | 'cli.progress.sample_failed_done' | 'cli.run.invalid_repeat' | 'cli.run.invalid_judge_repeat' | 'cli.run.no_debias_length_active' | 'cli.run.invalid_bootstrap_samples' | 'cli.run.bootstrap_samples_too_large' | 'cli.run.dry_run_no_scores' | 'cli.run.skill_section' | 'cli.run.run_section' | 'cli.run.batch_complete' | 'cli.run.batch_verdict_header' | 'cli.run.batch_child_report_missing' | 'cli.run.eval_complete' | 'cli.run.tally' | 'cli.run.report_saved' | 'cli.run.evidence_recorded' | 'cli.run.evidence_recorded_unbound' | 'cli.run.report_only_gate_skipped' | 'cli.run.report_server_running' | 'cli.run.report_server_view' | 'cli.run.report_server_stop' | 'cli.run.no_serve_in_non_tty' | 'cli.run.no_serve_view_hint' | 'cli.run.gold_load_failed' | 'cli.run.gold_load_issue' | 'cli.run.contamination_warning' | 'cli.run.skip_connectivity_warning';
|
|
3
3
|
export declare const runDict: Record<RunMessageKey, CliMessage>;
|
|
@@ -99,6 +99,14 @@ export const runDict = {
|
|
|
99
99
|
zh: '📄 报告已保存到: {path}\n',
|
|
100
100
|
en: '📄 Report saved to: {path}\n',
|
|
101
101
|
},
|
|
102
|
+
'cli.run.evidence_recorded': {
|
|
103
|
+
zh: '🔖 已为受管 skill「{name}」记录评测证据 → measurable\n',
|
|
104
|
+
en: '🔖 Recorded eval evidence for managed skill "{name}" → measurable\n',
|
|
105
|
+
},
|
|
106
|
+
'cli.run.evidence_recorded_unbound': {
|
|
107
|
+
zh: '🔖 受管 skill「{name}」:评测内容与当前安装版本指纹不一致,证据已留存但不绑当前版本\n',
|
|
108
|
+
en: '🔖 Managed skill "{name}": eval content differs from the installed version; evidence kept but not bound to current\n',
|
|
109
|
+
},
|
|
102
110
|
'cli.run.report_only_gate_skipped': {
|
|
103
111
|
zh: 'ℹ 已启用 report-only 模式:保留 verdict 输出,但本次不使用 verdict 改写 exit code。\n',
|
|
104
112
|
en: 'ℹ Report-only mode enabled: verdict is still printed, but it will not affect the exit code.\n',
|
|
@@ -56,8 +56,9 @@ import { type GenMessageKey } from './i18n-dict/gen.js';
|
|
|
56
56
|
import { type HelpMessageKey } from './i18n-dict/help.js';
|
|
57
57
|
import { type InitMessageKey } from './i18n-dict/init.js';
|
|
58
58
|
import { type InstallMessageKey } from './i18n-dict/install.js';
|
|
59
|
+
import { type ListMessageKey } from './i18n-dict/list.js';
|
|
59
60
|
import { type RunMessageKey } from './i18n-dict/run.js';
|
|
60
61
|
import type { CliMessage } from './i18n-dict/types.js';
|
|
61
62
|
export type { CliMessage } from './i18n-dict/types.js';
|
|
62
|
-
export type CliMessageKey = CommonMessageKey | EvolveMessageKey | GenMessageKey | HelpMessageKey | InitMessageKey | InstallMessageKey | RunMessageKey;
|
|
63
|
+
export type CliMessageKey = CommonMessageKey | EvolveMessageKey | GenMessageKey | HelpMessageKey | InitMessageKey | InstallMessageKey | ListMessageKey | RunMessageKey;
|
|
63
64
|
export declare const CLI_DICT: Record<CliMessageKey, CliMessage>;
|
|
@@ -56,6 +56,7 @@ import { genDict } from './i18n-dict/gen.js';
|
|
|
56
56
|
import { helpDict } from './i18n-dict/help.js';
|
|
57
57
|
import { initDict } from './i18n-dict/init.js';
|
|
58
58
|
import { installDict } from './i18n-dict/install.js';
|
|
59
|
+
import { listDict } from './i18n-dict/list.js';
|
|
59
60
|
import { runDict } from './i18n-dict/run.js';
|
|
60
61
|
export const CLI_DICT = {
|
|
61
62
|
...commonDict,
|
|
@@ -64,5 +65,6 @@ export const CLI_DICT = {
|
|
|
64
65
|
...helpDict,
|
|
65
66
|
...initDict,
|
|
66
67
|
...installDict,
|
|
68
|
+
...listDict,
|
|
67
69
|
...runDict,
|
|
68
70
|
};
|
|
@@ -24,6 +24,13 @@ import { configVariantsToSpecs } from '../../../inputs/eval-config.js';
|
|
|
24
24
|
* 引导用户改用 --control-cwd / --treatment-cwd 或 eval.yaml 的 variant.cwd。git 修订语法 `@{...}`
|
|
25
25
|
* 由 parseVariantCwd 保护、不误判。cwd 由调用方在边界注入(见 eval-runner)。 */
|
|
26
26
|
function cliVariantSpec(rawExpr, role, cwd) {
|
|
27
|
+
// 远端 git 不走 CLI 字符串(URL 的 `:`/`@` 会被 parseGitInput / parseVariantCwd 误切)——
|
|
28
|
+
// 先于 @cwd guard 拦协议 URL 与 scp 形式 `git@host:path`,给明确的「走 eval.yaml」指引,
|
|
29
|
+
// 而非让 scp 形式落到下面 @cwd 的迁移报错(误导)。绝对本地路径 `/abs/skill.md` 不在此列。
|
|
30
|
+
if (/^(https?|ssh|git):\/\//i.test(rawExpr) || /^[\w.-]+@[\w.-]+:/.test(rawExpr)) {
|
|
31
|
+
throw new Error(`远端 git 源不支持经 --${role} 字符串传入: "${rawExpr}"。请在 eval.yaml 的 variant 上用结构化 `
|
|
32
|
+
+ `git: { url, ref, spec } 字段(避开 URL 里的 : / @ 被误切),CLI 仅支持本地路径与 git:<ref>:<spec>。`);
|
|
33
|
+
}
|
|
27
34
|
// 探测旧 name@cwd 形态报错。注:含合法 `@`(非 `@{`)的路径(如 /x/@dir/skill.md)会被一并
|
|
28
35
|
// 误判 —— 属 pre-existing 限制(这类路径本就不被 @cwd 支持),报错文案仍指向迁移指引。
|
|
29
36
|
if (parseVariantCwd(rawExpr).cwd !== undefined) {
|
package/dist/cli/lib/shared.js
CHANGED
|
@@ -5,7 +5,7 @@ export function requireEvaluationReport(report, id, lang) {
|
|
|
5
5
|
console.error(tCli('cli.common.report_not_found', lang, { id }));
|
|
6
6
|
throw new CliExit(1);
|
|
7
7
|
}
|
|
8
|
-
if (report.
|
|
8
|
+
if (report.kind === 'batch-evaluation') {
|
|
9
9
|
console.error(lang === 'zh'
|
|
10
10
|
? `报告 ${id} 是 BatchEvaluationReport。该命令需要单次 EvaluationReport;请使用其中的 child reportId。`
|
|
11
11
|
: `Report ${id} is a BatchEvaluationReport. This command requires an EvaluationReport; use a child reportId from the batch.`);
|
package/dist/doctor/index.js
CHANGED
|
@@ -26,10 +26,10 @@ export function resolveDoctorTargets(target, cwd) {
|
|
|
26
26
|
const skillsDir = join(cwd, 'skills');
|
|
27
27
|
if (existsSync(skillsDir) && statSync(skillsDir).isDirectory()) {
|
|
28
28
|
const variants = discoverVariants(skillsDir).filter((v) => v !== 'baseline');
|
|
29
|
-
return variants.length > 0 ? resolveArtifacts(skillsDir, variants, { strictBaseline: false }) : [];
|
|
29
|
+
return variants.length > 0 ? resolveArtifacts(skillsDir, variants, { strictBaseline: false, materialize: false }) : [];
|
|
30
30
|
}
|
|
31
31
|
const variants = discoverVariants(cwd).filter((v) => v !== 'baseline');
|
|
32
|
-
return variants.length > 0 ? resolveArtifacts(cwd, variants, { strictBaseline: false }) : [];
|
|
32
|
+
return variants.length > 0 ? resolveArtifacts(cwd, variants, { strictBaseline: false, materialize: false }) : [];
|
|
33
33
|
}
|
|
34
34
|
const absTarget = resolve(target);
|
|
35
35
|
if (!existsSync(absTarget)) {
|
|
@@ -38,7 +38,7 @@ export function resolveDoctorTargets(target, cwd) {
|
|
|
38
38
|
const stat = statSync(absTarget);
|
|
39
39
|
if (stat.isFile() && absTarget.endsWith('.md')) {
|
|
40
40
|
// 单文件:用 file-path 模式(走 resolveArtifacts 的 "包含 /" 分支)
|
|
41
|
-
return resolveArtifacts(dirname(absTarget), [absTarget], { strictBaseline: false });
|
|
41
|
+
return resolveArtifacts(dirname(absTarget), [absTarget], { strictBaseline: false, materialize: false });
|
|
42
42
|
}
|
|
43
43
|
if (stat.isDirectory()) {
|
|
44
44
|
// 目录自身就是 directory-skill (含 SKILL.md): 按单个 skill 解析,
|
|
@@ -48,10 +48,10 @@ export function resolveDoctorTargets(target, cwd) {
|
|
|
48
48
|
// skillRoot 丢失,assets/foo.md 这类相对依赖会锚到 doctor 的 cwd 而不是
|
|
49
49
|
// skill 目录, 误报缺文件。
|
|
50
50
|
if (existsSync(join(absTarget, 'SKILL.md'))) {
|
|
51
|
-
return resolveArtifacts(dirname(absTarget), [basename(absTarget)], { strictBaseline: false });
|
|
51
|
+
return resolveArtifacts(dirname(absTarget), [basename(absTarget)], { strictBaseline: false, materialize: false });
|
|
52
52
|
}
|
|
53
53
|
const variants = discoverVariants(absTarget).filter((v) => v !== 'baseline');
|
|
54
|
-
return variants.length > 0 ? resolveArtifacts(absTarget, variants, { strictBaseline: false }) : [];
|
|
54
|
+
return variants.length > 0 ? resolveArtifacts(absTarget, variants, { strictBaseline: false, materialize: false }) : [];
|
|
55
55
|
}
|
|
56
56
|
throw new Error(`doctor target must be a .md file or directory: ${target}`);
|
|
57
57
|
}
|
|
@@ -229,7 +229,7 @@ export async function runDoctor(opts) {
|
|
|
229
229
|
? 'warnings_only'
|
|
230
230
|
: 'passed';
|
|
231
231
|
return {
|
|
232
|
-
|
|
232
|
+
kind: 'doctor',
|
|
233
233
|
schemaVersion: DOCTOR_REPORT_SCHEMA_VERSION,
|
|
234
234
|
id: nextReportId(),
|
|
235
235
|
timestamp: new Date().toISOString(),
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* Executor result cache.
|
|
3
3
|
*
|
|
4
4
|
* Caches successful executor results to disk to avoid redundant API calls.
|
|
5
|
-
* Cache key
|
|
6
|
-
* runtime + mocks + mocksStrict + effort).
|
|
5
|
+
* Cache key v6 = sha256(model + system + prompt + cwd + allowedSkills + executor +
|
|
6
|
+
* runtime + mocks + mocksStrict + effort + artifactContentHash).
|
|
7
7
|
* Loaded into memory on init, flushed to disk on save().
|
|
8
8
|
*
|
|
9
9
|
* Prefix bumps intentionally invalidate old entries when construct-validity
|
|
@@ -15,6 +15,11 @@
|
|
|
15
15
|
* 报告 meta 标的 effort 跟实际跑的 effort 不一致 — 测量可比性污染);
|
|
16
16
|
* 同时 cache.set 不再砍 turns / toolCalls — 工具类断言 + diagnostic 要看 trace,
|
|
17
17
|
* 砍掉的话 cached rerun 会让工具断言为空、diagnostic 没真实证据,跟 cold run 不一致
|
|
18
|
+
* - v6: artifact 内容指纹(contentHash)。`system` 只含 SKILL.md 正文,但本地 dir-skill 的
|
|
19
|
+
* references/ 资产是真实运行时输入(cwd=skillRoot,agent 可读),改资产只动 contentHash、
|
|
20
|
+
* 不动 system → 旧 key 会命中旧输出、贴到新 artifactHashes 上,形成静默测量污染。把
|
|
21
|
+
* contentHash 纳入 key,资产变即重跑。git skill 的 contentHash 只随 SKILL.md 变(其资产不
|
|
22
|
+
* 暴露给 executor、本就不该触发重跑),口径自洽
|
|
18
23
|
*/
|
|
19
24
|
import type { ExecutorCache } from '../types/index.js';
|
|
20
25
|
export declare function createCache(cacheDir: string): ExecutorCache;
|
|
@@ -27,4 +32,7 @@ mocksStrict?: boolean,
|
|
|
27
32
|
/** Executor effort 也进 key:effort 'low'/'medium'/'high' 改变 LLM 思考预算,
|
|
28
33
|
* 输出/工具调用/分数都可能不同,跨 effort 共享 cache 会让报告 meta 标的 effort
|
|
29
34
|
* 跟实际生成的 effort 不一致,违反"两份报告比分数前先比 cliVersion / effort"语义。 */
|
|
30
|
-
effort?: string
|
|
35
|
+
effort?: string,
|
|
36
|
+
/** artifact 内容指纹(整树 / 单文件哈)。本地 dir-skill 改 references/ 资产只动此值、不动 system,
|
|
37
|
+
* 不进 key 会让改资产后命中旧输出 → 静默污染。空(baseline / 无 skill)等价无指纹。 */
|
|
38
|
+
artifactContentHash?: string): string;
|
package/dist/eval-core/cache.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* Executor result cache.
|
|
3
3
|
*
|
|
4
4
|
* Caches successful executor results to disk to avoid redundant API calls.
|
|
5
|
-
* Cache key
|
|
6
|
-
* runtime + mocks + mocksStrict + effort).
|
|
5
|
+
* Cache key v6 = sha256(model + system + prompt + cwd + allowedSkills + executor +
|
|
6
|
+
* runtime + mocks + mocksStrict + effort + artifactContentHash).
|
|
7
7
|
* Loaded into memory on init, flushed to disk on save().
|
|
8
8
|
*
|
|
9
9
|
* Prefix bumps intentionally invalidate old entries when construct-validity
|
|
@@ -15,6 +15,11 @@
|
|
|
15
15
|
* 报告 meta 标的 effort 跟实际跑的 effort 不一致 — 测量可比性污染);
|
|
16
16
|
* 同时 cache.set 不再砍 turns / toolCalls — 工具类断言 + diagnostic 要看 trace,
|
|
17
17
|
* 砍掉的话 cached rerun 会让工具断言为空、diagnostic 没真实证据,跟 cold run 不一致
|
|
18
|
+
* - v6: artifact 内容指纹(contentHash)。`system` 只含 SKILL.md 正文,但本地 dir-skill 的
|
|
19
|
+
* references/ 资产是真实运行时输入(cwd=skillRoot,agent 可读),改资产只动 contentHash、
|
|
20
|
+
* 不动 system → 旧 key 会命中旧输出、贴到新 artifactHashes 上,形成静默测量污染。把
|
|
21
|
+
* contentHash 纳入 key,资产变即重跑。git skill 的 contentHash 只随 SKILL.md 变(其资产不
|
|
22
|
+
* 暴露给 executor、本就不该触发重跑),口径自洽
|
|
18
23
|
*/
|
|
19
24
|
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
|
|
20
25
|
import { join } from 'node:path';
|
|
@@ -114,7 +119,10 @@ mocksStrict,
|
|
|
114
119
|
/** Executor effort 也进 key:effort 'low'/'medium'/'high' 改变 LLM 思考预算,
|
|
115
120
|
* 输出/工具调用/分数都可能不同,跨 effort 共享 cache 会让报告 meta 标的 effort
|
|
116
121
|
* 跟实际生成的 effort 不一致,违反"两份报告比分数前先比 cliVersion / effort"语义。 */
|
|
117
|
-
effort
|
|
122
|
+
effort,
|
|
123
|
+
/** artifact 内容指纹(整树 / 单文件哈)。本地 dir-skill 改 references/ 资产只动此值、不动 system,
|
|
124
|
+
* 不进 key 会让改资产后命中旧输出 → 静默污染。空(baseline / 无 skill)等价无指纹。 */
|
|
125
|
+
artifactContentHash) {
|
|
118
126
|
// allowedSkills 序列化:undefined → "" / [] → "[]" / [...] → 排序后 JSON。
|
|
119
127
|
// 排序保证 ["a","b"] 和 ["b","a"] 命中同一缓存(语义等价)。
|
|
120
128
|
const isoStr = allowedSkills === undefined
|
|
@@ -130,8 +138,8 @@ effort) {
|
|
|
130
138
|
// executor + runtime + effort 进 cache key:同 model 名走不同 executor 或同 executor
|
|
131
139
|
// 换 binary/SDK 版本时输出可能不同,旧 cache 不可复用。
|
|
132
140
|
const hash = createHash('sha256')
|
|
133
|
-
.update(`${model || ''}\n${system || ''}\n${prompt || ''}\n${cwd || ''}\n${isoStr}\n${executor || ''}\n${runtimeFingerprint || ''}\n${mockStr}\n${strictStr}\n${effortStr}`)
|
|
141
|
+
.update(`${model || ''}\n${system || ''}\n${prompt || ''}\n${cwd || ''}\n${isoStr}\n${executor || ''}\n${runtimeFingerprint || ''}\n${mockStr}\n${strictStr}\n${effortStr}\n${artifactContentHash || ''}`)
|
|
134
142
|
.digest('hex')
|
|
135
143
|
.slice(0, 16);
|
|
136
|
-
return `
|
|
144
|
+
return `v6:${hash}`;
|
|
137
145
|
}
|