@shahimat/harness 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/dist/checks/context-index.js +34 -18
- package/dist/cli.js +7 -0
- package/dist/commands/expect.d.ts +3 -0
- package/dist/commands/expect.js +303 -0
- package/dist/commands/types.d.ts +2 -0
- package/dist/expect/batch.d.ts +44 -0
- package/dist/expect/batch.js +157 -0
- package/dist/expect/evidence.d.ts +5 -0
- package/dist/expect/evidence.js +21 -0
- package/dist/expect/index.d.ts +4 -0
- package/dist/expect/index.js +4 -0
- package/dist/expect/judge.d.ts +24 -0
- package/dist/expect/judge.js +50 -0
- package/dist/expect/levels.d.ts +25 -0
- package/dist/expect/levels.js +43 -0
- package/dist/expect/lint.d.ts +15 -0
- package/dist/expect/lint.js +38 -0
- package/dist/expect/repo.d.ts +60 -0
- package/dist/expect/repo.js +139 -0
- package/dist/expect/schema.d.ts +13 -0
- package/dist/expect/schema.js +47 -0
- package/dist/expect/verdicts.d.ts +35 -0
- package/dist/expect/verdicts.js +23 -0
- package/dist/prompt-index.d.ts +2 -0
- package/dist/prompt-index.js +57 -2
- package/package.json +14 -1
package/README.md
CHANGED
|
@@ -25,6 +25,16 @@ npx -y @shahimat/harness@<version> <команда> --root .
|
|
|
25
25
|
- `harness adapters sync` — файлы под Claude Code и Codex из
|
|
26
26
|
`AGENTS.md` и `.agents/`: `CLAUDE.md`, копии skills, субагенты,
|
|
27
27
|
выключенная memory агента.
|
|
28
|
+
- `harness expect init` — каркас ожиданий в проекте: конфиг vitest,
|
|
29
|
+
пример файла с бюджетом, npm-скрипты, `.gitignore`.
|
|
30
|
+
- `harness expect report | lint | judge [--apply]` — ожидания в форме
|
|
31
|
+
vitest (`expectations/*.expect.ts`): дерево со статусами, проверка
|
|
32
|
+
заголовков, пакетная смысловая проверка судьёй-агентом с кэшем
|
|
33
|
+
вердиктов. Помощники для проверок — `@shahimat/harness/expect`
|
|
34
|
+
(нужен vitest в проекте): `crit` / `warn` / `note` / `budget`,
|
|
35
|
+
`expectAtMost` (info при > 90% порога), зонды по репозиторию,
|
|
36
|
+
`judged`, проверка JSON Schema (`expectSchema`, `dataFiles`,
|
|
37
|
+
`frontmatters`).
|
|
28
38
|
- `harness index build | check --raw-base <url>` — индекс
|
|
29
39
|
`prompts/index.yml` репозитория промптов.
|
|
30
40
|
|
|
@@ -34,7 +44,7 @@ npx -y @shahimat/harness@<version> <команда> --root .
|
|
|
34
44
|
## `context/harness.yml`
|
|
35
45
|
|
|
36
46
|
```yaml
|
|
37
|
-
version: 0.
|
|
47
|
+
version: 0.3.0
|
|
38
48
|
adapters: [claude-code, codex]
|
|
39
49
|
checks:
|
|
40
50
|
changelog:
|
|
@@ -229,36 +229,55 @@ class Collector {
|
|
|
229
229
|
this.report(manifest, `index_layers: dir «${String(dir)}» — нужен путь внутри проекта`);
|
|
230
230
|
continue;
|
|
231
231
|
}
|
|
232
|
+
const format = (isRecord(item) ? item['format'] : undefined) ?? 'md';
|
|
233
|
+
if (format !== 'md' && format !== 'yml') {
|
|
234
|
+
this.report(manifest, `index_layers: format «${String(format)}» — md или yml`);
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
232
237
|
const taken = layers.find((l) => l.section === section);
|
|
233
238
|
if (taken !== undefined) {
|
|
234
239
|
this.report(manifest, `index_layers: section «${section}» уже объявлена в ${taken.manifest}`);
|
|
235
240
|
continue;
|
|
236
241
|
}
|
|
237
|
-
layers.push({ section, dir: dir.replace(/\/+$/, ''), manifest });
|
|
242
|
+
layers.push({ section, dir: dir.replace(/\/+$/, ''), format, manifest });
|
|
238
243
|
}
|
|
239
244
|
}
|
|
240
245
|
return layers;
|
|
241
246
|
}
|
|
242
247
|
/**
|
|
243
|
-
* Файл слоя
|
|
244
|
-
*
|
|
248
|
+
* Файл слоя модуля: `md` — frontmatter `id`, `namespace`, строка индекса —
|
|
249
|
+
* первый `# …`; `yml` — поля `id`, `namespace`, `summary`. `id` = путь
|
|
250
|
+
* внутри слоя без расширения, `/` → `-`.
|
|
245
251
|
*/
|
|
246
252
|
async collectLayer(layer) {
|
|
247
|
-
|
|
253
|
+
const ext = layer.format === 'md' ? '.md' : '.yml';
|
|
254
|
+
for (const file of await listFiles(this.root, layer.dir, ext)) {
|
|
248
255
|
let data;
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
data =
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
this.
|
|
255
|
-
continue;
|
|
256
|
+
let title;
|
|
257
|
+
if (layer.format === 'yml') {
|
|
258
|
+
data = await this.readYaml(file);
|
|
259
|
+
if (data === null)
|
|
260
|
+
continue;
|
|
261
|
+
title = this.oneLine(file, data, 'summary');
|
|
256
262
|
}
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
263
|
+
else {
|
|
264
|
+
const text = await readFile(path.join(this.root, file), 'utf8');
|
|
265
|
+
try {
|
|
266
|
+
data = parseFrontmatter(text);
|
|
267
|
+
}
|
|
268
|
+
catch (error) {
|
|
269
|
+
this.report(file, `невалидный frontmatter: ${error.message.split('\n')[0]}`);
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
if (data === null) {
|
|
273
|
+
this.report(file, 'нет frontmatter (id, namespace)');
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
title = firstHeading(text);
|
|
277
|
+
if (title === null)
|
|
278
|
+
this.report(file, 'нет заголовка «# …» (одна строка для индекса)');
|
|
260
279
|
}
|
|
261
|
-
const expected = file.slice(layer.dir.length + 1, -
|
|
280
|
+
const expected = file.slice(layer.dir.length + 1, -ext.length).split('/').join('-');
|
|
262
281
|
const id = data['id'];
|
|
263
282
|
if (typeof id !== 'string' || id === '') {
|
|
264
283
|
this.report(file, 'нет поля id');
|
|
@@ -267,9 +286,6 @@ class Collector {
|
|
|
267
286
|
if (id !== expected)
|
|
268
287
|
this.report(file, `id «${id}» не совпадает с путём «${expected}»`);
|
|
269
288
|
const ns = this.checkNamespace(file, data);
|
|
270
|
-
const title = firstHeading(text);
|
|
271
|
-
if (title === null)
|
|
272
|
-
this.report(file, 'нет заголовка «# …» (одна строка для индекса)');
|
|
273
289
|
if (ns === null)
|
|
274
290
|
continue;
|
|
275
291
|
const sections = this.namespace(ns).layers;
|
package/dist/cli.js
CHANGED
|
@@ -4,6 +4,7 @@ import { runAdapters } from './commands/adapters.js';
|
|
|
4
4
|
import { runChangelog } from './commands/changelog.js';
|
|
5
5
|
import { runCheck } from './commands/check.js';
|
|
6
6
|
import { runContext } from './commands/context.js';
|
|
7
|
+
import { runExpect } from './commands/expect.js';
|
|
7
8
|
import { runIndex } from './commands/prompt-index.js';
|
|
8
9
|
import { UsageError } from './diagnostics.js';
|
|
9
10
|
const USAGE = `Использование:
|
|
@@ -11,6 +12,7 @@ const USAGE = `Использование:
|
|
|
11
12
|
harness adapters sync [--root <dir>]
|
|
12
13
|
harness changelog lint [--root <dir>]
|
|
13
14
|
harness context index [--root <dir>]
|
|
15
|
+
harness expect init | report | lint | judge [--apply] [--root <dir>]
|
|
14
16
|
harness index build --raw-base <url> [--root <dir>] [--out <file>]
|
|
15
17
|
harness index check --raw-base <url> [--root <dir>]
|
|
16
18
|
|
|
@@ -23,6 +25,7 @@ export async function main(argv, now = new Date()) {
|
|
|
23
25
|
root: { type: 'string', default: '.' },
|
|
24
26
|
out: { type: 'string' },
|
|
25
27
|
'raw-base': { type: 'string' },
|
|
28
|
+
apply: { type: 'boolean' },
|
|
26
29
|
help: { type: 'boolean', short: 'h' },
|
|
27
30
|
},
|
|
28
31
|
});
|
|
@@ -35,6 +38,8 @@ export async function main(argv, now = new Date()) {
|
|
|
35
38
|
args.out = values.out;
|
|
36
39
|
if (values['raw-base'] !== undefined)
|
|
37
40
|
args.rawBase = values['raw-base'];
|
|
41
|
+
if (values.apply === true)
|
|
42
|
+
args.apply = true;
|
|
38
43
|
const [group, command] = positionals;
|
|
39
44
|
switch (group) {
|
|
40
45
|
case 'adapters':
|
|
@@ -45,6 +50,8 @@ export async function main(argv, now = new Date()) {
|
|
|
45
50
|
return runChangelog(command, args);
|
|
46
51
|
case 'context':
|
|
47
52
|
return runContext(command, args);
|
|
53
|
+
case 'expect':
|
|
54
|
+
return runExpect(command, args);
|
|
48
55
|
case 'index':
|
|
49
56
|
return runIndex(command, args);
|
|
50
57
|
default:
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
import { spawnSync } from 'node:child_process';
|
|
2
|
+
import { existsSync, readFileSync, rmSync } from 'node:fs';
|
|
3
|
+
import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { createRequire } from 'node:module';
|
|
5
|
+
import { tmpdir } from 'node:os';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { UsageError } from '../diagnostics.js';
|
|
8
|
+
import { applyAnswers, applyLintAnswer, planBatches, renderBatch, renderLintBatch } from '../expect/batch.js';
|
|
9
|
+
import { readEvidence, shaOf } from '../expect/evidence.js';
|
|
10
|
+
import { COLLECT_ENV, idOf } from '../expect/judge.js';
|
|
11
|
+
import { BUDGET_TITLE } from '../expect/levels.js';
|
|
12
|
+
import { lintTitles } from '../expect/lint.js';
|
|
13
|
+
import { loadVerdicts, saveVerdicts, VERDICTS_PATH } from '../expect/verdicts.js';
|
|
14
|
+
export const EXPECT_CONFIG = 'expectations/vitest.config.ts';
|
|
15
|
+
const JUDGE_DIR = 'expectations/.judge';
|
|
16
|
+
/** Запускает vitest проекта на файлах ожиданий и возвращает результаты. */
|
|
17
|
+
function runVitest(root, env = {}) {
|
|
18
|
+
if (!existsSync(path.join(root, EXPECT_CONFIG)))
|
|
19
|
+
throw new UsageError(`нет ${EXPECT_CONFIG}`);
|
|
20
|
+
const require = createRequire(path.join(root, 'package.json'));
|
|
21
|
+
let vitestPkg;
|
|
22
|
+
try {
|
|
23
|
+
vitestPkg = require.resolve('vitest/package.json');
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
throw new UsageError('vitest не установлен в проекте: npm i -D vitest');
|
|
27
|
+
}
|
|
28
|
+
const bin = JSON.parse(readFileSync(vitestPkg, 'utf8')).bin['vitest'] ?? '';
|
|
29
|
+
const out = path.join(tmpdir(), `harness-expect-${process.pid}-${Date.now()}.json`);
|
|
30
|
+
spawnSync(process.execPath, [path.join(path.dirname(vitestPkg), bin), 'run', '--root', root, '--config', EXPECT_CONFIG, '--reporter=json', `--outputFile=${out}`], { cwd: root, env: { ...process.env, ...env, HARNESS_EXPECT_ROOT: root }, stdio: ['ignore', 'ignore', 'inherit'] });
|
|
31
|
+
if (!existsSync(out))
|
|
32
|
+
throw new Error('vitest не записал отчёт');
|
|
33
|
+
const data = JSON.parse(readFileSync(out, 'utf8'));
|
|
34
|
+
rmSync(out, { force: true });
|
|
35
|
+
return data.testResults.flatMap((file) => file.assertionResults.map((a) => {
|
|
36
|
+
const result = {
|
|
37
|
+
file: path.relative(root, file.name).split(path.sep).join('/'),
|
|
38
|
+
path: a.ancestorTitles,
|
|
39
|
+
title: a.title,
|
|
40
|
+
status: a.status,
|
|
41
|
+
message: (a.failureMessages[0] ?? '').split('\n')[0] ?? '',
|
|
42
|
+
// Обычный it() без обёртки — crit: формальная проверка по умолчанию.
|
|
43
|
+
level: a.meta?.level ?? 'crit',
|
|
44
|
+
notes: a.meta?.notes ?? [],
|
|
45
|
+
};
|
|
46
|
+
if (a.meta?.budget !== undefined)
|
|
47
|
+
result.budget = a.meta.budget;
|
|
48
|
+
return result;
|
|
49
|
+
}));
|
|
50
|
+
}
|
|
51
|
+
const fullName = (t) => [...t.path, t.title].join(' > ');
|
|
52
|
+
function byFile(results) {
|
|
53
|
+
const files = new Map();
|
|
54
|
+
for (const r of results)
|
|
55
|
+
files.set(r.file, [...(files.get(r.file) ?? []), r]);
|
|
56
|
+
return files;
|
|
57
|
+
}
|
|
58
|
+
const titlesSha = (tests) => shaOf(tests.map(fullName).sort().join('\n'));
|
|
59
|
+
const isPending = (t) => t.status === 'failed' && t.message.includes('нет свежего вердикта');
|
|
60
|
+
async function report(args) {
|
|
61
|
+
const all = runVitest(args.root);
|
|
62
|
+
const store = await loadVerdicts();
|
|
63
|
+
const titleIssues = lintTitles(all.filter((r) => r.title !== BUDGET_TITLE).map((r) => ({ file: r.file, path: r.path, title: r.title })));
|
|
64
|
+
let blocked = false;
|
|
65
|
+
let total = { tests: 0, crit: 0, warn: 0, info: 0, pending: 0 };
|
|
66
|
+
for (const [file, tests] of byFile(all)) {
|
|
67
|
+
const budget = tests.find((t) => t.title === BUDGET_TITLE)?.budget ?? { crit: 0, warn: 0 };
|
|
68
|
+
const items = tests.filter((t) => t.title !== BUDGET_TITLE);
|
|
69
|
+
const count = { crit: 0, warn: 0, info: 0, pending: 0 };
|
|
70
|
+
console.log(file);
|
|
71
|
+
let prev = [];
|
|
72
|
+
for (const t of items) {
|
|
73
|
+
t.path.forEach((p, i) => {
|
|
74
|
+
if (prev[i] !== p)
|
|
75
|
+
console.log(`${' '.repeat(i + 1)}${p}`);
|
|
76
|
+
});
|
|
77
|
+
prev = t.path;
|
|
78
|
+
const pad = ' '.repeat(t.path.length + 1);
|
|
79
|
+
if (isPending(t)) {
|
|
80
|
+
count.pending++;
|
|
81
|
+
console.log(`${pad}? ${t.title} — нужен вердикт судьи`);
|
|
82
|
+
}
|
|
83
|
+
else if (t.status === 'failed') {
|
|
84
|
+
count[t.level === 'warn' ? 'warn' : 'crit']++;
|
|
85
|
+
console.log(`${pad}${t.level === 'warn' ? '⚠' : '✗'} ${t.title} — ${t.message}`);
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
console.log(`${pad}${t.status === 'todo' ? '○' : t.status === 'passed' ? '✓' : '-'} ${t.title}`);
|
|
89
|
+
}
|
|
90
|
+
for (const n of t.notes)
|
|
91
|
+
console.log(`${pad} ℹ ${n}`);
|
|
92
|
+
count.info += t.notes.length;
|
|
93
|
+
}
|
|
94
|
+
for (const d of titleIssues.filter((i) => i.title.file === file)) {
|
|
95
|
+
count[d.level]++;
|
|
96
|
+
console.log(` ${d.level === 'crit' ? '✗' : '⚠'} заголовок «${d.title.title}» — ${d.message}`);
|
|
97
|
+
}
|
|
98
|
+
const lint = store.lint[file];
|
|
99
|
+
if (lint === undefined || lint.titles !== titlesSha(items)) {
|
|
100
|
+
count.pending++;
|
|
101
|
+
console.log(' ? смысловая проверка заголовков устарела — harness expect judge');
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
const name = new Map(items.map((t) => [idOf(fullName(t)), t.title]));
|
|
105
|
+
for (const c of lint.contradictions)
|
|
106
|
+
console.log(` ⚠ противоречие: «${name.get(c.a)}» ↔ «${name.get(c.b)}» — ${c.reason}`);
|
|
107
|
+
for (const u of lint.unmeasurable)
|
|
108
|
+
console.log(` ⚠ неизмеримо: «${name.get(u.id)}» — ${u.reason}`);
|
|
109
|
+
count.warn += lint.contradictions.length + lint.unmeasurable.length;
|
|
110
|
+
}
|
|
111
|
+
const over = count.crit > budget.crit || count.warn > budget.warn || count.pending > 0;
|
|
112
|
+
blocked ||= over;
|
|
113
|
+
console.log(` ${over ? '✗' : '✓'} crit ${count.crit}/${budget.crit} · warn ${count.warn}/${budget.warn} · info ${count.info}` +
|
|
114
|
+
(count.pending > 0 ? ` · ? ${count.pending} без вердикта` : ''));
|
|
115
|
+
total = {
|
|
116
|
+
tests: total.tests + items.length,
|
|
117
|
+
crit: total.crit + count.crit,
|
|
118
|
+
warn: total.warn + count.warn,
|
|
119
|
+
info: total.info + count.info,
|
|
120
|
+
pending: total.pending + count.pending,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
console.log(`\nожиданий ${total.tests}: crit ${total.crit} · warn ${total.warn} · info ${total.info} · ? ${total.pending} — ${blocked ? 'бюджет превышен' : 'в бюджете'}`);
|
|
124
|
+
return blocked ? 1 : 0;
|
|
125
|
+
}
|
|
126
|
+
async function lint(args) {
|
|
127
|
+
const titles = runVitest(args.root)
|
|
128
|
+
.filter((r) => r.title !== BUDGET_TITLE)
|
|
129
|
+
.map((r) => ({ file: r.file, path: r.path, title: r.title }));
|
|
130
|
+
const diags = lintTitles(titles);
|
|
131
|
+
for (const d of diags)
|
|
132
|
+
console.error(`${d.level} ${d.title.file}: «${d.title.title}» — ${d.message}`);
|
|
133
|
+
console.log(diags.length === 0 ? `expect lint: ok (${titles.length})` : `expect lint: нарушений ${diags.length}`);
|
|
134
|
+
return diags.length === 0 ? 0 : 1;
|
|
135
|
+
}
|
|
136
|
+
async function judgePrepare(args) {
|
|
137
|
+
const dir = path.join(args.root, JUDGE_DIR);
|
|
138
|
+
await rm(dir, { recursive: true, force: true });
|
|
139
|
+
const pendingDir = path.join(dir, 'pending');
|
|
140
|
+
const results = runVitest(args.root, { [COLLECT_ENV]: pendingDir });
|
|
141
|
+
const items = [];
|
|
142
|
+
if (existsSync(pendingDir)) {
|
|
143
|
+
for (const f of (await readdir(pendingDir)).sort())
|
|
144
|
+
items.push(JSON.parse(await readFile(path.join(pendingDir, f), 'utf8')));
|
|
145
|
+
await rm(pendingDir, { recursive: true, force: true });
|
|
146
|
+
}
|
|
147
|
+
const plan = { batches: [], lint: [] };
|
|
148
|
+
for (const [i, batch] of planBatches(items).entries()) {
|
|
149
|
+
const task = `${JUDGE_DIR}/batch-${i + 1}.md`;
|
|
150
|
+
const answer = `${JUDGE_DIR}/batch-${i + 1}.answer.json`;
|
|
151
|
+
await mkdir(dir, { recursive: true });
|
|
152
|
+
await writeFile(path.join(args.root, task), renderBatch(batch, answer), 'utf8');
|
|
153
|
+
plan.batches.push({ task, answer, items: batch.items });
|
|
154
|
+
}
|
|
155
|
+
const store = await loadVerdicts();
|
|
156
|
+
for (const [file, all] of byFile(results)) {
|
|
157
|
+
const tests = all.filter((t) => t.title !== BUDGET_TITLE);
|
|
158
|
+
const sha = titlesSha(tests);
|
|
159
|
+
if (store.lint[file]?.titles === sha)
|
|
160
|
+
continue;
|
|
161
|
+
const n = plan.lint.length + 1;
|
|
162
|
+
const task = `${JUDGE_DIR}/lint-${n}.md`;
|
|
163
|
+
const answer = `${JUDGE_DIR}/lint-${n}.answer.json`;
|
|
164
|
+
const titles = tests.map((t) => ({ id: idOf(fullName(t)), title: fullName(t) }));
|
|
165
|
+
await mkdir(dir, { recursive: true });
|
|
166
|
+
await writeFile(path.join(args.root, task), renderLintBatch(file, titles, answer), 'utf8');
|
|
167
|
+
plan.lint.push({ task, answer, file, titles: sha, ids: titles.map((t) => t.id) });
|
|
168
|
+
}
|
|
169
|
+
if (plan.batches.length === 0 && plan.lint.length === 0) {
|
|
170
|
+
console.log('expect judge: всё свежее, судья не нужен');
|
|
171
|
+
return 0;
|
|
172
|
+
}
|
|
173
|
+
await writeFile(path.join(dir, 'plan.json'), JSON.stringify(plan, null, 2), 'utf8');
|
|
174
|
+
console.log(`expect judge: вопросов ${items.length} в пакетах ${plan.batches.length}, проверок заголовков ${plan.lint.length}`);
|
|
175
|
+
for (const t of [...plan.batches, ...plan.lint])
|
|
176
|
+
console.log(` ${t.task} → ${t.answer}`);
|
|
177
|
+
console.log('Каждый файл — задание судье (субагенту), ответ — в указанный JSON. Затем: harness expect judge --apply');
|
|
178
|
+
return 1;
|
|
179
|
+
}
|
|
180
|
+
async function judgeApply(args) {
|
|
181
|
+
const planFile = path.join(args.root, JUDGE_DIR, 'plan.json');
|
|
182
|
+
if (!existsSync(planFile))
|
|
183
|
+
throw new UsageError('нет плана: сначала harness expect judge');
|
|
184
|
+
const plan = JSON.parse(await readFile(planFile, 'utf8'));
|
|
185
|
+
const store = await loadVerdicts();
|
|
186
|
+
const rejected = [];
|
|
187
|
+
const readJson = async (rel) => {
|
|
188
|
+
try {
|
|
189
|
+
return JSON.parse(await readFile(path.join(args.root, rel), 'utf8'));
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
return undefined;
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
for (const b of plan.batches) {
|
|
196
|
+
const answers = await readJson(b.answer);
|
|
197
|
+
if (answers === undefined) {
|
|
198
|
+
rejected.push(`${b.answer}: нет ответа`);
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
const texts = new Map();
|
|
202
|
+
for (const ref of new Set(b.items.flatMap((i) => i.evidence.map((e) => e.ref))))
|
|
203
|
+
texts.set(ref, await readEvidence(ref));
|
|
204
|
+
const result = applyAnswers(b.items, answers, texts, args.now);
|
|
205
|
+
Object.assign(store.verdicts, result.accepted);
|
|
206
|
+
for (const r of result.rejected)
|
|
207
|
+
rejected.push(`${b.answer}: ${r.id} — ${r.reason}`);
|
|
208
|
+
}
|
|
209
|
+
for (const l of plan.lint) {
|
|
210
|
+
const answer = await readJson(l.answer);
|
|
211
|
+
const { verdict, error } = applyLintAnswer(l.titles, new Set(l.ids), answer, args.now);
|
|
212
|
+
if (verdict === null)
|
|
213
|
+
rejected.push(`${l.answer}: ${error ?? 'нет ответа'}`);
|
|
214
|
+
else
|
|
215
|
+
store.lint[l.file] = verdict;
|
|
216
|
+
}
|
|
217
|
+
await saveVerdicts(store);
|
|
218
|
+
for (const r of rejected)
|
|
219
|
+
console.error(r);
|
|
220
|
+
if (rejected.length > 0) {
|
|
221
|
+
console.error(`expect judge --apply: отклонено ${rejected.length}; принятое сохранено в ${VERDICTS_PATH}`);
|
|
222
|
+
return 1;
|
|
223
|
+
}
|
|
224
|
+
await rm(path.join(args.root, JUDGE_DIR), { recursive: true, force: true });
|
|
225
|
+
console.log(`expect judge --apply: вердикты сохранены в ${VERDICTS_PATH}`);
|
|
226
|
+
return 0;
|
|
227
|
+
}
|
|
228
|
+
const INIT_CONFIG = `import { defineConfig } from 'vitest/config';
|
|
229
|
+
|
|
230
|
+
// Ожидания: заголовки crit/warn — для человека, проверки — класс Check ниже в файле.
|
|
231
|
+
// Отчёт: npm run expect. Вердикты судьи: npm run expect:judge → expect:apply.
|
|
232
|
+
export default defineConfig({
|
|
233
|
+
test: {
|
|
234
|
+
include: ['expectations/**/*.expect.ts'],
|
|
235
|
+
testTimeout: 30_000,
|
|
236
|
+
},
|
|
237
|
+
});
|
|
238
|
+
`;
|
|
239
|
+
const initExample = (name) => `import { describe, it } from 'vitest';
|
|
240
|
+
import { budget } from '@shahimat/harness/expect';
|
|
241
|
+
|
|
242
|
+
budget({ crit: 0, warn: 0 });
|
|
243
|
+
|
|
244
|
+
describe('${name}', () => {
|
|
245
|
+
it.todo('первое ожидание: одно утверждение до 200 символов, да/нет или число');
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
// ─────────────────────────────── проверки ───────────────────────────────
|
|
249
|
+
|
|
250
|
+
class Check {}
|
|
251
|
+
void Check;
|
|
252
|
+
`;
|
|
253
|
+
/** Каркас ожиданий в проекте; существующие файлы не трогаются. */
|
|
254
|
+
async function init(args) {
|
|
255
|
+
const done = [];
|
|
256
|
+
const put = async (rel, content) => {
|
|
257
|
+
const abs = path.join(args.root, rel);
|
|
258
|
+
if (existsSync(abs))
|
|
259
|
+
return;
|
|
260
|
+
await mkdir(path.dirname(abs), { recursive: true });
|
|
261
|
+
await writeFile(abs, content, 'utf8');
|
|
262
|
+
done.push(`создан ${rel}`);
|
|
263
|
+
};
|
|
264
|
+
const name = path.basename(args.root);
|
|
265
|
+
await put(EXPECT_CONFIG, INIT_CONFIG);
|
|
266
|
+
await put(`expectations/${name}.expect.ts`, initExample(name));
|
|
267
|
+
const gitignore = path.join(args.root, '.gitignore');
|
|
268
|
+
const ignored = existsSync(gitignore) ? await readFile(gitignore, 'utf8') : '';
|
|
269
|
+
if (!ignored.split('\n').includes(`${JUDGE_DIR}/`)) {
|
|
270
|
+
await writeFile(gitignore, `${ignored}${ignored === '' || ignored.endsWith('\n') ? '' : '\n'}${JUDGE_DIR}/\n`, 'utf8');
|
|
271
|
+
done.push(`.gitignore: ${JUDGE_DIR}/`);
|
|
272
|
+
}
|
|
273
|
+
const pkgFile = path.join(args.root, 'package.json');
|
|
274
|
+
if (existsSync(pkgFile)) {
|
|
275
|
+
const pkg = JSON.parse(await readFile(pkgFile, 'utf8'));
|
|
276
|
+
const scripts = { expect: 'harness expect report --root .', 'expect:judge': 'harness expect judge --root .', 'expect:apply': 'harness expect judge --apply --root .' };
|
|
277
|
+
const added = Object.entries(scripts).filter(([k]) => pkg.scripts?.[k] === undefined);
|
|
278
|
+
if (added.length > 0) {
|
|
279
|
+
pkg.scripts = { ...pkg.scripts, ...Object.fromEntries(added) };
|
|
280
|
+
await writeFile(pkgFile, `${JSON.stringify(pkg, null, 2)}\n`, 'utf8');
|
|
281
|
+
done.push(`package.json: скрипты ${added.map(([k]) => k).join(', ')}`);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
for (const line of done)
|
|
285
|
+
console.log(line);
|
|
286
|
+
console.log(done.length === 0 ? 'expect init: всё уже на месте' : 'expect init: готово');
|
|
287
|
+
console.log('Зависимости: npm i -D vitest @shahimat/harness [@shahimat/expectations]');
|
|
288
|
+
return 0;
|
|
289
|
+
}
|
|
290
|
+
export async function runExpect(command, args) {
|
|
291
|
+
switch (command) {
|
|
292
|
+
case 'init':
|
|
293
|
+
return init(args);
|
|
294
|
+
case 'report':
|
|
295
|
+
return report(args);
|
|
296
|
+
case 'lint':
|
|
297
|
+
return lint(args);
|
|
298
|
+
case 'judge':
|
|
299
|
+
return args.apply === true ? judgeApply(args) : judgePrepare(args);
|
|
300
|
+
default:
|
|
301
|
+
throw new UsageError(`неизвестная команда: expect ${command ?? ''}`.trimEnd());
|
|
302
|
+
}
|
|
303
|
+
}
|
package/dist/commands/types.d.ts
CHANGED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { PendingItem } from './judge.ts';
|
|
2
|
+
import type { LintVerdict, Verdict } from './verdicts.ts';
|
|
3
|
+
export interface BatchLimits {
|
|
4
|
+
maxItems: number;
|
|
5
|
+
maxBytes: number;
|
|
6
|
+
}
|
|
7
|
+
export declare const DEFAULT_LIMITS: BatchLimits;
|
|
8
|
+
export interface Batch {
|
|
9
|
+
items: PendingItem[];
|
|
10
|
+
/** Уникальные доказательства пакета — судья читает каждое один раз. */
|
|
11
|
+
refs: string[];
|
|
12
|
+
bytes: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Пакеты для судьи: ожидания с одинаковыми доказательствами — вместе,
|
|
16
|
+
* группы складываются в пакеты до лимита вопросов и объёма доказательств.
|
|
17
|
+
*/
|
|
18
|
+
export declare function planBatches(items: PendingItem[], limits?: BatchLimits): Batch[];
|
|
19
|
+
export declare function renderBatch(batch: Batch, answerFile: string): string;
|
|
20
|
+
export interface Answer {
|
|
21
|
+
id: string;
|
|
22
|
+
verdict: boolean;
|
|
23
|
+
ref: string;
|
|
24
|
+
quote: string;
|
|
25
|
+
reason: string;
|
|
26
|
+
}
|
|
27
|
+
export interface ApplyResult {
|
|
28
|
+
accepted: Record<string, Verdict>;
|
|
29
|
+
rejected: {
|
|
30
|
+
id: string;
|
|
31
|
+
reason: string;
|
|
32
|
+
}[];
|
|
33
|
+
}
|
|
34
|
+
/** Проверяет ответы судьи; цитата должна дословно встречаться в доказательстве. */
|
|
35
|
+
export declare function applyAnswers(items: PendingItem[], answers: unknown, evidenceText: Map<string, string>, now: Date): ApplyResult;
|
|
36
|
+
/** Смысловая проверка заголовков одного файла ожиданий — один проход. */
|
|
37
|
+
export declare function renderLintBatch(file: string, titles: {
|
|
38
|
+
id: string;
|
|
39
|
+
title: string;
|
|
40
|
+
}[], answerFile: string): string;
|
|
41
|
+
export declare function applyLintAnswer(titlesSha: string, ids: Set<string>, answer: unknown, now: Date): {
|
|
42
|
+
verdict: LintVerdict | null;
|
|
43
|
+
error: string | null;
|
|
44
|
+
};
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { containsQuote } from './evidence.js';
|
|
2
|
+
export const DEFAULT_LIMITS = { maxItems: 25, maxBytes: 80_000 };
|
|
3
|
+
/**
|
|
4
|
+
* Пакеты для судьи: ожидания с одинаковыми доказательствами — вместе,
|
|
5
|
+
* группы складываются в пакеты до лимита вопросов и объёма доказательств.
|
|
6
|
+
*/
|
|
7
|
+
export function planBatches(items, limits = DEFAULT_LIMITS) {
|
|
8
|
+
const groups = new Map();
|
|
9
|
+
for (const item of [...items].sort((a, b) => (a.id < b.id ? -1 : 1))) {
|
|
10
|
+
const key = item.evidence.map((e) => e.ref).sort().join('\n');
|
|
11
|
+
groups.set(key, [...(groups.get(key) ?? []), item]);
|
|
12
|
+
}
|
|
13
|
+
const batches = [];
|
|
14
|
+
const sizes = new Map();
|
|
15
|
+
const bytesOf = (refs) => [...refs].reduce((n, r) => n + (sizes.get(r) ?? 0), 0);
|
|
16
|
+
let current = { items: [], refs: new Set() };
|
|
17
|
+
const flush = () => {
|
|
18
|
+
if (current.items.length === 0)
|
|
19
|
+
return;
|
|
20
|
+
batches.push({ items: current.items, refs: [...current.refs].sort(), bytes: bytesOf(current.refs) });
|
|
21
|
+
current = { items: [], refs: new Set() };
|
|
22
|
+
};
|
|
23
|
+
for (const group of groups.values()) {
|
|
24
|
+
for (const e of group[0]?.evidence ?? [])
|
|
25
|
+
sizes.set(e.ref, e.bytes);
|
|
26
|
+
for (const item of group) {
|
|
27
|
+
const refs = new Set([...current.refs, ...item.evidence.map((e) => e.ref)]);
|
|
28
|
+
if (current.items.length >= limits.maxItems || (current.items.length > 0 && bytesOf(refs) > limits.maxBytes))
|
|
29
|
+
flush();
|
|
30
|
+
current.items.push(item);
|
|
31
|
+
for (const e of item.evidence)
|
|
32
|
+
current.refs.add(e.ref);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
flush();
|
|
36
|
+
return batches;
|
|
37
|
+
}
|
|
38
|
+
export function renderBatch(batch, answerFile) {
|
|
39
|
+
const lines = [
|
|
40
|
+
'# Пакет для судьи ожиданий',
|
|
41
|
+
'',
|
|
42
|
+
'Ты — судья. Прочитай доказательства (каждое один раз) и ответь на',
|
|
43
|
+
'каждый вопрос: выполнено ли утверждение — да (true) или нет (false).',
|
|
44
|
+
'Отвечай только по доказательствам, без догадок.',
|
|
45
|
+
'',
|
|
46
|
+
'Для каждого ответа — дословная цитата (8–300 символов) из одного',
|
|
47
|
+
'доказательства и его ссылка (`ref`). Цитата проверяется программно:',
|
|
48
|
+
'не найдена в тексте — ответ отклоняется. Для «нет» — цитата того, что',
|
|
49
|
+
'противоречит или ближе всего к утверждению, и причина.',
|
|
50
|
+
'',
|
|
51
|
+
`Ответ — JSON-массив в файл \`${answerFile}\`:`,
|
|
52
|
+
'',
|
|
53
|
+
'```json',
|
|
54
|
+
'[{"id": "<id>", "verdict": true, "ref": "<ref>", "quote": "<цитата>", "reason": "<одна строка>"}]',
|
|
55
|
+
'```',
|
|
56
|
+
'',
|
|
57
|
+
'## Доказательства',
|
|
58
|
+
'',
|
|
59
|
+
'Ссылка `path#Заголовок` — раздел markdown с этим заголовком.',
|
|
60
|
+
'',
|
|
61
|
+
...batch.refs.map((r) => `- \`${r}\``),
|
|
62
|
+
'',
|
|
63
|
+
'## Вопросы',
|
|
64
|
+
'',
|
|
65
|
+
];
|
|
66
|
+
for (const item of batch.items) {
|
|
67
|
+
lines.push(`- \`${item.id}\` — ${item.question.split(' > ').join(' → ')}`);
|
|
68
|
+
lines.push(` доказательства: ${item.evidence.map((e) => `\`${e.ref}\``).join(', ')}`);
|
|
69
|
+
}
|
|
70
|
+
return `${lines.join('\n')}\n`;
|
|
71
|
+
}
|
|
72
|
+
/** Проверяет ответы судьи; цитата должна дословно встречаться в доказательстве. */
|
|
73
|
+
export function applyAnswers(items, answers, evidenceText, now) {
|
|
74
|
+
const result = { accepted: {}, rejected: [] };
|
|
75
|
+
if (!Array.isArray(answers)) {
|
|
76
|
+
result.rejected.push({ id: '*', reason: 'ответ — не JSON-массив' });
|
|
77
|
+
return result;
|
|
78
|
+
}
|
|
79
|
+
const byId = new Map(items.map((i) => [i.id, i]));
|
|
80
|
+
for (const raw of answers) {
|
|
81
|
+
const id = String(raw.id ?? '');
|
|
82
|
+
const item = byId.get(id);
|
|
83
|
+
if (item === undefined) {
|
|
84
|
+
result.rejected.push({ id, reason: 'неизвестный id' });
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
if (typeof raw.verdict !== 'boolean') {
|
|
88
|
+
result.rejected.push({ id, reason: 'verdict — не true/false' });
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
const ref = String(raw.ref ?? '');
|
|
92
|
+
if (!item.evidence.some((e) => e.ref === ref)) {
|
|
93
|
+
result.rejected.push({ id, reason: `ref «${ref}» не из доказательств вопроса` });
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
const quote = String(raw.quote ?? '');
|
|
97
|
+
if (quote.length > 300 || !containsQuote(evidenceText.get(ref) ?? '', quote)) {
|
|
98
|
+
result.rejected.push({ id, reason: 'цитата не найдена в доказательстве дословно' });
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
result.accepted[id] = {
|
|
102
|
+
question: item.question,
|
|
103
|
+
evidence: Object.fromEntries(item.evidence.map((e) => [e.ref, e.sha])),
|
|
104
|
+
verdict: raw.verdict,
|
|
105
|
+
quote,
|
|
106
|
+
ref,
|
|
107
|
+
reason: String(raw.reason ?? ''),
|
|
108
|
+
at: now.toISOString().slice(0, 10),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
for (const item of items) {
|
|
112
|
+
if (!(item.id in result.accepted) && !result.rejected.some((r) => r.id === item.id)) {
|
|
113
|
+
result.rejected.push({ id: item.id, reason: 'нет ответа' });
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return result;
|
|
117
|
+
}
|
|
118
|
+
/** Смысловая проверка заголовков одного файла ожиданий — один проход. */
|
|
119
|
+
export function renderLintBatch(file, titles, answerFile) {
|
|
120
|
+
return [
|
|
121
|
+
'# Проверка заголовков ожиданий',
|
|
122
|
+
'',
|
|
123
|
+
`Файл: \`${file}\`. Каждая строка — ожидание человека к реализации.`,
|
|
124
|
+
'Найди:',
|
|
125
|
+
'',
|
|
126
|
+
'1. Противоречия — пары ожиданий, которые не могут быть выполнены',
|
|
127
|
+
' одновременно.',
|
|
128
|
+
'2. Неизмеримые — ожидания, на которые нельзя ответить «да/нет» или',
|
|
129
|
+
' числом (размыто, оценочно, без порога).',
|
|
130
|
+
'',
|
|
131
|
+
'Ничего не нашёл — пустые массивы. Не выдумывай проблем.',
|
|
132
|
+
'',
|
|
133
|
+
`Ответ — JSON в файл \`${answerFile}\`:`,
|
|
134
|
+
'',
|
|
135
|
+
'```json',
|
|
136
|
+
'{"contradictions": [{"a": "<id>", "b": "<id>", "reason": "<строка>"}], "unmeasurable": [{"id": "<id>", "reason": "<строка>"}]}',
|
|
137
|
+
'```',
|
|
138
|
+
'',
|
|
139
|
+
'## Ожидания',
|
|
140
|
+
'',
|
|
141
|
+
...titles.map((t) => `- \`${t.id}\` — ${t.title}`),
|
|
142
|
+
'',
|
|
143
|
+
].join('\n');
|
|
144
|
+
}
|
|
145
|
+
export function applyLintAnswer(titlesSha, ids, answer, now) {
|
|
146
|
+
const a = answer;
|
|
147
|
+
if (a === null || typeof a !== 'object' || !Array.isArray(a.contradictions) || !Array.isArray(a.unmeasurable)) {
|
|
148
|
+
return { verdict: null, error: 'ответ — не {contradictions: [], unmeasurable: []}' };
|
|
149
|
+
}
|
|
150
|
+
const bad = [...a.contradictions.flatMap((c) => [c.a, c.b]), ...a.unmeasurable.map((u) => u.id)].filter((id) => !ids.has(id));
|
|
151
|
+
if (bad.length > 0)
|
|
152
|
+
return { verdict: null, error: `неизвестные id: ${bad.join(', ')}` };
|
|
153
|
+
return {
|
|
154
|
+
verdict: { titles: titlesSha, contradictions: a.contradictions, unmeasurable: a.unmeasurable, at: now.toISOString().slice(0, 10) },
|
|
155
|
+
error: null,
|
|
156
|
+
};
|
|
157
|
+
}
|