@payconductor/houndsman 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/commands/golden-command.ts +92 -0
- package/src/cli/commands/hunt-command.ts +85 -1
- package/src/cli/help.ts +3 -0
- package/src/cli/index.ts +2 -0
- package/src/core/brief.ts +35 -7
- package/src/core/commands/compare.ts +33 -0
- package/src/core/commands/golden.ts +176 -0
- package/src/core/commands/scaffold.ts +2 -0
- package/src/core/compare.ts +67 -0
- package/src/core/config.ts +10 -0
- package/src/core/engine/finalize-pipeline.ts +25 -1
- package/src/core/findings-state.ts +9 -1
- package/src/core/golden.ts +346 -0
- package/src/core/hunt/hunt-runner.ts +546 -49
- package/src/core/hunt/index.ts +38 -1
- package/src/core/hunt/maki-prompt.ts +289 -8
- package/src/core/hunt/maki-runner.ts +93 -16
- package/src/core/hunt/review-checks.ts +460 -0
- package/src/core/project-config.ts +10 -0
- package/src/core/registry-check.ts +30 -6
- package/src/core/report-builder.ts +9 -1
- package/src/core/severity.ts +245 -0
- package/src/core/storage/types.ts +1 -0
- package/src/index.ts +22 -0
- package/src/skill/OUTPUT-FORMAT.md +20 -1
- package/src/skill/README.md +1 -0
- package/src/skill/hunt.md +60 -0
- package/src/skill/train.md +3 -0
- package/src/tests/__snapshots__/baseline.test.ts.snap +24 -9
- package/src/tests/compare.test.ts +23 -0
- package/src/tests/findings-state.test.ts +82 -0
- package/src/tests/golden.test.ts +195 -0
- package/src/tests/hunt-maki.test.ts +652 -2
- package/src/tests/review-scripts.test.ts +19 -0
- package/src/tests/severity.test.ts +235 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@payconductor/houndsman",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"description": "Orquestrador de revisão de Pull Requests: classifica o diff, despacha watchdogs especializados por domínio e consolida o relatório.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Comando `golden` — avalia as runs da branch contra defeitos congelados.
|
|
3
|
+
*
|
|
4
|
+
* Exemplo:
|
|
5
|
+
* houndsman golden .houndsman/golden/asaas-split.json
|
|
6
|
+
* houndsman golden .houndsman/golden/asaas-split.json --variant otimizado-4
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { runGolden } from "../../core/commands/golden";
|
|
10
|
+
import { ArgParser } from "../arg-parser";
|
|
11
|
+
import type { CliContext, Command } from "./command";
|
|
12
|
+
|
|
13
|
+
/** Flags com valor (não entram nos positionais). */
|
|
14
|
+
const VALUE_FLAGS = new Set(["--branch", "--variant", "--slug", "--config"]);
|
|
15
|
+
|
|
16
|
+
export const GOLDEN_HELP = `Uso: houndsman golden <arquivo.json> [flags]
|
|
17
|
+
|
|
18
|
+
Avalia uma run (ou todas as variantes da branch) contra um golden set —
|
|
19
|
+
defeitos já confirmados no código — e reporta recall/precisão.
|
|
20
|
+
|
|
21
|
+
Flags:
|
|
22
|
+
--branch <nome> Branch alvo (default: a atual do git)
|
|
23
|
+
--variant <rótulo> Avalia só uma variante
|
|
24
|
+
--slug <slug> Avalia uma run específica
|
|
25
|
+
--json Saída JSON (sem markdown)
|
|
26
|
+
|
|
27
|
+
O relatório é gravado como \`golden-<slug>.md|json\` na run avaliada.
|
|
28
|
+
|
|
29
|
+
Formato do arquivo:
|
|
30
|
+
{ "version": 1, "title": "...", "commit": "abc1234",
|
|
31
|
+
"expected": [{ "id": "seller-api-key", "file": "src/a.ts",
|
|
32
|
+
"line": 116, "tolerance": 30, "match": "sellerApiKey",
|
|
33
|
+
"severity": "blocking", "note": "..." }] 1}
|
|
34
|
+
|
|
35
|
+
Exemplos:
|
|
36
|
+
houndsman golden .houndsman/golden/asaas-split.json
|
|
37
|
+
houndsman golden .houndsman/golden/asaas-split.json --variant otimizado-5`;
|
|
38
|
+
|
|
39
|
+
/** Primeiro argumento posicional (ignora flags com valor). */
|
|
40
|
+
function positionalFile(args: readonly string[]): string | undefined {
|
|
41
|
+
for (let index = 0; index < args.length; index++) {
|
|
42
|
+
const arg = args[index];
|
|
43
|
+
if (arg.startsWith("--")) {
|
|
44
|
+
if (VALUE_FLAGS.has(arg)) index++;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
return arg;
|
|
48
|
+
}
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export class GoldenCommand implements Command {
|
|
53
|
+
readonly name = "golden";
|
|
54
|
+
|
|
55
|
+
async run(args: string[], context: CliContext): Promise<number> {
|
|
56
|
+
const parser = ArgParser.from(args);
|
|
57
|
+
if (parser.has("--help") || parser.has("-h")) {
|
|
58
|
+
context.logger.info(GOLDEN_HELP);
|
|
59
|
+
return 0;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const file = positionalFile(args);
|
|
63
|
+
if (!file) {
|
|
64
|
+
context.logger.info(GOLDEN_HELP);
|
|
65
|
+
return 1;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const engine = await context.engine();
|
|
69
|
+
try {
|
|
70
|
+
const result = runGolden(engine.storage, engine.repo, {
|
|
71
|
+
file,
|
|
72
|
+
branch: parser.value("--branch"),
|
|
73
|
+
variant: parser.value("--variant"),
|
|
74
|
+
slug: parser.value("--slug"),
|
|
75
|
+
});
|
|
76
|
+
if (parser.has("--json")) {
|
|
77
|
+
context.logger.info(result.json.trimEnd());
|
|
78
|
+
} else {
|
|
79
|
+
context.logger.info(result.markdown.trimEnd());
|
|
80
|
+
context.logger.success(
|
|
81
|
+
`Golden salvo em ${result.artifacts?.markdown} (run ${result.target.slug}).`,
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
return 0;
|
|
85
|
+
} catch (error) {
|
|
86
|
+
context.logger.error(error instanceof Error ? error.message : String(error));
|
|
87
|
+
return 1;
|
|
88
|
+
} finally {
|
|
89
|
+
engine.close();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
@@ -57,6 +57,8 @@ Flags:
|
|
|
57
57
|
--hunt-concurrency Processos maki simultâneos
|
|
58
58
|
--hunt-timeout Timeout por watchdog (ms)
|
|
59
59
|
--no-retry Não repete saída inválida
|
|
60
|
+
--no-checklist Sem a passada extra de fluxos/invariantes não verificados
|
|
61
|
+
--no-second-opinion Sem a passada adversarial do revisor de fluxo
|
|
60
62
|
--fail-on-blocking Exit 1 se houver bloqueantes
|
|
61
63
|
|
|
62
64
|
Instalação:
|
|
@@ -237,6 +239,10 @@ export class HuntCommand implements Command {
|
|
|
237
239
|
concurrency,
|
|
238
240
|
timeoutMs,
|
|
239
241
|
retries,
|
|
242
|
+
checklist: parser.has("--no-checklist") ? false : project.huntChecklist,
|
|
243
|
+
secondOpinion: parser.has("--no-second-opinion")
|
|
244
|
+
? false
|
|
245
|
+
: project.huntSecondOpinion,
|
|
240
246
|
runner: new SpawnMakiRunner(binary.path),
|
|
241
247
|
onEvent: (event) => this.logEvent(board, event, context.logger),
|
|
242
248
|
});
|
|
@@ -267,6 +273,56 @@ export class HuntCommand implements Command {
|
|
|
267
273
|
`${report.totals.findings} finding(s), $${report.totals.costUsd.toFixed(4)}, ` +
|
|
268
274
|
`${Math.round(report.durationMs / 1000)}s.`,
|
|
269
275
|
);
|
|
276
|
+
const checklistCost = report.watchdogs.reduce(
|
|
277
|
+
(sum, watchdog) => sum + (watchdog.checklist?.costUsd ?? 0),
|
|
278
|
+
0,
|
|
279
|
+
);
|
|
280
|
+
const opinionCost = report.watchdogs.reduce(
|
|
281
|
+
(sum, watchdog) => sum + (watchdog.secondOpinion?.costUsd ?? 0),
|
|
282
|
+
0,
|
|
283
|
+
);
|
|
284
|
+
if (checklistCost + opinionCost > 0) {
|
|
285
|
+
context.logger.info(
|
|
286
|
+
`Passadas extras: checklist $${checklistCost.toFixed(4)} · ` +
|
|
287
|
+
`segunda opinião $${opinionCost.toFixed(4)}.`,
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
const unchecked = report.watchdogs.filter(
|
|
291
|
+
(watchdog) =>
|
|
292
|
+
(watchdog.checks?.missing ?? 0) + (watchdog.rules?.missing ?? 0) > 0,
|
|
293
|
+
);
|
|
294
|
+
if (unchecked.length > 0) {
|
|
295
|
+
context.logger.warn(
|
|
296
|
+
`Checklist de intenção/regras incompleto em ${unchecked.length} watchdog(s): ` +
|
|
297
|
+
unchecked
|
|
298
|
+
.map(
|
|
299
|
+
(watchdog) =>
|
|
300
|
+
`${watchdog.watchdogId} (itens ${watchdog.checks?.missing ?? 0}, regras ${watchdog.rules?.missing ?? 0})`,
|
|
301
|
+
)
|
|
302
|
+
.join(", "),
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
const businessReview = report.watchdogs.find(
|
|
306
|
+
(watchdog) => watchdog.business !== undefined,
|
|
307
|
+
)?.business;
|
|
308
|
+
if (businessReview && !businessReview.accepted) {
|
|
309
|
+
context.logger.info(
|
|
310
|
+
"Modelo de intenção devolvido pelo revisor foi descartado " +
|
|
311
|
+
`(inferior ao já gravado: ${report.business ?? "n/d"}).`,
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
const incomplete = report.watchdogs
|
|
315
|
+
.filter((watchdog) => (watchdog.coverage?.undeclared ?? 0) > 0)
|
|
316
|
+
.map(
|
|
317
|
+
(watchdog) =>
|
|
318
|
+
`${watchdog.watchdogId} ${watchdog.coverage?.reviewed ?? 0}/${watchdog.coverage?.total ?? 0}`,
|
|
319
|
+
);
|
|
320
|
+
if (incomplete.length > 0) {
|
|
321
|
+
context.logger.info(
|
|
322
|
+
`Cobertura declarada incompleta: ${incomplete.join(", ")} ` +
|
|
323
|
+
"(detalhe em `hunt-report.json`).",
|
|
324
|
+
);
|
|
325
|
+
}
|
|
270
326
|
|
|
271
327
|
const failed = report.watchdogs.filter(
|
|
272
328
|
(watchdog) => watchdog.status === "failed",
|
|
@@ -405,7 +461,9 @@ export class HuntCommand implements Command {
|
|
|
405
461
|
`harness: ${project.huntEnabled ? "ligado" : "desligado"} · ` +
|
|
406
462
|
`concorrência ${project.huntConcurrency} · ` +
|
|
407
463
|
`timeout ${Math.round(project.huntTimeoutMs / 1000)}s · ` +
|
|
408
|
-
`retries ${project.huntRetries}
|
|
464
|
+
`retries ${project.huntRetries} · ` +
|
|
465
|
+
`checklist ${project.huntChecklist ? "on" : "off"} · ` +
|
|
466
|
+
`segunda opinião ${project.huntSecondOpinion ? "on" : "off"}.`,
|
|
409
467
|
);
|
|
410
468
|
return 0;
|
|
411
469
|
}
|
|
@@ -431,6 +489,32 @@ export class HuntCommand implements Command {
|
|
|
431
489
|
case "start":
|
|
432
490
|
board.add(event.watchdogId, "revisando");
|
|
433
491
|
return;
|
|
492
|
+
case "checklist":
|
|
493
|
+
if (event.status === "start") {
|
|
494
|
+
logger.step(
|
|
495
|
+
`${event.watchdogId}: ${event.missing} item(s) e ${event.missingRules} regra(s) sem verificação — passada de checklist…`,
|
|
496
|
+
);
|
|
497
|
+
} else if (event.status === "failed") {
|
|
498
|
+
logger.warn(
|
|
499
|
+
`${event.watchdogId}: checklist falhou (${event.error ?? "erro"}); relatório da primeira passada mantido.`,
|
|
500
|
+
);
|
|
501
|
+
}
|
|
502
|
+
return;
|
|
503
|
+
case "second-opinion":
|
|
504
|
+
if (event.status === "start") {
|
|
505
|
+
logger.step(
|
|
506
|
+
`${event.watchdogId}: passada adversarial (credencial/estado, capacidade, identificador, idempotência, contrato)…`,
|
|
507
|
+
);
|
|
508
|
+
} else if (event.status === "done") {
|
|
509
|
+
logger.info(
|
|
510
|
+
`${event.watchdogId}: segunda opinião adicionou ${event.findings ?? 0} finding(s).`,
|
|
511
|
+
);
|
|
512
|
+
} else {
|
|
513
|
+
logger.warn(
|
|
514
|
+
`${event.watchdogId}: segunda opinião falhou (${event.error ?? "erro"}); relatório anterior mantido.`,
|
|
515
|
+
);
|
|
516
|
+
}
|
|
517
|
+
return;
|
|
434
518
|
case "retry":
|
|
435
519
|
board.retry(
|
|
436
520
|
event.watchdogId,
|
package/src/cli/help.ts
CHANGED
|
@@ -6,6 +6,7 @@ Comandos:
|
|
|
6
6
|
--finalize Consolida os findings e grava o resumo final
|
|
7
7
|
context Coleta o contexto da PR via gh (Passo 0; --pr/--worktree)
|
|
8
8
|
compare [a] [b] Compara duas runs (A/B) por slug ou --branch/--a/--b
|
|
9
|
+
golden <arquivo> Avalia as runs contra um golden set (recall/precisão)
|
|
9
10
|
findings <sub> Triagem/correção dos findings (list · show · set · done · reset · sync · summary · batch · verify · submit · path)
|
|
10
11
|
hunt [flags] Harness local (maki): prepare → watchdogs via maki → finalize
|
|
11
12
|
(a skill /houndsman hunt continua existindo p/ agentes externos)
|
|
@@ -43,6 +44,8 @@ Flags de hunt (harness local):
|
|
|
43
44
|
--hunt-concurrency Processos maki simultâneos
|
|
44
45
|
--hunt-timeout Timeout por watchdog (ms)
|
|
45
46
|
--no-retry Não repete saída inválida
|
|
47
|
+
--no-checklist Sem a passada extra de fluxos/invariantes não verificados
|
|
48
|
+
--no-second-opinion Sem a passada adversarial do revisor de fluxo
|
|
46
49
|
|
|
47
50
|
Flags de findings:
|
|
48
51
|
--decision <d> pending | execute | ignore
|
package/src/cli/index.ts
CHANGED
|
@@ -39,6 +39,7 @@ import { CompareCommand } from "./commands/compare-command";
|
|
|
39
39
|
import { ContextCommand } from "./commands/context-command";
|
|
40
40
|
import { DoctorCommand } from "./commands/doctor-command";
|
|
41
41
|
import { FindingsCommand } from "./commands/findings-command";
|
|
42
|
+
import { GoldenCommand } from "./commands/golden-command";
|
|
42
43
|
import { HelpCommand } from "./commands/help-command";
|
|
43
44
|
import { HuntCommand } from "./commands/hunt-command";
|
|
44
45
|
import { CommandRegistry } from "./commands/registry";
|
|
@@ -63,6 +64,7 @@ function buildRegistry(): CommandRegistry {
|
|
|
63
64
|
.register(new HuntCommand())
|
|
64
65
|
.register(new ContextCommand())
|
|
65
66
|
.register(new CompareCommand())
|
|
67
|
+
.register(new GoldenCommand())
|
|
66
68
|
.register(new FindingsCommand())
|
|
67
69
|
.register(new StatusCommand())
|
|
68
70
|
.register(new WatchdogCommand())
|
package/src/core/brief.ts
CHANGED
|
@@ -40,22 +40,36 @@ export const PROCEDURE: readonly string[] = [
|
|
|
40
40
|
* O diff já cobre o essencial; o resto é para chamador/contexto de um símbolo
|
|
41
41
|
* citado. Sem teto, o subagente entrava em loop de leitura e cada brief levava
|
|
42
42
|
* minutos — o `general` era o pior caso (PR inteira).
|
|
43
|
+
*
|
|
44
|
+
* O teto mínimo é o piso, não o valor fixo: um domínio com 37 arquivos não se
|
|
45
|
+
* revisa com 10 leituras (o `general` pulava 12 de 37 e perdia o rastreio de
|
|
46
|
+
* contrato). O tamanho do domínio define quantas leituras o brief concede.
|
|
43
47
|
*/
|
|
44
48
|
const EXPLORATION_BUDGET = {
|
|
45
|
-
general: { reads: 10, searches:
|
|
46
|
-
domain: { reads: 6, searches: 6 },
|
|
49
|
+
general: { reads: 10, searches: 12, maxReads: 24, filesPerRead: 2 },
|
|
50
|
+
domain: { reads: 6, searches: 6, maxReads: 14, filesPerRead: 2 },
|
|
47
51
|
} as const;
|
|
48
52
|
|
|
53
|
+
/** Teto de leituras para o domínio (cresce com o nº de arquivos, até o máximo). */
|
|
54
|
+
function readsFor(
|
|
55
|
+
budget: (typeof EXPLORATION_BUDGET)[keyof typeof EXPLORATION_BUDGET],
|
|
56
|
+
files: number,
|
|
57
|
+
): number {
|
|
58
|
+
const needed = Math.ceil(files / budget.filesPerRead);
|
|
59
|
+
return Math.min(budget.maxReads, Math.max(budget.reads, needed));
|
|
60
|
+
}
|
|
61
|
+
|
|
49
62
|
/** Seção com o teto de leituras/buscas do brief (visível para o modelo). */
|
|
50
63
|
function explorationSection(args: WatchdogRenderArgs): string[] {
|
|
51
64
|
const budget =
|
|
52
65
|
args.watchdogId === GENERAL_WATCHDOG_ID
|
|
53
66
|
? EXPLORATION_BUDGET.general
|
|
54
67
|
: EXPLORATION_BUDGET.domain;
|
|
68
|
+
const reads = readsFor(budget, args.files.length);
|
|
55
69
|
return [
|
|
56
70
|
"## Orçamento de exploração",
|
|
57
71
|
"",
|
|
58
|
-
`> O diff já está no brief. Limite **neste brief**: ${
|
|
72
|
+
`> O diff já está no brief. Limite **neste brief**: ${reads} leitura(s) de arquivo`,
|
|
59
73
|
`> e ${budget.searches} busca(s) (grep).`,
|
|
60
74
|
"> Abra arquivo só para chamador/contexto de um símbolo citado nos trechos.",
|
|
61
75
|
"> Se a prova não estiver no brief nem nesse orçamento, **não é achado** —",
|
|
@@ -171,11 +185,24 @@ function codeItem(value: string): string {
|
|
|
171
185
|
return `- \`${value}\``;
|
|
172
186
|
}
|
|
173
187
|
|
|
174
|
-
/** Linha de regra com emoji de severidade. */
|
|
175
|
-
function ruleItem(rule: Rule): string {
|
|
176
|
-
return `- ${SEVERITY_EMOJI[rule.severity]} **${rule.severity}** — ${rule.text}`;
|
|
188
|
+
/** Linha de regra com número e emoji de severidade (o número é a identidade no checklist). */
|
|
189
|
+
function ruleItem(rule: Rule, index: number): string {
|
|
190
|
+
return `- ${index}. ${SEVERITY_EMOJI[rule.severity]} **${rule.severity}** — ${rule.text}`;
|
|
177
191
|
}
|
|
178
192
|
|
|
193
|
+
/**
|
|
194
|
+
* Contrato de severidade do output — a severidade é derivada, não escolhida.
|
|
195
|
+
*
|
|
196
|
+
* Finding de regra carrega a severidade declarada em `Rule.severity`; emergente
|
|
197
|
+
* carrega `impactClass`, mapeado em `core/severity.ts`. Isso impede o mesmo
|
|
198
|
+
* código de sair `blocking` numa review e `attention` na seguinte.
|
|
199
|
+
*/
|
|
200
|
+
export const SEVERITY_GUIDE: readonly string[] = [
|
|
201
|
+
"- **Finding de regra:** copie `rule` exatamente como no brief. A severidade é a **declarada na regra** — outro valor é ignorado e corrigido pelo motor.",
|
|
202
|
+
"- **Finding emergente** (`Lógica:`/`Domínio:`): preencha `impactClass` — `flow-break` (fluxo/capacidade não honrada), `contract-drift` (contrato/estado divergente), `degradation` (borda/erro frágil) ou `cosmetic`.",
|
|
203
|
+
"- `blocking` fora de regra continua exigindo `scenario` + `evidence`; sem os dois, o motor rebaixa para `attention`.",
|
|
204
|
+
];
|
|
205
|
+
|
|
179
206
|
/** Seção opcional (`## <título>` + itens) — vazia se não houver itens. */
|
|
180
207
|
function optionalList(title: string, items: readonly string[] | undefined): string[] {
|
|
181
208
|
if (!items || items.length === 0) return [];
|
|
@@ -587,9 +614,10 @@ export function defaultRenderBrief(
|
|
|
587
614
|
"",
|
|
588
615
|
...[...def.rules]
|
|
589
616
|
.sort((a, b) => severityRank(a.severity) - severityRank(b.severity))
|
|
590
|
-
.map(ruleItem),
|
|
617
|
+
.map((rule, index) => ruleItem(rule, index + 1)),
|
|
591
618
|
"",
|
|
592
619
|
);
|
|
620
|
+
lines.push("## Como declarar a severidade", "", ...SEVERITY_GUIDE, "");
|
|
593
621
|
lines.push(
|
|
594
622
|
"## Docs de contexto (carregar sob demanda)",
|
|
595
623
|
"",
|
|
@@ -46,10 +46,42 @@ type ReportShape = { durationMs?: number };
|
|
|
46
46
|
type HuntShape = {
|
|
47
47
|
model?: string;
|
|
48
48
|
durationMs?: number;
|
|
49
|
+
businessScore?: number;
|
|
49
50
|
totals?: { totalTokens?: number; costUsd?: number };
|
|
51
|
+
watchdogs?: {
|
|
52
|
+
checks?: {
|
|
53
|
+
expected?: number;
|
|
54
|
+
verified?: number;
|
|
55
|
+
missing?: number;
|
|
56
|
+
gaps?: number;
|
|
57
|
+
};
|
|
58
|
+
coverage?: { reviewed?: number; skipped?: number; undeclared?: number };
|
|
59
|
+
checklist?: { costUsd?: number };
|
|
60
|
+
secondOpinion?: { costUsd?: number; findings?: number };
|
|
61
|
+
}[];
|
|
50
62
|
};
|
|
51
63
|
type AiUsageShape = { total?: { totalTokens?: number } };
|
|
52
64
|
|
|
65
|
+
/** Agrega o diagnóstico do `hunt-report.json` (checklist + cobertura declarada). */
|
|
66
|
+
function harnessOf(hunt: HuntShape): NonNullable<ReviewSnapshot["harness"]> {
|
|
67
|
+
const watchdogs = hunt.watchdogs ?? [];
|
|
68
|
+
const sum = (pick: (item: (typeof watchdogs)[number]) => number | undefined) =>
|
|
69
|
+
watchdogs.reduce((total, item) => total + (pick(item) ?? 0), 0);
|
|
70
|
+
return {
|
|
71
|
+
checksExpected: sum((item) => item.checks?.expected),
|
|
72
|
+
checksVerified: sum((item) => item.checks?.verified),
|
|
73
|
+
checksGaps: sum((item) => item.checks?.gaps),
|
|
74
|
+
reviewedFiles: sum((item) => item.coverage?.reviewed),
|
|
75
|
+
skippedFiles: sum((item) => item.coverage?.skipped),
|
|
76
|
+
undeclaredFiles: sum((item) => item.coverage?.undeclared),
|
|
77
|
+
businessScore: hunt.businessScore,
|
|
78
|
+
extraCostUsd:
|
|
79
|
+
sum((item) => item.checklist?.costUsd) +
|
|
80
|
+
sum((item) => item.secondOpinion?.costUsd),
|
|
81
|
+
secondOpinionFindings: sum((item) => item.secondOpinion?.findings),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
53
85
|
/** Resolve a run pelo slug dentro do repositório. */
|
|
54
86
|
function bySlug(
|
|
55
87
|
storage: StorageProvider,
|
|
@@ -149,6 +181,7 @@ function snapshotOf(
|
|
|
149
181
|
model: hunt?.model ?? manifest?.ai?.model,
|
|
150
182
|
findings,
|
|
151
183
|
bySeverity: countBySeverity(findings),
|
|
184
|
+
harness: hunt ? harnessOf(hunt) : undefined,
|
|
152
185
|
coverage: {
|
|
153
186
|
watchdogs: watchdogs.length,
|
|
154
187
|
files: files.size,
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Comando `golden` (biblioteca) — avalia runs contra defeitos congelados.
|
|
3
|
+
*
|
|
4
|
+
* A contagem de findings de duas variantes nunca é comparável: cada execução
|
|
5
|
+
* amostra pontos diferentes do mesmo diff. Com um golden set (arquivo do
|
|
6
|
+
* consumidor), a comparação vira recall/precisão sobre os defeitos que já
|
|
7
|
+
* foram confirmados no código.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { readFileSync } from "node:fs";
|
|
11
|
+
import { resolve } from "node:path";
|
|
12
|
+
import {
|
|
13
|
+
commitMatches,
|
|
14
|
+
type GoldenFinding,
|
|
15
|
+
type GoldenRunScore,
|
|
16
|
+
type GoldenSet,
|
|
17
|
+
goldenToJson,
|
|
18
|
+
parseGoldenSet,
|
|
19
|
+
renderGoldenReport,
|
|
20
|
+
scoreGolden,
|
|
21
|
+
} from "../golden";
|
|
22
|
+
import { normalizePath } from "../hunt/review-checks";
|
|
23
|
+
import { currentBranch } from "../pr-context";
|
|
24
|
+
import type { RepoIdentity, ReviewRecord, StorageProvider } from "../storage/types";
|
|
25
|
+
|
|
26
|
+
/** Seleção das runs a avaliar. */
|
|
27
|
+
export type GoldenOptions = {
|
|
28
|
+
/** Caminho do arquivo de golden set. */
|
|
29
|
+
file: string;
|
|
30
|
+
/** Branch alvo (default: a atual). */
|
|
31
|
+
branch?: string;
|
|
32
|
+
/** Rótulo da variante; ausente = todas as runs da branch. */
|
|
33
|
+
variant?: string;
|
|
34
|
+
/** Slug explícito (vence `variant`). */
|
|
35
|
+
slug?: string;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/** Resultado do comando. */
|
|
39
|
+
export type GoldenResult = {
|
|
40
|
+
markdown: string;
|
|
41
|
+
json: string;
|
|
42
|
+
set: GoldenSet;
|
|
43
|
+
runs: GoldenRunScore[];
|
|
44
|
+
/** Run onde o relatório foi gravado (a mais recente avaliada). */
|
|
45
|
+
target: ReviewRecord;
|
|
46
|
+
artifacts?: { markdown: string; json: string };
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/** Lê e valida o arquivo de golden set. */
|
|
50
|
+
export function loadGoldenSet(file: string): {
|
|
51
|
+
set: GoldenSet;
|
|
52
|
+
warnings: string[];
|
|
53
|
+
} {
|
|
54
|
+
let content: string;
|
|
55
|
+
try {
|
|
56
|
+
content = readFileSync(resolve(file), "utf8");
|
|
57
|
+
} catch (error) {
|
|
58
|
+
throw new Error(
|
|
59
|
+
`Golden set ilegível: ${error instanceof Error ? error.message : String(error)}`,
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
let raw: unknown;
|
|
63
|
+
try {
|
|
64
|
+
raw = JSON.parse(content);
|
|
65
|
+
} catch {
|
|
66
|
+
throw new Error(`Golden set não é JSON válido: ${file}`);
|
|
67
|
+
}
|
|
68
|
+
const parsed = parseGoldenSet(raw);
|
|
69
|
+
if (!parsed.set) {
|
|
70
|
+
throw new Error(
|
|
71
|
+
`Golden set inválido (${file}): ${parsed.warnings.join(" ")}`.trim(),
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
return { set: parsed.set, warnings: parsed.warnings };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Findings presentes de uma run, no formato da avaliação. */
|
|
78
|
+
export function goldenFindingsOf(
|
|
79
|
+
storage: StorageProvider,
|
|
80
|
+
review: ReviewRecord,
|
|
81
|
+
): GoldenFinding[] {
|
|
82
|
+
const state = storage.readFindings(review.id);
|
|
83
|
+
return Object.values(state?.findings ?? {})
|
|
84
|
+
.filter((finding) => finding.present)
|
|
85
|
+
.map((finding) => ({
|
|
86
|
+
fingerprint: finding.fingerprint,
|
|
87
|
+
file: normalizePath(finding.file),
|
|
88
|
+
line: finding.line ?? undefined,
|
|
89
|
+
rule: finding.rule,
|
|
90
|
+
severity: finding.severity,
|
|
91
|
+
message: finding.message,
|
|
92
|
+
evidence: finding.evidence,
|
|
93
|
+
watchdogs: finding.watchdogIds,
|
|
94
|
+
}));
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Runs alvo: slug > variante > todas as da branch. */
|
|
98
|
+
function selectReviews(
|
|
99
|
+
storage: StorageProvider,
|
|
100
|
+
repo: RepoIdentity,
|
|
101
|
+
options: GoldenOptions,
|
|
102
|
+
): ReviewRecord[] {
|
|
103
|
+
const reviews = storage.listReviews(repo.key);
|
|
104
|
+
if (options.slug) {
|
|
105
|
+
const found = reviews.find((review) => review.slug === options.slug);
|
|
106
|
+
if (!found) {
|
|
107
|
+
throw new Error(`Nenhuma run com slug '${options.slug}' neste repositório.`);
|
|
108
|
+
}
|
|
109
|
+
return [found];
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const branch = options.branch ?? currentBranch();
|
|
113
|
+
const ofBranch = reviews.filter((review) => review.branch === branch);
|
|
114
|
+
const variant = options.variant?.trim();
|
|
115
|
+
const selected = variant
|
|
116
|
+
? ofBranch.filter((review) => review.variant === variant)
|
|
117
|
+
: ofBranch;
|
|
118
|
+
if (selected.length === 0) {
|
|
119
|
+
throw new Error(
|
|
120
|
+
`Nenhuma revisão para '${branch}'${variant ? ` (variante '${variant}')` : ""}.`,
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
return selected;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Rótulo curto da run (`otimizado-4` ou `principal`). */
|
|
127
|
+
function labelOf(review: ReviewRecord): string {
|
|
128
|
+
return review.variant?.trim() || "principal";
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Executa a avaliação e grava o relatório na run mais recente selecionada. */
|
|
132
|
+
export function runGolden(
|
|
133
|
+
storage: StorageProvider,
|
|
134
|
+
repo: RepoIdentity,
|
|
135
|
+
options: GoldenOptions,
|
|
136
|
+
): GoldenResult {
|
|
137
|
+
const { set } = loadGoldenSet(options.file);
|
|
138
|
+
const reviews = selectReviews(storage, repo, options);
|
|
139
|
+
const runs: GoldenRunScore[] = reviews.map((review) => {
|
|
140
|
+
const findings = goldenFindingsOf(storage, review);
|
|
141
|
+
const commit = review.headSha ?? review.reviewedSha;
|
|
142
|
+
return {
|
|
143
|
+
slug: review.slug,
|
|
144
|
+
label: labelOf(review),
|
|
145
|
+
variant: review.variant,
|
|
146
|
+
scope: review.branch,
|
|
147
|
+
commit,
|
|
148
|
+
commitMismatch: !commitMatches(set.commit, commit),
|
|
149
|
+
findings: findings.length,
|
|
150
|
+
score: scoreGolden(set, findings),
|
|
151
|
+
};
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
const markdown = renderGoldenReport(set, runs);
|
|
155
|
+
const json = goldenToJson(set, runs);
|
|
156
|
+
const target = reviews[0];
|
|
157
|
+
const base = `golden-${target.slug}`;
|
|
158
|
+
storage.writeArtifact(target.id, {
|
|
159
|
+
name: `${base}.md`,
|
|
160
|
+
kind: "golden",
|
|
161
|
+
content: markdown,
|
|
162
|
+
});
|
|
163
|
+
storage.writeArtifact(target.id, {
|
|
164
|
+
name: `${base}.json`,
|
|
165
|
+
kind: "golden",
|
|
166
|
+
content: json,
|
|
167
|
+
});
|
|
168
|
+
return {
|
|
169
|
+
markdown,
|
|
170
|
+
json,
|
|
171
|
+
set,
|
|
172
|
+
runs,
|
|
173
|
+
target,
|
|
174
|
+
artifacts: { markdown: `${base}.md`, json: `${base}.json` },
|
|
175
|
+
};
|
|
176
|
+
}
|
|
@@ -426,10 +426,12 @@ export const DOC_MAPPINGS: readonly (readonly [RegExp, readonly string[]])[] = [
|
|
|
426
426
|
|
|
427
427
|
const PROJECT_GITIGNORE = `# Diretórios gerados pelo houndsman — \`core/\` é regenerado por \`houndsman update\`.
|
|
428
428
|
# \`work/\` e \`worktrees/\` são runtime de revisão (staging e worktrees).
|
|
429
|
+
# \`golden/\` é massa de avaliação local (defeitos congelados por commit).
|
|
429
430
|
core/
|
|
430
431
|
.runs/
|
|
431
432
|
work/
|
|
432
433
|
worktrees/
|
|
434
|
+
golden/
|
|
433
435
|
`;
|
|
434
436
|
|
|
435
437
|
// ---------------------------------------------------------------------------
|
package/src/core/compare.ts
CHANGED
|
@@ -38,6 +38,23 @@ export type ReviewSnapshot = {
|
|
|
38
38
|
prepareMs?: number;
|
|
39
39
|
/** Duração do harness maki (`hunt-report.json`). */
|
|
40
40
|
huntMs?: number;
|
|
41
|
+
/**
|
|
42
|
+
* Diagnóstico do harness local: checklist de intenção e cobertura declarada.
|
|
43
|
+
* Ausente no modo skill (não há `hunt-report.json`).
|
|
44
|
+
*/
|
|
45
|
+
harness?: {
|
|
46
|
+
checksExpected?: number;
|
|
47
|
+
checksVerified?: number;
|
|
48
|
+
checksGaps?: number;
|
|
49
|
+
reviewedFiles?: number;
|
|
50
|
+
skippedFiles?: number;
|
|
51
|
+
undeclaredFiles?: number;
|
|
52
|
+
businessScore?: number;
|
|
53
|
+
/** Custo somado das passadas extras (checklist + segunda opinião). */
|
|
54
|
+
extraCostUsd?: number;
|
|
55
|
+
/** Findings adicionados pela segunda opinião. */
|
|
56
|
+
secondOpinionFindings?: number;
|
|
57
|
+
};
|
|
41
58
|
tokens?: number;
|
|
42
59
|
costUsd?: number;
|
|
43
60
|
};
|
|
@@ -123,6 +140,22 @@ function formatMs(ms: number | undefined): string {
|
|
|
123
140
|
return `${minutes}m${String(Math.round(seconds % 60)).padStart(2, "0")}s`;
|
|
124
141
|
}
|
|
125
142
|
|
|
143
|
+
/** Contagem simples (`—` quando não medida). */
|
|
144
|
+
function formatCount(value: number | undefined): string {
|
|
145
|
+
return value === undefined || !Number.isFinite(value) ? "—" : String(value);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Razão verificada/esperada (`4/6`). */
|
|
149
|
+
function formatRatio(verified: number | undefined, expected: number | undefined): string {
|
|
150
|
+
if (verified === undefined || expected === undefined) return "—";
|
|
151
|
+
return `${verified}/${expected}`;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Custo em dólar (`$0.1234`). */
|
|
155
|
+
function formatCost(value: number | undefined): string {
|
|
156
|
+
return value === undefined || !Number.isFinite(value) ? "—" : `$${value.toFixed(4)}`;
|
|
157
|
+
}
|
|
158
|
+
|
|
126
159
|
function formatDelta(
|
|
127
160
|
a: number | undefined,
|
|
128
161
|
b: number | undefined,
|
|
@@ -206,6 +239,40 @@ export function renderComparison(comparison: ReviewComparison): string {
|
|
|
206
239
|
metricRow("Duração total", formatMs(a.durationMs), formatMs(b.durationMs)),
|
|
207
240
|
metricRow("Prepare", formatMs(a.prepareMs), formatMs(b.prepareMs)),
|
|
208
241
|
metricRow("Harness", formatMs(a.huntMs), formatMs(b.huntMs)),
|
|
242
|
+
...(a.harness || b.harness
|
|
243
|
+
? [
|
|
244
|
+
metricRow(
|
|
245
|
+
"Checklist verificado",
|
|
246
|
+
formatRatio(a.harness?.checksVerified, a.harness?.checksExpected),
|
|
247
|
+
formatRatio(b.harness?.checksVerified, b.harness?.checksExpected),
|
|
248
|
+
),
|
|
249
|
+
metricRow(
|
|
250
|
+
"Arquivos revisados (declarado)",
|
|
251
|
+
formatCount(a.harness?.reviewedFiles),
|
|
252
|
+
formatCount(b.harness?.reviewedFiles),
|
|
253
|
+
),
|
|
254
|
+
metricRow(
|
|
255
|
+
"Arquivos sem declaração",
|
|
256
|
+
formatCount(a.harness?.undeclaredFiles),
|
|
257
|
+
formatCount(b.harness?.undeclaredFiles),
|
|
258
|
+
),
|
|
259
|
+
metricRow(
|
|
260
|
+
"Modelo de intenção (score)",
|
|
261
|
+
formatCount(a.harness?.businessScore),
|
|
262
|
+
formatCount(b.harness?.businessScore),
|
|
263
|
+
),
|
|
264
|
+
metricRow(
|
|
265
|
+
"Passadas extras (custo)",
|
|
266
|
+
formatCost(a.harness?.extraCostUsd),
|
|
267
|
+
formatCost(b.harness?.extraCostUsd),
|
|
268
|
+
),
|
|
269
|
+
metricRow(
|
|
270
|
+
"Segunda opinião (findings)",
|
|
271
|
+
formatCount(a.harness?.secondOpinionFindings),
|
|
272
|
+
formatCount(b.harness?.secondOpinionFindings),
|
|
273
|
+
),
|
|
274
|
+
]
|
|
275
|
+
: []),
|
|
209
276
|
metricRow(
|
|
210
277
|
"Tokens",
|
|
211
278
|
a.tokens !== undefined ? String(a.tokens) : "—",
|
package/src/core/config.ts
CHANGED
|
@@ -146,6 +146,16 @@ export type HoundsmanHunt = {
|
|
|
146
146
|
timeoutMs?: number;
|
|
147
147
|
/** Tentativas extras quando a saída do watchdog é inválida (default: 1). */
|
|
148
148
|
retries?: number;
|
|
149
|
+
/**
|
|
150
|
+
* Passada extra por watchdog para cobrir fluxos/invariantes/regras que a
|
|
151
|
+
* primeira resposta não verificou (default: ligada).
|
|
152
|
+
*/
|
|
153
|
+
checklist?: boolean;
|
|
154
|
+
/**
|
|
155
|
+
* Passada adversarial do `general` nas 5 classes (credencial/estado,
|
|
156
|
+
* capacidade, identificador, idempotência, contrato). Default: ligada.
|
|
157
|
+
*/
|
|
158
|
+
secondOpinion?: boolean;
|
|
149
159
|
/** Caminho explícito do binário (`HOUNDSMAN_MAKI_PATH` tem precedência). */
|
|
150
160
|
binaryPath?: string;
|
|
151
161
|
};
|