ai-dev-harness 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COLLEAGUE_TRIAL_GUIDE.md +44 -16
- package/ONLINE_INSTALL_GUIDE.md +458 -0
- package/README.md +157 -22
- package/dist/agent.js +38 -19
- package/dist/approval.js +64 -0
- package/dist/audit.js +1 -0
- package/dist/checkpoint.js +83 -0
- package/dist/cli.js +322 -81
- package/dist/config.js +39 -1
- package/dist/context.js +14 -5
- package/dist/contextScore.js +44 -0
- package/dist/diagnostics.js +58 -0
- package/dist/doctor.js +100 -0
- package/dist/feedback.js +5 -0
- package/dist/integration.js +62 -2
- package/dist/loop.js +116 -0
- package/dist/mcp.js +1 -1
- package/dist/metrics.js +91 -0
- package/dist/replay.js +27 -0
- package/dist/runPaths.js +30 -0
- package/dist/summarize.js +139 -11
- package/package.json +3 -2
- package/scripts/read-evidence.ps1 +37 -1
- package/scripts/run-harness.ps1 +1 -1
- package/templates/default-project/TOOLS.md +6 -3
- package/templates/default-project/skills/add_feature/context.yaml +18 -15
- package/templates/default-project/skills/fix_bug/context.yaml +18 -15
- package/templates/default-project/skills/update_docs/context.yaml +16 -12
- package/templates/default-project/skills/write_tests/context.yaml +17 -14
- package/templates/integrations/claude/SKILL.md +47 -11
- package/templates/integrations/codex/AGENTS_BLOCK.md +42 -11
- package/templates/integrations/cursor/ai-dev-harness.mdc +55 -0
- package/templates/integrations/gemini/GEMINI_BLOCK.md +46 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { writeFile } from "node:fs/promises";
|
|
2
|
+
function hasValueAfter(label, output) {
|
|
3
|
+
const escaped = label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
4
|
+
const match = output.match(new RegExp(`${escaped}\\s*[::]?\\s*([^\\n]+)`, "i"));
|
|
5
|
+
if (!match)
|
|
6
|
+
return false;
|
|
7
|
+
const value = match[1].trim();
|
|
8
|
+
return Boolean(value && !/^[-(]?\s*(none|n\/a|unknown|无|没有|未使用|缺失|不适用)/i.test(value));
|
|
9
|
+
}
|
|
10
|
+
function hasAnyValueAfter(labels, output) {
|
|
11
|
+
return labels.some((label) => hasValueAfter(label, output));
|
|
12
|
+
}
|
|
13
|
+
function hasAnyPattern(patterns, output) {
|
|
14
|
+
return patterns.some((pattern) => pattern.test(output));
|
|
15
|
+
}
|
|
16
|
+
export function createContextScore(output) {
|
|
17
|
+
const requiredFields = {
|
|
18
|
+
codegraphToolsUsed: hasAnyValueAfter(["CodeGraph tools used", "CodeGraph 工具", "CodeGraph 使用工具", "代码图谱工具"], output) ||
|
|
19
|
+
hasAnyPattern([/codegraph[_-]?\w+/i, /codegraph_explore/i], output),
|
|
20
|
+
blastRadiusSummary: hasAnyValueAfter(["Blast-radius summary", "Blast radius summary", "影响范围", "影响面", "变更影响范围"], output) ||
|
|
21
|
+
hasAnyPattern([/blast[_ -]?radius/i, /影响范围\s*[::]\s*\S+/i], output),
|
|
22
|
+
knowledgeConceptIds: hasAnyValueAfter(["Candidate concept_ids", "Concept IDs", "候选 concept_ids", "知识库 concept_ids", "知识概念"], output) ||
|
|
23
|
+
hasAnyPattern([/concept[_ -]?ids?\s*[::]\s*\S+/i, /\/[^ \n]+\/(overview|features|metrics|api|playbooks|references)\//i], output),
|
|
24
|
+
missingContextDeclared: hasAnyValueAfter(["Missing context", "缺失上下文", "上下文缺口", "未确认上下文"], output) ||
|
|
25
|
+
hasAnyPattern([/Missing context\s*[::]/i, /缺失上下文\s*[::]/i, /上下文缺口\s*[::]/i], output)
|
|
26
|
+
};
|
|
27
|
+
const missing = Object.entries(requiredFields)
|
|
28
|
+
.filter(([, present]) => !present)
|
|
29
|
+
.map(([key]) => key);
|
|
30
|
+
const presentCount = Object.values(requiredFields).filter(Boolean).length;
|
|
31
|
+
return {
|
|
32
|
+
status: presentCount >= 4 ? "strong" : presentCount >= 2 ? "acceptable" : "weak",
|
|
33
|
+
requiredFields,
|
|
34
|
+
missing,
|
|
35
|
+
recommendation: missing.length
|
|
36
|
+
? `下一轮请补充以下上下文证据:${missing.join(", ")}。`
|
|
37
|
+
: "上下文证据结构完整。"
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
export async function writeContextScore(path, output) {
|
|
41
|
+
const score = createContextScore(output);
|
|
42
|
+
await writeFile(path, `${JSON.stringify(score, null, 2)}\n`, "utf8");
|
|
43
|
+
return score;
|
|
44
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { writeFile } from "node:fs/promises";
|
|
2
|
+
import { truncate } from "./utils.js";
|
|
3
|
+
const FILE_PATTERN = /(?:^|\s)([A-Za-z]:[\\/][^\s:]+|\b[\w./\\-]+\.(?:ts|tsx|js|jsx|json|md|yaml|yml|py|java|cc|cpp|h|hpp|cs|go|rs))(?::(\d+))?/g;
|
|
4
|
+
function uniq(values) {
|
|
5
|
+
return [...new Set(values.filter(Boolean))].slice(0, 20);
|
|
6
|
+
}
|
|
7
|
+
function errorLines(log) {
|
|
8
|
+
return log
|
|
9
|
+
.split(/\r?\n/)
|
|
10
|
+
.filter((line) => /error|failed|failure|exception|assert|not ok|FAIL|Error:/i.test(line))
|
|
11
|
+
.slice(0, 30)
|
|
12
|
+
.map((line) => truncate(line, 500));
|
|
13
|
+
}
|
|
14
|
+
function suspectedFiles(log) {
|
|
15
|
+
const files = [];
|
|
16
|
+
for (const match of log.matchAll(FILE_PATTERN))
|
|
17
|
+
files.push(match[1].replace(/\\/g, "/"));
|
|
18
|
+
return uniq(files);
|
|
19
|
+
}
|
|
20
|
+
function testNames(log) {
|
|
21
|
+
const names = [];
|
|
22
|
+
for (const line of log.split(/\r?\n/)) {
|
|
23
|
+
const subtest = line.match(/# Subtest:\s*(.+)$/);
|
|
24
|
+
if (subtest)
|
|
25
|
+
names.push(subtest[1].trim());
|
|
26
|
+
const jest = line.match(/(?:FAIL|✕|×)\s+(.+)$/);
|
|
27
|
+
if (jest)
|
|
28
|
+
names.push(jest[1].trim());
|
|
29
|
+
}
|
|
30
|
+
return uniq(names);
|
|
31
|
+
}
|
|
32
|
+
export function createVerifyDiagnostics(verify) {
|
|
33
|
+
const failedCommands = verify.commands
|
|
34
|
+
.filter((command) => command.exitCode !== 0)
|
|
35
|
+
.map((command) => {
|
|
36
|
+
const log = `${command.stdout}\n${command.stderr}`;
|
|
37
|
+
const errors = errorLines(log);
|
|
38
|
+
return {
|
|
39
|
+
name: command.name,
|
|
40
|
+
command: command.command,
|
|
41
|
+
exitCode: command.exitCode,
|
|
42
|
+
failureSummary: errors[0] ?? truncate(log.trim() || "验证命令失败但没有输出。", 1000),
|
|
43
|
+
errorLines: errors,
|
|
44
|
+
suspectedFiles: suspectedFiles(log),
|
|
45
|
+
testNames: testNames(log),
|
|
46
|
+
rawLogRef: "verify.json"
|
|
47
|
+
};
|
|
48
|
+
});
|
|
49
|
+
return {
|
|
50
|
+
status: verify.status,
|
|
51
|
+
failedCommands
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
export async function writeVerifyDiagnostics(path, verify) {
|
|
55
|
+
const diagnostics = createVerifyDiagnostics(verify);
|
|
56
|
+
await writeFile(path, `${JSON.stringify(diagnostics, null, 2)}\n`, "utf8");
|
|
57
|
+
return diagnostics;
|
|
58
|
+
}
|
package/dist/doctor.js
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { git, pathExists, runCommand } from "./utils.js";
|
|
3
|
+
const agentLabels = ["codex", "claude", "cursor", "gemini"];
|
|
4
|
+
function commandHead(command) {
|
|
5
|
+
const trimmed = command.trim();
|
|
6
|
+
const quoted = trimmed.match(/^"([^"]+)"/);
|
|
7
|
+
if (quoted)
|
|
8
|
+
return quoted[1];
|
|
9
|
+
return trimmed.split(/\s+/)[0] ?? "";
|
|
10
|
+
}
|
|
11
|
+
async function commandAvailable(command, cwd) {
|
|
12
|
+
const head = commandHead(command);
|
|
13
|
+
if (!head)
|
|
14
|
+
return false;
|
|
15
|
+
const probe = process.platform === "win32" ? `where ${head}` : `command -v ${head}`;
|
|
16
|
+
const result = await runCommand(probe, cwd, undefined, 5000);
|
|
17
|
+
return result.exitCode === 0;
|
|
18
|
+
}
|
|
19
|
+
function renderStatus(status) {
|
|
20
|
+
if (status === "ok")
|
|
21
|
+
return "OK";
|
|
22
|
+
if (status === "warn")
|
|
23
|
+
return "WARN";
|
|
24
|
+
return "FAIL";
|
|
25
|
+
}
|
|
26
|
+
export async function runDoctor(cwd, config) {
|
|
27
|
+
const checks = [];
|
|
28
|
+
const isGitRepo = Boolean(await git(cwd, "rev-parse --show-toplevel"));
|
|
29
|
+
checks.push({
|
|
30
|
+
name: "Git 仓库",
|
|
31
|
+
status: isGitRepo ? "ok" : "fail",
|
|
32
|
+
detail: isGitRepo ? "当前目录是 Git 仓库。" : "当前目录不是 Git 仓库,请在业务仓库根目录运行。"
|
|
33
|
+
});
|
|
34
|
+
const gitStatus = isGitRepo ? await git(cwd, "status --short") : "";
|
|
35
|
+
checks.push({
|
|
36
|
+
name: "Git 工作区",
|
|
37
|
+
status: gitStatus ? "warn" : "ok",
|
|
38
|
+
detail: gitStatus ? "当前存在未提交改动,restore 会拒绝自动恢复运行前已有改动。" : "当前工作区 clean。"
|
|
39
|
+
});
|
|
40
|
+
checks.push({
|
|
41
|
+
name: "harness.yaml",
|
|
42
|
+
status: (await pathExists(path.join(cwd, "harness.yaml"))) ? "ok" : "fail",
|
|
43
|
+
detail: (await pathExists(path.join(cwd, "harness.yaml"))) ? "配置文件存在。" : "未找到 harness.yaml,请先运行 harness init。"
|
|
44
|
+
});
|
|
45
|
+
const skillNames = ["fix_bug", "add_feature", "write_tests", "update_docs"];
|
|
46
|
+
const missingSkills = [];
|
|
47
|
+
for (const skill of skillNames) {
|
|
48
|
+
if (!(await pathExists(path.join(cwd, "skills", skill, "SKILL.md"))))
|
|
49
|
+
missingSkills.push(skill);
|
|
50
|
+
}
|
|
51
|
+
checks.push({
|
|
52
|
+
name: "内置 Skills",
|
|
53
|
+
status: missingSkills.length ? "fail" : "ok",
|
|
54
|
+
detail: missingSkills.length ? `缺少 Skill:${missingSkills.join(", ")}。` : "内置 Skill 文件完整。"
|
|
55
|
+
});
|
|
56
|
+
checks.push({
|
|
57
|
+
name: "对话集成脚本",
|
|
58
|
+
status: (await pathExists(path.join(cwd, ".harness", "ai-dev-harness", "scripts", "run-harness.ps1"))) ? "ok" : "warn",
|
|
59
|
+
detail: (await pathExists(path.join(cwd, ".harness", "ai-dev-harness", "scripts", "run-harness.ps1")))
|
|
60
|
+
? "run-harness.ps1 已安装。"
|
|
61
|
+
: "未找到对话集成脚本;CLI 仍可用,可运行 harness install-integration --agent all。"
|
|
62
|
+
});
|
|
63
|
+
for (const agent of agentLabels) {
|
|
64
|
+
const command = config.agents[agent].command;
|
|
65
|
+
const available = await commandAvailable(command, cwd);
|
|
66
|
+
checks.push({
|
|
67
|
+
name: `${agent} 执行器`,
|
|
68
|
+
status: available ? "ok" : "warn",
|
|
69
|
+
detail: available ? `命令可用:${command}` : `命令不可用或不在 PATH:${command}。不用该 agent 可忽略。`
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
const mcpUrls = [
|
|
73
|
+
["CodeGraph MCP", config.mcp.codegraph.url],
|
|
74
|
+
["Knowledge Graph MCP", config.mcp.knowledgeGraph.url]
|
|
75
|
+
];
|
|
76
|
+
for (const [name, url] of mcpUrls) {
|
|
77
|
+
const isDefaultLocal = /localhost:733[12]/.test(url);
|
|
78
|
+
checks.push({
|
|
79
|
+
name,
|
|
80
|
+
status: url && !isDefaultLocal ? "ok" : "warn",
|
|
81
|
+
detail: url
|
|
82
|
+
? isDefaultLocal
|
|
83
|
+
? `当前仍是默认地址:${url}。本地试点请改成真实 MCP source metadata。`
|
|
84
|
+
: `已配置地址:${url}。Harness 只记录 metadata,不主动连接。`
|
|
85
|
+
: "未配置 MCP 地址。"
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
const verifyCommands = config.verify.defaultCommands;
|
|
89
|
+
const onlyGitStatus = verifyCommands.length === 1 && verifyCommands[0].command === "git status --short";
|
|
90
|
+
checks.push({
|
|
91
|
+
name: "验证命令强度",
|
|
92
|
+
status: onlyGitStatus ? "warn" : "ok",
|
|
93
|
+
detail: onlyGitStatus
|
|
94
|
+
? "当前只有 git status,适合冒烟但验证较弱;试点真实任务建议补充 test/lint/typecheck。"
|
|
95
|
+
: `已配置 ${verifyCommands.length} 条默认验证命令。`
|
|
96
|
+
});
|
|
97
|
+
const failed = checks.some((check) => check.status === "fail");
|
|
98
|
+
const text = ["AI Dev Harness Doctor", ...checks.map((check) => `- [${renderStatus(check.status)}] ${check.name}: ${check.detail}`)].join("\n");
|
|
99
|
+
return { checks, failed, text };
|
|
100
|
+
}
|
package/dist/feedback.js
ADDED
package/dist/integration.js
CHANGED
|
@@ -10,7 +10,7 @@ export async function installIntegration(options) {
|
|
|
10
10
|
created.push(...scriptResult.created);
|
|
11
11
|
updated.push(...scriptResult.updated);
|
|
12
12
|
skipped.push(...scriptResult.skipped);
|
|
13
|
-
if (options.agent === "claude" || options.agent === "both") {
|
|
13
|
+
if (options.agent === "claude" || options.agent === "both" || options.agent === "all") {
|
|
14
14
|
const result = await installClaudeSkill(options.targetDir, options.harnessCommand, options.packageRoot);
|
|
15
15
|
if (result === "created")
|
|
16
16
|
created.push(".claude/skills/ai-dev-harness/SKILL.md");
|
|
@@ -19,7 +19,7 @@ export async function installIntegration(options) {
|
|
|
19
19
|
else
|
|
20
20
|
skipped.push(".claude/skills/ai-dev-harness/SKILL.md");
|
|
21
21
|
}
|
|
22
|
-
if (options.agent === "codex" || options.agent === "both") {
|
|
22
|
+
if (options.agent === "codex" || options.agent === "both" || options.agent === "all") {
|
|
23
23
|
const result = await installCodexRules(options.targetDir, options.harnessCommand, options.packageRoot);
|
|
24
24
|
if (result === "created")
|
|
25
25
|
created.push("AGENTS.md");
|
|
@@ -28,6 +28,24 @@ export async function installIntegration(options) {
|
|
|
28
28
|
else
|
|
29
29
|
skipped.push("AGENTS.md");
|
|
30
30
|
}
|
|
31
|
+
if (options.agent === "cursor" || options.agent === "all") {
|
|
32
|
+
const result = await installCursorRules(options.targetDir, options.harnessCommand, options.packageRoot);
|
|
33
|
+
if (result === "created")
|
|
34
|
+
created.push(".cursor/rules/ai-dev-harness.mdc");
|
|
35
|
+
else if (result === "updated")
|
|
36
|
+
updated.push(".cursor/rules/ai-dev-harness.mdc");
|
|
37
|
+
else
|
|
38
|
+
skipped.push(".cursor/rules/ai-dev-harness.mdc");
|
|
39
|
+
}
|
|
40
|
+
if (options.agent === "gemini" || options.agent === "all") {
|
|
41
|
+
const result = await installGeminiRules(options.targetDir, options.harnessCommand, options.packageRoot);
|
|
42
|
+
if (result === "created")
|
|
43
|
+
created.push("GEMINI.md");
|
|
44
|
+
else if (result === "updated")
|
|
45
|
+
updated.push("GEMINI.md");
|
|
46
|
+
else
|
|
47
|
+
skipped.push("GEMINI.md");
|
|
48
|
+
}
|
|
31
49
|
return { created, updated, skipped };
|
|
32
50
|
}
|
|
33
51
|
async function installScriptAssets(targetDir, packageRoot, harnessCliPath) {
|
|
@@ -92,6 +110,38 @@ async function installCodexRules(targetDir, harnessCommand, packageRoot) {
|
|
|
92
110
|
await appendFile(agentsPath, `\n\n${block}`, "utf8");
|
|
93
111
|
return "updated";
|
|
94
112
|
}
|
|
113
|
+
async function installCursorRules(targetDir, harnessCommand, packageRoot) {
|
|
114
|
+
const rulesPath = path.join(targetDir, ".cursor", "rules", "ai-dev-harness.mdc");
|
|
115
|
+
await mkdir(path.dirname(rulesPath), { recursive: true });
|
|
116
|
+
const next = await cursorRules(harnessCommand, packageRoot);
|
|
117
|
+
if (await pathExists(rulesPath)) {
|
|
118
|
+
const current = await readFile(rulesPath, "utf8");
|
|
119
|
+
if (current === next)
|
|
120
|
+
return "skipped";
|
|
121
|
+
await writeFile(rulesPath, next, "utf8");
|
|
122
|
+
return "updated";
|
|
123
|
+
}
|
|
124
|
+
await writeFile(rulesPath, next, "utf8");
|
|
125
|
+
return "created";
|
|
126
|
+
}
|
|
127
|
+
async function installGeminiRules(targetDir, harnessCommand, packageRoot) {
|
|
128
|
+
const geminiPath = path.join(targetDir, "GEMINI.md");
|
|
129
|
+
const block = await geminiBlock(harnessCommand, packageRoot);
|
|
130
|
+
if (!(await pathExists(geminiPath))) {
|
|
131
|
+
await writeFile(geminiPath, `# Project Agent Rules\n\n${block}`, "utf8");
|
|
132
|
+
return "created";
|
|
133
|
+
}
|
|
134
|
+
const current = await readFile(geminiPath, "utf8");
|
|
135
|
+
if (current.includes("## AI Dev Harness")) {
|
|
136
|
+
const next = current.replace(/(?:\r?\n){0,2}## AI Dev Harness[\s\S]*$/m, `\n\n${block}`);
|
|
137
|
+
if (next === current)
|
|
138
|
+
return "skipped";
|
|
139
|
+
await writeFile(geminiPath, next, "utf8");
|
|
140
|
+
return "updated";
|
|
141
|
+
}
|
|
142
|
+
await appendFile(geminiPath, `\n\n${block}`, "utf8");
|
|
143
|
+
return "updated";
|
|
144
|
+
}
|
|
95
145
|
async function claudeSkill(harnessCommand, packageRoot) {
|
|
96
146
|
return renderTemplate(await readTemplateFile(path.join("integrations", "claude", "SKILL.md"), packageRoot), {
|
|
97
147
|
HARNESS_COMMAND: harnessCommand
|
|
@@ -102,3 +152,13 @@ async function codexBlock(harnessCommand, packageRoot) {
|
|
|
102
152
|
HARNESS_COMMAND: harnessCommand
|
|
103
153
|
});
|
|
104
154
|
}
|
|
155
|
+
async function cursorRules(harnessCommand, packageRoot) {
|
|
156
|
+
return renderTemplate(await readTemplateFile(path.join("integrations", "cursor", "ai-dev-harness.mdc"), packageRoot), {
|
|
157
|
+
HARNESS_COMMAND: harnessCommand
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
async function geminiBlock(harnessCommand, packageRoot) {
|
|
161
|
+
return renderTemplate(await readTemplateFile(path.join("integrations", "gemini", "GEMINI_BLOCK.md"), packageRoot), {
|
|
162
|
+
HARNESS_COMMAND: harnessCommand
|
|
163
|
+
});
|
|
164
|
+
}
|
package/dist/loop.js
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { writeFile } from "node:fs/promises";
|
|
2
|
+
export function evaluateIteration(options) {
|
|
3
|
+
let failureType = "none";
|
|
4
|
+
let reason = "Agent 执行成功、输出契约通过、验证通过。";
|
|
5
|
+
if (options.agentResult.status !== "success") {
|
|
6
|
+
failureType = "agent_failed";
|
|
7
|
+
reason = `Agent 执行失败:${options.agentResult.error ?? `exitCode=${options.agentResult.exitCode}`}`;
|
|
8
|
+
}
|
|
9
|
+
else if (options.contractCheck.status !== "passed") {
|
|
10
|
+
failureType = "contract_failed";
|
|
11
|
+
reason = `Agent 输出缺少必需章节:${options.contractCheck.missingSections.join(", ") || "unknown"}`;
|
|
12
|
+
}
|
|
13
|
+
else if (options.approval?.required) {
|
|
14
|
+
failureType = "approval_required";
|
|
15
|
+
reason = `触发审批门禁:${options.approval.reasons.map((item) => item.type).join(", ")}`;
|
|
16
|
+
}
|
|
17
|
+
else if (options.contextScore?.status === "weak") {
|
|
18
|
+
failureType = "context_weak";
|
|
19
|
+
reason = `上下文证据不足:${options.contextScore.missing.join(", ") || "unknown"}`;
|
|
20
|
+
}
|
|
21
|
+
else if (options.verifyResult.status !== "passed") {
|
|
22
|
+
failureType = "verify_failed";
|
|
23
|
+
const failed = options.verifyResult.commands.filter((command) => command.exitCode !== 0).map((command) => command.name);
|
|
24
|
+
const diagnostic = options.verifyDiagnostics?.failedCommands[0]?.failureSummary;
|
|
25
|
+
reason = `验证未通过:${failed.join(", ") || options.verifyResult.status}${diagnostic ? `;${diagnostic}` : ""}`;
|
|
26
|
+
}
|
|
27
|
+
if (failureType === "none") {
|
|
28
|
+
return {
|
|
29
|
+
iteration: options.iteration,
|
|
30
|
+
status: "passed",
|
|
31
|
+
failureType,
|
|
32
|
+
reason,
|
|
33
|
+
canRework: false,
|
|
34
|
+
nextAction: "summarize",
|
|
35
|
+
signals: {
|
|
36
|
+
agentStatus: options.agentResult.status,
|
|
37
|
+
agentExitCode: options.agentResult.exitCode,
|
|
38
|
+
contractStatus: options.contractCheck.status,
|
|
39
|
+
verifyStatus: options.verifyResult.status,
|
|
40
|
+
contextScoreStatus: options.contextScore?.status,
|
|
41
|
+
approvalRequired: Boolean(options.approval?.required),
|
|
42
|
+
changedFiles: options.agentResult.changedFiles
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
if (options.previous?.failureType === failureType) {
|
|
47
|
+
failureType = "same_failure_repeated";
|
|
48
|
+
reason = `连续两轮出现相同失败,停止自循环:${options.previous.failureType}`;
|
|
49
|
+
}
|
|
50
|
+
else if (options.iteration >= options.maxIterations) {
|
|
51
|
+
failureType = "max_iterations_reached";
|
|
52
|
+
reason = `达到最大自循环轮次 ${options.maxIterations},停止。最后失败原因:${reason}`;
|
|
53
|
+
}
|
|
54
|
+
const canRework = failureType !== "same_failure_repeated" && failureType !== "max_iterations_reached" && failureType !== "approval_required";
|
|
55
|
+
return {
|
|
56
|
+
iteration: options.iteration,
|
|
57
|
+
status: canRework ? "rework_needed" : "stopped",
|
|
58
|
+
failureType,
|
|
59
|
+
reason,
|
|
60
|
+
canRework,
|
|
61
|
+
nextAction: canRework ? "rework" : "stop",
|
|
62
|
+
signals: {
|
|
63
|
+
agentStatus: options.agentResult.status,
|
|
64
|
+
agentExitCode: options.agentResult.exitCode,
|
|
65
|
+
contractStatus: options.contractCheck.status,
|
|
66
|
+
verifyStatus: options.verifyResult.status,
|
|
67
|
+
contextScoreStatus: options.contextScore?.status,
|
|
68
|
+
approvalRequired: Boolean(options.approval?.required),
|
|
69
|
+
changedFiles: options.agentResult.changedFiles
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
export function renderReworkFeedback(evaluation) {
|
|
74
|
+
return `上一轮自纠偏评估:
|
|
75
|
+
- iteration: ${evaluation.iteration}
|
|
76
|
+
- failure_type: ${evaluation.failureType}
|
|
77
|
+
- reason: ${evaluation.reason}
|
|
78
|
+
- next_action: ${evaluation.nextAction}
|
|
79
|
+
|
|
80
|
+
请只围绕上述失败原因做最小 rework:
|
|
81
|
+
1. 重新检查目标、上下文和计划是否偏离。
|
|
82
|
+
2. 优先修复验证失败、输出契约缺失、上下文证据不足或 Agent 执行错误。
|
|
83
|
+
3. 不扩大任务范围,不做无关重构。
|
|
84
|
+
4. 如果需要扩大范围、改公共接口、或目标不明确,请停止并说明需要人工确认。`;
|
|
85
|
+
}
|
|
86
|
+
export async function writeLoopArtifacts(options) {
|
|
87
|
+
await writeFile(options.reviewPath, `${JSON.stringify(options.review, null, 2)}\n`, "utf8");
|
|
88
|
+
const latest = options.review.iterations.at(-1);
|
|
89
|
+
const report = `# 自纠偏循环报告
|
|
90
|
+
|
|
91
|
+
## 结果
|
|
92
|
+
- 状态: ${options.review.status}
|
|
93
|
+
- 最大轮次: ${options.review.maxIterations}
|
|
94
|
+
- 实际轮次: ${options.review.iterations.length}
|
|
95
|
+
- 最终原因: ${latest?.reason ?? "无"}
|
|
96
|
+
|
|
97
|
+
## 轮次明细
|
|
98
|
+
${options.review.iterations
|
|
99
|
+
.map((item) => `### Iteration ${item.iteration}
|
|
100
|
+
- 状态: ${item.status}
|
|
101
|
+
- 失败类型: ${item.failureType}
|
|
102
|
+
- 原因: ${item.reason}
|
|
103
|
+
- 下一步: ${item.nextAction}
|
|
104
|
+
- Agent: ${item.signals.agentStatus} (${item.signals.agentExitCode ?? "n/a"})
|
|
105
|
+
- 契约: ${item.signals.contractStatus}
|
|
106
|
+
- 验证: ${item.signals.verifyStatus}
|
|
107
|
+
- 修改文件: ${item.signals.changedFiles.join(", ") || "无"}`)
|
|
108
|
+
.join("\n\n")}
|
|
109
|
+
|
|
110
|
+
## 人工介入建议
|
|
111
|
+
- 如果状态为 stopped,请先看 \`execution.log\`、\`verify.json\`、\`agent_contract.json\` 和本报告中的失败类型。
|
|
112
|
+
- 如果失败类型是 \`same_failure_repeated\`,优先补充上下文、调整 Skill 或让用户确认目标。
|
|
113
|
+
- 如果失败类型是 \`max_iterations_reached\`,不要继续盲目重跑,先做人工评审。
|
|
114
|
+
`;
|
|
115
|
+
await writeFile(options.reportPath, report, "utf8");
|
|
116
|
+
}
|
package/dist/mcp.js
CHANGED
|
@@ -9,7 +9,7 @@ function endpointSummary(endpoint) {
|
|
|
9
9
|
function agentDrivenResult(source, endpoint, skill) {
|
|
10
10
|
const request = {
|
|
11
11
|
mode: "agent-driven",
|
|
12
|
-
instruction: "
|
|
12
|
+
instruction: "The current execution agent should call this MCP server with its own configured MCP client before invoking Harness. Harness records the configured source but does not implement MCP transport.",
|
|
13
13
|
endpoint: endpointSummary(endpoint),
|
|
14
14
|
suggestedQueries: skill.contextQueries
|
|
15
15
|
};
|
package/dist/metrics.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { readdir, readFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { pathExists } from "./utils.js";
|
|
4
|
+
async function readJson(filePath) {
|
|
5
|
+
try {
|
|
6
|
+
return JSON.parse(await readFile(filePath, "utf8"));
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
function increment(map, key) {
|
|
13
|
+
map[key] = (map[key] ?? 0) + 1;
|
|
14
|
+
}
|
|
15
|
+
export async function collectRunMetrics(cwd, config) {
|
|
16
|
+
const runsRoot = path.join(cwd, config.audit.path);
|
|
17
|
+
const entries = (await pathExists(runsRoot)) ? await readdir(runsRoot, { withFileTypes: true }) : [];
|
|
18
|
+
const runs = [];
|
|
19
|
+
for (const entry of entries.filter((item) => item.isDirectory())) {
|
|
20
|
+
const runDir = path.join(runsRoot, entry.name);
|
|
21
|
+
const intake = await readJson(path.join(runDir, "intake.json"));
|
|
22
|
+
const review = await readJson(path.join(runDir, "loop_review.json"));
|
|
23
|
+
const verify = await readJson(path.join(runDir, "verify.json"));
|
|
24
|
+
const last = review?.iterations?.at(-1);
|
|
25
|
+
runs.push({
|
|
26
|
+
runId: entry.name,
|
|
27
|
+
skill: intake?.skill,
|
|
28
|
+
agent: intake?.agent,
|
|
29
|
+
status: review?.status ?? "unknown",
|
|
30
|
+
iterations: review?.iterations?.length ?? 0,
|
|
31
|
+
failureType: last?.failureType ?? "unknown",
|
|
32
|
+
contractStatus: last?.signals?.contractStatus,
|
|
33
|
+
verifyStatus: last?.signals?.verifyStatus,
|
|
34
|
+
contextScore: last?.signals?.contextScoreStatus,
|
|
35
|
+
approvalRequired: Boolean(last?.signals?.approvalRequired),
|
|
36
|
+
failedVerifyCommands: verify?.commands?.filter((command) => command.exitCode !== 0).map((command) => command.name) ?? []
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
const failureTypes = {};
|
|
40
|
+
const skillStatus = {};
|
|
41
|
+
const agentStatus = {};
|
|
42
|
+
const verifyFailures = {};
|
|
43
|
+
const contextScores = {};
|
|
44
|
+
for (const run of runs) {
|
|
45
|
+
increment(failureTypes, run.failureType);
|
|
46
|
+
if (run.skill)
|
|
47
|
+
increment(skillStatus, `${run.skill}:${run.status}`);
|
|
48
|
+
if (run.agent)
|
|
49
|
+
increment(agentStatus, `${run.agent}:${run.status}`);
|
|
50
|
+
for (const command of run.failedVerifyCommands)
|
|
51
|
+
increment(verifyFailures, command);
|
|
52
|
+
if (run.contextScore)
|
|
53
|
+
increment(contextScores, run.contextScore);
|
|
54
|
+
}
|
|
55
|
+
const total = runs.length;
|
|
56
|
+
const passed = runs.filter((run) => run.status === "passed").length;
|
|
57
|
+
const firstPass = runs.filter((run) => run.status === "passed" && run.iterations === 1).length;
|
|
58
|
+
const contractPassed = runs.filter((run) => run.contractStatus === "passed").length;
|
|
59
|
+
const averageIterations = total ? runs.reduce((sum, run) => sum + run.iterations, 0) / total : 0;
|
|
60
|
+
return {
|
|
61
|
+
runs,
|
|
62
|
+
summary: {
|
|
63
|
+
totalRuns: total,
|
|
64
|
+
successRate: total ? passed / total : 0,
|
|
65
|
+
firstPassRate: total ? firstPass / total : 0,
|
|
66
|
+
agentContractPassRate: total ? contractPassed / total : 0,
|
|
67
|
+
averageIterations,
|
|
68
|
+
failureTypes,
|
|
69
|
+
skillStatus,
|
|
70
|
+
agentStatus,
|
|
71
|
+
verifyFailureTopN: Object.fromEntries(Object.entries(verifyFailures).sort((a, b) => b[1] - a[1]).slice(0, 10)),
|
|
72
|
+
approvalGateCount: runs.filter((run) => run.approvalRequired).length,
|
|
73
|
+
contextScores
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
export function renderMetricsText(metrics) {
|
|
78
|
+
return `运行指标
|
|
79
|
+
- 总运行次数: ${metrics.summary.totalRuns}
|
|
80
|
+
- 成功率: ${Number(metrics.summary.successRate).toFixed(2)}
|
|
81
|
+
- 一次通过率: ${Number(metrics.summary.firstPassRate).toFixed(2)}
|
|
82
|
+
- Agent 契约通过率: ${Number(metrics.summary.agentContractPassRate).toFixed(2)}
|
|
83
|
+
- 平均循环轮次: ${Number(metrics.summary.averageIterations).toFixed(2)}
|
|
84
|
+
- 失败类型分布: ${JSON.stringify(metrics.summary.failureTypes)}
|
|
85
|
+
- Skill 状态: ${JSON.stringify(metrics.summary.skillStatus)}
|
|
86
|
+
- Agent 状态: ${JSON.stringify(metrics.summary.agentStatus)}
|
|
87
|
+
- 验证失败 Top N: ${JSON.stringify(metrics.summary.verifyFailureTopN)}
|
|
88
|
+
- Approval Gate 触发次数: ${metrics.summary.approvalGateCount}
|
|
89
|
+
- Context Score 分布: ${JSON.stringify(metrics.summary.contextScores)}
|
|
90
|
+
`;
|
|
91
|
+
}
|
package/dist/replay.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
export async function buildReplayPlan(runDir, overrideAgent, dry = false) {
|
|
4
|
+
const intake = JSON.parse(await readFile(path.join(runDir, "intake.json"), "utf8"));
|
|
5
|
+
const agent = overrideAgent ?? intake.agent;
|
|
6
|
+
if (agent !== "codex" && agent !== "claude" && agent !== "cursor" && agent !== "gemini") {
|
|
7
|
+
throw new Error(`历史运行记录中的 agent 无效:${agent}`);
|
|
8
|
+
}
|
|
9
|
+
if (!intake.task || !intake.skill)
|
|
10
|
+
throw new Error(`历史运行记录缺少 task 或 skill:${runDir}`);
|
|
11
|
+
return {
|
|
12
|
+
sourceRun: runDir,
|
|
13
|
+
task: intake.task,
|
|
14
|
+
skill: intake.skill,
|
|
15
|
+
agent,
|
|
16
|
+
dry
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export function renderReplayPlan(plan) {
|
|
20
|
+
return `Replay Plan
|
|
21
|
+
- Source run: ${plan.sourceRun}
|
|
22
|
+
- Task: ${plan.task}
|
|
23
|
+
- Skill: ${plan.skill}
|
|
24
|
+
- Agent: ${plan.agent}
|
|
25
|
+
- Dry run: ${plan.dry ? "yes" : "no"}
|
|
26
|
+
`;
|
|
27
|
+
}
|
package/dist/runPaths.js
CHANGED
|
@@ -15,8 +15,38 @@ export function makeRunPaths(cwd, config, task) {
|
|
|
15
15
|
agentContract: path.join(runDir, "agent_contract.json"),
|
|
16
16
|
gitDiff: path.join(runDir, "git_diff.patch"),
|
|
17
17
|
verify: path.join(runDir, "verify.json"),
|
|
18
|
+
verifyDiagnostics: path.join(runDir, "verify_diagnostics.json"),
|
|
19
|
+
contextScore: path.join(runDir, "context_score.json"),
|
|
20
|
+
approvalRequired: path.join(runDir, "approval_required.json"),
|
|
21
|
+
feedback: path.join(runDir, "feedback.jsonl"),
|
|
22
|
+
checkpoint: path.join(runDir, "checkpoint.json"),
|
|
23
|
+
preIterationDiff: path.join(runDir, "pre_iteration_diff.patch"),
|
|
24
|
+
postIterationDiff: path.join(runDir, "post_iteration_diff.patch"),
|
|
18
25
|
summary: path.join(runDir, "summary.md"),
|
|
19
26
|
persistSuggestions: path.join(runDir, "persist_suggestions.md"),
|
|
27
|
+
loopReview: path.join(runDir, "loop_review.json"),
|
|
28
|
+
loopReport: path.join(runDir, "loop_report.md"),
|
|
20
29
|
audit: path.join(runDir, "audit.jsonl")
|
|
21
30
|
};
|
|
22
31
|
}
|
|
32
|
+
export function makeIterationPaths(paths, iteration) {
|
|
33
|
+
const iterationDir = path.join(paths.runDir, `iteration-${iteration}`);
|
|
34
|
+
return {
|
|
35
|
+
iteration,
|
|
36
|
+
iterationDir,
|
|
37
|
+
contextPack: path.join(iterationDir, "context_pack.md"),
|
|
38
|
+
agentPrompt: path.join(iterationDir, "agent_prompt.md"),
|
|
39
|
+
plan: path.join(iterationDir, "plan.md"),
|
|
40
|
+
executionLog: path.join(iterationDir, "execution.log"),
|
|
41
|
+
agentContract: path.join(iterationDir, "agent_contract.json"),
|
|
42
|
+
gitDiff: path.join(iterationDir, "git_diff.patch"),
|
|
43
|
+
verify: path.join(iterationDir, "verify.json"),
|
|
44
|
+
verifyDiagnostics: path.join(iterationDir, "verify_diagnostics.json"),
|
|
45
|
+
contextScore: path.join(iterationDir, "context_score.json"),
|
|
46
|
+
approvalRequired: path.join(iterationDir, "approval_required.json"),
|
|
47
|
+
feedback: path.join(iterationDir, "feedback.jsonl"),
|
|
48
|
+
checkpoint: path.join(iterationDir, "checkpoint.json"),
|
|
49
|
+
preIterationDiff: path.join(iterationDir, "pre_iteration_diff.patch"),
|
|
50
|
+
postIterationDiff: path.join(iterationDir, "post_iteration_diff.patch")
|
|
51
|
+
};
|
|
52
|
+
}
|