ai-dev-harness 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COLLEAGUE_TRIAL_GUIDE.md +175 -0
- package/ONLINE_INSTALL_GUIDE.md +458 -0
- package/README.md +253 -54
- package/dist/agent.js +74 -49
- package/dist/approval.js +64 -0
- package/dist/audit.js +1 -0
- package/dist/checkpoint.js +83 -0
- package/dist/cli.js +326 -99
- package/dist/config.js +118 -9
- package/dist/context.js +62 -21
- package/dist/contextScore.js +44 -0
- package/dist/diagnostics.js +58 -0
- package/dist/doctor.js +100 -0
- package/dist/feedback.js +5 -0
- package/dist/init.js +8 -20
- package/dist/integration.js +84 -93
- package/dist/loop.js +116 -0
- package/dist/mcp.js +29 -112
- package/dist/metrics.js +91 -0
- package/dist/replay.js +27 -0
- package/dist/runPaths.js +31 -0
- package/dist/summarize.js +170 -39
- package/dist/templates.js +39 -215
- package/package.json +5 -2
- package/scripts/read-evidence.ps1 +45 -3
- package/scripts/run-harness.ps1 +9 -6
- package/templates/default-project/AGENTS.md +9 -0
- package/templates/default-project/SKILLS.md +13 -0
- package/templates/default-project/TOOLS.md +11 -0
- package/templates/default-project/skills/add_feature/SKILL.md +34 -0
- package/templates/default-project/skills/add_feature/context.yaml +33 -0
- package/templates/default-project/skills/add_feature/verify.yaml +2 -0
- package/templates/default-project/skills/fix_bug/SKILL.md +45 -0
- package/templates/default-project/skills/fix_bug/context.yaml +33 -0
- package/templates/default-project/skills/fix_bug/verify.yaml +2 -0
- package/templates/default-project/skills/update_docs/SKILL.md +28 -0
- package/templates/default-project/skills/update_docs/context.yaml +30 -0
- package/templates/default-project/skills/update_docs/verify.yaml +2 -0
- package/templates/default-project/skills/write_tests/SKILL.md +30 -0
- package/templates/default-project/skills/write_tests/context.yaml +32 -0
- package/templates/default-project/skills/write_tests/verify.yaml +2 -0
- package/templates/integrations/claude/SKILL.md +115 -0
- package/templates/integrations/codex/AGENTS_BLOCK.md +82 -0
- package/templates/integrations/cursor/ai-dev-harness.mdc +55 -0
- package/templates/integrations/gemini/GEMINI_BLOCK.md +46 -0
package/dist/integration.js
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
|
-
import { appendFile,
|
|
2
|
+
import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
|
+
import { readTemplateFile, renderTemplate } from "./templates.js";
|
|
3
4
|
import { pathExists } from "./utils.js";
|
|
4
5
|
export async function installIntegration(options) {
|
|
5
6
|
const created = [];
|
|
6
7
|
const updated = [];
|
|
7
8
|
const skipped = [];
|
|
8
|
-
const scriptResult = await installScriptAssets(options.targetDir, options.packageRoot ?? process.cwd());
|
|
9
|
+
const scriptResult = await installScriptAssets(options.targetDir, options.packageRoot ?? process.cwd(), options.harnessCliPath);
|
|
9
10
|
created.push(...scriptResult.created);
|
|
10
11
|
updated.push(...scriptResult.updated);
|
|
11
12
|
skipped.push(...scriptResult.skipped);
|
|
12
|
-
if (options.agent === "claude" || options.agent === "both") {
|
|
13
|
-
const result = await installClaudeSkill(options.targetDir, options.harnessCommand);
|
|
13
|
+
if (options.agent === "claude" || options.agent === "both" || options.agent === "all") {
|
|
14
|
+
const result = await installClaudeSkill(options.targetDir, options.harnessCommand, options.packageRoot);
|
|
14
15
|
if (result === "created")
|
|
15
16
|
created.push(".claude/skills/ai-dev-harness/SKILL.md");
|
|
16
17
|
else if (result === "updated")
|
|
@@ -18,8 +19,8 @@ export async function installIntegration(options) {
|
|
|
18
19
|
else
|
|
19
20
|
skipped.push(".claude/skills/ai-dev-harness/SKILL.md");
|
|
20
21
|
}
|
|
21
|
-
if (options.agent === "codex" || options.agent === "both") {
|
|
22
|
-
const result = await installCodexRules(options.targetDir, options.harnessCommand);
|
|
22
|
+
if (options.agent === "codex" || options.agent === "both" || options.agent === "all") {
|
|
23
|
+
const result = await installCodexRules(options.targetDir, options.harnessCommand, options.packageRoot);
|
|
23
24
|
if (result === "created")
|
|
24
25
|
created.push("AGENTS.md");
|
|
25
26
|
else if (result === "updated")
|
|
@@ -27,9 +28,27 @@ export async function installIntegration(options) {
|
|
|
27
28
|
else
|
|
28
29
|
skipped.push("AGENTS.md");
|
|
29
30
|
}
|
|
31
|
+
if (options.agent === "cursor" || options.agent === "all") {
|
|
32
|
+
const result = await installCursorRules(options.targetDir, options.harnessCommand, options.packageRoot);
|
|
33
|
+
if (result === "created")
|
|
34
|
+
created.push(".cursor/rules/ai-dev-harness.mdc");
|
|
35
|
+
else if (result === "updated")
|
|
36
|
+
updated.push(".cursor/rules/ai-dev-harness.mdc");
|
|
37
|
+
else
|
|
38
|
+
skipped.push(".cursor/rules/ai-dev-harness.mdc");
|
|
39
|
+
}
|
|
40
|
+
if (options.agent === "gemini" || options.agent === "all") {
|
|
41
|
+
const result = await installGeminiRules(options.targetDir, options.harnessCommand, options.packageRoot);
|
|
42
|
+
if (result === "created")
|
|
43
|
+
created.push("GEMINI.md");
|
|
44
|
+
else if (result === "updated")
|
|
45
|
+
updated.push("GEMINI.md");
|
|
46
|
+
else
|
|
47
|
+
skipped.push("GEMINI.md");
|
|
48
|
+
}
|
|
30
49
|
return { created, updated, skipped };
|
|
31
50
|
}
|
|
32
|
-
async function installScriptAssets(targetDir, packageRoot) {
|
|
51
|
+
async function installScriptAssets(targetDir, packageRoot, harnessCliPath) {
|
|
33
52
|
const created = [];
|
|
34
53
|
const updated = [];
|
|
35
54
|
const skipped = [];
|
|
@@ -42,7 +61,8 @@ async function installScriptAssets(targetDir, packageRoot) {
|
|
|
42
61
|
const rel = path.join(".harness", "ai-dev-harness", "scripts", asset);
|
|
43
62
|
if (!(await pathExists(source)))
|
|
44
63
|
continue;
|
|
45
|
-
const
|
|
64
|
+
const raw = await readFile(source, "utf8");
|
|
65
|
+
const next = renderTemplate(raw, { HARNESS_CLI_PATH: harnessCliPath ?? "{{HARNESS_CLI_PATH}}" });
|
|
46
66
|
if (await pathExists(target)) {
|
|
47
67
|
const current = await readFile(target, "utf8");
|
|
48
68
|
if (current === next) {
|
|
@@ -53,28 +73,28 @@ async function installScriptAssets(targetDir, packageRoot) {
|
|
|
53
73
|
updated.push(rel);
|
|
54
74
|
continue;
|
|
55
75
|
}
|
|
56
|
-
await
|
|
76
|
+
await writeFile(target, next, "utf8");
|
|
57
77
|
created.push(rel);
|
|
58
78
|
}
|
|
59
79
|
return { created, updated, skipped };
|
|
60
80
|
}
|
|
61
|
-
async function installClaudeSkill(targetDir, harnessCommand) {
|
|
81
|
+
async function installClaudeSkill(targetDir, harnessCommand, packageRoot) {
|
|
62
82
|
const skillPath = path.join(targetDir, ".claude", "skills", "ai-dev-harness", "SKILL.md");
|
|
63
83
|
await mkdir(path.dirname(skillPath), { recursive: true });
|
|
64
84
|
if (await pathExists(skillPath)) {
|
|
65
85
|
const current = await readFile(skillPath, "utf8");
|
|
66
|
-
const next = claudeSkill(harnessCommand);
|
|
86
|
+
const next = await claudeSkill(harnessCommand, packageRoot);
|
|
67
87
|
if (current === next)
|
|
68
88
|
return "skipped";
|
|
69
89
|
await writeFile(skillPath, next, "utf8");
|
|
70
90
|
return "updated";
|
|
71
91
|
}
|
|
72
|
-
await writeFile(skillPath, claudeSkill(harnessCommand), "utf8");
|
|
92
|
+
await writeFile(skillPath, await claudeSkill(harnessCommand, packageRoot), "utf8");
|
|
73
93
|
return "created";
|
|
74
94
|
}
|
|
75
|
-
async function installCodexRules(targetDir, harnessCommand) {
|
|
95
|
+
async function installCodexRules(targetDir, harnessCommand, packageRoot) {
|
|
76
96
|
const agentsPath = path.join(targetDir, "AGENTS.md");
|
|
77
|
-
const block = codexBlock(harnessCommand);
|
|
97
|
+
const block = await codexBlock(harnessCommand, packageRoot);
|
|
78
98
|
if (!(await pathExists(agentsPath))) {
|
|
79
99
|
await writeFile(agentsPath, `# Project Agent Rules\n\n${block}`, "utf8");
|
|
80
100
|
return "created";
|
|
@@ -90,84 +110,55 @@ async function installCodexRules(targetDir, harnessCommand) {
|
|
|
90
110
|
await appendFile(agentsPath, `\n\n${block}`, "utf8");
|
|
91
111
|
return "updated";
|
|
92
112
|
}
|
|
93
|
-
function
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
powershell -ExecutionPolicy Bypass -File ".harness/ai-dev-harness/scripts/read-evidence.ps1" -Run <run_id_or_path>
|
|
140
|
-
\`\`\`
|
|
141
|
-
|
|
142
|
-
Do not bypass Harness for implementation tasks unless the user explicitly asks for direct manual work.
|
|
143
|
-
`;
|
|
113
|
+
async function installCursorRules(targetDir, harnessCommand, packageRoot) {
|
|
114
|
+
const rulesPath = path.join(targetDir, ".cursor", "rules", "ai-dev-harness.mdc");
|
|
115
|
+
await mkdir(path.dirname(rulesPath), { recursive: true });
|
|
116
|
+
const next = await cursorRules(harnessCommand, packageRoot);
|
|
117
|
+
if (await pathExists(rulesPath)) {
|
|
118
|
+
const current = await readFile(rulesPath, "utf8");
|
|
119
|
+
if (current === next)
|
|
120
|
+
return "skipped";
|
|
121
|
+
await writeFile(rulesPath, next, "utf8");
|
|
122
|
+
return "updated";
|
|
123
|
+
}
|
|
124
|
+
await writeFile(rulesPath, next, "utf8");
|
|
125
|
+
return "created";
|
|
126
|
+
}
|
|
127
|
+
async function installGeminiRules(targetDir, harnessCommand, packageRoot) {
|
|
128
|
+
const geminiPath = path.join(targetDir, "GEMINI.md");
|
|
129
|
+
const block = await geminiBlock(harnessCommand, packageRoot);
|
|
130
|
+
if (!(await pathExists(geminiPath))) {
|
|
131
|
+
await writeFile(geminiPath, `# Project Agent Rules\n\n${block}`, "utf8");
|
|
132
|
+
return "created";
|
|
133
|
+
}
|
|
134
|
+
const current = await readFile(geminiPath, "utf8");
|
|
135
|
+
if (current.includes("## AI Dev Harness")) {
|
|
136
|
+
const next = current.replace(/(?:\r?\n){0,2}## AI Dev Harness[\s\S]*$/m, `\n\n${block}`);
|
|
137
|
+
if (next === current)
|
|
138
|
+
return "skipped";
|
|
139
|
+
await writeFile(geminiPath, next, "utf8");
|
|
140
|
+
return "updated";
|
|
141
|
+
}
|
|
142
|
+
await appendFile(geminiPath, `\n\n${block}`, "utf8");
|
|
143
|
+
return "updated";
|
|
144
|
+
}
|
|
145
|
+
async function claudeSkill(harnessCommand, packageRoot) {
|
|
146
|
+
return renderTemplate(await readTemplateFile(path.join("integrations", "claude", "SKILL.md"), packageRoot), {
|
|
147
|
+
HARNESS_COMMAND: harnessCommand
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
async function codexBlock(harnessCommand, packageRoot) {
|
|
151
|
+
return renderTemplate(await readTemplateFile(path.join("integrations", "codex", "AGENTS_BLOCK.md"), packageRoot), {
|
|
152
|
+
HARNESS_COMMAND: harnessCommand
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
async function cursorRules(harnessCommand, packageRoot) {
|
|
156
|
+
return renderTemplate(await readTemplateFile(path.join("integrations", "cursor", "ai-dev-harness.mdc"), packageRoot), {
|
|
157
|
+
HARNESS_COMMAND: harnessCommand
|
|
158
|
+
});
|
|
144
159
|
}
|
|
145
|
-
function
|
|
146
|
-
return
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
- \`fix_bug\`
|
|
151
|
-
- \`add_feature\`
|
|
152
|
-
- \`write_tests\`
|
|
153
|
-
- \`update_docs\`
|
|
154
|
-
|
|
155
|
-
Run from the repository root:
|
|
156
|
-
|
|
157
|
-
\`\`\`bash
|
|
158
|
-
powershell -ExecutionPolicy Bypass -File ".harness/ai-dev-harness/scripts/run-harness.ps1" -Task "<task>" -Skill <skill> -Agent codex -Live
|
|
159
|
-
\`\`\`
|
|
160
|
-
|
|
161
|
-
Fallback direct command:
|
|
162
|
-
|
|
163
|
-
\`\`\`bash
|
|
164
|
-
${harnessCommand} run --task "<task>" --skill <skill> --agent codex --live
|
|
165
|
-
\`\`\`
|
|
166
|
-
|
|
167
|
-
For read-only analysis, include "Do not modify repository files" in the task and usually use \`update_docs\`.
|
|
168
|
-
|
|
169
|
-
After every Harness run, report the evidence directory, inspect \`summary.md\`, \`verify.json\`, and \`audit.jsonl\`, and report MCP degradation errors if present.
|
|
170
|
-
Use \`.harness/ai-dev-harness/scripts/read-evidence.ps1\` to print the evidence summary when useful.
|
|
171
|
-
Do not write directly to Knowledge Graph in v1; use \`persist_suggestions.md\`.
|
|
172
|
-
`;
|
|
160
|
+
async function geminiBlock(harnessCommand, packageRoot) {
|
|
161
|
+
return renderTemplate(await readTemplateFile(path.join("integrations", "gemini", "GEMINI_BLOCK.md"), packageRoot), {
|
|
162
|
+
HARNESS_COMMAND: harnessCommand
|
|
163
|
+
});
|
|
173
164
|
}
|
package/dist/loop.js
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { writeFile } from "node:fs/promises";
|
|
2
|
+
export function evaluateIteration(options) {
|
|
3
|
+
let failureType = "none";
|
|
4
|
+
let reason = "Agent 执行成功、输出契约通过、验证通过。";
|
|
5
|
+
if (options.agentResult.status !== "success") {
|
|
6
|
+
failureType = "agent_failed";
|
|
7
|
+
reason = `Agent 执行失败:${options.agentResult.error ?? `exitCode=${options.agentResult.exitCode}`}`;
|
|
8
|
+
}
|
|
9
|
+
else if (options.contractCheck.status !== "passed") {
|
|
10
|
+
failureType = "contract_failed";
|
|
11
|
+
reason = `Agent 输出缺少必需章节:${options.contractCheck.missingSections.join(", ") || "unknown"}`;
|
|
12
|
+
}
|
|
13
|
+
else if (options.approval?.required) {
|
|
14
|
+
failureType = "approval_required";
|
|
15
|
+
reason = `触发审批门禁:${options.approval.reasons.map((item) => item.type).join(", ")}`;
|
|
16
|
+
}
|
|
17
|
+
else if (options.contextScore?.status === "weak") {
|
|
18
|
+
failureType = "context_weak";
|
|
19
|
+
reason = `上下文证据不足:${options.contextScore.missing.join(", ") || "unknown"}`;
|
|
20
|
+
}
|
|
21
|
+
else if (options.verifyResult.status !== "passed") {
|
|
22
|
+
failureType = "verify_failed";
|
|
23
|
+
const failed = options.verifyResult.commands.filter((command) => command.exitCode !== 0).map((command) => command.name);
|
|
24
|
+
const diagnostic = options.verifyDiagnostics?.failedCommands[0]?.failureSummary;
|
|
25
|
+
reason = `验证未通过:${failed.join(", ") || options.verifyResult.status}${diagnostic ? `;${diagnostic}` : ""}`;
|
|
26
|
+
}
|
|
27
|
+
if (failureType === "none") {
|
|
28
|
+
return {
|
|
29
|
+
iteration: options.iteration,
|
|
30
|
+
status: "passed",
|
|
31
|
+
failureType,
|
|
32
|
+
reason,
|
|
33
|
+
canRework: false,
|
|
34
|
+
nextAction: "summarize",
|
|
35
|
+
signals: {
|
|
36
|
+
agentStatus: options.agentResult.status,
|
|
37
|
+
agentExitCode: options.agentResult.exitCode,
|
|
38
|
+
contractStatus: options.contractCheck.status,
|
|
39
|
+
verifyStatus: options.verifyResult.status,
|
|
40
|
+
contextScoreStatus: options.contextScore?.status,
|
|
41
|
+
approvalRequired: Boolean(options.approval?.required),
|
|
42
|
+
changedFiles: options.agentResult.changedFiles
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
if (options.previous?.failureType === failureType) {
|
|
47
|
+
failureType = "same_failure_repeated";
|
|
48
|
+
reason = `连续两轮出现相同失败,停止自循环:${options.previous.failureType}`;
|
|
49
|
+
}
|
|
50
|
+
else if (options.iteration >= options.maxIterations) {
|
|
51
|
+
failureType = "max_iterations_reached";
|
|
52
|
+
reason = `达到最大自循环轮次 ${options.maxIterations},停止。最后失败原因:${reason}`;
|
|
53
|
+
}
|
|
54
|
+
const canRework = failureType !== "same_failure_repeated" && failureType !== "max_iterations_reached" && failureType !== "approval_required";
|
|
55
|
+
return {
|
|
56
|
+
iteration: options.iteration,
|
|
57
|
+
status: canRework ? "rework_needed" : "stopped",
|
|
58
|
+
failureType,
|
|
59
|
+
reason,
|
|
60
|
+
canRework,
|
|
61
|
+
nextAction: canRework ? "rework" : "stop",
|
|
62
|
+
signals: {
|
|
63
|
+
agentStatus: options.agentResult.status,
|
|
64
|
+
agentExitCode: options.agentResult.exitCode,
|
|
65
|
+
contractStatus: options.contractCheck.status,
|
|
66
|
+
verifyStatus: options.verifyResult.status,
|
|
67
|
+
contextScoreStatus: options.contextScore?.status,
|
|
68
|
+
approvalRequired: Boolean(options.approval?.required),
|
|
69
|
+
changedFiles: options.agentResult.changedFiles
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
export function renderReworkFeedback(evaluation) {
|
|
74
|
+
return `上一轮自纠偏评估:
|
|
75
|
+
- iteration: ${evaluation.iteration}
|
|
76
|
+
- failure_type: ${evaluation.failureType}
|
|
77
|
+
- reason: ${evaluation.reason}
|
|
78
|
+
- next_action: ${evaluation.nextAction}
|
|
79
|
+
|
|
80
|
+
请只围绕上述失败原因做最小 rework:
|
|
81
|
+
1. 重新检查目标、上下文和计划是否偏离。
|
|
82
|
+
2. 优先修复验证失败、输出契约缺失、上下文证据不足或 Agent 执行错误。
|
|
83
|
+
3. 不扩大任务范围,不做无关重构。
|
|
84
|
+
4. 如果需要扩大范围、改公共接口、或目标不明确,请停止并说明需要人工确认。`;
|
|
85
|
+
}
|
|
86
|
+
export async function writeLoopArtifacts(options) {
|
|
87
|
+
await writeFile(options.reviewPath, `${JSON.stringify(options.review, null, 2)}\n`, "utf8");
|
|
88
|
+
const latest = options.review.iterations.at(-1);
|
|
89
|
+
const report = `# 自纠偏循环报告
|
|
90
|
+
|
|
91
|
+
## 结果
|
|
92
|
+
- 状态: ${options.review.status}
|
|
93
|
+
- 最大轮次: ${options.review.maxIterations}
|
|
94
|
+
- 实际轮次: ${options.review.iterations.length}
|
|
95
|
+
- 最终原因: ${latest?.reason ?? "无"}
|
|
96
|
+
|
|
97
|
+
## 轮次明细
|
|
98
|
+
${options.review.iterations
|
|
99
|
+
.map((item) => `### Iteration ${item.iteration}
|
|
100
|
+
- 状态: ${item.status}
|
|
101
|
+
- 失败类型: ${item.failureType}
|
|
102
|
+
- 原因: ${item.reason}
|
|
103
|
+
- 下一步: ${item.nextAction}
|
|
104
|
+
- Agent: ${item.signals.agentStatus} (${item.signals.agentExitCode ?? "n/a"})
|
|
105
|
+
- 契约: ${item.signals.contractStatus}
|
|
106
|
+
- 验证: ${item.signals.verifyStatus}
|
|
107
|
+
- 修改文件: ${item.signals.changedFiles.join(", ") || "无"}`)
|
|
108
|
+
.join("\n\n")}
|
|
109
|
+
|
|
110
|
+
## 人工介入建议
|
|
111
|
+
- 如果状态为 stopped,请先看 \`execution.log\`、\`verify.json\`、\`agent_contract.json\` 和本报告中的失败类型。
|
|
112
|
+
- 如果失败类型是 \`same_failure_repeated\`,优先补充上下文、调整 Skill 或让用户确认目标。
|
|
113
|
+
- 如果失败类型是 \`max_iterations_reached\`,不要继续盲目重跑,先做人工评审。
|
|
114
|
+
`;
|
|
115
|
+
await writeFile(options.reportPath, report, "utf8");
|
|
116
|
+
}
|
package/dist/mcp.js
CHANGED
|
@@ -1,49 +1,25 @@
|
|
|
1
1
|
import { appendFile } from "node:fs/promises";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
method: "POST",
|
|
9
|
-
headers: { "content-type": "application/json", accept: "application/json, text/event-stream" },
|
|
10
|
-
body: JSON.stringify(body),
|
|
11
|
-
signal: controller.signal
|
|
12
|
-
});
|
|
13
|
-
const text = await response.text();
|
|
14
|
-
if (!response.ok)
|
|
15
|
-
throw new Error(`HTTP ${response.status}: ${truncate(text, 1000)}`);
|
|
16
|
-
return parseMcpResponse(text, response.headers.get("content-type") ?? "");
|
|
17
|
-
}
|
|
18
|
-
finally {
|
|
19
|
-
clearTimeout(timer);
|
|
20
|
-
}
|
|
2
|
+
function endpointSummary(endpoint) {
|
|
3
|
+
return {
|
|
4
|
+
type: endpoint.type ?? "http",
|
|
5
|
+
url: endpoint.url,
|
|
6
|
+
headerNames: Object.keys(endpoint.headers ?? {})
|
|
7
|
+
};
|
|
21
8
|
}
|
|
22
|
-
function
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
});
|
|
37
|
-
}
|
|
38
|
-
try {
|
|
39
|
-
return JSON.parse(text);
|
|
40
|
-
}
|
|
41
|
-
catch {
|
|
42
|
-
return text;
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
function summarize(data) {
|
|
46
|
-
return truncate(typeof data === "string" ? data : JSON.stringify(data, null, 2), 6000);
|
|
9
|
+
function agentDrivenResult(source, endpoint, skill) {
|
|
10
|
+
const request = {
|
|
11
|
+
mode: "agent-driven",
|
|
12
|
+
instruction: "The current execution agent should call this MCP server with its own configured MCP client before invoking Harness. Harness records the configured source but does not implement MCP transport.",
|
|
13
|
+
endpoint: endpointSummary(endpoint),
|
|
14
|
+
suggestedQueries: skill.contextQueries
|
|
15
|
+
};
|
|
16
|
+
return {
|
|
17
|
+
source,
|
|
18
|
+
ok: true,
|
|
19
|
+
request,
|
|
20
|
+
responseSummary: `${source} is configured as an agent-driven MCP context source.`,
|
|
21
|
+
data: request
|
|
22
|
+
};
|
|
47
23
|
}
|
|
48
24
|
export class CodeGraphSource {
|
|
49
25
|
config;
|
|
@@ -52,41 +28,10 @@ export class CodeGraphSource {
|
|
|
52
28
|
this.config = config;
|
|
53
29
|
this.queryLogPath = queryLogPath;
|
|
54
30
|
}
|
|
55
|
-
async getImpactContext(
|
|
56
|
-
const
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
skill: skill.name,
|
|
60
|
-
queries: skill.contextQueries,
|
|
61
|
-
changedFiles
|
|
62
|
-
};
|
|
63
|
-
return this.query("codegraph", this.config.mcp.codegraph.url, request);
|
|
64
|
-
}
|
|
65
|
-
async query(source, url, request) {
|
|
66
|
-
try {
|
|
67
|
-
const data = await postJson(url, request, this.config.mcp.timeoutMs);
|
|
68
|
-
const result = {
|
|
69
|
-
source,
|
|
70
|
-
ok: true,
|
|
71
|
-
degraded: false,
|
|
72
|
-
request,
|
|
73
|
-
responseSummary: summarize(data),
|
|
74
|
-
data
|
|
75
|
-
};
|
|
76
|
-
await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
|
|
77
|
-
return result;
|
|
78
|
-
}
|
|
79
|
-
catch (error) {
|
|
80
|
-
const result = {
|
|
81
|
-
source,
|
|
82
|
-
ok: false,
|
|
83
|
-
degraded: true,
|
|
84
|
-
request,
|
|
85
|
-
error: error instanceof Error ? error.message : String(error)
|
|
86
|
-
};
|
|
87
|
-
await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
|
|
88
|
-
return result;
|
|
89
|
-
}
|
|
31
|
+
async getImpactContext(_task, skill) {
|
|
32
|
+
const result = agentDrivenResult("codegraph", this.config.mcp.codegraph, skill);
|
|
33
|
+
await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
|
|
34
|
+
return result;
|
|
90
35
|
}
|
|
91
36
|
}
|
|
92
37
|
export class KnowledgeGraphSource {
|
|
@@ -96,38 +41,10 @@ export class KnowledgeGraphSource {
|
|
|
96
41
|
this.config = config;
|
|
97
42
|
this.queryLogPath = queryLogPath;
|
|
98
43
|
}
|
|
99
|
-
async searchKnowledge(
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
skill: skill.name,
|
|
104
|
-
queries: skill.contextQueries,
|
|
105
|
-
codeContext
|
|
106
|
-
};
|
|
107
|
-
try {
|
|
108
|
-
const data = await postJson(this.config.mcp.knowledgeGraph.url, request, this.config.mcp.timeoutMs);
|
|
109
|
-
const result = {
|
|
110
|
-
source: "knowledgeGraph",
|
|
111
|
-
ok: true,
|
|
112
|
-
degraded: false,
|
|
113
|
-
request,
|
|
114
|
-
responseSummary: summarize(data),
|
|
115
|
-
data
|
|
116
|
-
};
|
|
117
|
-
await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
|
|
118
|
-
return result;
|
|
119
|
-
}
|
|
120
|
-
catch (error) {
|
|
121
|
-
const result = {
|
|
122
|
-
source: "knowledgeGraph",
|
|
123
|
-
ok: false,
|
|
124
|
-
degraded: true,
|
|
125
|
-
request,
|
|
126
|
-
error: error instanceof Error ? error.message : String(error)
|
|
127
|
-
};
|
|
128
|
-
await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
|
|
129
|
-
return result;
|
|
130
|
-
}
|
|
44
|
+
async searchKnowledge(_task, skill) {
|
|
45
|
+
const result = agentDrivenResult("knowledgeGraph", this.config.mcp.knowledgeGraph, skill);
|
|
46
|
+
await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
|
|
47
|
+
return result;
|
|
131
48
|
}
|
|
132
49
|
}
|
|
133
50
|
export class ContextSource {
|
|
@@ -139,7 +56,7 @@ export class ContextSource {
|
|
|
139
56
|
}
|
|
140
57
|
async searchTaskContext(task, skill) {
|
|
141
58
|
const codeContext = await this.codeGraph.getImpactContext(task, skill);
|
|
142
|
-
const knowledgeContext = await this.knowledgeGraph.searchKnowledge(task, skill
|
|
59
|
+
const knowledgeContext = await this.knowledgeGraph.searchKnowledge(task, skill);
|
|
143
60
|
return { codeContext, knowledgeContext };
|
|
144
61
|
}
|
|
145
62
|
}
|
package/dist/metrics.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { readdir, readFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { pathExists } from "./utils.js";
|
|
4
|
+
async function readJson(filePath) {
|
|
5
|
+
try {
|
|
6
|
+
return JSON.parse(await readFile(filePath, "utf8"));
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
function increment(map, key) {
|
|
13
|
+
map[key] = (map[key] ?? 0) + 1;
|
|
14
|
+
}
|
|
15
|
+
export async function collectRunMetrics(cwd, config) {
|
|
16
|
+
const runsRoot = path.join(cwd, config.audit.path);
|
|
17
|
+
const entries = (await pathExists(runsRoot)) ? await readdir(runsRoot, { withFileTypes: true }) : [];
|
|
18
|
+
const runs = [];
|
|
19
|
+
for (const entry of entries.filter((item) => item.isDirectory())) {
|
|
20
|
+
const runDir = path.join(runsRoot, entry.name);
|
|
21
|
+
const intake = await readJson(path.join(runDir, "intake.json"));
|
|
22
|
+
const review = await readJson(path.join(runDir, "loop_review.json"));
|
|
23
|
+
const verify = await readJson(path.join(runDir, "verify.json"));
|
|
24
|
+
const last = review?.iterations?.at(-1);
|
|
25
|
+
runs.push({
|
|
26
|
+
runId: entry.name,
|
|
27
|
+
skill: intake?.skill,
|
|
28
|
+
agent: intake?.agent,
|
|
29
|
+
status: review?.status ?? "unknown",
|
|
30
|
+
iterations: review?.iterations?.length ?? 0,
|
|
31
|
+
failureType: last?.failureType ?? "unknown",
|
|
32
|
+
contractStatus: last?.signals?.contractStatus,
|
|
33
|
+
verifyStatus: last?.signals?.verifyStatus,
|
|
34
|
+
contextScore: last?.signals?.contextScoreStatus,
|
|
35
|
+
approvalRequired: Boolean(last?.signals?.approvalRequired),
|
|
36
|
+
failedVerifyCommands: verify?.commands?.filter((command) => command.exitCode !== 0).map((command) => command.name) ?? []
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
const failureTypes = {};
|
|
40
|
+
const skillStatus = {};
|
|
41
|
+
const agentStatus = {};
|
|
42
|
+
const verifyFailures = {};
|
|
43
|
+
const contextScores = {};
|
|
44
|
+
for (const run of runs) {
|
|
45
|
+
increment(failureTypes, run.failureType);
|
|
46
|
+
if (run.skill)
|
|
47
|
+
increment(skillStatus, `${run.skill}:${run.status}`);
|
|
48
|
+
if (run.agent)
|
|
49
|
+
increment(agentStatus, `${run.agent}:${run.status}`);
|
|
50
|
+
for (const command of run.failedVerifyCommands)
|
|
51
|
+
increment(verifyFailures, command);
|
|
52
|
+
if (run.contextScore)
|
|
53
|
+
increment(contextScores, run.contextScore);
|
|
54
|
+
}
|
|
55
|
+
const total = runs.length;
|
|
56
|
+
const passed = runs.filter((run) => run.status === "passed").length;
|
|
57
|
+
const firstPass = runs.filter((run) => run.status === "passed" && run.iterations === 1).length;
|
|
58
|
+
const contractPassed = runs.filter((run) => run.contractStatus === "passed").length;
|
|
59
|
+
const averageIterations = total ? runs.reduce((sum, run) => sum + run.iterations, 0) / total : 0;
|
|
60
|
+
return {
|
|
61
|
+
runs,
|
|
62
|
+
summary: {
|
|
63
|
+
totalRuns: total,
|
|
64
|
+
successRate: total ? passed / total : 0,
|
|
65
|
+
firstPassRate: total ? firstPass / total : 0,
|
|
66
|
+
agentContractPassRate: total ? contractPassed / total : 0,
|
|
67
|
+
averageIterations,
|
|
68
|
+
failureTypes,
|
|
69
|
+
skillStatus,
|
|
70
|
+
agentStatus,
|
|
71
|
+
verifyFailureTopN: Object.fromEntries(Object.entries(verifyFailures).sort((a, b) => b[1] - a[1]).slice(0, 10)),
|
|
72
|
+
approvalGateCount: runs.filter((run) => run.approvalRequired).length,
|
|
73
|
+
contextScores
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
export function renderMetricsText(metrics) {
|
|
78
|
+
return `运行指标
|
|
79
|
+
- 总运行次数: ${metrics.summary.totalRuns}
|
|
80
|
+
- 成功率: ${Number(metrics.summary.successRate).toFixed(2)}
|
|
81
|
+
- 一次通过率: ${Number(metrics.summary.firstPassRate).toFixed(2)}
|
|
82
|
+
- Agent 契约通过率: ${Number(metrics.summary.agentContractPassRate).toFixed(2)}
|
|
83
|
+
- 平均循环轮次: ${Number(metrics.summary.averageIterations).toFixed(2)}
|
|
84
|
+
- 失败类型分布: ${JSON.stringify(metrics.summary.failureTypes)}
|
|
85
|
+
- Skill 状态: ${JSON.stringify(metrics.summary.skillStatus)}
|
|
86
|
+
- Agent 状态: ${JSON.stringify(metrics.summary.agentStatus)}
|
|
87
|
+
- 验证失败 Top N: ${JSON.stringify(metrics.summary.verifyFailureTopN)}
|
|
88
|
+
- Approval Gate 触发次数: ${metrics.summary.approvalGateCount}
|
|
89
|
+
- Context Score 分布: ${JSON.stringify(metrics.summary.contextScores)}
|
|
90
|
+
`;
|
|
91
|
+
}
|
package/dist/replay.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
export async function buildReplayPlan(runDir, overrideAgent, dry = false) {
|
|
4
|
+
const intake = JSON.parse(await readFile(path.join(runDir, "intake.json"), "utf8"));
|
|
5
|
+
const agent = overrideAgent ?? intake.agent;
|
|
6
|
+
if (agent !== "codex" && agent !== "claude" && agent !== "cursor" && agent !== "gemini") {
|
|
7
|
+
throw new Error(`历史运行记录中的 agent 无效:${agent}`);
|
|
8
|
+
}
|
|
9
|
+
if (!intake.task || !intake.skill)
|
|
10
|
+
throw new Error(`历史运行记录缺少 task 或 skill:${runDir}`);
|
|
11
|
+
return {
|
|
12
|
+
sourceRun: runDir,
|
|
13
|
+
task: intake.task,
|
|
14
|
+
skill: intake.skill,
|
|
15
|
+
agent,
|
|
16
|
+
dry
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export function renderReplayPlan(plan) {
|
|
20
|
+
return `Replay Plan
|
|
21
|
+
- Source run: ${plan.sourceRun}
|
|
22
|
+
- Task: ${plan.task}
|
|
23
|
+
- Skill: ${plan.skill}
|
|
24
|
+
- Agent: ${plan.agent}
|
|
25
|
+
- Dry run: ${plan.dry ? "yes" : "no"}
|
|
26
|
+
`;
|
|
27
|
+
}
|