ai-dev-harness 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/COLLEAGUE_TRIAL_GUIDE.md +175 -0
  2. package/ONLINE_INSTALL_GUIDE.md +458 -0
  3. package/README.md +253 -54
  4. package/dist/agent.js +74 -49
  5. package/dist/approval.js +64 -0
  6. package/dist/audit.js +1 -0
  7. package/dist/checkpoint.js +83 -0
  8. package/dist/cli.js +326 -99
  9. package/dist/config.js +118 -9
  10. package/dist/context.js +62 -21
  11. package/dist/contextScore.js +44 -0
  12. package/dist/diagnostics.js +58 -0
  13. package/dist/doctor.js +100 -0
  14. package/dist/feedback.js +5 -0
  15. package/dist/init.js +8 -20
  16. package/dist/integration.js +84 -93
  17. package/dist/loop.js +116 -0
  18. package/dist/mcp.js +29 -112
  19. package/dist/metrics.js +91 -0
  20. package/dist/replay.js +27 -0
  21. package/dist/runPaths.js +31 -0
  22. package/dist/summarize.js +170 -39
  23. package/dist/templates.js +39 -215
  24. package/package.json +5 -2
  25. package/scripts/read-evidence.ps1 +45 -3
  26. package/scripts/run-harness.ps1 +9 -6
  27. package/templates/default-project/AGENTS.md +9 -0
  28. package/templates/default-project/SKILLS.md +13 -0
  29. package/templates/default-project/TOOLS.md +11 -0
  30. package/templates/default-project/skills/add_feature/SKILL.md +34 -0
  31. package/templates/default-project/skills/add_feature/context.yaml +33 -0
  32. package/templates/default-project/skills/add_feature/verify.yaml +2 -0
  33. package/templates/default-project/skills/fix_bug/SKILL.md +45 -0
  34. package/templates/default-project/skills/fix_bug/context.yaml +33 -0
  35. package/templates/default-project/skills/fix_bug/verify.yaml +2 -0
  36. package/templates/default-project/skills/update_docs/SKILL.md +28 -0
  37. package/templates/default-project/skills/update_docs/context.yaml +30 -0
  38. package/templates/default-project/skills/update_docs/verify.yaml +2 -0
  39. package/templates/default-project/skills/write_tests/SKILL.md +30 -0
  40. package/templates/default-project/skills/write_tests/context.yaml +32 -0
  41. package/templates/default-project/skills/write_tests/verify.yaml +2 -0
  42. package/templates/integrations/claude/SKILL.md +115 -0
  43. package/templates/integrations/codex/AGENTS_BLOCK.md +82 -0
  44. package/templates/integrations/cursor/ai-dev-harness.mdc +55 -0
  45. package/templates/integrations/gemini/GEMINI_BLOCK.md +46 -0
@@ -1,16 +1,17 @@
1
1
  import path from "node:path";
2
- import { appendFile, copyFile, mkdir, readFile, writeFile } from "node:fs/promises";
2
+ import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
3
+ import { readTemplateFile, renderTemplate } from "./templates.js";
3
4
  import { pathExists } from "./utils.js";
4
5
  export async function installIntegration(options) {
5
6
  const created = [];
6
7
  const updated = [];
7
8
  const skipped = [];
8
- const scriptResult = await installScriptAssets(options.targetDir, options.packageRoot ?? process.cwd());
9
+ const scriptResult = await installScriptAssets(options.targetDir, options.packageRoot ?? process.cwd(), options.harnessCliPath);
9
10
  created.push(...scriptResult.created);
10
11
  updated.push(...scriptResult.updated);
11
12
  skipped.push(...scriptResult.skipped);
12
- if (options.agent === "claude" || options.agent === "both") {
13
- const result = await installClaudeSkill(options.targetDir, options.harnessCommand);
13
+ if (options.agent === "claude" || options.agent === "both" || options.agent === "all") {
14
+ const result = await installClaudeSkill(options.targetDir, options.harnessCommand, options.packageRoot);
14
15
  if (result === "created")
15
16
  created.push(".claude/skills/ai-dev-harness/SKILL.md");
16
17
  else if (result === "updated")
@@ -18,8 +19,8 @@ export async function installIntegration(options) {
18
19
  else
19
20
  skipped.push(".claude/skills/ai-dev-harness/SKILL.md");
20
21
  }
21
- if (options.agent === "codex" || options.agent === "both") {
22
- const result = await installCodexRules(options.targetDir, options.harnessCommand);
22
+ if (options.agent === "codex" || options.agent === "both" || options.agent === "all") {
23
+ const result = await installCodexRules(options.targetDir, options.harnessCommand, options.packageRoot);
23
24
  if (result === "created")
24
25
  created.push("AGENTS.md");
25
26
  else if (result === "updated")
@@ -27,9 +28,27 @@ export async function installIntegration(options) {
27
28
  else
28
29
  skipped.push("AGENTS.md");
29
30
  }
31
+ if (options.agent === "cursor" || options.agent === "all") {
32
+ const result = await installCursorRules(options.targetDir, options.harnessCommand, options.packageRoot);
33
+ if (result === "created")
34
+ created.push(".cursor/rules/ai-dev-harness.mdc");
35
+ else if (result === "updated")
36
+ updated.push(".cursor/rules/ai-dev-harness.mdc");
37
+ else
38
+ skipped.push(".cursor/rules/ai-dev-harness.mdc");
39
+ }
40
+ if (options.agent === "gemini" || options.agent === "all") {
41
+ const result = await installGeminiRules(options.targetDir, options.harnessCommand, options.packageRoot);
42
+ if (result === "created")
43
+ created.push("GEMINI.md");
44
+ else if (result === "updated")
45
+ updated.push("GEMINI.md");
46
+ else
47
+ skipped.push("GEMINI.md");
48
+ }
30
49
  return { created, updated, skipped };
31
50
  }
32
- async function installScriptAssets(targetDir, packageRoot) {
51
+ async function installScriptAssets(targetDir, packageRoot, harnessCliPath) {
33
52
  const created = [];
34
53
  const updated = [];
35
54
  const skipped = [];
@@ -42,7 +61,8 @@ async function installScriptAssets(targetDir, packageRoot) {
42
61
  const rel = path.join(".harness", "ai-dev-harness", "scripts", asset);
43
62
  if (!(await pathExists(source)))
44
63
  continue;
45
- const next = await readFile(source, "utf8");
64
+ const raw = await readFile(source, "utf8");
65
+ const next = renderTemplate(raw, { HARNESS_CLI_PATH: harnessCliPath ?? "{{HARNESS_CLI_PATH}}" });
46
66
  if (await pathExists(target)) {
47
67
  const current = await readFile(target, "utf8");
48
68
  if (current === next) {
@@ -53,28 +73,28 @@ async function installScriptAssets(targetDir, packageRoot) {
53
73
  updated.push(rel);
54
74
  continue;
55
75
  }
56
- await copyFile(source, target);
76
+ await writeFile(target, next, "utf8");
57
77
  created.push(rel);
58
78
  }
59
79
  return { created, updated, skipped };
60
80
  }
61
- async function installClaudeSkill(targetDir, harnessCommand) {
81
+ async function installClaudeSkill(targetDir, harnessCommand, packageRoot) {
62
82
  const skillPath = path.join(targetDir, ".claude", "skills", "ai-dev-harness", "SKILL.md");
63
83
  await mkdir(path.dirname(skillPath), { recursive: true });
64
84
  if (await pathExists(skillPath)) {
65
85
  const current = await readFile(skillPath, "utf8");
66
- const next = claudeSkill(harnessCommand);
86
+ const next = await claudeSkill(harnessCommand, packageRoot);
67
87
  if (current === next)
68
88
  return "skipped";
69
89
  await writeFile(skillPath, next, "utf8");
70
90
  return "updated";
71
91
  }
72
- await writeFile(skillPath, claudeSkill(harnessCommand), "utf8");
92
+ await writeFile(skillPath, await claudeSkill(harnessCommand, packageRoot), "utf8");
73
93
  return "created";
74
94
  }
75
- async function installCodexRules(targetDir, harnessCommand) {
95
+ async function installCodexRules(targetDir, harnessCommand, packageRoot) {
76
96
  const agentsPath = path.join(targetDir, "AGENTS.md");
77
- const block = codexBlock(harnessCommand);
97
+ const block = await codexBlock(harnessCommand, packageRoot);
78
98
  if (!(await pathExists(agentsPath))) {
79
99
  await writeFile(agentsPath, `# Project Agent Rules\n\n${block}`, "utf8");
80
100
  return "created";
@@ -90,84 +110,55 @@ async function installCodexRules(targetDir, harnessCommand) {
90
110
  await appendFile(agentsPath, `\n\n${block}`, "utf8");
91
111
  return "updated";
92
112
  }
93
- function claudeSkill(harnessCommand) {
94
- return `---
95
- name: ai-dev-harness
96
- description: Trigger the auditable AI Dev Harness CLI from Claude Code conversations and stream the Harness process in the current chat.
97
- ---
98
-
99
- # AI Dev Harness
100
-
101
- Use this skill when the user asks to use Harness, AI Dev Harness, an audited run, CodeGraph/Knowledge Graph context, or one of these skills: \`fix_bug\`, \`add_feature\`, \`write_tests\`, \`update_docs\`.
102
-
103
- ## Command
104
-
105
- Run from the repository root:
106
-
107
- \`\`\`bash
108
- powershell -ExecutionPolicy Bypass -File ".harness/ai-dev-harness/scripts/run-harness.ps1" -Task "<task>" -Skill <skill> -Agent claude -Live
109
- \`\`\`
110
-
111
- Fallback direct command:
112
-
113
- \`\`\`bash
114
- ${harnessCommand} run --task "<task>" --skill <skill> --agent claude --live
115
- \`\`\`
116
-
117
- ## Skill Mapping
118
-
119
- - Defect or regression: \`fix_bug\`
120
- - Scoped feature: \`add_feature\`
121
- - Add or improve tests: \`write_tests\`
122
- - Read-only analysis or docs: \`update_docs\`
123
-
124
- For read-only analysis, include "Do not modify repository files" in the task.
125
-
126
- ## Reporting
127
-
128
- After the command finishes, inspect and report:
129
-
130
- - Evidence directory printed by Harness
131
- - \`summary.md\`
132
- - \`verify.json\`
133
- - \`audit.jsonl\`
134
- - MCP degradation errors, if present
135
-
136
- You may use:
137
-
138
- \`\`\`bash
139
- powershell -ExecutionPolicy Bypass -File ".harness/ai-dev-harness/scripts/read-evidence.ps1" -Run <run_id_or_path>
140
- \`\`\`
141
-
142
- Do not bypass Harness for implementation tasks unless the user explicitly asks for direct manual work.
143
- `;
113
+ async function installCursorRules(targetDir, harnessCommand, packageRoot) {
114
+ const rulesPath = path.join(targetDir, ".cursor", "rules", "ai-dev-harness.mdc");
115
+ await mkdir(path.dirname(rulesPath), { recursive: true });
116
+ const next = await cursorRules(harnessCommand, packageRoot);
117
+ if (await pathExists(rulesPath)) {
118
+ const current = await readFile(rulesPath, "utf8");
119
+ if (current === next)
120
+ return "skipped";
121
+ await writeFile(rulesPath, next, "utf8");
122
+ return "updated";
123
+ }
124
+ await writeFile(rulesPath, next, "utf8");
125
+ return "created";
126
+ }
127
+ async function installGeminiRules(targetDir, harnessCommand, packageRoot) {
128
+ const geminiPath = path.join(targetDir, "GEMINI.md");
129
+ const block = await geminiBlock(harnessCommand, packageRoot);
130
+ if (!(await pathExists(geminiPath))) {
131
+ await writeFile(geminiPath, `# Project Agent Rules\n\n${block}`, "utf8");
132
+ return "created";
133
+ }
134
+ const current = await readFile(geminiPath, "utf8");
135
+ if (current.includes("## AI Dev Harness")) {
136
+ const next = current.replace(/(?:\r?\n){0,2}## AI Dev Harness[\s\S]*$/m, `\n\n${block}`);
137
+ if (next === current)
138
+ return "skipped";
139
+ await writeFile(geminiPath, next, "utf8");
140
+ return "updated";
141
+ }
142
+ await appendFile(geminiPath, `\n\n${block}`, "utf8");
143
+ return "updated";
144
+ }
145
+ async function claudeSkill(harnessCommand, packageRoot) {
146
+ return renderTemplate(await readTemplateFile(path.join("integrations", "claude", "SKILL.md"), packageRoot), {
147
+ HARNESS_COMMAND: harnessCommand
148
+ });
149
+ }
150
+ async function codexBlock(harnessCommand, packageRoot) {
151
+ return renderTemplate(await readTemplateFile(path.join("integrations", "codex", "AGENTS_BLOCK.md"), packageRoot), {
152
+ HARNESS_COMMAND: harnessCommand
153
+ });
154
+ }
155
+ async function cursorRules(harnessCommand, packageRoot) {
156
+ return renderTemplate(await readTemplateFile(path.join("integrations", "cursor", "ai-dev-harness.mdc"), packageRoot), {
157
+ HARNESS_COMMAND: harnessCommand
158
+ });
144
159
  }
145
- function codexBlock(harnessCommand) {
146
- return `## AI Dev Harness
147
-
148
- Use the AI Dev Harness CLI when the user asks for Harness, an audited run, CodeGraph/Knowledge Graph assisted work, or one of these skills:
149
-
150
- - \`fix_bug\`
151
- - \`add_feature\`
152
- - \`write_tests\`
153
- - \`update_docs\`
154
-
155
- Run from the repository root:
156
-
157
- \`\`\`bash
158
- powershell -ExecutionPolicy Bypass -File ".harness/ai-dev-harness/scripts/run-harness.ps1" -Task "<task>" -Skill <skill> -Agent codex -Live
159
- \`\`\`
160
-
161
- Fallback direct command:
162
-
163
- \`\`\`bash
164
- ${harnessCommand} run --task "<task>" --skill <skill> --agent codex --live
165
- \`\`\`
166
-
167
- For read-only analysis, include "Do not modify repository files" in the task and usually use \`update_docs\`.
168
-
169
- After every Harness run, report the evidence directory, inspect \`summary.md\`, \`verify.json\`, and \`audit.jsonl\`, and report MCP degradation errors if present.
170
- Use \`.harness/ai-dev-harness/scripts/read-evidence.ps1\` to print the evidence summary when useful.
171
- Do not write directly to Knowledge Graph in v1; use \`persist_suggestions.md\`.
172
- `;
160
+ async function geminiBlock(harnessCommand, packageRoot) {
161
+ return renderTemplate(await readTemplateFile(path.join("integrations", "gemini", "GEMINI_BLOCK.md"), packageRoot), {
162
+ HARNESS_COMMAND: harnessCommand
163
+ });
173
164
  }
package/dist/loop.js ADDED
@@ -0,0 +1,116 @@
1
+ import { writeFile } from "node:fs/promises";
2
+ export function evaluateIteration(options) {
3
+ let failureType = "none";
4
+ let reason = "Agent 执行成功、输出契约通过、验证通过。";
5
+ if (options.agentResult.status !== "success") {
6
+ failureType = "agent_failed";
7
+ reason = `Agent 执行失败:${options.agentResult.error ?? `exitCode=${options.agentResult.exitCode}`}`;
8
+ }
9
+ else if (options.contractCheck.status !== "passed") {
10
+ failureType = "contract_failed";
11
+ reason = `Agent 输出缺少必需章节:${options.contractCheck.missingSections.join(", ") || "unknown"}`;
12
+ }
13
+ else if (options.approval?.required) {
14
+ failureType = "approval_required";
15
+ reason = `触发审批门禁:${options.approval.reasons.map((item) => item.type).join(", ")}`;
16
+ }
17
+ else if (options.contextScore?.status === "weak") {
18
+ failureType = "context_weak";
19
+ reason = `上下文证据不足:${options.contextScore.missing.join(", ") || "unknown"}`;
20
+ }
21
+ else if (options.verifyResult.status !== "passed") {
22
+ failureType = "verify_failed";
23
+ const failed = options.verifyResult.commands.filter((command) => command.exitCode !== 0).map((command) => command.name);
24
+ const diagnostic = options.verifyDiagnostics?.failedCommands[0]?.failureSummary;
25
+ reason = `验证未通过:${failed.join(", ") || options.verifyResult.status}${diagnostic ? `;${diagnostic}` : ""}`;
26
+ }
27
+ if (failureType === "none") {
28
+ return {
29
+ iteration: options.iteration,
30
+ status: "passed",
31
+ failureType,
32
+ reason,
33
+ canRework: false,
34
+ nextAction: "summarize",
35
+ signals: {
36
+ agentStatus: options.agentResult.status,
37
+ agentExitCode: options.agentResult.exitCode,
38
+ contractStatus: options.contractCheck.status,
39
+ verifyStatus: options.verifyResult.status,
40
+ contextScoreStatus: options.contextScore?.status,
41
+ approvalRequired: Boolean(options.approval?.required),
42
+ changedFiles: options.agentResult.changedFiles
43
+ }
44
+ };
45
+ }
46
+ if (options.previous?.failureType === failureType) {
47
+ failureType = "same_failure_repeated";
48
+ reason = `连续两轮出现相同失败,停止自循环:${options.previous.failureType}`;
49
+ }
50
+ else if (options.iteration >= options.maxIterations) {
51
+ failureType = "max_iterations_reached";
52
+ reason = `达到最大自循环轮次 ${options.maxIterations},停止。最后失败原因:${reason}`;
53
+ }
54
+ const canRework = failureType !== "same_failure_repeated" && failureType !== "max_iterations_reached" && failureType !== "approval_required";
55
+ return {
56
+ iteration: options.iteration,
57
+ status: canRework ? "rework_needed" : "stopped",
58
+ failureType,
59
+ reason,
60
+ canRework,
61
+ nextAction: canRework ? "rework" : "stop",
62
+ signals: {
63
+ agentStatus: options.agentResult.status,
64
+ agentExitCode: options.agentResult.exitCode,
65
+ contractStatus: options.contractCheck.status,
66
+ verifyStatus: options.verifyResult.status,
67
+ contextScoreStatus: options.contextScore?.status,
68
+ approvalRequired: Boolean(options.approval?.required),
69
+ changedFiles: options.agentResult.changedFiles
70
+ }
71
+ };
72
+ }
73
+ export function renderReworkFeedback(evaluation) {
74
+ return `上一轮自纠偏评估:
75
+ - iteration: ${evaluation.iteration}
76
+ - failure_type: ${evaluation.failureType}
77
+ - reason: ${evaluation.reason}
78
+ - next_action: ${evaluation.nextAction}
79
+
80
+ 请只围绕上述失败原因做最小 rework:
81
+ 1. 重新检查目标、上下文和计划是否偏离。
82
+ 2. 优先修复验证失败、输出契约缺失、上下文证据不足或 Agent 执行错误。
83
+ 3. 不扩大任务范围,不做无关重构。
84
+ 4. 如果需要扩大范围、改公共接口、或目标不明确,请停止并说明需要人工确认。`;
85
+ }
86
+ export async function writeLoopArtifacts(options) {
87
+ await writeFile(options.reviewPath, `${JSON.stringify(options.review, null, 2)}\n`, "utf8");
88
+ const latest = options.review.iterations.at(-1);
89
+ const report = `# 自纠偏循环报告
90
+
91
+ ## 结果
92
+ - 状态: ${options.review.status}
93
+ - 最大轮次: ${options.review.maxIterations}
94
+ - 实际轮次: ${options.review.iterations.length}
95
+ - 最终原因: ${latest?.reason ?? "无"}
96
+
97
+ ## 轮次明细
98
+ ${options.review.iterations
99
+ .map((item) => `### Iteration ${item.iteration}
100
+ - 状态: ${item.status}
101
+ - 失败类型: ${item.failureType}
102
+ - 原因: ${item.reason}
103
+ - 下一步: ${item.nextAction}
104
+ - Agent: ${item.signals.agentStatus} (${item.signals.agentExitCode ?? "n/a"})
105
+ - 契约: ${item.signals.contractStatus}
106
+ - 验证: ${item.signals.verifyStatus}
107
+ - 修改文件: ${item.signals.changedFiles.join(", ") || "无"}`)
108
+ .join("\n\n")}
109
+
110
+ ## 人工介入建议
111
+ - 如果状态为 stopped,请先看 \`execution.log\`、\`verify.json\`、\`agent_contract.json\` 和本报告中的失败类型。
112
+ - 如果失败类型是 \`same_failure_repeated\`,优先补充上下文、调整 Skill 或让用户确认目标。
113
+ - 如果失败类型是 \`max_iterations_reached\`,不要继续盲目重跑,先做人工评审。
114
+ `;
115
+ await writeFile(options.reportPath, report, "utf8");
116
+ }
package/dist/mcp.js CHANGED
@@ -1,49 +1,25 @@
1
1
  import { appendFile } from "node:fs/promises";
2
- import { truncate } from "./utils.js";
3
- async function postJson(url, body, timeoutMs) {
4
- const controller = new AbortController();
5
- const timer = setTimeout(() => controller.abort(), timeoutMs);
6
- try {
7
- const response = await fetch(url, {
8
- method: "POST",
9
- headers: { "content-type": "application/json", accept: "application/json, text/event-stream" },
10
- body: JSON.stringify(body),
11
- signal: controller.signal
12
- });
13
- const text = await response.text();
14
- if (!response.ok)
15
- throw new Error(`HTTP ${response.status}: ${truncate(text, 1000)}`);
16
- return parseMcpResponse(text, response.headers.get("content-type") ?? "");
17
- }
18
- finally {
19
- clearTimeout(timer);
20
- }
2
+ function endpointSummary(endpoint) {
3
+ return {
4
+ type: endpoint.type ?? "http",
5
+ url: endpoint.url,
6
+ headerNames: Object.keys(endpoint.headers ?? {})
7
+ };
21
8
  }
22
- function parseMcpResponse(text, contentType) {
23
- if (contentType.includes("text/event-stream")) {
24
- const dataLines = text
25
- .split(/\r?\n/)
26
- .filter((line) => line.startsWith("data:"))
27
- .map((line) => line.slice(5).trim())
28
- .filter((line) => line && line !== "[DONE]");
29
- return dataLines.map((line) => {
30
- try {
31
- return JSON.parse(line);
32
- }
33
- catch {
34
- return line;
35
- }
36
- });
37
- }
38
- try {
39
- return JSON.parse(text);
40
- }
41
- catch {
42
- return text;
43
- }
44
- }
45
- function summarize(data) {
46
- return truncate(typeof data === "string" ? data : JSON.stringify(data, null, 2), 6000);
9
+ function agentDrivenResult(source, endpoint, skill) {
10
+ const request = {
11
+ mode: "agent-driven",
12
+ instruction: "The current execution agent should call this MCP server with its own configured MCP client before invoking Harness. Harness records the configured source but does not implement MCP transport.",
13
+ endpoint: endpointSummary(endpoint),
14
+ suggestedQueries: skill.contextQueries
15
+ };
16
+ return {
17
+ source,
18
+ ok: true,
19
+ request,
20
+ responseSummary: `${source} is configured as an agent-driven MCP context source.`,
21
+ data: request
22
+ };
47
23
  }
48
24
  export class CodeGraphSource {
49
25
  config;
@@ -52,41 +28,10 @@ export class CodeGraphSource {
52
28
  this.config = config;
53
29
  this.queryLogPath = queryLogPath;
54
30
  }
55
- async getImpactContext(task, skill, changedFiles) {
56
- const request = {
57
- type: "codegraph.impact",
58
- task,
59
- skill: skill.name,
60
- queries: skill.contextQueries,
61
- changedFiles
62
- };
63
- return this.query("codegraph", this.config.mcp.codegraph.url, request);
64
- }
65
- async query(source, url, request) {
66
- try {
67
- const data = await postJson(url, request, this.config.mcp.timeoutMs);
68
- const result = {
69
- source,
70
- ok: true,
71
- degraded: false,
72
- request,
73
- responseSummary: summarize(data),
74
- data
75
- };
76
- await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
77
- return result;
78
- }
79
- catch (error) {
80
- const result = {
81
- source,
82
- ok: false,
83
- degraded: true,
84
- request,
85
- error: error instanceof Error ? error.message : String(error)
86
- };
87
- await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
88
- return result;
89
- }
31
+ async getImpactContext(_task, skill) {
32
+ const result = agentDrivenResult("codegraph", this.config.mcp.codegraph, skill);
33
+ await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
34
+ return result;
90
35
  }
91
36
  }
92
37
  export class KnowledgeGraphSource {
@@ -96,38 +41,10 @@ export class KnowledgeGraphSource {
96
41
  this.config = config;
97
42
  this.queryLogPath = queryLogPath;
98
43
  }
99
- async searchKnowledge(task, skill, codeContext) {
100
- const request = {
101
- type: "knowledge.search",
102
- task,
103
- skill: skill.name,
104
- queries: skill.contextQueries,
105
- codeContext
106
- };
107
- try {
108
- const data = await postJson(this.config.mcp.knowledgeGraph.url, request, this.config.mcp.timeoutMs);
109
- const result = {
110
- source: "knowledgeGraph",
111
- ok: true,
112
- degraded: false,
113
- request,
114
- responseSummary: summarize(data),
115
- data
116
- };
117
- await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
118
- return result;
119
- }
120
- catch (error) {
121
- const result = {
122
- source: "knowledgeGraph",
123
- ok: false,
124
- degraded: true,
125
- request,
126
- error: error instanceof Error ? error.message : String(error)
127
- };
128
- await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
129
- return result;
130
- }
44
+ async searchKnowledge(_task, skill) {
45
+ const result = agentDrivenResult("knowledgeGraph", this.config.mcp.knowledgeGraph, skill);
46
+ await appendFile(this.queryLogPath, `${JSON.stringify(result)}\n`, "utf8");
47
+ return result;
131
48
  }
132
49
  }
133
50
  export class ContextSource {
@@ -139,7 +56,7 @@ export class ContextSource {
139
56
  }
140
57
  async searchTaskContext(task, skill) {
141
58
  const codeContext = await this.codeGraph.getImpactContext(task, skill);
142
- const knowledgeContext = await this.knowledgeGraph.searchKnowledge(task, skill, codeContext.data ?? codeContext.error);
59
+ const knowledgeContext = await this.knowledgeGraph.searchKnowledge(task, skill);
143
60
  return { codeContext, knowledgeContext };
144
61
  }
145
62
  }
@@ -0,0 +1,91 @@
1
+ import { readdir, readFile } from "node:fs/promises";
2
+ import path from "node:path";
3
+ import { pathExists } from "./utils.js";
4
+ async function readJson(filePath) {
5
+ try {
6
+ return JSON.parse(await readFile(filePath, "utf8"));
7
+ }
8
+ catch {
9
+ return undefined;
10
+ }
11
+ }
12
+ function increment(map, key) {
13
+ map[key] = (map[key] ?? 0) + 1;
14
+ }
15
+ export async function collectRunMetrics(cwd, config) {
16
+ const runsRoot = path.join(cwd, config.audit.path);
17
+ const entries = (await pathExists(runsRoot)) ? await readdir(runsRoot, { withFileTypes: true }) : [];
18
+ const runs = [];
19
+ for (const entry of entries.filter((item) => item.isDirectory())) {
20
+ const runDir = path.join(runsRoot, entry.name);
21
+ const intake = await readJson(path.join(runDir, "intake.json"));
22
+ const review = await readJson(path.join(runDir, "loop_review.json"));
23
+ const verify = await readJson(path.join(runDir, "verify.json"));
24
+ const last = review?.iterations?.at(-1);
25
+ runs.push({
26
+ runId: entry.name,
27
+ skill: intake?.skill,
28
+ agent: intake?.agent,
29
+ status: review?.status ?? "unknown",
30
+ iterations: review?.iterations?.length ?? 0,
31
+ failureType: last?.failureType ?? "unknown",
32
+ contractStatus: last?.signals?.contractStatus,
33
+ verifyStatus: last?.signals?.verifyStatus,
34
+ contextScore: last?.signals?.contextScoreStatus,
35
+ approvalRequired: Boolean(last?.signals?.approvalRequired),
36
+ failedVerifyCommands: verify?.commands?.filter((command) => command.exitCode !== 0).map((command) => command.name) ?? []
37
+ });
38
+ }
39
+ const failureTypes = {};
40
+ const skillStatus = {};
41
+ const agentStatus = {};
42
+ const verifyFailures = {};
43
+ const contextScores = {};
44
+ for (const run of runs) {
45
+ increment(failureTypes, run.failureType);
46
+ if (run.skill)
47
+ increment(skillStatus, `${run.skill}:${run.status}`);
48
+ if (run.agent)
49
+ increment(agentStatus, `${run.agent}:${run.status}`);
50
+ for (const command of run.failedVerifyCommands)
51
+ increment(verifyFailures, command);
52
+ if (run.contextScore)
53
+ increment(contextScores, run.contextScore);
54
+ }
55
+ const total = runs.length;
56
+ const passed = runs.filter((run) => run.status === "passed").length;
57
+ const firstPass = runs.filter((run) => run.status === "passed" && run.iterations === 1).length;
58
+ const contractPassed = runs.filter((run) => run.contractStatus === "passed").length;
59
+ const averageIterations = total ? runs.reduce((sum, run) => sum + run.iterations, 0) / total : 0;
60
+ return {
61
+ runs,
62
+ summary: {
63
+ totalRuns: total,
64
+ successRate: total ? passed / total : 0,
65
+ firstPassRate: total ? firstPass / total : 0,
66
+ agentContractPassRate: total ? contractPassed / total : 0,
67
+ averageIterations,
68
+ failureTypes,
69
+ skillStatus,
70
+ agentStatus,
71
+ verifyFailureTopN: Object.fromEntries(Object.entries(verifyFailures).sort((a, b) => b[1] - a[1]).slice(0, 10)),
72
+ approvalGateCount: runs.filter((run) => run.approvalRequired).length,
73
+ contextScores
74
+ }
75
+ };
76
+ }
77
+ export function renderMetricsText(metrics) {
78
+ return `运行指标
79
+ - 总运行次数: ${metrics.summary.totalRuns}
80
+ - 成功率: ${Number(metrics.summary.successRate).toFixed(2)}
81
+ - 一次通过率: ${Number(metrics.summary.firstPassRate).toFixed(2)}
82
+ - Agent 契约通过率: ${Number(metrics.summary.agentContractPassRate).toFixed(2)}
83
+ - 平均循环轮次: ${Number(metrics.summary.averageIterations).toFixed(2)}
84
+ - 失败类型分布: ${JSON.stringify(metrics.summary.failureTypes)}
85
+ - Skill 状态: ${JSON.stringify(metrics.summary.skillStatus)}
86
+ - Agent 状态: ${JSON.stringify(metrics.summary.agentStatus)}
87
+ - 验证失败 Top N: ${JSON.stringify(metrics.summary.verifyFailureTopN)}
88
+ - Approval Gate 触发次数: ${metrics.summary.approvalGateCount}
89
+ - Context Score 分布: ${JSON.stringify(metrics.summary.contextScores)}
90
+ `;
91
+ }
package/dist/replay.js ADDED
@@ -0,0 +1,27 @@
1
+ import { readFile } from "node:fs/promises";
2
+ import path from "node:path";
3
+ export async function buildReplayPlan(runDir, overrideAgent, dry = false) {
4
+ const intake = JSON.parse(await readFile(path.join(runDir, "intake.json"), "utf8"));
5
+ const agent = overrideAgent ?? intake.agent;
6
+ if (agent !== "codex" && agent !== "claude" && agent !== "cursor" && agent !== "gemini") {
7
+ throw new Error(`历史运行记录中的 agent 无效:${agent}`);
8
+ }
9
+ if (!intake.task || !intake.skill)
10
+ throw new Error(`历史运行记录缺少 task 或 skill:${runDir}`);
11
+ return {
12
+ sourceRun: runDir,
13
+ task: intake.task,
14
+ skill: intake.skill,
15
+ agent,
16
+ dry
17
+ };
18
+ }
19
+ export function renderReplayPlan(plan) {
20
+ return `Replay Plan
21
+ - Source run: ${plan.sourceRun}
22
+ - Task: ${plan.task}
23
+ - Skill: ${plan.skill}
24
+ - Agent: ${plan.agent}
25
+ - Dry run: ${plan.dry ? "yes" : "no"}
26
+ `;
27
+ }