@rallycry/conveyor-agent 7.1.3 → 7.1.4

package/dist/task-audit-handler-URD2BOC4.js

@@ -0,0 +1,381 @@
+import {
+  textResult
+} from "./chunk-C5YAMQJ2.js";
+import {
+  createHarness,
+  createServiceLogger,
+  defineTool
+} from "./chunk-U6AYNVS7.js";
+
+// src/runner/task-audit-handler.ts
+import { z } from "zod";
+var logger = createServiceLogger("TaskAudit");
+var FALLBACK_MODEL = "claude-sonnet-4-20250514";
+var TASK_AUDIT_SYSTEM_PROMPT = `You are a chess-style analysis engine for AI agent task execution. You grade each turn of an agent's work trace like a chess engine grades moves: Correct, Neutral, or Blunder.
+
+You receive the agent's full execution trace (tool calls, reasoning, errors) along with the task plan, description, and human messages. Your job is to objectively assess the quality of the agent's work.
+
+## Grading Scale
+
+- **Correct** (\u265F): Sound moves toward completion \u2014 necessary file reads, proper tool use, following plan steps, logical reasoning, adapting to feedback
+- **Neutral** (\xB7): No significant positive or negative effect \u2014 exploratory reads, clarifying questions, minor tangents that don't waste resources
+- **Blunder** (??): Wasted effort or wrong direction \u2014 incorrect edits, reading irrelevant files repeatedly, ignoring the plan, repeated failures without adapting
+
+## Phase Identification
+
+Classify each turn into a phase based on the trace context:
+- **planning**: Events before the first file modification (Edit/Write tool use) \u2014 reading files, analyzing requirements, reasoning about approach
+- **building**: File modification events and related reasoning \u2014 writing code, running tests, fixing errors
+- **human**: Turns that are direct responses to user/human chat messages
+
+## Specific Blunder Examples
+- Reading the same file 5+ times without making meaningful changes
+- Making edits that are immediately reverted or overwritten
+- Ignoring explicit plan steps or user corrections
+- Repeated failed tool calls (isError: true) without changing approach
+- Using wrong APIs despite project patterns visible in the trace
+- Spending many turns on tangential work not in the plan
+
+## Specific Correct Examples
+- Reading context files before implementation
+- Following plan steps in logical sequence
+- Proper error handling and recovery after tool failures
+- Adapting approach based on user feedback
+- Efficient tool usage with clear purpose
+
+## Output Format
+
+After analyzing all turns, output a JSON block:
+
+\`\`\`json
+{
+  "turnGrades": [
+    {
+      "turnIndex": 0,
+      "phase": "planning",
+      "grade": "correct",
+      "reasoning": "Read relevant config files to understand project structure before making changes",
+      "eventType": "tool_use",
+      "eventSummary": "Read package.json and tsconfig.json"
+    }
+  ],
+  "summary": "Brief 2-3 sentence assessment of overall execution quality",
+  "planningAccuracy": 0.85,
+  "buildingAccuracy": 0.72,
+  "humanAccuracy": 1.0
+}
+\`\`\`
+
+**Accuracy Calculation**: For each phase, accuracy = correct / (correct + blunder). Neutral turns are excluded from the ratio. Return null if a phase has no correct or blunder turns.
+
+## Suggestion Guidelines
+
+After grading, if you identify actionable cross-cutting improvements that would help future tasks (not one-off fixes), call the \`create_suggestion\` tool. Focus on:
+- Recurring patterns of wasted effort
+- Missing documentation or context that caused confusion
+- Process improvements that would prevent common blunders
+- Tool or workflow optimizations`;
+function buildSessionInfoSection(info) {
+  const lines = ["## Session Info"];
+  if (info.model) lines.push(`- Model: ${info.model}`);
+  if (info.totalCostUsd !== null) lines.push(`- Total Cost: $${info.totalCostUsd.toFixed(4)}`);
+  if (info.planningTurns !== null) lines.push(`- Planning Turns: ${info.planningTurns}`);
+  if (info.buildingTurns !== null) lines.push(`- Building Turns: ${info.buildingTurns}`);
+  if (info.locDiff !== null) lines.push(`- Lines Changed: ${info.locDiff}`);
+  lines.push("");
+  return lines.join("\n");
+}
+function buildTraceSection(trace) {
+  const lines = ["## Agent Trace\n"];
+  if (trace.length === 0) {
+    lines.push("No agent trace data available.\n");
+  } else {
+    for (let i = 0; i < trace.length; i++) {
+      const event = trace[i];
+      const dataStr = JSON.stringify(event.data);
+      const truncated = dataStr.length > 2e3 ? dataStr.slice(0, 2e3) + "..." : dataStr;
+      lines.push(`### Turn ${i} [${event.type}] @ ${event.timestamp}`);
+      lines.push(truncated);
+      lines.push("");
+    }
+  }
+  return lines.join("\n");
+}
+function buildTaskAuditPrompt(task) {
+  const parts = [`# Task Audit: ${task.taskTitle}
+`];
+  if (task.taskDescription) parts.push(`## Description
+${task.taskDescription}
+`);
+  if (task.taskPlan) parts.push(`## Plan
+${task.taskPlan}
+`);
+  parts.push(`## Task Status: ${task.taskStatus}`);
+  if (task.storyPointValue !== null) parts.push(`## Story Points: ${task.storyPointValue}`);
+  parts.push("");
+  if (task.sessionInfo) parts.push(buildSessionInfoSection(task.sessionInfo));
+  if (task.agentInstructions) parts.push(`## Agent Instructions
+${task.agentInstructions}
+`);
+  if (task.humanMessages.length > 0) {
+    parts.push("## Human Messages\n");
+    for (const msg of task.humanMessages) {
+      parts.push(`[${msg.createdAt}] **${msg.role}**: ${msg.content}`);
+    }
+    parts.push("");
+  }
+  parts.push(buildTraceSection(task.agentTrace));
+  parts.push("Grade each turn above, then output the structured JSON result.");
+  return parts.join("\n");
+}
+function buildProjectSuggestionTool(connection, projectId) {
+  return defineTool(
+    "create_suggestion",
+    "Suggest a cross-cutting improvement for the project based on patterns found during audit. If a similar suggestion exists, your vote is added instead of creating a duplicate.",
+    {
+      title: z.string().describe("Short title for the suggestion"),
+      description: z.string().optional().describe("Details about the suggestion"),
+      tag_names: z.array(z.string()).optional().describe("Tag names to categorize the suggestion")
+    },
+    async ({ title, description, tag_names }) => {
+      try {
+        const result = await connection.call("createProjectSuggestion", {
+          projectId,
+          title,
+          description,
+          tagNames: tag_names
+        });
+        if (result.merged) {
+          return textResult(
+            `Merged into existing suggestion (ID: ${result.mergedIntoId ?? result.id}).`
+          );
+        }
+        return textResult(`Suggestion created (ID: ${result.id}).`);
+      } catch (error) {
+        return textResult(
+          `Failed to create suggestion: ${error instanceof Error ? error.message : "Unknown error"}`
+        );
+      }
+    }
+  );
+}
+async function fetchModel(connection) {
+  try {
+    const ctx = await connection.call("getProjectAgentContext", {
+      projectId: connection.projectId
+    });
+    return ctx.model || FALLBACK_MODEL;
+  } catch {
+    return FALLBACK_MODEL;
+  }
+}
+function toAuditResult(parsed) {
+  if (!parsed || !Array.isArray(parsed.turnGrades)) return null;
+  return {
+    turnGrades: parsed.turnGrades,
+    summary: parsed.summary ?? "",
+    planningAccuracy: parsed.planningAccuracy ?? null,
+    buildingAccuracy: parsed.buildingAccuracy ?? null,
+    humanAccuracy: parsed.humanAccuracy ?? null
+  };
+}
+function extractJsonFromResponse(text) {
+  const codeBlockMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/);
+  const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : text;
+  try {
+    const result = toAuditResult(JSON.parse(jsonStr));
+    if (result) return result;
+  } catch {
+  }
+  const objectMatch = text.match(/\{[\s\S]*"turnGrades"[\s\S]*\}/);
+  if (objectMatch) {
+    try {
+      const result = toAuditResult(JSON.parse(objectMatch[0]));
+      if (result) return result;
+    } catch {
+    }
+  }
+  throw new Error("Could not parse task audit grades from Claude response");
+}
+function computeGradeCounts(turnGrades) {
+  const counts = {
+    planningCorrect: 0,
+    planningNeutral: 0,
+    planningBlunder: 0,
+    buildingCorrect: 0,
+    buildingNeutral: 0,
+    buildingBlunder: 0,
+    humanCorrect: 0,
+    humanNeutral: 0,
+    humanBlunder: 0
+  };
+  for (const tg of turnGrades) {
+    const key = `${tg.phase}${tg.grade.charAt(0).toUpperCase()}${tg.grade.slice(1)}`;
+    if (key in counts) {
+      counts[key]++;
+    }
+  }
+  return counts;
+}
+async function collectResponseFromEvents(events, connection, requestId, taskId) {
+  const responseParts = [];
+  let costUsd = null;
+  let model = null;
+  for await (const event of events) {
+    if (event.type === "system") {
+      const sysEvent = event;
+      if (sysEvent.subtype === "init" && sysEvent.model) {
+        model = sysEvent.model;
+      }
+    }
+    if (event.type === "assistant") {
+      const { message } = event;
+      for (const block of message.content) {
+        if (block.type === "text" && block.text) {
+          responseParts.push(block.text);
+        } else if (block.type === "tool_use" && block.name) {
+          const inputStr = typeof block.input === "string" ? block.input : JSON.stringify(block.input);
+          await connection.call("reportTaskAuditProgress", {
+            projectId: connection.projectId,
+            requestId,
+            taskId,
+            activity: `${block.name}: ${inputStr.slice(0, 500)}`
+          }).catch(() => {
+          });
+        }
+      }
+    }
+    if (event.type === "result") {
+      const resultEvent = event;
+      if (resultEvent.subtype === "success" && resultEvent.total_cost_usd !== void 0) {
+        costUsd = resultEvent.total_cost_usd;
+      }
+      break;
+    }
+  }
+  return { text: responseParts.join("\n\n").trim(), costUsd, model };
+}
+function buildErrorResult(projectId, requestId, taskId, error) {
+  return {
+    projectId,
+    requestId,
+    taskId,
+    summary: "",
+    turnGrades: [],
+    planningAccuracy: null,
+    buildingAccuracy: null,
+    humanAccuracy: null,
+    planningCorrect: 0,
+    planningNeutral: 0,
+    planningBlunder: 0,
+    buildingCorrect: 0,
+    buildingNeutral: 0,
+    buildingBlunder: 0,
+    humanCorrect: 0,
+    humanNeutral: 0,
+    humanBlunder: 0,
+    suggestionIds: [],
+    auditCostUsd: null,
+    model: null,
+    error
+  };
+}
+async function auditSingleTask(taskPayload, request, connection, projectDir, model) {
+  const { requestId, projectId } = request;
+  await connection.call("reportTaskAuditProgress", {
+    projectId: connection.projectId,
+    requestId,
+    taskId: taskPayload.taskId,
+    activity: "Starting task audit..."
+  }).catch(() => {
+  });
+  const suggestionIds = [];
+  const suggestionTool = buildProjectSuggestionTool(connection, projectId);
+  const wrappedSuggestionTool = defineTool(
+    suggestionTool.name,
+    suggestionTool.description,
+    suggestionTool.schema,
+    async (input) => {
+      const result = await suggestionTool.handler(input);
+      const text = result.content?.[0]?.text ?? "";
+      const idMatch = text.match(/ID: ([a-z0-9]+)/i);
+      if (idMatch) suggestionIds.push(idMatch[1]);
+      return result;
+    }
+  );
+  const harness = createHarness();
+  const mcpServer = harness.createMcpServer({
+    name: "task-audit",
+    tools: [wrappedSuggestionTool]
+  });
+  const events = harness.executeQuery({
+    prompt: buildTaskAuditPrompt(taskPayload),
+    options: {
+      model,
+      systemPrompt: TASK_AUDIT_SYSTEM_PROMPT,
+      cwd: projectDir,
+      permissionMode: "bypassPermissions",
+      allowDangerouslySkipPermissions: true,
+      tools: { type: "preset", preset: "claude_code" },
+      mcpServers: { "task-audit": mcpServer },
+      maxTurns: 6,
+      maxBudgetUsd: 3,
+      disallowedTools: ["Edit", "Write", "Bash", "NotebookEdit", "MultiEdit"]
+    }
+  });
+  const response = await collectResponseFromEvents(
+    events,
+    connection,
+    requestId,
+    taskPayload.taskId
+  );
+  if (!response.text) throw new Error("No response from Claude");
+  const parsed = extractJsonFromResponse(response.text);
+  const counts = computeGradeCounts(parsed.turnGrades);
+  logger.info("Task audit complete", {
+    requestId,
+    taskId: taskPayload.taskId,
+    grades: parsed.turnGrades.length,
+    suggestions: suggestionIds.length
+  });
+  await connection.call("reportTaskAuditResult", {
+    projectId: connection.projectId,
+    requestId,
+    taskId: taskPayload.taskId,
+    summary: parsed.summary,
+    turnGrades: parsed.turnGrades,
+    planningAccuracy: parsed.planningAccuracy,
+    buildingAccuracy: parsed.buildingAccuracy,
+    humanAccuracy: parsed.humanAccuracy,
+    ...counts,
+    suggestionIds,
+    auditCostUsd: response.costUsd,
+    model: response.model
+  });
+}
+async function handleTaskAudit(request, connection, projectDir) {
+  const { requestId } = request;
+  logger.info("Starting task audit", { requestId, taskCount: request.tasks.length });
+  const model = await fetchModel(connection);
+  for (const taskPayload of request.tasks) {
+    try {
+      await auditSingleTask(taskPayload, request, connection, projectDir, model);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      logger.error("Task audit failed for task", {
+        requestId,
+        taskId: taskPayload.taskId,
+        error: msg
+      });
+      await connection.call(
+        "reportTaskAuditResult",
+        buildErrorResult(connection.projectId, requestId, taskPayload.taskId, msg)
+      ).catch(() => {
+      });
+    }
+  }
+}
+export {
+  handleTaskAudit
+};
+//# sourceMappingURL=task-audit-handler-URD2BOC4.js.map

package/dist/task-audit-handler-URD2BOC4.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/runner/task-audit-handler.ts"],"sourcesContent":["import { z } from \"zod\";\nimport { createHarness, defineTool } from \"../harness/index.js\";\nimport type { ProjectConnection } from \"../connection/project-connection.js\";\nimport type { TaskAuditRunnerRequest, TaskAuditPayload, TaskAuditTurnGrade } from \"@project/shared\";\nimport { textResult } from \"../tools/helpers.js\";\nimport { createServiceLogger } from \"../utils/logger.js\";\n\nconst logger = createServiceLogger(\"TaskAudit\");\n\nconst FALLBACK_MODEL = \"claude-sonnet-4-20250514\";\n\n// ── System prompt ────────────────────────────────────────────────────────\n\nconst TASK_AUDIT_SYSTEM_PROMPT = `You are a chess-style analysis engine for AI agent task execution. You grade each turn of an agent's work trace like a chess engine grades moves: Correct, Neutral, or Blunder.\n\nYou receive the agent's full execution trace (tool calls, reasoning, errors) along with the task plan, description, and human messages. Your job is to objectively assess the quality of the agent's work.\n\n## Grading Scale\n\n- **Correct** (♟): Sound moves toward completion — necessary file reads, proper tool use, following plan steps, logical reasoning, adapting to feedback\n- **Neutral** (·): No significant positive or negative effect — exploratory reads, clarifying questions, minor tangents that don't waste resources\n- **Blunder** (??): Wasted effort or wrong direction — incorrect edits, reading irrelevant files repeatedly, ignoring the plan, repeated failures without adapting\n\n## Phase Identification\n\nClassify each turn into a phase based on the trace context:\n- **planning**: Events before the first file modification (Edit/Write tool use) — reading files, analyzing requirements, reasoning about approach\n- **building**: File modification events and related reasoning — writing code, running tests, fixing errors\n- **human**: Turns that are direct responses to user/human chat messages\n\n## Specific Blunder Examples\n- Reading the same file 5+ times without making meaningful changes\n- Making edits that are immediately reverted or overwritten\n- Ignoring explicit plan steps or user corrections\n- Repeated failed tool calls (isError: true) without changing approach\n- Using wrong APIs despite project patterns visible in the trace\n- Spending many turns on tangential work not in the plan\n\n## Specific Correct Examples\n- Reading context files before implementation\n- Following plan steps in logical sequence\n- Proper error handling and recovery after tool failures\n- Adapting approach based on user feedback\n- Efficient tool usage with clear purpose\n\n## Output Format\n\nAfter analyzing all turns, output a JSON block:\n\n\\`\\`\\`json\n{\n  \"turnGrades\": [\n    {\n      \"turnIndex\": 0,\n      \"phase\": \"planning\",\n      \"grade\": \"correct\",\n      \"reasoning\": \"Read relevant config files to understand project structure before making changes\",\n      \"eventType\": \"tool_use\",\n      \"eventSummary\": \"Read package.json and tsconfig.json\"\n    }\n  ],\n  \"summary\": \"Brief 2-3 sentence assessment of overall execution quality\",\n  \"planningAccuracy\": 0.85,\n  \"buildingAccuracy\": 0.72,\n  \"humanAccuracy\": 1.0\n}\n\\`\\`\\`\n\n**Accuracy Calculation**: For each phase, accuracy = correct / (correct + blunder). Neutral turns are excluded from the ratio. Return null if a phase has no correct or blunder turns.\n\n## Suggestion Guidelines\n\nAfter grading, if you identify actionable cross-cutting improvements that would help future tasks (not one-off fixes), call the \\`create_suggestion\\` tool. Focus on:\n- Recurring patterns of wasted effort\n- Missing documentation or context that caused confusion\n- Process improvements that would prevent common blunders\n- Tool or workflow optimizations`;\n\n// ── Prompt section builders ─────────────────────────────────────────────\n\nfunction buildSessionInfoSection(info: NonNullable<TaskAuditPayload[\"sessionInfo\"]>): string {\n  const lines: string[] = [\"## Session Info\"];\n  if (info.model) lines.push(`- Model: ${info.model}`);\n  if (info.totalCostUsd !== null) lines.push(`- Total Cost: $${info.totalCostUsd.toFixed(4)}`);\n  if (info.planningTurns !== null) lines.push(`- Planning Turns: ${info.planningTurns}`);\n  if (info.buildingTurns !== null) lines.push(`- Building Turns: ${info.buildingTurns}`);\n  if (info.locDiff !== null) lines.push(`- Lines Changed: ${info.locDiff}`);\n  lines.push(\"\");\n  return lines.join(\"\\n\");\n}\n\nfunction buildTraceSection(trace: TaskAuditPayload[\"agentTrace\"]): string {\n  const lines: string[] = [\"## Agent Trace\\n\"];\n  if (trace.length === 0) {\n    lines.push(\"No agent trace data available.\\n\");\n  } else {\n    for (let i = 0; i < trace.length; i++) {\n      const event = trace[i];\n      const dataStr = JSON.stringify(event.data);\n      const truncated = dataStr.length > 2000 ? dataStr.slice(0, 2000) + \"...\" : dataStr;\n      lines.push(`### Turn ${i} [${event.type}] @ ${event.timestamp}`);\n      lines.push(truncated);\n      lines.push(\"\");\n    }\n  }\n  return lines.join(\"\\n\");\n}\n\n// ── Prompt builder ──────────────────────────────────────────────────────\n\nfunction buildTaskAuditPrompt(task: TaskAuditPayload): string {\n  const parts: string[] = [`# Task Audit: ${task.taskTitle}\\n`];\n\n  if (task.taskDescription) parts.push(`## Description\\n${task.taskDescription}\\n`);\n  if (task.taskPlan) parts.push(`## Plan\\n${task.taskPlan}\\n`);\n\n  parts.push(`## Task Status: ${task.taskStatus}`);\n  if (task.storyPointValue !== null) parts.push(`## Story Points: ${task.storyPointValue}`);\n  parts.push(\"\");\n\n  if (task.sessionInfo) parts.push(buildSessionInfoSection(task.sessionInfo));\n  if (task.agentInstructions) parts.push(`## Agent Instructions\\n${task.agentInstructions}\\n`);\n\n  if (task.humanMessages.length > 0) {\n    parts.push(\"## Human Messages\\n\");\n    for (const msg of task.humanMessages) {\n      parts.push(`[${msg.createdAt}] **${msg.role}**: ${msg.content}`);\n    }\n    parts.push(\"\");\n  }\n\n  parts.push(buildTraceSection(task.agentTrace));\n  parts.push(\"Grade each turn above, then output the structured JSON result.\");\n\n  return parts.join(\"\\n\");\n}\n\n// ── Suggestion MCP tool ─────────────────────────────────────────────────\n\nfunction buildProjectSuggestionTool(connection: ProjectConnection, projectId: string) {\n  return defineTool(\n    \"create_suggestion\",\n    \"Suggest a cross-cutting improvement for the project based on patterns found during audit. If a similar suggestion exists, your vote is added instead of creating a duplicate.\",\n    {\n      title: z.string().describe(\"Short title for the suggestion\"),\n      description: z.string().optional().describe(\"Details about the suggestion\"),\n      tag_names: z.array(z.string()).optional().describe(\"Tag names to categorize the suggestion\"),\n    },\n    async ({ title, description, tag_names }) => {\n      try {\n        const result = await connection.call(\"createProjectSuggestion\", {\n          projectId,\n          title,\n          description,\n          tagNames: tag_names,\n        });\n        if (result.merged) {\n          return textResult(\n            `Merged into existing suggestion (ID: ${result.mergedIntoId ?? result.id}).`,\n          );\n        }\n        return textResult(`Suggestion created (ID: ${result.id}).`);\n      } catch (error) {\n        return textResult(\n          `Failed to create suggestion: ${error instanceof Error ? error.message : \"Unknown error\"}`,\n        );\n      }\n    },\n  );\n}\n\n// ── Fetch model ─────────────────────────────────────────────────────────\n\nasync function fetchModel(connection: ProjectConnection): Promise<string> {\n  try {\n    const ctx = await connection.call(\"getProjectAgentContext\", {\n      projectId: connection.projectId,\n    });\n    return ctx.model || FALLBACK_MODEL;\n  } catch {\n    return FALLBACK_MODEL;\n  }\n}\n\n// ── JSON extraction ─────────────────────────────────────────────────────\n\ninterface AuditJsonResult {\n  turnGrades: TaskAuditTurnGrade[];\n  summary: string;\n  planningAccuracy: number | null;\n  buildingAccuracy: number | null;\n  humanAccuracy: number | null;\n}\n\nfunction toAuditResult(parsed: Record<string, unknown>): AuditJsonResult | null {\n  if (!parsed || !Array.isArray(parsed.turnGrades)) return null;\n  return {\n    turnGrades: parsed.turnGrades as TaskAuditTurnGrade[],\n    summary: (parsed.summary as string) ?? \"\",\n    planningAccuracy: (parsed.planningAccuracy as number) ?? null,\n    buildingAccuracy: (parsed.buildingAccuracy as number) ?? null,\n    humanAccuracy: (parsed.humanAccuracy as number) ?? null,\n  };\n}\n\nfunction extractJsonFromResponse(text: string): AuditJsonResult {\n  const codeBlockMatch = text.match(/```(?:json)?\\s*\\n?([\\s\\S]*?)```/);\n  const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : text;\n\n  try {\n    const result = toAuditResult(JSON.parse(jsonStr));\n    if (result) return result;\n  } catch {\n    // Try fallback: find a JSON object in raw text\n  }\n\n  const objectMatch = text.match(/\\{[\\s\\S]*\"turnGrades\"[\\s\\S]*\\}/);\n  if (objectMatch) {\n    try {\n      const result = toAuditResult(JSON.parse(objectMatch[0]));\n      if (result) return result;\n    } catch {\n      // Fall through\n    }\n  }\n\n  throw new Error(\"Could not parse task audit grades from Claude response\");\n}\n\n// ── Compute grade counts ────────────────────────────────────────────────\n\nfunction computeGradeCounts(turnGrades: TaskAuditTurnGrade[]) {\n  const counts = {\n    planningCorrect: 0,\n    planningNeutral: 0,\n    planningBlunder: 0,\n    buildingCorrect: 0,\n    buildingNeutral: 0,\n    buildingBlunder: 0,\n    humanCorrect: 0,\n    humanNeutral: 0,\n    humanBlunder: 0,\n  };\n\n  for (const tg of turnGrades) {\n    const key =\n      `${tg.phase}${tg.grade.charAt(0).toUpperCase()}${tg.grade.slice(1)}` as keyof typeof counts;\n    if (key in counts) {\n      counts[key]++;\n    }\n  }\n\n  return counts;\n}\n\n// ── Event stream processing ─────────────────────────────────────────────\n\nasync function collectResponseFromEvents(\n  events: AsyncIterable<{ type: string }>,\n  connection: ProjectConnection,\n  requestId: string,\n  taskId: string,\n): Promise<{ text: string; costUsd: number | null; model: string | null }> {\n  const responseParts: string[] = [];\n  let costUsd: number | null = null;\n  let model: string | null = null;\n\n  for await (const event of events) {\n    if (event.type === \"system\") {\n      const sysEvent = event as { type: string; subtype?: string; model?: string };\n      if (sysEvent.subtype === \"init\" && sysEvent.model) {\n        model = sysEvent.model;\n      }\n    }\n\n    if (event.type === \"assistant\") {\n      const { message } = event as {\n        type: string;\n        message: {\n          content: Array<{ type: string; text?: string; name?: string; input?: unknown }>;\n        };\n      };\n      for (const block of message.content) {\n        if (block.type === \"text\" && block.text) {\n          responseParts.push(block.text);\n        } else if (block.type === \"tool_use\" && block.name) {\n          const inputStr =\n            typeof block.input === \"string\" ? block.input : JSON.stringify(block.input);\n          await connection\n            .call(\"reportTaskAuditProgress\", {\n              projectId: connection.projectId,\n              requestId,\n              taskId,\n              activity: `${block.name}: ${inputStr.slice(0, 500)}`,\n            })\n            .catch(() => {});\n        }\n      }\n    }\n\n    if (event.type === \"result\") {\n      const resultEvent = event as {\n        type: string;\n        subtype?: string;\n        total_cost_usd?: number;\n      };\n      if (resultEvent.subtype === \"success\" && resultEvent.total_cost_usd !== undefined) {\n        costUsd = resultEvent.total_cost_usd;\n      }\n      break;\n    }\n  }\n\n  return { text: responseParts.join(\"\\n\\n\").trim(), costUsd, model };\n}\n\n// ── Error result helper ─────────────────────────────────────────────────\n\nfunction buildErrorResult(projectId: string, requestId: string, taskId: string, error: string) {\n  return {\n    projectId,\n    requestId,\n    taskId,\n    summary: \"\",\n    turnGrades: [] as TaskAuditTurnGrade[],\n    planningAccuracy: null,\n    buildingAccuracy: null,\n    humanAccuracy: null,\n    planningCorrect: 0,\n    planningNeutral: 0,\n    planningBlunder: 0,\n    buildingCorrect: 0,\n    buildingNeutral: 0,\n    buildingBlunder: 0,\n    humanCorrect: 0,\n    humanNeutral: 0,\n    humanBlunder: 0,\n    suggestionIds: [] as string[],\n    auditCostUsd: null,\n    model: null,\n    error,\n  };\n}\n\n// ── Per-task audit execution ────────────────────────────────────────────\n\nasync function auditSingleTask(\n  taskPayload: TaskAuditPayload,\n  request: TaskAuditRunnerRequest,\n  connection: ProjectConnection,\n  projectDir: string,\n  model: string,\n): Promise<void> {\n  const { requestId, projectId } = request;\n\n  await connection\n    .call(\"reportTaskAuditProgress\", {\n      projectId: connection.projectId,\n      requestId,\n      taskId: taskPayload.taskId,\n      activity: \"Starting task audit...\",\n    })\n    .catch(() => {});\n\n  const suggestionIds: string[] = [];\n  const suggestionTool = buildProjectSuggestionTool(connection, projectId);\n\n  // Wrap the suggestion tool to capture created IDs\n  const wrappedSuggestionTool = defineTool(\n    suggestionTool.name,\n    suggestionTool.description,\n    suggestionTool.schema as z.ZodRawShape,\n    async (input: Record<string, unknown>) => {\n      const result = await suggestionTool.handler(input);\n      const text = result.content?.[0]?.text ?? \"\";\n      const idMatch = text.match(/ID: ([a-z0-9]+)/i);\n      if (idMatch) suggestionIds.push(idMatch[1]);\n      return result;\n    },\n  );\n\n  const harness = createHarness();\n  const mcpServer = harness.createMcpServer({\n    name: \"task-audit\",\n    tools: [wrappedSuggestionTool],\n  });\n\n  const events = harness.executeQuery({\n    prompt: buildTaskAuditPrompt(taskPayload),\n    options: {\n      model,\n      systemPrompt: TASK_AUDIT_SYSTEM_PROMPT,\n      cwd: projectDir,\n      permissionMode: \"bypassPermissions\",\n      allowDangerouslySkipPermissions: true,\n      tools: { type: \"preset\" as const, preset: \"claude_code\" as const },\n      mcpServers: { \"task-audit\": mcpServer },\n      maxTurns: 6,\n      maxBudgetUsd: 3,\n      disallowedTools: [\"Edit\", \"Write\", \"Bash\", \"NotebookEdit\", \"MultiEdit\"],\n    },\n  });\n\n  const response = await collectResponseFromEvents(\n    events,\n    connection,\n    requestId,\n    taskPayload.taskId,\n  );\n  if (!response.text) throw new Error(\"No response from Claude\");\n\n  const parsed = extractJsonFromResponse(response.text);\n  const counts = computeGradeCounts(parsed.turnGrades);\n\n  logger.info(\"Task audit complete\", {\n    requestId,\n    taskId: taskPayload.taskId,\n    grades: parsed.turnGrades.length,\n    suggestions: suggestionIds.length,\n  });\n\n  await connection.call(\"reportTaskAuditResult\", {\n    projectId: connection.projectId,\n    requestId,\n    taskId: taskPayload.taskId,\n    summary: parsed.summary,\n    turnGrades: parsed.turnGrades,\n    planningAccuracy: parsed.planningAccuracy,\n    buildingAccuracy: parsed.buildingAccuracy,\n    humanAccuracy: parsed.humanAccuracy,\n    ...counts,\n    suggestionIds,\n    auditCostUsd: response.costUsd,\n    model: response.model,\n  });\n}\n\n// ── Main handler ────────────────────────────────────────────────────────\n\nexport async function handleTaskAudit(\n  request: TaskAuditRunnerRequest,\n  connection: ProjectConnection,\n  projectDir: string,\n): Promise<void> {\n  const { requestId } = request;\n\n  logger.info(\"Starting task audit\", { requestId, taskCount: request.tasks.length });\n\n  const model = await fetchModel(connection);\n\n  for (const taskPayload of request.tasks) {\n    try {\n      await auditSingleTask(taskPayload, request, connection, projectDir, model);\n    } catch (err) {\n      const msg = err instanceof Error ? err.message : String(err);\n      logger.error(\"Task audit failed for task\", {\n        requestId,\n        taskId: taskPayload.taskId,\n        error: msg,\n      });\n      await connection\n        .call(\n          \"reportTaskAuditResult\",\n          buildErrorResult(connection.projectId, requestId, taskPayload.taskId, msg),\n        )\n        .catch(() => {});\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;AAAA,SAAS,SAAS;AAOlB,IAAM,SAAS,oBAAoB,WAAW;AAE9C,IAAM,iBAAiB;AAIvB,IAAM,2BAA2B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmEjC,SAAS,wBAAwB,MAA4D;AAC3F,QAAM,QAAkB,CAAC,iBAAiB;AAC1C,MAAI,KAAK,MAAO,OAAM,KAAK,YAAY,KAAK,KAAK,EAAE;AACnD,MAAI,KAAK,iBAAiB,KAAM,OAAM,KAAK,kBAAkB,KAAK,aAAa,QAAQ,CAAC,CAAC,EAAE;AAC3F,MAAI,KAAK,kBAAkB,KAAM,OAAM,KAAK,qBAAqB,KAAK,aAAa,EAAE;AACrF,MAAI,KAAK,kBAAkB,KAAM,OAAM,KAAK,qBAAqB,KAAK,aAAa,EAAE;AACrF,MAAI,KAAK,YAAY,KAAM,OAAM,KAAK,oBAAoB,KAAK,OAAO,EAAE;AACxE,QAAM,KAAK,EAAE;AACb,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,kBAAkB,OAA+C;AACxE,QAAM,QAAkB,CAAC,kBAAkB;AAC3C,MAAI,MAAM,WAAW,GAAG;AACtB,UAAM,KAAK,kCAAkC;AAAA,EAC/C,OAAO;AACL,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,YAAM,QAAQ,MAAM,CAAC;AACrB,YAAM,UAAU,KAAK,UAAU,MAAM,IAAI;AACzC,YAAM,YAAY,QAAQ,SAAS,MAAO,QAAQ,MAAM,GAAG,GAAI,IAAI,QAAQ;AAC3E,YAAM,KAAK,YAAY,CAAC,KAAK,MAAM,IAAI,OAAO,MAAM,SAAS,EAAE;AAC/D,YAAM,KAAK,SAAS;AACpB,YAAM,KAAK,EAAE;AAAA,IACf;AAAA,EACF;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,SAAS,qBAAqB,MAAgC;AAC5D,QAAM,QAAkB,CAAC,iBAAiB,KAAK,SAAS;AAAA,CAAI;AAE5D,MAAI,KAAK,gBAAiB,OAAM,KAAK;AAAA,EAAmB,KAAK,eAAe;AAAA,CAAI;AAChF,MAAI,KAAK,SAAU,OAAM,KAAK;AAAA,EAAY,KAAK,QAAQ;AAAA,CAAI;AAE3D,QAAM,KAAK,mBAAmB,KAAK,UAAU,EAAE;AAC/C,MAAI,KAAK,oBAAoB,KAAM,OAAM,KAAK,oBAAoB,KAAK,eAAe,EAAE;AACxF,QAAM,KAAK,EAAE;AAEb,MAAI,KAAK,YAAa,OAAM,KAAK,wBAAwB,KAAK,WAAW,CAAC;AAC1E,MAAI,KAAK,kBAAmB,OAAM,KAAK;AAAA,EAA0B,KAAK,iBAAiB;AAAA,CAAI;AAE3F,MAAI,KAAK,cAAc,SAAS,GAAG;AACjC,UAAM,KAAK,qBAAqB;AAChC,eAAW,OAAO,KAAK,eAAe;AACpC,YAAM,KAAK,IAAI,IAAI,SAAS,OAAO,IAAI,IAAI,OAAO,IAAI,OAAO,EAAE;AAAA,IACjE;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,KAAK,kBAAkB,KAAK,UAAU,CAAC;AAC7C,QAAM,KAAK,gEAAgE;AAE3E,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,SAAS,2BAA2B,YAA+B,WAAmB;AACpF,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,MACE,OAAO,EAAE,OAAO,EAAE,SAAS,gCAAgC;AAAA,MAC3D,aAAa,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,8BAA8B;AAAA,MAC1E,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,SAAS,wCAAwC;AAAA,IAC7F;AAAA,IACA,OAAO,EAAE,OAAO,aAAa,UAAU,MAAM;AAC3C,UAAI;AACF,cAAM,SAAS,MAAM,WAAW,KAAK,2BAA2B;AAAA,UAC9D;AAAA,UACA;AAAA,UACA;AAAA,UACA,UAAU;AAAA,QACZ,CAAC;AACD,YAAI,OAAO,QAAQ;AACjB,iBAAO;AAAA,YACL,wCAAwC,OAAO,gBAAgB,OAAO,EAAE;AAAA,UAC1E;AAAA,QACF;AACA,eAAO,WAAW,2BAA2B,OAAO,EAAE,IAAI;AAAA,MAC5D,SAAS,OAAO;AACd,eAAO;AAAA,UACL,gCAAgC,iBAAiB,QAAQ,MAAM,UAAU,eAAe;AAAA,QAC1F;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAIA,eAAe,WAAW,YAAgD;AACxE,MAAI;AACF,UAAM,MAAM,MAAM,WAAW,KAAK,0BAA0B;AAAA,MAC1D,WAAW,WAAW;AAAA,IACxB,CAAC;AACD,WAAO,IAAI,SAAS;AAAA,EACtB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAYA,SAAS,cAAc,QAAyD;AAC9E,MAAI,CAAC,UAAU,CAAC,MAAM,QAAQ,OAAO,UAAU,EAAG,QAAO;AACzD,SAAO;AAAA,IACL,YAAY,OAAO;AAAA,IACnB,SAAU,OAAO,WAAsB;AAAA,IACvC,kBAAmB,OAAO,oBAA+B;AAAA,IACzD,kBAAmB,OAAO,oBAA+B;AAAA,IACzD,eAAgB,OAAO,iBAA4B;AAAA,EACrD;AACF;AAEA,SAAS,wBAAwB,MAA+B;AAC9D,QAAM,iBAAiB,KAAK,MAAM,iCAAiC;AACnE,QAAM,UAAU,iBAAiB,eAAe,CAAC,EAAE,KAAK,IAAI;AAE5D,MAAI;AACF,UAAM,SAAS,cAAc,KAAK,MAAM,OAAO,CAAC;AAChD,QAAI,OAAQ,QAAO;AAAA,EACrB,QAAQ;AAAA,EAER;AAEA,QAAM,cAAc,KAAK,MAAM,gCAAgC;AAC/D,MAAI,aAAa;AACf,QAAI;AACF,YAAM,SAAS,cAAc,KAAK,MAAM,YAAY,CAAC,CAAC,CAAC;AACvD,UAAI,OAAQ,QAAO;AAAA,IACrB,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,QAAM,IAAI,MAAM,wDAAwD;AAC1E;AAIA,SAAS,mBAAmB,YAAkC;AAC5D,QAAM,SAAS;AAAA,IACb,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,cAAc;AAAA,IACd,cAAc;AAAA,IACd,cAAc;AAAA,EAChB;AAEA,aAAW,MAAM,YAAY;AAC3B,UAAM,MACJ,GAAG,GAAG,KAAK,GAAG,GAAG,MAAM,OAAO,CAAC,EAAE,YAAY,CAAC,GAAG,GAAG,MAAM,MAAM,CAAC,CAAC;AACpE,QAAI,OAAO,QAAQ;AACjB,aAAO,GAAG;AAAA,IACZ;AAAA,EACF;AAEA,SAAO;AACT;AAIA,eAAe,0BACb,QACA,YACA,WACA,QACyE;AACzE,QAAM,gBAA0B,CAAC;AACjC,MAAI,UAAyB;AAC7B,MAAI,QAAuB;AAE3B,mBAAiB,SAAS,QAAQ;AAChC,QAAI,MAAM,SAAS,UAAU;AAC3B,YAAM,WAAW;AACjB,UAAI,SAAS,YAAY,UAAU,SAAS,OAAO;AACjD,gBAAQ,SAAS;AAAA,MACnB;AAAA,IACF;AAEA,QAAI,MAAM,SAAS,aAAa;AAC9B,YAAM,EAAE,QAAQ,IAAI;AAMpB,iBAAW,SAAS,QAAQ,SAAS;AACnC,YAAI,MAAM,SAAS,UAAU,MAAM,MAAM;AACvC,wBAAc,KAAK,MAAM,IAAI;AAAA,QAC/B,WAAW,MAAM,SAAS,cAAc,MAAM,MAAM;AAClD,gBAAM,WACJ,OAAO,MAAM,UAAU,WAAW,MAAM,QAAQ,KAAK,UAAU,MAAM,KAAK;AAC5E,gBAAM,WACH,KAAK,2BAA2B;AAAA,YAC/B,WAAW,WAAW;AAAA,YACtB;AAAA,YACA;AAAA,YACA,UAAU,GAAG,MAAM,IAAI,KAAK,SAAS,MAAM,GAAG,GAAG,CAAC;AAAA,UACpD,CAAC,EACA,MAAM,MAAM;AAAA,UAAC,CAAC;AAAA,QACnB;AAAA,MACF;AAAA,IACF;AAEA,QAAI,MAAM,SAAS,UAAU;AAC3B,YAAM,cAAc;AAKpB,UAAI,YAAY,YAAY,aAAa,YAAY,mBAAmB,QAAW;AACjF,kBAAU,YAAY;AAAA,MACxB;AACA;AAAA,IACF;AAAA,EACF;AAEA,SAAO,EAAE,MAAM,cAAc,KAAK,MAAM,EAAE,KAAK,GAAG,SAAS,MAAM;AACnE;AAIA,SAAS,iBAAiB,WAAmB,WAAmB,QAAgB,OAAe;AAC7F,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,IACT,YAAY,CAAC;AAAA,IACb,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,IAClB,eAAe;AAAA,IACf,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,cAAc;AAAA,IACd,cAAc;AAAA,IACd,cAAc;AAAA,IACd,eAAe,CAAC;AAAA,IAChB,cAAc;AAAA,IACd,OAAO;AAAA,IACP;AAAA,EACF;AACF;AAIA,eAAe,gBACb,aACA,SACA,YACA,YACA,OACe;AACf,QAAM,EAAE,WAAW,UAAU,IAAI;AAEjC,QAAM,WACH,KAAK,2BAA2B;AAAA,IAC/B,WAAW,WAAW;AAAA,IACtB;AAAA,IACA,QAAQ,YAAY;AAAA,IACpB,UAAU;AAAA,EACZ,CAAC,EACA,MAAM,MAAM;AAAA,EAAC,CAAC;AAEjB,QAAM,gBAA0B,CAAC;AACjC,QAAM,iBAAiB,2BAA2B,YAAY,SAAS;AAGvE,QAAM,wBAAwB;AAAA,IAC5B,eAAe;AAAA,IACf,eAAe;AAAA,IACf,eAAe;AAAA,IACf,OAAO,UAAmC;AACxC,YAAM,SAAS,MAAM,eAAe,QAAQ,KAAK;AACjD,YAAM,OAAO,OAAO,UAAU,CAAC,GAAG,QAAQ;AAC1C,YAAM,UAAU,KAAK,MAAM,kBAAkB;AAC7C,UAAI,QAAS,eAAc,KAAK,QAAQ,CAAC,CAAC;AAC1C,aAAO;AAAA,IACT;AAAA,EACF;AAEA,QAAM,UAAU,cAAc;AAC9B,QAAM,YAAY,QAAQ,gBAAgB;AAAA,IACxC,MAAM;AAAA,IACN,OAAO,CAAC,qBAAqB;AAAA,EAC/B,CAAC;AAED,QAAM,SAAS,QAAQ,aAAa;AAAA,IAClC,QAAQ,qBAAqB,WAAW;AAAA,IACxC,SAAS;AAAA,MACP;AAAA,MACA,cAAc;AAAA,MACd,KAAK;AAAA,MACL,gBAAgB;AAAA,MAChB,iCAAiC;AAAA,MACjC,OAAO,EAAE,MAAM,UAAmB,QAAQ,cAAuB;AAAA,MACjE,YAAY,EAAE,cAAc,UAAU;AAAA,MACtC,UAAU;AAAA,MACV,cAAc;AAAA,MACd,iBAAiB,CAAC,QAAQ,SAAS,QAAQ,gBAAgB,WAAW;AAAA,IACxE;AAAA,EACF,CAAC;AAED,QAAM,WAAW,MAAM;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,IACA,YAAY;AAAA,EACd;AACA,MAAI,CAAC,SAAS,KAAM,OAAM,IAAI,MAAM,yBAAyB;AAE7D,QAAM,SAAS,wBAAwB,SAAS,IAAI;AACpD,QAAM,SAAS,mBAAmB,OAAO,UAAU;AAEnD,SAAO,KAAK,uBAAuB;AAAA,IACjC;AAAA,IACA,QAAQ,YAAY;AAAA,IACpB,QAAQ,OAAO,WAAW;AAAA,IAC1B,aAAa,cAAc;AAAA,EAC7B,CAAC;AAED,QAAM,WAAW,KAAK,yBAAyB;AAAA,IAC7C,WAAW,WAAW;AAAA,IACtB;AAAA,IACA,QAAQ,YAAY;AAAA,IACpB,SAAS,OAAO;AAAA,IAChB,YAAY,OAAO;AAAA,IACnB,kBAAkB,OAAO;AAAA,IACzB,kBAAkB,OAAO;AAAA,IACzB,eAAe,OAAO;AAAA,IACtB,GAAG;AAAA,IACH;AAAA,IACA,cAAc,SAAS;AAAA,IACvB,OAAO,SAAS;AAAA,EAClB,CAAC;AACH;AAIA,eAAsB,gBACpB,SACA,YACA,YACe;AACf,QAAM,EAAE,UAAU,IAAI;AAEtB,SAAO,KAAK,uBAAuB,EAAE,WAAW,WAAW,QAAQ,MAAM,OAAO,CAAC;AAEjF,QAAM,QAAQ,MAAM,WAAW,UAAU;AAEzC,aAAW,eAAe,QAAQ,OAAO;AACvC,QAAI;AACF,YAAM,gBAAgB,aAAa,SAAS,YAAY,YAAY,KAAK;AAAA,IAC3E,SAAS,KAAK;AACZ,YAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,aAAO,MAAM,8BAA8B;AAAA,QACzC;AAAA,QACA,QAAQ,YAAY;AAAA,QACpB,OAAO;AAAA,MACT,CAAC;AACD,YAAM,WACH;AAAA,QACC;AAAA,QACA,iBAAiB,WAAW,WAAW,WAAW,YAAY,QAAQ,GAAG;AAAA,MAC3E,EACC,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACnB;AAAA,EACF;AACF;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rallycry/conveyor-agent",
-  "version": "7.1.3",
+  "version": "7.1.4",
   "description": "Conveyor Agent Runner v7.0 - Agent-as-User architecture with BaseService patterns",
   "keywords": [
     "agent",