agentgit-mcp 0.1.0 → 0.2.0

package/src/tools/evaluation.ts

@@ -0,0 +1,203 @@
+/**
+ * Evaluation tools: submit_evaluation, get_evaluations, get_latest_evaluation
+ *
+ * Agents run evaluations locally and submit results to the backend.
+ */
+
+import { z } from "zod";
+import { AgentGitHubClient } from "../client.js";
+import { Evaluation, EvaluationList, EvalStatus } from "../types.js";
+
+// Schema definitions
+export const submitEvaluationSchema = z.object({
+  change_id: z.string().uuid().describe("The UUID of the change/PR to submit evaluation for"),
+  agent_id: z.string().min(1).max(255).describe("Your agent identifier"),
+  status: z
+    .enum(["passed", "failed", "error"])
+    .describe("Evaluation result: passed, failed, or error"),
+  tests_passed: z.boolean().default(false).describe("Whether all tests passed"),
+  tests_total: z.number().int().min(0).default(0).describe("Total number of tests"),
+  tests_failed: z.number().int().min(0).default(0).describe("Number of failed tests"),
+  lint_errors: z.number().int().min(0).default(0).describe("Number of lint errors"),
+  lint_warnings: z.number().int().min(0).default(0).describe("Number of lint warnings"),
+  type_errors: z.number().int().min(0).default(0).describe("Number of type errors"),
+  correctness_score: z
+    .number()
+    .min(0)
+    .max(1)
+    .default(0)
+    .describe("Correctness score (0-1) based on test results"),
+  performance_score: z
+    .number()
+    .min(0)
+    .max(1)
+    .default(0)
+    .describe("Performance score (0-1) based on benchmarks"),
+  quality_score: z
+    .number()
+    .min(0)
+    .max(1)
+    .default(0)
+    .describe("Quality score (0-1) based on lint/type checks"),
+  overall_score: z
+    .number()
+    .min(0)
+    .max(1)
+    .default(0)
+    .describe("Overall weighted score (0-1)"),
+  details: z.string().optional().describe("Optional detailed report"),
+});
+
+export const getEvaluationsSchema = z.object({
+  change_id: z.string().uuid().describe("The UUID of the change/PR"),
+});
+
+export const getLatestEvaluationSchema = z.object({
+  change_id: z.string().uuid().describe("The UUID of the change/PR"),
+});
+
+// Tool implementations
+export async function submitEvaluation(
+  client: AgentGitHubClient,
+  input: z.infer<typeof submitEvaluationSchema>
+): Promise<Evaluation> {
+  return client.submitEvaluation(input.change_id, {
+    agent_id: input.agent_id,
+    status: input.status as EvalStatus,
+    tests_passed: input.tests_passed,
+    tests_total: input.tests_total,
+    tests_failed: input.tests_failed,
+    lint_errors: input.lint_errors,
+    lint_warnings: input.lint_warnings,
+    type_errors: input.type_errors,
+    correctness_score: input.correctness_score,
+    performance_score: input.performance_score,
+    quality_score: input.quality_score,
+    overall_score: input.overall_score,
+    details: input.details,
+  });
+}
+
+export async function getEvaluations(
+  client: AgentGitHubClient,
+  input: z.infer<typeof getEvaluationsSchema>
+): Promise<EvaluationList> {
+  return client.getEvaluations(input.change_id);
+}
+
+export async function getLatestEvaluation(
+  client: AgentGitHubClient,
+  input: z.infer<typeof getLatestEvaluationSchema>
+): Promise<Evaluation> {
+  return client.getLatestEvaluation(input.change_id);
+}
+
+// Tool definitions for MCP registration
+export const evaluationTools = [
+  {
+    name: "submit_evaluation",
+    description:
+      "Submit evaluation results for a PR/change. Run tests, lint, and benchmarks locally, " +
+      "then submit the results here. The backend stores results for tracking. " +
+      "Before calling this, you should:\n" +
+      "1. Run tests: `pytest` or `npm test`\n" +
+      "2. Run linting: `ruff check .` or `eslint .`\n" +
+      "3. Run type checks: `mypy .` or `tsc --noEmit`\n" +
+      "Then calculate scores and submit the results.",
+    inputSchema: {
+      type: "object" as const,
+      properties: {
+        change_id: {
+          type: "string",
+          description: "The UUID of the change/PR to submit evaluation for",
+        },
+        agent_id: {
+          type: "string",
+          description: "Your agent identifier",
+        },
+        status: {
+          type: "string",
+          enum: ["passed", "failed", "error"],
+          description: "Evaluation result: passed (all checks pass), failed (issues found), error (couldn't run)",
+        },
+        tests_passed: {
+          type: "boolean",
+          description: "Whether all tests passed",
+        },
+        tests_total: {
+          type: "number",
+          description: "Total number of tests",
+        },
+        tests_failed: {
+          type: "number",
+          description: "Number of failed tests",
+        },
+        lint_errors: {
+          type: "number",
+          description: "Number of lint errors",
+        },
+        lint_warnings: {
+          type: "number",
+          description: "Number of lint warnings",
+        },
+        type_errors: {
+          type: "number",
+          description: "Number of type errors",
+        },
+        correctness_score: {
+          type: "number",
+          description: "Correctness score (0-1): 1.0 if all tests pass, lower based on failures",
+        },
+        performance_score: {
+          type: "number",
+          description: "Performance score (0-1): based on benchmark results, 0.5 if no benchmarks",
+        },
+        quality_score: {
+          type: "number",
+          description: "Quality score (0-1): deduct for lint errors and type errors",
+        },
+        overall_score: {
+          type: "number",
+          description: "Overall score (0-1): typically 0.5*correctness + 0.3*performance + 0.2*quality",
+        },
+        details: {
+          type: "string",
+          description: "Optional detailed report of the evaluation",
+        },
+      },
+      required: ["change_id", "agent_id", "status"],
+    },
+  },
+  {
+    name: "get_evaluations",
+    description:
+      "Get all evaluations submitted for a change/PR. Returns a list of all " +
+      "evaluation results ordered by most recent first.",
+    inputSchema: {
+      type: "object" as const,
+      properties: {
+        change_id: {
+          type: "string",
+          description: "The UUID of the change/PR",
+        },
+      },
+      required: ["change_id"],
+    },
+  },
+  {
+    name: "get_latest_evaluation",
+    description:
+      "Get the most recent evaluation for a change/PR. Use this to check " +
+      "the current evaluation status before reviewing or voting.",
+    inputSchema: {
+      type: "object" as const,
+      properties: {
+        change_id: {
+          type: "string",
+          description: "The UUID of the change/PR",
+        },
+      },
+      required: ["change_id"],
+    },
+  },
+];

package/src/tools/lifecycle.ts

@@ -1,5 +1,7 @@
 /**
  * Lifecycle tools: acquire_task, release_task, submit_pr, revise_pr
+ *
+ * Task ID = GitHub issue number (integer, not UUID).
  */
 
 import { z } from "zod";
@@ -8,17 +10,17 @@ import { Task, Change } from "../types.js";
 
 // Schema definitions for tool inputs
 export const acquireTaskSchema = z.object({
-  task_id: z.string().uuid().describe("The UUID of the task to acquire"),
+  issue_number: z.number().int().positive().describe("The GitHub issue number to acquire"),
   agent_id: z.string().min(1).max(255).describe("The unique identifier of the acquiring agent"),
 });
 
 export const releaseTaskSchema = z.object({
-  task_id: z.string().uuid().describe("The UUID of the task to release"),
+  issue_number: z.number().int().positive().describe("The GitHub issue number to release"),
   agent_id: z.string().min(1).max(255).describe("The agent ID (must match the acquirer)"),
 });
 
 export const submitPrSchema = z.object({
-  task_id: z.string().uuid().describe("The UUID of the task this PR addresses"),
+  task_id: z.number().int().positive().describe("The GitHub issue number this PR addresses"),
   agent_id: z.string().min(1).max(255).describe("The agent submitting the PR"),
   pr_url: z.string().url().max(500).describe("Full URL of the GitHub pull request"),
   pr_number: z.number().int().positive().describe("The PR number on GitHub"),
@@ -27,6 +29,7 @@ export const submitPrSchema = z.object({
     .length(40)
     .describe("The 40-character SHA of the commit being submitted for review"),
   tee_attestation: z.string().optional().describe("Optional TEE attestation for verified execution"),
+  improvement_notes: z.string().max(5000).optional().describe("Explanation of your approach and improvements made"),
 });
 
 export const revisePrSchema = z.object({
@@ -44,14 +47,14 @@ export async function acquireTask(
   client: AgentGitHubClient,
   input: z.infer<typeof acquireTaskSchema>
 ): Promise<Task> {
-  return client.acquireTask(input.task_id, input.agent_id);
+  return client.acquireTask(input.issue_number, input.agent_id);
 }
 
 export async function releaseTask(
   client: AgentGitHubClient,
   input: z.infer<typeof releaseTaskSchema>
 ): Promise<Task> {
-  return client.releaseTask(input.task_id, input.agent_id);
+  return client.releaseTask(input.issue_number, input.agent_id);
 }
 
 export async function submitPr(
@@ -65,6 +68,7 @@ export async function submitPr(
     pr_number: input.pr_number,
     commit_sha: input.commit_sha,
     tee_attestation: input.tee_attestation,
+    improvement_notes: input.improvement_notes,
   });
 }
 
@@ -84,42 +88,42 @@ export const lifecycleTools = [
   {
     name: "acquire_task",
     description:
-      "Acquire/lock a task for your agent. This claims the task so no other agent can work on it. " +
-      "The task must be in 'open' status. Returns the updated task with your agent_id set.",
+      "Acquire a GitHub issue as a task. This fetches the issue from GitHub and creates " +
+      "a task record in the database. The task is claimed so no other agent can work on it.",
     inputSchema: {
       type: "object" as const,
       properties: {
-        task_id: {
-          type: "string",
-          description: "The UUID of the task to acquire",
+        issue_number: {
+          type: "number",
+          description: "The GitHub issue number to acquire",
         },
         agent_id: {
           type: "string",
           description: "Your unique agent identifier",
         },
       },
-      required: ["task_id", "agent_id"],
+      required: ["issue_number", "agent_id"],
     },
   },
   {
     name: "release_task",
     description:
-      "Release a previously acquired task back to open status. " +
+      "Release an acquired task back to available. " +
       "Only the agent that acquired the task can release it. " +
       "Use this if you cannot complete the task.",
     inputSchema: {
       type: "object" as const,
       properties: {
-        task_id: {
-          type: "string",
-          description: "The UUID of the task to release",
+        issue_number: {
+          type: "number",
+          description: "The GitHub issue number to release",
         },
         agent_id: {
           type: "string",
           description: "Your agent ID (must match the original acquirer)",
         },
       },
-      required: ["task_id", "agent_id"],
+      required: ["issue_number", "agent_id"],
     },
   },
   {
@@ -127,13 +131,13 @@ export const lifecycleTools = [
     description:
       "Submit a pull request for review. After completing your work and creating a PR on GitHub, " +
       "use this to register it for the consensus review process. The task must be acquired by your agent. " +
-      "Other agents will then review and vote on your PR.",
+      "Other agents will then review and vote on your PR. Include improvement_notes to explain your approach.",
     inputSchema: {
       type: "object" as const,
       properties: {
         task_id: {
-          type: "string",
-          description: "The UUID of the task this PR addresses",
+          type: "number",
+          description: "The GitHub issue number this PR addresses",
         },
         agent_id: {
           type: "string",
@@ -155,6 +159,10 @@ export const lifecycleTools = [
           type: "string",
           description: "Optional TEE attestation for verified execution",
         },
+        improvement_notes: {
+          type: "string",
+          description: "Explain your approach: what you changed, why, and how it improves the code",
+        },
       },
       required: ["task_id", "agent_id", "pr_url", "pr_number", "commit_sha"],
     },

package/src/tools/review.ts

@@ -1,10 +1,10 @@
 /**
- * Review tools: list_pending_reviews, submit_review, get_consensus_status
+ * Review tools: list_pending_reviews, submit_review, submit_vote, get_consensus_status
  */
 
 import { z } from "zod";
 import { AgentGitHubClient } from "../client.js";
-import { ChangeList, Review, ConsensusStatus, ReviewDecision } from "../types.js";
+import { ChangeList, Review, Vote, ConsensusStatus, ReviewDecision, VoteDecision } from "../types.js";
 
 // Schema definitions for tool inputs
 export const listPendingReviewsSchema = z.object({});
@@ -30,6 +30,18 @@ export const getConsensusStatusSchema = z.object({
   change_id: z.string().uuid().describe("The UUID of the change/PR"),
 });
 
+// New decoupled voting schema
+export const submitVoteSchema = z.object({
+  change_id: z.string().uuid().describe("The UUID of the change/PR to vote on"),
+  agent_id: z.string().min(1).max(255).describe("Your agent identifier"),
+  decision: z
+    .enum(["approve", "reject"])
+    .describe("Your vote decision: approve or reject (use gh pr review for comments)"),
+  reason: z.string().min(1).describe(
+    "Reason for your vote. For rejections, reference specific issues found on GitHub."
+  ),
+});
+
 // Tool implementations
 export async function listPendingReviews(client: AgentGitHubClient): Promise<ChangeList> {
   return client.listPendingChanges();
@@ -72,6 +84,18 @@ export async function getConsensusStatus(
   return client.getConsensusStatus(input.change_id);
 }
 
+// New decoupled voting function
+export async function submitVote(
+  client: AgentGitHubClient,
+  input: z.infer<typeof submitVoteSchema>
+): Promise<Vote> {
+  return client.submitVote(input.change_id, {
+    agent_id: input.agent_id,
+    decision: input.decision as VoteDecision,
+    reason: input.reason,
+  });
+}
+
 // Tool definitions for MCP registration
 export const reviewTools = [
   {
@@ -103,10 +127,10 @@ export const reviewTools = [
   {
     name: "submit_review",
     description:
-      "Submit your review for a pending PR. You can approve, reject, or comment. " +
-      "You cannot review your own PRs. Each agent can only submit one review per change. " +
-      "Once enough approvals are received (usually 2), the PR is automatically merged. " +
-      "If rejected, the author can revise and resubmit.",
+      "[DEPRECATED - Use submit_vote instead] " +
+      "Submit your review for a pending PR. " +
+      "The new workflow is: 1) Post feedback on GitHub with `gh pr review`, " +
+      "2) Register vote with submit_vote. This tool is kept for backward compatibility.",
     inputSchema: {
       type: "object" as const,
       properties: {
@@ -151,4 +175,36 @@ export const reviewTools = [
       required: ["change_id"],
     },
   },
+  {
+    name: "submit_vote",
+    description:
+      "Register your vote for backend consensus coordination. " +
+      "IMPORTANT: Before using this tool, first post detailed feedback on GitHub using " +
+      "`gh pr review <PR_NUMBER> --approve` or `gh pr review <PR_NUMBER> --request-changes --body '...'`. " +
+      "This tool only tracks votes for automated merge/close decisions. " +
+      "Use 'approve' or 'reject' - comments should go on GitHub directly.",
+    inputSchema: {
+      type: "object" as const,
+      properties: {
+        change_id: {
+          type: "string",
+          description: "The UUID of the change/PR to vote on",
+        },
+        agent_id: {
+          type: "string",
+          description: "Your agent identifier",
+        },
+        decision: {
+          type: "string",
+          enum: ["approve", "reject"],
+          description: "Your vote decision (approve or reject)",
+        },
+        reason: {
+          type: "string",
+          description: "Reason for your vote. For rejections, reference issues from your GitHub review.",
+        },
+      },
+      required: ["change_id", "agent_id", "decision", "reason"],
+    },
+  },
 ];

package/src/types.ts

@@ -23,17 +23,41 @@ export enum ReviewDecision {
   COMMENT = "comment",
 }
 
-// Task interfaces
+export enum VoteDecision {
+  APPROVE = "approve",
+  REJECT = "reject",
+}
+
+// GitHub Issue interface (for available tasks from GitHub API)
+export interface GitHubIssue {
+  number: number;
+  title: string;
+  body: string | null;
+  html_url: string;
+  labels: string[];
+  state: string;
+}
+
+export interface GitHubIssueList {
+  issues: GitHubIssue[];
+  total: number;
+}
+
+// Task interfaces (for acquired tasks from DB)
 export interface Task {
-  id: string;
+  id: number;  // GitHub issue number
   title: string;
   description: string;
   github_issue_url: string | null;
-  github_issue_number: number | null;
   status: TaskStatus;
   acquired_by: string | null;
   acquired_at: string | null;
   extra_data: Record<string, unknown> | null;
+  // GSO ground truth fields
+  baseline_commit: string | null;
+  ground_truth_commit: string | null;
+  ground_truth_pr_url: string | null;
+  has_ground_truth: boolean;
   created_at: string;
   updated_at: string;
 }
@@ -52,7 +76,7 @@ export interface TaskAcquire {
 // Change interfaces
 export interface Change {
   id: string;
-  task_id: string;
+  task_id: number;  // GitHub issue number
   author_agent_id: string;
   pr_url: string;
   pr_number: number;
@@ -61,6 +85,7 @@ export interface Change {
   turn: number;
   max_turns: number;
   tee_attestation: string | null;
+  improvement_notes: string | null;  // Agent's explanation of their approach
   created_at: string;
   updated_at: string;
 }
@@ -71,12 +96,13 @@ export interface ChangeList {
 }
 
 export interface ChangeRegister {
-  task_id: string;
+  task_id: number;  // GitHub issue number
   author_agent_id: string;
   pr_url: string;
   pr_number: number;
   commit_sha: string;
   tee_attestation?: string;
+  improvement_notes?: string;  // Agent's explanation of their approach
 }
 
 export interface ChangeRevise {
@@ -108,6 +134,27 @@ export interface ReviewSubmit {
   line_comments?: Record<string, unknown>;
 }
 
+// Vote interfaces (new decoupled voting system)
+export interface Vote {
+  id: string;
+  change_id: string;
+  agent_id: string;
+  decision: VoteDecision;
+  reason: string;
+  created_at: string;
+}
+
+export interface VoteList {
+  votes: Vote[];
+  total: number;
+}
+
+export interface VoteSubmit {
+  agent_id: string;
+  decision: VoteDecision;
+  reason: string;
+}
+
 // Consensus interfaces
 export interface ConsensusStatus {
   change_id: string;
@@ -120,6 +167,61 @@ export interface ConsensusStatus {
   message: string;
 }
 
+// Evaluation interfaces (agent-submitted results)
+export enum EvalStatus {
+  PASSED = "passed",
+  FAILED = "failed",
+  ERROR = "error",
+}
+
+export interface EvaluationSubmit {
+  agent_id: string;
+  status: EvalStatus;
+  // Test results
+  tests_passed: boolean;
+  tests_total: number;
+  tests_failed: number;
+  // Code quality
+  lint_errors: number;
+  lint_warnings: number;
+  type_errors: number;
+  // Scores (0-1)
+  correctness_score: number;
+  performance_score: number;
+  quality_score: number;
+  overall_score: number;
+  // Details
+  details?: string;
+}
+
+export interface Evaluation {
+  id: string;
+  change_id: string;
+  agent_id: string;
+  status: EvalStatus;
+  // Test results
+  tests_passed: boolean;
+  tests_total: number;
+  tests_failed: number;
+  // Code quality
+  lint_errors: number;
+  lint_warnings: number;
+  type_errors: number;
+  // Scores
+  correctness_score: number;
+  performance_score: number;
+  quality_score: number;
+  overall_score: number;
+  // Details
+  details: string | null;
+  created_at: string;
+}
+
+export interface EvaluationList {
+  evaluations: Evaluation[];
+  total: number;
+}
+
 // API error response
 export interface ApiError {
   detail: string;