@almightygpt/core 0.2.0

package/src/review/run-worker-reviewer.ts

@@ -0,0 +1,528 @@
+/**
+ * Worker / Reviewer two-role review pipeline.
+ *
+ * This is the headline AlmightyGPT flow: one AI does the work, a different
+ * AI critiques it, and the human gets a single artifact with both passes.
+ *
+ * Pipeline:
+ *   1. Load config + create run folder (type: review-worker-reviewer).
+ *   2. Resolve Worker adapter (--worker or config.defaults.worker).
+ *   3. Resolve Reviewer adapter (--reviewer or config.defaults.reviewer).
+ *      Same vendor on both sides is allowed but discouraged via a warning.
+ *   4. Collect git diff (+ optional range).
+ *   5. Redact secrets.
+ *   6. Build context manifest, write input.md + context-manifest.json.
+ *   7. Pre-flight budget check.
+ *   8. Worker stage: assemble Worker memory, run, persist output.
+ *   9. Reviewer stage: assemble Reviewer memory, give it the Worker output
+ *      and the diff, run, persist output.
+ *  10. Shallow-review detection on the Reviewer's response.
+ *  11. Write the human review file (Worker plan summary lives inside the
+ *      Reviewer's response per the prompt's required-sections list).
+ *  12. Write run.json with both metrics + totals.
+ *
+ * Events are emitted throughout via the optional onEvent callback so the
+ * CLI / future extension can drive progress UIs without polling.
+ */
+
+import { AdapterError, type Adapter } from "../adapters/types.js";
+import { MockAdapter } from "../adapters/mock.js";
+import { OpenAIAdapter } from "../adapters/openai.js";
+import { ClaudeAdapter } from "../adapters/claude.js";
+import { GeminiAdapter } from "../adapters/gemini.js";
+import { loadConfig } from "../config/load.js";
+import { redactSecrets } from "../context/redact.js";
+import {
+  buildContextManifest,
+  writeContextManifest,
+} from "../context/manifest.js";
+import {
+  collectGitContext,
+  createRunFolder,
+  writeAgentOutput,
+  writeRunInput,
+  writeRunMetadata,
+} from "../runs/folder.js";
+import type { AgentMetrics, RunMetadata } from "../runs/types.js";
+import { collectGitDiff } from "./diff.js";
+import { assembleMemory } from "./memory.js";
+import {
+  buildReviewerSystemFraming,
+  buildReviewerOfWorkerUserMessage,
+  buildWorkerSystemFraming,
+  buildWorkerUserMessage,
+} from "./prompts.js";
+import { writeHumanReviewFile } from "./write.js";
+import { BudgetTracker, BudgetExceededError } from "./budget.js";
+import type { ReviewEventHandler } from "./events.js";
+
+export interface WorkerReviewerOptions {
+  repoRoot: string;
+  topic: string;
+  /** Worker agent name. Falls back to config.defaults.worker. */
+  worker?: string;
+  /** Reviewer agent name. Falls back to config.defaults.reviewer. */
+  reviewer?: string;
+  /** Optional git range like "HEAD~1..HEAD". */
+  range?: string;
+  /** Bypass the git status safety check / refuse-overwrite check. */
+  force?: boolean;
+  /** Optional event stream subscriber. */
+  onEvent?: ReviewEventHandler;
+}
+
+export interface WorkerReviewerResult {
+  reviewPath: string;
+  reviewBytes: number;
+  runId: string;
+  runFolder: string;
+  worker: { name: string; provider: string; model: string };
+  reviewer: { name: string; provider: string; model: string };
+  metrics: AgentMetrics[];
+  totals: {
+    tokensIn: number;
+    tokensOut: number;
+    costUsd: number;
+    latencyMs: number;
+  };
+  filesReviewed: string[];
+  redactionsTotal: number;
+  shallowWarning?: string;
+  sameProviderWarning?: string;
+  memoryMissing: string[];
+}
+
+export async function runWorkerReviewerReview(
+  opts: WorkerReviewerOptions,
+): Promise<WorkerReviewerResult> {
+  const emit = opts.onEvent ?? (() => {});
+  const config = await loadConfig(opts.repoRoot);
+
+  const workerName = opts.worker ?? config.defaults.worker;
+  const reviewerName = opts.reviewer ?? config.defaults.reviewer;
+  if (!workerName) {
+    throw new Error(
+      "No worker specified. Pass --worker <name> or set defaults.worker in .almightygpt/config.yaml.",
+    );
+  }
+  if (!reviewerName) {
+    throw new Error(
+      "No reviewer specified. Pass --reviewer <name> or set defaults.reviewer in .almightygpt/config.yaml.",
+    );
+  }
+
+  const workerCfg = config.agents[workerName];
+  const reviewerCfg = config.agents[reviewerName];
+  if (!workerCfg) {
+    throw new Error(
+      `Worker "${workerName}" not found in .almightygpt/config.yaml agents map.`,
+    );
+  }
+  if (!reviewerCfg) {
+    throw new Error(
+      `Reviewer "${reviewerName}" not found in .almightygpt/config.yaml agents map.`,
+    );
+  }
+  if (!workerCfg.enabled) {
+    throw new Error(`Worker "${workerName}" is disabled in config.`);
+  }
+  if (!reviewerCfg.enabled) {
+    throw new Error(`Reviewer "${reviewerName}" is disabled in config.`);
+  }
+
+  const workerAdapter = makeAdapter(workerName, workerCfg.provider);
+  const reviewerAdapter = makeAdapter(reviewerName, reviewerCfg.provider);
+
+  if (!(await workerAdapter.isAvailable())) {
+    throw new AdapterError(
+      `Worker adapter "${workerName}" (${workerCfg.provider}) is not available. ` +
+        envHintFor(workerCfg.provider),
+      workerName,
+    );
+  }
+  if (!(await reviewerAdapter.isAvailable())) {
+    throw new AdapterError(
+      `Reviewer adapter "${reviewerName}" (${reviewerCfg.provider}) is not available. ` +
+        envHintFor(reviewerCfg.provider),
+      reviewerName,
+    );
+  }
+
+  const sameProviderWarning =
+    workerCfg.provider === reviewerCfg.provider
+      ? `Worker and Reviewer are both ${workerCfg.provider}. Cross-vendor pairing catches more issues.`
+      : undefined;
+
+  const runFolder = await createRunFolder({
+    repoRoot: opts.repoRoot,
+    runsDir: config.runsDir,
+    topic: opts.topic,
+    type: "review-worker-reviewer",
+  });
+
+  emit({
+    type: "run_started",
+    runId: runFolder.id,
+    runType: "review-worker-reviewer",
+    topic: opts.topic,
+    reviewsDir: config.reviewsDir,
+    runFolder: runFolder.relPath,
+  });
+
+  const git = await collectGitContext(opts.repoRoot);
+  const baseMeta: RunMetadata = {
+    id: runFolder.id,
+    type: "review-worker-reviewer",
+    createdAt: new Date().toISOString(),
+    workspacePath: opts.repoRoot,
+    topic: opts.topic,
+    git,
+    input: opts.range
+      ? { source: "diff-range", range: opts.range }
+      : { source: "diff" },
+    agents: Object.entries(config.agents).map(([name, a]) => ({
+      name,
+      role: a.role,
+      provider: a.provider,
+      enabled: a.enabled,
+    })),
+    adapterVersions: [
+      { adapter: workerAdapter.name, version: "0.0.0" },
+      { adapter: reviewerAdapter.name, version: "0.0.0" },
+    ],
+    status: "running",
+    metrics: [],
+    totals: { tokensIn: 0, tokensOut: 0, costUsd: 0, latencyMs: 0 },
+    budget: config.budget,
+  };
+  await writeRunMetadata(runFolder.absPath, baseMeta);
+
+  try {
+    const diffArgs: { range?: string } = {};
+    if (opts.range) diffArgs.range = opts.range;
+    const diffResult = await collectGitDiff(opts.repoRoot, diffArgs);
+    if (diffResult.empty && !opts.range) {
+      throw new Error(
+        "No uncommitted changes. Stage changes, pick a range with --range, or commit something to review.",
+      );
+    }
+
+    const redaction = config.security.redactSecrets
+      ? redactSecrets(diffResult.diff)
+      : { text: diffResult.diff, redactions: [], totalCount: 0 };
+    emit({
+      type: "redaction_complete",
+      totalCount: redaction.totalCount,
+      byKind: redaction.redactions,
+    });
+
+    const manifest = buildContextManifest({
+      inputSource: opts.range ? "diff-range" : "diff",
+      filesIncluded: diffResult.files.map((p) => ({ path: p, bytes: 0 })),
+      filesSkipped: [],
+      diffText: redaction.text,
+      redaction: {
+        enabled: config.security.redactSecrets,
+        totalCount: redaction.totalCount,
+        byKind: redaction.redactions,
+      },
+    });
+    const manifestRelPath = await writeContextManifest(
+      runFolder.absPath,
+      manifest,
+    );
+    await writeRunInput(runFolder.absPath, redaction.text);
+
+    const budget = new BudgetTracker(config.budget);
+
+    // ─── Worker stage ───────────────────────────────────────────────
+    const workerMemory = await assembleMemory(
+      opts.repoRoot,
+      workerCfg.memoryFile,
+    );
+    const workerSystem =
+      buildWorkerSystemFraming() + "\n\n" + workerMemory.text;
+    const workerUser = buildWorkerUserMessage({
+      topic: opts.topic,
+      diff: redaction.text,
+      files: diffResult.files,
+    });
+
+    budget.preflightCheck({
+      model: "gpt-4o",
+      estimatedTokensIn: Math.ceil(
+        (workerSystem.length + workerUser.length) / 4,
+      ),
+      maxOutputTokens: 4096,
+    });
+
+    emit({
+      type: "agent_started",
+      role: "worker",
+      agent: workerName,
+      provider: workerAdapter.provider,
+    });
+    const workerOut = await workerAdapter.execute({
+      role: "worker",
+      systemPrompt: workerSystem,
+      userMessage: workerUser,
+    });
+    budget.record({
+      tokensIn: workerOut.tokensIn,
+      tokensOut: workerOut.tokensOut,
+      costUsd: workerOut.costUsd,
+    });
+    await writeAgentOutput(runFolder.absPath, "worker", workerOut.content);
+    emit({
+      type: "agent_completed",
+      role: "worker",
+      agent: workerName,
+      provider: workerAdapter.provider,
+      model: workerOut.modelUsed,
+      outputPath: `${runFolder.relPath}/outputs/worker.md`,
+      tokensIn: workerOut.tokensIn,
+      tokensOut: workerOut.tokensOut,
+      costUsd: workerOut.costUsd,
+      latencyMs: workerOut.latencyMs,
+    });
+
+    // ─── Reviewer stage ─────────────────────────────────────────────
+    const reviewerMemory = await assembleMemory(
+      opts.repoRoot,
+      reviewerCfg.memoryFile,
+    );
+    const reviewerSystem =
+      buildReviewerSystemFraming() + "\n\n" + reviewerMemory.text;
+    const reviewerUser = buildReviewerOfWorkerUserMessage({
+      topic: opts.topic,
+      diff: redaction.text,
+      files: diffResult.files,
+      workerOutput: workerOut.content,
+      workerAgent: workerName,
+      workerProvider: workerAdapter.provider,
+    });
+
+    budget.preflightCheck({
+      model: "gpt-4o",
+      estimatedTokensIn: Math.ceil(
+        (reviewerSystem.length + reviewerUser.length) / 4,
+      ),
+      maxOutputTokens: 4096,
+    });
+
+    emit({
+      type: "agent_started",
+      role: "reviewer",
+      agent: reviewerName,
+      provider: reviewerAdapter.provider,
+    });
+    const reviewerOut = await reviewerAdapter.execute({
+      role: "reviewer",
+      systemPrompt: reviewerSystem,
+      userMessage: reviewerUser,
+    });
+    budget.record({
+      tokensIn: reviewerOut.tokensIn,
+      tokensOut: reviewerOut.tokensOut,
+      costUsd: reviewerOut.costUsd,
+    });
+    await writeAgentOutput(runFolder.absPath, "reviewer", reviewerOut.content);
+    emit({
+      type: "agent_completed",
+      role: "reviewer",
+      agent: reviewerName,
+      provider: reviewerAdapter.provider,
+      model: reviewerOut.modelUsed,
+      outputPath: `${runFolder.relPath}/outputs/reviewer.md`,
+      tokensIn: reviewerOut.tokensIn,
+      tokensOut: reviewerOut.tokensOut,
+      costUsd: reviewerOut.costUsd,
+      latencyMs: reviewerOut.latencyMs,
+    });
+
+    const shallowWarning = detectShallowReview(
+      reviewerOut.content,
+      config.review.requireConcreteWeaknesses,
+    );
+
+    // Write human review file using the Reviewer's response. The Worker
+    // plan summary is one of the required sections in the Reviewer's prompt,
+    // so it's already embedded in the response.
+    const writeOpts: Parameters<typeof writeHumanReviewFile>[0] = {
+      repoRoot: opts.repoRoot,
+      reviewsDir: config.reviewsDir,
+      topic: opts.topic,
+      reviewerName,
+      reviewerProvider: reviewerAdapter.provider,
+      modelUsed: reviewerOut.modelUsed,
+      body: reviewerOut.content,
+      metrics: {
+        tokensIn: workerOut.tokensIn + reviewerOut.tokensIn,
+        tokensOut: workerOut.tokensOut + reviewerOut.tokensOut,
+        costUsd: workerOut.costUsd + reviewerOut.costUsd,
+        latencyMs: workerOut.latencyMs + reviewerOut.latencyMs,
+      },
+      runFolder: runFolder.relPath,
+    };
+    if (shallowWarning) writeOpts.shallowWarning = shallowWarning;
+    if (opts.force) writeOpts.force = opts.force;
+    const written = await writeHumanReviewFile(writeOpts);
+
+    const reviewWrittenEvent: import("./events.js").ReviewWrittenEvent = {
+      type: "review_written",
+      reviewPath: written.path,
+      bytes: written.bytes,
+    };
+    if (shallowWarning) reviewWrittenEvent.shallowWarning = shallowWarning;
+    emit(reviewWrittenEvent);
+
+    const totals = {
+      tokensIn: workerOut.tokensIn + reviewerOut.tokensIn,
+      tokensOut: workerOut.tokensOut + reviewerOut.tokensOut,
+      costUsd: workerOut.costUsd + reviewerOut.costUsd,
+      latencyMs: workerOut.latencyMs + reviewerOut.latencyMs,
+    };
+
+    const metrics: AgentMetrics[] = [
+      {
+        agent: workerName,
+        role: "worker",
+        provider: workerAdapter.provider,
+        model: workerOut.modelUsed,
+        tokensIn: workerOut.tokensIn,
+        tokensOut: workerOut.tokensOut,
+        costUsd: workerOut.costUsd,
+        latencyMs: workerOut.latencyMs,
+      },
+      {
+        agent: reviewerName,
+        role: "reviewer",
+        provider: reviewerAdapter.provider,
+        model: reviewerOut.modelUsed,
+        tokensIn: reviewerOut.tokensIn,
+        tokensOut: reviewerOut.tokensOut,
+        costUsd: reviewerOut.costUsd,
+        latencyMs: reviewerOut.latencyMs,
+      },
+    ];
+
+    const finalMeta: RunMetadata = {
+      ...baseMeta,
+      finishedAt: new Date().toISOString(),
+      status: "completed",
+      contextManifestPath: manifestRelPath,
+      reviewPath: written.path,
+      metrics,
+      totals,
+    };
+    await writeRunMetadata(runFolder.absPath, finalMeta);
+
+    emit({
+      type: "run_completed",
+      runId: runFolder.id,
+      reviewPath: written.path,
+      runFolder: runFolder.relPath,
+      totals,
+    });
+
+    const result: WorkerReviewerResult = {
+      reviewPath: written.path,
+      reviewBytes: written.bytes,
+      runId: runFolder.id,
+      runFolder: runFolder.relPath,
+      worker: {
+        name: workerName,
+        provider: workerAdapter.provider,
+        model: workerOut.modelUsed,
+      },
+      reviewer: {
+        name: reviewerName,
+        provider: reviewerAdapter.provider,
+        model: reviewerOut.modelUsed,
+      },
+      metrics,
+      totals,
+      filesReviewed: diffResult.files,
+      redactionsTotal: redaction.totalCount,
+      memoryMissing: [
+        ...new Set([...workerMemory.missing, ...reviewerMemory.missing]),
+      ],
+    };
+    if (shallowWarning) result.shallowWarning = shallowWarning;
+    if (sameProviderWarning) result.sameProviderWarning = sameProviderWarning;
+    return result;
+  } catch (err) {
+    const failedMeta: RunMetadata = {
+      ...baseMeta,
+      finishedAt: new Date().toISOString(),
+      status: err instanceof BudgetExceededError ? "aborted_budget" : "failed",
+      error: {
+        name: err instanceof Error ? err.name : "Error",
+        message: err instanceof Error ? err.message : String(err),
+      },
+    };
+    await writeRunMetadata(runFolder.absPath, failedMeta);
+    emit({
+      type: "run_failed",
+      runId: runFolder.id,
+      error: {
+        name: err instanceof Error ? err.name : "Error",
+        message: err instanceof Error ? err.message : String(err),
+      },
+    });
+    throw err;
+  }
+}
+
+function makeAdapter(name: string, provider: string): Adapter {
+  switch (provider) {
+    case "openai":
+      return new OpenAIAdapter(name);
+    case "anthropic":
+      return new ClaudeAdapter(name);
+    case "google":
+      return new GeminiAdapter(name);
+    case "mock":
+      return new MockAdapter();
+    default:
+      throw new Error(
+        `Provider "${provider}" not supported. Use "openai", "anthropic", "google", or "mock".`,
+      );
+  }
+}
+
+function envHintFor(provider: string): string {
+  switch (provider) {
+    case "openai":
+      return "Export OPENAI_API_KEY in your environment.";
+    case "anthropic":
+      return "Export ANTHROPIC_API_KEY in your environment.";
+    case "google":
+      return "Export GOOGLE_API_KEY (or GEMINI_API_KEY) in your environment.";
+    default:
+      return "";
+  }
+}
+
+function detectShallowReview(
+  content: string,
+  minWeaknesses: number,
+): string | undefined {
+  const fileRefs = content.match(/[\w./_-]+\.(ts|tsx|js|jsx|py|go|rb|md|json|yaml|yml)(?::\d+)?/g);
+  const fileRefCount = fileRefs ? fileRefs.length : 0;
+
+  const weaknessSection = content.match(
+    /##\s+Concrete\s+Weaknesses\s*\n([\s\S]*?)(\n##|$)/i,
+  );
+  const weaknessCount = weaknessSection
+    ? (weaknessSection[1]!.match(/^\s*\d+\.\s+/gm) ?? []).length
+    : 0;
+
+  if (fileRefCount === 0) {
+    return `Reviewer output contains zero file/line references. Likely shallow.`;
+  }
+  if (weaknessCount < minWeaknesses) {
+    return `Reviewer listed ${weaknessCount} concrete weaknesses; config requires ${minWeaknesses}. Output may be shallow.`;
+  }
+  return undefined;
+}