karajan-code 2.34.0 → 3.1.0

package/src/commands/init.js

@@ -12,6 +12,7 @@ import { createWizard, isTTY } from "../utils/wizard.js";
 import { runCommand } from "../utils/process.js";
 import { getInstallCommand } from "../utils/os-detect.js";
 import { detectOsLocale, SUPPORTED_LANGUAGES } from "../utils/locale.js";
+import { buildTelemetryPayload, sendTelemetryEvent } from "../utils/telemetry.js";
 import { detectRtk } from "../utils/rtk-detect.js";
 import { installRtk } from "../utils/rtk-install.js";
 import { detectProjectStack } from "../utils/stack-detect.js";
@@ -256,6 +257,42 @@ async function askBoardSecurity(wizard, config, logger) {
   logger.info(`  -> HU Board port: ${config.hu_board.port}`);
 }
 
+/**
+ * Explicit, opt-in telemetry consent. We do NOT inherit a "Karajan-installed
+ * users want to help" assumption — silence and missing config keys both mean
+ * "no". The prompt is rendered in the OS locale (EN/ES today; falls back to
+ * EN for everything else) so users see it in their own language even before
+ * they've set `config.language`. See src/utils/telemetry.js and the
+ * "Privacy & Telemetry" section of README.md for what gets sent.
+ */
+async function askTelemetry(wizard, config, logger) {
+  const locale = detectOsLocale();
+  const prompts = {
+    es: {
+      question: "¿Quieres ayudar a mejorar Karajan enviando telemetría anónima al proyecto?",
+      detail: "  (versión, OS, comandos, duración del pipeline — sin código, sin tareas, sin datos personales)",
+      previewIntro: "  Ejemplo del payload exacto que se enviaría al ejecutar `kj run` (puedes inspeccionar más con `kj telemetry preview`):",
+      enabled: "  -> Telemetría: activada (¡gracias!)",
+      disabled: "  -> Telemetría: desactivada",
+    },
+    en: {
+      question: "Help improve Karajan by sending anonymous telemetry to the project?",
+      detail: "  (version, OS, commands, pipeline duration — no code, no tasks, no personal data)",
+      previewIntro: "  Example of the exact payload that would be sent when you run `kj run` (inspect more with `kj telemetry preview`):",
+      enabled: "  -> Telemetry: enabled (thank you!)",
+      disabled: "  -> Telemetry: disabled",
+    },
+  };
+  const t = prompts[locale] || prompts.en;
+  logger.info(t.detail);
+  const example = buildTelemetryPayload("cli_command", { version: "x.y.z", command: "run" });
+  logger.info(t.previewIntro);
+  logger.info(`    ${JSON.stringify(example)}`);
+  const enabled = await wizard.confirm(t.question, false);
+  config.telemetry = enabled;
+  logger.info(enabled ? t.enabled : t.disabled);
+}
+
 async function runWizard(config, logger) {
   const agents = await detectAvailableAgents();
   const available = agents.filter((a) => a.available);
@@ -318,6 +355,10 @@ async function runWizard(config, logger) {
       await askBoardSecurity(wizard, config, logger);
     }
 
+    logger.info("");
+    logger.info("Privacy:");
+    await askTelemetry(wizard, config, logger);
+
     logger.info("");
   } finally {
     wizard.close();
@@ -735,7 +776,6 @@ export async function initCommand({ logger, flags = {} }) {
   await writeInitConfig(configPath, config);
 
   // Telemetry: anonymous install event (non-blocking)
-  const { sendTelemetryEvent } = await import("../utils/telemetry.js");
   const { readFileSync } = await import("node:fs");
   const { fileURLToPath } = await import("node:url");
   try {

package/src/commands/plan/generate.js

@@ -21,14 +21,14 @@ import { formatPlan, formatHuTable } from "./_shared.js";
 /**
  * kj plan "task" — generate plan + HUs
  */
-export async function planGenerateCommand({ task, config, logger, json, context, flags = {} }) {
+export async function planGenerateCommand({ task, config, logger, json, context, flags = {}, sourceCanvasPath = null }) {
   const { withCliRunLog } = await import("../../utils/cli-run-log.js");
   return withCliRunLog("plan", { projectDir: config?.projectDir, logger }, async ({ runLog }) => {
-    return planGenerateImpl({ task, config, logger, json, context, runLog, flags });
+    return planGenerateImpl({ task, config, logger, json, context, runLog, flags, sourceCanvasPath });
   });
 }
 
-async function planGenerateImpl({ task, config, logger, json, context, runLog, flags = {} }) {
+async function planGenerateImpl({ task, config, logger, json, context, runLog, flags = {}, sourceCanvasPath = null }) {
   // Auto-GC: prune orphan plans (project dir gone) + old finalised plans/
   // sessions/HU batches before we start. Silent unless something was
   // removed; one-liner summary on stderr in that case so --json stays
@@ -164,6 +164,12 @@ async function planGenerateImpl({ task, config, logger, json, context, runLog, f
   plan.approach = parsed?.approach || (typeof parsed === "string" ? parsed : (typeof result.output === "string" ? result.output : null));
   plan.risks = parsed?.risks || [];
   plan.outOfScope = parsed?.outOfScope || [];
+  // KJC-TSK-0348: persist the SPEC.canvas.md path so `kj sync --apply`
+  // can later locate it and append drift notes. Only set when the plan
+  // actually came from a Canvas SPEC; flat-task plans stay unstamped.
+  if (typeof sourceCanvasPath === "string" && sourceCanvasPath.length > 0) {
+    plan.sourceCanvasPath = sourceCanvasPath;
+  }
 
   // Convert planner steps → HUs, carrying the structured acceptance_tests
   // the LLM was asked to emit. If the LLM fell back to legacy behaviour

package/src/commands/rag-mcp.js

@@ -0,0 +1,18 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { pathToFileURL } from "node:url";
+
+/**
+ * `kj rag mcp` — spawn the standalone RAG MCP server inline.
+ *
+ * KJC-TSK-0492 PR3. Adds CLI discoverability for the `kj-rag-mcp` binary
+ * so external IDE users can find it via the same `kj rag <subcommand>`
+ * surface they already know. The implementation dynamic-imports the
+ * binary entry so behavior stays identical to running `kj-rag-mcp`
+ * directly — same handlers, same transport, same orphan guard.
+ */
+export async function ragMcpCommand() {
+  const moduleDir = path.dirname(fileURLToPath(import.meta.url));
+  const binPath = path.resolve(moduleDir, "../../bin/kj-rag-mcp.js");
+  await import(pathToFileURL(binPath).href);
+}

package/src/commands/sync.js

@@ -1,12 +1,15 @@
 /**
  * `kj sync` — closed-loop drift detection between code and Canvas/plan.
- * Issue #540 MVP: detection + report only. No --apply yet.
+ * Issue #540 MVP: detection + report.
+ * KJC-TSK-0348: `--apply` writes drift notes back into the source Canvas
+ * markdown (with `.bak` backup) so the user can review + commit.
  */
 
 import fs from "node:fs";
 import path from "node:path";
 import { listPlans, loadPlan } from "../plan/plan-store.js";
 import { classifyDrift, formatDriftReport } from "../sync/detect-drift.js";
+import { applyCanvasPatch } from "../sync/apply-canvas-patch.js";
 
 const SKIP_DIRS = new Set(["node_modules", ".git", "dist", "build", "coverage", ".karajan", ".kj", "tmp", ".cache", ".astro", ".next"]);
 
@@ -36,8 +39,11 @@ function listSourceFiles(rootDir, out = []) {
  * @param {object} args.logger
  * @param {string} [args.planId] — when set, sync against this plan; else use the latest
  * @param {boolean} [args.json]
+ * @param {boolean} [args.apply] — when true, write a drift patch back into
+ *                                 the Canvas markdown the plan came from
+ *                                 (with `.bak` backup).
  */
-export async function syncCommand({ config, logger, planId, json }) {
+export async function syncCommand({ config, logger, planId, json, apply }) {
   const projectDir = config?.projectDir || process.cwd();
 
   // Resolve which plan to sync against.
@@ -60,14 +66,36 @@ export async function syncCommand({ config, logger, planId, json }) {
   const files = listSourceFiles(projectDir);
   const report = classifyDrift(plan, files);
 
+  let applyResult = null;
+  if (apply) {
+    if (!plan.sourceCanvasPath) {
+      throw new Error(
+        "kj sync --apply: this plan has no sourceCanvasPath — it was not generated from a SPEC.canvas.md. "
+        + "Re-run `kj plan generate --task-file SPEC.canvas.md` so the plan remembers the Canvas it came from."
+      );
+    }
+    applyResult = await applyCanvasPatch({ canvasPath: plan.sourceCanvasPath, report });
+  }
+
   if (json) {
     console.log(JSON.stringify({
       planId: plan.planId,
       projectDir,
       ...report,
+      apply: applyResult,
     }, null, 2));
   } else {
     console.log(formatDriftReport(report, plan));
+    if (applyResult) {
+      if (applyResult.skipped) {
+        console.log("\nkj sync --apply: no actionable drift, Canvas left untouched.");
+      } else {
+        console.log(
+          `\nkj sync --apply: wrote ${applyResult.mutations} drift note(s) to ${applyResult.patchedPath} `
+          + `(backup at ${applyResult.backupPath}).`
+        );
+      }
+    }
   }
 
   // Exit 0 always — drift is informational, not an error. CI users

package/src/commands/telemetry.js

@@ -0,0 +1,70 @@
+import { TELEMETRY_ENDPOINT, buildTelemetryPayload, isTelemetryEnabled } from "../utils/telemetry.js";
+
+/**
+ * Render the exact payload that would be POSTed for every event type.
+ *
+ * Read-only by design: this never opens a socket. It exists so users
+ * can inspect what telemetry sends before (or after) opting in,
+ * without grepping the source. The contract here MUST stay in sync
+ * with the producers in src/cli/_shared.js and the pipeline emitter
+ * in src/orchestrator/flow-runner.js — if a new field appears there,
+ * mirror it in the example below.
+ */
+function buildPreview(version) {
+  const v = version || "x.y.z";
+  return {
+    endpoint: TELEMETRY_ENDPOINT,
+    events: [
+      buildTelemetryPayload("install", { version: v }),
+      buildTelemetryPayload("cli_command", { version: v, command: "run" }),
+      buildTelemetryPayload("pipeline_complete", {
+        version: v,
+        mode: "standard",
+        agent: "claude",
+        duration_s: 42,
+        success: true,
+        taskType: "feature"
+      })
+    ]
+  };
+}
+
+export async function telemetryPreviewCommand({ config, logger, version, json }) {
+  const preview = buildPreview(version);
+  if (json) {
+    logger.info(JSON.stringify(preview, null, 2));
+    return preview;
+  }
+  const enabled = isTelemetryEnabled(config);
+  logger.info(`Telemetry endpoint: ${preview.endpoint}`);
+  logger.info(`Telemetry enabled:  ${enabled ? "yes" : "no"}`);
+  logger.info("");
+  logger.info("Exact payloads that would be POSTed (one per event type):");
+  logger.info("");
+  for (const payload of preview.events) {
+    logger.info(`  [${payload.event}]`);
+    logger.info(`    ${JSON.stringify(payload)}`);
+    logger.info("");
+  }
+  logger.info("No data was sent — `kj telemetry preview` never opens a socket.");
+  return preview;
+}
+
+export async function telemetryStatusCommand({ config, logger, json }) {
+  const enabled = isTelemetryEnabled(config);
+  const report = {
+    enabled,
+    endpoint: TELEMETRY_ENDPOINT,
+    flipHint: enabled
+      ? "Set `telemetry: false` in ~/.karajan/kj.config.yml (or re-run `kj init`) to turn it off."
+      : "Set `telemetry: true` in ~/.karajan/kj.config.yml (or re-run `kj init`) to turn it on."
+  };
+  if (json) {
+    logger.info(JSON.stringify(report, null, 2));
+    return report;
+  }
+  logger.info(`Telemetry: ${enabled ? "enabled" : "disabled"}`);
+  logger.info(`Endpoint:  ${report.endpoint}`);
+  logger.info(report.flipHint);
+  return report;
+}

package/src/config/defaults.js

@@ -70,6 +70,11 @@ const DEFAULTS = {
   development: {
     methodology: "tdd",
     require_test_changes: true,
+    // KJC-TSK-0398: opt-in red-then-green gate. When true, the pipeline
+    // stashes source files and re-runs the test suite to confirm the new
+    // tests actually fail without the implementation. Off by default to
+    // preserve current iteration latency.
+    require_red_then_green: false,
     test_file_patterns: ["/tests/", "/__tests__/", ".test.", ".spec."],
     source_file_extensions: [".js", ".jsx", ".ts", ".tsx", ".py", ".go", ".java", ".rb", ".php", ".cs"]
   },
@@ -196,7 +201,10 @@ const DEFAULTS = {
     // hook. Both default ON; flip to false to opt out without uninstalling.
     autoUpdate: { onCommit: true, onRun: true }
   },
-  telemetry: true,
+  // Opt-in. Set by the `kj init` wizard (explicit consent prompt in the OS
+  // locale). When the key is absent or false, no events are sent. See
+  // src/utils/telemetry.js and the Privacy & Telemetry section of README.md.
+  telemetry: false,
   guards: {
     output: {
       enabled: true,

package/src/config/schema.js

@@ -115,6 +115,8 @@ const PipelineMap = v.optional(v.looseObject({
 const Development = v.optional(v.looseObject({
   methodology: v.optional(Methodology),
   require_test_changes: v.optional(v.boolean()),
+  // KJC-TSK-0398: opt-in flag; module lands in PR1, integration in PR3.
+  require_red_then_green: v.optional(v.boolean()),
   test_file_patterns: v.optional(v.array(v.string())),
   source_file_extensions: v.optional(v.array(v.string())),
 }));

package/src/orchestrator/config-init.js

@@ -229,6 +229,7 @@ export function resolvePipelineFlags(config) {
     securityEnabled: Boolean(config.pipeline?.security?.enabled),
     impeccableEnabled: Boolean(config.pipeline?.impeccable?.enabled),
     perfEnabled: Boolean(config.pipeline?.perf?.enabled),
+    toolJudgeEnabled: Boolean(config.pipeline?.tool_judge?.enabled),
     reviewerEnabled: config.pipeline?.reviewer?.enabled !== false,
     discoverEnabled: Boolean(config.pipeline?.discover?.enabled),
     architectEnabled: Boolean(config.pipeline?.architect?.enabled),

package/src/orchestrator/drivers/iteration-phases/quality-gates.js

@@ -19,6 +19,8 @@ import {
 } from "../../iteration-stages.js";
 import { runImpeccableStage } from "../../post-loop-stages.js";
 import { runPerfStage } from "../../stages/perf-stage.js";
+import { runTddDisciplineStage } from "../../stages/tdd-discipline-stage.js";
+import { runToolJudgeStage } from "../../stages/tool-judge-stage.js";
 import { tryCiComment } from "../../ci-integration.js";
 
 export async function runQualityGateStages({ config, logger, emitter, eventBase, session, trackBudget, i, askQuestion, repeatDetector, budgetSummary, sonarState, task, stageResults, coderRole, pipelineFlags, brainCtx }) {
@@ -26,6 +28,16 @@ export async function runQualityGateStages({ config, logger, emitter, eventBase,
   if (tddResult.action === "pause") return { action: "return", result: tddResult.result };
   if (tddResult.action === "continue") return { action: "continue" };
 
+  // KJC-TSK-0398 PR3: opt-in red-then-green check.
+  if (config.development?.require_red_then_green) {
+    const disc = await runTddDisciplineStage({
+      config, logger, emitter, eventBase,
+      sourceFiles: tddResult.sourceFiles || [],
+      testFiles: tddResult.testFiles || [],
+    });
+    if (disc.action === "continue") return { action: "continue" };
+  }
+
   // Sonar runs for code tasks per policy. Since v2.7.4 it is NOT
   // toggleable via config — that's intrinsic to Karajan. The taskType
   // policy (resolved_policies.sonar) is the single source of truth.
@@ -84,5 +96,16 @@ export async function runQualityGateStages({ config, logger, emitter, eventBase,
     if (perfResult.action === "continue") return { action: "continue" };
   }
 
+  // KJC-TSK-0375 PR3: opt-in tool-call quality judge. Non-blocking.
+  if (pipelineFlags?.toolJudgeEnabled) {
+    const judgeResult = await runToolJudgeStage({
+      config, logger, emitter, eventBase, session, coderRole, trackBudget,
+      iteration: i, task,
+    });
+    if (judgeResult.stageResult) {
+      stageResults.toolJudge = judgeResult.stageResult;
+    }
+  }
+
   return { action: "ok" };
 }

package/src/orchestrator/stages/coder-stage.js

@@ -8,7 +8,7 @@ import { CoderRole } from "../../roles/coder-role.js";
 import { RefactorerRole } from "../../roles/refactorer-role.js";
 import { addCheckpoint, markSessionStatus, saveSession } from "../../session/store.js";
 import {
-  setReviewerFeedback, incrementRetryCount, resetRetryCount,
+  setReviewerFeedback, setCoderTranscript, incrementRetryCount, resetRetryCount,
 } from "../../session/mutators.js";
 import { generateDiff, getUntrackedFiles } from "../../review/diff-generator.js";
 import { evaluateTddPolicy } from "../../review/tdd-policy.js";
@@ -19,6 +19,7 @@ import { invokeSolomon } from "../solomon-escalation.js";
 import { detectRateLimit } from "../../utils/rate-limit-detector.js";
 import { createStallDetector } from "../../utils/stall-detector.js";
 import { buildStandbyState } from "../../brain/standby-store.js";
+import { applyQuotaSimulation } from "../../utils/quota-simulator.js";
 import { snapshotHomeTopLevel, detectNewHomeEntries, formatLeakMessage, verifyLeaksAgainstTranscript, detectTranscriptCdLeaks } from "../fs-leak-detector.js";
 
 export async function runCoderStage({ coderRoleInstance, coderRole, config, logger, emitter, eventBase, session, plannedTask, trackBudget, iteration, brainCtx, acceptanceTests = null, adrs = null, specSection = null, reviewerFindings = null, huId = null }) {
@@ -76,6 +77,7 @@ export async function runCoderStage({ coderRoleInstance, coderRole, config, logg
     coderStall.stop();
   }
   trackBudget({ role: "coder", provider: coderRole.provider, model: coderRole.model, result: coderExecResult.result, duration_ms: Date.now() - coderStart });
+  applyQuotaSimulation(coderExecResult, { iteration, agent: coderRole.provider });
 
   if (!coderExecResult.ok) {
     const details = coderExecResult.result?.error || coderExecResult.summary || "unknown error";
@@ -221,6 +223,12 @@ export async function runCoderStage({ coderRoleInstance, coderRole, config, logg
     logger?.warn?.(`[verification-gate] Skipped: ${err?.message || err}`);
   }
 
+  // KJC-TSK-0375 PR3: stash the coder's raw transcript on the session so
+  // the optional tool-judge stage can extract structured tool calls later
+  // without re-invoking the coder. Cleared between iterations by the coder
+  // overwriting it; ignored when the tool-judge flag is off.
+  setCoderTranscript(session, coderTranscript);
+
   await addCheckpoint(session, { stage: "coder", iteration, note: "Coder applied changes", provider: coderRole.provider, model: coderRole.model || null, filesChanged });
   emitProgress(
     emitter,
@@ -281,6 +289,7 @@ export async function runRefactorerStage({ refactorerRole, config, logger, emitt
     refactorerStall.stop();
   }
   trackBudget({ role: "refactorer", provider: refactorerRole.provider, model: refactorerRole.model, result: refResult.result, duration_ms: Date.now() - refactorerStart });
+  applyQuotaSimulation(refResult, { iteration, agent: refactorerRole.provider });
   if (!refResult.ok) {
     const details = refResult.result?.error || refResult.summary || "unknown error";
     // Quota cap → hibernate cleanly, same as the coder stage above.
@@ -456,5 +465,5 @@ export async function runTddCheckStage({ config, logger, emitter, eventBase, ses
     return handleTddFailure({ tddEval, config, logger, emitter, eventBase, session, iteration, askQuestion, task, brainCtx });
   }
 
-  return { action: "ok" };
+  return { action: "ok", sourceFiles: tddEval.sourceFiles, testFiles: tddEval.testFiles };
 }

package/src/orchestrator/stages/reviewer-stage.js

@@ -13,6 +13,7 @@ import { runReviewerWithFallback } from "../reviewer-fallback.js";
 import { invokeSolomon } from "../solomon-escalation.js";
 import { detectRateLimit } from "../../utils/rate-limit-detector.js";
 import { createStallDetector } from "../../utils/stall-detector.js";
+import { applyQuotaSimulation } from "../../utils/quota-simulator.js";
 
 function categorizeIssues(issues) {
   const categories = { security: 0, correctness: 0, tests: 0, style: 0, other: 0 };
@@ -269,6 +270,16 @@ export async function runReviewerStage({ reviewerRole, config, logger, emitter,
     reviewerStall.stop();
   }
 
+  if (applyQuotaSimulation(reviewerExec.execResult, { iteration, agent: reviewerRole.provider })) {
+    // Reviewer wraps the agent call: detectRateLimit reads `lastAttempt.result.error`,
+    // so propagate the synthetic line into the most recent attempt too.
+    const last = reviewerExec.attempts?.at(-1);
+    if (last) {
+      if (!last.result || typeof last.result !== "object") last.result = {};
+      last.result.error = reviewerExec.execResult.result.error;
+    }
+  }
+
   if (!reviewerExec.execResult?.ok) {
     const lastAttempt = reviewerExec.attempts.at(-1);
     const details =

package/src/orchestrator/stages/tdd-discipline-stage.js

@@ -0,0 +1,56 @@
+// TDD discipline stage (KJC-TSK-0398 PR3).
+// Runs the red-then-green check when `development.require_red_then_green`
+// is true. Wraps `verifyTddDiscipline` with the surgical stash helper and
+// the project's detected test framework. Gate is opt-in; an unsupported
+// framework reports a warning and does not block the iteration.
+
+import { detectTestFramework } from "../../utils/project-detect.js";
+import { verifyTddDiscipline } from "../../review/tdd-discipline.js";
+import { stashFilesAside, restoreFiles } from "../../review/git-stash-helper.js";
+import { runCommand } from "../../utils/process.js";
+import { emitProgress, makeEvent } from "../../utils/events.js";
+
+const TDD_DISCIPLINE_COMMANDS = {
+  vitest: ["npx", ["vitest", "run", "--reporter=verbose"]],
+  jest: ["npx", ["jest", "--verbose"]],
+  pytest: ["pytest", ["--tb=short"]],
+};
+
+export async function runTddDisciplineStage({
+  config, logger, emitter, eventBase, sourceFiles = [], testFiles = [],
+  detectFramework = detectTestFramework,
+  runTestsImpl,
+  stashImpl = stashFilesAside,
+  restoreImpl = restoreFiles,
+} = {}) {
+  const projectDir = config?.projectDir || process.cwd();
+  const detection = await detectFramework(projectDir);
+  const framework = detection?.framework || null;
+
+  const runTests = runTestsImpl || (async () => {
+    const cmd = TDD_DISCIPLINE_COMMANDS[framework];
+    if (!cmd) return { exitCode: 0, output: "" };
+    const res = await runCommand(cmd[0], cmd[1], { cwd: projectDir, reject: false });
+    return { exitCode: res.exitCode ?? 0, output: `${res.stdout || ""}\n${res.stderr || ""}` };
+  });
+
+  const result = await verifyTddDiscipline({
+    projectDir, sourceFiles, testFiles, framework,
+    runTests,
+    stashFiles: (files) => stashImpl(files, { projectDir }),
+    restoreFiles: (token) => restoreImpl(token, { projectDir }),
+  });
+
+  emitProgress(emitter, makeEvent("tdd:discipline", { ...eventBase, stage: "tdd" }, {
+    status: result.ok ? "ok" : "fail",
+    message: result.message || `TDD discipline: ${result.reason}`,
+    detail: { reason: result.reason, framework },
+  }));
+
+  if (result.warning) logger?.warn?.(result.warning);
+  if (!result.ok) {
+    logger?.warn?.(`TDD discipline violation: ${result.reason}`);
+    return { action: "continue", result };
+  }
+  return { action: "ok", result };
+}

package/src/orchestrator/stages/tool-judge-stage.js

@@ -0,0 +1,53 @@
+// KJC-TSK-0375 PR3 — ToolJudgeStage.
+// Opt-in (`pipeline.tool_judge.enabled`). Scores the coder's tool calls
+// extracted from `session.last_coder_transcript`. Non-blocking: a low
+// score emits a warn event and surfaces in summary but never gates the
+// iteration loop. Wiring lives in `quality-gates.js`.
+
+import { ToolCorrectnessJudgeRole } from "../../roles/tool-correctness-judge-role.js";
+import { extractToolCalls } from "../../utils/tool-call-extractor.js";
+import { emitProgress, makeEvent } from "../../utils/events.js";
+
+export async function runToolJudgeStage({
+  config, logger, emitter, eventBase, session, coderRole, trackBudget,
+  iteration, task,
+  extractFn = extractToolCalls,
+  RoleClass = ToolCorrectnessJudgeRole,
+} = {}) {
+  logger?.setContext?.({ iteration, stage: "tool-judge" });
+  emitProgress(emitter, makeEvent("toolJudge:start", { ...eventBase, stage: "tool-judge" }, {
+    message: "Tool-judge scoring coder tool calls",
+  }));
+
+  const transcript = session?.last_coder_transcript || "";
+  const provider = coderRole?.provider || "claude";
+  const toolCalls = extractFn({ transcript, provider });
+
+  const role = new RoleClass({ config, logger, emitter });
+  const roleResult = await role.execute({ task, toolCalls });
+
+  try { trackBudget?.("tool-judge", roleResult.usage || { tokens_in: 0, tokens_out: 0, cost_usd: 0 }); } catch { /* best-effort */ }
+
+  const r = roleResult.result;
+  const status = r?.skipped ? "skip" : r?.lowQuality ? "warn" : roleResult.ok ? "ok" : "fail";
+
+  emitProgress(emitter, makeEvent("toolJudge:end", { ...eventBase, stage: "tool-judge" }, {
+    status,
+    message: roleResult.summary || "tool-judge done",
+    detail: { score: r?.score ?? null, threshold: r?.threshold ?? null, skipped: Boolean(r?.skipped), callCount: toolCalls.length },
+  }));
+
+  return {
+    action: "ok",
+    stageResult: {
+      ok: roleResult.ok !== false,
+      score: r?.score ?? null,
+      lowQuality: Boolean(r?.lowQuality),
+      skipped: Boolean(r?.skipped),
+      threshold: r?.threshold ?? null,
+      summary: roleResult.summary || "",
+      breakdown: r?.breakdown || [],
+      callCount: toolCalls.length,
+    },
+  };
+}

package/src/prompts/tool-correctness-judge.js

@@ -0,0 +1,31 @@
+// KJC-TSK-0375 — Prompt builder for the ToolCorrectnessJudge role.
+// Inspired by deepeval's tool-correctness metric.
+
+const PREAMBLE = "IMPORTANT: You are running as a Karajan sub-agent. "
+  + "Do NOT use any MCP tools, do NOT mention Karajan. "
+  + "Your job is to judge another agent's tool choices.";
+
+export function buildToolCorrectnessJudgePrompt({ task, toolCalls, instructions }) {
+  const calls = toolCalls
+    .map((c, i) => `${i + 1}. **${c.tool}** — args: ${JSON.stringify(c.args)}`)
+    .join("\n");
+  const body = [
+    "You are the ToolCorrectnessJudge for Karajan Code.",
+    "Input: (a) a task description and (b) the ordered tool calls an LLM coder",
+    "made (Read/Edit/Write/Bash/Glob/Grep/MultiEdit). Judge each call on:",
+    "  - tool choice: right tool for the intent?",
+    "  - argument quality: paths plausible, commands valid, no garbage placeholders?",
+    "Per-call verdict: `correct=true|false` + one-sentence `reason`.",
+    "Aggregate into a single 0-100 score:",
+    "  - 100 = every call right with clean args",
+    "  - 70-99 = mostly right, minor missteps",
+    "  - 40-69 = several wrong choices, budget wasted",
+    "  - 0-39 = predominantly bad, retry advised",
+    "Return a single valid JSON object and nothing else.",
+    'Schema: {"score":0-100,"summary":string,"breakdown":[{"tool":string,"args":object,"correct":boolean,"reason":string}]}',
+  ].join("\n");
+  const sections = [PREAMBLE];
+  if (instructions) sections.push(instructions);
+  sections.push(body, `## Task\n${task}`, `## Tool calls (${toolCalls.length})\n${calls || "(no tool calls captured)"}`);
+  return sections.join("\n\n");
+}