karajan-code 1.41.0 → 1.42.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "karajan-code",
-  "version": "1.41.0",
+  "version": "1.42.0",
   "description": "Local multi-agent coding orchestrator with TDD, SonarQube, and code review pipeline",
   "type": "module",
   "license": "AGPL-3.0",

package/src/audit/basal-cost.js

@@ -0,0 +1,190 @@
+import fs from "node:fs";
+import path from "node:path";
+import { execFile, execFileSync } from "node:child_process";
+import { getKarajanHome } from "../utils/paths.js";
+
+const SOURCE_EXTENSIONS = new Set([
+  ".js", ".mjs", ".cjs", ".ts", ".mts", ".cts", ".tsx", ".jsx",
+  ".py", ".rb", ".go", ".rs", ".java", ".kt", ".swift", ".c", ".cpp", ".h",
+  ".vue", ".svelte", ".astro", ".php", ".cs", ".sh"
+]);
+
+const EXCLUDE_DIRS = new Set([
+  "node_modules", ".git", "dist", "build", "coverage", ".next",
+  ".nuxt", ".output", "__pycache__", ".cache", ".parcel-cache"
+]);
+
+function slugify(projectDir) {
+  return path.basename(projectDir).replace(/[^a-zA-Z0-9_-]/g, "_");
+}
+
+function walkSourceFiles(dir) {
+  // Use git ls-files for speed (instant vs recursive walk)
+  try {
+    const output = execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard"], { cwd: dir, encoding: "utf8", timeout: 5000 });
+    return output.trim().split("\n")
+      .filter(f => f && SOURCE_EXTENSIONS.has(path.extname(f)) && !EXCLUDE_DIRS.has(f.split("/")[0]))
+      .map(f => path.join(dir, f));
+  } catch {
+    // Fallback to recursive walk if not a git repo
+    const files = [];
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return files;
+    }
+    for (const entry of entries) {
+      if (EXCLUDE_DIRS.has(entry.name)) continue;
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        files.push(...walkSourceFiles(full));
+      } else if (SOURCE_EXTENSIONS.has(path.extname(entry.name))) {
+        files.push(full);
+      }
+    }
+    return files;
+  }
+}
+
+function countLines(filePath) {
+  try {
+    const content = fs.readFileSync(filePath, "utf8");
+    return content.split("\n").length;
+  } catch {
+    return 0;
+  }
+}
+
+function countDependencies(projectDir) {
+  const pkgPath = path.join(projectDir, "package.json");
+  try {
+    const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8"));
+    const deps = Object.keys(pkg.dependencies || {});
+    const devDeps = Object.keys(pkg.devDependencies || {});
+    return { dependencies: deps.length, devDependencies: devDeps.length, total: deps.length + devDeps.length };
+  } catch {
+    return { dependencies: 0, devDependencies: 0, total: 0 };
+  }
+}
+
+function runDepcheck(projectDir) {
+  return new Promise((resolve) => {
+    execFile("npx", ["depcheck", "--json"], { cwd: projectDir, timeout: 30000 }, (err, stdout) => {
+      if (err && !stdout) {
+        resolve({ unused: [], note: "depcheck not available or failed" });
+        return;
+      }
+      try {
+        const result = JSON.parse(stdout);
+        const unused = [
+          ...(result.dependencies || []),
+          ...(result.devDependencies || [])
+        ];
+        resolve({ unused });
+      } catch {
+        resolve({ unused: [], note: "depcheck output not parseable" });
+      }
+    });
+  });
+}
+
+function findDeadExports(sourceFiles) {
+  const exportMap = new Map(); // file -> [exportName]
+  const importedNames = new Set();
+
+  const exportRe = /export\s+(?:default\s+)?(?:function|class|const|let|var|async\s+function)\s+(\w+)/g;
+  const namedExportRe = /export\s*\{([^}]+)\}/g;
+  const importRe = /import\s+\{([^}]+)\}\s+from/g;
+  const defaultImportRe = /import\s+(\w+)\s+from/g;
+
+  for (const file of sourceFiles) {
+    let content;
+    try {
+      content = fs.readFileSync(file, "utf8");
+    } catch {
+      continue;
+    }
+
+    const exports = [];
+    for (const match of content.matchAll(exportRe)) {
+      exports.push(match[1]);
+    }
+    for (const match of content.matchAll(namedExportRe)) {
+      const names = match[1].split(",").map(n => n.trim().split(/\s+as\s+/).pop().trim());
+      exports.push(...names);
+    }
+    if (exports.length > 0) {
+      exportMap.set(file, exports);
+    }
+
+    for (const match of content.matchAll(importRe)) {
+      const names = match[1].split(",").map(n => n.trim().split(/\s+as\s+/)[0].trim());
+      for (const name of names) importedNames.add(name);
+    }
+    for (const match of content.matchAll(defaultImportRe)) {
+      importedNames.add(match[1]);
+    }
+  }
+
+  const dead = [];
+  for (const [file, exports] of exportMap) {
+    for (const name of exports) {
+      if (!importedNames.has(name)) {
+        dead.push({ file, name });
+      }
+    }
+  }
+  return dead;
+}
+
+export async function measureBasalCost(projectDir) {
+  const sourceFiles = walkSourceFiles(projectDir);
+  let totalLines = 0;
+  for (const f of sourceFiles) {
+    totalLines += countLines(f);
+  }
+
+  const depInfo = countDependencies(projectDir);
+  const depcheck = await runDepcheck(projectDir);
+  const deadExports = findDeadExports(sourceFiles);
+
+  return {
+    totalLines,
+    totalFiles: sourceFiles.length,
+    dependencies: depInfo,
+    unusedDependencies: depcheck,
+    deadExports
+  };
+}
+
+function auditDir() {
+  return path.join(getKarajanHome(), "audits");
+}
+
+export function loadPreviousAudit(projectDir) {
+  const file = path.join(auditDir(), slugify(projectDir), "latest.json");
+  try {
+    return JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+export function saveAuditSnapshot(projectDir, metrics) {
+  const dir = path.join(auditDir(), slugify(projectDir));
+  fs.mkdirSync(dir, { recursive: true });
+  const snapshot = { ...metrics, timestamp: new Date().toISOString() };
+  fs.writeFileSync(path.join(dir, "latest.json"), JSON.stringify(snapshot, null, 2));
+  return snapshot;
+}
+
+export function computeGrowthDelta(current, previous) {
+  if (!previous) return null;
+  return {
+    lines: current.totalLines - previous.totalLines,
+    files: current.totalFiles - previous.totalFiles,
+    deps: current.dependencies.total - (previous.dependencies?.total ?? 0),
+    since: previous.timestamp || null
+  };
+}

package/src/hu/lazy-planner.js

@@ -0,0 +1,104 @@
+/**
+ * Lazy HU Planner — refines HUs one at a time using context from completed HUs.
+ *
+ * Instead of fully planning all HUs upfront, only the first HU is decomposed
+ * with full acceptance criteria. Subsequent HUs are refined lazily before
+ * execution, incorporating learnings from previously completed HUs.
+ */
+import { createAgent } from "../agents/index.js";
+import { extractFirstJson } from "../utils/json-extract.js";
+
+const SUBAGENT_PREAMBLE = [
+  "IMPORTANT: You are running as a Karajan sub-agent.",
+  "Do NOT ask about using Karajan, do NOT mention Karajan, do NOT suggest orchestration.",
+  "Do NOT use any MCP tools. Focus only on refining the user story."
+].join(" ");
+
+/**
+ * Build a prompt asking the AI to refine an HU based on context from completed HUs.
+ * @param {object} hu - The HU to refine (with at least id, original.text or certified.text).
+ * @param {Array<object>} completedHus - Array of completed HU results with summaries.
+ * @returns {string} The assembled refinement prompt.
+ */
+export function buildRefinementPrompt(hu, completedHus) {
+  const sections = [SUBAGENT_PREAMBLE];
+
+  sections.push("## HU Refinement");
+  sections.push(
+    "You are a senior product owner. Refine the following user story by adding or updating its acceptance criteria.",
+    "Use the context from previously completed HUs to make the criteria more precise and avoid redundant work."
+  );
+
+  if (completedHus.length > 0) {
+    sections.push("## Previously Completed HUs");
+    for (const completed of completedHus) {
+      const title = completed.certified?.title || completed.id;
+      const summary = completed.resultSummary || completed.certified?.text || completed.original?.text || "No summary available";
+      sections.push(`### ${completed.id} — ${title}\nResult: ${summary}`);
+    }
+  }
+
+  const huText = hu.certified?.text || hu.original?.text || `Implement HU ${hu.id}`;
+  sections.push(`## HU to Refine\n### ${hu.id}\n${huText}`);
+
+  sections.push(
+    "Return a single valid JSON object and nothing else.",
+    `JSON schema: {"id":string,"title":string,"text":string,"acceptanceCriteria":[string]}`,
+    "Rules:",
+    "- Keep the same HU id",
+    "- Refine the description to be more specific based on what was learned",
+    "- Add concrete, testable acceptance criteria",
+    "- Avoid repeating work already done in completed HUs",
+    "- The text field should be the full story description ready for implementation"
+  );
+
+  return sections.join("\n\n");
+}
+
+/**
+ * Refine an HU using AI, incorporating context from completed HUs.
+ * @param {object} hu - The HU story object to refine.
+ * @param {Array<object>} completedHus - Completed HU stories with results.
+ * @param {object} config - Pipeline configuration.
+ * @param {object} logger - Logger instance.
+ * @returns {Promise<object>} Updated HU object with refined description and AC.
+ */
+export async function refineHuWithContext(hu, completedHus, config, logger) {
+  const provider = config?.roles?.hu_reviewer?.provider || config?.roles?.coder?.provider || "claude";
+  const prompt = buildRefinementPrompt(hu, completedHus);
+
+  logger.info(`Lazy planner: refining HU ${hu.id} with context from ${completedHus.length} completed HU(s)`);
+
+  const agent = createAgent(provider, config, logger);
+  let result;
+  try {
+    result = await agent.runTask({ prompt, role: "hu-reviewer" });
+  } catch (err) {
+    logger.warn(`Lazy planner: refinement failed for HU ${hu.id}: ${err.message}`);
+    return hu;
+  }
+
+  if (!result.ok) {
+    logger.warn(`Lazy planner: refinement returned not-ok for HU ${hu.id}`);
+    return hu;
+  }
+
+  const parsed = extractFirstJson(result.output);
+  if (!parsed || !parsed.text) {
+    logger.warn(`Lazy planner: could not parse refinement output for HU ${hu.id}`);
+    return hu;
+  }
+
+  // Update the HU with refined content
+  const refined = { ...hu };
+  refined.certified = {
+    ...(hu.certified || {}),
+    text: parsed.text,
+    title: parsed.title || hu.certified?.title || hu.id,
+    acceptanceCriteria: parsed.acceptanceCriteria || []
+  };
+  refined.needsRefinement = false;
+
+  logger.info(`Lazy planner: HU ${hu.id} refined successfully`);
+  return refined;
+}

package/src/orchestrator/hu-sub-pipeline.js

@@ -5,6 +5,7 @@
 import { topologicalSort } from "../hu/graph.js";
 import { updateStoryStatus, loadHuBatch, saveHuBatch, HU_STATUS } from "../hu/store.js";
 import { emitProgress, makeEvent } from "../utils/events.js";
+import { refineHuWithContext } from "../hu/lazy-planner.js";
 
 /**
  * Determine whether the HU reviewer result requires a sub-pipeline
@@ -72,7 +73,7 @@ function buildHuTask(story) {
  * @param {object} params.logger - Logger instance
  * @returns {Promise<{approved: boolean, results: object[], blockedIds: string[]}>}
  */
-export async function runHuSubPipeline({ huReviewerResult, runIterationFn, emitter, eventBase, logger }) {
+export async function runHuSubPipeline({ huReviewerResult, runIterationFn, emitter, eventBase, logger, config = null }) {
   const batchSessionId = huReviewerResult.batchSessionId;
   let batch;
   try {
@@ -108,6 +109,40 @@ export async function runHuSubPipeline({ huReviewerResult, runIterationFn, emitt
       continue;
     }
 
+    // --- Lazy refinement: refine HU if it needs it, using completed HU context ---
+    if (story.needsRefinement && config) {
+      const completedHus = results
+        .filter(r => r.approved)
+        .map(r => {
+          const completedStory = batch.stories.find(s => s.id === r.huId);
+          return {
+            ...completedStory,
+            resultSummary: r.result?.summary || r.result?.reason || null
+          };
+        });
+
+      emitProgress(emitter, makeEvent("hu:refine-start", { ...eventBase, stage: "hu-sub-pipeline" }, {
+        message: `Refining HU ${story.id} with context from ${completedHus.length} completed HU(s)`,
+        detail: { huId: story.id, completedCount: completedHus.length }
+      }));
+
+      try {
+        const refined = await refineHuWithContext(story, completedHus, config, logger);
+        // Apply refinement to the batch story in-place
+        Object.assign(story, refined);
+        story.needsRefinement = false;
+        await saveHuBatch(batchSessionId, batch);
+
+        emitProgress(emitter, makeEvent("hu:refine-end", { ...eventBase, stage: "hu-sub-pipeline" }, {
+          message: `HU ${story.id} refined successfully`,
+          detail: { huId: story.id }
+        }));
+      } catch (err) {
+        logger.warn(`Lazy refinement failed for HU ${story.id}: ${err.message} — proceeding with original`);
+        story.needsRefinement = false;
+      }
+    }
+
     const huTask = buildHuTask(story);
 
     // --- hu:start ---

package/src/orchestrator/pre-loop-stages.js

@@ -646,11 +646,17 @@ export async function runHuReviewerStage({ config, logger, emitter, eventBase, s
       config, logger, emitter, eventBase, session, coderRole, trackBudget
     });
     if (decomposed && decomposed.length > 1) {
-      stories = decomposed.map(hu => ({
-        id: hu.id,
-        text: `As a ${hu.role}, I want to ${hu.goal}, so that ${hu.benefit}\n\nTitle: ${hu.title}\nAcceptance Criteria:\n${hu.acceptanceCriteria.map(ac => `- ${ac}`).join("\n")}`,
-        blocked_by: hu.dependsOn || []
-      }));
+      stories = decomposed.map((hu, idx) => {
+        const isFirst = idx === 0;
+        const fullText = `As a ${hu.role}, I want to ${hu.goal}, so that ${hu.benefit}\n\nTitle: ${hu.title}\nAcceptance Criteria:\n${hu.acceptanceCriteria.map(ac => `- ${ac}`).join("\n")}`;
+        const skeletonText = `As a ${hu.role}, I want to ${hu.goal}, so that ${hu.benefit}\n\nTitle: ${hu.title}`;
+        return {
+          id: hu.id,
+          text: isFirst ? fullText : skeletonText,
+          blocked_by: hu.dependsOn || [],
+          needsRefinement: !isFirst
+        };
+      });
     } else {
       stories = [{ id: "HU-AUTO-001", text: session.task }];
     }

package/src/orchestrator.js

@@ -1426,7 +1426,8 @@ export async function runFlow({ task, config, logger, flags = {}, emitter = null
         runIterationFn: async (huTask) => runIterationLoop(ctx, { task: huTask, askQuestion, emitter, logger }),
         emitter,
         eventBase: ctx.eventBase,
-        logger
+        logger,
+        config: ctx.config
       });
 
       emitProgress(emitter, makeEvent("hu:sub-pipeline:end", { ...ctx.eventBase, stage: "hu-sub-pipeline" }, {

package/src/prompts/audit.js

@@ -14,7 +14,7 @@ const VALID_SCORES = new Set(["A", "B", "C", "D", "F"]);
 const VALID_SEVERITIES = new Set(["critical", "high", "medium", "low"]);
 const VALID_IMPACT = new Set(["high", "medium", "low"]);
 
-export function buildAuditPrompt({ task, instructions, dimensions = null, context = null }) {
+export function buildAuditPrompt({ task, instructions, dimensions = null, context = null, basalCost = null, growthDelta = null }) {
   const sections = [SUBAGENT_PREAMBLE];
 
   if (instructions) {
@@ -112,6 +112,40 @@ export function buildAuditPrompt({ task, instructions, dimensions = null, contex
     `Only include dimensions you were asked to analyze: ${activeDimensions.join(", ")}`
   );
 
+  if (basalCost) {
+    const lines = [
+      "## Basal Cost",
+      `- Total source lines: ${basalCost.totalLines}`,
+      `- Total source files: ${basalCost.totalFiles}`,
+      `- Dependencies: ${basalCost.dependencies?.total ?? 0} (${basalCost.dependencies?.dependencies ?? 0} prod + ${basalCost.dependencies?.devDependencies ?? 0} dev)`
+    ];
+    if (basalCost.unusedDependencies?.unused?.length > 0) {
+      lines.push(`- Unused dependencies: ${basalCost.unusedDependencies.unused.join(", ")}`);
+    } else if (basalCost.unusedDependencies?.note) {
+      lines.push(`- Unused dependencies: ${basalCost.unusedDependencies.note}`);
+    }
+    if (basalCost.deadExports?.length > 0) {
+      lines.push(`- Dead exports (exported but never imported): ${basalCost.deadExports.length}`);
+      for (const de of basalCost.deadExports.slice(0, 20)) {
+        lines.push(`  - ${de.name} in ${de.file}`);
+      }
+      if (basalCost.deadExports.length > 20) {
+        lines.push(`  - ... and ${basalCost.deadExports.length - 20} more`);
+      }
+    }
+    if (growthDelta) {
+      lines.push("");
+      lines.push("### Growth since last audit");
+      lines.push(`- Lines: ${growthDelta.lines >= 0 ? "+" : ""}${growthDelta.lines}`);
+      lines.push(`- Files: ${growthDelta.files >= 0 ? "+" : ""}${growthDelta.files}`);
+      lines.push(`- Dependencies: ${growthDelta.deps >= 0 ? "+" : ""}${growthDelta.deps}`);
+      if (growthDelta.since) lines.push(`- Since: ${growthDelta.since}`);
+    }
+    lines.push("");
+    lines.push("Flag if basal cost is growing faster than feature delivery. Recommend elimination of unused code and dependencies.");
+    sections.push(lines.join("\n"));
+  }
+
   if (context) {
     sections.push(`## Context\n${context}`);
   }

package/src/roles/audit-role.js

@@ -1,6 +1,7 @@
 import { BaseRole } from "./base-role.js";
 import { createAgent as defaultCreateAgent } from "../agents/index.js";
 import { buildAuditPrompt, parseAuditOutput, AUDIT_DIMENSIONS } from "../prompts/audit.js";
+import { measureBasalCost, loadPreviousAudit, saveAuditSnapshot, computeGrowthDelta } from "../audit/basal-cost.js";
 
 function resolveProvider(config) {
   return (
@@ -57,7 +58,18 @@ export class AuditRole extends BaseRole {
     const provider = resolveProvider(this.config);
     const agent = this._createAgent(provider, this.config, this.logger);
 
-    const prompt = buildAuditPrompt({ task, instructions: this.instructions, dimensions, context });
+    const projectDir = this.config?.projectDir || process.cwd();
+    let basalCost = null;
+    let growthDelta = null;
+    try {
+      basalCost = await measureBasalCost(projectDir);
+      const previous = loadPreviousAudit(projectDir);
+      growthDelta = computeGrowthDelta(basalCost, previous);
+    } catch {
+      // basal cost is best-effort, don't fail the audit
+    }
+
+    const prompt = buildAuditPrompt({ task, instructions: this.instructions, dimensions, context, basalCost, growthDelta });
     const runArgs = { prompt, role: "audit" };
     if (onOutput) runArgs.onOutput = onOutput;
     const result = await agent.runTask(runArgs);
@@ -82,12 +94,18 @@ export class AuditRole extends BaseRole {
         };
       }
 
+      if (basalCost) {
+        try { saveAuditSnapshot(projectDir, basalCost); } catch { /* best-effort */ }
+      }
+
       return {
         ok: true,
         result: {
           summary: parsed.summary,
           dimensions: parsed.dimensions,
           topRecommendations: parsed.topRecommendations,
+          basalCost: basalCost || undefined,
+          growthDelta: growthDelta || undefined,
           provider
         },
         summary: buildSummary(parsed),