@karajan-family/code 4.21.0

package/src/utils/telemetry.js

@@ -0,0 +1,124 @@
+export const TELEMETRY_ENDPOINT = "https://karajan-code.web.app/api/telemetry";
+
+/**
+ * Build the exact JSON payload that would be POSTed for an event.
+ * Pure function: deterministic given (eventName, data) — exposed so
+ * `kj telemetry preview` can render the literal bytes the backend
+ * receives without performing any network call.
+ *
+ * The `data.version` field is folded into `v`; the rest of the
+ * event-specific keys ride through `...data` after the common ones,
+ * matching the order sent over the wire.
+ *
+ * @param {string} eventName
+ * @param {object} [data]
+ * @returns {object}
+ */
+export function buildTelemetryPayload(eventName, data = {}) {
+  const { version, ...rest } = data;
+  return {
+    event: eventName,
+    v: version || "unknown",
+    os: process.platform,
+    node: process.version,
+    ts: Date.now(),
+    ...rest
+  };
+}
+
+/**
+ * Φ0-H (KJC-TSK-0526): aggregate cache-hit ratio per role from the budget
+ * summary the orchestrator already builds. Returns `{ cached_pct_coder,
+ * cached_pct_reviewer, cached_pct_total }` with values rounded to one
+ * decimal, or `null` per slot when there is no measurable input (rather
+ * than `0`, so the backend can distinguish "no cache" from "no signal").
+ *
+ * Provider-agnostic by design: the BudgetTracker entries already
+ * normalise `cached_tokens` for Anthropic (`cache_read_input_tokens`),
+ * OpenAI / Codex (`prompt_tokens_details.cached_tokens`), Gemini
+ * (`cachedContentTokenCount`) and the LiteLLM-routed providers
+ * (aider, opencode), so this helper does not need to know which
+ * provider produced the run.
+ *
+ * @param {{ breakdown_by_role?: object, total_cached_tokens?: number } | null | undefined} summary
+ * @returns {{ cached_pct_coder: number|null, cached_pct_reviewer: number|null, cached_pct_total: number|null }}
+ */
+export function computeCachedPct(summary) {
+  const empty = { cached_pct_coder: null, cached_pct_reviewer: null, cached_pct_total: null };
+  if (!summary || typeof summary !== "object") return empty;
+  const ratio = (cached, tokensIn) => {
+    const c = Number(cached);
+    const t = Number(tokensIn);
+    if (!Number.isFinite(c) || !Number.isFinite(t) || t <= 0 || c < 0) return null;
+    return Math.round((c / t) * 1000) / 10;
+  };
+  const byRole = summary.breakdown_by_role || {};
+  const coder = byRole.coder || null;
+  const reviewer = byRole.reviewer || null;
+  let totalCached = 0;
+  let totalIn = 0;
+  for (const role of Object.keys(byRole)) {
+    const r = byRole[role];
+    totalCached += Number(r?.cached_tokens || 0);
+    totalIn += Number(r?.tokens_in || 0);
+  }
+  return {
+    cached_pct_coder: coder ? ratio(coder.cached_tokens, coder.tokens_in) : null,
+    cached_pct_reviewer: reviewer ? ratio(reviewer.cached_tokens, reviewer.tokens_in) : null,
+    cached_pct_total: ratio(totalCached, totalIn)
+  };
+}
+
+/**
+ * Send an anonymous telemetry event. Non-blocking, fire-and-forget.
+ * Never throws, never blocks the pipeline.
+ *
+ * @param {string} eventName - Event name (e.g. "install", "pipeline_complete", "cli_command")
+ * @param {object} data - Event-specific data
+ * @param {object} [config] - Karajan config (checked for telemetry opt-out)
+ */
+export async function sendTelemetryEvent(eventName, data, config) {
+  if (!isTelemetryEnabled(config)) return;
+
+  try {
+    const payload = buildTelemetryPayload(eventName, data);
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 3000);
+    try {
+      await fetch(TELEMETRY_ENDPOINT, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(payload),
+        signal: controller.signal
+      });
+    } finally {
+      clearTimeout(timeout);
+    }
+  } catch (err) {
+    // Audit follow-up: was a silent catch{}. Telemetry MUST stay
+    // fire-and-forget (never block the user's pipeline, never throw),
+    // but completely silent failures hide DNS/network bugs that matter
+    // when debugging the telemetry pipeline itself. Surface the error
+    // to stderr only when KJ_DEBUG=1 is explicitly set, so normal runs
+    // are still silent.
+    if (process.env.KJ_DEBUG === "1") {
+      // Stderr (not console — telemetry runs from CLI commands and is
+      // legitimate user-visible diagnostic output under KJ_DEBUG).
+      process.stderr.write(`[telemetry-debug] ${eventName}: ${err.message}\n`);
+    }
+  }
+}
+
+/**
+ * Check if telemetry is enabled (opt-in model).
+ *
+ * Returns true ONLY when the user explicitly answered "yes" in the
+ * `kj init` wizard and we persisted `telemetry: true` to their config.
+ * Anything else — undefined, null, missing key, false — disables sending.
+ *
+ * @param {object} [config] - Karajan config
+ * @returns {boolean}
+ */
+export function isTelemetryEnabled(config) {
+  return config?.telemetry === true;
+}

package/src/utils/templates-root.js

@@ -0,0 +1,61 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { isSea, getAsset } from "node:sea";
+import { getKarajanHome } from "./paths.js";
+
+// npm install / source tree: the real templates/ directory next to src/.
+const SOURCE_ROOT = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  "..",
+  "..",
+  "templates",
+);
+
+const INDEX_ASSET = "templates-index.json";
+
+/**
+ * Root directory of the built-in templates (KJC-BUG-0104).
+ *
+ * The SEA binary ships only the JS bundle — templates/ never travels as
+ * files, and `import.meta.dirname`-relative resolution lands on the
+ * binary's own directory (e.g. `$HOME/templates/skills` → ENOENT). In SEA
+ * builds the templates travel as SEA assets (see scripts/build-sea.mjs);
+ * they are extracted once per version into ~/.karajan/embedded-templates/
+ * and served from there, so every fs-based consumer keeps working.
+ */
+export function getTemplatesRoot() {
+  if (!isSeaRuntime()) return SOURCE_ROOT;
+  const index = JSON.parse(getAsset(INDEX_ASSET, "utf8"));
+  const destRoot = path.join(getKarajanHome(), "embedded-templates", index.version);
+  return extractEmbeddedTemplates({
+    destRoot,
+    files: index.files,
+    readAsset: (rel) => getAsset(`templates/${rel}`, "utf8"),
+  });
+}
+
+function isSeaRuntime() {
+  try {
+    return isSea();
+  } catch {
+    // isSea() unexpectedly threw ⇒ treat as npm / source tree, never block.
+    return false;
+  }
+}
+
+/**
+ * Extract the embedded template files into destRoot (idempotent: a
+ * .complete marker skips re-extraction). Exported for unit testing.
+ */
+export function extractEmbeddedTemplates({ destRoot, files, readAsset }) {
+  const marker = path.join(destRoot, ".complete");
+  if (fs.existsSync(marker)) return destRoot;
+  for (const rel of files) {
+    const dest = path.join(destRoot, rel);
+    fs.mkdirSync(path.dirname(dest), { recursive: true });
+    fs.writeFileSync(dest, readAsset(rel));
+  }
+  fs.writeFileSync(marker, "");
+  return destRoot;
+}

package/src/utils/tool-call-extractor.js

@@ -0,0 +1,56 @@
+// KJC-TSK-0375 PR2 — Extract structured tool calls from an agent transcript.
+//
+// The ToolCorrectnessJudgeRole (PR1) consumes a list of `{tool, args}`
+// objects. The coder writes a raw transcript (provider-dependent format) to
+// disk per iteration. This module bridges the two: parse the transcript,
+// return the ordered list of tool invocations.
+//
+// Provider-agnostic interface; Claude stream-json is the only format
+// implemented today. New providers add a branch below — keep parsers
+// pure (no I/O), so the role tests can mock transcripts trivially.
+
+const PROVIDER_PARSERS = {
+  claude: parseClaudeStreamJson,
+};
+
+/**
+ * Extract tool calls from a transcript string.
+ *
+ * @param {object} args
+ * @param {string} args.transcript - raw agent output (stdout/stderr buffer).
+ * @param {string} [args.provider] - "claude" (default). Unknown returns [].
+ * @returns {Array<{tool:string, args:object}>}
+ */
+export function extractToolCalls({ transcript, provider = "claude" } = {}) {
+  if (typeof transcript !== "string" || transcript.length === 0) return [];
+  const parser = PROVIDER_PARSERS[provider];
+  if (!parser) return [];
+  return parser(transcript);
+}
+
+// Claude emits one JSON event per line. We care about `assistant` events
+// whose `message.content` is an array containing `tool_use` blocks with
+// `{type:"tool_use", name, input}`. Lines that fail to parse, that are
+// not `assistant`, or whose content is not an array are silently skipped:
+// the transcript also carries `system`, `result`, `rate_limit_event`, etc.
+function parseClaudeStreamJson(transcript) {
+  const out = [];
+  for (const raw of transcript.split("\n")) {
+    const line = raw.trim();
+    if (!line || line[0] !== "{") continue;
+    let obj;
+    try { obj = JSON.parse(line); } catch { continue; }
+    if (obj?.type !== "assistant") continue;
+    const content = obj?.message?.content;
+    if (!Array.isArray(content)) continue;
+    for (const block of content) {
+      if (block?.type === "tool_use" && typeof block.name === "string") {
+        out.push({
+          tool: block.name,
+          args: block.input && typeof block.input === "object" ? block.input : {},
+        });
+      }
+    }
+  }
+  return out;
+}

package/src/utils/tool-governor.js

@@ -0,0 +1,116 @@
+/**
+ * tool-governor — resource governance for external tools launched by kj
+ * (KJC-TSK-0668), born from a real semgrep CPU storm: single-flight across
+ * processes (pid-stamped lockfile, stale locks stolen), jobs cap (half the
+ * CPUs), nice +10 best-effort, and hard timeouts that kill the PROCESS TREE
+ * (own process group on POSIX). In-process cousin: async-lock.js.
+ */
+
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const DEFAULT_LOCKS_DIR = path.join(os.homedir(), ".karajan", "locks");
+const NICENESS = 10;
+
+/** Parallelism cap for scanner `--jobs` flags: half the CPUs, min 1. */
+export function jobsCap(cpuCount = os.cpus().length) {
+  return Math.max(1, Math.floor(cpuCount / 2));
+}
+
+function isPidAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return err.code === "EPERM"; // alive but owned by another user
+  }
+}
+
+/** Machine-wide lock for `tool` → {release}; ELOCKTIMEOUT if the live holder outlasts `timeoutMs`. */
+export async function acquireToolLock(tool, { locksDir = DEFAULT_LOCKS_DIR, timeoutMs = 180_000, pollMs = 500 } = {}) {
+  fs.mkdirSync(locksDir, { recursive: true });
+  const lockPath = path.join(locksDir, `${tool}.lock`);
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    try {
+      // wx = atomic create-or-fail: the filesystem is the arbiter.
+      const fd = fs.openSync(lockPath, "wx");
+      fs.writeSync(fd, JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() }));
+      fs.closeSync(fd);
+      return { release: () => fs.rmSync(lockPath, { force: true }) };
+    } catch (err) {
+      if (err.code !== "EEXIST") throw err;
+      let holder = null;
+      try {
+        holder = JSON.parse(fs.readFileSync(lockPath, "utf8"));
+      } catch { /* unreadable → stale */ }
+      if (!holder || !Number.isInteger(holder.pid) || !isPidAlive(holder.pid)) {
+        fs.rmSync(lockPath, { force: true });
+        continue;
+      }
+      if (Date.now() >= deadline) {
+        const e = new Error(`${tool} is already running (pid ${holder.pid}, since ${holder.startedAt}) — gave up waiting after ${timeoutMs}ms`);
+        e.code = "ELOCKTIMEOUT";
+        throw e;
+      }
+      await new Promise((r) => setTimeout(r, pollMs));
+    }
+  }
+}
+
+/**
+ * Governed spawn: own process group (POSIX), lowered priority, timeout kills
+ * the whole tree. Rejections mirror execFile's contract (err.code /
+ * err.killed / err.stdout) so scanner collectors keep their error handling.
+ */
+export function governedExecFile(bin, args, { cwd, timeout = 0, maxBuffer = 16 * 1024 * 1024, niceness = NICENESS } = {}) {
+  return new Promise((resolve, reject) => {
+    const posix = process.platform !== "win32";
+    const child = spawn(bin, args, { cwd, detached: posix, stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    let killedByGovernor = false, timer = null;
+    const killTree = () => {
+      killedByGovernor = true;
+      try {
+        if (posix) process.kill(-child.pid, "SIGKILL");
+        else child.kill("SIGKILL");
+      } catch { /* tree already gone */ }
+    };
+    if (timeout > 0) timer = setTimeout(killTree, timeout);
+    child.on("spawn", () => {
+      try {
+        os.setPriority(child.pid, niceness);
+      } catch { /* best-effort */ }
+    });
+    const bufferBlown = () => stdout.length + stderr.length > maxBuffer;
+    child.stdout.on("data", (d) => { stdout += d; if (bufferBlown()) killTree(); });
+    child.stderr.on("data", (d) => { stderr += d; if (bufferBlown()) killTree(); });
+    child.on("error", (err) => {
+      if (timer) clearTimeout(timer);
+      reject(Object.assign(err, { stdout, stderr }));
+    });
+    child.on("close", (code) => {
+      if (timer) clearTimeout(timer);
+      if (killedByGovernor) {
+        reject(Object.assign(new Error(`${bin} killed by governor after ${timeout}ms (process tree included)`), { killed: true, stdout, stderr }));
+      } else if (code === 0) {
+        resolve({ stdout, stderr });
+      } else {
+        reject(Object.assign(new Error(`${bin} exited with code ${code}`), { code, stdout, stderr }));
+      }
+    });
+  });
+}
+
+/** Lock + governed exec + guaranteed release. `opts.bin` overrides the executable. */
+export async function runGoverned(tool, args, opts = {}) {
+  const lock = await acquireToolLock(tool, opts.lock);
+  try {
+    return await governedExecFile(opts.bin || tool, args, opts);
+  } finally {
+    lock.release();
+  }
+}

package/src/utils/tool-installer.js

@@ -0,0 +1,93 @@
+// Executor primitives for `kj install-tools` (KJC-TSK-0606).
+//
+// Two ways to run an install step:
+//  · runInstallCommand(cmd)                 — captured: stdout/stderr collected,
+//                                             for non-privileged commands.
+//  · runInstallCommand(cmd, {interactive})  — tty inherited: the child owns the
+//                                             terminal so `sudo` prompts the user
+//                                             directly. kj NEVER sees the password.
+//
+// And a downloader for tools shipped as a single static binary (osv-scanner,
+// semgrep) when no package manager is available: fetch → temp file → chmod +x →
+// atomic rename into ~/.local/bin. On any failure it leaves no partial file.
+
+import { execFile, spawn } from "node:child_process";
+import { promisify } from "node:util";
+import os from "node:os";
+import path from "node:path";
+import fs from "node:fs/promises";
+
+const execFileAsync = promisify(execFile);
+
+/** Directory where downloaded standalone binaries are placed. */
+export function binDir() {
+  return path.join(os.homedir(), ".local", "bin");
+}
+
+/**
+ * Run an install command. Commands here are simple (no nested quoting), so we
+ * tokenize on whitespace.
+ *
+ * @param {string} command
+ * @param {{ interactive?: boolean }} [opts]
+ *   interactive → inherit the parent's stdio so sudo can prompt for the password
+ *   on the user's own terminal; kj never captures or logs it.
+ */
+export async function runInstallCommand(command, opts = {}) {
+  const [bin, ...args] = command.split(/\s+/);
+  if (opts.interactive) return runInherited(bin, args);
+  try {
+    const { stdout, stderr } = await execFileAsync(bin, args, {
+      timeout: 300_000,
+      maxBuffer: 32 * 1024 * 1024,
+    });
+    return { ok: true, stdout, stderr };
+  } catch (err) {
+    return { ok: false, error: err?.shortMessage || err?.message || String(err), stderr: err?.stderr };
+  }
+}
+
+function runInherited(bin, args) {
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn(bin, args, { stdio: "inherit", timeout: 600_000 });
+    } catch (err) {
+      resolve({ ok: false, error: err?.message || String(err) });
+      return;
+    }
+    child.on("error", (err) => resolve({ ok: false, error: err?.message || String(err) }));
+    child.on("close", (code) => resolve({ ok: code === 0, code }));
+  });
+}
+
+/**
+ * Download a single static binary and place it (executable) into `dir`.
+ * Writes to a temp file first and renames atomically; on any failure the temp
+ * file is removed so no partial artifact is left behind.
+ *
+ * @param {{ url: string, name: string, dir?: string }} args
+ */
+export async function downloadBinary({ url, name, dir = binDir() }) {
+  let res;
+  try {
+    res = await fetch(url);
+  } catch (err) {
+    return { ok: false, error: `download failed: ${err?.message || String(err)}` };
+  }
+  if (!res.ok) return { ok: false, error: `download failed: HTTP ${res.status}` };
+
+  await fs.mkdir(dir, { recursive: true });
+  const dest = path.join(dir, name);
+  const tmp = path.join(dir, `.${name}.download`);
+  try {
+    const buf = Buffer.from(await res.arrayBuffer());
+    await fs.writeFile(tmp, buf);
+    await fs.chmod(tmp, 0o755);
+    await fs.rename(tmp, dest);
+    return { ok: true, dest };
+  } catch (err) {
+    await fs.rm(tmp, { force: true }).catch(() => {});
+    return { ok: false, error: `install failed: ${err?.message || String(err)}` };
+  }
+}