@jhlee0619/codexloop 0.1.0

package/scripts/lib/args.mjs

@@ -0,0 +1,202 @@
+// Lightweight CLI argument parser used by scripts/loop-companion.mjs.
+//
+// Design:
+// - Supports `--key=value`, `--key value`, `--flag`, short `-k` aliases, and `--` passthrough.
+// - Unknown long flags fall through to positionals so callers can forward them.
+// - `splitRawArgumentString` tokenizes a raw $ARGUMENTS string with quote + backslash handling.
+
+export function parseArgs(argv, config = {}) {
+  const valueOptions = new Set(config.valueOptions ?? []);
+  const booleanOptions = new Set(config.booleanOptions ?? []);
+  const repeatableOptions = new Set(config.repeatableOptions ?? []);
+  const aliasMap = config.aliasMap ?? {};
+
+  const options = {};
+  const positionals = [];
+  let passthrough = false;
+
+  const assign = (key, value) => {
+    if (repeatableOptions.has(key)) {
+      if (!Array.isArray(options[key])) {
+        options[key] = [];
+      }
+      options[key].push(value);
+    } else {
+      options[key] = value;
+    }
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const token = argv[index];
+
+    if (passthrough) {
+      positionals.push(token);
+      continue;
+    }
+
+    if (token === "--") {
+      passthrough = true;
+      continue;
+    }
+
+    if (!token.startsWith("-") || token === "-") {
+      positionals.push(token);
+      continue;
+    }
+
+    if (token.startsWith("--")) {
+      const [rawKey, inlineValue] = splitKeyValue(token.slice(2));
+      const key = aliasMap[rawKey] ?? rawKey;
+
+      if (booleanOptions.has(key)) {
+        const value = inlineValue === undefined ? true : inlineValue !== "false";
+        assign(key, value);
+        continue;
+      }
+
+      if (valueOptions.has(key) || repeatableOptions.has(key)) {
+        const nextValue = inlineValue ?? argv[index + 1];
+        if (nextValue === undefined) {
+          throw new Error(`Missing value for --${rawKey}`);
+        }
+        assign(key, nextValue);
+        if (inlineValue === undefined) {
+          index += 1;
+        }
+        continue;
+      }
+
+      positionals.push(token);
+      continue;
+    }
+
+    const shortKey = token.slice(1);
+    const key = aliasMap[shortKey] ?? shortKey;
+
+    if (booleanOptions.has(key)) {
+      assign(key, true);
+      continue;
+    }
+
+    if (valueOptions.has(key) || repeatableOptions.has(key)) {
+      const nextValue = argv[index + 1];
+      if (nextValue === undefined) {
+        throw new Error(`Missing value for -${shortKey}`);
+      }
+      assign(key, nextValue);
+      index += 1;
+      continue;
+    }
+
+    positionals.push(token);
+  }
+
+  return { options, positionals };
+}
+
+function splitKeyValue(raw) {
+  const equalsIndex = raw.indexOf("=");
+  if (equalsIndex === -1) {
+    return [raw, undefined];
+  }
+  return [raw.slice(0, equalsIndex), raw.slice(equalsIndex + 1)];
+}
+
+export function splitRawArgumentString(raw) {
+  if (raw == null) {
+    return [];
+  }
+  const input = String(raw);
+  const tokens = [];
+  let current = "";
+  let quote = null;
+  let escaping = false;
+  let started = false;
+
+  const push = () => {
+    if (started) {
+      tokens.push(current);
+      current = "";
+      started = false;
+    }
+  };
+
+  for (const character of input) {
+    if (escaping) {
+      current += character;
+      started = true;
+      escaping = false;
+      continue;
+    }
+    if (character === "\\") {
+      escaping = true;
+      continue;
+    }
+    if (quote) {
+      if (character === quote) {
+        quote = null;
+      } else {
+        current += character;
+        started = true;
+      }
+      continue;
+    }
+    if (character === "'" || character === "\"") {
+      quote = character;
+      started = true;
+      continue;
+    }
+    if (/\s/.test(character)) {
+      push();
+      continue;
+    }
+    current += character;
+    started = true;
+  }
+
+  if (escaping) {
+    current += "\\";
+    started = true;
+  }
+  push();
+  return tokens;
+}
+
+export function normalizeArgv(argv) {
+  if (argv.length === 1) {
+    const [raw] = argv;
+    if (!raw || !raw.trim()) {
+      return [];
+    }
+    return splitRawArgumentString(raw);
+  }
+  return argv;
+}
+
+export function parseDuration(value) {
+  if (value == null) return null;
+  if (typeof value === "number") return value;
+  const match = String(value).trim().match(/^(\d+(?:\.\d+)?)(ms|s|m|h|d)?$/i);
+  if (!match) {
+    throw new Error(`Invalid duration: ${value}`);
+  }
+  const amount = Number(match[1]);
+  const unit = (match[2] ?? "ms").toLowerCase();
+  const scale = { ms: 1, s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 }[unit];
+  return Math.floor(amount * scale);
+}
+
+export function parseInteger(value, { name = "value", min, max } = {}) {
+  if (value == null) return null;
+  const parsed = Number.parseInt(String(value), 10);
+  if (!Number.isFinite(parsed)) {
+    throw new Error(`Invalid integer for ${name}: ${value}`);
+  }
+  if (min != null && parsed < min) {
+    throw new Error(`${name} must be >= ${min} (got ${parsed})`);
+  }
+  if (max != null && parsed > max) {
+    throw new Error(`${name} must be <= ${max} (got ${parsed})`);
+  }
+  return parsed;
+}

package/scripts/lib/codex-exec.mjs

@@ -0,0 +1,318 @@
+// Thin wrapper around the `codex` CLI used for every step of the iteration
+// loop (evaluate / suggest / rank). Always runs in `exec --json --ephemeral`
+// mode with a JSON Schema attached via `--output-schema`, and extracts the
+// structured response from `--output-last-message` for reliability.
+//
+// The binary is resolved via `resolveCodexBin()` which honors
+// `CODEXLOOP_CODEX_BIN` — this is how tests swap in the mock fixture without
+// changing runtime code.
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import process from "node:process";
+
+import { runCommand, sleep } from "./process.mjs";
+import { getSchemaPath, renderPrompt } from "./prompts.mjs";
+
+const DEFAULT_CODEX_BIN = "codex";
+
+const STEP_SCHEMAS = {
+  evaluate: "evaluation",
+  suggest: "proposal",
+  rank: "ranking"
+};
+
+export function resolveCodexBin() {
+  return process.env.CODEXLOOP_CODEX_BIN || DEFAULT_CODEX_BIN;
+}
+
+export class CodexError extends Error {
+  constructor(message, { kind = "unknown", stderr = "", status = null, attempts = 1 } = {}) {
+    super(message);
+    this.name = "CodexError";
+    this.kind = kind;
+    this.stderr = stderr;
+    this.status = status;
+    this.attempts = attempts;
+  }
+}
+
+export function classifyCodexFailure(stderr, status) {
+  const text = String(stderr || "").toLowerCase();
+  if (text.includes("rate limit") || text.includes("too many requests") || text.includes("429")) {
+    return "rate-limit";
+  }
+  if (
+    text.includes("unauthorized") ||
+    text.includes("not logged in") ||
+    text.includes("not authenticated") ||
+    text.includes("auth") && text.includes("fail")
+  ) {
+    return "auth";
+  }
+  if (
+    text.includes("network") ||
+    text.includes("timed out") ||
+    text.includes("timeout") ||
+    text.includes("econn") ||
+    text.includes("etimedout") ||
+    text.includes("dns")
+  ) {
+    return "network";
+  }
+  if (text.includes("schema") || text.includes("jsonschema")) {
+    return "schema";
+  }
+  return status === 0 ? "parse" : "unknown";
+}
+
+function isTransient(kind) {
+  return kind === "rate-limit" || kind === "network";
+}
+
+function backoffMs(attempt) {
+  return Math.min(5000, 500 * 2 ** Math.max(0, attempt - 1));
+}
+
+export function checkCodexAvailable({ cwd = process.cwd() } = {}) {
+  const bin = resolveCodexBin();
+  const result = runCommand(bin, ["--version"], { cwd, timeoutMs: 5000 });
+  if (result.error && result.error.code === "ENOENT") {
+    return {
+      available: false,
+      kind: "missing",
+      detail: `${bin} not found on PATH. Install it with 'npm install -g @openai/codex' or run /codex:setup from the codex plugin.`
+    };
+  }
+  if (result.error) {
+    return { available: false, kind: "error", detail: result.error.message };
+  }
+  if (result.status !== 0) {
+    const detail = (result.stderr || result.stdout || `exit ${result.status}`).trim();
+    return { available: false, kind: "error", detail };
+  }
+  return { available: true, version: (result.stdout || result.stderr || "").trim() };
+}
+
+// Raw wrapper. Spawns `codex exec --json --output-schema … < prompt`,
+// returns the parsed structured output plus token usage and duration.
+//
+// `model` is forwarded via `--model` and `reasoningEffort` is forwarded via
+// `-c model_reasoning_effort=<value>`. Both are optional; if unset, codex
+// falls back to the defaults in ~/.codex/config.toml.
+export async function runCodex({
+  prompt,
+  schemaName,
+  cwd,
+  sandbox = "read-only",
+  model = null,
+  reasoningEffort = null,
+  timeoutMs = 180_000,
+  retries = 2,
+  maxBuffer = 64 * 1024 * 1024
+} = {}) {
+  if (typeof prompt !== "string" || !prompt.trim()) {
+    throw new CodexError("runCodex requires a non-empty prompt string", { kind: "usage" });
+  }
+  if (!cwd) {
+    throw new CodexError("runCodex requires a cwd", { kind: "usage" });
+  }
+
+  const bin = resolveCodexBin();
+  const schemaPath = schemaName ? getSchemaPath(schemaName) : null;
+  const lastMessageFile = path.join(
+    os.tmpdir(),
+    `codexloop-last-${process.pid}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}.txt`
+  );
+
+  const args = [
+    "exec",
+    "--json",
+    "--sandbox", sandbox,
+    "--ephemeral",
+    "--skip-git-repo-check",
+    "--cd", cwd,
+    "--color", "never",
+    "--output-last-message", lastMessageFile
+  ];
+  if (schemaPath) {
+    args.push("--output-schema", schemaPath);
+  }
+  if (model) {
+    args.push("--model", model);
+  }
+  if (reasoningEffort) {
+    args.push("-c", `model_reasoning_effort=${reasoningEffort}`);
+  }
+  args.push("-"); // prompt on stdin
+
+  const maxAttempts = Math.max(1, retries + 1);
+  let lastError = null;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    const startedAt = Date.now();
+    const result = runCommand(bin, args, {
+      cwd,
+      input: prompt,
+      stdio: ["pipe", "pipe", "pipe"],
+      timeoutMs,
+      maxBuffer
+    });
+    const durationMs = Date.now() - startedAt;
+
+    if (result.error) {
+      const kind = result.error.code === "ENOENT" ? "missing" : "spawn";
+      lastError = new CodexError(
+        `${bin} failed to spawn: ${result.error.message}`,
+        { kind, stderr: result.stderr, attempts: attempt }
+      );
+      break; // spawn errors are not retry-worthy
+    }
+
+    const events = parseJsonlEvents(result.stdout);
+    const usage = extractUsage(events);
+
+    if (result.status !== 0) {
+      const kind = classifyCodexFailure(result.stderr, result.status);
+      lastError = new CodexError(
+        `${bin} exec exited ${result.status}: ${(result.stderr || "").trim().slice(0, 500) || "(no stderr)"}`,
+        { kind, stderr: result.stderr, status: result.status, attempts: attempt }
+      );
+      if (!isTransient(kind) || attempt >= maxAttempts) {
+        break;
+      }
+      await sleep(backoffMs(attempt));
+      continue;
+    }
+
+    let rawOutput = "";
+    try {
+      if (fs.existsSync(lastMessageFile)) {
+        rawOutput = fs.readFileSync(lastMessageFile, "utf8");
+      }
+    } catch {}
+    if (!rawOutput.trim()) {
+      rawOutput = extractLastAgentMessage(events) ?? "";
+    }
+    try { fs.unlinkSync(lastMessageFile); } catch {}
+
+    if (!rawOutput.trim()) {
+      lastError = new CodexError("Codex returned no agent message", {
+        kind: "empty-output",
+        stderr: result.stderr,
+        status: 0,
+        attempts: attempt
+      });
+      if (attempt >= maxAttempts) break;
+      await sleep(backoffMs(attempt));
+      continue;
+    }
+
+    let data;
+    try {
+      data = parseJsonOutput(rawOutput);
+    } catch (err) {
+      lastError = new CodexError(`Failed to parse Codex JSON output: ${err.message}`, {
+        kind: "parse",
+        stderr: result.stderr,
+        status: 0,
+        attempts: attempt
+      });
+      if (attempt >= maxAttempts) break;
+      await sleep(backoffMs(attempt));
+      continue;
+    }
+
+    return {
+      data,
+      usage,
+      events,
+      durationMs,
+      attempts: attempt,
+      rawOutput
+    };
+  }
+
+  try { fs.unlinkSync(lastMessageFile); } catch {}
+  throw lastError ?? new CodexError("Unknown codex exec failure", { kind: "unknown" });
+}
+
+// Higher-level helper keyed by iteration step. Loads the matching prompt
+// template, renders it with `vars`, and dispatches to runCodex with the
+// right schema name.
+export async function codexCall(step, vars, opts = {}) {
+  const schemaName = STEP_SCHEMAS[step];
+  if (!schemaName) {
+    throw new CodexError(`Unknown codex step: ${step}`, { kind: "usage" });
+  }
+  const prompt = renderPrompt(step, vars);
+  return runCodex({
+    prompt,
+    schemaName,
+    ...opts
+  });
+}
+
+function parseJsonlEvents(stdout) {
+  if (!stdout) return [];
+  const events = [];
+  for (const line of stdout.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      events.push(JSON.parse(trimmed));
+    } catch {
+      // Malformed progress line from Codex — ignore.
+    }
+  }
+  return events;
+}
+
+function extractLastAgentMessage(events) {
+  for (let i = events.length - 1; i >= 0; i -= 1) {
+    const evt = events[i];
+    if (evt?.type === "item.completed" && evt.item?.type === "agent_message" && typeof evt.item.text === "string") {
+      return evt.item.text;
+    }
+  }
+  return null;
+}
+
+function extractUsage(events) {
+  for (let i = events.length - 1; i >= 0; i -= 1) {
+    const evt = events[i];
+    if (evt?.type === "turn.completed" && evt.usage) {
+      return {
+        inputTokens: evt.usage.input_tokens ?? 0,
+        cachedInputTokens: evt.usage.cached_input_tokens ?? 0,
+        outputTokens: evt.usage.output_tokens ?? 0
+      };
+    }
+  }
+  return { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 };
+}
+
+function parseJsonOutput(raw) {
+  const trimmed = String(raw).trim();
+  if (!trimmed) throw new Error("empty output");
+  try {
+    return JSON.parse(trimmed);
+  } catch {}
+
+  // Some Codex versions wrap JSON in a ```json fence; strip that.
+  const fence = /^```(?:json)?\s*([\s\S]*?)\s*```$/m.exec(trimmed);
+  if (fence) {
+    try {
+      return JSON.parse(fence[1]);
+    } catch {}
+  }
+
+  // Last resort: find the outermost {...} block.
+  const start = trimmed.indexOf("{");
+  const end = trimmed.lastIndexOf("}");
+  if (start !== -1 && end !== -1 && end > start) {
+    return JSON.parse(trimmed.slice(start, end + 1));
+  }
+  throw new Error("output is not valid JSON");
+}

package/scripts/lib/convergence.mjs

@@ -0,0 +1,153 @@
+// Per-iteration quality score + loop stopping criteria.
+//
+// Quality score q_i in [0, 1] weighted sum of six terms:
+//   testPassRate, issueReduction, winnerConfidence,
+//   typeClean, lintClean, distanceFromGoal
+//
+// Stop triggers (first match wins):
+//   1. goal-met
+//   2. regression (single drop >0.10 with validate.regression true)
+//      → divergence if previous iteration also regressed
+//   3. negligible-improvement  (|Δq| < ε for stableWindow iters AND q >= 0.75)
+//   4. plateau                 (same Δq rule, q < 0.75)
+//   5. budget-iterations
+//   6. budget-time
+//   7. budget-calls
+
+export const QUALITY_WEIGHTS = Object.freeze({
+  testPassRate: 0.30,
+  issueReduction: 0.20,
+  winnerConfidence: 0.15,
+  typeClean: 0.15,
+  lintClean: 0.10,
+  distanceFromGoal: 0.10
+});
+
+export const REGRESSION_DROP = 0.10;
+export const PLATEAU_QUALITY_THRESHOLD = 0.75;
+
+function clamp01(x) {
+  if (!Number.isFinite(x)) return 0;
+  if (x < 0) return 0;
+  if (x > 1) return 1;
+  return x;
+}
+
+export function computeQualityScore(iteration, state) {
+  const v = iteration.validate;
+  const e = iteration.evaluate;
+
+  let testPassRate = 0.5;
+  if (v && Number.isFinite(v.passingTests) && Number.isFinite(v.failingTests)) {
+    const total = v.passingTests + v.failingTests;
+    testPassRate = total > 0 ? v.passingTests / total : 1;
+  } else if (v?.passed === true) {
+    testPassRate = 1;
+  } else if (v?.passed === false) {
+    testPassRate = 0;
+  } else if (e && Number.isFinite(e.passingTests) && Number.isFinite(e.failingTests)) {
+    const total = e.passingTests + e.failingTests;
+    testPassRate = total > 0 ? e.passingTests / total : 1;
+  }
+
+  const openIssuesNow = Array.isArray(e?.openIssues) ? e.openIssues.length : 0;
+  const initial = Math.max(1, state?.openIssuesInitial ?? openIssuesNow);
+  const issueReduction = 1 - Math.min(1, openIssuesNow / initial);
+
+  const winnerConfidence = iteration.ranking?.winner?.confidence;
+  const winnerConfidenceTerm = Number.isFinite(winnerConfidence) ? winnerConfidence : 0.5;
+
+  const typeErrors = v?.typeErrors ?? e?.typeErrors ?? 0;
+  const typeClean = typeErrors === 0 ? 1 : Math.max(0, 1 - Math.min(1, typeErrors / 10));
+
+  const lintErrors = v?.lintErrors ?? e?.lintErrors ?? 0;
+  const lintClean = lintErrors === 0 ? 1 : Math.max(0, 1 - Math.min(1, lintErrors / 20));
+
+  const distance = clamp01(e?.distanceFromGoal ?? 0.5);
+  const distanceFromGoal = 1 - distance;
+
+  const terms = {
+    testPassRate: clamp01(testPassRate),
+    issueReduction: clamp01(issueReduction),
+    winnerConfidence: clamp01(winnerConfidenceTerm),
+    typeClean: clamp01(typeClean),
+    lintClean: clamp01(lintClean),
+    distanceFromGoal: clamp01(distanceFromGoal)
+  };
+
+  let total = 0;
+  for (const key of Object.keys(QUALITY_WEIGHTS)) {
+    total += QUALITY_WEIGHTS[key] * terms[key];
+  }
+  return { score: Number(total.toFixed(6)), terms };
+}
+
+export function checkStopping(state) {
+  const iterations = state.iterations ?? [];
+  const last = iterations[iterations.length - 1];
+
+  // Budget-iterations: check even if no iteration ran yet
+  if ((state.budget?.consumed?.iterations ?? 0) >= (state.budget?.maxIterations ?? Infinity)) {
+    return { shouldStop: true, reason: "budget-iterations" };
+  }
+
+  // Budget-time
+  const startedAtMs = state.budget?.consumed?.startedAtMs;
+  if (
+    startedAtMs != null &&
+    Date.now() - startedAtMs >= (state.budget?.maxElapsedMs ?? Infinity)
+  ) {
+    return { shouldStop: true, reason: "budget-time" };
+  }
+
+  // Budget-calls
+  if (
+    (state.budget?.consumed?.codexCalls ?? 0) >= (state.budget?.maxCodexCalls ?? Infinity)
+  ) {
+    return { shouldStop: true, reason: "budget-calls" };
+  }
+
+  if (!last) {
+    return { shouldStop: false };
+  }
+
+  // goal-met: trust only if validate didn't fail
+  const goalMetByVerdict =
+    last.evaluate?.verdict === "goal-met" || last.stopReason === "goal-met";
+  if (goalMetByVerdict && last.validate?.passed !== false) {
+    return { shouldStop: true, reason: "goal-met" };
+  }
+
+  // regression / divergence
+  if (
+    last.qualityDelta != null &&
+    last.qualityDelta < -REGRESSION_DROP &&
+    last.validate?.regression === true
+  ) {
+    const prev = iterations[iterations.length - 2];
+    if (prev?.validate?.regression === true) {
+      return { shouldStop: true, reason: "divergence" };
+    }
+  }
+
+  // negligible-improvement / plateau
+  const epsilon = state.convergence?.epsilon ?? 0.02;
+  const window = state.convergence?.stableWindow ?? 2;
+  if (iterations.length >= window + 1) {
+    const recent = iterations.slice(-window - 1);
+    const deltas = [];
+    for (let i = 1; i < recent.length; i += 1) {
+      const d = (recent[i].qualityScore ?? 0) - (recent[i - 1].qualityScore ?? 0);
+      deltas.push(Math.abs(d));
+    }
+    if (deltas.every((d) => d < epsilon)) {
+      const q = last.qualityScore ?? 0;
+      if (q >= PLATEAU_QUALITY_THRESHOLD) {
+        return { shouldStop: true, reason: "negligible-improvement" };
+      }
+      return { shouldStop: true, reason: "plateau" };
+    }
+  }
+
+  return { shouldStop: false };
+}