chainlesschain 0.162.147 → 0.162.149

package/src/commands/eval.js

@@ -0,0 +1,248 @@
+/**
+ * `cc eval` — reliability eval harness (Phase 7). Runs a suite of self-checking
+ * coding tasks against the real agent and reports the task-success rate.
+ *
+ * Each task gets a fresh temp workspace; the agent is invoked headlessly
+ * (`cc agent -p` in that cwd with acceptEdits so it can write files) and the
+ * task's objective check() decides pass/fail. `--dry-run` swaps in a no-op agent
+ * so the harness + report can be exercised without a model.
+ */
+
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { spawn } from "child_process";
+import { runEvalSuite } from "../lib/eval/runner.js";
+import { getSuite } from "../lib/eval/tasks.js";
+import {
+  TelemetryRecorder,
+  formatTelemetry,
+} from "../lib/telemetry/span-recorder.js";
+import { computeTrend, formatTrend } from "../lib/eval/trend.js";
+
+/** Append a run summary as one JSONL line to the history file. */
+function appendHistory(file, record) {
+  fs.appendFileSync(file, JSON.stringify(record) + "\n", "utf8");
+}
+
+/** Read a JSONL eval-history file into an array of run records (skips bad lines). */
+function readHistory(file) {
+  if (!fs.existsSync(file)) return [];
+  const out = [];
+  for (const line of fs.readFileSync(file, "utf8").split(/\r?\n/)) {
+    const t = line.trim();
+    if (!t) continue;
+    try {
+      out.push(JSON.parse(t));
+    } catch {
+      /* skip a corrupt history line rather than abort the report */
+    }
+  }
+  return out;
+}
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+// This file is src/commands/eval.js → the CLI entry is bin/chainlesschain.js.
+const BIN = path.resolve(__dirname, "..", "..", "bin", "chainlesschain.js");
+
+/**
+ * Build a runAgent that shells out to a headless `cc agent -p` in the task's
+ * workspace. acceptEdits lets it write files without an interactive prompt.
+ */
+function makeHeadlessRunAgent(opts = {}) {
+  return function runAgent({ prompt, cwd, timeoutMs }) {
+    return new Promise((resolve) => {
+      const args = [
+        BIN,
+        "agent",
+        "-p",
+        prompt,
+        "--permission-mode",
+        "acceptEdits",
+        "--output-format",
+        "text",
+      ];
+      if (opts.model) args.push("--model", opts.model);
+      if (opts.provider) args.push("--provider", opts.provider);
+      const child = spawn(process.execPath, args, {
+        cwd,
+        env: { ...process.env, CLAUDECODE: "1" },
+        windowsHide: true,
+      });
+      let out = "";
+      let err = "";
+      let settled = false;
+      const done = (result) => {
+        if (settled) return;
+        settled = true;
+        resolve(result);
+      };
+      const timer = setTimeout(() => {
+        try {
+          child.kill("SIGTERM");
+        } catch {
+          /* already gone */
+        }
+        done({
+          ok: false,
+          output: out,
+          error: `timed out after ${timeoutMs || 120000}ms`,
+        });
+      }, timeoutMs || 120000);
+      if (typeof timer.unref === "function") timer.unref();
+      child.stdout?.on("data", (d) => (out += d.toString("utf8")));
+      child.stderr?.on("data", (d) => (err += d.toString("utf8")));
+      child.on("error", (e) => {
+        clearTimeout(timer);
+        done({ ok: false, output: out, error: e.message });
+      });
+      child.on("close", (code) => {
+        clearTimeout(timer);
+        done({ ok: code === 0, output: out, error: code === 0 ? null : err });
+      });
+    });
+  };
+}
+
+export function registerEvalCommand(program, { logger } = {}) {
+  const log = logger || console;
+  program
+    .command("eval")
+    .description(
+      "Run the reliability eval suite (self-checking coding tasks) and report the task-success rate",
+    )
+    .option("--suite <name>", "Suite to run", "builtin")
+    .option("--model <model>", "Model for the agent runs")
+    .option("--provider <provider>", "Provider for the agent runs")
+    .option("--json", "Output the summary as JSON")
+    .option("--keep", "Keep the per-task temp workspaces (debug)")
+    .option(
+      "--otlp <file>",
+      "Write OpenTelemetry (OTLP/JSON) spans for the run to a file",
+    )
+    .option(
+      "--dry-run",
+      "Use a no-op agent (exercise the harness/report without a model)",
+    )
+    .option(
+      "--history <file>",
+      "Append this run's summary (JSONL) for trend tracking",
+    )
+    .option(
+      "--label <label>",
+      "Tag this run in the history (e.g. a version/commit)",
+    )
+    .option(
+      "--trend",
+      "Report the pass-rate trend from --history instead of running (CI gate)",
+    )
+    .option(
+      "--regression-threshold <pct>",
+      "With --trend: pass-rate drop (in points) that fails the gate on its own",
+      "0",
+    )
+    .action(async (options) => {
+      // Trend mode: don't run the suite — read the recorded history and report
+      // the pass-rate trend + per-task regressions. This is the release-pipeline
+      // consumer (the runs themselves need a real model; the report is pure).
+      if (options.trend) {
+        if (!options.history) {
+          (log.error || console.error)("--trend requires --history <file>");
+          process.exitCode = 1;
+          return;
+        }
+        const runs = readHistory(options.history);
+        const threshold =
+          Math.max(0, Number(options.regressionThreshold) || 0) / 100;
+        const trend = computeTrend(runs, { regressionThreshold: threshold });
+        if (options.json) console.log(JSON.stringify(trend, null, 2));
+        else (log.log || console.log)(formatTrend(trend));
+        if (trend.regressed) process.exitCode = 1;
+        return;
+      }
+
+      let tasks;
+      try {
+        tasks = getSuite(options.suite);
+      } catch (err) {
+        log.error ? log.error(err.message) : console.error(err.message);
+        process.exitCode = 1;
+        return;
+      }
+      const runAgent = options.dryRun
+        ? async () => ({ ok: true, output: "(dry-run: no agent)" })
+        : makeHeadlessRunAgent({
+            model: options.model,
+            provider: options.provider,
+          });
+
+      // OTel-shaped telemetry for the run (per-task span + failure class).
+      const recorder = new TelemetryRecorder({ serviceName: "cc-eval" });
+
+      const summary = await runEvalSuite(tasks, {
+        runAgent,
+        recorder,
+        keepWorkspaces: options.keep === true,
+        onResult: options.json
+          ? undefined
+          : (r) =>
+              (log.info || console.log)(
+                `  ${r.pass ? "✔" : "✗"} ${r.id} (${r.ms}ms)` +
+                  (r.error
+                    ? ` — ${r.error}`
+                    : r.detail
+                      ? ` — ${r.detail}`
+                      : ""),
+              ),
+      });
+
+      if (options.otlp) {
+        try {
+          fs.writeFileSync(
+            options.otlp,
+            JSON.stringify(recorder.toOtlp(), null, 2),
+            "utf8",
+          );
+        } catch (e) {
+          (log.error || console.error)(`  otlp write failed: ${e.message}`);
+        }
+      }
+
+      // Append to the trend history (a compact record — pass counts + per-task
+      // pass/fail + a timestamp/label — so `cc eval --trend` can chart it).
+      if (options.history) {
+        try {
+          appendHistory(options.history, {
+            ranAt: new Date().toISOString(),
+            label: options.label || null,
+            passed: summary.passed,
+            total: summary.total,
+            passRate: summary.passRate,
+            results: (summary.results || []).map((r) => ({
+              id: r.id,
+              pass: r.pass,
+            })),
+          });
+        } catch (e) {
+          (log.error || console.error)(`  history write failed: ${e.message}`);
+        }
+      }
+
+      if (options.json) {
+        console.log(JSON.stringify(summary, null, 2));
+      } else {
+        // Per-task lines already streamed via onResult; print the summary +
+        // the telemetry metrics (durations / failure classification).
+        const pct = (summary.passRate * 100).toFixed(1);
+        (log.log || console.log)(
+          `\nEval: ${summary.passed}/${summary.total} passed (${pct}%) in ${summary.ms}ms`,
+        );
+        if (summary.telemetry) {
+          (log.log || console.log)("\n" + formatTelemetry(summary.telemetry));
+        }
+      }
+      // Non-zero exit when not every task passed — usable as a CI gate.
+      if (summary.passed < summary.total) process.exitCode = 1;
+    });
+  return program;
+}

package/src/commands/plugin.js

@@ -495,6 +495,360 @@ export function registerPluginCommand(program) {
         process.exit(1);
       }
     });
+
+  // plugin validate — parse a plugin's unified manifest and report every
+  // component it contributes (skills/agents/hooks/mcp/lsp/monitors/bin/settings),
+  // plus path-traversal / schema problems. Optional signature/hash verification
+  // reuses the real crypto in plugin-security. No DB, no install — pure inspection.
+  plugin
+    .command("validate <dir>")
+    .description("Validate a plugin manifest and list its components")
+    .option("--sha256 <hex>", "Expected SHA-256 of the manifest file")
+    .option("--signature <path>", "Detached Ed25519 signature of the manifest")
+    .option("--public-key <path>", "Public key for signature verification")
+    .option("--json", "Output as JSON")
+    .action(async (dir, options) => {
+      const { parsePluginManifest, summarizeComponents } =
+        await import("../lib/plugin-runtime/manifest.js");
+      const manifest = parsePluginManifest(dir);
+
+      // Optional integrity/signature check on the manifest file itself.
+      // verifyPluginManifest THROWS on any mismatch/failure and returns a
+      // details object on success — so a caught error means verification failed.
+      let verification = null;
+      if (manifest.manifestPath && (options.sha256 || options.signature)) {
+        try {
+          const v = verifyPluginManifest({
+            manifestFile: manifest.manifestPath,
+            expectedSha256: options.sha256,
+            signatureFile: options.signature,
+            publicKeyFile: options.publicKey,
+          });
+          verification = {
+            ok: true,
+            sha256: v?.sha256 || null,
+            signatureVerified: v?.signatureVerified === true,
+          };
+        } catch (err) {
+          verification = { ok: false, reason: err.message };
+          manifest.ok = false;
+          manifest.errors.push(`manifest verification failed: ${err.message}`);
+        }
+      }
+
+      const counts = summarizeComponents(manifest);
+
+      if (options.json) {
+        console.log(
+          JSON.stringify(
+            { ...manifest, componentCounts: counts, verification },
+            null,
+            2,
+          ),
+        );
+        if (!manifest.ok) process.exitCode = 1;
+        return;
+      }
+
+      const m = manifest.metadata || {};
+      logger.log(
+        chalk.bold(`Plugin: `) +
+          `${chalk.cyan(m.name || "(no name)")} ${chalk.gray("v" + (m.version || "?"))}`,
+      );
+      if (m.description) logger.log(chalk.gray("  " + m.description));
+      logger.log(
+        chalk.gray(`  manifest: ${manifest.manifestPath || "(none)"}`),
+      );
+      logger.log(chalk.bold("\nComponents:"));
+      for (const [kind, n] of Object.entries(counts)) {
+        const mark = n > 0 ? chalk.green(String(n)) : chalk.gray("0");
+        logger.log(`  ${kind.padEnd(10)} ${mark}`);
+      }
+      if (verification) {
+        logger.log(
+          chalk.bold("\nVerification: ") +
+            (verification.ok ? chalk.green("passed") : chalk.red("FAILED")),
+        );
+      }
+      for (const w of manifest.warnings) logger.log(chalk.yellow(`  ⚠ ${w}`));
+      for (const e of manifest.errors) logger.log(chalk.red(`  ✖ ${e}`));
+      if (manifest.ok) {
+        logger.success("\nManifest is valid.");
+      } else {
+        logger.error("\nManifest is INVALID.");
+        process.exitCode = 1;
+      }
+    });
+
+  // ── unified plugin runtime install lifecycle (Phase 3) ──
+  // These operate on the scope version dirs (user/project/local), independent
+  // of the legacy DB-backed `install/list/remove` above. A plugin installed
+  // here has its skills/hooks/lsp components picked up by the agent.
+
+  const SCOPES = "user|project|local";
+
+  // plugin add <source> — install from a local dir, git URL, or owner/repo
+  plugin
+    .command("add <source>")
+    .description(
+      `Install a plugin from a local dir, git URL, or owner/repo[#ref] (scope: ${SCOPES})`,
+    )
+    .option("--scope <scope>", "Install scope (user|project|local)", "user")
+    .option("--force", "Reinstall over an existing immutable version")
+    .option("--sha256 <hex>", "Expected SHA-256 of the manifest file")
+    .option("--signature <path>", "Detached Ed25519 signature of the manifest")
+    .option("--public-key <path>", "Public key for signature verification")
+    .option("--json", "Output as JSON")
+    .action(async (source, options) => {
+      const { installFromSource } =
+        await import("../lib/plugin-runtime/install.js");
+      const signature =
+        options.sha256 || options.signature || options.publicKey
+          ? {
+              sha256: options.sha256,
+              signatureFile: options.signature,
+              publicKeyFile: options.publicKey,
+              requireSignature: Boolean(options.signature),
+            }
+          : null;
+      try {
+        const res = installFromSource(source, {
+          scope: options.scope,
+          cwd: process.cwd(),
+          force: options.force === true,
+          signature,
+        });
+        if (options.json) {
+          console.log(JSON.stringify(res, null, 2));
+        } else {
+          logger.success(
+            `Installed ${res.name} v${res.version} (${res.scope} scope)` +
+              (res.signatureVerified ? chalk.green(" ✔ signed") : ""),
+          );
+          logger.log(chalk.gray(`  → ${res.dir}`));
+          for (const w of res.warnings || [])
+            logger.log(chalk.yellow(`  ⚠ ${w}`));
+        }
+      } catch (err) {
+        logger.error(`Install failed: ${err.message}`);
+        process.exitCode = 1;
+      }
+    });
+
+  // plugin installed — list runtime-installed plugins across scopes
+  plugin
+    .command("installed")
+    .description("List plugins installed in the unified runtime (scope dirs)")
+    .option("--json", "Output as JSON")
+    .action(async (options) => {
+      const { listInstalled } =
+        await import("../lib/plugin-runtime/install.js");
+      const rows = listInstalled({ cwd: process.cwd() });
+      if (options.json) {
+        console.log(JSON.stringify(rows, null, 2));
+        return;
+      }
+      if (rows.length === 0) {
+        logger.info("No plugins installed. Add one with: cc plugin add <dir>");
+        return;
+      }
+      const { isPluginTrusted } =
+        await import("../lib/plugin-runtime/trust.js");
+      logger.log(chalk.bold(`Installed plugins (${rows.length}):`));
+      for (const r of rows) {
+        const ok = r.ok ? chalk.green("✔") : chalk.red("✖");
+        const trust = isPluginTrusted(r)
+          ? chalk.green("trusted")
+          : chalk.yellow("untrusted");
+        logger.log(
+          `  ${ok} ${chalk.cyan(r.name)} v${r.version} ${chalk.gray(`[${r.scope}]`)} ${trust}`,
+        );
+      }
+    });
+
+  // plugin trust <name> — allow a plugin's code-bearing components to run
+  plugin
+    .command("trust <name>")
+    .description("Trust a plugin so its hooks / LSP servers may run")
+    .option("--scope <scope>", "Scope of the plugin", "project")
+    .option("--list", "List all trusted plugins instead")
+    .action(async (name, options) => {
+      const { trustPlugin, listTrust } =
+        await import("../lib/plugin-runtime/trust.js");
+      const { getActiveVersion } =
+        await import("../lib/plugin-runtime/install.js");
+      if (options.list) {
+        for (const t of listTrust()) {
+          logger.log(`  ${chalk.cyan(t.name)} v${t.version} [${t.scope}]`);
+        }
+        return;
+      }
+      const version = getActiveVersion(name, {
+        scope: options.scope,
+        cwd: process.cwd(),
+      });
+      if (!version) {
+        logger.error(`${name} is not installed at ${options.scope} scope`);
+        process.exitCode = 1;
+        return;
+      }
+      trustPlugin(name, { scope: options.scope, version });
+      logger.success(`Trusted ${name} v${version} (${options.scope} scope)`);
+    });
+
+  // plugin untrust <name> — revoke trust
+  plugin
+    .command("untrust <name>")
+    .description(
+      "Revoke trust for a plugin (its hooks / LSP servers stop running)",
+    )
+    .option("--scope <scope>", "Scope of the plugin", "project")
+    .action(async (name, options) => {
+      const { untrustPlugin } = await import("../lib/plugin-runtime/trust.js");
+      const res = untrustPlugin(name, { scope: options.scope });
+      if (res.removed) {
+        logger.success(`Revoked trust for ${name} (${options.scope} scope)`);
+      } else {
+        logger.info(`${name} was not trusted at ${options.scope} scope`);
+      }
+    });
+
+  // plugin monitors — list (and optionally run) trusted plugins' background
+  // monitors. `--run --seconds N` actually starts the supervisor for N seconds,
+  // prints captured output, then reaps everything (verifies no leaked process).
+  plugin
+    .command("monitors")
+    .description("List installed plugins' background monitors (trusted only)")
+    .option("--json", "Output as JSON")
+    .option("--run", "Actually run the monitors for a few seconds, then reap")
+    .option("--seconds <n>", "With --run: how long to run", "3")
+    .action(async (options) => {
+      const { collectPluginMonitors } =
+        await import("../lib/plugin-runtime/monitors.js");
+      const monitors = collectPluginMonitors({ cwd: process.cwd() });
+      if (options.json && !options.run) {
+        console.log(JSON.stringify(monitors, null, 2));
+        return;
+      }
+      if (monitors.length === 0) {
+        logger.info(
+          "No monitors from trusted plugins. (Untrusted project plugins are skipped — `cc plugin trust <name>`.)",
+        );
+        return;
+      }
+      logger.log(chalk.bold(`Plugin monitors (${monitors.length}):`));
+      for (const m of monitors) {
+        const cadence =
+          m.mode === "interval"
+            ? `every ${m.intervalMs || 60000}ms`
+            : "long-running";
+        logger.log(
+          `  ${chalk.cyan(m.id)} ${chalk.gray(`[${m.scope}]`)} ${m.command} ${m.args.join(" ")} ${chalk.gray(`(${cadence})`)}`,
+        );
+      }
+      if (!options.run) return;
+
+      const { PluginMonitorSupervisor } =
+        await import("../lib/plugin-monitor-supervisor.js");
+      const secs = Math.max(1, parseInt(options.seconds, 10) || 3);
+      const sup = new PluginMonitorSupervisor();
+      const started = sup.start(monitors);
+      logger.log(
+        chalk.gray(`\nRunning ${started.length} monitor(s) for ${secs}s…`),
+      );
+      await new Promise((r) => setTimeout(r, secs * 1000));
+      const out = sup.drainOutputs();
+      sup.stopAll();
+      logger.log(chalk.bold(`Captured ${out.length} output line(s):`));
+      for (const rec of out.slice(0, 50)) {
+        logger.log(
+          `  ${chalk.gray(`[${rec.monitor}/${rec.stream}]`)} ${rec.line}`,
+        );
+      }
+      logger.success("Monitors reaped — no process left running.");
+    });
+
+  // plugin uninstall <name> — remove a plugin (or one version) from a scope
+  plugin
+    .command("uninstall <name>")
+    .description(`Uninstall a runtime plugin from a scope (${SCOPES})`)
+    .option("--scope <scope>", "Scope to remove from", "user")
+    .option("--version <version>", "Remove only this version (default: all)")
+    .action(async (name, options) => {
+      const { uninstall } = await import("../lib/plugin-runtime/install.js");
+      try {
+        const res = uninstall(name, {
+          scope: options.scope,
+          cwd: process.cwd(),
+          version: options.version,
+        });
+        logger.success(
+          `Uninstalled ${name} (${res.removed.join(", ") || "nothing"}) from ${options.scope} scope`,
+        );
+      } catch (err) {
+        logger.error(`Uninstall failed: ${err.message}`);
+        process.exitCode = 1;
+      }
+    });
+
+  // plugin upgrade <source> — re-fetch a source and install its newer version
+  plugin
+    .command("upgrade <source>")
+    .description(
+      "Update a runtime plugin from its source (local dir or git); repoints .active",
+    )
+    .option("--scope <scope>", `Scope to update in (${SCOPES})`, "user")
+    .option("--force", "Reinstall even if the version is unchanged")
+    .action(async (source, options) => {
+      const { updatePlugin } = await import("../lib/plugin-runtime/install.js");
+      try {
+        const res = updatePlugin(source, {
+          scope: options.scope,
+          cwd: process.cwd(),
+          force: options.force,
+        });
+        if (res.updated) {
+          logger.success(
+            `Updated ${res.name}: ${res.previousVersion ? `v${res.previousVersion} → ` : ""}v${res.version} (${options.scope} scope)`,
+          );
+        } else if (res.reinstalled) {
+          logger.success(
+            `Reinstalled ${res.name} v${res.version} (${options.scope} scope)`,
+          );
+        } else {
+          logger.info(
+            `${res.name} is already up to date at v${res.version} (use --force to reinstall)`,
+          );
+        }
+      } catch (err) {
+        logger.error(`Upgrade failed: ${err.message}`);
+        process.exitCode = 1;
+      }
+    });
+
+  // plugin use <name> <version> — pin the active version (rollback / switch)
+  plugin
+    .command("use <name> <version>")
+    .description("Pin a plugin's active version (rollback or switch)")
+    .option("--scope <scope>", "Scope", "user")
+    .action(async (name, version, options) => {
+      const { setActiveVersion } =
+        await import("../lib/plugin-runtime/install.js");
+      try {
+        setActiveVersion(name, version, {
+          scope: options.scope,
+          cwd: process.cwd(),
+        });
+        logger.success(
+          `${name} active version → v${version} (${options.scope} scope)`,
+        );
+      } catch (err) {
+        logger.error(`Failed: ${err.message}`);
+        process.exitCode = 1;
+      }
+    });
+
+  return plugin;
 }
 
 // === Iter26 V2 governance overlay ===

package/src/commands/serve.js

@@ -27,6 +27,14 @@ export function registerServeCommand(program) {
       "Allow non-localhost connections (requires --token)",
     )
     .option("--project <path>", "Default project root for sessions")
+    .option(
+      "--remote-session-relay-url <url>",
+      "Signaling relay URL for Remote Session pairing",
+    )
+    .option(
+      "--remote-session-peer-id <id>",
+      "Stable relay peer ID for this local runtime",
+    )
     .option(
       "--http-port <port>",
       "Hosted HTTP port for Phase 5 envelope SSE (disabled if unset)",
@@ -47,6 +55,8 @@ export function registerServeCommand(program) {
           project: opts.project,
           httpPort: opts.httpPort ? parseInt(opts.httpPort, 10) : null,
           bundlePath: opts.bundle || null,
+          remoteSessionRelayUrl: opts.remoteSessionRelayUrl || null,
+          remoteSessionPeerId: opts.remoteSessionPeerId || null,
         });
         await runtime.startServer();
       } catch (err) {