chainlesschain 0.166.46 → 0.166.48

package/src/commands/llm.js

@@ -1,6 +1,6 @@
 /**
  * LLM management commands
- * chainlesschain llm models|test|providers|add-provider|switch
+ * chainlesschain llm models|test|capabilities|providers|add-provider|switch
  */
 
 import ora from "ora";
@@ -48,6 +48,7 @@ import {
   saveLlmConnection,
 } from "../lib/llm-connection-config.js";
 import { probeLlmConnection } from "../lib/llm-connection-probe.js";
+import { resolveModelCapabilityProfile } from "../lib/model-capabilities.js";
 
 /**
  * Resolve the effective `cc llm test` target from CLI flags + persisted config.
@@ -111,9 +112,111 @@ export function resolveLlmTestTarget(
   };
 }
 
+/** Resolve a static profile without reading credentials or provider key envs. */
+export function resolveLlmCapabilities(
+  options = {},
+  config = {},
+  builtIns = BUILT_IN_PROVIDERS,
+) {
+  if (options.baseUrl !== undefined) {
+    try {
+      const endpoint = new URL(options.baseUrl);
+      if (!["http:", "https:"].includes(endpoint.protocol)) throw new Error();
+    } catch {
+      throw new TypeError("--base-url must be an absolute HTTP(S) URL");
+    }
+  }
+  const llm = config.llm || {};
+  const { provider, model, baseUrl } = resolveLlmTestTarget(
+    {
+      provider: options.provider,
+      model: options.model,
+      baseUrl: options.baseUrl,
+    },
+    {
+      llm: { provider: llm.provider, model: llm.model, baseUrl: llm.baseUrl },
+    },
+    builtIns,
+    {},
+  );
+  return resolveModelCapabilityProfile({
+    provider,
+    model,
+    baseUrl,
+    contextMemoryModelWindowTokens: options.contextWindow,
+    maxOutputTokens: options.maxOutputTokens,
+  });
+}
+
+export function formatLlmCapabilities(profile) {
+  const protocolNote =
+    profile.runtimeProtocol === "openai-responses"
+      ? "native Responses adapter selected from the static profile; runtime not verified"
+      : "transport selection is static; runtime not verified";
+  return [
+    "LLM capability profile (static; runtime not verified)",
+    `Provider: ${profile.provider}`,
+    `Model: ${profile.model}`,
+    `Context window: ${profile.contextWindowTokens} tokens (${profile.windowSource}; ${profile.windowAssumed ? "assumed/operator-declared, not runtime verified" : "static metadata, not runtime verified"})`,
+    `Runtime protocol: ${profile.runtimeProtocol}; ${protocolNote}`,
+    `Request output cap: ${profile.requestMaxOutputTokens ?? "not explicitly set"}`,
+    `Planned output reserve: ${profile.plannedOutputReserveTokens} tokens (${profile.outputReserveSource}); planning reserve is not an enforced request cap`,
+    `Advertised maximum output: ${profile.advertisedMaxOutputTokens ?? "unknown"}`,
+    "Fallback values are estimates, not a readiness or execution guarantee.",
+    ...profile.limitations.map((limitation) => `Note: ${limitation}`),
+    `Profile digest: ${profile.digest}`,
+  ].join("\n");
+}
+
 export function registerLlmCommand(program) {
   const llm = program.command("llm").description("LLM provider management");
 
+  llm
+    .command("capabilities")
+    .description(
+      "Inspect a static model capability profile without network or runtime verification",
+    )
+    .option("--provider <provider>", "LLM provider (default: configured)")
+    .option("--model <model>", "Model (default: configured/provider)")
+    .option(
+      "--base-url <url>",
+      "API base URL for static profile selection only",
+    )
+    .option(
+      "--context-window <tokens>",
+      "Operator-declared context window, not runtime verified",
+    )
+    .option(
+      "--max-output-tokens <tokens>",
+      "Explicit request output cap for this diagnostic profile",
+    )
+    .option("--json", "Output the static capability profile as JSON")
+    .action((options) => {
+      let config;
+      try {
+        // Do not resolve OS secrets or create .corrupted backups on a read-only
+        // diagnostic. Also avoid echoing JSON parser errors containing secrets.
+        config = loadConfig({ resolveSecrets: false, failIfUnavailable: true });
+      } catch {
+        logger.error(
+          "Could not read LLM configuration; repair the configuration before inspecting capabilities.",
+        );
+        process.exitCode = 1;
+        return;
+      }
+      try {
+        const profile = resolveLlmCapabilities(options, config);
+        if (options.json) {
+          console.log(JSON.stringify(profile, null, 2));
+        } else {
+          logger.log(formatLlmCapabilities(profile));
+        }
+      } catch (error) {
+        logger.error(error.message);
+        process.exitCode = 1;
+      }
+    });
+
   llm
     .command("configure")
     .description(

package/src/commands/plugin.js

@@ -1,10 +1,11 @@
 /**
  * Plugin management commands
- * chainlesschain plugin list|install|remove|enable|disable|update|info|search|registry|summary
+ * chainlesschain plugin list|install|remove|enable|disable|update|info|search|registry|summary|eval
  */
 
 import chalk from "chalk";
 import fs from "node:fs";
+import path from "node:path";
 import { logger } from "../lib/logger.js";
 import {
   enforcePluginPolicy,
@@ -1435,6 +1436,165 @@ export function registerPluginCommand(program) {
   // component it contributes (skills/agents/hooks/mcp/lsp/monitors/bin/settings),
   // plus path-traversal / schema problems. Optional signature/hash verification
   // reuses the real crypto in plugin-security. No DB, no install — pure inspection.
+  plugin
+    .command("eval <dir>")
+    .description(
+      "Run a local control/candidate effectiveness eval for a plugin",
+    )
+    .option(
+      "--suite <file>",
+      "Suite JSON inside the plugin root (default: evals/suite.json)",
+    )
+    .option(
+      "--holdout <file>",
+      "Reviewer-owned holdout suite outside the plugin payload",
+    )
+    .option("--provider <provider>", "Fixed provider for both eval arms")
+    .option("--model <model>", "Fixed model for both eval arms")
+    .option(
+      "--min-pass-rate-delta <pct>",
+      "Override the suite's minimum candidate gain in percentage points",
+    )
+    .option(
+      "--samples <n>",
+      "Run each control/candidate task 1-20 times (default: suite or 1)",
+    )
+    .option(
+      "--arm-order <order>",
+      "balanced, control-first, or candidate-first (default: balanced)",
+    )
+    .option("--json", "Output the bound report as JSON")
+    .option("--html <file>", "Write a local HTML report")
+    .option("--keep", "Keep per-task temporary workspaces")
+    .option(
+      "--dry-run",
+      "Validate and exercise both arms without a model (never passes the evidence gate)",
+    )
+    .action(async (dir, options) => {
+      try {
+        if (!options.dryRun && (!options.provider || !options.model)) {
+          throw new Error(
+            "plugin eval requires explicit --provider and --model for comparable real runs",
+          );
+        }
+        let minPassRateDelta;
+        if (options.minPassRateDelta !== undefined) {
+          const percentage = Number(options.minPassRateDelta);
+          if (
+            !Number.isFinite(percentage) ||
+            percentage < -100 ||
+            percentage > 100
+          ) {
+            throw new Error(
+              "--min-pass-rate-delta must be between -100 and 100",
+            );
+          }
+          minPassRateDelta = percentage / 100;
+        }
+        let sampleRuns;
+        if (options.samples !== undefined) {
+          sampleRuns = Number(options.samples);
+          if (
+            !Number.isSafeInteger(sampleRuns) ||
+            sampleRuns < 1 ||
+            sampleRuns > 20
+          ) {
+            throw new Error("--samples must be an integer between 1 and 20");
+          }
+        }
+        const {
+          attachPluginEvalHoldout,
+          loadPluginEvalDefinition,
+          renderPluginEvalHtml,
+          runPluginEval,
+        } = await import("../lib/eval/plugin-suite.js");
+        const { makeHeadlessRunAgent } = await import("./eval.js");
+        const definition = attachPluginEvalHoldout(
+          loadPluginEvalDefinition(dir, {
+            suiteFile: options.suite,
+          }),
+          options.holdout,
+        );
+        const dryRunAgent = async () => ({
+          ok: true,
+          output: `${JSON.stringify({
+            type: "result",
+            subtype: "success",
+            is_error: false,
+            result: "(dry-run: no agent)",
+            tool_calls: [],
+            usage: {},
+            total_cost_usd: 0,
+            num_turns: 0,
+          })}\n`,
+        });
+        const shared = {
+          provider: options.provider,
+          model: options.model,
+          ephemeral: true,
+        };
+        const controlRunAgent = options.dryRun
+          ? dryRunAgent
+          : makeHeadlessRunAgent({
+              ...shared,
+              env: { CC_PLUGINS: "0", CC_PLUGIN_SCOPES: "" },
+            });
+        const candidateRunAgent = options.dryRun
+          ? dryRunAgent
+          : makeHeadlessRunAgent({
+              ...shared,
+              env: { CC_PLUGINS: "1", CC_PLUGIN_SCOPES: "project" },
+            });
+        const report = await runPluginEval(definition, {
+          controlRunAgent,
+          candidateRunAgent,
+          provider: options.provider || null,
+          model: options.model || null,
+          dryRun: options.dryRun === true,
+          keepWorkspaces: options.keep === true,
+          ...(minPassRateDelta === undefined ? {} : { minPassRateDelta }),
+          ...(sampleRuns === undefined ? {} : { sampleRuns }),
+          ...(options.armOrder === undefined
+            ? {}
+            : { armOrder: options.armOrder }),
+          onResult: options.json
+            ? undefined
+            : ({ arm, sample, result }) =>
+                logger.info(
+                  `  ${arm.padEnd(9)} #${sample} ${result.pass ? "✔" : "✗"} ${result.id} (${result.ms}ms)`,
+                ),
+        });
+        if (options.html) {
+          fs.writeFileSync(
+            path.resolve(options.html),
+            renderPluginEvalHtml(report),
+            "utf8",
+          );
+        }
+        if (options.json) {
+          console.log(JSON.stringify(report, null, 2));
+        } else {
+          logger.log(
+            `\nPlugin eval: ${report.status} (${report.comparison.outcome}, delta ${(report.comparison.passRateDelta * 100).toFixed(1)} points)`,
+          );
+          logger.log(
+            `  Control: ${report.arms.control.effectivePassed}/${report.arms.control.total}; Candidate: ${report.arms.candidate.effectivePassed}/${report.arms.candidate.total}`,
+          );
+          logger.log(`  Payload: ${report.plugin.payloadDigest}`);
+          logger.log(`  Suite:   ${report.suite.digest}`);
+          if (report.reasons.length) {
+            logger.warn(`  Reasons: ${report.reasons.join(", ")}`);
+          }
+          if (options.html)
+            logger.info(`  HTML: ${path.resolve(options.html)}`);
+        }
+        if (!report.passed) process.exitCode = 1;
+      } catch (error) {
+        logger.error(`Plugin eval failed: ${error.message}`);
+        process.exitCode = 1;
+      }
+    });
+
   plugin
     .command("validate <dir>")
     .description("Validate a plugin manifest and list its components")

package/src/commands/sandbox.js

@@ -4,10 +4,15 @@
  */
 
 import chalk from "chalk";
-import { intArg, floatArg } from "../lib/cli-arg.js";
+import { intArg } from "../lib/cli-arg.js";
 import ora from "ora";
 import { logger } from "../lib/logger.js";
 import { bootstrap, shutdown } from "../runtime/bootstrap.js";
+import {
+  assessAgentSandboxCapabilities,
+  normalizeAgentSandboxMode,
+  probeSandboxAvailability,
+} from "../lib/agent-sandbox.js";
 import {
   createSandbox,
   acquireSandbox,
@@ -38,11 +43,125 @@ import {
   getSandboxStatsV2,
 } from "../lib/sandbox-v2.js";
 
-export function registerSandboxCommand(program) {
+function csvList(value) {
+  return String(value || "")
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter(Boolean);
+}
+
+function printCapabilityEntries(label, entries) {
+  logger.log(`${chalk.bold(label)} (${entries.length})`);
+  if (entries.length === 0) {
+    logger.log("  none");
+    return;
+  }
+  for (const entry of entries) {
+    const suffix = entry.message
+      ? ` — ${entry.message}`
+      : entry.enforcement
+        ? ` — ${entry.enforcement}`
+        : entry.entries
+          ? ` — ${entry.entries} rule(s)`
+          : "";
+    logger.log(`  ${entry.id}${suffix}`);
+  }
+}
+
+export function registerSandboxCommand(program, dependencies = {}) {
   const sandbox = program
     .command("sandbox")
     .description("Security sandbox v2 — isolated agent execution environments");
 
+  // sandbox capabilities — read-only agent-shell backend preflight
+  sandbox
+    .command("capabilities")
+    .description(
+      "Report requested, enforceable, applied, and unsupported agent-shell sandbox capabilities",
+    )
+    .option(
+      "--engine <engine>",
+      "Agent shell backend (docker|bubblewrap)",
+      "docker",
+    )
+    .option(
+      "--mode <mode>",
+      "Agent sandbox posture (off|workspace-write|strict)",
+      "workspace-write",
+    )
+    .option("--network", "Request unrestricted network access")
+    .option("--allow-read <paths>", "Comma-separated additional read paths")
+    .option("--deny-read <paths>", "Comma-separated denied read paths")
+    .option("--allow-write <paths>", "Comma-separated additional write paths")
+    .option("--deny-write <paths>", "Comma-separated denied write paths")
+    .option("--allowed-domains <domains>", "Comma-separated allowed domains")
+    .option("--denied-domains <domains>", "Comma-separated denied domains")
+    .option("--no-probe", "Do not probe backend availability")
+    .option("--json", "Output the versioned report as JSON")
+    .action((options) => {
+      try {
+        const sandboxConfig = normalizeAgentSandboxMode(
+          options.mode,
+          options.mode === "off" ? false : true,
+          {
+            cwd: process.cwd(),
+            network: options.network === true,
+            settings: {
+              engine: options.engine,
+              filesystem: {
+                allowRead: csvList(options.allowRead),
+                denyRead: csvList(options.denyRead),
+                allowWrite: csvList(options.allowWrite),
+                denyWrite: csvList(options.denyWrite),
+              },
+              network: {
+                allowedDomains: csvList(options.allowedDomains),
+                deniedDomains: csvList(options.deniedDomains),
+              },
+            },
+          },
+        );
+        const availability =
+          sandboxConfig && options.probe !== false
+            ? (
+                dependencies.probeSandboxAvailability ||
+                probeSandboxAvailability
+              )(sandboxConfig)
+            : null;
+        const report = assessAgentSandboxCapabilities(sandboxConfig, {
+          availability,
+          host: dependencies.host,
+        });
+
+        if (options.json) {
+          console.log(JSON.stringify(report, null, 2));
+        } else {
+          logger.log(chalk.bold("Agent Shell Sandbox Capabilities"));
+          logger.log(
+            `  Host:    ${report.host.platform} ${report.host.release} (${report.host.arch})`,
+          );
+          logger.log(
+            `  Backend: ${report.backend.engine || "none"} (${report.backend.availabilityChecked ? (report.backend.available ? "available" : "unavailable") : "not probed"})`,
+          );
+          logger.log(`  Status:  ${report.status}`);
+          logger.log(
+            `  Execution observed: ${report.execution.observed ? "yes" : "no"}`,
+          );
+          printCapabilityEntries("Requested", report.requested);
+          printCapabilityEntries("Enforceable", report.enforceable);
+          printCapabilityEntries("Applied", report.applied);
+          printCapabilityEntries("Unsupported", report.unsupported);
+        }
+
+        if (["unsupported", "unavailable"].includes(report.status)) {
+          process.exitCode = 2;
+        }
+      } catch (error) {
+        logger.error(`Failed: ${error.message}`);
+        process.exitCode = 2;
+      }
+    });
+
   // sandbox create <agent-id>
   sandbox
     .command("create")

package/src/commands/serve.js

@@ -5,6 +5,7 @@
 
 import { logger } from "../lib/logger.js";
 import { createAgentRuntimeFactory } from "../runtime/runtime-factory.js";
+import { captureSkillRuntimeDependencies } from "../lib/evolution/skill-runtime-revalidation.js";
 import { captureAgentSkillOutcomeIndex } from "../lib/evolution/agent-evolution-runtime-composition-brand.js";
 import { captureSkillVectorAuthority } from "../lib/skill-vector-authority.js";
 import { captureSkillRetrievalRevocationReader } from "../lib/evolution/skill-retrieval-revocation-authority.js";
@@ -184,7 +185,11 @@ export function registerServeCommand(program, dependencies = {}) {
               tlsKeyPath: opts.appServerTlsKey,
               maxConnections: parseInt(opts.maxConnections, 10),
               store,
-              kernelFactory: () => new CliAgentKernelAdapter({ cwd }),
+              kernelFactory: () =>
+                new CliAgentKernelAdapter({
+                  cwd,
+                  sessionOptions: captureSkillRuntimeDependencies(dependencies),
+                }),
               evolutionCompositionFactory,
               skillOutcomeIndex,
               skillVectorAuthority,
@@ -218,7 +223,10 @@ export function registerServeCommand(program, dependencies = {}) {
           try {
             await runStdioAppServer({
               store,
-              kernel: new CliAgentKernelAdapter({ cwd }),
+              kernel: new CliAgentKernelAdapter({
+                cwd,
+                sessionOptions: captureSkillRuntimeDependencies(dependencies),
+              }),
               evolutionCompositionFactory,
               skillOutcomeIndex,
               skillVectorAuthority,
@@ -232,6 +240,7 @@ export function registerServeCommand(program, dependencies = {}) {
           return;
         }
         const runtime = createAgentRuntimeFactory({
+          ...captureSkillRuntimeDependencies(dependencies),
           deps:
             evolutionCompositionFactory === null
               ? {}

package/src/commands/skill.js

@@ -61,6 +61,10 @@ import { CLI_PACK_DOMAINS } from "../lib/skill-packs/schema.js";
 import { registerRecordReplayCommands } from "./record-replay.js";
 import { routeSkillDescriptors } from "../lib/skill-retrieval-router.js";
 import { resolveSkillOutcomeAuthority } from "../lib/skill-outcome-authority.js";
+import {
+  assertSkillRuntimeAdmission,
+  captureSkillRuntimeDependencies,
+} from "../lib/evolution/skill-runtime-revalidation.js";
 import {
   captureSkillVectorAuthority,
   unavailableSkillVectorEvidence,
@@ -277,6 +281,8 @@ export async function runControlledSkill(options = {}) {
     loader,
     executeTool = null,
     llmOptions = null,
+    skillRuntimeAdmission = null,
+    skillRuntimeAdmissionRequired = false,
     signal = null,
   } = options;
   throwIfAborted(signal, "Skill command interrupted before discovery");
@@ -299,8 +305,24 @@ export async function runControlledSkill(options = {}) {
   }
   const executionLease = loader.acquireSkillExecution?.(skill, { signal });
   const executionSignal = executionLease?.signal || signal;
+  const selectedSkillName = skill.id || skill.dirName;
+  const runtimeAdmissionContext = {
+    llmOptions,
+    effectiveAllowedToolNames: ["read_file", "search_files", "list_dir"],
+  };
+  const assertRuntime = () =>
+    assertSkillRuntimeAdmission({
+      skill,
+      loader,
+      admission: skillRuntimeAdmission,
+      required: skillRuntimeAdmissionRequired,
+      expectedSkillName: selectedSkillName,
+      context: runtimeAdmissionContext,
+    });
   try {
+    assertRuntime();
     const executionContext = {
+      ...runtimeAdmissionContext,
       loadedBecause: "run_skill",
       bodyIncluded: false,
       signal: executionSignal,
@@ -315,6 +337,7 @@ export async function runControlledSkill(options = {}) {
       skill = loader.materializeSkill(skill, executionContext);
     }
     executionLease?.assertActive?.();
+    assertRuntime();
     if (skill.isolation !== true) {
       return {
         error:
@@ -343,6 +366,8 @@ export async function runControlledSkill(options = {}) {
         {
           cwd,
           skillLoader: loader,
+          skillRuntimeAdmission,
+          skillRuntimeAdmissionRequired,
           ...(executionSignal ? { signal: executionSignal } : {}),
           ...(llmOptions ? { llmOptions: { ...llmOptions } } : {}),
         },
@@ -369,6 +394,8 @@ export async function runControlledSkill(options = {}) {
 }
 
 export function registerSkillCommand(program, dependencies = {}) {
+  const runtimeAdmissionDependencies =
+    captureSkillRuntimeDependencies(dependencies);
   const skillVectorAuthority =
     dependencies.skillVectorAuthority == null
       ? null
@@ -641,6 +668,7 @@ export function registerSkillCommand(program, dependencies = {}) {
     .option("-y, --yes", "Authorize the current isolated skill digest")
     .action(async (name, inputParts, options) => {
       const runLoader = new CLISkillLoader({
+        ...runtimeAdmissionDependencies,
         reauthorizeSkill: createCliSkillReauthorizer({
           assumeYes: options.yes === true,
         }),
@@ -654,6 +682,7 @@ export function registerSkillCommand(program, dependencies = {}) {
           cwd: process.cwd(),
           loader: runLoader,
           llmOptions,
+          ...runtimeAdmissionDependencies,
         });
       } catch (error) {
         result = {

package/src/data/changelog.json

@@ -1,6 +1,32 @@
 {
   "source": "CHANGELOG.md",
   "releases": [
+    {
+      "productVersion": null,
+      "cliVersion": "0.166.48",
+      "cliVersions": ["0.166.48"],
+      "date": null,
+      "title": "",
+      "sections": [
+        {
+          "heading": "Added - cc CLI 0.166.48: make evaluation, recovery, and governed recall durable",
+          "body": "- Add repeated balanced sampling, confidence intervals, and reviewer-owned\n  holdout suites to `cc plugin eval` through `--samples`, `--arm-order`, and\n  `--holdout`.\n- Persist standalone Headless deferred questions across process restarts, with\n  binding checks and one-time answer consumption.\n- Add a file-identity-bound read-only index for paged background Agent status\n  queries, with authoritative full-scan fallback on cache misses or races.\n- Add host-supplied, revision/digest-bound semantic recall candidates while\n  preserving canonical lifecycle, scope, expiry, and sink enforcement.\n- Publish Context Memory Kernel `0.1.3` before the CLI so public installs\n  resolve the new governed semantic-recall contract."
+        }
+      ]
+    },
+    {
+      "productVersion": null,
+      "cliVersion": "0.166.47",
+      "cliVersions": ["0.166.47"],
+      "date": null,
+      "title": "",
+      "sections": [
+        {
+          "heading": "Added - cc CLI 0.166.47: govern Evolution runtime and scale background sessions",
+          "body": "- Add the durable Evolution ledger v2, evidence-bound Wiki maintenance, and\n  runtime revalidation so governed learning and deployment decisions retain\n  verifiable state across restarts.\n- Add native OpenAI Responses transport together with versioned model\n  capability profiles, explicit model-egress inventory, and bounded runtime\n  capability reporting.\n- Add bound plugin-evaluation and deferred-question flows for Agent sessions,\n  while keeping deployment and evidence admission fail-closed.\n- Paginate `cc background session list` so installations with large background\n  session histories can navigate results without loading the entire list."
+        }
+      ]
+    },
     {
       "productVersion": null,
       "cliVersion": "0.166.46",

package/src/gateways/ws/session-protocol.js

@@ -535,9 +535,21 @@ async function ensureSessionHandler(
   const { WebSocketInteractionAdapter } =
     await import("../../lib/interaction-adapter.js");
   const enablePhase5Envelopes = await _isPhase5EnvelopesEnabled();
+  const stateSnapshot =
+    typeof server.sessionManager.getSessionStateSnapshot === "function"
+      ? server.sessionManager.getSessionStateSnapshot(session.id)
+      : null;
   session.interaction = new WebSocketInteractionAdapter(ws, session.id, {
     enablePhase5Envelopes,
     envelopeBus: server.envelopeBus || null,
+    deferredState: stateSnapshot,
+    onDeferredQuestionChange: ({ type, payload }) => {
+      server.sessionManager?.recordSessionStateEvent?.(
+        session.id,
+        type,
+        payload,
+      );
+    },
     onPendingApprovalChange: ({ type, payload }) => {
       if (
         server.sessionManager &&
@@ -651,6 +663,7 @@ async function ensureSessionHandler(
         evolutionCompositionFactory: server.evolutionCompositionFactory,
         skillOutcomeIndex: server.skillOutcomeIndex,
         skillVectorAuthority: server.skillVectorAuthority,
+        ...server.runtimeAdmissionDependencies,
       });
       sessionHostLease = null;
       sessionBudgetRoot = null;