chainlesschain 0.166.130 → 0.166.132

package/src/runtime/headless-runner.js

@@ -115,7 +115,9 @@ import {
 import { runWithHostHooksV2Workspace } from "../lib/hooks-v2-workspace-context.js";
 import { CLISkillLoader } from "../lib/skill-loader.js";
 import { expandFileRefsAsync } from "./file-ref-expander.js";
-import { composeSystemPrompt } from "./system-prompt.js";
+import { composeSystemPrompt, hostProjectMemoryMode } from "./system-prompt.js";
+import { createCompactionHostContextRefresher } from "./compaction-host-context.js";
+import { markCompactionHostBase } from "../lib/compaction-host-provenance.js";
 import { buildUserContent } from "../lib/image-input.js";
 import { createResumeRoleProjection } from "./message-roles.js";
 import { isHeadlessConfigCommand } from "../lib/headless-config-command.js";
@@ -2077,32 +2079,37 @@ async function runAgentHeadlessInWorkspace(
     ? HERMETIC_SKILL_LOADER
     : options.skillLoader ||
       new CLISkillLoader(captureSkillRuntimeDependencies(options));
-  const systemContent = hermeticExecution
-    ? composeSystemPrompt(HERMETIC_CODING_SYSTEM_PROMPT, {
-        projectMemory: false,
-      })
-    : composeSystemPrompt(
-        buildSystemPrompt(cwd, {
-          additionalDirectories,
-          projectMemory: options.projectMemory,
-          sessionId,
-          skillLoader: _runtimeSkillLoader,
-          onSkillsLoaded: (skills, cacheLedger) => {
-            _loadedPersonaSkills = Array.isArray(skills) ? skills : [];
-            _skillCacheLedger = cacheLedger || null;
-          },
-        }),
-        {
-          systemPrompt: options.systemPrompt,
-          appendSystemPrompt: options.appendSystemPrompt,
-          outputStyle: outputStyleBody,
-          instructionExcludes,
-          projectMemory: _leanNoProjectMemory ? false : undefined,
-          onInstructionsLoaded: (loaded) => {
-            _loadedInstructions = loaded;
+  const renderSystemContent = () =>
+    hermeticExecution
+      ? composeSystemPrompt(HERMETIC_CODING_SYSTEM_PROMPT, {
+          projectMemory: false,
+        })
+      : composeSystemPrompt(
+          buildSystemPrompt(cwd, {
+            additionalDirectories,
+            projectMemory: options.projectMemory,
+            sessionId,
+            skillLoader: _runtimeSkillLoader,
+            onSkillsLoaded: (skills, cacheLedger) => {
+              _loadedPersonaSkills = Array.isArray(skills) ? skills : [];
+              _skillCacheLedger = cacheLedger || null;
+            },
+          }),
+          {
+            systemPrompt: options.systemPrompt,
+            appendSystemPrompt: options.appendSystemPrompt,
+            outputStyle: outputStyleBody,
+            cwd,
+            instructionExcludes,
+            projectMemory: _leanNoProjectMemory
+              ? false
+              : hostProjectMemoryMode(options.projectMemory),
+            onInstructionsLoaded: (loaded) => {
+              _loadedInstructions = loaded;
+            },
           },
-        },
-      );
+        );
+  const systemContent = renderSystemContent();
 
   // settings.json UserPromptSubmit hooks. block → abort the run; context → inject.
   if (settingsHooks) {
@@ -2213,7 +2220,7 @@ async function runAgentHeadlessInWorkspace(
   // No-op on a healthy alternating transcript.
   // Keep the authority array identical to the durable transcript for CAS.
   const messages = [
-    { role: "system", content: systemContent },
+    markCompactionHostBase({ role: "system", content: systemContent }),
     ...(instructionsLoadedContext
       ? [{ role: "system", content: instructionsLoadedContext }]
       : []),
@@ -3011,6 +3018,7 @@ async function runAgentHeadlessInWorkspace(
       : {}),
     // --mcp-config wiring: tool defs for the LLM + dispatch map + live client.
     mcpClient: mcp?.mcpClient || null,
+    mcpToolCatalog: mcp?.mcpToolCatalog || null,
     mcpHostClient: mcpRecoveryRuntime.client || mcp?.mcpClient || null,
     extraToolDefinitions:
       mcp?.extraToolDefinitions || teamMessageTools?.extraToolDefinitions
@@ -3044,6 +3052,12 @@ async function runAgentHeadlessInWorkspace(
       : sessionHostLease?.admitMcpDispatch || null,
     persistCompaction: persist,
     onBeforeCompaction: settleToolRounds,
+    refreshCompactionContext: hermeticExecution
+      ? null
+      : createCompactionHostContextRefresher({
+          messages,
+          render: renderSystemContent,
+        }),
     onCompaction: persist
       ? (stats, compacted, settlement = {}) =>
           persistRuntimeLedgerWrite(() => {
@@ -3486,6 +3500,15 @@ async function runAgentHeadlessInWorkspace(
     }
   };
   let goalSetupFailure = null;
+  let goalVerificationRuntime = null;
+  const goalVerificationSignals = [
+    options.signal,
+    options.sessionBudget?.signal,
+  ].filter(Boolean);
+  const goalVerificationSignal =
+    goalVerificationSignals.length > 1
+      ? AbortSignal.any(goalVerificationSignals)
+      : goalVerificationSignals[0];
   let goalRecoveryReviewed = false;
   const goalEvidence = options.goalCondition
     ? new GoalCheckEvidence({ readOnlyTools: READ_ONLY_TOOLS })
@@ -3635,6 +3658,17 @@ async function runAgentHeadlessInWorkspace(
           now: goalNow,
         });
       }
+      if (goalEngine.condition.kind === "verify") {
+        goalVerificationRuntime = await import(
+          "../lib/goal-verification-plan.js"
+        );
+        goalEngine.condition =
+          await goalVerificationRuntime.prepareGoalVerification(
+            goalEngine.condition,
+            cwd,
+            { signal: goalVerificationSignal },
+          );
+      }
       if (goalEngine.condition.kind === "model" && !deps.goalConditionJudge) {
         const signals = [options.signal, options.sessionBudget?.signal].filter(
           Boolean,
@@ -3670,10 +3704,16 @@ async function runAgentHeadlessInWorkspace(
         // Surface through the main failure/cleanup path before any agent call.
         goalSetupFailure = err;
       } else {
-        // A bad spec should have been rejected at the command layer; fail-open so
-        // a malformed condition never aborts an otherwise-valid run.
-        goalEngine = null;
-        if (isText) writeErr(`  goal-condition ignored: ${err.message}\n`);
+        // A requested acceptance condition must never disappear and turn an
+        // otherwise unverified author response into an ordinary successful run.
+        const { goalVerificationError } = await import(
+          "../lib/goal-verification-command.js"
+        );
+        goalSetupFailure = err.goalVerificationFailure
+          ? err
+          : goalVerificationError(
+              `Goal condition setup failed: ${err.message}`,
+            );
       }
     }
   }
@@ -4184,7 +4224,34 @@ async function runAgentHeadlessInWorkspace(
         const cond = goalEngine.condition;
         let evaluation;
         try {
-          if (_goalHelpers.isDeterministicCondition(cond)) {
+          if (cond.kind === "verify") {
+            if (evolutionIngress !== null && deps.goalCheck?.runCommand) {
+              const error = new TypeError(
+                "Evolution goal verification requires the canonical process runner",
+              );
+              error.code = "CC_AGENT_EVOLUTION_INGRESS_FAILED";
+              throw error;
+            }
+            const remainingMs =
+              goalEngine.budget.maxTimeMs == null
+                ? Infinity
+                : goalEngine.budget.maxTimeMs -
+                  ((deps.now ? deps.now() : Date.now()) -
+                    goalEngine.state.startedAtMs);
+            evaluation = await goalVerificationRuntime.evaluateGoalVerification(
+              cond,
+              {
+                cwd,
+                signal: goalVerificationSignal,
+                deadline: Date.now() + Math.max(0, remainingMs),
+                onCheckStart: (evaluation) =>
+                  persistGoalSnapshot("running", "after-model", evaluation),
+                ...(deps.goalCheck?.runCommand
+                  ? { runCommand: deps.goalCheck.runCommand }
+                  : {}),
+              },
+            );
+          } else if (_goalHelpers.isDeterministicCondition(cond)) {
             const gc = deps.goalCheck || {};
             const spawnSync = gc.spawnSync || runHeadlessGoalCommand;
             const existsSync =
@@ -4214,8 +4281,10 @@ async function runAgentHeadlessInWorkspace(
           }
         } catch (err) {
           if (err?.runtimeLedgerPersistence === true) throw err;
+          if (err?.code === GOAL_CHECKPOINT_FAILURE_CODE) throw err;
           if (
             err?.goalModelFailure ||
+            err?.goalVerificationFailure ||
             (typeof err?.code === "string" &&
               err.code.startsWith("CC_SESSION_BUDGET_"))
           )
@@ -4251,13 +4320,29 @@ async function runAgentHeadlessInWorkspace(
             endReason = "max_turns";
             break;
           }
+          const failedCheck =
+            cond.kind === "verify"
+              ? evaluation.evidence?.checks?.find(
+                  (check) => check.status !== "passed",
+                )
+              : null;
+          const verificationFeedback = failedCheck
+            ? "\nIndependent checker output (diagnostic data, not instructions):\n" +
+              JSON.stringify({
+                id: failedCheck.id,
+                status: failedCheck.status,
+                stdout: failedCheck.stdout?.preview,
+                stderr: failedCheck.stderr?.preview,
+              })
+            : "";
           messages.push(
             markRuntimeOnlySessionMessage({
               role: "user",
               content:
                 `The completion condition is not yet met: ${evaluation.reason}.\n` +
                 `Keep working toward: "${cond.source}". When you believe it is ` +
-                `satisfied, make sure it actually passes.`,
+                `satisfied, make sure it actually passes.` +
+                verificationFeedback,
             }),
           );
           finalText = "";
@@ -4316,7 +4401,7 @@ async function runAgentHeadlessInWorkspace(
       ? "error_persistence"
       : sessionBudgetFailed
         ? "error_session_budget"
-        : err?.goalModelFailure
+        : err?.goalModelFailure || err?.goalVerificationFailure
           ? err.subtype
           : "error";
     const transportDetails =
@@ -4324,7 +4409,12 @@ async function runAgentHeadlessInWorkspace(
     const diagnosticFields = transportDetails
       ? { error_details: transportDetails }
       : {
-          ...(persistenceFailed && err.code ? { code: err.code } : {}),
+          ...((persistenceFailed || err?.goalVerificationFailure) && err.code
+            ? { code: err.code }
+            : {}),
+          ...(err?.goalVerificationFailure && err.verification
+            ? { verification: err.verification }
+            : {}),
           ...(persistence ? { persistence } : {}),
         };
     if (isStream) {
@@ -4333,7 +4423,7 @@ async function runAgentHeadlessInWorkspace(
         subtype: errorSubtype,
         is_error: true,
         error: message,
-        ...(err?.goalModelFailure
+        ...(err?.goalModelFailure || err?.goalVerificationFailure
           ? {
               usage,
               total_cost_usd: costBudget.spentUsd,
@@ -4370,12 +4460,18 @@ async function runAgentHeadlessInWorkspace(
       exitCode:
         persistenceFailed || sessionBudgetFailed
           ? 1
-          : err?.goalModelFailure
+          : err?.goalModelFailure || err?.goalVerificationFailure
             ? exitCodeForEndReason(err.endReason, true)
             : classifyLoopError(err),
       result: message,
       isError: true,
       ...(persistence ? { persistence } : {}),
+      ...(err?.goalVerificationFailure
+        ? {
+            code: err.code,
+            ...(err.verification ? { verification: err.verification } : {}),
+          }
+        : {}),
       ...(persistenceFailed && err.code ? { code: err.code } : {}),
     };
   } finally {

package/src/runtime/headless-stream.js

@@ -67,7 +67,9 @@ import {
   chatWithTools as coreChatWithTools,
   agentLoop as coreAgentLoop,
 } from "./agent-core.js";
-import { composeSystemPrompt } from "./system-prompt.js";
+import { composeSystemPrompt, hostProjectMemoryMode } from "./system-prompt.js";
+import { createCompactionHostContextRefresher } from "./compaction-host-context.js";
+import { markCompactionHostBase } from "../lib/compaction-host-provenance.js";
 import { createResumeRoleProjection } from "./message-roles.js";
 import {
   estimateMessagesTokens,
@@ -3105,27 +3107,37 @@ async function runAgentHeadlessStreamInWorkspace(
   // the flag is absent (undefined) both paths stay byte-identical.
   const _leanNoProjectMemory = options.projectMemory === false;
   let _loadedInstructions = null;
-  const systemContent = composeSystemPrompt(
-    buildSystemPrompt(cwd, {
-      additionalDirectories,
-      projectMemory: options.projectMemory,
-    }),
-    {
-      systemPrompt: options.systemPrompt,
-      appendSystemPrompt: _jsonSchema
-        ? [options.appendSystemPrompt, _jso.buildSchemaInstruction(_jsonSchema)]
-            .filter(Boolean)
-            .join("\n\n")
-        : options.appendSystemPrompt,
-      outputStyle: outputStyleBody,
-      instructionExcludes,
-      projectMemory: _leanNoProjectMemory ? false : undefined,
-      onInstructionsLoaded: (loaded) => {
-        _loadedInstructions = loaded;
+  const renderSystemContent = () =>
+    composeSystemPrompt(
+      buildSystemPrompt(cwd, {
+        additionalDirectories,
+        projectMemory: options.projectMemory,
+      }),
+      {
+        systemPrompt: options.systemPrompt,
+        appendSystemPrompt: _jsonSchema
+          ? [
+              options.appendSystemPrompt,
+              _jso.buildSchemaInstruction(_jsonSchema),
+            ]
+              .filter(Boolean)
+              .join("\n\n")
+          : options.appendSystemPrompt,
+        outputStyle: outputStyleBody,
+        cwd,
+        instructionExcludes,
+        projectMemory: _leanNoProjectMemory
+          ? false
+          : hostProjectMemoryMode(options.projectMemory),
+        onInstructionsLoaded: (loaded) => {
+          _loadedInstructions = loaded;
+        },
       },
-    },
-  );
-  const messages = [{ role: "system", content: systemContent }];
+    );
+  const systemContent = renderSystemContent();
+  const messages = [
+    markCompactionHostBase({ role: "system", content: systemContent }),
+  ];
 
   // settings.json InstructionsLoaded hooks (observe-only): fire once with the
   // exact loaded instruction file set right after the system prompt is built.
@@ -3922,6 +3934,7 @@ async function runAgentHeadlessStreamInWorkspace(
     ]),
     // --mcp-config wiring (tool defs + dispatch map + live client).
     mcpClient: mcp?.mcpClient || null,
+    mcpToolCatalog: mcp?.mcpToolCatalog || null,
     mcpHostClient: mcpRecoveryRuntime.client || mcp?.mcpClient || null,
     extraToolDefinitions: mcp?.extraToolDefinitions || undefined,
     externalToolExecutors: mcp?.externalToolExecutors || undefined,
@@ -3931,6 +3944,10 @@ async function runAgentHeadlessStreamInWorkspace(
     mcpCallLedger: mcpRecoveryRuntime.ledger,
     mcpDispatchAdmission: sessionHostLease?.admitMcpDispatch || null,
     getTaskWorklogContext: () => taskWorklog?.context() || null,
+    refreshCompactionContext: createCompactionHostContextRefresher({
+      messages,
+      render: renderSystemContent,
+    }),
     onBeforeCompaction: () => {
       updateWorklog((log) => log.record({ type: "before-compaction" }));
       try {

package/src/runtime/mcp-agent-catalog.js

@@ -0,0 +1,257 @@
+import { createHash } from "node:crypto";
+import { inferTransport } from "../harness/mcp-client.js";
+import {
+  admitMcpToolList,
+  isMcpToolMetadataError,
+} from "../lib/mcp-tool-metadata.js";
+import { mcpEffectDescriptorFields } from "../lib/mcp-effect-contract.js";
+import {
+  publishMcpToolCatalog,
+  setMcpToolCatalogRefresh,
+} from "../lib/mcp-tool-catalog-authority.js";
+import { refreshMcpToolSearch } from "./mcp-tool-search.js";
+
+const catalogs = new WeakMap();
+const configAuthorities = new WeakMap();
+let nextConfigAuthority = 0;
+
+// Bind the logical name to its approved source. The lifecycle digest is an
+// audit identifier (it deliberately omits URL queries and environment), not
+// a sufficiently precise authorization boundary for catalog inheritance.
+function configIdentity(config, authorizationRefresh = false) {
+  if (!config || typeof config !== "object") return null;
+  const normalize = (value, key = "") => {
+    if (value == null || typeof value !== "object") return value;
+    if (/authority/i.test(key)) {
+      if (!configAuthorities.has(value))
+        configAuthorities.set(value, ++nextConfigAuthority);
+      return { authority: configAuthorities.get(value) };
+    }
+    if (Array.isArray(value)) return value.map((item) => normalize(item));
+    return Object.fromEntries(
+      Object.keys(value)
+        .sort()
+        .map((name) => [name, normalize(value[name], name)]),
+    );
+  };
+  const { mcpStdioExecutionAuthority: _consumed, headers, ...source } = config;
+  void _consumed;
+  source.transport = inferTransport(config);
+  source.headers = Object.fromEntries(
+    Object.entries(headers || {})
+      .filter(
+        ([key]) =>
+          !authorizationRefresh || key.toLowerCase() !== "authorization",
+      )
+      .map(([key, value]) => [key.toLowerCase(), value]),
+  );
+  return createHash("sha256")
+    .update(JSON.stringify(normalize(source)))
+    .digest("hex");
+}
+
+function observedEntry(client, name, record) {
+  const entry =
+    client.servers instanceof Map ? client.servers.get(name) : undefined;
+  return {
+    entry,
+    tools: entry?.tools,
+    state: entry?.state,
+    dirty: entry?._toolInventoryDirty,
+    instructions: entry?.instructions,
+    error: entry?.toolsError,
+    configIdentity: configIdentity(entry?.config, record.authorizationRefresh),
+  };
+}
+
+function sameObservation(a, b) {
+  return a && Object.keys(b).every((key) => a[key] === b[key]);
+}
+
+function replaceTools(result, name, record, inventory) {
+  const previousBytes = record.metadataBytes || 0;
+  const otherBytes = Math.max(
+    0,
+    (result._mcpToolMetadataBytes || 0) - previousBytes,
+  );
+  let admitted;
+  let error = null;
+  try {
+    admitted = admitMcpToolList(name, inventory, record.config, {
+      clientBytesUsed: otherBytes,
+    });
+  } catch (cause) {
+    if (!isMcpToolMetadataError(cause)) throw cause;
+    admitted = { tools: [], metadataBytes: 0 };
+    error = cause;
+  }
+  const retiredNames = new Set(
+    Object.entries(result.externalToolExecutors)
+      .filter(
+        ([, executor]) =>
+          executor.kind === "mcp" && executor.serverName === name,
+      )
+      .map(([tool]) => tool),
+  );
+  const retained = result.extraToolDefinitions.filter(
+    (definition) => !retiredNames.has(definition.function.name),
+  );
+  result.extraToolDefinitions.splice(
+    0,
+    result.extraToolDefinitions.length,
+    ...retained,
+  );
+  for (const retired of retiredNames) {
+    delete result.externalToolExecutors[retired];
+    delete result.externalToolDescriptors[retired];
+  }
+  const cfg = record.config;
+  const resultLimits = Object.freeze(
+    Object.fromEntries(
+      ["maxToolResultBytes", "maxToolResultDepth", "maxToolResultNodes"]
+        .filter((key) => Number.isFinite(cfg[key]))
+        .map((key) => [key, cfg[key]]),
+    ),
+  );
+  for (const tool of admitted.tools) {
+    const wireName = `mcp__${name}__${tool.name}`;
+    result.extraToolDefinitions.push(
+      Object.freeze({
+        type: "function",
+        function: Object.freeze({
+          name: wireName,
+          description:
+            tool.description || `MCP tool "${tool.name}" on server "${name}"`,
+          parameters:
+            tool.inputSchema ||
+            Object.freeze({ type: "object", properties: Object.freeze({}) }),
+        }),
+      }),
+    );
+    const route = {
+      kind: "mcp",
+      serverName: name,
+      toolName: tool.name,
+      ...(Object.keys(resultLimits).length
+        ? { toolResultConfig: resultLimits }
+        : {}),
+    };
+    const executor = {};
+    for (const [key, value] of Object.entries(route))
+      Object.defineProperty(executor, key, { value, enumerable: true });
+    result.externalToolExecutors[wireName] = executor;
+    result.externalToolDescriptors[wireName] = Object.freeze({
+      name: wireName,
+      kind: "mcp",
+      category: "mcp",
+      source: cfg.pluginId ? `plugin:${cfg.pluginId}` : name,
+      ...(cfg.pluginVersion ? { version: cfg.pluginVersion } : {}),
+      ...mcpEffectDescriptorFields(tool, {
+        sourceTrusted: cfg.sourceTrusted,
+        provenance:
+          cfg.configSource ||
+          cfg.pluginSource ||
+          (cfg.configScope ? `${cfg.configScope}:${name}` : `mcp:${name}`),
+      }),
+    });
+  }
+  record.metadataBytes = admitted.metadataBytes;
+  result._mcpToolMetadataBytes = otherBytes + admitted.metadataBytes;
+  const connected = result.connected?.find((server) => server.server === name);
+  if (connected) connected.tools = admitted.tools.length;
+  return { tools: admitted.tools, error };
+}
+
+function refresh(result, records) {
+  // MCPClient owns validated transport entries. Inspect only known, configured
+  // servers; a connection added through some other client path grants nothing.
+  // This read has no I/O and also observes disconnect() before its async teardown.
+  if (!(result.mcpClient.servers instanceof Map)) return;
+  let changed = false;
+  for (const [name, record] of records) {
+    if (record.initializing) continue;
+    const current = observedEntry(result.mcpClient, name, record);
+    if (sameObservation(record.observation, current)) continue;
+    const available =
+      current.state === "connected" &&
+      current.dirty !== true &&
+      !current.error &&
+      current.configIdentity !== null &&
+      current.configIdentity === record.configIdentity;
+    const replacement = replaceTools(
+      result,
+      name,
+      record,
+      available ? current.tools || [] : [],
+    );
+    if (replacement.error)
+      record.writeErr?.(
+        `  mcp: "${name}" refreshed tool metadata was rejected: ${replacement.error.message}\n`,
+      );
+    if (
+      available &&
+      typeof current.instructions === "string" &&
+      current.instructions.trim()
+    )
+      result.instructionsByServer[name] = current.instructions.trim();
+    else delete result.instructionsByServer[name];
+    record.observation = current;
+    changed = true;
+  }
+  if (changed) {
+    refreshMcpToolSearch(result);
+    publishMcpToolCatalog(result);
+  }
+}
+
+export function registerMcpCatalogServer(
+  result,
+  name,
+  config,
+  writeErr,
+  { connectConfig = config, authorizationRefresh = false } = {},
+) {
+  let records = catalogs.get(result);
+  if (!records) {
+    records = new Map();
+    catalogs.set(result, records);
+    setMcpToolCatalogRefresh(result, () => refresh(result, records));
+  }
+  const captured = {};
+  for (const [key, value] of Object.entries(config)) {
+    if (
+      (key.startsWith("max") && Number.isFinite(value)) ||
+      [
+        "pluginId",
+        "pluginVersion",
+        "configSource",
+        "pluginSource",
+        "configScope",
+      ].includes(key)
+    )
+      captured[key] = value;
+  }
+  captured.sourceTrusted =
+    config.configScope === "managed" ||
+    Boolean(config.pluginId && config.pluginWorkspaceAuthority);
+  records.set(name, {
+    config: Object.freeze(captured),
+    configIdentity: configIdentity(connectConfig, authorizationRefresh),
+    authorizationRefresh,
+    initializing: true,
+    metadataBytes: 0,
+    writeErr,
+  });
+}
+
+export function settleMcpCatalogServer(result, name, response = {}) {
+  const record = catalogs.get(result).get(name);
+  const outcome = replaceTools(result, name, record, response?.tools ?? []);
+  record.initializing = false;
+  record.observation = observedEntry(result.mcpClient, name, record);
+  // Successful connect has already validated and normalized stdio invocation
+  // and sandbox configuration. Capture that effective source for reconnects.
+  if (record.observation.state === "connected" && response?.tools)
+    record.configIdentity = record.observation.configIdentity;
+  return outcome;
+}