impel-cli 0.20.42 → 0.20.44

package/scripts/profile-native-codex.mjs

@@ -7,14 +7,22 @@ import os from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
+import {
+  MANAGED_AGENT_MCP_SERVER,
+  NATIVE_AGENT_ANSWER_TOOL,
+  nativeToolNamespace,
+  resolveManagedCodexAgentProfile,
+} from "../src/agents.js";
 import { nativeSpawnInvocation } from "../src/nativeProcess.js";
-import { summarizeRawAttempt } from "./analyze-native-codex.mjs";
+import { parseStrictJsonLines, summarizeRawAttempt } from "./analyze-native-codex.mjs";
 
 const PROFILE_SCHEMA = "impel.native-codex-profile.v1";
 const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000;
 const DEFAULT_GRACE_MS = 5_000;
 const MAX_CAPTURE_BYTES = 256 * 1024 * 1024;
+const MAX_CONFIG_BYTES = 1024 * 1024;
 const TENANT_PREFLIGHT_RETRY_DELAYS_MS = [250, 750, 1_500];
+const ISOLATED_PROFILE_PREFIX = "impel-native-codex-benchmark-";
 
 function privateDirectory(directory) {
   if (fs.existsSync(directory) && fs.lstatSync(directory).isSymbolicLink()) {
@@ -29,6 +37,16 @@ function privateWrite(filePath, contents) {
   try { fs.chmodSync(filePath, 0o600); } catch { /* Best effort on Windows. */ }
 }
 
+function privateCopy(source, destination) {
+  const stat = fs.lstatSync(source);
+  if (!stat.isFile() || stat.size > MAX_CONFIG_BYTES) {
+    throw new Error("the selected Impel config must be a bounded regular file");
+  }
+  privateDirectory(path.dirname(destination));
+  fs.copyFileSync(source, destination, fs.constants.COPYFILE_EXCL);
+  try { fs.chmodSync(destination, 0o600); } catch { /* Best effort on Windows. */ }
+}
+
 function openPrivate(filePath) {
   const descriptor = fs.openSync(filePath, "wx", 0o600);
   try { fs.chmodSync(filePath, 0o600); } catch { /* Best effort on Windows. */ }
@@ -233,6 +251,350 @@ export async function assertExpectedTenant({ expectedTenant, impelBinary = "impe
   return actual;
 }
 
+function selectedConfigPath(environment = process.env) {
+  const home = environment.HOME || environment.USERPROFILE || os.homedir();
+  const root = environment.IMPEL_CONFIG_DIR || path.join(home, ".config", "impel");
+  return path.join(root, "config.json");
+}
+
+function isolatedBenchmarkPaths(root, tenantId) {
+  const home = path.join(root, "home");
+  const configRoot = path.join(root, "config");
+  return {
+    root,
+    home,
+    configRoot,
+    codexHome: path.join(configRoot, "cli", "tenants", tenantId, "codex"),
+    nativeCodexHome: path.join(root, "native-codex"),
+    appHome: path.join(root, "apps"),
+  };
+}
+
+function isolatedBenchmarkEnvironment(options, paths, telemetryPath, baseEnvironment) {
+  return {
+    ...baseEnvironment,
+    HOME: paths.home,
+    USERPROFILE: paths.home,
+    XDG_CONFIG_HOME: path.join(paths.home, ".config"),
+    IMPEL_CONFIG_DIR: paths.configRoot,
+    IMPEL_APP_HOME: paths.appHome,
+    // `impel codex` runs against the tenant's generated profile directory. An
+    // explicit CODEX_HOME overrides that derivation, so it must point at the
+    // isolated tenant profile; pointing it anywhere else starts Codex with no
+    // MCP servers, and the cohort silently measures model prose instead of
+    // agent latency.
+    CODEX_HOME: paths.codexHome,
+    IMPEL_SKIP_UPDATE_CHECK: "1",
+    IMPEL_NATIVE_AGENT_TELEMETRY_PATH: telemetryPath,
+    IMPEL_NATIVE_HOST: options.mode === "direct"
+      ? "codex-direct-profile"
+      : "codex-compatible-agent",
+    IMPEL_NATIVE_HOST_BUILD: options.hostBuild,
+    IMPEL_NATIVE_BENCHMARK: "1",
+  };
+}
+
+function profileEnvironmentEntry(contents, name, value) {
+  return contents.includes(`${JSON.stringify(name)} = ${JSON.stringify(value)}`);
+}
+
+function regularFileContents(filePath, missingMessage) {
+  try {
+    const stat = fs.lstatSync(filePath);
+    if (!stat.isFile() || stat.isSymbolicLink()) throw new Error();
+    return fs.readFileSync(filePath, "utf8");
+  } catch {
+    throw new Error(missingMessage);
+  }
+}
+
+function tomlSection(contents, sectionName) {
+  const header = `[${sectionName}]`;
+  const lines = contents.split(/\r?\n/u);
+  const matches = lines.flatMap((line, index) => line.trim() === header ? [index] : []);
+  if (matches.length !== 1) return null;
+  const start = matches[0] + 1;
+  const relativeEnd = lines.slice(start).findIndex((line) => /^\s*\[/u.test(line));
+  return lines.slice(start, relativeEnd === -1 ? lines.length : start + relativeEnd).join("\n");
+}
+
+function tomlAssignment(section, name) {
+  if (section === null) return null;
+  const escaped = name.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+  const matches = [...section.matchAll(new RegExp(`^\\s*${escaped}\\s*=\\s*(.+?)\\s*$`, "gmu"))];
+  return matches.length === 1 ? matches[0][1] : null;
+}
+
+function tomlStringList(value) {
+  if (typeof value !== "string" || !/^\s*\[.*\]\s*$/su.test(value)) return null;
+  try {
+    const parsed = JSON.parse(value);
+    return Array.isArray(parsed) && parsed.every((entry) => typeof entry === "string")
+      ? parsed
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+function assertSelectedProfileContract(contents, environment, policyFingerprint) {
+  const mcpSectionName = `mcp_servers.${MANAGED_AGENT_MCP_SERVER}`;
+  const mcpSection = tomlSection(contents, mcpSectionName);
+  if (mcpSection === null) {
+    throw new Error(`the isolated managed Codex agent profile has no [${mcpSectionName}] block`);
+  }
+  if (!tomlAssignment(mcpSection, "command") || !tomlAssignment(mcpSection, "args")) {
+    throw new Error("the isolated managed Codex agent profile has an incomplete MCP command binding");
+  }
+  if (tomlAssignment(mcpSection, "tool_timeout_sec") !== "120") {
+    throw new Error("the isolated managed Codex agent profile has no reviewed MCP tool timeout");
+  }
+  const enabledTools = tomlStringList(tomlAssignment(mcpSection, "enabled_tools"));
+  if (!enabledTools?.includes(NATIVE_AGENT_ANSWER_TOOL)) {
+    throw new Error(
+      `the isolated managed Codex agent profile does not enable ${NATIVE_AGENT_ANSWER_TOOL}`,
+    );
+  }
+  const mcpEnvironment = tomlAssignment(mcpSection, "env");
+  for (const name of [
+    "IMPEL_NATIVE_AGENT_TELEMETRY_PATH",
+    "IMPEL_NATIVE_HOST",
+    "IMPEL_NATIVE_HOST_BUILD",
+    "IMPEL_NATIVE_BENCHMARK",
+  ]) {
+    if (!profileEnvironmentEntry(mcpEnvironment || "", name, environment[name])) {
+      throw new Error(`the isolated managed Codex agent MCP env did not bake ${name}`);
+    }
+  }
+  const mcpArgs = tomlAssignment(mcpSection, "args") || "";
+  if (!mcpArgs.includes(JSON.stringify("--policy-fingerprint"))
+    || !mcpArgs.includes(JSON.stringify(policyFingerprint))) {
+    throw new Error("the isolated managed Codex agent profile lost its policy fingerprint binding");
+  }
+
+  const codeModeSection = tomlSection(contents, "features.code_mode");
+  if (tomlAssignment(codeModeSection, "enabled") !== "true") {
+    throw new Error("the isolated managed Codex agent profile does not enable reviewed code mode");
+  }
+  const namespaces = tomlStringList(
+    tomlAssignment(codeModeSection, "direct_only_tool_namespaces"),
+  );
+  if (!namespaces?.includes(nativeToolNamespace())) {
+    throw new Error(
+      `the isolated managed Codex agent profile does not bind ${nativeToolNamespace()} as a direct-only namespace`,
+    );
+  }
+}
+
+export function verifyIsolatedBenchmarkProfile({
+  codexHome,
+  expectedTenant,
+  agent,
+  environment,
+}) {
+  const manifestPath = path.join(codexHome, "agents", "impel-managed", ".manifest.json");
+  let manifest;
+  try {
+    const stat = fs.lstatSync(manifestPath);
+    if (!stat.isFile() || stat.isSymbolicLink()) throw new Error();
+    manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+  } catch {
+    throw new Error("the isolated managed Codex agent manifest was not generated");
+  }
+  if (manifest?.client !== "codex"
+    || manifest.tenantId !== expectedTenant
+    || manifest.directProfiles !== true
+    || manifest.directCodeMode !== true
+    || manifest.nativeBenchmark !== true
+    || !Array.isArray(manifest.files)
+    || !Array.isArray(manifest.profiles)
+    || !manifest.contentDigests
+    || typeof manifest.contentDigests !== "object"
+    || Array.isArray(manifest.contentDigests)
+    || !manifest.nativeAgentTelemetryEnvironment
+    || typeof manifest.nativeAgentTelemetryEnvironment !== "object"
+    || Array.isArray(manifest.nativeAgentTelemetryEnvironment)) {
+    throw new Error("the isolated managed Codex agent manifest is missing its benchmark telemetry contract");
+  }
+  for (const name of [
+    "IMPEL_NATIVE_AGENT_TELEMETRY_PATH",
+    "IMPEL_NATIVE_HOST",
+    "IMPEL_NATIVE_HOST_BUILD",
+  ]) {
+    if (manifest.nativeAgentTelemetryEnvironment[name] !== environment[name]) {
+      throw new Error(`the isolated managed Codex agent manifest did not retain ${name}`);
+    }
+  }
+  const matches = (Array.isArray(manifest.agents) ? manifest.agents : []).filter((record) =>
+    record
+    && !record.retired
+    && [record.agentId, record.title, record.profileName].includes(agent)
+  );
+  if (matches.length !== 1) {
+    throw new Error(`the isolated managed Codex agent ${JSON.stringify(agent)} was not generated exactly once`);
+  }
+  const [record] = matches;
+  if (!/^[a-f0-9]{64}$/u.test(record.policyFingerprint || "")) {
+    throw new Error("the isolated managed Codex agent manifest has no policy fingerprint");
+  }
+  let resolved;
+  try {
+    resolved = resolveManagedCodexAgentProfile(codexHome, expectedTenant, agent);
+  } catch (error) {
+    throw new Error(`the isolated managed Codex agent profile failed its managed integrity check: ${error.message}`);
+  }
+  const profileFileName = resolved.profileFileName;
+  if (typeof profileFileName !== "string"
+    || path.basename(profileFileName) !== profileFileName
+    || !profileFileName.endsWith(".config.toml")) {
+    throw new Error("the isolated managed Codex agent profile mapping is invalid");
+  }
+  const profilePath = path.join(codexHome, profileFileName);
+  const contents = regularFileContents(
+    profilePath,
+    "the isolated managed Codex agent profile was not written as a regular file",
+  );
+  const definitionFileName = `${resolved.profileName}.toml`;
+  if (!manifest.files.includes(definitionFileName)) {
+    throw new Error("the isolated managed Codex agent definition is absent from its manifest");
+  }
+  const definitionPath = path.join(codexHome, "agents", definitionFileName);
+  const definitionContents = regularFileContents(
+    definitionPath,
+    "the isolated managed Codex agent definition was not written as a regular file",
+  );
+  if (manifest.contentDigests?.[`agents/${definitionFileName}`]
+    !== crypto.createHash("sha256").update(fs.readFileSync(definitionPath)).digest("hex")) {
+    throw new Error("the isolated managed Codex agent definition failed its managed integrity check");
+  }
+  assertSelectedProfileContract(contents, environment, record.policyFingerprint);
+  try {
+    assertSelectedProfileContract(definitionContents, environment, record.policyFingerprint);
+  } catch (error) {
+    throw new Error(error.message.replace("agent profile", "agent definition"));
+  }
+  return { manifestPath, profilePath, profileFileName, definitionPath };
+}
+
+async function generateIsolatedBenchmarkProfile(options, isolation) {
+  // `managedSkillProfiles` only emits the "Impel isolated Codex" target when a
+  // tenant is selected AND its profile directory already exists, so a fresh
+  // isolated root would otherwise generate the native profile only — leaving
+  // Codex with no impel_agent MCP server and the cohort measuring model prose.
+  const select = impelInvocation(options.impelBinary, [
+    "tenant",
+    "use",
+    options.expectedTenant,
+  ], isolation.environment);
+  const selected = await runCaptured(select.command, select.args, {
+    environment: isolation.environment,
+  });
+  if (selected.code !== 0) {
+    throw new Error(
+      `\`impel tenant use ${options.expectedTenant}\` exited with status ${selected.code ?? "unknown"}`,
+    );
+  }
+  fs.mkdirSync(isolation.codexHome, { recursive: true, mode: 0o700 });
+
+  const invocation = impelInvocation(options.impelBinary, [
+    "agents",
+    "sync",
+    "codex",
+    "--skip-apps",
+  ], isolation.environment);
+  const result = await runCaptured(invocation.command, invocation.args, {
+    environment: isolation.environment,
+  });
+  if (result.code !== 0) {
+    throw new Error(`\`impel agents sync codex\` exited with status ${result.code ?? "unknown"}`);
+  }
+}
+
+/**
+ * Run a cohort with the CLI's existing IMPEL_CONFIG_DIR override pointed at a
+ * private temporary root. Only config.json is copied in for authentication;
+ * no managed profile from the operator's config root is read or written.
+ */
+export async function withIsolatedBenchmarkProfile(options, sessionDir, callback, {
+  baseEnvironment = process.env,
+  operatorConfigPath = selectedConfigPath(baseEnvironment),
+  prepareProfile = generateIsolatedBenchmarkProfile,
+  assertTenant = assertExpectedTenant,
+  stderr = process.stderr,
+  temporaryParent = os.tmpdir(),
+} = {}) {
+  const root = fs.mkdtempSync(path.join(temporaryParent, ISOLATED_PROFILE_PREFIX));
+  const paths = isolatedBenchmarkPaths(root, options.expectedTenant);
+  const telemetryPath = path.join(sessionDir, "telemetry.jsonl");
+  const environment = isolatedBenchmarkEnvironment(
+    options,
+    paths,
+    telemetryPath,
+    baseEnvironment,
+  );
+  let removed = false;
+  const removeRoot = () => {
+    if (removed) return;
+    removed = true;
+    fs.rmSync(root, { recursive: true, force: true });
+  };
+  // Synchronous exit cleanup also covers Ctrl-C/termination paths where the
+  // event loop cannot continue far enough to unwind the async callback.
+  process.once("exit", removeRoot);
+  let prepared = false;
+  try {
+    privateDirectory(paths.home);
+    privateDirectory(paths.configRoot);
+    stderr.write(
+      `Using isolated benchmark profile ${paths.codexHome}; cohort telemetry: ${telemetryPath}\n`,
+    );
+    try {
+      privateCopy(operatorConfigPath, path.join(paths.configRoot, "config.json"));
+      // Explicit sync discovers initialized isolated CLI roots. Creating only
+      // this empty destination lets the production sync command generate the
+      // complete agent catalog without launching Codex or touching a live
+      // profile.
+      privateDirectory(paths.codexHome);
+      await assertTenant({
+        expectedTenant: options.expectedTenant,
+        impelBinary: options.impelBinary,
+        environment,
+      });
+      await prepareProfile(options, {
+        ...paths,
+        telemetryPath,
+        environment,
+      });
+      const verified = verifyIsolatedBenchmarkProfile({
+        codexHome: paths.codexHome,
+        expectedTenant: options.expectedTenant,
+        agent: options.agent,
+        environment,
+      });
+      // Profile preparation must not contribute any events to the measured
+      // cohort. The generated profile keeps pointing at this same path.
+      fs.rmSync(telemetryPath, { force: true });
+      prepared = true;
+      return await callback({
+        ...paths,
+        ...verified,
+        telemetryPath,
+        environment,
+      });
+    } catch (error) {
+      if (prepared) throw error;
+      throw new Error(
+        `could not generate the isolated benchmark profile for tenant ${JSON.stringify(options.expectedTenant)}: `
+        + `${error?.message || error}. Verify Impel auth, network access, the selected tenant, and `
+        + "that `impel agents sync codex` can fetch this tenant's agent catalog.",
+      );
+    }
+  } finally {
+    process.off("exit", removeRoot);
+    removeRoot();
+  }
+}
+
 function listRollouts(root, startedAtMs, endedAtMs) {
   if (!fs.existsSync(root)) return [];
   const results = [];
@@ -439,35 +801,141 @@ export function safeAttemptRecord({ options, index, processResult, summary, json
   };
 }
 
+export function profiledCodexAttemptOptions(options, telemetryPath, environment = process.env) {
+  return {
+    codexArgs: [
+      "codex",
+      "--benchmark",
+      ...(options.mode === "direct" ? ["--agent", options.agent] : []),
+      "exec",
+      "--json",
+      ...options.extraArgs,
+      "-",
+    ],
+    environment: {
+      ...environment,
+      IMPEL_NATIVE_AGENT_TELEMETRY_PATH: telemetryPath,
+      IMPEL_NATIVE_HOST: options.mode === "direct" ? "codex-direct-profile" : "codex-compatible-agent",
+      IMPEL_NATIVE_HOST_BUILD: options.hostBuild,
+      IMPEL_NATIVE_BENCHMARK: "1",
+    },
+  };
+}
+
+function attemptTelemetryWindow(attempt, index) {
+  const startedAtMs = attempt?.processResult?.startedAtMs ?? Date.parse(attempt?.startedAt || "");
+  const endedAtMs = attempt?.processResult?.endedAtMs ?? Date.parse(attempt?.endedAt || "");
+  if (!Number.isFinite(startedAtMs) || !Number.isFinite(endedAtMs) || endedAtMs < startedAtMs) {
+    throw new Error(`attempt ${index + 1} has an invalid telemetry window`);
+  }
+  return { index, startedAtMs, endedAtMs };
+}
+
+function eventTimestampMs(event) {
+  const value = Date.parse(event?.timestamp || "");
+  return Number.isFinite(value) ? value : null;
+}
+
+function windowOwner(windows, timestampMs) {
+  if (timestampMs === null) return null;
+  const candidates = windows.filter(({ startedAtMs, endedAtMs }) =>
+    timestampMs >= startedAtMs && timestampMs <= endedAtMs
+  );
+  if (candidates.length === 0) return null;
+  // Concurrent attempts can overlap. Prefer the attempt whose process start is
+  // closest to the event, then use input order as a stable tie breaker.
+  return candidates.sort((left, right) =>
+    (timestampMs - left.startedAtMs) - (timestampMs - right.startedAtMs)
+    || left.index - right.index
+  )[0].index;
+}
+
+/**
+ * Attribute a shared cohort telemetry stream without duplicating events.
+ * A correlation's first timestamp anchors the whole correlation to one
+ * attempt; records without a correlation ID fall back to attempt windows.
+ */
+export function splitCohortTelemetryEvents(attempts, telemetryText) {
+  const windows = attempts.map(attemptTelemetryWindow);
+  const split = attempts.map(() => []);
+  const events = telemetryText.trim()
+    ? parseStrictJsonLines(telemetryText, "native-agent cohort telemetry")
+    : [];
+  const correlationOwner = new Map();
+  for (const event of events) {
+    if (typeof event.correlationId !== "string" || !event.correlationId) continue;
+    if (!correlationOwner.has(event.correlationId)) {
+      correlationOwner.set(
+        event.correlationId,
+        windowOwner(windows, eventTimestampMs(event)),
+      );
+    } else if (correlationOwner.get(event.correlationId) === null) {
+      const owner = windowOwner(windows, eventTimestampMs(event));
+      if (owner !== null) correlationOwner.set(event.correlationId, owner);
+    }
+  }
+  for (const event of events) {
+    const owner = typeof event.correlationId === "string" && event.correlationId
+      ? correlationOwner.get(event.correlationId) ?? null
+      : windowOwner(windows, eventTimestampMs(event));
+    if (owner !== null) split[owner].push(event);
+  }
+  return split;
+}
+
+export function finalizeProfileAttempts(attempts, telemetryText) {
+  const telemetryByAttempt = splitCohortTelemetryEvents(attempts, telemetryText);
+  return attempts.map((attempt, attemptIndex) => {
+    if (attempt?.profileAttempt !== true) return attempt;
+    const attributedTelemetryText = telemetryByAttempt[attemptIndex]
+      .map((event) => JSON.stringify(event))
+      .join("\n");
+    let summary = null;
+    let parseError = null;
+    try {
+      summary = summarizeRawAttempt({
+        stdoutText: attempt.stdoutText,
+        telemetryText: attributedTelemetryText,
+        rolloutTexts: attempt.rolloutTexts,
+      });
+    } catch (error) {
+      parseError = error;
+    }
+    const record = safeAttemptRecord({
+      options: attempt.options,
+      index: attempt.index,
+      processResult: attempt.processResult,
+      summary,
+      jsonValid: parseError === null,
+      parseError,
+    });
+    privateWrite(path.join(attempt.attemptDir, "record.json"), `${JSON.stringify(record, null, 2)}\n`);
+    return record;
+  });
+}
+
 async function profileAttempt(options, sessionDir, index) {
+  const environment = options.environment || process.env;
   await assertExpectedTenant({
     expectedTenant: options.expectedTenant,
     impelBinary: options.impelBinary,
-    environment: process.env,
+    environment,
   });
   const attemptName = `${options.cohort}-${String(index + 1 + options.attemptOffset).padStart(3, "0")}`;
   const attemptDir = path.join(sessionDir, attemptName);
   privateDirectory(attemptDir);
   const stdoutPath = path.join(attemptDir, "stdout.jsonl");
   const stderrPath = path.join(attemptDir, "stderr.log");
-  const telemetryPath = path.join(attemptDir, "telemetry.jsonl");
-  const codexArgs = [
-    "codex",
-    ...(options.mode === "direct" ? ["--agent", options.agent] : []),
-    "exec",
-    "--json",
-    ...options.extraArgs,
-    "-",
-  ];
-  const invocation = impelInvocation(options.impelBinary, codexArgs, process.env);
+  const attemptOptions = profiledCodexAttemptOptions(
+    options,
+    options.telemetryPath,
+    environment,
+  );
+  const { codexArgs } = attemptOptions;
+  const invocation = impelInvocation(options.impelBinary, codexArgs, attemptOptions.environment);
   const processResult = await runProfiledProcess({
     ...invocation,
-    environment: {
-      ...process.env,
-      IMPEL_NATIVE_AGENT_TELEMETRY_PATH: telemetryPath,
-      IMPEL_NATIVE_HOST: options.mode === "direct" ? "codex-direct-profile" : "codex-compatible-agent",
-      IMPEL_NATIVE_HOST_BUILD: options.hostBuild,
-    },
+    environment: attemptOptions.environment,
     input: options.prompt,
     stdoutPath,
     stderrPath,
@@ -481,27 +949,15 @@ async function profileAttempt(options, sessionDir, index) {
     stdoutText,
     destination: path.join(attemptDir, "rollouts"),
   });
-  let summary = null;
-  let parseError = null;
-  try {
-    summary = summarizeRawAttempt({
-      stdoutText,
-      telemetryText: fs.existsSync(telemetryPath) ? fs.readFileSync(telemetryPath, "utf8") : "",
-      rolloutTexts: rolloutPaths.map((rolloutPath) => fs.readFileSync(rolloutPath, "utf8")),
-    });
-  } catch (error) {
-    parseError = error;
-  }
-  const record = safeAttemptRecord({
+  return {
+    profileAttempt: true,
     options,
     index,
+    attemptDir,
     processResult,
-    summary,
-    jsonValid: parseError === null,
-    parseError,
-  });
-  privateWrite(path.join(attemptDir, "record.json"), `${JSON.stringify(record, null, 2)}\n`);
-  return record;
+    stdoutText,
+    rolloutTexts: rolloutPaths.map((rolloutPath) => fs.readFileSync(rolloutPath, "utf8")),
+  };
 }
 
 function profilerFailureClass(error) {
@@ -514,16 +970,20 @@ function profilerFailureClass(error) {
   return "attempt-infrastructure";
 }
 
-export async function runWorkers(options, sessionDir, profileAttemptImpl = profileAttempt) {
+export async function runWorkers(options, sessionDir, profileAttemptImpl = profileAttempt, {
+  startIndex = 0,
+  endIndex = options.attempts,
+  concurrency = options.concurrency,
+} = {}) {
   const results = new Array(options.attempts);
-  let next = 0;
+  let next = startIndex;
   let abort = null;
   const worker = async () => {
     for (;;) {
       if (abort) return;
       const index = next;
       next += 1;
-      if (index >= options.attempts) return;
+      if (index >= endIndex) return;
       try {
         results[index] = await profileAttemptImpl(options, sessionDir, index);
       } catch (error) {
@@ -534,10 +994,98 @@ export async function runWorkers(options, sessionDir, profileAttemptImpl = profi
       }
     }
   };
-  await Promise.all(Array.from({ length: options.concurrency }, () => worker()));
+  const workerCount = Math.min(concurrency, Math.max(0, endIndex - startIndex));
+  await Promise.all(Array.from({ length: workerCount }, () => worker()));
   return { attempts: results.filter(Boolean), abort };
 }
 
+const MISSING_MANAGED_AGENT_MESSAGE =
+  "the isolated profile did not expose the managed agent tool; the cohort would measure model prose, not agent latency";
+
+export function firstAttemptIntegrityAbort(record) {
+  const answerCalls = record?.toolCounts?.[NATIVE_AGENT_ANSWER_TOOL] || 0;
+  const terminalResults = record?.terminalResults || 0;
+  if (answerCalls > 0 && terminalResults > 0) return null;
+  return {
+    failureClass: "managed-agent-integrity",
+    attemptId: record?.attemptId || null,
+    message: `${MISSING_MANAGED_AGENT_MESSAGE} (first attempt: ${answerCalls} ${NATIVE_AGENT_ANSWER_TOOL} call${answerCalls === 1 ? "" : "s"}, ${terminalResults} terminal result${terminalResults === 1 ? "" : "s"}); aborting remaining attempts`,
+  };
+}
+
+function readTelemetryText(telemetryPath) {
+  return fs.existsSync(telemetryPath) ? fs.readFileSync(telemetryPath, "utf8") : "";
+}
+
+/**
+ * The first attempt is deliberately run alone. Until its rollout proves that
+ * the fixed answer tool reached a terminal result, starting concurrent work
+ * would risk collecting a whole cohort of plausible-looking model prose.
+ */
+export async function runGuardedCohort(options, sessionDir, {
+  profileAttemptImpl = profileAttempt,
+  readTelemetry = readTelemetryText,
+} = {}) {
+  const first = await runWorkers(options, sessionDir, profileAttemptImpl, {
+    startIndex: 0,
+    endIndex: 1,
+    concurrency: 1,
+  });
+  const firstRecords = finalizeProfileAttempts(
+    first.attempts,
+    readTelemetry(options.telemetryPath),
+  );
+  if (first.abort) return { attempts: firstRecords, abort: first.abort };
+  const integrityAbort = firstAttemptIntegrityAbort(firstRecords[0]);
+  if (integrityAbort) return { attempts: firstRecords, abort: integrityAbort };
+  if (options.attempts === 1) return { attempts: firstRecords, abort: null };
+
+  const remaining = await runWorkers(options, sessionDir, profileAttemptImpl, {
+    startIndex: 1,
+    endIndex: options.attempts,
+    concurrency: options.concurrency,
+  });
+  const remainingRecords = finalizeProfileAttempts(
+    remaining.attempts,
+    readTelemetry(options.telemetryPath),
+  );
+  return {
+    attempts: [...firstRecords, ...remainingRecords],
+    abort: remaining.abort,
+  };
+}
+
+export function buildProfileAggregate({ sessionId, generatedAt, options, result }) {
+  const telemetryCaptured = result.attempts.some(
+    (attempt) => (attempt.telemetryEventCount || 0) > 0,
+  );
+  return {
+    schema: PROFILE_SCHEMA,
+    sessionId,
+    generatedAt,
+    tenantId: options.expectedTenant,
+    sloClass: options.sloClass,
+    mode: options.mode,
+    cohort: options.cohort,
+    cliVersion: options.hostBuild,
+    promptSha256: options.promptSha256,
+    complete: result.abort === null && result.attempts.length === options.attempts,
+    telemetryCaptured,
+    expectedAttempts: options.attempts,
+    abort: result.abort,
+    attempts: result.attempts,
+  };
+}
+
+export function telemetryCaptureWarning(aggregate) {
+  const terminalResults = aggregate.attempts.reduce(
+    (total, attempt) => total + (attempt.terminalResults || 0),
+    0,
+  );
+  if (!aggregate.complete || aggregate.telemetryCaptured || terminalResults === 0) return null;
+  return "WARNING: the cohort completed with managed-agent terminal results but captured zero telemetry events; aggregate telemetryCaptured=false";
+}
+
 async function restoreTenant(originalTenant, options) {
   let actual;
   try { actual = await currentTenant(options.impelBinary, process.env); } catch { return false; }
@@ -572,8 +1120,6 @@ async function main(argv) {
     throw new Error("could not establish the Impel CLI build under test");
   }
   options.hostBuild = version.stdout.trim();
-  const configRoot = process.env.IMPEL_CONFIG_DIR || path.join(os.homedir(), ".config", "impel");
-  options.rolloutRoot = path.join(configRoot, "cli", "tenants", options.expectedTenant, "codex", "sessions");
   const outputRoot = path.resolve(options.outputDir);
   privateDirectory(outputRoot);
   const sessionId = `codex-${new Date().toISOString().replace(/[:.]/gu, "-")}-${crypto.randomBytes(4).toString("hex")}`;
@@ -581,28 +1127,29 @@ async function main(argv) {
   privateDirectory(sessionDir);
   let restored = false;
   try {
-    const result = await runWorkers(options, sessionDir);
-    const aggregate = {
-      schema: PROFILE_SCHEMA,
+    const result = await withIsolatedBenchmarkProfile(options, sessionDir, async (isolation) => {
+      options.environment = isolation.environment;
+      options.telemetryPath = isolation.telemetryPath;
+      options.rolloutRoot = path.join(isolation.codexHome, "sessions");
+      return runGuardedCohort(options, sessionDir);
+    });
+    const aggregate = buildProfileAggregate({
       sessionId,
       generatedAt: new Date().toISOString(),
-      tenantId: options.expectedTenant,
-      sloClass: options.sloClass,
-      mode: options.mode,
-      cohort: options.cohort,
-      cliVersion: options.hostBuild,
-      promptSha256: options.promptSha256,
-      complete: result.abort === null && result.attempts.length === options.attempts,
-      expectedAttempts: options.attempts,
-      abort: result.abort,
-      attempts: result.attempts,
-    };
+      options,
+      result,
+    });
     const aggregatePath = path.join(sessionDir, "aggregate.json");
     privateWrite(aggregatePath, `${JSON.stringify(aggregate, null, 2)}\n`);
     process.stderr.write(`Private Codex profile written under ${sessionDir}\n`);
+    const telemetryWarning = telemetryCaptureWarning(aggregate);
+    if (telemetryWarning) process.stderr.write(`${telemetryWarning}\n`);
     process.stdout.write(`${JSON.stringify(aggregate)}\n`);
     if (!aggregate.complete) {
-      throw new Error(`cohort aborted (${aggregate.abort?.failureClass || "incomplete"}); partial aggregate retained`);
+      const detail = aggregate.abort?.message ? `: ${aggregate.abort.message}` : "";
+      throw new Error(
+        `cohort aborted (${aggregate.abort?.failureClass || "incomplete"})${detail}; partial aggregate retained`,
+      );
     }
   } finally {
     restored = await restoreTenant(originalTenant, options);