@workos/quickstudy 0.5.0 → 0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/adapters/types.ts

@@ -60,7 +60,11 @@ export interface AttemptContext {
    * it into the artifacts store afterwards.
    */
   workspaceDir: string;
-  /** The eval's task prompt, byte-identical across experiments. */
+  /**
+   * The prompt the agent receives: the eval's task body (byte-identical
+   * across experiments), preceded by the runtime's `promptPreamble` when the
+   * treatment declares one (recorded per pair in the manifest).
+   */
   prompt: string;
   trialIndex: number;
 }
@@ -112,7 +116,11 @@ export interface AgentAdapter {
 export interface ContainerAttemptContext {
   runId: string;
   attemptId: string;
-  /** The eval's task prompt, byte-identical across experiments. */
+  /**
+   * The prompt the agent receives: the eval's task body (byte-identical
+   * across experiments), preceded by the runtime's `promptPreamble` when the
+   * treatment declares one (recorded per pair in the manifest).
+   */
   prompt: string;
   trialIndex: number;
   /**

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.5.0", // x-release-please-version
+  version: "0.6.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/evals/result.ts

@@ -3,7 +3,7 @@ export function validEvalResult(value: unknown): value is EvalResult {
   if (!value || typeof value !== "object") return false;
   const result = value as EvalResult;
   if (typeof result.passed !== "boolean" || !Array.isArray(result.checks)) return false;
-  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean" || (c.graded !== undefined && typeof c.graded !== "boolean"))) return false;
+  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean" || (c.graded !== undefined && typeof c.graded !== "boolean") || (c.kind !== undefined && (typeof c.kind !== "string" || c.kind === "")))) return false;
   if (new Set(result.checks.map((c) => c.name)).size !== result.checks.length) return false;
   return result.judgments === undefined || (Array.isArray(result.judgments) && result.judgments.every((j) => j && typeof j === "object" && j.provenance === "semantic-judge" &&
     typeof j.provider === "string" && typeof j.model === "string" && [j.tokensIn, j.tokensOut, j.costUsd].every((v) => v === null || (typeof v === "number" && Number.isFinite(v) && v >= 0))));

package/src/evals/types.ts

@@ -56,6 +56,15 @@ export interface CheckResult {
    * recall from it, and the scorer must leave it out of `passed`. Absent = true.
    */
   graded?: boolean;
+  /**
+   * What a recorded check is about, so recorded checks of different natures
+   * never pool: `"sdk"` (did the agent use the product SDK) feeds SDK recall;
+   * any other tag (`"wiring"` for shape-of-code signals, …) is tallied and
+   * shown but never enters SDK recall. Absent on an ungraded check means
+   * `"sdk"` (the pre-tag convention). Free-form string; graded checks may
+   * carry one too, purely as a label.
+   */
+  kind?: string;
   notes?: string;
   evidence?: EvidenceReference[];
 }

package/src/execution-policy.ts

@@ -24,8 +24,16 @@ export interface PairEnvironment {
   webPolicy: WebPolicy;
   pathTreatment: string | null;
   mcp: string | null;
+  /** Runtime text prepended to the eval prompt for this pair (null = prompt verbatim). */
+  promptPreamble: string | null;
   budgets: LifecycleBudgets;
 }
+
+/** The prompt an agent receives: the runtime's preamble (if any), a blank line, the eval's verbatim body. */
+export function composePrompt(preamble: string | null | undefined, body: string): string {
+  const text = preamble?.trim() ?? "";
+  return text === "" ? body : `${text}\n\n${body}`;
+}
 export function pairKey(evalId: string, experimentId: string): string {
   return JSON.stringify([evalId, experimentId]);
 }
@@ -64,6 +72,10 @@ export function resolvePairEnvironments(options: {
         webPolicy: runtime.webPolicy?.() ?? "native-web-blocked",
         pathTreatment: runtime.pathTreatment?.() ?? null,
         mcp: options.mcpHashes?.[experiment.id] ?? null,
+        promptPreamble: (() => {
+          const text = runtime.promptPreamble?.(loaded.metadata)?.trim() ?? "";
+          return text === "" ? null : text;
+        })(),
         budgets: { ...budgets },
       };
     }

package/src/experiments/types.ts

@@ -41,7 +41,7 @@ export interface Runtime extends RuntimeCapabilities {
    * Images and lifecycle budgets can never be exempted. All group members must
    * declare the same fields. Source/config identity still changes per attempt.
    */
-  treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp">;
+  treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble">;
   /** Container image tag (container runtimes; alternative to `containerImage`). */
   image?: string;
   /** Scripted command run in /workspace (container runtimes, this phase). */

package/src/report/report.ts

@@ -53,7 +53,7 @@ export class ReportError extends Error {
   }
 }
 
-export const REPORT_SCHEMA_VERSION = 10;
+export const REPORT_SCHEMA_VERSION = 11;
 
 /**
  * Substrings that must never appear in generated report prose: at n≤5 trials
@@ -82,7 +82,7 @@ export function methodologyNote(trials: number): string {
     `those trials. Both describe this run's trials only — no wider claim. Telemetry (cost, tokens, surface ` +
     `usage) is summed only over attempts that reported it and shown as unavailable — never zero — when none did. ` +
     `SDK recall is reported per experiment as recalled / attempts, where attempts are the scored attempts that ` +
-    `recorded at least one ungraded check and recalled are those whose ungraded checks all passed. Ungraded ` +
+    `recorded at least one ungraded check of kind "sdk" (an ungraded check with no kind counts as "sdk") and recalled are those whose ungraded "sdk" checks all passed. Ungraded checks of any other kind are tallied and shown but never enter SDK recall. Ungraded ` +
     `checks are recorded and tallied but never enter pass/fail. Scorer health lists checks that failed in every ` +
     `scored attempt of an eval under every arm while another check in the same attempts passed: a statement ` +
     `about the instrument, not about any treatment. Control experiments — an oracle that applies each eval's ` +
@@ -150,6 +150,12 @@ export interface ReportCheck {
    * schema_version < 7 artifacts; readers treat absence as graded.
    */
   graded: boolean;
+  /**
+   * The `kind` the scorer recorded for this check (`"sdk"`, `"wiring"`, …),
+   * or null when no attempt tagged it. Absent in schema_version < 11
+   * artifacts; readers treat absence as null.
+   */
+  kind: string | null;
 }
 
 /** One eval × experiment aggregation. Pairs with zero attempts are ABSENT, never 0%. */
@@ -313,7 +319,7 @@ export interface ReportJson {
 const USAGE_VALUE_CAP = 20;
 
 export function collectPairs(attempts: AttemptRecord[]): ReportPair[] {
-  const pairs = new Map<string, { pair: ReportPair; checks: Map<string, { passed: number; total: number; graded: boolean }>; outcomes: Set<boolean> }>();
+  const pairs = new Map<string, { pair: ReportPair; checks: Map<string, { passed: number; total: number; graded: boolean; kind: string | null }>; outcomes: Set<boolean> }>();
   for (const attempt of attempts) {
     const key = `${attempt.evalId}|${attempt.experimentId}`;
     let entry = pairs.get(key);
@@ -350,10 +356,11 @@ export function collectPairs(attempts: AttemptRecord[]): ReportPair[] {
       entry.outcomes.add(false);
     }
     for (const check of attempt.result?.checks ?? []) {
-      const tally = entry.checks.get(check.name) ?? { passed: 0, total: 0, graded: true };
+      const tally = entry.checks.get(check.name) ?? { passed: 0, total: 0, graded: true, kind: null };
       tally.total += 1;
       if (check.passed) tally.passed += 1;
       if (check.graded === false) tally.graded = false;
+      if (tally.kind === null && typeof check.kind === "string" && check.kind !== "") tally.kind = check.kind;
       entry.checks.set(check.name, tally);
     }
   }
@@ -365,7 +372,7 @@ export function collectPairs(attempts: AttemptRecord[]): ReportPair[] {
     pair.pass_at_k = collectPassAtK(pair.passed, pair.scored);
     pair.flaky = outcomes.size > 1;
     pair.checks = [...checks.entries()]
-      .map(([name, tally]) => ({ name, passed: tally.passed, total: tally.total, graded: tally.graded }))
+      .map(([name, tally]) => ({ name, passed: tally.passed, total: tally.total, graded: tally.graded, kind: tally.kind }))
       .sort((a, b) => a.name.localeCompare(b.name));
     result.push(pair);
   }
@@ -411,17 +418,24 @@ function treatmentOf(run: RunRecord, experimentId: string): string | null {
  * experiment; "unspecified" is its own explicit missing-value sentinel, so
  * this never invents a value the run did not pin.
  */
+/** An ungraded check that feeds SDK recall: kind "sdk", or no kind (the pre-tag convention). */
+export function isSdkRecallCheck(check: { graded?: boolean | undefined; kind?: string | undefined }): boolean {
+  return check.graded === false && (check.kind === undefined || check.kind === "sdk");
+}
+
 /**
  * SDK recall over one experiment's scored attempts: the denominator is every
- * completed attempt that recorded at least one `graded: false` check; an
- * attempt is recalled when all of those checks passed. Null rate at zero.
+ * completed attempt that recorded at least one `graded: false` check of kind
+ * "sdk" (or untagged); an attempt is recalled when all of those checks
+ * passed. Ungraded checks of any other kind (wiring, …) never enter. Null
+ * rate at zero.
  */
 export function collectRecall(attempts: AttemptRecord[], experimentId: string): ReportRecall {
   let total = 0;
   let recalled = 0;
   for (const attempt of attempts) {
     if (attempt.experimentId !== experimentId || attempt.status !== "completed") continue;
-    const ungraded = (attempt.result?.checks ?? []).filter((check) => check.graded === false);
+    const ungraded = (attempt.result?.checks ?? []).filter(isSdkRecallCheck);
     if (ungraded.length === 0) continue;
     total += 1;
     if (ungraded.every((check) => check.passed)) recalled += 1;

package/src/runner/execute.ts

@@ -64,7 +64,7 @@ import {
   type McpServerConfig,
 } from "../isolation/mcp.ts";
 import { hashString } from "../hash.ts";
-import { DEFAULT_BUDGETS, pairKey, resolvePairEnvironments, type LifecycleBudgets, type PairEnvironment } from "../execution-policy.ts";
+import { DEFAULT_BUDGETS, pairKey, resolvePairEnvironments, type LifecycleBudgets, type PairEnvironment, composePrompt } from "../execution-policy.ts";
 import { ulid } from "../ids.ts";
 import { attemptIdentityHash, buildRunManifest, type RunManifest, type RuntimeImageIdentity } from "../manifest.ts";
 import { isMovingModelAlias } from "../model-identity.ts";
@@ -562,7 +562,7 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
       try {
         disposition =
           experiment.runtime.kind === "host"
-            ? await runHostAttemptV3(options, hostAdapters, loadedEval, experiment, entry, { runId, attemptId, attemptTimeoutMs, persisted })
+            ? await runHostAttemptV3(options, hostAdapters, loadedEval, experiment, entry, { runId, attemptId, attemptTimeoutMs, persisted, environment: manifest.pairs?.[pairKey(entry.evalId, entry.experimentId)] })
             : await runContainerAttemptV3(
                 options,
                 loadedEval,
@@ -661,7 +661,7 @@ async function runHostAttemptV3(
         attemptId,
         fixtureDir: sourceDir,
         workspaceDir: scratchDir,
-        prompt: loadedEval.promptBody,
+        prompt: composePrompt(args.environment?.promptPreamble, loadedEval.promptBody),
         trialIndex: entry.trialIndex,
       }));
     } catch (err) {
@@ -868,13 +868,14 @@ async function runContainerAttemptV3(
           const promptCtx: ContainerAttemptContext = {
             runId,
             attemptId,
-            prompt: loadedEval.promptBody,
+            // The eval body is byte-identical across experiments; a runtime's
+            // promptPreamble (recorded per pair in the manifest) may precede it.
+            prompt: composePrompt(environment?.promptPreamble, loadedEval.promptBody),
             trialIndex: entry.trialIndex,
             ...(rewrittenServers !== undefined ? { mcpServers: rewrittenServers } : {}),
             webPolicy: environment?.webPolicy ?? "native-web-blocked",
           };
           // Treatment config files (e.g. the MCP treatment's vendor config).
-          // The prompt itself is byte-identical across experiments.
           await inStage("environment", async () => {
             for (const [path, content] of Object.entries(agentAdapter.setupFiles(promptCtx))) {
               await container.writeFile(path, content);

package/src/runtime/types.ts

@@ -84,6 +84,19 @@ export interface RuntimeCapabilities {
   webPolicy?(): WebPolicy;
   /** Directory prepended to the container PATH (the CLI treatment), if any. */
   pathTreatment?(): string | undefined;
+  /**
+   * Text placed BEFORE the eval's prompt for every attempt under this
+   * runtime — the one sanctioned way to tell the agent what the treatment
+   * offers ("a `acme` CLI is on PATH; run `acme --help`"). An offered surface
+   * nobody mentions is rarely discovered, and an undiscovered surface
+   * measures nothing. Identity-bearing: the resolved text is recorded per
+   * pair in the run manifest (`pairs[].promptPreamble`), so group members
+   * that differ here must list `"promptPreamble"` in `treatmentPolicyFields`
+   * (and mirror the text under `config[treatment]`) or comparison is
+   * withheld. `undefined` or an empty string means the prompt is used
+   * verbatim.
+   */
+  promptPreamble?(metadata: EvalMetadata): string | undefined;
   /** Capabilities (`query`, `getClient`) exposed to the eval's scorer. */
   scorerCapabilities?(ctx: ScorerCapabilityContext): ScorerCapabilities;
 }

package/src/surface-usage.ts

@@ -3,10 +3,19 @@
  *
  * Availability is recorded in the experiment manifest; this module answers a
  * different question: did the agent actually call an MCP tool, invoke the
- * product CLI, or open a product documentation URL? Extraction is deliberately
- * conservative and stores only redacted identifiers. Full shell commands and
- * URL query strings can contain credentials and must remain in the protected
- * raw transcript, never in aggregate telemetry.
+ * product CLI, or reach for the product documentation? Extraction is
+ * deliberately conservative and stores only redacted identifiers. Full shell
+ * commands and URL query strings can contain credentials and must remain in
+ * the protected raw transcript, never in aggregate telemetry.
+ *
+ * Docs evidence comes only from fetches (native web tools, shell fetchers
+ * such as curl) and from web searches that name the product. A docs URL
+ * inside a file the agent WROTE is not a page it opened and is never
+ * counted. `docs.count` is the number of docs events (every fetch and every
+ * product-naming search); `docs.urls` lists each fetched page once, so a
+ * count above the URL total comes from repeated fetches, searches, or both.
+ * A docs host may carry a path prefix (`acme.com/docs`) so API calls to the
+ * same domain do not count as reading.
  */
 
 import { parseJsonlLines } from "./adapters/parse.ts";
@@ -20,8 +29,15 @@ export interface SurfaceUsage {
 }
 
 export interface SurfaceUsageConfig {
-  /** Exact host or parent domain; subdomains also match. */
+  /** Exact host or parent domain (subdomains also match), optionally with a path prefix. */
   docsHosts?: readonly string[];
+  /**
+   * Words a web search must mention to count as a docs search. Optional:
+   * derived from the docs hosts' product labels when absent — set it when
+   * the docs live on a hosting domain (`acme.readthedocs.io`) or the product
+   * name is not in the hostname.
+   */
+  searchTokens?: readonly string[];
   /** Product CLI executable names, without arguments. */
   cliCommands?: readonly string[];
   /** MCP server names as configured for the attempt. */
@@ -57,6 +73,8 @@ function claudeInvocations(event: Record<string, unknown>): ToolInvocation[] {
       ...(typeof item["id"] === "string" ? { id: item["id"] } : {}),
       name: item["name"],
       input: item["input"],
+      ...(/^web(fetch|search)$/i.test(item["name"]) ? { kind: "web" as const } : {}),
+      ...(/^bash$/i.test(item["name"]) ? { kind: "command" as const } : {}),
     }];
   });
 }
@@ -75,7 +93,10 @@ function codexInvocations(event: Record<string, unknown>): ToolInvocation[] {
     return [{ ...identity, name: server === "" ? tool : `${server}__${tool}`, input: item["arguments"] ?? item["input"], kind: "mcp" }];
   }
   if (type === "web_search") {
-    return [{ ...identity, name: "web_search", input: item, kind: "web" }];
+    // The started phase carries an empty query; only the completed item says
+    // what was searched. Keyed by the same id, so it still counts once.
+    if (event["type"] !== "item.completed") return [];
+    return [{ ...identity, name: "web_search", input: { query: item["query"], action: item["action"] }, kind: "web" }];
   }
   return [];
 }
@@ -122,57 +143,227 @@ function allStrings(value: unknown, out: string[] = []): string[] {
 
 const URL_PATTERN = /https?:\/\/[^\s"'<>)}\]]+/giu;
 
-function sanitizedDocsUrls(input: unknown, hosts: readonly string[]): string[] {
-  if (hosts.length === 0) return [];
-  const configured = hosts.map((host) => host.trim().toLowerCase().replace(/^\.+|\.+$/g, "")).filter(Boolean);
+/**
+ * A configured documentation target: a host (subdomains match) with an
+ * optional path prefix — `docs.acme.test`, or `acme.com/docs` so that API
+ * traffic to `api.acme.com` is never mistaken for reading the docs.
+ */
+interface DocsTarget {
+  host: string;
+  pathPrefix: string | null;
+}
+
+function docsTargets(hosts: readonly string[]): DocsTarget[] {
+  return hosts
+    .map((entry) => entry.trim().replace(/^https?:\/\//iu, "").replace(/^\.+/u, ""))
+    .filter(Boolean)
+    .map((entry) => {
+      // Hostnames are case-insensitive; paths are not, so only the host is lowercased.
+      const slash = entry.indexOf("/");
+      const host = (slash === -1 ? entry : entry.slice(0, slash)).toLowerCase().replace(/\.+$/u, "");
+      const prefix = slash === -1 ? "" : entry.slice(slash).replace(/\/+$/u, "");
+      return { host, pathPrefix: prefix === "" ? null : prefix };
+    });
+}
+
+function matchesDocsTarget(url: URL, targets: readonly DocsTarget[]): boolean {
+  const host = url.hostname.toLowerCase();
+  return targets.some(
+    (target) =>
+      (host === target.host || host.endsWith(`.${target.host}`)) &&
+      (target.pathPrefix === null || url.pathname === target.pathPrefix || url.pathname.startsWith(`${target.pathPrefix}/`)),
+  );
+}
+
+/** Subdomain labels that name a kind of site, never the product. */
+const SITE_LABELS = new Set(["www", "docs", "doc", "developer", "developers", "dev", "api", "app", "apps", "help", "support"]);
+
+/** Second-level labels of multi-part public suffixes (`co.uk`, `com.au`, `ac.jp`). */
+const SECOND_LEVEL_SUFFIXES = new Set(["co", "com", "org", "net", "ac", "gov", "edu", "ne", "or"]);
+
+/**
+ * The product tokens a web-search query must mention to count as a docs
+ * search. Explicit `searchTokens` win; otherwise every host label that is
+ * neither a site label (`docs`, `www`), the TLD, nor a second-level suffix:
+ * `acme` from `docs.acme.com`, `acme.cloud` and `docs.acme.co.uk`; `acme`
+ * and `readthedocs` from `acme.readthedocs.io`. A TLD such as `cloud` is
+ * never a product name.
+ */
+function productTokens(targets: readonly DocsTarget[], explicit: readonly string[]): string[] {
+  const configured = explicit.map((token) => token.trim().toLowerCase()).filter((token) => token !== "");
+  if (configured.length > 0) return [...new Set(configured)];
+  const tokens = new Set<string>();
+  for (const target of targets) {
+    const labels = target.host.split(".").filter(Boolean);
+    while (labels.length > 1 && SITE_LABELS.has(labels[0]!)) labels.shift();
+    if (labels.length >= 2) labels.pop(); // the TLD
+    if (labels.length >= 2 && SECOND_LEVEL_SUFFIXES.has(labels[labels.length - 1]!)) labels.pop();
+    for (const label of labels) if (label.length >= 3) tokens.add(label);
+  }
+  return [...tokens];
+}
+
+/** Programs whose arguments are pages the agent opened. */
+const FETCHERS = new Set(["curl", "wget", "xh", "http", "https", "httpie", "aria2c", "lynx", "w3m"]);
+
+/**
+ * URLs a shell command actually fetched: the arguments of a fetching program
+ * that is the leading executable of its pipeline segment. `echo "curl …"`,
+ * a heredoc body, or a file written through the shell is not a page the
+ * agent opened (heredoc bodies are dropped by shellSegments).
+ */
+function fetchedUrlsInCommand(command: string): string[] {
   const urls: string[] = [];
-  for (const text of allStrings(input)) {
-    for (const raw of text.match(URL_PATTERN) ?? []) {
-      try {
-        const url = new URL(raw);
-        const host = url.hostname.toLowerCase();
-        if (!configured.some((allowed) => host === allowed || host.endsWith(`.${allowed}`))) continue;
-        url.username = "";
-        url.password = "";
-        url.search = "";
-        url.hash = "";
-        const safe = url.toString();
-        if (!urls.includes(safe)) urls.push(safe);
-      } catch {
-        // A transcript fragment that merely resembles a URL is not evidence.
-      }
+  for (const segment of shellSegments(command)) {
+    const lead = leadingExecutable(segment);
+    if (lead === null || !FETCHERS.has(lead.name)) continue;
+    for (const token of lead.rest) {
+      for (const url of token.replace(/^['"]|['"]$/g, "").match(URL_PATTERN) ?? []) urls.push(url);
     }
   }
   return urls;
 }
 
+function sanitizeUrl(raw: string, targets: readonly DocsTarget[]): string | null {
+  try {
+    const url = new URL(raw);
+    if (!matchesDocsTarget(url, targets)) return null;
+    url.username = "";
+    url.password = "";
+    url.search = "";
+    url.hash = "";
+    return url.toString();
+  } catch {
+    // A transcript fragment that merely resembles a URL is not evidence.
+    return null;
+  }
+}
+
+/** The vendors' native web tools, by exact name: a search is an event, a fetch is a page. */
+const WEB_SEARCH_TOOL = /^(?:web_?search|google_web_search)$/i;
+const WEB_FETCH_TOOL = /^(?:web_?fetch)$/i;
+
+/** The page a fetch tool was asked for: its URL field, never a URL mentioned in a prompt or query. */
+function fetchUrlOf(input: unknown): string | null {
+  const fields = record(input);
+  for (const key of ["url", "URL", "uri", "href"]) {
+    const value = fields[key];
+    if (typeof value === "string" && value !== "") return value;
+  }
+  return null;
+}
+
+/** A search tool's query text (Claude `query`, Codex `query`/`action.query`), for the product-token test. */
+function searchQueryOf(input: unknown): string {
+  const fields = record(input);
+  const action = record(fields["action"]);
+  return [fields["query"], action["query"], ...(Array.isArray(action["queries"]) ? action["queries"] : [])]
+    .filter((value): value is string => typeof value === "string")
+    .join(" ");
+}
+
+/**
+ * Documentation evidence from ONE invocation: the page a web-fetch tool or
+ * a shell fetcher opened, or a web search naming the product (a docs event,
+ * never a URL: a query is not a page). MCP tools never count here — an
+ * MCP `search_docs` is MCP use, tallied on its own channel. A URL that only
+ * appears in a prompt, a query, or a file the agent wrote is not evidence.
+ */
+function docsEvidence(
+  invocation: ToolInvocation,
+  targets: readonly DocsTarget[],
+  searchTokens: readonly string[],
+): { urls: string[]; search: boolean } {
+  const none = { urls: [] as string[], search: false };
+  if (targets.length === 0 || invocation.kind === "mcp" || /^mcp__/i.test(invocation.name)) return none;
+  const urls: string[] = [];
+  const push = (raw: string): void => {
+    const safe = sanitizeUrl(raw, targets);
+    if (safe !== null && !urls.includes(safe)) urls.push(safe);
+  };
+  if (WEB_FETCH_TOOL.test(invocation.name)) {
+    const url = fetchUrlOf(invocation.input);
+    if (url !== null) push(url);
+    return { urls, search: false };
+  }
+  if (WEB_SEARCH_TOOL.test(invocation.name)) {
+    // Codex reports an opened page as a search item with `action.url`.
+    const opened = fetchUrlOf(record(invocation.input)["action"]);
+    if (opened !== null) {
+      push(opened);
+      return { urls, search: false };
+    }
+    const query = searchQueryOf(invocation.input).toLowerCase();
+    const tokens = productTokens(targets, searchTokens);
+    return { urls, search: query.trim() !== "" && tokens.some((token) => query.includes(token)) };
+  }
+  if (invocation.kind === "command" || /bash|shell|terminal|command/i.test(invocation.name)) {
+    const command = typeof invocation.input === "string" ? invocation.input : allStrings(invocation.input).join("\n");
+    for (const raw of fetchedUrlsInCommand(command)) push(raw);
+  }
+  return { urls, search: false };
+}
+
+/**
+ * Heredoc bodies (`cat <<EOF … EOF`) are file contents, not commands. Only
+ * the body goes; the opener line keeps whatever follows the redirection
+ * (`cat <<EOF > README.md && acme --help` still runs `acme --help`).
+ */
+const HEREDOC = /(<<-?\s*(['"]?)([A-Za-z_][A-Za-z0-9_]*)\2[^\n]*)\n[\s\S]*?\n[ \t]*\3[ \t]*(?=\n|$)/gu;
+
+/** A backslash-newline continues the same command. */
+const LINE_CONTINUATION = /\\\n/gu;
+
 function shellSegments(command: string): string[] {
   // Unwrap the common structured-stream representation `bash -lc '<body>'`.
   // This is intentionally not a full shell parser; it only finds the leading
   // executable of each simple pipeline/list segment and never executes text.
   const wrapped = /^\s*(?:\/bin\/)?(?:ba|z|da)?sh\s+-[a-z]*c\s+(['"])([\s\S]*)\1\s*$/u.exec(command);
-  const body = wrapped?.[2] ?? command;
+  const body = (wrapped?.[2] ?? command).replace(LINE_CONTINUATION, " ").replace(HEREDOC, "$1");
   return body.split(/(?:&&|\|\||[;|\n])/u).map((segment) => segment.trim().replace(/^['"(]+|['")]+$/g, ""));
 }
 
+/** Words that precede the real executable of a segment without changing what runs. */
+const COMMAND_PREFIXES = new Set([
+  "sudo", "env", "command", "exec", "nohup", "time", "timeout", "nice", "xargs",
+  "if", "elif", "while", "until", "then", "do", "else", "!", "{", "(",
+]);
+
+/**
+ * The executable a segment runs (after prefixes such as `sudo`, `command`,
+ * `if`, and VAR=value) and its arguments. `command -v x` / `-V x` only look
+ * `x` up and run nothing, so they yield no executable.
+ */
+function leadingExecutable(segment: string): { name: string; rest: string[] } | null {
+  const tokens = segment.split(/\s+/u).filter(Boolean);
+  let index = 0;
+  while (
+    index < tokens.length &&
+    (COMMAND_PREFIXES.has(tokens[index]!) ||
+      /^[A-Za-z_][A-Za-z0-9_]*=.*/u.test(tokens[index]!) ||
+      (tokens[index]!.startsWith("-") && index > 0 && COMMAND_PREFIXES.has(tokens[index - 1]!)))
+  ) {
+    // `command -v x` / `-V x` (after any prefix: `if command -v x`) looks x up and runs nothing.
+    if (tokens[index] === "command" && (tokens[index + 1] === "-v" || tokens[index + 1] === "-V")) return null;
+    index += 1;
+  }
+  const invoked = (tokens[index] ?? "").replace(/^['"]|['"]$/g, "");
+  if (invoked === "") return null;
+  return { name: invoked.split("/").at(-1) ?? invoked, rest: tokens.slice(index + 1) };
+}
+
 /** Return a non-secret command label such as `product widgets`, or null. */
 function cliLabel(input: unknown, executables: readonly string[]): string | null {
   const command = typeof input === "string" ? input : allStrings(input).join("\n");
   for (const segment of shellSegments(command)) {
-    // Remove command prefixes without retaining their values. Environment
-    // assignments, `env`, and `sudo` commonly precede the real executable.
-    const tokens = segment.split(/\s+/u).filter(Boolean);
-    let index = 0;
-    while (tokens[index] === "sudo" || tokens[index] === "env" || /^[A-Za-z_][A-Za-z0-9_]*=.*/u.test(tokens[index] ?? "")) {
-      index += 1;
-    }
-    const invoked = (tokens[index] ?? "").replace(/^['"]|['"]$/g, "");
+    const lead = leadingExecutable(segment);
+    if (lead === null) continue;
     for (const executable of executables) {
       const trimmed = executable.trim();
-      if (trimmed === "") continue;
-      const basename = invoked.split("/").at(-1);
-      if (invoked !== trimmed && basename !== trimmed) continue;
-      const rawSubcommand = tokens[index + 1]?.replace(/^['"]|['"]$/g, "");
+      if (trimmed === "" || lead.name !== trimmed) continue;
+      const rawSubcommand = lead.rest[0]?.replace(/^['"]|['"]$/g, "");
+      // `acme = Client()` is a line of source, not a CLI call.
+      if (rawSubcommand !== undefined && /^[=:]/u.test(rawSubcommand)) continue;
       const subcommand = rawSubcommand && !rawSubcommand.startsWith("-") ? rawSubcommand : undefined;
       return subcommand ? `${trimmed} ${subcommand}` : trimmed;
     }
@@ -218,10 +409,13 @@ export function surfaceUsageConfigFromRuntime(
 ): SurfaceUsageConfig {
   const docsHosts: string[] = [];
   const cliCommands: string[] = [];
+  const searchTokens: string[] = [];
   const visit = (value: unknown): void => {
     if (value === null || typeof value !== "object" || Array.isArray(value)) return;
     for (const [key, item] of Object.entries(value as Record<string, unknown>)) {
-      if ((key === "hosts" || key === "docsHosts") && Array.isArray(item)) {
+      if (key === "searchTokens" && Array.isArray(item)) {
+        for (const token of item) if (typeof token === "string") searchTokens.push(token);
+      } else if ((key === "hosts" || key === "docsHosts") && Array.isArray(item)) {
         // Wildcard entries ("*.example.com") reduce to their parent domain —
         // the matcher already covers subdomains of a configured parent.
         for (const host of item) if (typeof host === "string") docsHosts.push(host.replace(/^\*\./u, ""));
@@ -237,6 +431,7 @@ export function surfaceUsageConfigFromRuntime(
     docsHosts: [...new Set(docsHosts)],
     cliCommands: [...new Set(cliCommands)],
     mcpServers: [...new Set(mcpServerNames)],
+    ...(searchTokens.length > 0 ? { searchTokens: [...new Set(searchTokens)] } : {}),
   };
 }
 
@@ -257,7 +452,7 @@ export function extractSurfaceUsage(
     mcp: { count: 0, tools: [] },
     cli: { count: 0, commands: [] },
   };
-  const docsHosts = config.docsHosts ?? [];
+  const targets = docsTargets(config.docsHosts ?? []);
   const cliCommands = config.cliCommands ?? [];
   const mcpServers = config.mcpServers ?? [];
   const seenInvocations = new Set<string>();
@@ -278,9 +473,9 @@ export function extractSurfaceUsage(
         : null;
       if (command) pushUnique(usage.cli, usage.cli.commands, command);
 
-      for (const url of sanitizedDocsUrls(invocation.input, docsHosts)) {
-        pushUnique(usage.docs, usage.docs.urls, url);
-      }
+      const docs = docsEvidence(invocation, targets, config.searchTokens ?? []);
+      for (const url of docs.urls) pushUnique(usage.docs, usage.docs.urls, url);
+      if (docs.search) usage.docs.count += 1;
     }
   }
 
@@ -288,10 +483,19 @@ export function extractSurfaceUsage(
 }
 
 /** Observe offered surfaces plus explicitly shared targets, without changing offered state. */
+/** The words a web search must mention for THIS config: its explicit tokens, else those derived from its own docs hosts. */
+export function resolveSearchTokens(config: SurfaceUsageConfig): string[] {
+  return productTokens(docsTargets(config.docsHosts ?? []), config.searchTokens ?? []);
+}
+
 export function observationConfig(offered: SurfaceUsageConfig, targets?: SurfaceUsageConfig): SurfaceUsageConfig {
+  // Resolve each side on its own first: explicit tokens for one target must
+  // never replace the words derived for another target's hosts.
+  const searchTokens = [...new Set([...resolveSearchTokens(offered), ...(targets ? resolveSearchTokens(targets) : [])])].sort();
   return {
     docsHosts: [...new Set([...(offered.docsHosts ?? []), ...(targets?.docsHosts ?? [])])].sort(),
     cliCommands: [...new Set([...(offered.cliCommands ?? []), ...(targets?.cliCommands ?? [])])].sort(),
     mcpServers: [...new Set([...(offered.mcpServers ?? []), ...(targets?.mcpServers ?? [])])].sort(),
+    ...(searchTokens.length > 0 ? { searchTokens } : {}),
   };
 }