@prismatic-io/lux 0.0.2-preview.17 → 0.0.2-preview.20

package/src/index.ts

@@ -258,9 +258,11 @@ export {
   type DriverRuntime,
   defineDriver,
   driverRef,
+  type HarnessSelection,
   type ResolvedFixtures,
   ResolvedFixturesSchema,
   type StartContext,
+  selectModelAndReasoningEffort,
 } from "./core/driver.js";
 export {
   type DriverMetric,
@@ -344,6 +346,38 @@ export {
   type OptimizerInput,
 } from "./core/experiment.js";
 export { type GlobalIdParts, tryParseGlobalId } from "./core/global-id.js";
+export {
+  type CatalogModels,
+  catalogEfforts,
+  defaultCatalogEffort,
+  defineHarnessCatalog,
+  HARNESS_EFFORTS,
+  type HarnessCatalog,
+  type ReasoningEffort,
+  ReasoningEffortSchema,
+} from "./core/harness-catalog.js";
+export {
+  antigravityCatalog,
+  type BuiltInHarnessDriver,
+  builtInHarnessCatalogs,
+  claudeCodeCatalog,
+  codexCatalog,
+  copilotCatalog,
+  cursorCatalog,
+  harnessCatalogFor,
+} from "./core/harness-catalogs/index.js";
+export {
+  type BuiltInHarnessMatrixEntry,
+  BuiltInHarnessMatrixEntrySchema,
+  type CustomHarnessMatrixEntry,
+  expandHarnessMatrix,
+  type HarnessMatrix,
+  type HarnessMatrixEntry,
+  HarnessMatrixEntrySchema,
+  HarnessMatrixSchema,
+  type HarnessPermutation,
+  permutationId,
+} from "./core/harness-matrix.js";
 export {
   defineFixtures,
   type LifecycleFixture,
@@ -379,14 +413,11 @@ export {
 export {
   DEFAULT_CLAUDE_MODEL,
   DEFAULT_CODEX_MODEL,
-  DEFAULT_CODEX_REASONING_EFFORT,
   defaultHarnessModel,
   type HarnessModelConfig,
   HarnessModelConfigSchema,
   type ModelCliProvider,
   ModelCliProviderSchema,
-  type ReasoningEffort,
-  ReasoningEffortSchema,
 } from "./core/model-cli.js";
 export {
   type AssertionProgress,
@@ -472,6 +503,11 @@ export {
   type WireEvent,
   WireEventSchema,
 } from "./core/wire.js";
+export {
+  type AntigravityDriverConfig,
+  AntigravityDriverConfigSchema,
+  antigravityDriver,
+} from "./drivers/antigravity/index.js";
 export {
   type ClaudeCodeDriverConfig,
   ClaudeCodeDriverConfigSchema,
@@ -482,6 +518,11 @@ export {
   CodexDriverConfigSchema,
   codexDriver,
 } from "./drivers/codex/index.js";
+export {
+  type CopilotDriverConfig,
+  CopilotDriverConfigSchema,
+  copilotDriver,
+} from "./drivers/copilot/index.js";
 export {
   type CursorDriverConfig,
   CursorDriverConfigSchema,

package/src/orchestrator/comparison-identity.ts

@@ -36,6 +36,7 @@ export const runComparisonIdentity = (metadata: RunMetadata): string =>
     assertionsHash: metadata.provenance?.assertionsHash,
     fixturesHash: metadata.provenance?.fixturesHash,
     profileId: metadata.provenance?.profileId,
+    permutationId: metadata.provenance?.permutationId,
     driverConfigHash: metadata.provenance?.driverConfigHash,
     answererConfigHash: metadata.provenance?.answererConfigHash,
     evaluatorConfigHash: metadata.provenance?.evaluatorConfigHash,

package/src/orchestrator/config.ts

@@ -18,13 +18,16 @@ import {
   type DriverPlugin,
   DriverRefSchema,
   formatZodIssues,
+  HarnessMatrixSchema,
   HarnessModelConfigSchema,
   type LifecycleHooksPlugin,
   lifecycleHooksPluginIssue,
   Registry,
 } from "../core/index.js";
+import { antigravityDriver } from "../drivers/antigravity/index.js";
 import { claudeCodeDriver } from "../drivers/claude-code/index.js";
 import { codexDriver } from "../drivers/codex/index.js";
+import { copilotDriver } from "../drivers/copilot/index.js";
 import { cursorDriver } from "../drivers/cursor/index.js";
 import { mcpProbeDriver } from "../drivers/mcp/index.js";
 import { subprocessDriver } from "../drivers/subprocess/index.js";
@@ -80,6 +83,8 @@ const LuxConfigSchema = z
     subjectRoot: z.string().optional(),
     defaultAnswerer: AnswererRefSchema.default("persona"),
     defaultDriver: DriverRefSchema.optional(),
+    /** Driver, model, and effort permutations every case runs through by default. */
+    matrix: HarnessMatrixSchema.optional(),
     /** Shared provider/model defaults for persona answering and rubric judging. */
     harness: HarnessModelConfigSchema.optional(),
     concurrency: z.number().int().min(1).default(1),
@@ -166,6 +171,7 @@ export const loadLuxConfig = async (cwd: string = process.cwd()): Promise<Resolv
     ...(validated.fixturesRoot ? { fixturesRoot: resolve(root, validated.fixturesRoot) } : {}),
     defaultAnswerer: validated.defaultAnswerer,
     ...(validated.defaultDriver ? { defaultDriver: validated.defaultDriver } : {}),
+    ...(validated.matrix ? { matrix: validated.matrix } : {}),
     ...(validated.harness ? { harness: validated.harness } : {}),
     concurrency: validated.concurrency,
     ...(validated.environment ? { environment: validated.environment } : {}),
@@ -181,6 +187,8 @@ const builtinDrivers: DriverPlugin<never>[] = [
   claudeCodeDriver,
   codexDriver,
   cursorDriver,
+  copilotDriver,
+  antigravityDriver,
   subprocessDriver,
   mcpProbeDriver,
 ];

package/src/orchestrator/doctor.ts

@@ -4,7 +4,15 @@ import { resolve } from "node:path";
 import { glob } from "tinyglobby";
 import { z } from "zod";
 import { RubricConfigSchema } from "../assertions/rubric/index.js";
-import type { EvalCase, Registry } from "../core/index.js";
+import {
+  casePermutations,
+  type EvalCase,
+  expandHarnessMatrix,
+  type HarnessPermutation,
+  permutationDriverRef,
+  permutationId,
+  type Registry,
+} from "../core/index.js";
 import { registryForEvalCase } from "./case-extensions.js";
 import { buildRegistryFromConfig, loadLuxConfig, type ResolvedLuxConfig } from "./config.js";
 import { type DiscoveredCase, type DiscoveryResult, discoverCases } from "./discover.js";
@@ -17,6 +25,12 @@ import {
 } from "./experiment-loader.js";
 import { cliIdentity } from "./experiment-runtime-identity.js";
 import { assertionConfigForSchema } from "./grading.js";
+import {
+  describeHarnessCatalogDrift,
+  harnessCatalogDrift,
+  harnessCatalogProbes,
+  hasHarnessCatalogDrift,
+} from "./harness-catalog-drift.js";
 import { loadEvalCase } from "./loader.js";
 import { validatePluginConfig, validatePluginConfigAsync } from "./validate-plugin-config.js";
 
@@ -62,6 +76,7 @@ export const DoctorOptionsSchema = z
   .object({
     cwd: z.string().min(1),
     campaigns: z.array(z.string()).optional(),
+    static: z.boolean().optional(),
   })
   .strict();
 export type DoctorOptions = z.input<typeof DoctorOptionsSchema>;
@@ -131,7 +146,28 @@ const effectiveDriverReference = (local: unknown, fallback: unknown): unknown =>
   };
 };
 
-const caseDriverReferences = (local: unknown, config: ResolvedLuxConfig): unknown[] => {
+const boundPermutation = (
+  permutation: HarnessPermutation,
+  config: ResolvedLuxConfig,
+  registry: Registry,
+): unknown => {
+  const plugin = registry.getDriver(permutation.driver);
+  if (!plugin) throw new Error(`unknown driver '${permutation.driver}'`);
+  return permutationDriverRef(permutation, config.defaultDriver, plugin);
+};
+
+const caseDriverReferences = (
+  authored: Pick<EvalCase, "driver" | "matrix">,
+  config: ResolvedLuxConfig,
+  registry: Registry,
+): unknown[] => {
+  const local = authored.driver;
+  const permutations = casePermutations(authored, config.matrix);
+  if (permutations.length > 0) {
+    return permutations.map((permutation) =>
+      effectiveDriverReference(local, boundPermutation(permutation, config, registry)),
+    );
+  }
   const fallbacks = [
     config.defaultDriver,
     ...Object.values(config.profiles ?? {}).map((profile) => profile.driver),
@@ -191,6 +227,17 @@ const configuredCliTargets = (config: ResolvedLuxConfig, registry: Registry): Cl
   };
   for (const command of driverCommands(config.defaultDriver, registry))
     add("default driver", command);
+  for (const permutation of expandHarnessMatrix(config.matrix ?? [])) {
+    try {
+      for (const command of driverCommands(
+        boundPermutation(permutation, config, registry),
+        registry,
+      ))
+        add(`matrix '${permutationId(permutation)}' driver`, command);
+    } catch {
+      /* Reported by validateConfiguredMatrix. */
+    }
+  }
   add("default answerer", answererCommand(config.defaultAnswerer));
   if (config.harness) {
     add(
@@ -356,6 +403,32 @@ const validateConfiguredDrivers = (
   }
 };
 
+const validateConfiguredMatrix = (
+  config: ResolvedLuxConfig,
+  registry: Registry,
+  recorder: DoctorRecorder,
+): void => {
+  for (const permutation of expandHarnessMatrix(config.matrix ?? [])) {
+    const id = permutationId(permutation);
+    try {
+      const reference = boundPermutation(permutation, config, registry);
+      const plugin = registry.getDriver(permutation.driver);
+      validatePluginConfig(
+        plugin?.configSchema,
+        rawRefConfig(reference),
+        `matrix '${id}' driver '${permutation.driver}'`,
+      );
+    } catch (error) {
+      recorder.add(
+        `matrix:${id}`,
+        "fail",
+        `matrix permutation '${id}' is not runnable`,
+        firstLine(error),
+      );
+    }
+  }
+};
+
 const validateConfiguredAnswerers = (
   config: ResolvedLuxConfig,
   registry: Registry,
@@ -575,7 +648,7 @@ const inspectCase = async (
   try {
     const authored = await loadEvalCase(discovered.path);
     const caseRegistry = registryForEvalCase(registry, authored);
-    const drivers = caseDriverReferences(authored.driver, config);
+    const drivers = caseDriverReferences(authored, config, caseRegistry);
     const answerer = authored.answerer ?? config.defaultAnswerer;
     validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder, authored.requires);
     validateCaseAnswerer(discovered.id, answerer, caseRegistry, recorder);
@@ -658,6 +731,54 @@ const inspectCliTargets = async (
   }
 };
 
+const catalogDriverReferences = (config: ResolvedLuxConfig): unknown[] => [
+  ...configuredDriverReferences(config).map(([, reference]) => reference),
+  ...(config.matrix ?? []).map((entry) => ({ name: entry.driver, config: entry.config })),
+];
+
+const configuredDriverCommand = (config: ResolvedLuxConfig, driver: string): string | undefined => {
+  for (const reference of catalogDriverReferences(config)) {
+    if (refName(reference) !== driver) continue;
+    const command = (rawRefConfig(reference) as { command?: unknown } | undefined)?.command;
+    if (typeof command === "string") return command;
+  }
+  return undefined;
+};
+
+const inspectHarnessCatalogs = async (
+  config: ResolvedLuxConfig,
+  recorder: DoctorRecorder,
+): Promise<void> => {
+  const configured = new Set(
+    catalogDriverReferences(config)
+      .map(refName)
+      .filter((name): name is string => name !== undefined),
+  );
+  for (const probe of harnessCatalogProbes) {
+    const driver = probe.catalog.driver;
+    if (!configured.has(driver)) continue;
+    const live = await probe.read(configuredDriverCommand(config, driver));
+    if (!live) {
+      recorder.add(
+        `catalog:${driver}`,
+        "warn",
+        `${driver} harness catalog (verified ${probe.catalog.verified}) could not be compared with the installed CLI`,
+        `${probe.source} was unavailable`,
+      );
+      continue;
+    }
+    const drift = harnessCatalogDrift(probe.catalog, live);
+    recorder.add(
+      `catalog:${driver}`,
+      hasHarnessCatalogDrift(drift) ? "warn" : "pass",
+      hasHarnessCatalogDrift(drift)
+        ? `${driver} harness catalog (verified ${probe.catalog.verified}) differs from the installed CLI`
+        : `${driver} harness catalog matches the installed CLI`,
+      describeHarnessCatalogDrift(drift) || undefined,
+    );
+  }
+};
+
 const isBlockingCampaignWarning = (warning: string): boolean =>
   warning.includes("below confidence minSamples=") || warning.includes("cannot attain confidence");
 
@@ -666,6 +787,7 @@ const inspectCampaign = async (
   config: ResolvedLuxConfig,
   registry: Registry,
   recorder: DoctorRecorder,
+  staticChecks: boolean,
 ): Promise<void> => {
   try {
     const loaded = await loadExperimentCampaign(path);
@@ -679,7 +801,9 @@ const inspectCampaign = async (
         placeholders.join(", "),
       );
     }
-    const plan = await planExperimentCampaign({ config, registry, loaded }, mode);
+    const plan = await planExperimentCampaign({ config, registry, loaded }, mode, {
+      static: staticChecks,
+    });
     recorder.campaigns.push({ path, campaignId: loaded.campaign.id, mode, plan });
     recorder.add(
       `campaign:${loaded.campaign.id}`,
@@ -711,6 +835,7 @@ const inspectCampaigns = async (
   config: ResolvedLuxConfig,
   registry: Registry,
   recorder: DoctorRecorder,
+  staticChecks: boolean,
 ): Promise<void> => {
   let paths: string[];
   try {
@@ -733,7 +858,7 @@ const inspectCampaigns = async (
     );
   }
   for (const path of paths) {
-    await inspectCampaign(path, config, registry, recorder);
+    await inspectCampaign(path, config, registry, recorder, staticChecks);
   }
 };
 
@@ -747,14 +872,30 @@ export const doctorProject = async (options: DoctorOptions): Promise<DoctorRepor
   if (!registry) return recorder.finish(config.rootDir);
 
   validateConfiguredDrivers(config, registry, recorder);
+  validateConfiguredMatrix(config, registry, recorder);
   validateConfiguredAnswerers(config, registry, recorder);
   const caseCliTargets = await inspectCases(config, registry, recorder);
-  await inspectCliTargets(
-    [...configuredCliTargets(config, registry), ...caseCliTargets],
+  if (parsedOptions.static) {
+    recorder.add(
+      "static-checks",
+      "warn",
+      "static checks only; installed executables and campaign execution identity were not checked",
+    );
+  } else {
+    await inspectCliTargets(
+      [...configuredCliTargets(config, registry), ...caseCliTargets],
+      config,
+      recorder,
+    );
+    await inspectHarnessCatalogs(config, recorder);
+  }
+  await inspectCampaigns(
+    parsedOptions.campaigns ?? [],
     config,
+    registry,
     recorder,
+    parsedOptions.static ?? false,
   );
-  await inspectCampaigns(parsedOptions.campaigns ?? [], config, registry, recorder);
   return recorder.finish(config.rootDir);
 };
 

package/src/orchestrator/experiment-context.ts

@@ -3,17 +3,18 @@ import type { ResolvedLuxConfig } from "./config.js";
 import type { discoverCases } from "./discover.js";
 import type { LoadedExperimentCampaign } from "./experiment-loader.js";
 
-export type EvaluationCorpus =
+export type CorpusSelection =
   | {
       kind: "agent";
       cases: Awaited<ReturnType<typeof discoverCases>>["cases"];
-      hash: string;
     }
   | {
       kind: "grader";
       hash: string;
     };
 
+export type EvaluationCorpus = CorpusSelection & { hash: string };
+
 export type CampaignExecutionOptions = {
   config: ResolvedLuxConfig;
   registry: Registry;

package/src/orchestrator/experiment-corpus.ts

@@ -13,7 +13,11 @@ import {
 import { assertDisjointAnnotationSplits, selectGraderAnnotations } from "./annotation-loader.js";
 import { discoverCases, selectCases } from "./discover.js";
 import { experimentDriverIdentity } from "./driver-identity.js";
-import type { CampaignExecutionOptions, EvaluationCorpus } from "./experiment-context.js";
+import type {
+  CampaignExecutionOptions,
+  CorpusSelection,
+  EvaluationCorpus,
+} from "./experiment-context.js";
 import { profileOrThrow } from "./experiment-context.js";
 import { cliIdentity, referencedAnswererCli } from "./experiment-runtime-identity.js";
 import { fixtureIdentity } from "./fixture-identity.js";
@@ -252,6 +256,21 @@ export const graderCorpusHash = async (
 export const prepareCorpus = async (
   execution: CampaignExecutionOptions,
 ): Promise<EvaluationCorpus> => {
+  const corpus = await inspectCorpus(execution);
+  if (corpus.kind === "grader") return corpus;
+  return {
+    ...corpus,
+    hash: await agentCorpusHash(
+      execution,
+      corpus.cases,
+      selectorsBySplit(execution.loaded.campaign),
+    ),
+  };
+};
+
+export const inspectCorpus = async (
+  execution: CampaignExecutionOptions,
+): Promise<CorpusSelection> => {
   const campaign = execution.loaded.campaign;
   const splits = selectorsBySplit(campaign);
   if (campaign.evaluation.kind === "grader") {
@@ -344,7 +363,6 @@ export const prepareCorpus = async (
   return {
     kind: "agent",
     cases: discovery.cases,
-    hash: await agentCorpusHash(execution, discovery.cases, splits),
   };
 };
 

package/src/orchestrator/experiment-identity.ts

@@ -253,9 +253,9 @@ const authoredConfigDependencyHash = async (
   }
   return authoredDependencyGraphHash(configRoot, [...entrypoints]);
 };
-const validateAgentCaseProfileDrivers = async (
+export const validateAgentCaseProfileDrivers = async (
   execution: CampaignExecutionOptions,
-  corpus: Extract<EvaluationCorpus, { kind: "agent" }>,
+  corpus: Pick<Extract<EvaluationCorpus, { kind: "agent" }>, "cases">,
 ): Promise<void> => {
   const selected = [
     ...new Map(

package/src/orchestrator/experiment-preflight.ts

@@ -0,0 +1,27 @@
+import { assertValidationArgumentsBound } from "./candidates.js";
+import {
+  type CampaignExecutionOptions,
+  type CorpusSelection,
+  profileOrThrow,
+} from "./experiment-context.js";
+import { inspectCorpus, prepareCorpus } from "./experiment-corpus.js";
+import { evaluationIdentity, validateAgentCaseProfileDrivers } from "./experiment-identity.js";
+
+export const preparePlanningCorpus = async (
+  execution: CampaignExecutionOptions,
+  mode: "experiment" | "optimize",
+  staticChecks: boolean,
+): Promise<CorpusSelection> => {
+  if (!staticChecks) {
+    const corpus = await prepareCorpus(execution);
+    await evaluationIdentity(execution, corpus, mode);
+    return corpus;
+  }
+  const corpus = await inspectCorpus(execution);
+  const campaign = execution.loaded.campaign;
+  for (const profileId of campaign.profiles) profileOrThrow(execution.config, profileId);
+  if (corpus.kind === "agent") await validateAgentCaseProfileDrivers(execution, corpus);
+  for (const validation of campaign.subject.validate)
+    assertValidationArgumentsBound(execution.loaded.subjectRoot, validation.args);
+  return corpus;
+};

package/src/orchestrator/experiment.ts

@@ -48,6 +48,7 @@ import {
   resolvedProfileSnapshot,
   runtimeCliReferences,
 } from "./experiment-identity.js";
+import { preparePlanningCorpus } from "./experiment-preflight.js";
 import {
   type ExperimentEvaluation,
   loadExperimentCandidates,
@@ -182,6 +183,7 @@ const evaluateAcross = async (options: {
 export const planExperimentCampaign = async (
   execution: CampaignExecutionOptions,
   mode: "experiment" | "optimize",
+  options: { static?: boolean } = {},
 ): Promise<ExperimentCampaignPlan> => {
   const campaign = execution.loaded.campaign;
   if (mode === "optimize") {
@@ -192,11 +194,7 @@ export const planExperimentCampaign = async (
   }
   const baseline = await captureSeedCandidate(execution.loaded.subjectRoot, campaign.subject);
   await assertGraderSubject(execution, baseline);
-  const corpus = await prepareCorpus(execution);
-  // Plan is the deterministic preflight for execution. Build the exact same
-  // identity now so profile, behavior-tree, and custom CLI failures happen
-  // before a user approves any model spend.
-  await evaluationIdentity(execution, corpus, mode);
+  const corpus = await preparePlanningCorpus(execution, mode, options.static ?? false);
   const splitSelectors = selectorsBySplit(campaign);
   const graderAnnotations = execution.loaded.graderAnnotations?.annotations ?? [];
   const splits = splitSelectors.map(({ name, selector }) => ({

package/src/orchestrator/harness-catalog-drift.ts

@@ -0,0 +1,132 @@
+import { readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { z } from "zod";
+import {
+  type ClosedHarnessCatalog,
+  codexCatalog,
+  copilotCatalog,
+  exec,
+  type ReasoningEffort,
+  tryParseJson,
+} from "../core/index.js";
+
+export type LiveHarnessModel = {
+  id: string;
+  efforts?: readonly ReasoningEffort[] | undefined;
+};
+
+export type HarnessCatalogDrift = {
+  unlisted: string[];
+  mismatched: string[];
+  unavailable: string[];
+};
+
+export const harnessCatalogDrift = (
+  catalog: ClosedHarnessCatalog,
+  live: readonly LiveHarnessModel[],
+): HarnessCatalogDrift => {
+  const liveIds = new Set(live.map((model) => model.id));
+  return {
+    unlisted: live.filter((model) => !(model.id in catalog.models)).map((model) => model.id),
+    mismatched: live
+      .filter(
+        (model) =>
+          model.efforts !== undefined &&
+          model.id in catalog.models &&
+          model.efforts.join(",") !== catalog.models[model.id]?.join(","),
+      )
+      .map((model) => `${model.id} (${model.efforts?.join(", ") || "no effort"})`),
+    unavailable: Object.keys(catalog.models).filter((id) => !liveIds.has(id)),
+  };
+};
+
+export const hasHarnessCatalogDrift = (drift: HarnessCatalogDrift): boolean =>
+  drift.unlisted.length > 0 || drift.mismatched.length > 0;
+
+export const describeHarnessCatalogDrift = (drift: HarnessCatalogDrift): string =>
+  [
+    drift.unlisted.length > 0 ? `not in catalog: ${drift.unlisted.join(", ")}` : undefined,
+    drift.mismatched.length > 0 ? `efforts differ: ${drift.mismatched.join("; ")}` : undefined,
+    drift.unavailable.length > 0 ? `not offered here: ${drift.unavailable.join(", ")}` : undefined,
+  ]
+    .filter((part) => part !== undefined)
+    .join(". ");
+
+const CodexModelsCacheSchema = z.object({
+  models: z.array(
+    z.looseObject({
+      slug: z.string(),
+      visibility: z.string().optional(),
+      supported_reasoning_levels: z.array(z.looseObject({ effort: z.string() })).default([]),
+    }),
+  ),
+});
+
+const codexModelsCachePath = (env: NodeJS.ProcessEnv = process.env): string =>
+  join(env.CODEX_HOME ?? join(homedir(), ".codex"), "models_cache.json");
+
+export const parseCodexModelsCache = (text: string): LiveHarnessModel[] | undefined => {
+  const parsed = CodexModelsCacheSchema.safeParse(tryParseJson(text));
+  if (!parsed.success) return undefined;
+  return parsed.data.models
+    .filter((model) => model.visibility !== "hide")
+    .map((model) => ({
+      id: model.slug,
+      efforts: model.supported_reasoning_levels.map((level) => level.effort as ReasoningEffort),
+    }));
+};
+
+const readCodexLiveModels = async (
+  path = codexModelsCachePath(),
+): Promise<LiveHarnessModel[] | undefined> => {
+  try {
+    return parseCodexModelsCache(await readFile(path, "utf8"));
+  } catch {
+    return undefined;
+  }
+};
+
+export const parseCopilotModelValues = (helpText: string): LiveHarnessModel[] | undefined => {
+  const lines = helpText.split("\n");
+  const start = lines.findIndex((line) => /^\s*`model`:/.test(line));
+  if (start < 0) return undefined;
+  const models: LiveHarnessModel[] = [];
+  for (const line of lines.slice(start + 1)) {
+    const match = /^\s*-\s*"([^"]+)"\s*$/.exec(line);
+    if (!match?.[1]) break;
+    models.push({ id: match[1] });
+  }
+  return models.length > 0 ? models : undefined;
+};
+
+const readCopilotLiveModels = async (
+  command = "copilot",
+): Promise<LiveHarnessModel[] | undefined> => {
+  try {
+    const result = await exec(command, ["help", "config"], { timeoutMs: 15_000 });
+    if (result.exitCode !== 0 || result.timedOut) return undefined;
+    return parseCopilotModelValues(result.stdout);
+  } catch {
+    return undefined;
+  }
+};
+
+export type HarnessCatalogProbe = {
+  catalog: ClosedHarnessCatalog;
+  source: string;
+  read: (command: string | undefined) => Promise<LiveHarnessModel[] | undefined>;
+};
+
+export const harnessCatalogProbes: readonly HarnessCatalogProbe[] = [
+  {
+    catalog: codexCatalog,
+    source: "models_cache.json",
+    read: () => readCodexLiveModels(),
+  },
+  {
+    catalog: copilotCatalog,
+    source: "help config",
+    read: (command) => readCopilotLiveModels(command),
+  },
+];