@fre4x/benchmark 1.1.0-beta.0 → 1.1.0-beta.3

package/dist/index.js

@@ -3223,8 +3223,8 @@ var require_utils = __commonJS({
       }
       return ind;
     }
-    function removeDotSegments(path2) {
-      let input = path2;
+    function removeDotSegments(path4) {
+      let input = path4;
       const output = [];
       let nextSlash = -1;
       let len = 0;
@@ -3423,8 +3423,8 @@ var require_schemes = __commonJS({
         wsComponent.secure = void 0;
       }
       if (wsComponent.resourceName) {
-        const [path2, query] = wsComponent.resourceName.split("?");
-        wsComponent.path = path2 && path2 !== "/" ? path2 : void 0;
+        const [path4, query] = wsComponent.resourceName.split("?");
+        wsComponent.path = path4 && path4 !== "/" ? path4 : void 0;
         wsComponent.query = query;
         wsComponent.resourceName = void 0;
       }
@@ -7878,10 +7878,10 @@ function mergeDefs(...defs) {
 function cloneDef(schema) {
   return mergeDefs(schema._zod.def);
 }
-function getElementAtPath(obj, path2) {
-  if (!path2)
+function getElementAtPath(obj, path4) {
+  if (!path4)
     return obj;
-  return path2.reduce((acc, key) => acc?.[key], obj);
+  return path4.reduce((acc, key) => acc?.[key], obj);
 }
 function promiseAllObject(promisesObj) {
   const keys = Object.keys(promisesObj);
@@ -8264,11 +8264,11 @@ function aborted(x, startIndex = 0) {
   }
   return false;
 }
-function prefixIssues(path2, issues) {
+function prefixIssues(path4, issues) {
   return issues.map((iss) => {
     var _a2;
     (_a2 = iss).path ?? (_a2.path = []);
-    iss.path.unshift(path2);
+    iss.path.unshift(path4);
     return iss;
   });
 }
@@ -8451,7 +8451,7 @@ function formatError(error48, mapper = (issue2) => issue2.message) {
 }
 function treeifyError(error48, mapper = (issue2) => issue2.message) {
   const result = { errors: [] };
-  const processError = (error49, path2 = []) => {
+  const processError = (error49, path4 = []) => {
     var _a2, _b;
     for (const issue2 of error49.issues) {
       if (issue2.code === "invalid_union" && issue2.errors.length) {
@@ -8461,7 +8461,7 @@ function treeifyError(error48, mapper = (issue2) => issue2.message) {
       } else if (issue2.code === "invalid_element") {
         processError({ issues: issue2.issues }, issue2.path);
       } else {
-        const fullpath = [...path2, ...issue2.path];
+        const fullpath = [...path4, ...issue2.path];
         if (fullpath.length === 0) {
           result.errors.push(mapper(issue2));
           continue;
@@ -8493,8 +8493,8 @@ function treeifyError(error48, mapper = (issue2) => issue2.message) {
 }
 function toDotPath(_path) {
   const segs = [];
-  const path2 = _path.map((seg) => typeof seg === "object" ? seg.key : seg);
-  for (const seg of path2) {
+  const path4 = _path.map((seg) => typeof seg === "object" ? seg.key : seg);
+  for (const seg of path4) {
     if (typeof seg === "number")
       segs.push(`[${seg}]`);
     else if (typeof seg === "symbol")
@@ -20471,13 +20471,13 @@ function resolveRef(ref, ctx) {
   if (!ref.startsWith("#")) {
     throw new Error("External $ref is not supported, only local refs (#/...) are allowed");
   }
-  const path2 = ref.slice(1).split("/").filter(Boolean);
-  if (path2.length === 0) {
+  const path4 = ref.slice(1).split("/").filter(Boolean);
+  if (path4.length === 0) {
     return ctx.rootSchema;
   }
   const defsKey = ctx.version === "draft-2020-12" ? "$defs" : "definitions";
-  if (path2[0] === defsKey) {
-    const key = path2[1];
+  if (path4[0] === defsKey) {
+    const key = path4[1];
     if (!key || !ctx.defs[key]) {
       throw new Error(`Reference not found: ${ref}`);
     }
@@ -21269,8 +21269,8 @@ function getErrorMap2() {
 
 // ../node_modules/zod/v3/helpers/parseUtil.js
 var makeIssue = (params) => {
-  const { data, path: path2, errorMaps, issueData } = params;
-  const fullPath = [...path2, ...issueData.path || []];
+  const { data, path: path4, errorMaps, issueData } = params;
+  const fullPath = [...path4, ...issueData.path || []];
   const fullIssue = {
     ...issueData,
     path: fullPath
@@ -21385,11 +21385,11 @@ var errorUtil;
 
 // ../node_modules/zod/v3/types.js
 var ParseInputLazyPath = class {
-  constructor(parent, value, path2, key) {
+  constructor(parent, value, path4, key) {
     this._cachedPath = [];
     this.parent = parent;
     this.data = value;
-    this._path = path2;
+    this._path = path4;
     this._key = key;
   }
   get path() {
@@ -30432,45 +30432,161 @@ var StdioServerTransport = class {
   }
 };
 
-// src/gaia.ts
-import { readFile } from "node:fs/promises";
+// src/attemptStore.ts
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
 
 // src/types.ts
-var challengeSummarySchema = external_exports.object({
-  challenge_id: external_exports.string(),
-  benchmark_id: external_exports.string(),
-  title: external_exports.string(),
-  description: external_exports.string(),
-  version: external_exports.string(),
-  total_questions: external_exports.number().int().min(1),
-  difficulty_levels: external_exports.array(external_exports.number().int().min(1).max(3)),
-  supports_assets: external_exports.boolean()
-});
+var primitiveValueSchema = external_exports.union([external_exports.string(), external_exports.number(), external_exports.boolean()]);
+var benchmarkFamilySchema = external_exports.enum(["code", "web", "os"]);
+var runnerKindSchema = external_exports.enum([
+  "code_runner",
+  "browser_runner",
+  "os_runner"
+]);
+var responseFormatSchema = external_exports.enum(["text", "json"]);
+var assetEncodingSchema = external_exports.enum(["utf8", "base64"]);
+var assetTransportSchema = external_exports.enum(["inline"]);
+var assetMaterializationSchema = external_exports.enum(["inline", "workspace_file"]);
+var checkerKindSchema = external_exports.enum([
+  "exact_text",
+  "normalized_text",
+  "regex_match",
+  "contains_all_text",
+  "json_field_equals",
+  "runner_fact_equals",
+  "runner_log_contains_text"
+]);
+var catalogModeSchema = external_exports.enum([
+  "builtin",
+  "file",
+  "remote_url",
+  "remote_cache",
+  "mock"
+]);
 var assetSchema = external_exports.object({
   asset_id: external_exports.string(),
   name: external_exports.string(),
   mime_type: external_exports.string(),
-  encoding: external_exports.enum(["utf8", "base64"]).default("utf8"),
+  transport: assetTransportSchema.default("inline"),
+  materialization: assetMaterializationSchema.default("inline"),
+  encoding: assetEncodingSchema.default("utf8"),
   content: external_exports.string(),
   description: external_exports.string().optional()
 });
-var questionSchema = external_exports.object({
-  question_id: external_exports.string(),
+var checkerBaseSchema = external_exports.object({
+  checker_id: external_exports.string()
+});
+var exactTextCheckerSchema = checkerBaseSchema.extend({
+  kind: external_exports.literal("exact_text"),
+  expected: external_exports.string(),
+  case_sensitive: external_exports.boolean().default(true)
+});
+var normalizedTextCheckerSchema = checkerBaseSchema.extend({
+  kind: external_exports.literal("normalized_text"),
+  expected: external_exports.string()
+});
+var regexMatchCheckerSchema = checkerBaseSchema.extend({
+  kind: external_exports.literal("regex_match"),
+  pattern: external_exports.string(),
+  flags: external_exports.string().optional()
+});
+var containsAllTextCheckerSchema = checkerBaseSchema.extend({
+  kind: external_exports.literal("contains_all_text"),
+  terms: external_exports.array(external_exports.string()).min(1)
+});
+var jsonFieldEqualsCheckerSchema = checkerBaseSchema.extend({
+  kind: external_exports.literal("json_field_equals"),
+  field: external_exports.string(),
+  expected: primitiveValueSchema
+});
+var runnerFactEqualsCheckerSchema = checkerBaseSchema.extend({
+  kind: external_exports.literal("runner_fact_equals"),
+  fact: external_exports.string(),
+  expected: primitiveValueSchema
+});
+var runnerLogContainsTextCheckerSchema = checkerBaseSchema.extend({
+  kind: external_exports.literal("runner_log_contains_text"),
+  terms: external_exports.array(external_exports.string()).min(1)
+});
+var checkerSchema = external_exports.discriminatedUnion("kind", [
+  exactTextCheckerSchema,
+  normalizedTextCheckerSchema,
+  regexMatchCheckerSchema,
+  containsAllTextCheckerSchema,
+  jsonFieldEqualsCheckerSchema,
+  runnerFactEqualsCheckerSchema,
+  runnerLogContainsTextCheckerSchema
+]);
+var runnerArtifactKindSchema = external_exports.enum([
+  "materialized_asset",
+  "submission",
+  "runner_report"
+]);
+var taskSchema = external_exports.object({
+  task_id: external_exports.string(),
+  title: external_exports.string(),
   prompt: external_exports.string(),
-  expected_answer: external_exports.string(),
-  grading_mode: external_exports.enum(["exact", "normalized"]).default("normalized"),
+  response_format: responseFormatSchema.default("text"),
   difficulty: external_exports.number().int().min(1).max(3),
-  assets: external_exports.array(assetSchema).default([])
+  assets: external_exports.array(assetSchema).default([]),
+  checkers: external_exports.array(checkerSchema).min(1)
 });
-var challengeDefinitionSchema = challengeSummarySchema.extend({
-  questions: external_exports.array(questionSchema).min(1)
+var challengeDefinitionSchema = external_exports.object({
+  challenge_id: external_exports.string(),
+  benchmark_id: external_exports.string(),
+  family: benchmarkFamilySchema,
+  runner_kind: runnerKindSchema,
+  title: external_exports.string(),
+  description: external_exports.string(),
+  version: external_exports.string(),
+  source: external_exports.string().default("external"),
+  tasks: external_exports.array(taskSchema).min(1)
 });
-var submissionRecordSchema = external_exports.object({
-  question_id: external_exports.string(),
+var challengeSummarySchema = external_exports.object({
+  challenge_id: external_exports.string(),
+  benchmark_id: external_exports.string(),
+  family: benchmarkFamilySchema,
+  runner_kind: runnerKindSchema,
+  title: external_exports.string(),
+  description: external_exports.string(),
+  version: external_exports.string(),
+  source: external_exports.string(),
+  total_tasks: external_exports.number().int().min(1),
+  difficulty_levels: external_exports.array(external_exports.number().int().min(1).max(3)),
+  supports_assets: external_exports.boolean(),
+  checker_kinds: external_exports.array(checkerKindSchema).min(1)
+});
+var checkerEvidenceSchema = external_exports.object({
+  checker_id: external_exports.string(),
+  kind: checkerKindSchema,
+  passed: external_exports.boolean(),
+  message: external_exports.string()
+});
+var runnerExecutionArtifactSchema = external_exports.object({
+  artifact_id: external_exports.string(),
+  kind: runnerArtifactKindSchema,
+  path: external_exports.string(),
+  mime_type: external_exports.string(),
+  description: external_exports.string().optional()
+});
+var runnerExecutionSchema = external_exports.object({
+  runner_kind: runnerKindSchema,
+  workspace_dir: external_exports.string(),
+  logs: external_exports.array(external_exports.string()),
+  facts: external_exports.record(external_exports.string(), primitiveValueSchema),
+  artifacts: external_exports.array(runnerExecutionArtifactSchema)
+});
+var evaluationRecordSchema = external_exports.object({
+  task_id: external_exports.string(),
   solution: external_exports.string(),
   accepted: external_exports.boolean(),
   score_delta: external_exports.number().int().min(0),
-  submitted_at: external_exports.string()
+  submitted_at: external_exports.string(),
+  evidence: external_exports.array(checkerEvidenceSchema),
+  execution: runnerExecutionSchema
 });
 var attemptStatusSchema = external_exports.enum(["active", "completed", "cancelled"]);
 var persistedAttemptSchema = external_exports.object({
@@ -30478,17 +30594,18 @@ var persistedAttemptSchema = external_exports.object({
   benchmark_id: external_exports.string(),
   challenge_id: external_exports.string(),
   challenge_title: external_exports.string(),
+  family: benchmarkFamilySchema,
+  runner_kind: runnerKindSchema,
   dataset_version: external_exports.string(),
+  source: external_exports.string(),
   status: attemptStatusSchema,
-  current_question_index: external_exports.number().int().min(0),
-  total_questions: external_exports.number().int().min(1),
+  revision: external_exports.number().int().min(1),
+  current_task_index: external_exports.number().int().min(0),
+  total_tasks: external_exports.number().int().min(1),
   started_at: external_exports.string(),
   updated_at: external_exports.string(),
-  questions: external_exports.array(questionSchema),
-  submissions: external_exports.array(submissionRecordSchema)
-});
-var persistedAttemptsFileSchema = external_exports.object({
-  attempts: external_exports.array(persistedAttemptSchema)
+  tasks: external_exports.array(taskSchema),
+  evaluations: external_exports.array(evaluationRecordSchema)
 });
 var listChallengesInputSchema = paginationSchema;
 var startChallengeInputSchema = external_exports.object({
@@ -30496,210 +30613,773 @@ var startChallengeInputSchema = external_exports.object({
 });
 var submitSolutionInputSchema = external_exports.object({
   attempt_id: external_exports.string().min(1).describe("Attempt id"),
-  solution: external_exports.string().min(1).describe("Answer text")
+  solution: external_exports.string().min(1).describe("Answer text or JSON")
 });
 var getAssetInputSchema = external_exports.object({
   attempt_id: external_exports.string().min(1).describe("Attempt id"),
   asset_id: external_exports.string().min(1).describe("Asset id")
 });
-var getAttemptInputSchema = external_exports.object({
+var getAttemptInputSchema = paginationSchema.extend({
   attempt_id: external_exports.string().min(1).describe("Attempt id")
 });
 var cancelAttemptInputSchema = external_exports.object({
   attempt_id: external_exports.string().min(1).describe("Attempt id")
 });
-
-// src/gaia.ts
-var DEFAULT_GAIA_CHALLENGES = [
-  {
-    challenge_id: "gaia_public_validation_sample",
-    benchmark_id: "gaia",
-    title: "GAIA Public Validation Sample",
-    description: "A GAIA-style multi-step sample with asset reading and exact grading.",
-    version: "sample-v1",
-    total_questions: 2,
-    difficulty_levels: [1, 2],
-    supports_assets: true,
-    questions: [
-      {
-        question_id: "gaia-q1",
-        prompt: "Use the attached CSV and return only the full name with the highest total score.",
-        expected_answer: "Mina Park",
-        grading_mode: "normalized",
-        difficulty: 1,
-        assets: [
-          {
-            asset_id: "gaia-q1-scores",
-            name: "lab_scores.csv",
-            mime_type: "text/csv",
-            encoding: "utf8",
-            description: "Score table for three candidates.",
-            content: [
-              "name,logic,search,tool_use",
-              "Mina Park,8,9,10",
-              "Jon Ortiz,9,7,8",
-              "Asha Singh,7,8,9"
-            ].join("\n")
-          }
-        ]
-      },
-      {
-        question_id: "gaia-q2",
-        prompt: "Read the attached briefing and reply with the two-word checkpoint callsign in lowercase.",
-        expected_answer: "ember clock",
-        grading_mode: "normalized",
-        difficulty: 2,
-        assets: [
-          {
-            asset_id: "gaia-q2-brief",
-            name: "mission_brief.txt",
-            mime_type: "text/plain",
-            encoding: "utf8",
-            description: "Briefing memo with a repeated callsign.",
-            content: [
-              "Field memo",
-              "",
-              "Checkpoint callsign: ember clock.",
-              "All couriers must repeat ember clock before hand-off.",
-              "The phrase ember clock remains active until rotation."
-            ].join("\n")
-          }
-        ]
-      }
-    ]
-  }
-];
-async function loadChallengesFromFile(filePath) {
-  const raw = await readFile(filePath, "utf8");
-  const parsed = JSON.parse(raw);
-  if (!Array.isArray(parsed)) {
-    throw new Error("GAIA data file must contain a JSON array.");
-  }
-  return parsed.map((entry) => challengeDefinitionSchema.parse(entry));
-}
-var GaiaAdapter = class {
-  benchmarkId = "gaia";
-  dataFile = process.env.BENCHMARK_GAIA_DATA_FILE;
-  async getChallenges() {
-    if (!this.dataFile) {
-      return DEFAULT_GAIA_CHALLENGES;
-    }
-    return await loadChallengesFromFile(this.dataFile);
-  }
-  async listChallenges() {
-    const challenges = await this.getChallenges();
-    return challenges.map(
-      ({ questions: _questions, ...summary }) => summary
-    );
-  }
-  async getChallengeDefinition(challengeId) {
-    const challenges = await this.getChallenges();
-    return challenges.find(
-      (challenge) => challenge.challenge_id === challengeId
-    ) ?? null;
-  }
-};
-function createGaiaAdapter() {
-  return new GaiaAdapter();
+var getCatalogStatusInputSchema = external_exports.object({});
+var syncCatalogInputSchema = external_exports.object({
+  force: external_exports.boolean().default(false).describe("Ignore cache freshness and refetch")
+});
+function summarizeChallenge(challenge) {
+  const difficultyLevels = [
+    ...new Set(challenge.tasks.map((task) => task.difficulty))
+  ].sort((left, right) => left - right);
+  const checkerKinds = [
+    ...new Set(
+      challenge.tasks.flatMap(
+        (task) => task.checkers.map((checker) => checker.kind)
+      )
+    )
+  ];
+  return {
+    challenge_id: challenge.challenge_id,
+    benchmark_id: challenge.benchmark_id,
+    family: challenge.family,
+    runner_kind: challenge.runner_kind,
+    title: challenge.title,
+    description: challenge.description,
+    version: challenge.version,
+    source: challenge.source,
+    total_tasks: challenge.tasks.length,
+    difficulty_levels: difficultyLevels,
+    supports_assets: challenge.tasks.some((task) => task.assets.length > 0),
+    checker_kinds: checkerKinds
+  };
 }
 
 // src/attemptStore.ts
-import { mkdir, readFile as readFile2, rm, writeFile } from "node:fs/promises";
-import { randomUUID } from "node:crypto";
-import os from "node:os";
-import path from "node:path";
 function nowIso() {
   return (/* @__PURE__ */ new Date()).toISOString();
 }
 function getStateDir() {
   return process.env.BENCHMARK_STATE_DIR ?? path.join(os.tmpdir(), "fre4x-benchmark-state");
 }
-function getStateFilePath() {
-  return path.join(getStateDir(), "attempts.json");
+function getAttemptWorkspaceDir(attemptId) {
+  return path.join(getStateDir(), "artifacts", attemptId);
+}
+function getAttemptsDir() {
+  return path.join(getStateDir(), "attempts");
+}
+function getLocksDir() {
+  return path.join(getStateDir(), "locks");
+}
+function getAttemptFilePath(attemptId) {
+  return path.join(getAttemptsDir(), `${attemptId}.json`);
+}
+function getLockDirPath(attemptId) {
+  return path.join(getLocksDir(), attemptId);
 }
-async function readStateFile() {
-  const filePath = getStateFilePath();
+async function sleep(milliseconds) {
+  await new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+async function ensureStateDirs() {
+  await mkdir(getAttemptsDir(), { recursive: true });
+  await mkdir(getLocksDir(), { recursive: true });
+}
+async function readAttemptFile(attemptId) {
+  const filePath = getAttemptFilePath(attemptId);
   try {
-    const raw = await readFile2(filePath, "utf8");
+    const raw = await readFile(filePath, "utf8");
     const parsed = JSON.parse(raw);
-    return persistedAttemptsFileSchema.parse(parsed);
+    return persistedAttemptSchema.parse(parsed);
   } catch (error48) {
     if (error48.code === "ENOENT") {
-      return { attempts: [] };
+      return null;
     }
     throw error48;
   }
 }
-async function writeStateFile(state) {
-  const dir = getStateDir();
-  await mkdir(dir, { recursive: true });
-  await writeFile(getStateFilePath(), JSON.stringify(state, null, 2), "utf8");
+async function writeAttemptFile(attempt) {
+  await ensureStateDirs();
+  const filePath = getAttemptFilePath(attempt.attempt_id);
+  const tempPath = `${filePath}.${randomUUID()}.tmp`;
+  const serialized = JSON.stringify(attempt, null, 2);
+  await writeFile(tempPath, serialized, "utf8");
+  await rename(tempPath, filePath);
+}
+async function withAttemptLock(attemptId, operation) {
+  await ensureStateDirs();
+  const lockDir = getLockDirPath(attemptId);
+  let retries = 0;
+  while (true) {
+    try {
+      await mkdir(lockDir);
+      break;
+    } catch (error48) {
+      const code = error48.code;
+      if (code !== "EEXIST") {
+        throw error48;
+      }
+      retries += 1;
+      if (retries > 200) {
+        throw new Error(`Attempt ${attemptId} is locked for too long.`);
+      }
+      await sleep(10);
+    }
+  }
+  try {
+    return await operation();
+  } finally {
+    await rm(lockDir, { recursive: true, force: true });
+  }
 }
 async function createAttempt(challenge) {
-  const state = await readStateFile();
   const timestamp = nowIso();
   const attempt = {
     attempt_id: `attempt_${randomUUID()}`,
     benchmark_id: challenge.benchmark_id,
     challenge_id: challenge.challenge_id,
     challenge_title: challenge.title,
+    family: challenge.family,
+    runner_kind: challenge.runner_kind,
     dataset_version: challenge.version,
+    source: challenge.source,
     status: "active",
-    current_question_index: 0,
-    total_questions: challenge.questions.length,
+    revision: 1,
+    current_task_index: 0,
+    total_tasks: challenge.tasks.length,
     started_at: timestamp,
     updated_at: timestamp,
-    questions: challenge.questions,
-    submissions: []
+    tasks: challenge.tasks,
+    evaluations: []
   };
-  state.attempts.push(attempt);
-  await writeStateFile(state);
+  await writeAttemptFile(attempt);
+  await mkdir(getAttemptWorkspaceDir(attempt.attempt_id), {
+    recursive: true
+  });
   return attempt;
 }
 async function getAttempt(attemptId) {
-  const state = await readStateFile();
-  return state.attempts.find((attempt) => attempt.attempt_id === attemptId) ?? null;
+  return await readAttemptFile(attemptId);
 }
-async function saveAttempt(updated) {
-  const state = await readStateFile();
-  const index = state.attempts.findIndex(
-    (attempt) => attempt.attempt_id === updated.attempt_id
-  );
-  if (index === -1) {
-    throw new Error(`Attempt ${updated.attempt_id} not found.`);
+async function appendEvaluation(attemptId, expectedTaskId, evaluation) {
+  return await withAttemptLock(attemptId, async () => {
+    const attempt = await readAttemptFile(attemptId);
+    if (!attempt) {
+      throw new Error(`Attempt ${attemptId} not found.`);
+    }
+    if (attempt.status !== "active") {
+      throw new Error(`Attempt ${attemptId} is ${attempt.status}.`);
+    }
+    const currentTask = attempt.tasks[attempt.current_task_index];
+    if (!currentTask) {
+      throw new Error(`Attempt ${attemptId} has no current task.`);
+    }
+    if (currentTask.task_id !== expectedTaskId) {
+      throw new Error(
+        `Attempt ${attemptId} advanced to ${currentTask.task_id} before this submission was stored.`
+      );
+    }
+    const nextAttempt = {
+      ...attempt,
+      revision: attempt.revision + 1,
+      evaluations: [...attempt.evaluations, evaluation],
+      current_task_index: attempt.current_task_index + 1,
+      updated_at: nowIso()
+    };
+    if (nextAttempt.current_task_index >= nextAttempt.total_tasks) {
+      nextAttempt.status = "completed";
+    }
+    await writeAttemptFile(nextAttempt);
+    return nextAttempt;
+  });
+}
+async function cancelAttempt(attemptId) {
+  return await withAttemptLock(attemptId, async () => {
+    const attempt = await readAttemptFile(attemptId);
+    if (!attempt) {
+      throw new Error(`Attempt ${attemptId} not found.`);
+    }
+    const nextAttempt = {
+      ...attempt,
+      revision: attempt.revision + 1,
+      status: "cancelled",
+      updated_at: nowIso()
+    };
+    await writeAttemptFile(nextAttempt);
+    return nextAttempt;
+  });
+}
+
+// src/catalog.ts
+import os2 from "node:os";
+import path2 from "node:path";
+import { fileURLToPath } from "node:url";
+import { mkdir as mkdir2, readFile as readFile2, rename as rename2, rm as rm2, stat, writeFile as writeFile2 } from "node:fs/promises";
+import { randomUUID as randomUUID2 } from "node:crypto";
+
+// src/mock.ts
+function isMockEnabled() {
+  return process.env.MOCK === "true" || process.env.BENCHMARK_MOCK === "true";
+}
+var IS_MOCK = isMockEnabled();
+
+// src/catalog.ts
+var cachedCatalogSchema = external_exports.object({
+  url: external_exports.string().url(),
+  fetched_at: external_exports.string(),
+  challenges: external_exports.array(challengeDefinitionSchema)
+});
+var builtinChallengesPromise = null;
+function getConfiguredCatalogFile() {
+  return process.env.BENCHMARK_CATALOG_FILE ?? process.env.BENCHMARK_GAIA_DATA_FILE ?? null;
+}
+function getConfiguredCatalogUrl() {
+  return process.env.BENCHMARK_CATALOG_URL ?? null;
+}
+function getStateDir2() {
+  return process.env.BENCHMARK_STATE_DIR ?? path2.join(os2.tmpdir(), "fre4x-benchmark-state");
+}
+function getCacheDir() {
+  return process.env.BENCHMARK_CACHE_DIR ?? path2.join(getStateDir2(), "catalog-cache");
+}
+function getCacheFilePath() {
+  return path2.join(getCacheDir(), "catalog.json");
+}
+function getCacheTtlSeconds() {
+  const raw = process.env.BENCHMARK_CACHE_TTL_SECONDS;
+  if (!raw) {
+    return 3600;
+  }
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : 3600;
+}
+async function ensureCacheDir() {
+  await mkdir2(getCacheDir(), { recursive: true });
+}
+async function loadChallengesFromFile(filePath) {
+  const raw = await readFile2(filePath, "utf8");
+  const parsed = JSON.parse(raw);
+  if (!Array.isArray(parsed)) {
+    throw new Error("Benchmark catalog file must contain a JSON array.");
   }
-  state.attempts[index] = updated;
-  await writeStateFile(state);
+  return parsed.map((entry) => challengeDefinitionSchema.parse(entry));
 }
-async function appendSubmission(attempt, submission) {
-  const nextAttempt = {
-    ...attempt,
-    submissions: [...attempt.submissions, submission],
-    current_question_index: attempt.current_question_index + 1,
-    updated_at: nowIso()
+async function getBuiltinChallenges() {
+  builtinChallengesPromise ??= (async () => {
+    const candidatePaths = [
+      fileURLToPath(new URL("./fallback-catalog.json", import.meta.url)),
+      fileURLToPath(
+        new URL("../catalogs/expanded-catalog.json", import.meta.url)
+      )
+    ];
+    for (const candidatePath of candidatePaths) {
+      try {
+        return await loadChallengesFromFile(candidatePath);
+      } catch (error48) {
+        if (error48.code === "ENOENT") {
+          continue;
+        }
+        throw error48;
+      }
+    }
+    throw new Error("No bundled fallback catalog was found.");
+  })();
+  return await builtinChallengesPromise;
+}
+async function mergeWithBuiltins(externalChallenges) {
+  const builtinChallenges = await getBuiltinChallenges();
+  const merged = /* @__PURE__ */ new Map();
+  for (const challenge of builtinChallenges) {
+    merged.set(challenge.challenge_id, challenge);
+  }
+  for (const challenge of externalChallenges) {
+    merged.set(challenge.challenge_id, challenge);
+  }
+  return [...merged.values()];
+}
+async function readCachedCatalog() {
+  try {
+    const raw = await readFile2(getCacheFilePath(), "utf8");
+    const parsed = JSON.parse(raw);
+    return cachedCatalogSchema.parse(parsed);
+  } catch (error48) {
+    if (error48.code === "ENOENT") {
+      return null;
+    }
+    throw error48;
+  }
+}
+async function writeCachedCatalog(url2, challenges) {
+  await ensureCacheDir();
+  const cache = {
+    url: url2,
+    fetched_at: (/* @__PURE__ */ new Date()).toISOString(),
+    challenges
   };
-  if (nextAttempt.current_question_index >= nextAttempt.total_questions) {
-    nextAttempt.status = "completed";
+  const filePath = getCacheFilePath();
+  const tempPath = `${filePath}.${randomUUID2()}.tmp`;
+  await writeFile2(tempPath, JSON.stringify(cache, null, 2), "utf8");
+  await rename2(tempPath, filePath);
+  return cache;
+}
+function getAgeSeconds(timestamp) {
+  const now = Date.now();
+  const value = Date.parse(timestamp);
+  if (!Number.isFinite(value)) {
+    return Number.MAX_SAFE_INTEGER;
+  }
+  return Math.max(0, Math.floor((now - value) / 1e3));
+}
+async function isCacheFresh(cache) {
+  if (!cache) {
+    return false;
+  }
+  return getAgeSeconds(cache.fetched_at) <= getCacheTtlSeconds();
+}
+async function fetchChallengesFromUrl(url2) {
+  const response = await fetch(url2, {
+    headers: {
+      accept: "application/json"
+    }
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Catalog fetch failed with status ${response.status} ${response.statusText}.`
+    );
+  }
+  const parsed = await response.json();
+  if (!Array.isArray(parsed)) {
+    throw new Error("Remote catalog must contain a JSON array.");
+  }
+  return parsed.map((entry) => challengeDefinitionSchema.parse(entry));
+}
+async function loadRemoteCatalog(forceRefresh) {
+  const configuredUrl = getConfiguredCatalogUrl();
+  if (!configuredUrl) {
+    return {
+      challenges: await getBuiltinChallenges(),
+      mode: "builtin",
+      lastError: null
+    };
+  }
+  const cache = await readCachedCatalog();
+  if (cache && !forceRefresh && await isCacheFresh(cache)) {
+    return {
+      challenges: await mergeWithBuiltins(cache.challenges),
+      mode: "remote_cache",
+      lastError: null
+    };
+  }
+  try {
+    const remoteChallenges = await fetchChallengesFromUrl(configuredUrl);
+    await writeCachedCatalog(configuredUrl, remoteChallenges);
+    return {
+      challenges: await mergeWithBuiltins(remoteChallenges),
+      mode: "remote_url",
+      lastError: null
+    };
+  } catch (error48) {
+    if (cache) {
+      const message = error48 instanceof Error ? error48.message : "Remote catalog fetch failed.";
+      return {
+        challenges: await mergeWithBuiltins(cache.challenges),
+        mode: "remote_cache",
+        lastError: message
+      };
+    }
+    throw error48;
   }
-  await saveAttempt(nextAttempt);
-  return nextAttempt;
 }
-async function cancelAttempt(attempt) {
-  const nextAttempt = {
-    ...attempt,
-    status: "cancelled",
-    updated_at: nowIso()
+async function loadCatalog(forceRefresh = false) {
+  if (isMockEnabled()) {
+    return {
+      challenges: await getBuiltinChallenges(),
+      mode: "mock",
+      lastError: null
+    };
+  }
+  const catalogFile = getConfiguredCatalogFile();
+  if (catalogFile) {
+    return {
+      challenges: await mergeWithBuiltins(
+        await loadChallengesFromFile(catalogFile)
+      ),
+      mode: "file",
+      lastError: null
+    };
+  }
+  return await loadRemoteCatalog(forceRefresh);
+}
+async function listChallenges() {
+  const catalog = await loadCatalog();
+  return catalog.challenges.map((challenge) => summarizeChallenge(challenge));
+}
+async function getChallengeDefinition(challengeId) {
+  const catalog = await loadCatalog();
+  return catalog.challenges.find(
+    (challenge) => challenge.challenge_id === challengeId
+  ) ?? null;
+}
+async function syncCatalog(forceRefresh) {
+  if (isMockEnabled()) {
+    return {
+      status: await getCatalogStatus(),
+      challenges: await getBuiltinChallenges()
+    };
+  }
+  const configuredFile = getConfiguredCatalogFile();
+  if (configuredFile) {
+    const challenges = await mergeWithBuiltins(
+      await loadChallengesFromFile(configuredFile)
+    );
+    return {
+      status: await getCatalogStatus(challenges, "file", null),
+      challenges
+    };
+  }
+  const configuredUrl = getConfiguredCatalogUrl();
+  if (!configuredUrl) {
+    throw new Error(
+      "BENCHMARK_CATALOG_URL is required for remote catalog sync."
+    );
+  }
+  const loaded = await loadCatalog(forceRefresh);
+  return {
+    status: await getCatalogStatus(
+      loaded.challenges,
+      loaded.mode,
+      loaded.lastError
+    ),
+    challenges: loaded.challenges
+  };
+}
+async function getCatalogStatus(knownChallenges, forcedMode, lastErrorOverride) {
+  const configuredFile = getConfiguredCatalogFile();
+  const configuredUrl = getConfiguredCatalogUrl();
+  const cacheFile = getCacheFilePath();
+  const ttlSeconds = getCacheTtlSeconds();
+  const cache = await readCachedCatalog();
+  const cacheFetchedAt = cache?.fetched_at ?? null;
+  const cacheAgeSeconds = cacheFetchedAt ? getAgeSeconds(cacheFetchedAt) : null;
+  const mode = forcedMode ? catalogModeSchema.parse(forcedMode) : isMockEnabled() ? "mock" : configuredFile ? "file" : configuredUrl ? cache ? "remote_cache" : "remote_url" : "builtin";
+  let challengeCount = knownChallenges?.length ?? null;
+  if (challengeCount === null) {
+    if (mode === "mock" || mode === "builtin") {
+      challengeCount = (await getBuiltinChallenges()).length;
+    } else if (configuredFile) {
+      try {
+        challengeCount = (await mergeWithBuiltins(
+          await loadChallengesFromFile(configuredFile)
+        )).length;
+      } catch {
+        challengeCount = null;
+      }
+    } else if (cache) {
+      challengeCount = (await mergeWithBuiltins(cache.challenges)).length;
+    }
+  }
+  let cacheExists = false;
+  try {
+    await stat(cacheFile);
+    cacheExists = true;
+  } catch (error48) {
+    if (error48.code !== "ENOENT") {
+      throw error48;
+    }
+  }
+  return {
+    mode,
+    configured_file: configuredFile,
+    configured_url: configuredUrl,
+    cache_file: cacheFile,
+    cache_exists: cacheExists,
+    cache_ttl_seconds: mode === "remote_cache" || mode === "remote_url" ? ttlSeconds : null,
+    cache_fetched_at: cacheFetchedAt,
+    cache_age_seconds: cacheAgeSeconds,
+    can_sync: !isMockEnabled() && !configuredFile && Boolean(configuredUrl),
+    challenge_count: challengeCount,
+    last_error: lastErrorOverride ?? null
   };
-  await saveAttempt(nextAttempt);
-  return nextAttempt;
 }
 
-// src/mock.ts
-var IS_MOCK = process.env.MOCK === "true" || process.env.BENCHMARK_MOCK === "true";
+// src/checkers.ts
+function normalizeText(value) {
+  return value.trim().toLowerCase().replace(/\s+/g, " ");
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function getPrimitiveAtPath(value, fieldPath) {
+  let current = value;
+  for (const segment of fieldPath.split(".")) {
+    if (!isRecord(current) || !(segment in current)) {
+      return void 0;
+    }
+    current = current[segment];
+  }
+  if (typeof current === "string" || typeof current === "number" || typeof current === "boolean") {
+    return current;
+  }
+  return void 0;
+}
+function renderPrimitive(value) {
+  return typeof value === "string" ? value : String(value);
+}
+function evaluateChecker(checker, solution, execution) {
+  switch (checker.kind) {
+    case "exact_text": {
+      const actual = checker.case_sensitive ? solution.trim() : solution.trim().toLowerCase();
+      const expected = checker.case_sensitive ? checker.expected.trim() : checker.expected.trim().toLowerCase();
+      const passed = actual === expected;
+      return {
+        checker_id: checker.checker_id,
+        kind: checker.kind,
+        passed,
+        message: passed ? "Exact text matched." : `Expected "${checker.expected}" but received "${solution.trim()}".`
+      };
+    }
+    case "normalized_text": {
+      const actual = normalizeText(solution);
+      const expected = normalizeText(checker.expected);
+      const passed = actual === expected;
+      return {
+        checker_id: checker.checker_id,
+        kind: checker.kind,
+        passed,
+        message: passed ? "Normalized text matched." : `Expected normalized "${checker.expected}" but received "${solution.trim()}".`
+      };
+    }
+    case "regex_match": {
+      try {
+        const expression = new RegExp(checker.pattern, checker.flags);
+        const passed = expression.test(solution);
+        return {
+          checker_id: checker.checker_id,
+          kind: checker.kind,
+          passed,
+          message: passed ? "Regex matched the solution." : `Pattern /${checker.pattern}/${checker.flags ?? ""} did not match.`
+        };
+      } catch (error48) {
+        const message = error48 instanceof Error ? error48.message : "Invalid regular expression.";
+        return {
+          checker_id: checker.checker_id,
+          kind: checker.kind,
+          passed: false,
+          message
+        };
+      }
+    }
+    case "contains_all_text": {
+      const actual = normalizeText(solution);
+      const missing = checker.terms.filter(
+        (term) => !actual.includes(normalizeText(term))
+      );
+      return {
+        checker_id: checker.checker_id,
+        kind: checker.kind,
+        passed: missing.length === 0,
+        message: missing.length === 0 ? "All required terms were present." : `Missing terms: ${missing.join(", ")}.`
+      };
+    }
+    case "json_field_equals": {
+      try {
+        const parsed = JSON.parse(solution);
+        const actual = getPrimitiveAtPath(parsed, checker.field);
+        const passed = actual === checker.expected;
+        return {
+          checker_id: checker.checker_id,
+          kind: checker.kind,
+          passed,
+          message: actual === void 0 ? `Field "${checker.field}" was missing or not primitive.` : passed ? `Field "${checker.field}" matched ${renderPrimitive(checker.expected)}.` : `Field "${checker.field}" expected ${renderPrimitive(checker.expected)} but received ${renderPrimitive(actual)}.`
+        };
+      } catch (error48) {
+        const message = error48 instanceof Error ? error48.message : "Solution was not valid JSON.";
+        return {
+          checker_id: checker.checker_id,
+          kind: checker.kind,
+          passed: false,
+          message
+        };
+      }
+    }
+    case "runner_fact_equals": {
+      const actual = execution.facts[checker.fact];
+      const passed = actual === checker.expected;
+      return {
+        checker_id: checker.checker_id,
+        kind: checker.kind,
+        passed,
+        message: actual === void 0 ? `Runner fact "${checker.fact}" was missing.` : passed ? `Runner fact "${checker.fact}" matched ${renderPrimitive(checker.expected)}.` : `Runner fact "${checker.fact}" expected ${renderPrimitive(checker.expected)} but received ${renderPrimitive(actual)}.`
+      };
+    }
+    case "runner_log_contains_text": {
+      const normalizedLogs = normalizeText(execution.logs.join("\n"));
+      const missing = checker.terms.filter(
+        (term) => !normalizedLogs.includes(normalizeText(term))
+      );
+      return {
+        checker_id: checker.checker_id,
+        kind: checker.kind,
+        passed: missing.length === 0,
+        message: missing.length === 0 ? "Runner logs contained all required terms." : `Runner logs missing terms: ${missing.join(", ")}.`
+      };
+    }
+  }
+}
+function evaluateTask(task, solution, execution) {
+  const evidence = task.checkers.map(
+    (checker) => evaluateChecker(checker, solution, execution)
+  );
+  return {
+    accepted: evidence.every((entry) => entry.passed),
+    evidence
+  };
+}
+
+// src/runners.ts
+import { mkdir as mkdir3, writeFile as writeFile3 } from "node:fs/promises";
+import path3 from "node:path";
+import { randomUUID as randomUUID3 } from "node:crypto";
+function nowStamp() {
+  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+}
+function createArtifact(kind, filePath, mimeType, description) {
+  return {
+    artifact_id: `${kind}_${randomUUID3()}`,
+    kind,
+    path: filePath,
+    mime_type: mimeType,
+    ...description ? { description } : {}
+  };
+}
+async function ensureTaskWorkspace(attemptId, taskId) {
+  const workspaceDir = path3.join(getAttemptWorkspaceDir(attemptId), taskId);
+  await mkdir3(workspaceDir, { recursive: true });
+  return workspaceDir;
+}
+async function materializeAsset(workspaceDir, taskId, asset) {
+  if (asset.materialization !== "workspace_file") {
+    return null;
+  }
+  const assetDir = path3.join(workspaceDir, "assets");
+  await mkdir3(assetDir, { recursive: true });
+  const fileName = `${taskId}-${asset.name}`;
+  const filePath = path3.join(assetDir, fileName);
+  await writeFile3(filePath, asset.content, "utf8");
+  return createArtifact(
+    "materialized_asset",
+    filePath,
+    asset.mime_type,
+    asset.description
+  );
+}
+async function writeSubmissionArtifact(workspaceDir, task, solution) {
+  const extension = task.response_format === "json" ? "json" : "txt";
+  const filePath = path3.join(workspaceDir, `submission.${extension}`);
+  await writeFile3(filePath, solution, "utf8");
+  return createArtifact(
+    "submission",
+    filePath,
+    task.response_format === "json" ? "application/json" : "text/plain",
+    "Submitted solution payload."
+  );
+}
+async function writeRunnerReport(workspaceDir, task, logs, facts) {
+  const reportPath = path3.join(
+    workspaceDir,
+    `runner-report-${nowStamp()}.json`
+  );
+  await writeFile3(
+    reportPath,
+    JSON.stringify(
+      {
+        task_id: task.task_id,
+        logs,
+        facts
+      },
+      null,
+      2
+    ),
+    "utf8"
+  );
+  return createArtifact(
+    "runner_report",
+    reportPath,
+    "application/json",
+    "Execution report with runner facts and logs."
+  );
+}
+function countWorkspaceAssets(task) {
+  return task.assets.filter(
+    (asset) => asset.materialization === "workspace_file"
+  ).length;
+}
+async function executeTaskRunner(attempt, task, solution) {
+  const workspaceDir = await ensureTaskWorkspace(
+    attempt.attempt_id,
+    task.task_id
+  );
+  const artifacts = [];
+  const logs = [
+    `runner ${attempt.runner_kind} started for task ${task.task_id}`
+  ];
+  for (const asset of task.assets) {
+    const artifact = await materializeAsset(
+      workspaceDir,
+      task.task_id,
+      asset
+    );
+    if (artifact) {
+      artifacts.push(artifact);
+      logs.push(
+        `materialized asset ${asset.asset_id} to ${artifact.path}`
+      );
+    } else {
+      logs.push(`kept asset ${asset.asset_id} inline`);
+    }
+  }
+  const submissionArtifact = await writeSubmissionArtifact(
+    workspaceDir,
+    task,
+    solution
+  );
+  artifacts.push(submissionArtifact);
+  logs.push(`wrote submission artifact ${submissionArtifact.path}`);
+  const facts = {
+    materialized_asset_count: countWorkspaceAssets(task),
+    total_asset_count: task.assets.length,
+    response_format: task.response_format,
+    family: attempt.family
+  };
+  if (attempt.runner_kind === "code_runner") {
+    facts.domain = "code";
+    logs.push("prepared code workspace inputs");
+  } else if (attempt.runner_kind === "browser_runner") {
+    facts.domain = "web";
+    logs.push("prepared browser snapshot workspace");
+  } else {
+    facts.domain = "os";
+    logs.push("prepared os workspace inventory");
+  }
+  const reportArtifact = await writeRunnerReport(
+    workspaceDir,
+    task,
+    logs,
+    facts
+  );
+  artifacts.push(reportArtifact);
+  logs.push(`wrote runner report ${reportArtifact.path}`);
+  return {
+    runner_kind: attempt.runner_kind,
+    workspace_dir: workspaceDir,
+    logs,
+    facts,
+    artifacts
+  };
+}
 
 // src/index.ts
 var PACKAGE_VERSION = getPackageVersion(import.meta.url);
-var ADAPTERS = [createGaiaAdapter()];
 var server = new McpServer({
   name: "@fre4x/benchmark",
   version: PACKAGE_VERSION
@@ -30715,29 +31395,47 @@ function toPublicAssetDescriptor(asset) {
     asset_id: asset.asset_id,
     name: asset.name,
     mime_type: asset.mime_type,
+    transport: asset.transport,
+    materialization: asset.materialization,
     ...asset.description ? { description: asset.description } : {}
   };
 }
-function toPublicQuestion(question) {
+function toPublicTask(task) {
   return {
-    question_id: question.question_id,
-    prompt: question.prompt,
-    difficulty: question.difficulty,
-    assets: question.assets.map(toPublicAssetDescriptor)
-  };
-}
-function normalizeAnswer(value) {
-  return value.trim().toLowerCase().replace(/\s+/g, " ");
-}
-function gradeQuestion(question, solution) {
-  if (question.grading_mode === "exact") {
-    return question.expected_answer.trim() === solution.trim();
-  }
-  return normalizeAnswer(question.expected_answer) === normalizeAnswer(solution);
+    task_id: task.task_id,
+    title: task.title,
+    prompt: task.prompt,
+    response_format: task.response_format,
+    difficulty: task.difficulty,
+    assets: task.assets.map(toPublicAssetDescriptor),
+    checker_kinds: task.checkers.map((checker) => checker.kind)
+  };
+}
+function formatRunnerExecution(execution) {
+  const logLines = execution.logs.map((entry) => `- ${entry}`).join("\n");
+  const factLines = Object.entries(execution.facts).map(([key, value]) => `- \`${key}\`: ${value}`).join("\n");
+  const artifactLines = execution.artifacts.map((artifact) => `- \`${artifact.kind}\`: ${artifact.path}`).join("\n");
+  return [
+    "## Runner Execution",
+    "",
+    `**Workspace Dir:** ${execution.workspace_dir}`,
+    "",
+    "### Facts",
+    "",
+    factLines || "_No facts recorded._",
+    "",
+    "### Logs",
+    "",
+    logLines || "_No logs recorded._",
+    "",
+    "### Artifacts",
+    "",
+    artifactLines || "_No artifacts recorded._"
+  ].join("\n");
 }
 function sumScore(attempt) {
-  return attempt.submissions.reduce(
-    (total, submission) => total + submission.score_delta,
+  return attempt.evaluations.reduce(
+    (total, evaluation) => total + evaluation.score_delta,
     0
   );
 }
@@ -30749,16 +31447,19 @@ function createGuidanceStep(action, recommendedTool, reason, exampleArguments) {
     ...exampleArguments ? { example_arguments: exampleArguments } : {}
   };
 }
-function getQuestionGuidance(attemptId, question) {
-  if (question.assets.length > 0) {
+function createSubmitExample(task) {
+  return task.response_format === "json" ? '{"field":"value"}' : "your final answer";
+}
+function getTaskGuidance(attemptId, task) {
+  if (task.assets.length > 0) {
     return {
       primary: createGuidanceStep(
         "review_asset",
         "benchmark_get_asset",
-        "This question includes an attachment.",
+        "This task includes an attachment.",
         {
           attempt_id: attemptId,
-          asset_id: question.assets[0].asset_id
+          asset_id: task.assets[0].asset_id
         }
       ),
       alternatives: [
@@ -30768,7 +31469,7 @@ function getQuestionGuidance(attemptId, question) {
           "Submit after reviewing the attachment.",
           {
             attempt_id: attemptId,
-            solution: "your final answer"
+            solution: createSubmitExample(task)
           }
         )
       ]
@@ -30778,10 +31479,10 @@ function getQuestionGuidance(attemptId, question) {
     primary: createGuidanceStep(
       "submit_solution",
       "benchmark_submit_solution",
-      "This question is ready for grading.",
+      "This task is ready for deterministic grading.",
       {
         attempt_id: attemptId,
-        solution: "your final answer"
+        solution: createSubmitExample(task)
       }
     ),
     alternatives: [
@@ -30789,24 +31490,17 @@ function getQuestionGuidance(attemptId, question) {
         "inspect_attempt",
         "benchmark_get_attempt",
         "Review the current attempt state.",
-        { attempt_id: attemptId }
+        {
+          attempt_id: attemptId,
+          limit: "20",
+          offset: "0"
+        }
       )
     ]
   };
 }
-function findAdapterForChallengeId(challengeId) {
-  return ADAPTERS.find(
-    (adapter) => challengeId.startsWith(`${adapter.benchmarkId}_`)
-  ) ?? null;
-}
-async function listAllChallenges() {
-  const groups = await Promise.all(
-    ADAPTERS.map(async (adapter) => await adapter.listChallenges())
-  );
-  return groups.flat();
-}
-function getCurrentQuestion(attempt) {
-  return attempt.questions[attempt.current_question_index] ?? null;
+function getCurrentTask(attempt) {
+  return attempt.tasks[attempt.current_task_index] ?? null;
 }
 function formatChallengeSummary(summary) {
   return [
@@ -30814,25 +31508,40 @@ function formatChallengeSummary(summary) {
     formatFields([
       ["Challenge ID", summary.challenge_id],
       ["Benchmark", summary.benchmark_id],
+      ["Family", summary.family],
+      ["Runner", summary.runner_kind],
       ["Version", summary.version],
-      ["Questions", summary.total_questions],
+      ["Source", summary.source],
+      ["Tasks", summary.total_tasks],
       ["Difficulties", summary.difficulty_levels.join(", ")],
+      ["Checker Kinds", summary.checker_kinds.join(", ")],
       ["Assets", summary.supports_assets ? "yes" : "no"]
     ]),
     "",
     summary.description
   ].join("\n");
 }
-function formatQuestionBlock(attemptId, questionIndex, totalQuestions, question) {
-  const assetLines = question.assets.length > 0 ? formatListItems(
-    question.assets,
-    (asset) => `- \`${asset.asset_id}\` (${asset.name}, ${asset.mime_type})`
+function formatTaskBlock(attemptId, taskIndex, totalTasks, task) {
+  const assetLines = task.assets.length > 0 ? formatListItems(
+    task.assets,
+    (asset) => `- \`${asset.asset_id}\` (${asset.name}, ${asset.mime_type}, ${asset.materialization})`
   ) : "_No assets attached._";
-  const guidance = getQuestionGuidance(attemptId, question);
+  const guidance = getTaskGuidance(attemptId, task);
   return [
-    `# Question ${questionIndex} of ${totalQuestions}`,
+    `# Task ${taskIndex} of ${totalTasks}`,
     "",
-    question.prompt,
+    `## ${task.title}`,
+    "",
+    task.prompt,
+    "",
+    formatFields([
+      ["Response Format", task.response_format],
+      ["Difficulty", task.difficulty],
+      [
+        "Checker Kinds",
+        task.checkers.map((checker) => checker.kind).join(", ")
+      ]
+    ]),
     "",
     "## Assets",
     "",
@@ -30845,9 +31554,71 @@ function formatQuestionBlock(attemptId, questionIndex, totalQuestions, question)
     `- **Reason**: ${guidance.primary.reason}`
   ].join("\n");
 }
+function formatCatalogStatus(status) {
+  return [
+    "# Catalog Status",
+    "",
+    formatFields([
+      ["Mode", status.mode],
+      ["Configured File", status.configured_file],
+      ["Configured URL", status.configured_url],
+      ["Cache File", status.cache_file],
+      ["Cache Exists", status.cache_exists ? "yes" : "no"],
+      ["Cache TTL Seconds", status.cache_ttl_seconds],
+      ["Cache Fetched At", status.cache_fetched_at],
+      ["Cache Age Seconds", status.cache_age_seconds],
+      ["Can Sync", status.can_sync ? "yes" : "no"],
+      ["Challenge Count", status.challenge_count],
+      ["Last Error", status.last_error]
+    ])
+  ].join("\n");
+}
+async function handleGetCatalogStatus() {
+  try {
+    const status = await getCatalogStatus();
+    return buildTextResult(formatCatalogStatus(status), {
+      catalog: status
+    });
+  } catch (error48) {
+    return createInternalError(error48);
+  }
+}
+async function handleSyncCatalog(params) {
+  try {
+    const result = await syncCatalog(params.force);
+    const status = result.status;
+    return buildTextResult(
+      [
+        "# Catalog Synced",
+        "",
+        formatFields([
+          ["Mode", status.mode],
+          ["Configured URL", status.configured_url],
+          ["Cache File", status.cache_file],
+          ["Challenge Count", status.challenge_count],
+          ["Last Error", status.last_error]
+        ])
+      ].join("\n"),
+      {
+        catalog: status,
+        synced_challenges: result.challenges.map(
+          (challenge) => challenge.challenge_id
+        )
+      }
+    );
+  } catch (error48) {
+    if (error48 instanceof Error && error48.message.includes("BENCHMARK_CATALOG_URL")) {
+      return createValidationError(
+        "BENCHMARK_CATALOG_URL",
+        "configure a remote catalog URL before syncing"
+      );
+    }
+    return createInternalError(error48);
+  }
+}
 async function handleListChallenges(params) {
   try {
-    const challenges = await listAllChallenges();
+    const challenges = await listChallenges();
     const paginated = applyPagination(challenges, params);
     const guidance = paginated.items.length > 0 ? createGuidanceStep(
       "start_challenge",
@@ -30860,7 +31631,7 @@ async function handleListChallenges(params) {
       "No benchmark suites are currently available."
     );
     const markdown = [
-      "# Benchmark Challenges",
+      "# Deterministic Benchmark Challenges",
       "",
       formatListItems(
         paginated.items,
@@ -30896,29 +31667,18 @@ async function handleListChallenges(params) {
 }
 async function handleStartChallenge(params) {
   try {
-    const adapter = findAdapterForChallengeId(params.challenge_id);
-    if (!adapter) {
-      return createNotFoundError(`challenge ${params.challenge_id}`);
-    }
-    const challenge = await adapter.getChallengeDefinition(
-      params.challenge_id
-    );
+    const challenge = await getChallengeDefinition(params.challenge_id);
     if (!challenge) {
       return createNotFoundError(`challenge ${params.challenge_id}`);
     }
     const attempt = await createAttempt(challenge);
-    const currentQuestion = getCurrentQuestion(attempt);
-    if (!currentQuestion) {
+    const currentTask = getCurrentTask(attempt);
+    if (!currentTask) {
       return createInternalError(
-        new Error(
-          `Challenge ${challenge.challenge_id} has no questions.`
-        )
+        new Error(`Challenge ${challenge.challenge_id} has no tasks.`)
       );
     }
-    const guidance = getQuestionGuidance(
-      attempt.attempt_id,
-      currentQuestion
-    );
+    const guidance = getTaskGuidance(attempt.attempt_id, currentTask);
     return buildTextResult(
       [
         `# ${challenge.title}`,
@@ -30926,15 +31686,17 @@ async function handleStartChallenge(params) {
         formatFields([
           ["Attempt ID", attempt.attempt_id],
           ["Benchmark", attempt.benchmark_id],
+          ["Family", attempt.family],
+          ["Runner", attempt.runner_kind],
           ["Version", attempt.dataset_version],
-          ["Questions", attempt.total_questions]
+          ["Tasks", attempt.total_tasks]
         ]),
         "",
-        formatQuestionBlock(
+        formatTaskBlock(
           attempt.attempt_id,
           1,
-          attempt.total_questions,
-          currentQuestion
+          attempt.total_tasks,
+          currentTask
         )
       ].join("\n"),
       {
@@ -30942,9 +31704,14 @@ async function handleStartChallenge(params) {
         challenge_id: attempt.challenge_id,
         benchmark_id: attempt.benchmark_id,
         challenge_title: attempt.challenge_title,
+        family: attempt.family,
+        runner_kind: attempt.runner_kind,
+        task_index: 1,
         question_index: 1,
-        total_questions: attempt.total_questions,
-        question: toPublicQuestion(currentQuestion),
+        total_tasks: attempt.total_tasks,
+        total_questions: attempt.total_tasks,
+        task: toPublicTask(currentTask),
+        question: toPublicTask(currentTask),
         done: false,
         guidance
       }
@@ -30965,26 +31732,39 @@ async function handleSubmitSolution(params) {
         `attempt is ${attempt.status} and cannot accept submissions`
       );
     }
-    const currentQuestion = getCurrentQuestion(attempt);
-    if (!currentQuestion) {
+    const currentTask = getCurrentTask(attempt);
+    if (!currentTask) {
       return createInternalError(
-        new Error(
-          `Attempt ${attempt.attempt_id} has no current question.`
-        )
+        new Error(`Attempt ${attempt.attempt_id} has no current task.`)
       );
     }
-    const accepted = gradeQuestion(currentQuestion, params.solution);
-    const scoreDelta = accepted ? 1 : 0;
-    const updatedAttempt = await appendSubmission(attempt, {
-      question_id: currentQuestion.question_id,
-      solution: params.solution,
-      accepted,
-      score_delta: scoreDelta,
-      submitted_at: (/* @__PURE__ */ new Date()).toISOString()
-    });
+    const execution = await executeTaskRunner(
+      attempt,
+      currentTask,
+      params.solution
+    );
+    const evaluation = evaluateTask(
+      currentTask,
+      params.solution,
+      execution
+    );
+    const scoreDelta = evaluation.accepted ? 1 : 0;
+    const updatedAttempt = await appendEvaluation(
+      attempt.attempt_id,
+      currentTask.task_id,
+      {
+        task_id: currentTask.task_id,
+        solution: params.solution,
+        accepted: evaluation.accepted,
+        score_delta: scoreDelta,
+        submitted_at: (/* @__PURE__ */ new Date()).toISOString(),
+        evidence: evaluation.evidence,
+        execution
+      }
+    );
     const totalScore = sumScore(updatedAttempt);
-    const nextQuestion = getCurrentQuestion(updatedAttempt);
-    if (!nextQuestion) {
+    const nextTask = getCurrentTask(updatedAttempt);
+    if (!nextTask) {
       const finalGuidance = {
         primary: createGuidanceStep(
           "complete",
@@ -30996,25 +31776,37 @@ async function handleSubmitSolution(params) {
             "inspect_attempt",
             "benchmark_get_attempt",
             "Review the completed attempt summary.",
-            { attempt_id: updatedAttempt.attempt_id }
+            {
+              attempt_id: updatedAttempt.attempt_id,
+              limit: "20",
+              offset: "0"
+            }
           )
         ]
       };
+      const evidenceLines2 = formatListItems(
+        evaluation.evidence,
+        (entry) => `- ${entry.passed ? "pass" : "fail"} \`${entry.checker_id}\`: ${entry.message}`
+      );
       return buildTextResult(
         [
           "# Challenge Complete",
           "",
           formatFields([
             ["Attempt ID", updatedAttempt.attempt_id],
-            ["Accepted", accepted ? "yes" : "no"],
+            ["Accepted", evaluation.accepted ? "yes" : "no"],
             ["Score Delta", scoreDelta],
             [
               "Total Score",
-              `${totalScore}/${updatedAttempt.total_questions}`
+              `${totalScore}/${updatedAttempt.total_tasks}`
             ]
           ]),
           "",
-          accepted ? "The final answer was accepted." : "The final answer was not accepted.",
+          "## Checker Evidence",
+          "",
+          evidenceLines2,
+          "",
+          formatRunnerExecution(execution),
           "",
           "## Next Step",
           "",
@@ -31024,22 +31816,25 @@ async function handleSubmitSolution(params) {
         ].join("\n"),
         {
           attempt_id: updatedAttempt.attempt_id,
-          accepted,
+          accepted: evaluation.accepted,
           score_delta: scoreDelta,
           total_score: totalScore,
           done: true,
+          evidence: evaluation.evidence,
+          execution,
           final_result: {
             total_score: totalScore,
-            max_score: updatedAttempt.total_questions,
-            passed: totalScore === updatedAttempt.total_questions
+            max_score: updatedAttempt.total_tasks,
+            passed: totalScore === updatedAttempt.total_tasks
           },
           guidance: finalGuidance
         }
       );
     }
-    const guidance = getQuestionGuidance(
-      updatedAttempt.attempt_id,
-      nextQuestion
+    const guidance = getTaskGuidance(updatedAttempt.attempt_id, nextTask);
+    const evidenceLines = formatListItems(
+      evaluation.evidence,
+      (entry) => `- ${entry.passed ? "pass" : "fail"} \`${entry.checker_id}\`: ${entry.message}`
     );
     return buildTextResult(
       [
@@ -31047,31 +31842,40 @@ async function handleSubmitSolution(params) {
         "",
         formatFields([
           ["Attempt ID", updatedAttempt.attempt_id],
-          ["Accepted", accepted ? "yes" : "no"],
+          ["Accepted", evaluation.accepted ? "yes" : "no"],
           ["Score Delta", scoreDelta],
           [
             "Total Score",
-            `${totalScore}/${updatedAttempt.total_questions}`
+            `${totalScore}/${updatedAttempt.total_tasks}`
           ]
         ]),
         "",
-        accepted ? "Answer accepted. Moving to the next question." : "Answer recorded. Moving to the next question.",
+        "## Checker Evidence",
+        "",
+        evidenceLines,
+        "",
+        formatRunnerExecution(execution),
         "",
-        formatQuestionBlock(
+        formatTaskBlock(
           updatedAttempt.attempt_id,
-          updatedAttempt.current_question_index + 1,
-          updatedAttempt.total_questions,
-          nextQuestion
+          updatedAttempt.current_task_index + 1,
+          updatedAttempt.total_tasks,
+          nextTask
         )
       ].join("\n"),
       {
         attempt_id: updatedAttempt.attempt_id,
-        accepted,
+        accepted: evaluation.accepted,
         score_delta: scoreDelta,
         total_score: totalScore,
-        question_index: updatedAttempt.current_question_index + 1,
-        total_questions: updatedAttempt.total_questions,
-        next_question: toPublicQuestion(nextQuestion),
+        task_index: updatedAttempt.current_task_index + 1,
+        question_index: updatedAttempt.current_task_index + 1,
+        total_tasks: updatedAttempt.total_tasks,
+        total_questions: updatedAttempt.total_tasks,
+        evidence: evaluation.evidence,
+        execution,
+        next_task: toPublicTask(nextTask),
+        next_question: toPublicTask(nextTask),
         done: false,
         guidance
       }
@@ -31086,15 +31890,15 @@ async function handleGetAsset(params) {
     if (!attempt) {
       return createNotFoundError(`attempt ${params.attempt_id}`);
     }
-    const question = attempt.questions.find(
+    const task = attempt.tasks.find(
       (candidate) => candidate.assets.some(
         (asset2) => asset2.asset_id === params.asset_id
       )
     );
-    const asset = question?.assets.find(
+    const asset = task?.assets.find(
       (candidate) => candidate.asset_id === params.asset_id
     );
-    if (!asset || !question) {
+    if (!asset || !task) {
       return createNotFoundError(`asset ${params.asset_id}`);
     }
     const guidance = {
@@ -31104,15 +31908,19 @@ async function handleGetAsset(params) {
         "Submit the answer after reviewing the asset.",
         {
           attempt_id: attempt.attempt_id,
-          solution: "your final answer"
+          solution: createSubmitExample(task)
         }
       ),
       alternatives: [
         createGuidanceStep(
           "inspect_attempt",
           "benchmark_get_attempt",
-          "Review the current question again.",
-          { attempt_id: attempt.attempt_id }
+          "Review the current task again.",
+          {
+            attempt_id: attempt.attempt_id,
+            limit: "20",
+            offset: "0"
+          }
         )
       ]
     };
@@ -31122,8 +31930,10 @@ async function handleGetAsset(params) {
         "",
         formatFields([
           ["Attempt ID", attempt.attempt_id],
-          ["Question ID", question.question_id],
+          ["Task ID", task.task_id],
           ["Mime Type", asset.mime_type],
+          ["Transport", asset.transport],
+          ["Materialization", asset.materialization],
           ["Encoding", asset.encoding]
         ]),
         "",
@@ -31143,6 +31953,8 @@ async function handleGetAsset(params) {
           asset_id: asset.asset_id,
           name: asset.name,
           mime_type: asset.mime_type,
+          transport: asset.transport,
+          materialization: asset.materialization,
           encoding: asset.encoding,
           description: asset.description,
           content: asset.content
@@ -31160,8 +31972,9 @@ async function handleGetAttempt(params) {
     if (!attempt) {
       return createNotFoundError(`attempt ${params.attempt_id}`);
     }
-    const currentQuestion = getCurrentQuestion(attempt);
-    const guidance = currentQuestion && attempt.status === "active" ? getQuestionGuidance(attempt.attempt_id, currentQuestion) : {
+    const currentTask = getCurrentTask(attempt);
+    const evaluationPage = applyPagination(attempt.evaluations, params);
+    const guidance = currentTask && attempt.status === "active" ? getTaskGuidance(attempt.attempt_id, currentTask) : {
       primary: createGuidanceStep(
         "complete",
         null,
@@ -31169,6 +31982,27 @@ async function handleGetAttempt(params) {
       ),
       alternatives: []
     };
+    const historyText = formatListItems(
+      evaluationPage.items,
+      (evaluation, index) => {
+        const evidenceLines = evaluation.evidence.map(
+          (entry) => `  - ${entry.passed ? "pass" : "fail"} \`${entry.checker_id}\`: ${entry.message}`
+        ).join("\n");
+        return [
+          `### Evaluation ${evaluationPage.offset + index + 1}`,
+          formatFields([
+            ["Task ID", evaluation.task_id],
+            ["Accepted", evaluation.accepted ? "yes" : "no"],
+            ["Score Delta", evaluation.score_delta]
+          ]),
+          "",
+          formatRunnerExecution(evaluation.execution),
+          "",
+          evidenceLines
+        ].join("\n");
+      },
+      "_No submissions recorded yet._"
+    );
     return buildTextResult(
       [
         "# Attempt Status",
@@ -31176,32 +32010,50 @@ async function handleGetAttempt(params) {
         formatFields([
           ["Attempt ID", attempt.attempt_id],
           ["Challenge", attempt.challenge_title],
+          ["Family", attempt.family],
+          ["Runner", attempt.runner_kind],
           ["Status", attempt.status],
+          ["Revision", attempt.revision],
           [
             "Progress",
-            `${attempt.current_question_index}/${attempt.total_questions}`
+            `${attempt.current_task_index}/${attempt.total_tasks}`
           ],
-          [
-            "Score",
-            `${sumScore(attempt)}/${attempt.total_questions}`
-          ]
+          ["Score", `${sumScore(attempt)}/${attempt.total_tasks}`]
         ]),
         "",
-        currentQuestion ? formatQuestionBlock(
+        currentTask ? formatTaskBlock(
           attempt.attempt_id,
-          attempt.current_question_index + 1,
-          attempt.total_questions,
-          currentQuestion
-        ) : "No current question is available."
-      ].join("\n"),
+          attempt.current_task_index + 1,
+          attempt.total_tasks,
+          currentTask
+        ) : "No current task is available.",
+        "",
+        "## Evaluation History",
+        "",
+        historyText,
+        evaluationPage.hasMore ? formatPaginationFooter(
+          evaluationPage.offset,
+          evaluationPage.limit,
+          evaluationPage.total
+        ) : ""
+      ].filter(Boolean).join("\n"),
       {
         attempt_id: attempt.attempt_id,
         challenge_id: attempt.challenge_id,
         status: attempt.status,
         total_score: sumScore(attempt),
-        total_questions: attempt.total_questions,
-        current_question_index: attempt.current_question_index,
-        current_question: currentQuestion ? toPublicQuestion(currentQuestion) : null,
+        total_tasks: attempt.total_tasks,
+        total_questions: attempt.total_tasks,
+        current_task_index: attempt.current_task_index,
+        current_question_index: attempt.current_task_index,
+        current_task: currentTask ? toPublicTask(currentTask) : null,
+        current_question: currentTask ? toPublicTask(currentTask) : null,
+        evaluations: evaluationPage.items,
+        history: evaluationPage.items,
+        history_total: evaluationPage.total,
+        offset: evaluationPage.offset,
+        limit: evaluationPage.limit,
+        has_more: evaluationPage.hasMore,
         guidance
       }
     );
@@ -31221,7 +32073,7 @@ async function handleCancelAttempt(params) {
         `attempt is already ${attempt.status}`
       );
     }
-    const cancelled = await cancelAttempt(attempt);
+    const cancelled = await cancelAttempt(attempt.attempt_id);
     return buildTextResult(
       [
         "# Attempt Cancelled",
@@ -31229,7 +32081,8 @@ async function handleCancelAttempt(params) {
         formatFields([
           ["Attempt ID", cancelled.attempt_id],
           ["Challenge", cancelled.challenge_title],
-          ["Status", cancelled.status]
+          ["Status", cancelled.status],
+          ["Revision", cancelled.revision]
         ])
       ].join("\n"),
       {
@@ -31253,7 +32106,7 @@ server.registerTool(
   "benchmark_list_challenges",
   {
     title: "List Challenges",
-    description: "List benchmark suites and challenge metadata.",
+    description: "List deterministic benchmark suites and metadata.",
     inputSchema: listChallengesInputSchema,
     annotations: {
       readOnlyHint: true,
@@ -31262,11 +32115,36 @@ server.registerTool(
   },
   handleListChallenges
 );
+server.registerTool(
+  "benchmark_get_catalog_status",
+  {
+    title: "Get Catalog Status",
+    description: "Inspect catalog source and cache status.",
+    inputSchema: getCatalogStatusInputSchema,
+    annotations: {
+      readOnlyHint: true,
+      idempotentHint: true
+    }
+  },
+  handleGetCatalogStatus
+);
+server.registerTool(
+  "benchmark_sync_catalog",
+  {
+    title: "Sync Catalog",
+    description: "Fetch and cache the remote benchmark catalog.",
+    inputSchema: syncCatalogInputSchema,
+    annotations: {
+      idempotentHint: true
+    }
+  },
+  handleSyncCatalog
+);
 server.registerTool(
   "benchmark_start_challenge",
   {
     title: "Start Challenge",
-    description: "Start an attempt and return the first question.",
+    description: "Start an attempt and return the first task.",
     inputSchema: startChallengeInputSchema
   },
   handleStartChallenge
@@ -31275,7 +32153,7 @@ server.registerTool(
   "benchmark_submit_solution",
   {
     title: "Submit Solution",
-    description: "Grade one answer and advance the attempt.",
+    description: "Grade one task and advance the attempt.",
     inputSchema: submitSolutionInputSchema
   },
   handleSubmitSolution
@@ -31297,7 +32175,7 @@ server.registerTool(
   "benchmark_get_attempt",
   {
     title: "Get Attempt",
-    description: "Inspect attempt status and current question.",
+    description: "Inspect attempt status, tasks, and evidence.",
     inputSchema: getAttemptInputSchema,
     annotations: {
       readOnlyHint: true,
@@ -31310,24 +32188,19 @@ server.registerTool(
   "benchmark_cancel_attempt",
   {
     title: "Cancel Attempt",
-    description: "Cancel an active benchmark attempt.",
+    description: "Cancel an active attempt.",
     inputSchema: cancelAttemptInputSchema
   },
   handleCancelAttempt
 );
 async function main() {
-  if (IS_MOCK) {
-    console.error(
-      "[benchmark] Running in MOCK mode \u2014 using built-in benchmark fixtures."
-    );
-  }
   const transport = new StdioServerTransport();
   await server.connect(transport);
   console.error("[benchmark] MCP server running via stdio");
 }
-if (process.env.NODE_ENV !== "test") {
+if (process.argv[1] && import.meta.url.endsWith(process.argv[1])) {
   main().catch((error48) => {
-    console.error("Server error:", error48);
+    console.error("[benchmark] fatal error", error48);
     process.exit(1);
   });
 }
@@ -31335,8 +32208,10 @@ export {
   handleCancelAttempt,
   handleGetAsset,
   handleGetAttempt,
+  handleGetCatalogStatus,
   handleListChallenges,
   handleStartChallenge,
   handleSubmitSolution,
+  handleSyncCatalog,
   server
 };