@fre4x/benchmark 1.1.0-beta.4 → 1.1.0

package/README.md

@@ -8,22 +8,22 @@ The rebuilt core is organized around challenge catalogs, typed task assets, and
 
 | Tool | Purpose |
 |------|---------|
-| `benchmark_list_challenges` | List deterministic benchmark suites with family, runner, and checker metadata |
-| `benchmark_get_catalog_status` | Inspect catalog source configuration, cache state, and availability |
-| `benchmark_sync_catalog` | Fetch and cache the remote benchmark catalog when a URL source is configured |
-| `benchmark_start_challenge` | Start an attempt and return the first task |
-| `benchmark_submit_solution` | Grade one task and return checker evidence plus the next task or final score |
-| `benchmark_get_asset` | Read an attached benchmark asset by `asset_id` |
-| `benchmark_get_attempt` | Inspect attempt status, current task, and paginated evaluation history |
-| `benchmark_cancel_attempt` | Cancel an active attempt |
+| `list_challenges` | List deterministic benchmark suites with family, runner, and checker metadata |
+| `get_catalog_status` | Inspect catalog source configuration, cache state, and availability |
+| `sync_catalog` | Fetch and cache the remote benchmark catalog when a URL source is configured |
+| `start_challenge` | Start an attempt and return the first task |
+| `submit_solution` | Grade one task and return checker evidence plus the next task or final score |
+| `get_asset` | Read an attached benchmark asset by `asset_id` |
+| `get_attempt` | Inspect attempt status, current task, and paginated evaluation history |
+| `cancel_attempt` | Cancel an active attempt |
 
 ## Workflow
 
-1. Call `benchmark_list_challenges`
+1. Call `list_challenges`
 2. Pick a `challenge_id`
-3. Call `benchmark_start_challenge`
-4. If the task has assets, call `benchmark_get_asset`
-5. Call `benchmark_submit_solution`
+3. Call `start_challenge`
+4. If the task has assets, call `get_asset`
+5. Call `submit_solution`
 6. Repeat until `done: true`
 
 Each response includes machine-readable guidance for the most likely next tool call.
@@ -77,7 +77,7 @@ BENCHMARK_MOCK=true
 
 `BENCHMARK_GAIA_DATA_FILE` is still accepted as a backward-compatible alias, but the rebuilt package is no longer GAIA-first.
 
-When `BENCHMARK_CATALOG_URL` is set, the package will reuse a fresh cached copy when available and can be explicitly refreshed with `benchmark_sync_catalog`.
+When `BENCHMARK_CATALOG_URL` is set, the package will reuse a fresh cached copy when available and can be explicitly refreshed with `sync_catalog`.
 
 ## Catalog shape
 

package/dist/index.js

@@ -31455,7 +31455,7 @@ function getTaskGuidance(attemptId, task) {
     return {
       primary: createGuidanceStep(
         "review_asset",
-        "benchmark_get_asset",
+        "get_asset",
         "This task includes an attachment.",
         {
           attempt_id: attemptId,
@@ -31465,7 +31465,7 @@ function getTaskGuidance(attemptId, task) {
       alternatives: [
         createGuidanceStep(
           "submit_solution",
-          "benchmark_submit_solution",
+          "submit_solution",
           "Submit after reviewing the attachment.",
           {
             attempt_id: attemptId,
@@ -31478,7 +31478,7 @@ function getTaskGuidance(attemptId, task) {
   return {
     primary: createGuidanceStep(
       "submit_solution",
-      "benchmark_submit_solution",
+      "submit_solution",
       "This task is ready for deterministic grading.",
       {
         attempt_id: attemptId,
@@ -31488,7 +31488,7 @@ function getTaskGuidance(attemptId, task) {
     alternatives: [
       createGuidanceStep(
         "inspect_attempt",
-        "benchmark_get_attempt",
+        "get_attempt",
         "Review the current attempt state.",
         {
           attempt_id: attemptId,
@@ -31622,7 +31622,7 @@ async function handleListChallenges(params) {
     const paginated = applyPagination(challenges, params);
     const guidance = paginated.items.length > 0 ? createGuidanceStep(
       "start_challenge",
-      "benchmark_start_challenge",
+      "start_challenge",
       "Begin with the first listed challenge.",
       { challenge_id: paginated.items[0].challenge_id }
     ) : createGuidanceStep(
@@ -31774,7 +31774,7 @@ async function handleSubmitSolution(params) {
         alternatives: [
           createGuidanceStep(
             "inspect_attempt",
-            "benchmark_get_attempt",
+            "get_attempt",
             "Review the completed attempt summary.",
             {
               attempt_id: updatedAttempt.attempt_id,
@@ -31904,7 +31904,7 @@ async function handleGetAsset(params) {
     const guidance = {
       primary: createGuidanceStep(
         "submit_solution",
-        "benchmark_submit_solution",
+        "submit_solution",
         "Submit the answer after reviewing the asset.",
         {
           attempt_id: attempt.attempt_id,
@@ -31914,7 +31914,7 @@ async function handleGetAsset(params) {
       alternatives: [
         createGuidanceStep(
           "inspect_attempt",
-          "benchmark_get_attempt",
+          "get_attempt",
           "Review the current task again.",
           {
             attempt_id: attempt.attempt_id,
@@ -32103,7 +32103,7 @@ async function handleCancelAttempt(params) {
   }
 }
 server.registerTool(
-  "benchmark_list_challenges",
+  "list_challenges",
   {
     title: "List Challenges",
     description: "List deterministic benchmark suites and metadata.",
@@ -32116,7 +32116,7 @@ server.registerTool(
   handleListChallenges
 );
 server.registerTool(
-  "benchmark_get_catalog_status",
+  "get_catalog_status",
   {
     title: "Get Catalog Status",
     description: "Inspect catalog source and cache status.",
@@ -32129,7 +32129,7 @@ server.registerTool(
   handleGetCatalogStatus
 );
 server.registerTool(
-  "benchmark_sync_catalog",
+  "sync_catalog",
   {
     title: "Sync Catalog",
     description: "Fetch and cache the remote benchmark catalog.",
@@ -32141,7 +32141,7 @@ server.registerTool(
   handleSyncCatalog
 );
 server.registerTool(
-  "benchmark_start_challenge",
+  "start_challenge",
   {
     title: "Start Challenge",
     description: "Start an attempt and return the first task.",
@@ -32150,7 +32150,7 @@ server.registerTool(
   handleStartChallenge
 );
 server.registerTool(
-  "benchmark_submit_solution",
+  "submit_solution",
   {
     title: "Submit Solution",
     description: "Grade one task and advance the attempt.",
@@ -32159,7 +32159,7 @@ server.registerTool(
   handleSubmitSolution
 );
 server.registerTool(
-  "benchmark_get_asset",
+  "get_asset",
   {
     title: "Get Asset",
     description: "Read an attached benchmark asset by id.",
@@ -32172,7 +32172,7 @@ server.registerTool(
   handleGetAsset
 );
 server.registerTool(
-  "benchmark_get_attempt",
+  "get_attempt",
   {
     title: "Get Attempt",
     description: "Inspect attempt status, tasks, and evidence.",
@@ -32185,7 +32185,7 @@ server.registerTool(
   handleGetAttempt
 );
 server.registerTool(
-  "benchmark_cancel_attempt",
+  "cancel_attempt",
   {
     title: "Cancel Attempt",
     description: "Cancel an active attempt.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fre4x/benchmark",
-  "version": "1.1.0-beta.4",
+  "version": "1.1.0",
   "description": "A deterministic benchmark MCP server for agent evaluation workflows.",
   "type": "module",
   "main": "dist/index.js",