@fre4x/benchmark 1.1.8 → 1.1.11

package/dist/index.js

@@ -6890,6 +6890,10 @@ var require_dist = __commonJS({
   }
 });
 
+// src/index.ts
+import { realpathSync } from "node:fs";
+import { fileURLToPath as fileURLToPath2 } from "node:url";
+
 // ../packages/shared/dist/errors.js
 function createValidationError(field, message) {
   return {
@@ -6926,7 +6930,7 @@ function createInternalError(error48) {
         type: "text",
         text: `Internal Error: ${message}
 
-Suggestion: If the problem persists, check server logs.`
+Suggestion: Use the error details above to correct tool arguments and retry the tool call.`
       }
     ]
   };
@@ -31327,159 +31331,7 @@ function evaluateTask(task, solution, execution) {
   };
 }
 
-// src/runners.ts
-import { mkdir as mkdir3, writeFile as writeFile3 } from "node:fs/promises";
-import path3 from "node:path";
-import { randomUUID as randomUUID3 } from "node:crypto";
-function nowStamp() {
-  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-}
-function createArtifact(kind, filePath, mimeType, description) {
-  return {
-    artifact_id: `${kind}_${randomUUID3()}`,
-    kind,
-    path: filePath,
-    mime_type: mimeType,
-    ...description ? { description } : {}
-  };
-}
-async function ensureTaskWorkspace(attemptId, taskId) {
-  const workspaceDir = path3.join(getAttemptWorkspaceDir(attemptId), taskId);
-  await mkdir3(workspaceDir, { recursive: true });
-  return workspaceDir;
-}
-async function materializeAsset(workspaceDir, taskId, asset) {
-  if (asset.materialization !== "workspace_file") {
-    return null;
-  }
-  const assetDir = path3.join(workspaceDir, "assets");
-  await mkdir3(assetDir, { recursive: true });
-  const fileName = `${taskId}-${asset.name}`;
-  const filePath = path3.join(assetDir, fileName);
-  await writeFile3(filePath, asset.content, "utf8");
-  return createArtifact(
-    "materialized_asset",
-    filePath,
-    asset.mime_type,
-    asset.description
-  );
-}
-async function writeSubmissionArtifact(workspaceDir, task, solution) {
-  const extension = task.response_format === "json" ? "json" : "txt";
-  const filePath = path3.join(workspaceDir, `submission.${extension}`);
-  await writeFile3(filePath, solution, "utf8");
-  return createArtifact(
-    "submission",
-    filePath,
-    task.response_format === "json" ? "application/json" : "text/plain",
-    "Submitted solution payload."
-  );
-}
-async function writeRunnerReport(workspaceDir, task, logs, facts) {
-  const reportPath = path3.join(
-    workspaceDir,
-    `runner-report-${nowStamp()}.json`
-  );
-  await writeFile3(
-    reportPath,
-    JSON.stringify(
-      {
-        task_id: task.task_id,
-        logs,
-        facts
-      },
-      null,
-      2
-    ),
-    "utf8"
-  );
-  return createArtifact(
-    "runner_report",
-    reportPath,
-    "application/json",
-    "Execution report with runner facts and logs."
-  );
-}
-function countWorkspaceAssets(task) {
-  return task.assets.filter(
-    (asset) => asset.materialization === "workspace_file"
-  ).length;
-}
-async function executeTaskRunner(attempt, task, solution) {
-  const workspaceDir = await ensureTaskWorkspace(
-    attempt.attempt_id,
-    task.task_id
-  );
-  const artifacts = [];
-  const logs = [
-    `runner ${attempt.runner_kind} started for task ${task.task_id}`
-  ];
-  const materializedAssets = await Promise.all(
-    task.assets.map(async (asset) => {
-      const artifact = await materializeAsset(
-        workspaceDir,
-        task.task_id,
-        asset
-      );
-      return { artifact, asset };
-    })
-  );
-  for (const { artifact, asset } of materializedAssets) {
-    if (artifact) {
-      artifacts.push(artifact);
-      logs.push(
-        `materialized asset ${asset.asset_id} to ${artifact.path}`
-      );
-    } else {
-      logs.push(`kept asset ${asset.asset_id} inline`);
-    }
-  }
-  const submissionArtifact = await writeSubmissionArtifact(
-    workspaceDir,
-    task,
-    solution
-  );
-  artifacts.push(submissionArtifact);
-  logs.push(`wrote submission artifact ${submissionArtifact.path}`);
-  const facts = {
-    materialized_asset_count: countWorkspaceAssets(task),
-    total_asset_count: task.assets.length,
-    response_format: task.response_format,
-    family: attempt.family
-  };
-  if (attempt.runner_kind === "code_runner") {
-    facts.domain = "code";
-    logs.push("prepared code workspace inputs");
-  } else if (attempt.runner_kind === "browser_runner") {
-    facts.domain = "web";
-    logs.push("prepared browser snapshot workspace");
-  } else {
-    facts.domain = "os";
-    logs.push("prepared os workspace inventory");
-  }
-  const reportArtifact = await writeRunnerReport(
-    workspaceDir,
-    task,
-    logs,
-    facts
-  );
-  artifacts.push(reportArtifact);
-  logs.push(`wrote runner report ${reportArtifact.path}`);
-  return {
-    runner_kind: attempt.runner_kind,
-    workspace_dir: workspaceDir,
-    logs,
-    facts,
-    artifacts
-  };
-}
-
-// src/index.ts
-var PACKAGE_VERSION = getPackageVersion(import.meta.url);
-var server = new McpServer({
-  name: "@fre4x/benchmark",
-  version: PACKAGE_VERSION
-});
+// src/format.ts
 function buildTextResult(text, structuredContent) {
   return {
     content: [{ type: "text", text: truncateToLimit(text) }],
@@ -31669,6 +31521,155 @@ function formatCatalogStatus(status) {
     ])
   ].join("\n");
 }
+
+// src/runners.ts
+import { mkdir as mkdir3, writeFile as writeFile3 } from "node:fs/promises";
+import path3 from "node:path";
+import { randomUUID as randomUUID3 } from "node:crypto";
+function nowStamp() {
+  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+}
+function createArtifact(kind, filePath, mimeType, description) {
+  return {
+    artifact_id: `${kind}_${randomUUID3()}`,
+    kind,
+    path: filePath,
+    mime_type: mimeType,
+    ...description ? { description } : {}
+  };
+}
+async function ensureTaskWorkspace(attemptId, taskId) {
+  const workspaceDir = path3.join(getAttemptWorkspaceDir(attemptId), taskId);
+  await mkdir3(workspaceDir, { recursive: true });
+  return workspaceDir;
+}
+async function materializeAsset(workspaceDir, taskId, asset) {
+  if (asset.materialization !== "workspace_file") {
+    return null;
+  }
+  const assetDir = path3.join(workspaceDir, "assets");
+  await mkdir3(assetDir, { recursive: true });
+  const fileName = `${taskId}-${asset.name}`;
+  const filePath = path3.join(assetDir, fileName);
+  await writeFile3(filePath, asset.content, "utf8");
+  return createArtifact(
+    "materialized_asset",
+    filePath,
+    asset.mime_type,
+    asset.description
+  );
+}
+async function writeSubmissionArtifact(workspaceDir, task, solution) {
+  const extension = task.response_format === "json" ? "json" : "txt";
+  const filePath = path3.join(workspaceDir, `submission.${extension}`);
+  await writeFile3(filePath, solution, "utf8");
+  return createArtifact(
+    "submission",
+    filePath,
+    task.response_format === "json" ? "application/json" : "text/plain",
+    "Submitted solution payload."
+  );
+}
+async function writeRunnerReport(workspaceDir, task, logs, facts) {
+  const reportPath = path3.join(
+    workspaceDir,
+    `runner-report-${nowStamp()}.json`
+  );
+  await writeFile3(
+    reportPath,
+    JSON.stringify(
+      {
+        task_id: task.task_id,
+        logs,
+        facts
+      },
+      null,
+      2
+    ),
+    "utf8"
+  );
+  return createArtifact(
+    "runner_report",
+    reportPath,
+    "application/json",
+    "Execution report with runner facts and logs."
+  );
+}
+function countWorkspaceAssets(task) {
+  return task.assets.filter(
+    (asset) => asset.materialization === "workspace_file"
+  ).length;
+}
+async function executeTaskRunner(attempt, task, solution) {
+  const workspaceDir = await ensureTaskWorkspace(
+    attempt.attempt_id,
+    task.task_id
+  );
+  const artifacts = [];
+  const logs = [
+    `runner ${attempt.runner_kind} started for task ${task.task_id}`
+  ];
+  const materializedAssets = await Promise.all(
+    task.assets.map(async (asset) => {
+      const artifact = await materializeAsset(
+        workspaceDir,
+        task.task_id,
+        asset
+      );
+      return { artifact, asset };
+    })
+  );
+  for (const { artifact, asset } of materializedAssets) {
+    if (artifact) {
+      artifacts.push(artifact);
+      logs.push(
+        `materialized asset ${asset.asset_id} to ${artifact.path}`
+      );
+    } else {
+      logs.push(`kept asset ${asset.asset_id} inline`);
+    }
+  }
+  const submissionArtifact = await writeSubmissionArtifact(
+    workspaceDir,
+    task,
+    solution
+  );
+  artifacts.push(submissionArtifact);
+  logs.push(`wrote submission artifact ${submissionArtifact.path}`);
+  const facts = {
+    materialized_asset_count: countWorkspaceAssets(task),
+    total_asset_count: task.assets.length,
+    response_format: task.response_format,
+    family: attempt.family
+  };
+  if (attempt.runner_kind === "code_runner") {
+    facts.domain = "code";
+    logs.push("prepared code workspace inputs");
+  } else if (attempt.runner_kind === "browser_runner") {
+    facts.domain = "web";
+    logs.push("prepared browser snapshot workspace");
+  } else {
+    facts.domain = "os";
+    logs.push("prepared os workspace inventory");
+  }
+  const reportArtifact = await writeRunnerReport(
+    workspaceDir,
+    task,
+    logs,
+    facts
+  );
+  artifacts.push(reportArtifact);
+  logs.push(`wrote runner report ${reportArtifact.path}`);
+  return {
+    runner_kind: attempt.runner_kind,
+    workspace_dir: workspaceDir,
+    logs,
+    facts,
+    artifacts
+  };
+}
+
+// src/handlers.ts
 async function handleGetCatalogStatus() {
   try {
     const status = await getCatalogStatus();
@@ -32198,6 +32199,13 @@ async function handleCancelAttempt(params) {
     return createInternalError(error48);
   }
 }
+
+// src/index.ts
+var PACKAGE_VERSION = getPackageVersion(import.meta.url);
+var server = new McpServer({
+  name: "@fre4x/benchmark",
+  version: PACKAGE_VERSION
+});
 server.registerTool(
   "list_challenges",
   {
@@ -32289,12 +32297,20 @@ server.registerTool(
   },
   handleCancelAttempt
 );
+function isMainModule(url2) {
+  if (!process.argv[1]) return false;
+  try {
+    return realpathSync(fileURLToPath2(url2)) === realpathSync(process.argv[1]);
+  } catch {
+    return false;
+  }
+}
 async function main() {
   const transport = new StdioServerTransport();
   await server.connect(transport);
   console.error("[benchmark] MCP server running via stdio");
 }
-if (process.argv[1] && import.meta.url.endsWith(process.argv[1])) {
+if (isMainModule(import.meta.url)) {
   main().catch((error48) => {
     console.error("[benchmark] fatal error", error48);
     process.exit(1);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fre4x/benchmark",
-  "version": "1.1.8",
+  "version": "1.1.11",
   "description": "A deterministic benchmark MCP server for agent evaluation workflows.",
   "type": "module",
   "main": "dist/index.js",