@datalayer/agent-runtimes 1.0.4 → 1.0.6

package/lib/specs/agents/index.js

@@ -31,7 +31,9 @@ export function getAgentSpecs(agentId) {
  */
 export function listAgentSpecs(prefix) {
     const specs = Object.values(AGENT_SPECS);
-    return prefix !== undefined ? specs.filter(s => s.id.startsWith(prefix)) : specs;
+    return prefix !== undefined
+        ? specs.filter(s => s.id.startsWith(prefix))
+        : specs;
 }
 /**
  * Collect all required environment variables for an agent spec.

package/lib/specs/benchmarks.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Benchmark Catalog
+ *
+ * Predefined evaluation benchmark configurations.
+ *
+ * This file is AUTO-GENERATED from YAML specifications.
+ * DO NOT EDIT MANUALLY - run 'make specs' to regenerate.
+ */
+import type { BenchmarkSpec } from '../types';
+export declare const AGENTBENCH_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const GPQA_DIAMOND_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const HUMANEVAL_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const MMLU_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const SWE_BENCH_VERIFIED_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const SWE_BENCH_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const TOOLBENCH_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const TRUTHFULQA_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const BENCHMARK_CATALOG: Record<string, BenchmarkSpec>;
+export declare function getBenchmarkSpecs(): BenchmarkSpec[];
+export declare function getBenchmarkSpec(benchmarkId: string): BenchmarkSpec | undefined;

package/lib/specs/benchmarks.js

@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+// ============================================================================
+// Benchmark Definitions
+// ============================================================================
+export const AGENTBENCH_BENCHMARK_SPEC_0_0_1 = {
+    id: 'agentbench',
+    version: '0.0.1',
+    name: 'AgentBench',
+    description: 'Multi-dimensional LLM-as-agent evaluation across 8 diverse environments including web browsing, operating system interaction, database queries, digital card games, lateral thinking, and household tasks.',
+    category: 'Agentic',
+    task_count: 4080,
+    metric: 'success_rate',
+    source: 'https://github.com/THUDM/AgentBench',
+    difficulty: 'hard',
+    languages: ['python', 'sql', 'bash'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: true,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['pass_rate', 'numeric'],
+    evaluators: ['precision-recall-evaluator:0.0.1', 'llm-judge:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const GPQA_DIAMOND_BENCHMARK_SPEC_0_0_1 = {
+    id: 'gpqa-diamond',
+    version: '0.0.1',
+    name: 'GPQA Diamond',
+    description: 'Graduate-level science questions crafted by domain experts. Tests advanced reasoning in physics, chemistry, and biology with questions that require PhD-level understanding to answer correctly.',
+    category: 'Knowledge',
+    task_count: 448,
+    metric: 'accuracy',
+    source: 'https://github.com/idavidrein/gpqa',
+    difficulty: 'expert',
+    languages: ['english'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: false,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['numeric'],
+    evaluators: ['precision-recall-evaluator:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const HUMANEVAL_BENCHMARK_SPEC_0_0_1 = {
+    id: 'humaneval',
+    version: '0.0.1',
+    name: 'HumanEval',
+    description: 'Python function implementation from docstrings. Measures functional correctness of code generation by testing against hand-written test cases. Widely used as a baseline for code generation benchmarks.',
+    category: 'Coding',
+    task_count: 164,
+    metric: 'pass@k',
+    source: 'https://github.com/openai/human-eval',
+    difficulty: 'medium',
+    languages: ['python'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: false,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['pass_rate'],
+    evaluators: ['precision-recall-evaluator:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const MMLU_BENCHMARK_SPEC_0_0_1 = {
+    id: 'mmlu',
+    version: '0.0.1',
+    name: 'MMLU',
+    description: 'Massive Multitask Language Understanding: 57-subject knowledge benchmark spanning STEM, humanities, social sciences, and more. Tests broad knowledge and reasoning across diverse academic domains.',
+    category: 'Knowledge',
+    task_count: 15908,
+    metric: 'accuracy',
+    source: 'https://github.com/hendrycks/test',
+    difficulty: 'medium',
+    languages: ['english'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: false,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['numeric'],
+    evaluators: ['precision-recall-evaluator:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const SWE_BENCH_VERIFIED_BENCHMARK_SPEC_0_0_1 = {
+    id: 'swe-bench-verified',
+    version: '0.0.1',
+    name: 'SWE-bench Verified',
+    description: 'Human-validated subset of SWE-bench with verified ground-truth patches. Provides higher confidence evaluation of software engineering capabilities by eliminating ambiguous or flawed test cases from the full benchmark.',
+    category: 'Coding',
+    task_count: 500,
+    metric: 'pass@1',
+    source: 'https://www.swebench.com/',
+    difficulty: 'hard',
+    languages: ['python'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: true,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['pass_rate'],
+    evaluators: ['precision-recall-evaluator:0.0.1', 'llm-judge:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const SWE_BENCH_BENCHMARK_SPEC_0_0_1 = {
+    id: 'swe-bench',
+    version: '0.0.1',
+    name: 'SWE-bench',
+    description: "Real-world software engineering tasks from GitHub issues. Tests an agent's ability to understand bug reports and feature requests, then produce working code patches that pass existing test suites.",
+    category: 'Coding',
+    task_count: 2294,
+    metric: 'pass@1',
+    source: 'https://www.swebench.com/',
+    difficulty: 'hard',
+    languages: ['python'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: true,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['pass_rate'],
+    evaluators: ['precision-recall-evaluator:0.0.1', 'llm-judge:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const TOOLBENCH_BENCHMARK_SPEC_0_0_1 = {
+    id: 'toolbench',
+    version: '0.0.1',
+    name: 'ToolBench',
+    description: 'Large-scale benchmark for tool-augmented LLMs covering 16000+ real-world APIs across 49 categories. Evaluates multi-step tool usage, API selection, argument generation, and response parsing in complex, chained workflows.',
+    category: 'Agentic',
+    task_count: 12657,
+    metric: 'pass_rate',
+    source: 'https://github.com/OpenBMB/ToolBench',
+    difficulty: 'hard',
+    languages: ['python', 'json'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: true,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['pass_rate', 'numeric'],
+    evaluators: ['precision-recall-evaluator:0.0.1', 'llm-judge:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const TRUTHFULQA_BENCHMARK_SPEC_0_0_1 = {
+    id: 'truthfulqa',
+    version: '0.0.1',
+    name: 'TruthfulQA',
+    description: 'Benchmark measuring whether a language model generates truthful answers to questions spanning 38 categories including health, law, finance, and politics. Designed to test resilience against common human misconceptions and falsehoods that models may have learned from training data.',
+    category: 'Safety',
+    task_count: 817,
+    metric: 'truthful_informative',
+    source: 'https://github.com/sylinrl/TruthfulQA',
+    difficulty: 'medium',
+    languages: ['english'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: false,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['categorical', 'numeric'],
+    evaluators: ['llm-judge:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+// ============================================================================
+// Benchmark Catalog
+// ============================================================================
+export const BENCHMARK_CATALOG = {
+    agentbench: AGENTBENCH_BENCHMARK_SPEC_0_0_1,
+    'gpqa-diamond': GPQA_DIAMOND_BENCHMARK_SPEC_0_0_1,
+    humaneval: HUMANEVAL_BENCHMARK_SPEC_0_0_1,
+    mmlu: MMLU_BENCHMARK_SPEC_0_0_1,
+    'swe-bench-verified': SWE_BENCH_VERIFIED_BENCHMARK_SPEC_0_0_1,
+    'swe-bench': SWE_BENCH_BENCHMARK_SPEC_0_0_1,
+    toolbench: TOOLBENCH_BENCHMARK_SPEC_0_0_1,
+    truthfulqa: TRUTHFULQA_BENCHMARK_SPEC_0_0_1,
+};
+export function getBenchmarkSpecs() {
+    return Object.values(BENCHMARK_CATALOG);
+}
+function resolveBenchmarkId(benchmarkId) {
+    if (benchmarkId in BENCHMARK_CATALOG)
+        return benchmarkId;
+    const idx = benchmarkId.lastIndexOf(':');
+    if (idx > 0) {
+        const base = benchmarkId.slice(0, idx);
+        if (base in BENCHMARK_CATALOG)
+            return base;
+    }
+    return benchmarkId;
+}
+export function getBenchmarkSpec(benchmarkId) {
+    return BENCHMARK_CATALOG[resolveBenchmarkId(benchmarkId)];
+}

package/lib/specs/envvars.js

@@ -11,7 +11,7 @@ export const ALPHAVANTAGE_API_KEY_SPEC_0_0_1 = {
     name: 'Alpha Vantage API Key',
     description: 'API key for accessing Alpha Vantage financial market data and stock information. Provides real-time and historical stock prices, forex data, and cryptocurrency information.',
     registrationUrl: 'https://www.alphavantage.co/support/#api-key',
-    tags: ["authentication", "api-key", "finance", "stocks", "market-data"],
+    tags: ['authentication', 'api-key', 'finance', 'stocks', 'market-data'],
     icon: 'key',
     emoji: '🔑',
 };
@@ -21,7 +21,7 @@ export const GITHUB_TOKEN_SPEC_0_0_1 = {
     name: 'GitHub Token',
     description: 'GitHub API token for repository management and code operations. Required for GitHub MCP server and GitHub skill to interact with GitHub repositories programmatically.',
     registrationUrl: 'https://github.com/settings/tokens',
-    tags: ["authentication", "token", "github", "git", "mcp-server", "skill"],
+    tags: ['authentication', 'token', 'github', 'git', 'mcp-server', 'skill'],
     icon: 'key',
     emoji: '🔑',
 };
@@ -31,7 +31,7 @@ export const GOOGLE_OAUTH_CLIENT_ID_SPEC_0_0_1 = {
     name: 'Google OAuth Client ID',
     description: 'OAuth 2.0 client ID for Google Workspace authentication. Required for Google Drive, Gmail, Calendar, and Docs integration through the Google Workspace MCP server.',
     registrationUrl: 'https://console.cloud.google.com/apis/credentials',
-    tags: ["authentication", "oauth", "google", "workspace", "client-id"],
+    tags: ['authentication', 'oauth', 'google', 'workspace', 'client-id'],
     icon: 'key',
     emoji: '🔑',
 };
@@ -41,7 +41,14 @@ export const GOOGLE_OAUTH_CLIENT_SECRET_SPEC_0_0_1 = {
     name: 'Google OAuth Client Secret',
     description: 'OAuth 2.0 client secret for Google Workspace authentication. Used in conjunction with client ID for secure API access to Google services.',
     registrationUrl: 'https://console.cloud.google.com/apis/credentials',
-    tags: ["authentication", "oauth", "google", "workspace", "client-secret", "security"],
+    tags: [
+        'authentication',
+        'oauth',
+        'google',
+        'workspace',
+        'client-secret',
+        'security',
+    ],
     icon: 'lock',
     emoji: '🔒',
 };
@@ -51,7 +58,7 @@ export const HF_TOKEN_SPEC_0_0_1 = {
     name: 'Hugging Face Token',
     description: 'Access token for Hugging Face API. Required for Hugging Face MCP server authentication. Create a READ token from your settings.',
     registrationUrl: 'https://huggingface.co/settings/tokens',
-    tags: ["authentication", "api-key", "huggingface", "machine-learning"],
+    tags: ['authentication', 'api-key', 'huggingface', 'machine-learning'],
     icon: 'key',
     emoji: '🔑',
 };
@@ -61,7 +68,7 @@ export const KAGGLE_TOKEN_SPEC_0_0_1 = {
     name: 'Kaggle API Token',
     description: 'API token for accessing Kaggle datasets, competitions, notebooks, and models. Required for Kaggle MCP server authentication.',
     registrationUrl: 'https://www.kaggle.com/settings/account',
-    tags: ["authentication", "api-key", "kaggle", "data"],
+    tags: ['authentication', 'api-key', 'kaggle', 'data'],
     icon: 'key',
     emoji: '🔑',
 };
@@ -71,7 +78,7 @@ export const SLACK_BOT_TOKEN_SPEC_0_0_1 = {
     name: 'Slack Bot Token',
     description: 'OAuth token for Slack bot authentication. Required for Slack MCP server to send messages, manage channels, and interact with workspace members.',
     registrationUrl: 'https://api.slack.com/apps',
-    tags: ["authentication", "oauth", "token", "slack", "messaging", "bot"],
+    tags: ['authentication', 'oauth', 'token', 'slack', 'messaging', 'bot'],
     icon: 'key',
     emoji: '🔑',
 };
@@ -80,7 +87,7 @@ export const SLACK_CHANNEL_IDS_SPEC_0_0_1 = {
     version: '0.0.1',
     name: 'Slack Channel IDs',
     description: 'Comma-separated list of Slack channel IDs that the bot is allowed to access. Restricts bot operations to specific channels for security and organization.',
-    tags: ["configuration", "slack", "channels", "identifier"],
+    tags: ['configuration', 'slack', 'channels', 'identifier'],
     icon: 'hash',
     emoji: undefined,
 };
@@ -90,7 +97,7 @@ export const SLACK_TEAM_ID_SPEC_0_0_1 = {
     name: 'Slack Team ID',
     description: 'Unique identifier for the Slack workspace (team). Required to specify which workspace the bot should connect to.',
     registrationUrl: 'https://api.slack.com/apps',
-    tags: ["configuration", "slack", "workspace", "identifier"],
+    tags: ['configuration', 'slack', 'workspace', 'identifier'],
     icon: 'organization',
     emoji: '🏢',
 };
@@ -100,7 +107,7 @@ export const TAVILY_API_KEY_SPEC_0_0_1 = {
     name: 'Tavily API Key',
     description: 'API key for Tavily web search and research capabilities. Required for web crawling, content extraction, and search operations.',
     registrationUrl: 'https://tavily.com/api-keys',
-    tags: ["authentication", "api-key", "search", "web", "research"],
+    tags: ['authentication', 'api-key', 'search', 'web', 'research'],
     icon: 'key',
     emoji: '🔑',
 };
@@ -108,16 +115,16 @@ export const TAVILY_API_KEY_SPEC_0_0_1 = {
 // Environment Variable Catalog
 // ============================================================================
 export const ENVVAR_CATALOG = {
-    'ALPHAVANTAGE_API_KEY': ALPHAVANTAGE_API_KEY_SPEC_0_0_1,
-    'GITHUB_TOKEN': GITHUB_TOKEN_SPEC_0_0_1,
-    'GOOGLE_OAUTH_CLIENT_ID': GOOGLE_OAUTH_CLIENT_ID_SPEC_0_0_1,
-    'GOOGLE_OAUTH_CLIENT_SECRET': GOOGLE_OAUTH_CLIENT_SECRET_SPEC_0_0_1,
-    'HF_TOKEN': HF_TOKEN_SPEC_0_0_1,
-    'KAGGLE_TOKEN': KAGGLE_TOKEN_SPEC_0_0_1,
-    'SLACK_BOT_TOKEN': SLACK_BOT_TOKEN_SPEC_0_0_1,
-    'SLACK_CHANNEL_IDS': SLACK_CHANNEL_IDS_SPEC_0_0_1,
-    'SLACK_TEAM_ID': SLACK_TEAM_ID_SPEC_0_0_1,
-    'TAVILY_API_KEY': TAVILY_API_KEY_SPEC_0_0_1,
+    ALPHAVANTAGE_API_KEY: ALPHAVANTAGE_API_KEY_SPEC_0_0_1,
+    GITHUB_TOKEN: GITHUB_TOKEN_SPEC_0_0_1,
+    GOOGLE_OAUTH_CLIENT_ID: GOOGLE_OAUTH_CLIENT_ID_SPEC_0_0_1,
+    GOOGLE_OAUTH_CLIENT_SECRET: GOOGLE_OAUTH_CLIENT_SECRET_SPEC_0_0_1,
+    HF_TOKEN: HF_TOKEN_SPEC_0_0_1,
+    KAGGLE_TOKEN: KAGGLE_TOKEN_SPEC_0_0_1,
+    SLACK_BOT_TOKEN: SLACK_BOT_TOKEN_SPEC_0_0_1,
+    SLACK_CHANNEL_IDS: SLACK_CHANNEL_IDS_SPEC_0_0_1,
+    SLACK_TEAM_ID: SLACK_TEAM_ID_SPEC_0_0_1,
+    TAVILY_API_KEY: TAVILY_API_KEY_SPEC_0_0_1,
 };
 function resolveEnvvarId(envvarId) {
     if (envvarId in ENVVAR_CATALOG)

package/lib/specs/evals.d.ts

@@ -1,20 +1,21 @@
 /**
  * Eval Catalog
  *
- * Predefined evaluation benchmark configurations.
+ * Predefined built-in evaluator configurations.
  *
  * This file is AUTO-GENERATED from YAML specifications.
  * DO NOT EDIT MANUALLY - run 'make specs' to regenerate.
  */
 import type { EvalSpec } from '../types';
-export declare const AGENTBENCH_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const GPQA_DIAMOND_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const HUMANEVAL_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const MMLU_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const SWE_BENCH_VERIFIED_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const SWE_BENCH_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const TOOLBENCH_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const TRUTHFULQA_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const CONFUSION_MATRIX_EVALUATOR_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const CONTAINS_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const EQUALS_EXPECTED_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const EQUALS_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const HAS_MATCHING_SPAN_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const IS_INSTANCE_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const LLM_JUDGE_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const MAX_DURATION_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const PRECISION_RECALL_EVALUATOR_EVAL_SPEC_0_0_1: EvalSpec;
 export declare const EVAL_CATALOG: Record<string, EvalSpec>;
 export declare function getEvalSpecs(): EvalSpec[];
 export declare function getEvalSpec(evalId: string): EvalSpec | undefined;

package/lib/specs/evals.js

@@ -5,114 +5,154 @@
 // ============================================================================
 // Eval Definitions
 // ============================================================================
-export const AGENTBENCH_EVAL_SPEC_0_0_1 = {
-    id: 'agentbench',
+export const CONFUSION_MATRIX_EVALUATOR_EVAL_SPEC_0_0_1 = {
+    id: 'confusion-matrix-evaluator',
     version: '0.0.1',
-    name: 'AgentBench',
-    description: 'Multi-dimensional LLM-as-agent evaluation across 8 diverse environments including web browsing, operating system interaction, database queries, digital card games, lateral thinking, and household tasks.',
-    category: 'Agentic',
-    task_count: 4080,
-    metric: 'success_rate',
-    source: 'https://github.com/THUDM/AgentBench',
-    difficulty: 'hard',
-    languages: ['python', 'sql', 'bash'],
+    name: 'Confusion Matrix Evaluator',
+    description: 'Aggregate evaluator for precision/recall style confusion-matrix reporting.',
+    category: 'Report',
+    evaluator_type: 'report',
+    pydantic_class: 'ConfusionMatrixEvaluator',
+    output_kind: 'report_table',
+    cost_tier: 'free',
+    latency: 'fast',
+    requires: ['expected_output'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const GPQA_DIAMOND_EVAL_SPEC_0_0_1 = {
-    id: 'gpqa-diamond',
+export const CONTAINS_EVAL_SPEC_0_0_1 = {
+    id: 'contains',
     version: '0.0.1',
-    name: 'GPQA Diamond',
-    description: 'Graduate-level science questions crafted by domain experts. Tests advanced reasoning in physics, chemistry, and biology with questions that require PhD-level understanding to answer correctly.',
-    category: 'Reasoning',
-    task_count: 448,
-    metric: 'accuracy',
-    source: 'https://github.com/idavidrein/gpqa',
-    difficulty: 'expert',
-    languages: ['english'],
+    name: 'Contains',
+    description: 'Assert that expected content appears in the model output.',
+    category: 'Comparison',
+    evaluator_type: 'case',
+    pydantic_class: 'ContainsEvaluator',
+    output_kind: 'boolean',
+    cost_tier: 'free',
+    latency: 'instant',
+    requires: ['expected_output'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const HUMANEVAL_EVAL_SPEC_0_0_1 = {
-    id: 'humaneval',
+export const EQUALS_EXPECTED_EVAL_SPEC_0_0_1 = {
+    id: 'equals-expected',
     version: '0.0.1',
-    name: 'HumanEval',
-    description: 'Python function implementation from docstrings. Measures functional correctness of code generation by testing against hand-written test cases. Widely used as a baseline for code generation benchmarks.',
-    category: 'Coding',
-    task_count: 164,
-    metric: 'pass@k',
-    source: 'https://github.com/openai/human-eval',
-    difficulty: 'medium',
-    languages: ['python'],
+    name: 'Equals Expected',
+    description: 'Compare model output against an expected value with strict matching.',
+    category: 'Comparison',
+    evaluator_type: 'case',
+    pydantic_class: 'EqualsExpectedEvaluator',
+    output_kind: 'boolean',
+    cost_tier: 'free',
+    latency: 'instant',
+    requires: ['expected_output'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const MMLU_EVAL_SPEC_0_0_1 = {
-    id: 'mmlu',
+export const EQUALS_EVAL_SPEC_0_0_1 = {
+    id: 'equals',
     version: '0.0.1',
-    name: 'MMLU',
-    description: 'Massive Multitask Language Understanding: 57-subject knowledge benchmark spanning STEM, humanities, social sciences, and more. Tests broad knowledge and reasoning across diverse academic domains.',
-    category: 'Knowledge',
-    task_count: 15908,
-    metric: 'accuracy',
-    source: 'https://github.com/hendrycks/test',
-    difficulty: 'medium',
-    languages: ['english'],
+    name: 'Equals',
+    description: 'Assert exact equality between expected and actual values.',
+    category: 'Comparison',
+    evaluator_type: 'case',
+    pydantic_class: 'EqualsEvaluator',
+    output_kind: 'boolean',
+    cost_tier: 'free',
+    latency: 'instant',
+    requires: ['expected_output'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const SWE_BENCH_VERIFIED_EVAL_SPEC_0_0_1 = {
-    id: 'swe-bench-verified',
+export const HAS_MATCHING_SPAN_EVAL_SPEC_0_0_1 = {
+    id: 'has-matching-span',
     version: '0.0.1',
-    name: 'SWE-bench Verified',
-    description: 'Human-validated subset of SWE-bench with verified ground-truth patches. Provides higher confidence evaluation of software engineering capabilities by eliminating ambiguous or flawed test cases from the full benchmark.',
-    category: 'Coding',
-    task_count: 500,
-    metric: 'pass@1',
-    source: 'https://www.swebench.com/',
-    difficulty: 'hard',
-    languages: ['python'],
+    name: 'Has Matching Span',
+    description: 'Validate expected spans in structured traces and tool-call transcripts.',
+    category: 'Span-Based',
+    evaluator_type: 'case',
+    pydantic_class: 'HasMatchingSpanEvaluator',
+    output_kind: 'boolean',
+    cost_tier: 'free',
+    latency: 'fast',
+    requires: ['trace'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const SWE_BENCH_EVAL_SPEC_0_0_1 = {
-    id: 'swe-bench',
+export const IS_INSTANCE_EVAL_SPEC_0_0_1 = {
+    id: 'is-instance',
     version: '0.0.1',
-    name: 'SWE-bench',
-    description: 'Real-world software engineering tasks from GitHub issues. Tests an agent\'s ability to understand bug reports and feature requests, then produce working code patches that pass existing test suites.',
-    category: 'Coding',
-    task_count: 2294,
-    metric: 'pass@1',
-    source: 'https://www.swebench.com/',
-    difficulty: 'hard',
-    languages: ['python'],
+    name: 'Is Instance',
+    description: 'Validate output type against an expected Python/JSON schema type.',
+    category: 'Type Validation',
+    evaluator_type: 'case',
+    pydantic_class: 'IsInstanceEvaluator',
+    output_kind: 'boolean',
+    cost_tier: 'free',
+    latency: 'instant',
+    requires: ['expected_type'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const TOOLBENCH_EVAL_SPEC_0_0_1 = {
-    id: 'toolbench',
+export const LLM_JUDGE_EVAL_SPEC_0_0_1 = {
+    id: 'llm-judge',
     version: '0.0.1',
-    name: 'ToolBench',
-    description: 'Large-scale benchmark for tool-augmented LLMs covering 16000+ real-world APIs across 49 categories. Evaluates multi-step tool usage, API selection, argument generation, and response parsing in complex, chained workflows.',
-    category: 'Agentic',
-    task_count: 12657,
-    metric: 'pass_rate',
-    source: 'https://github.com/OpenBMB/ToolBench',
-    difficulty: 'hard',
-    languages: ['python', 'json'],
+    name: 'LLM Judge',
+    description: 'Use an LLM-as-a-judge prompt to score quality and provide rationale.',
+    category: 'LLM-as-a-Judge',
+    evaluator_type: 'case',
+    pydantic_class: 'LLMJudgeEvaluator',
+    output_kind: 'score_and_assertion',
+    cost_tier: 'llm',
+    latency: 'slow',
+    requires: ['model'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: { threshold: 0.7 },
 };
-export const TRUTHFULQA_EVAL_SPEC_0_0_1 = {
-    id: 'truthfulqa',
+export const MAX_DURATION_EVAL_SPEC_0_0_1 = {
+    id: 'max-duration',
     version: '0.0.1',
-    name: 'TruthfulQA',
-    description: 'Benchmark measuring whether a language model generates truthful answers to questions spanning 38 categories including health, law, finance, and politics. Designed to test resilience against common human misconceptions and falsehoods that models may have learned from training data.',
-    category: 'Safety',
-    task_count: 817,
-    metric: 'truthful_informative',
-    source: 'https://github.com/sylinrl/TruthfulQA',
-    difficulty: 'medium',
-    languages: ['english'],
+    name: 'Max Duration',
+    description: 'Assert response latency remains below a configured duration threshold.',
+    category: 'Performance',
+    evaluator_type: 'case',
+    pydantic_class: 'MaxDurationEvaluator',
+    output_kind: 'boolean_with_reason',
+    cost_tier: 'free',
+    latency: 'instant',
+    requires: ['duration_ms'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: { max_duration_ms: 5000 },
+};
+export const PRECISION_RECALL_EVALUATOR_EVAL_SPEC_0_0_1 = {
+    id: 'precision-recall-evaluator',
+    version: '0.0.1',
+    name: 'Precision Recall Evaluator',
+    description: 'Aggregate evaluator for precision, recall, and pass-rate style benchmark reporting.',
+    category: 'Report',
+    evaluator_type: 'report',
+    pydantic_class: 'PrecisionRecallEvaluator',
+    output_kind: 'report_curve',
+    cost_tier: 'free',
+    latency: 'fast',
+    requires: ['expected_output'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
 // ============================================================================
 // Eval Catalog
 // ============================================================================
 export const EVAL_CATALOG = {
-    'agentbench': AGENTBENCH_EVAL_SPEC_0_0_1,
-    'gpqa-diamond': GPQA_DIAMOND_EVAL_SPEC_0_0_1,
-    'humaneval': HUMANEVAL_EVAL_SPEC_0_0_1,
-    'mmlu': MMLU_EVAL_SPEC_0_0_1,
-    'swe-bench-verified': SWE_BENCH_VERIFIED_EVAL_SPEC_0_0_1,
-    'swe-bench': SWE_BENCH_EVAL_SPEC_0_0_1,
-    'toolbench': TOOLBENCH_EVAL_SPEC_0_0_1,
-    'truthfulqa': TRUTHFULQA_EVAL_SPEC_0_0_1,
+    'confusion-matrix-evaluator': CONFUSION_MATRIX_EVALUATOR_EVAL_SPEC_0_0_1,
+    contains: CONTAINS_EVAL_SPEC_0_0_1,
+    'equals-expected': EQUALS_EXPECTED_EVAL_SPEC_0_0_1,
+    equals: EQUALS_EVAL_SPEC_0_0_1,
+    'has-matching-span': HAS_MATCHING_SPAN_EVAL_SPEC_0_0_1,
+    'is-instance': IS_INSTANCE_EVAL_SPEC_0_0_1,
+    'llm-judge': LLM_JUDGE_EVAL_SPEC_0_0_1,
+    'max-duration': MAX_DURATION_EVAL_SPEC_0_0_1,
+    'precision-recall-evaluator': PRECISION_RECALL_EVALUATOR_EVAL_SPEC_0_0_1,
 };
 export function getEvalSpecs() {
     return Object.values(EVAL_CATALOG);