@datalayer/agent-runtimes 1.0.5 → 1.0.6

package/lib/specs/benchmarks.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Benchmark Catalog
+ *
+ * Predefined evaluation benchmark configurations.
+ *
+ * This file is AUTO-GENERATED from YAML specifications.
+ * DO NOT EDIT MANUALLY - run 'make specs' to regenerate.
+ */
+import type { BenchmarkSpec } from '../types';
+export declare const AGENTBENCH_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const GPQA_DIAMOND_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const HUMANEVAL_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const MMLU_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const SWE_BENCH_VERIFIED_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const SWE_BENCH_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const TOOLBENCH_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const TRUTHFULQA_BENCHMARK_SPEC_0_0_1: BenchmarkSpec;
+export declare const BENCHMARK_CATALOG: Record<string, BenchmarkSpec>;
+export declare function getBenchmarkSpecs(): BenchmarkSpec[];
+export declare function getBenchmarkSpec(benchmarkId: string): BenchmarkSpec | undefined;

package/lib/specs/benchmarks.js

@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2025-2026 Datalayer, Inc.
+ * Distributed under the terms of the Modified BSD License.
+ */
+// ============================================================================
+// Benchmark Definitions
+// ============================================================================
+export const AGENTBENCH_BENCHMARK_SPEC_0_0_1 = {
+    id: 'agentbench',
+    version: '0.0.1',
+    name: 'AgentBench',
+    description: 'Multi-dimensional LLM-as-agent evaluation across 8 diverse environments including web browsing, operating system interaction, database queries, digital card games, lateral thinking, and household tasks.',
+    category: 'Agentic',
+    task_count: 4080,
+    metric: 'success_rate',
+    source: 'https://github.com/THUDM/AgentBench',
+    difficulty: 'hard',
+    languages: ['python', 'sql', 'bash'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: true,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['pass_rate', 'numeric'],
+    evaluators: ['precision-recall-evaluator:0.0.1', 'llm-judge:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const GPQA_DIAMOND_BENCHMARK_SPEC_0_0_1 = {
+    id: 'gpqa-diamond',
+    version: '0.0.1',
+    name: 'GPQA Diamond',
+    description: 'Graduate-level science questions crafted by domain experts. Tests advanced reasoning in physics, chemistry, and biology with questions that require PhD-level understanding to answer correctly.',
+    category: 'Knowledge',
+    task_count: 448,
+    metric: 'accuracy',
+    source: 'https://github.com/idavidrein/gpqa',
+    difficulty: 'expert',
+    languages: ['english'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: false,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['numeric'],
+    evaluators: ['precision-recall-evaluator:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const HUMANEVAL_BENCHMARK_SPEC_0_0_1 = {
+    id: 'humaneval',
+    version: '0.0.1',
+    name: 'HumanEval',
+    description: 'Python function implementation from docstrings. Measures functional correctness of code generation by testing against hand-written test cases. Widely used as a baseline for code generation benchmarks.',
+    category: 'Coding',
+    task_count: 164,
+    metric: 'pass@k',
+    source: 'https://github.com/openai/human-eval',
+    difficulty: 'medium',
+    languages: ['python'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: false,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['pass_rate'],
+    evaluators: ['precision-recall-evaluator:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const MMLU_BENCHMARK_SPEC_0_0_1 = {
+    id: 'mmlu',
+    version: '0.0.1',
+    name: 'MMLU',
+    description: 'Massive Multitask Language Understanding: 57-subject knowledge benchmark spanning STEM, humanities, social sciences, and more. Tests broad knowledge and reasoning across diverse academic domains.',
+    category: 'Knowledge',
+    task_count: 15908,
+    metric: 'accuracy',
+    source: 'https://github.com/hendrycks/test',
+    difficulty: 'medium',
+    languages: ['english'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: false,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['numeric'],
+    evaluators: ['precision-recall-evaluator:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const SWE_BENCH_VERIFIED_BENCHMARK_SPEC_0_0_1 = {
+    id: 'swe-bench-verified',
+    version: '0.0.1',
+    name: 'SWE-bench Verified',
+    description: 'Human-validated subset of SWE-bench with verified ground-truth patches. Provides higher confidence evaluation of software engineering capabilities by eliminating ambiguous or flawed test cases from the full benchmark.',
+    category: 'Coding',
+    task_count: 500,
+    metric: 'pass@1',
+    source: 'https://www.swebench.com/',
+    difficulty: 'hard',
+    languages: ['python'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: true,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['pass_rate'],
+    evaluators: ['precision-recall-evaluator:0.0.1', 'llm-judge:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const SWE_BENCH_BENCHMARK_SPEC_0_0_1 = {
+    id: 'swe-bench',
+    version: '0.0.1',
+    name: 'SWE-bench',
+    description: "Real-world software engineering tasks from GitHub issues. Tests an agent's ability to understand bug reports and feature requests, then produce working code patches that pass existing test suites.",
+    category: 'Coding',
+    task_count: 2294,
+    metric: 'pass@1',
+    source: 'https://www.swebench.com/',
+    difficulty: 'hard',
+    languages: ['python'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: true,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['pass_rate'],
+    evaluators: ['precision-recall-evaluator:0.0.1', 'llm-judge:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const TOOLBENCH_BENCHMARK_SPEC_0_0_1 = {
+    id: 'toolbench',
+    version: '0.0.1',
+    name: 'ToolBench',
+    description: 'Large-scale benchmark for tool-augmented LLMs covering 16000+ real-world APIs across 49 categories. Evaluates multi-step tool usage, API selection, argument generation, and response parsing in complex, chained workflows.',
+    category: 'Agentic',
+    task_count: 12657,
+    metric: 'pass_rate',
+    source: 'https://github.com/OpenBMB/ToolBench',
+    difficulty: 'hard',
+    languages: ['python', 'json'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: true,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['pass_rate', 'numeric'],
+    evaluators: ['precision-recall-evaluator:0.0.1', 'llm-judge:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+export const TRUTHFULQA_BENCHMARK_SPEC_0_0_1 = {
+    id: 'truthfulqa',
+    version: '0.0.1',
+    name: 'TruthfulQA',
+    description: 'Benchmark measuring whether a language model generates truthful answers to questions spanning 38 categories including health, law, finance, and politics. Designed to test resilience against common human misconceptions and falsehoods that models may have learned from training data.',
+    category: 'Safety',
+    task_count: 817,
+    metric: 'truthful_informative',
+    source: 'https://github.com/sylinrl/TruthfulQA',
+    difficulty: 'medium',
+    languages: ['english'],
+    dataset_source: 'hosted',
+    supports_live_monitoring: false,
+    supports_experiment_comparison: true,
+    evaluator_shapes: ['categorical', 'numeric'],
+    evaluators: ['llm-judge:0.0.1'],
+    recommended_windows: ['1h', '6h', '24h', '7d', '30d'],
+    trace_integration: true,
+    dataset_editability: 'read-only',
+    sdk_support: 'experimental',
+};
+// ============================================================================
+// Benchmark Catalog
+// ============================================================================
+export const BENCHMARK_CATALOG = {
+    agentbench: AGENTBENCH_BENCHMARK_SPEC_0_0_1,
+    'gpqa-diamond': GPQA_DIAMOND_BENCHMARK_SPEC_0_0_1,
+    humaneval: HUMANEVAL_BENCHMARK_SPEC_0_0_1,
+    mmlu: MMLU_BENCHMARK_SPEC_0_0_1,
+    'swe-bench-verified': SWE_BENCH_VERIFIED_BENCHMARK_SPEC_0_0_1,
+    'swe-bench': SWE_BENCH_BENCHMARK_SPEC_0_0_1,
+    toolbench: TOOLBENCH_BENCHMARK_SPEC_0_0_1,
+    truthfulqa: TRUTHFULQA_BENCHMARK_SPEC_0_0_1,
+};
+export function getBenchmarkSpecs() {
+    return Object.values(BENCHMARK_CATALOG);
+}
+function resolveBenchmarkId(benchmarkId) {
+    if (benchmarkId in BENCHMARK_CATALOG)
+        return benchmarkId;
+    const idx = benchmarkId.lastIndexOf(':');
+    if (idx > 0) {
+        const base = benchmarkId.slice(0, idx);
+        if (base in BENCHMARK_CATALOG)
+            return base;
+    }
+    return benchmarkId;
+}
+export function getBenchmarkSpec(benchmarkId) {
+    return BENCHMARK_CATALOG[resolveBenchmarkId(benchmarkId)];
+}

package/lib/specs/envvars.d.ts

@@ -8,7 +8,6 @@
  */
 import type { EnvvarSpec } from '../types';
 export declare const ALPHAVANTAGE_API_KEY_SPEC_0_0_1: EnvvarSpec;
-export declare const DATALAYER_API_KEY_SPEC_0_0_1: EnvvarSpec;
 export declare const GITHUB_TOKEN_SPEC_0_0_1: EnvvarSpec;
 export declare const GOOGLE_OAUTH_CLIENT_ID_SPEC_0_0_1: EnvvarSpec;
 export declare const GOOGLE_OAUTH_CLIENT_SECRET_SPEC_0_0_1: EnvvarSpec;

package/lib/specs/envvars.js

@@ -15,16 +15,6 @@ export const ALPHAVANTAGE_API_KEY_SPEC_0_0_1 = {
     icon: 'key',
     emoji: '🔑',
 };
-export const DATALAYER_API_KEY_SPEC_0_0_1 = {
-    id: 'DATALAYER_API_KEY',
-    version: '0.0.1',
-    name: 'Datalayer API Key',
-    description: 'API key for authenticating with Datalayer services, including datalayer-skills such as datalayer-whoami.',
-    registrationUrl: 'https://datalayer.app',
-    tags: ['authentication', 'api-key', 'datalayer', 'skill'],
-    icon: 'key',
-    emoji: '🔑',
-};
 export const GITHUB_TOKEN_SPEC_0_0_1 = {
     id: 'GITHUB_TOKEN',
     version: '0.0.1',
@@ -126,7 +116,6 @@ export const TAVILY_API_KEY_SPEC_0_0_1 = {
 // ============================================================================
 export const ENVVAR_CATALOG = {
     ALPHAVANTAGE_API_KEY: ALPHAVANTAGE_API_KEY_SPEC_0_0_1,
-    DATALAYER_API_KEY: DATALAYER_API_KEY_SPEC_0_0_1,
     GITHUB_TOKEN: GITHUB_TOKEN_SPEC_0_0_1,
     GOOGLE_OAUTH_CLIENT_ID: GOOGLE_OAUTH_CLIENT_ID_SPEC_0_0_1,
     GOOGLE_OAUTH_CLIENT_SECRET: GOOGLE_OAUTH_CLIENT_SECRET_SPEC_0_0_1,

package/lib/specs/evals.d.ts

@@ -1,20 +1,21 @@
 /**
  * Eval Catalog
  *
- * Predefined evaluation benchmark configurations.
+ * Predefined built-in evaluator configurations.
  *
  * This file is AUTO-GENERATED from YAML specifications.
  * DO NOT EDIT MANUALLY - run 'make specs' to regenerate.
  */
 import type { EvalSpec } from '../types';
-export declare const AGENTBENCH_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const GPQA_DIAMOND_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const HUMANEVAL_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const MMLU_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const SWE_BENCH_VERIFIED_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const SWE_BENCH_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const TOOLBENCH_EVAL_SPEC_0_0_1: EvalSpec;
-export declare const TRUTHFULQA_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const CONFUSION_MATRIX_EVALUATOR_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const CONTAINS_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const EQUALS_EXPECTED_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const EQUALS_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const HAS_MATCHING_SPAN_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const IS_INSTANCE_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const LLM_JUDGE_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const MAX_DURATION_EVAL_SPEC_0_0_1: EvalSpec;
+export declare const PRECISION_RECALL_EVALUATOR_EVAL_SPEC_0_0_1: EvalSpec;
 export declare const EVAL_CATALOG: Record<string, EvalSpec>;
 export declare function getEvalSpecs(): EvalSpec[];
 export declare function getEvalSpec(evalId: string): EvalSpec | undefined;

package/lib/specs/evals.js

@@ -5,114 +5,154 @@
 // ============================================================================
 // Eval Definitions
 // ============================================================================
-export const AGENTBENCH_EVAL_SPEC_0_0_1 = {
-    id: 'agentbench',
+export const CONFUSION_MATRIX_EVALUATOR_EVAL_SPEC_0_0_1 = {
+    id: 'confusion-matrix-evaluator',
     version: '0.0.1',
-    name: 'AgentBench',
-    description: 'Multi-dimensional LLM-as-agent evaluation across 8 diverse environments including web browsing, operating system interaction, database queries, digital card games, lateral thinking, and household tasks.',
-    category: 'Agentic',
-    task_count: 4080,
-    metric: 'success_rate',
-    source: 'https://github.com/THUDM/AgentBench',
-    difficulty: 'hard',
-    languages: ['python', 'sql', 'bash'],
+    name: 'Confusion Matrix Evaluator',
+    description: 'Aggregate evaluator for precision/recall style confusion-matrix reporting.',
+    category: 'Report',
+    evaluator_type: 'report',
+    pydantic_class: 'ConfusionMatrixEvaluator',
+    output_kind: 'report_table',
+    cost_tier: 'free',
+    latency: 'fast',
+    requires: ['expected_output'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const GPQA_DIAMOND_EVAL_SPEC_0_0_1 = {
-    id: 'gpqa-diamond',
+export const CONTAINS_EVAL_SPEC_0_0_1 = {
+    id: 'contains',
     version: '0.0.1',
-    name: 'GPQA Diamond',
-    description: 'Graduate-level science questions crafted by domain experts. Tests advanced reasoning in physics, chemistry, and biology with questions that require PhD-level understanding to answer correctly.',
-    category: 'Reasoning',
-    task_count: 448,
-    metric: 'accuracy',
-    source: 'https://github.com/idavidrein/gpqa',
-    difficulty: 'expert',
-    languages: ['english'],
+    name: 'Contains',
+    description: 'Assert that expected content appears in the model output.',
+    category: 'Comparison',
+    evaluator_type: 'case',
+    pydantic_class: 'ContainsEvaluator',
+    output_kind: 'boolean',
+    cost_tier: 'free',
+    latency: 'instant',
+    requires: ['expected_output'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const HUMANEVAL_EVAL_SPEC_0_0_1 = {
-    id: 'humaneval',
+export const EQUALS_EXPECTED_EVAL_SPEC_0_0_1 = {
+    id: 'equals-expected',
     version: '0.0.1',
-    name: 'HumanEval',
-    description: 'Python function implementation from docstrings. Measures functional correctness of code generation by testing against hand-written test cases. Widely used as a baseline for code generation benchmarks.',
-    category: 'Coding',
-    task_count: 164,
-    metric: 'pass@k',
-    source: 'https://github.com/openai/human-eval',
-    difficulty: 'medium',
-    languages: ['python'],
+    name: 'Equals Expected',
+    description: 'Compare model output against an expected value with strict matching.',
+    category: 'Comparison',
+    evaluator_type: 'case',
+    pydantic_class: 'EqualsExpectedEvaluator',
+    output_kind: 'boolean',
+    cost_tier: 'free',
+    latency: 'instant',
+    requires: ['expected_output'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const MMLU_EVAL_SPEC_0_0_1 = {
-    id: 'mmlu',
+export const EQUALS_EVAL_SPEC_0_0_1 = {
+    id: 'equals',
     version: '0.0.1',
-    name: 'MMLU',
-    description: 'Massive Multitask Language Understanding: 57-subject knowledge benchmark spanning STEM, humanities, social sciences, and more. Tests broad knowledge and reasoning across diverse academic domains.',
-    category: 'Knowledge',
-    task_count: 15908,
-    metric: 'accuracy',
-    source: 'https://github.com/hendrycks/test',
-    difficulty: 'medium',
-    languages: ['english'],
+    name: 'Equals',
+    description: 'Assert exact equality between expected and actual values.',
+    category: 'Comparison',
+    evaluator_type: 'case',
+    pydantic_class: 'EqualsEvaluator',
+    output_kind: 'boolean',
+    cost_tier: 'free',
+    latency: 'instant',
+    requires: ['expected_output'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const SWE_BENCH_VERIFIED_EVAL_SPEC_0_0_1 = {
-    id: 'swe-bench-verified',
+export const HAS_MATCHING_SPAN_EVAL_SPEC_0_0_1 = {
+    id: 'has-matching-span',
     version: '0.0.1',
-    name: 'SWE-bench Verified',
-    description: 'Human-validated subset of SWE-bench with verified ground-truth patches. Provides higher confidence evaluation of software engineering capabilities by eliminating ambiguous or flawed test cases from the full benchmark.',
-    category: 'Coding',
-    task_count: 500,
-    metric: 'pass@1',
-    source: 'https://www.swebench.com/',
-    difficulty: 'hard',
-    languages: ['python'],
+    name: 'Has Matching Span',
+    description: 'Validate expected spans in structured traces and tool-call transcripts.',
+    category: 'Span-Based',
+    evaluator_type: 'case',
+    pydantic_class: 'HasMatchingSpanEvaluator',
+    output_kind: 'boolean',
+    cost_tier: 'free',
+    latency: 'fast',
+    requires: ['trace'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const SWE_BENCH_EVAL_SPEC_0_0_1 = {
-    id: 'swe-bench',
+export const IS_INSTANCE_EVAL_SPEC_0_0_1 = {
+    id: 'is-instance',
     version: '0.0.1',
-    name: 'SWE-bench',
-    description: "Real-world software engineering tasks from GitHub issues. Tests an agent's ability to understand bug reports and feature requests, then produce working code patches that pass existing test suites.",
-    category: 'Coding',
-    task_count: 2294,
-    metric: 'pass@1',
-    source: 'https://www.swebench.com/',
-    difficulty: 'hard',
-    languages: ['python'],
+    name: 'Is Instance',
+    description: 'Validate output type against an expected Python/JSON schema type.',
+    category: 'Type Validation',
+    evaluator_type: 'case',
+    pydantic_class: 'IsInstanceEvaluator',
+    output_kind: 'boolean',
+    cost_tier: 'free',
+    latency: 'instant',
+    requires: ['expected_type'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
-export const TOOLBENCH_EVAL_SPEC_0_0_1 = {
-    id: 'toolbench',
+export const LLM_JUDGE_EVAL_SPEC_0_0_1 = {
+    id: 'llm-judge',
     version: '0.0.1',
-    name: 'ToolBench',
-    description: 'Large-scale benchmark for tool-augmented LLMs covering 16000+ real-world APIs across 49 categories. Evaluates multi-step tool usage, API selection, argument generation, and response parsing in complex, chained workflows.',
-    category: 'Agentic',
-    task_count: 12657,
-    metric: 'pass_rate',
-    source: 'https://github.com/OpenBMB/ToolBench',
-    difficulty: 'hard',
-    languages: ['python', 'json'],
+    name: 'LLM Judge',
+    description: 'Use an LLM-as-a-judge prompt to score quality and provide rationale.',
+    category: 'LLM-as-a-Judge',
+    evaluator_type: 'case',
+    pydantic_class: 'LLMJudgeEvaluator',
+    output_kind: 'score_and_assertion',
+    cost_tier: 'llm',
+    latency: 'slow',
+    requires: ['model'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: { threshold: 0.7 },
 };
-export const TRUTHFULQA_EVAL_SPEC_0_0_1 = {
-    id: 'truthfulqa',
+export const MAX_DURATION_EVAL_SPEC_0_0_1 = {
+    id: 'max-duration',
     version: '0.0.1',
-    name: 'TruthfulQA',
-    description: 'Benchmark measuring whether a language model generates truthful answers to questions spanning 38 categories including health, law, finance, and politics. Designed to test resilience against common human misconceptions and falsehoods that models may have learned from training data.',
-    category: 'Safety',
-    task_count: 817,
-    metric: 'truthful_informative',
-    source: 'https://github.com/sylinrl/TruthfulQA',
-    difficulty: 'medium',
-    languages: ['english'],
+    name: 'Max Duration',
+    description: 'Assert response latency remains below a configured duration threshold.',
+    category: 'Performance',
+    evaluator_type: 'case',
+    pydantic_class: 'MaxDurationEvaluator',
+    output_kind: 'boolean_with_reason',
+    cost_tier: 'free',
+    latency: 'instant',
+    requires: ['duration_ms'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: { max_duration_ms: 5000 },
+};
+export const PRECISION_RECALL_EVALUATOR_EVAL_SPEC_0_0_1 = {
+    id: 'precision-recall-evaluator',
+    version: '0.0.1',
+    name: 'Precision Recall Evaluator',
+    description: 'Aggregate evaluator for precision, recall, and pass-rate style benchmark reporting.',
+    category: 'Report',
+    evaluator_type: 'report',
+    pydantic_class: 'PrecisionRecallEvaluator',
+    output_kind: 'report_curve',
+    cost_tier: 'free',
+    latency: 'fast',
+    requires: ['expected_output'],
+    source: 'https://ai.pydantic.dev/evals/',
+    default_config: {},
 };
 // ============================================================================
 // Eval Catalog
 // ============================================================================
 export const EVAL_CATALOG = {
-    agentbench: AGENTBENCH_EVAL_SPEC_0_0_1,
-    'gpqa-diamond': GPQA_DIAMOND_EVAL_SPEC_0_0_1,
-    humaneval: HUMANEVAL_EVAL_SPEC_0_0_1,
-    mmlu: MMLU_EVAL_SPEC_0_0_1,
-    'swe-bench-verified': SWE_BENCH_VERIFIED_EVAL_SPEC_0_0_1,
-    'swe-bench': SWE_BENCH_EVAL_SPEC_0_0_1,
-    toolbench: TOOLBENCH_EVAL_SPEC_0_0_1,
-    truthfulqa: TRUTHFULQA_EVAL_SPEC_0_0_1,
+    'confusion-matrix-evaluator': CONFUSION_MATRIX_EVALUATOR_EVAL_SPEC_0_0_1,
+    contains: CONTAINS_EVAL_SPEC_0_0_1,
+    'equals-expected': EQUALS_EXPECTED_EVAL_SPEC_0_0_1,
+    equals: EQUALS_EVAL_SPEC_0_0_1,
+    'has-matching-span': HAS_MATCHING_SPAN_EVAL_SPEC_0_0_1,
+    'is-instance': IS_INSTANCE_EVAL_SPEC_0_0_1,
+    'llm-judge': LLM_JUDGE_EVAL_SPEC_0_0_1,
+    'max-duration': MAX_DURATION_EVAL_SPEC_0_0_1,
+    'precision-recall-evaluator': PRECISION_RECALL_EVALUATOR_EVAL_SPEC_0_0_1,
 };
 export function getEvalSpecs() {
     return Object.values(EVAL_CATALOG);

package/lib/specs/index.d.ts

@@ -13,7 +13,6 @@ export * from './memory';
 export * from './models';
 export * from './notifications';
 export * from './outputs';
-export * from './personas';
 export * from './skills';
 export * from './tools';
 export * from './triggers';

package/lib/specs/index.js

@@ -17,7 +17,6 @@ export * from './memory';
 export * from './models';
 export * from './notifications';
 export * from './outputs';
-export * from './personas';
 export * from './skills';
 export * from './tools';
 export * from './triggers';

package/lib/specs/models.d.ts

@@ -17,7 +17,6 @@ export declare const AIModels: {
     readonly AZURE_OPENAI_GPT_4_1: "azure-openai:gpt-4.1";
     readonly AZURE_OPENAI_GPT_4O_MINI: "azure-openai:gpt-4o-mini";
     readonly AZURE_OPENAI_GPT_4O: "azure-openai:gpt-4o";
-    readonly BEDROCK_US_ANTHROPIC_CLAUDE_3_5_HAIKU_20241022_V1_0: "bedrock:us.anthropic.claude-3-5-haiku-20241022-v1:0";
     readonly BEDROCK_US_ANTHROPIC_CLAUDE_OPUS_4_6_V1: "bedrock:us.anthropic.claude-opus-4-6-v1";
     readonly BEDROCK_US_ANTHROPIC_CLAUDE_OPUS_4_20250514_V1_0: "bedrock:us.anthropic.claude-opus-4-20250514-v1:0";
     readonly BEDROCK_US_ANTHROPIC_CLAUDE_SONNET_4_5_20250929_V1_0: "bedrock:us.anthropic.claude-sonnet-4-5-20250929-v1:0";
@@ -39,7 +38,6 @@ export declare const AZURE_OPENAI_GPT_4_1_NANO_0_0_1: AIModel;
 export declare const AZURE_OPENAI_GPT_4_1_0_0_1: AIModel;
 export declare const AZURE_OPENAI_GPT_4O_MINI_0_0_1: AIModel;
 export declare const AZURE_OPENAI_GPT_4O_0_0_1: AIModel;
-export declare const BEDROCK_US_ANTHROPIC_CLAUDE_3_5_HAIKU_20241022_V1_0_0_0_1: AIModel;
 export declare const BEDROCK_US_ANTHROPIC_CLAUDE_OPUS_4_6_V1_0_0_1: AIModel;
 export declare const BEDROCK_US_ANTHROPIC_CLAUDE_OPUS_4_20250514_V1_0_0_0_1: AIModel;
 export declare const BEDROCK_US_ANTHROPIC_CLAUDE_SONNET_4_5_20250929_V1_0_0_0_1: AIModel;

package/lib/specs/models.js

@@ -15,7 +15,6 @@ export const AIModels = {
     AZURE_OPENAI_GPT_4_1: 'azure-openai:gpt-4.1',
     AZURE_OPENAI_GPT_4O_MINI: 'azure-openai:gpt-4o-mini',
     AZURE_OPENAI_GPT_4O: 'azure-openai:gpt-4o',
-    BEDROCK_US_ANTHROPIC_CLAUDE_3_5_HAIKU_20241022_V1_0: 'bedrock:us.anthropic.claude-3-5-haiku-20241022-v1:0',
     BEDROCK_US_ANTHROPIC_CLAUDE_OPUS_4_6_V1: 'bedrock:us.anthropic.claude-opus-4-6-v1',
     BEDROCK_US_ANTHROPIC_CLAUDE_OPUS_4_20250514_V1_0: 'bedrock:us.anthropic.claude-opus-4-20250514-v1:0',
     BEDROCK_US_ANTHROPIC_CLAUDE_SONNET_4_5_20250929_V1_0: 'bedrock:us.anthropic.claude-sonnet-4-5-20250929-v1:0',
@@ -111,19 +110,6 @@ export const AZURE_OPENAI_GPT_4O_0_0_1 = {
     default: false,
     requiredEnvVars: ['AZURE_OPENAI_API_KEY', 'AZURE_OPENAI_ENDPOINT'],
 };
-export const BEDROCK_US_ANTHROPIC_CLAUDE_3_5_HAIKU_20241022_V1_0_0_0_1 = {
-    id: 'bedrock:us.anthropic.claude-3-5-haiku-20241022-v1:0',
-    version: '0.0.1',
-    name: 'Bedrock Claude Haiku 3.5',
-    description: 'Claude Haiku 3.5 via AWS Bedrock - fast and efficient',
-    provider: 'bedrock',
-    default: false,
-    requiredEnvVars: [
-        'AWS_ACCESS_KEY_ID',
-        'AWS_SECRET_ACCESS_KEY',
-        'AWS_DEFAULT_REGION',
-    ],
-};
 export const BEDROCK_US_ANTHROPIC_CLAUDE_OPUS_4_6_V1_0_0_1 = {
     id: 'bedrock:us.anthropic.claude-opus-4-6-v1',
     version: '0.0.1',
@@ -243,7 +229,6 @@ export const AI_MODEL_CATALOGUE = {
     'azure-openai:gpt-4.1': AZURE_OPENAI_GPT_4_1_0_0_1,
     'azure-openai:gpt-4o-mini': AZURE_OPENAI_GPT_4O_MINI_0_0_1,
     'azure-openai:gpt-4o': AZURE_OPENAI_GPT_4O_0_0_1,
-    'bedrock:us.anthropic.claude-3-5-haiku-20241022-v1:0': BEDROCK_US_ANTHROPIC_CLAUDE_3_5_HAIKU_20241022_V1_0_0_0_1,
     'bedrock:us.anthropic.claude-opus-4-6-v1': BEDROCK_US_ANTHROPIC_CLAUDE_OPUS_4_6_V1_0_0_1,
     'bedrock:us.anthropic.claude-opus-4-20250514-v1:0': BEDROCK_US_ANTHROPIC_CLAUDE_OPUS_4_20250514_V1_0_0_0_1,
     'bedrock:us.anthropic.claude-sonnet-4-5-20250929-v1:0': BEDROCK_US_ANTHROPIC_CLAUDE_SONNET_4_5_20250929_V1_0_0_0_1,

package/lib/specs/skills.d.ts

@@ -8,7 +8,6 @@
  */
 import type { SkillSpec } from '../types';
 export declare const CRAWL_SKILL_SPEC_0_0_1: SkillSpec;
-export declare const DATALAYER_WHOAMI_SKILL_SPEC_0_0_1: SkillSpec;
 export declare const EVENTS_SKILL_SPEC_0_0_1: SkillSpec;
 export declare const GITHUB_SKILL_SPEC_0_0_1: SkillSpec;
 export declare const JOKES_SKILL_SPEC_0_0_1: SkillSpec;

package/lib/specs/skills.js

@@ -22,23 +22,6 @@ export const CRAWL_SKILL_SPEC_0_0_1 = {
     emoji: '🌐',
     enabled: true,
 };
-export const DATALAYER_WHOAMI_SKILL_SPEC_0_0_1 = {
-    id: 'datalayer-whoami',
-    version: '0.0.1',
-    name: 'Datalayer Whoami Skill',
-    description: 'Retrieve the authenticated Datalayer user profile using the datalayer-skills package.',
-    module: 'datalayer_skills.skills.whoami',
-    package: undefined,
-    method: undefined,
-    path: undefined,
-    requiredEnvVars: ['DATALAYER_API_KEY:0.0.1'],
-    optionalEnvVars: [],
-    dependencies: ['datalayer_skills>=0.1.0'],
-    tags: ['datalayer', 'iam', 'identity', 'profile'],
-    icon: 'person',
-    emoji: '👤',
-    enabled: true,
-};
 export const EVENTS_SKILL_SPEC_0_0_1 = {
     id: 'events',
     version: '0.0.1',
@@ -129,7 +112,6 @@ export const TEXT_SUMMARIZER_SKILL_SPEC_0_0_1 = {
 // ============================================================================
 export const SKILLS_CATALOG = {
     crawl: CRAWL_SKILL_SPEC_0_0_1,
-    'datalayer-whoami': DATALAYER_WHOAMI_SKILL_SPEC_0_0_1,
     events: EVENTS_SKILL_SPEC_0_0_1,
     github: GITHUB_SKILL_SPEC_0_0_1,
     jokes: JOKES_SKILL_SPEC_0_0_1,