agent-evals 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56fb5b0637a74afe1bf8be4010c7ee38f9e0838d2dc0e449e01dae294fb2fdd9
-  data.tar.gz: 7d9dd13e1d1df76b9ce6b38813775285921505b12ba863c37b730d223ef552d1
+  metadata.gz: fce409377a09747fbca5b2cfce6beee74761377b73395773aa65a59409c05fcc
+  data.tar.gz: 6120e86241b88d81b0b057fd6bc6b5c429654dde2978b22c21a23f0c1b5ab25b
 SHA512:
-  metadata.gz: 82e38be379ec294b9f6d866ca3d2e7e83651a6790a6021a1010b430a53571389531b63c65faa1e58ba937de466f59c2a50187f04481ad2eaf3406efe5b9bedac
-  data.tar.gz: e955b8dfbdb2bb4194e7cf7712bc0cbba395b86d8534c24eb97493d66185dbbf8503c1bf62c648bd92e180274311b6a3b5d41fb57ffeb60ea6b3ab9273bb0c49
+  metadata.gz: e4c19f34da26df191552facd225fd003bfd0c7a59e172ecf87e1f6255ef274249f2ea6146d617f8426be1dd0c4d91d702624a3799f3573ce05613b631eb0b2b5
+  data.tar.gz: 7bfd5bf66da297c8d4b3e928cb67a046714a166f1df45a2da74b2e25b86f3bb3ef53309fb1511bf6149584355e2cd55e5591efddafcc0b1b06c15fea4da7e6b7

data/README.md

@@ -389,9 +389,9 @@ agent-evals completion zsh                     # shell completion (bash too)
 
 - `cases` prints one row per case: dataset, id, difficulty, the scorer's `summary`, word count.
 - `verify` reports empty inputs and missing or unparsable `expected.json`, then runs `adapter.verify(dataset)` and `scorer.verify`.
-- `run` and `compare` run one experiment per model and dataset. `compare --concurrency N` limits how many models run at once.
+- `run` and `compare` run one experiment per model and dataset, as many at once as the machine has CPUs. `-j N` / `--concurrency N` sets another limit, and `-j 0` starts them all together. `-p` is separate: the cases in flight inside each experiment.
 
-Flags shared by `run` and `compare`: `-m/--models`, `-d/--dataset` (every dataset when omitted), `-c/--case`, `-e/--effort`, `-t/--temperature`, `--max-tokens`, `-p/--parallelism`, `--config FILE`, `--project NAME`, `--tracker NAME`, `--local`, `--no-open`, `-n/--dry-run`, `-o/--output [FILE]`. `-l` and `-L` are reserved for logging and not yet implemented.
+Flags shared by `run` and `compare`: `-m/--models`, `-d/--dataset` (every dataset when omitted), `-c/--case`, `-e/--effort`, `-t/--temperature`, `--max-tokens`, `-p/--parallelism`, `-j/--concurrency`, `-C/--config FILE`, `--project NAME`, `--tracker NAME`, `--local`, `--no-open`, `-n/--dry-run`, `-o/--output [FILE]`. `-l` and `-L` are reserved for logging and not yet implemented.
 
 **Reading the output.** The leaderboard has one column per metric the scorer declares, ranked by the `primary` one, with a glossary built from each metric's `description`. STDOUT carries the tables. Progress bars and warnings go to STDERR, and anything a library prints while the bars redraw is saved to `log/<command>.warnings.<stamp>.log` instead of garbling the screen.
 

data/lib/agent/evals/cli/compare.rb

@@ -8,16 +8,14 @@ module Agent
       class Compare < ModelCommand
         desc "Run every case against several models and print a leaderboard"
 
-        option :concurrency, default: "0", desc: "Models running at once; 0 runs them all together"
-
         example [
           "                                     # everything .agent-eval.config.yml lists",
           "-m anthropic/claude-haiku-5.5,openai/gpt-4o-mini -t 0",
-          "-e low -c clear --concurrency 2",
+          "-e low -c clear -j 2                 # two experiments at a time",
           "-n                                   # show the matrix, spend nothing"
         ]
 
-        def call(models:, dataset:, dry_run:, concurrency:, **options)
+        def call(models:, dataset:, dry_run:, **options)
           config = lineup(options)
           contenders = contenders(config, models, options)
           datasets = datasets(dataset)
@@ -25,7 +23,7 @@ module Agent
 
           setup!(contenders, options)
           runs = datasets.flat_map { |name| contenders.map { experiment(it, config, options, dataset: name) } }
-          report(execute(runs, concurrency: Integer(concurrency)), output: options[:output], open: options[:open])
+          report(execute(runs, concurrency: concurrency(options)), output: options[:output], open: options[:open])
         end
       end
     end

data/lib/agent/evals/cli/model_command.rb

@@ -47,7 +47,7 @@ module Agent
             aliases: ["-d"],
             default: [],
             desc:    "Datasets, comma-separated; every dataset when omitted. Each model runs on each"
-          subclass.option :config, desc: "The lineup file (default: the suite's .agent-eval..agent-eval.config.yml)"
+          subclass.option :config, aliases: ["-C"], desc: "The lineup file (default: the suite's .agent-eval.config.yml)"
           subclass.option :effort, aliases: ["-e"], desc: "Reasoning effort: default, or one the provider accepts (overrides the config file)"
           subclass.option :temperature, aliases: ["-t"], desc: "Sampling temperature, or 'default' to send none (overrides the config file)"
           subclass.option :max_tokens, desc: "Output token cap, reasoning included (overrides the config file)"
@@ -57,6 +57,9 @@ module Agent
             default: [],
             desc:    "Only these cases, by id prefix: -c clear or -c clear/003,vague/001"
           subclass.option :parallelism, aliases: ["-p"], desc: "Cases in flight at once, per model (default: the config's defaults.parallelism)"
+          subclass.option :concurrency,
+            aliases: ["-j"],
+            desc:    "Experiments running at once; 0 runs them all together (default: the number of CPUs)"
           subclass.option :project,
             desc: "Braintrust project name; default is the BRAINTRUST_PROJECT_ID project, else the config file's project"
           subclass.option :open,
@@ -179,13 +182,26 @@ module Agent
         # @param runs [Array<Experiment>]
         # @param concurrency [Integer] experiments at once; 0 runs them all together
         # @return [Array<Experiment::Summary>]
-        def execute(runs, concurrency: 0)
+        def execute(runs, concurrency:)
           summaries = Array.new(runs.size)
           warnings.capture { bars(runs, summaries, concurrency) }
           summaries.compact.each { |s| s.errors.each { warnings.record("[#{s.name}] #{it}") } }
           summaries.compact
         end
 
+        # -j, else one experiment per CPU. Uncapped, a full lineup started
+        # every model x dataset pair at once: 140 experiments, each with its
+        # own pool of case threads, all fighting for the network and the screen.
+        #
+        # @return [Integer] experiments at once; 0 means all of them
+        # @raise [Error] on a negative or non-numeric -j
+        def concurrency(options)
+          value = options[:concurrency] || Etc.nprocessors
+          Integer(value).tap { raise Error, "--concurrency must be 0 or more, got #{value}" if it.negative? }
+        rescue ArgumentError, TypeError
+          raise Error, "--concurrency must be a whole number, got #{value.inspect}"
+        end
+
         # @return [Warnings] what this command has held back from the screen
         def warnings = @warnings ||= Warnings.new
 

data/lib/agent/evals/cli/run.rb

@@ -25,7 +25,7 @@ module Agent
 
           setup!(contenders, options)
           runs = datasets.flat_map { |name| contenders.map { experiment(it, config, options, dataset: name) } }
-          summaries = execute(runs)
+          summaries = execute(runs, concurrency: concurrency(options))
           report(summaries, output: options[:output], open: options[:open]) do
             summaries.each { breakdown(it, titled: summaries.size > 1) }
           end

data/lib/agent/evals/version.rb

@@ -2,6 +2,6 @@
 
 module Agent
   module Evals
-    VERSION = "0.1.1"
+    VERSION = "0.2.0"
   end
 end

data/lib/agent/evals.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "etc"
 require "json"
 require "dry/cli"
 require "dry/cli/help"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: agent-evals
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
   - Konstantin Gredeskoul