omniai-evals 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de97e262c2ca4119e0f158d0b4ed485ffeb9af5711edb529d099685fe277d615
-  data.tar.gz: ce16c93c7acab7e7f6c57680b65a14f04073ad21c7ad9f632efd746b60fb32a6
+  metadata.gz: 67cfea27bec341f1486a603f0cf4d3e77e31f0c5980ec819da55bef1e1b64a4e
+  data.tar.gz: 0b2c522085dd143abd823f2e0491629bc980c4882104379b3d9450ea353afa47
 SHA512:
-  metadata.gz: df80b33102d5c4e691f143136de334d35a9e3755e6bde89d71147de1feba1c2a334f1091c69995fb3945fd6b8740caae68a3d1e5d9f2d52805ee5ba8fb6a39a0
-  data.tar.gz: 2cfaaab87db9222a769db038b8b6d3800f570d3b351fd625e522f23157c9f5b21ef24b1fa7f5894516aeff8459404397d63b471ede0652aeaf555861e60afcca
+  metadata.gz: 485cb1d953a7da41ccc5e8a665e66ed6a73ce5f76ba2445c6c374a783a2f906d5ff7530b787a20037b66b66cedadd27ce0977bdcbaf7fec741edf68fb7ea83cc
+  data.tar.gz: bd7ff67a7fd6feeebf086258d5a97ee6269445389a9611a2b656db1b32b23bd457afaa6545919f79634689f0b46a1107e83bda38ee88b7938f642a65169fbcd3

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.2.0
+
+- `Adapters::ClaudeCLI` retries a rate limit: a process whose envelope has `api_error_status` 429 or 529 runs again, up to 5 processes, waiting 30s doubling each try plus up to 30s of jitter. `attempts` in the call's one record counts processes.
+- Out of attempts, the adapter's limit is spent: the call raises `ClaudeCLI::RateLimitSpent` (a `JudgeError`), no new call starts, and calls still waiting stop. A spent adapter stays spent, so a new run uses a new adapter. The gem never exits; the caller decides, using the error or `adapter.rate_limit.spent?`.
+
 ## 0.1.0
 
 Initial release.

data/README.md

@@ -119,6 +119,10 @@ bundle exec rake        # rspec + rubocop
 
 The six contract invariants have their own spec file, `spec/omniai/evals/invariants_spec.rb`.
 
+## Releasing
+
+Bump `VERSION` and `CHANGELOG.md` and merge. Dean then tags that commit on main `vX.Y.Z` and pushes the tag. The Release workflow runs the suite, checks the tag matches `VERSION` and is on main, then waits for Dean to approve the `release` environment before pushing the gem via RubyGems trusted publishing.
+
 ## License
 
 MIT. See `LICENSE`.

data/lib/omniai/evals/adapters/call_record.rb

@@ -8,24 +8,52 @@ module OmniAI
       # One per-call record for an adapter's optional `calls:` collector: stop codes and numbers
       # only, never result text or stderr (both can carry PHI).
       #
-      # `attempts` is always 1: neither adapter retries, and retries inside `claude -p` or the
-      # provider client are not reported.
+      # `attempts` counts `claude -p` processes: one more each time a process came back rate limited
+      # (see RateLimit). `latency_ms` covers them and their waits. Retries inside `claude -p` or the
+      # provider client are not reported, so Adapters::OmniAI always records 1.
       module CallRecord
         DURATION_KEYS = %w[duration_ms duration_api_ms].freeze
         # A call that finished but named no reason.
         UNREPORTED = "unreported"
 
-        # For Adapters::ClaudeCLI. The block receives a sink to hand the subprocess stdout to
-        # (including on a non-zero exit, so an error envelope's subtype is still recorded) and
-        # returns that stdout. The stdout stays here; it is parsed only for stop codes and numbers.
-        def self.capture(calls, timeout_error:, &block)
-          return yield(nil) if calls.nil?
+        # For Adapters::ClaudeCLI. Runs the block once per process, again while +rate_limit+ says to,
+        # and pushes exactly one record however that ends, from the last process to finish. The block
+        # receives a sink to hand the subprocess stdout to (including on a non-zero exit, so an error
+        # envelope's subtype is still recorded) and returns that stdout. Its errors are kept for the
+        # caller to raise; only what isn't a StandardError, like an Interrupt, goes straight up.
+        #
+        # @api private
+        # @return [Hash] the last process: :stdout, and :result or :error
+        def self.capture(calls, timeout_error:, rate_limit:, &process)
+          started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          attempts = 1
+          run = nil
+          begin
+            run = attempt(timeout_error, &process)
+            while rate_limit.run_again?(run[:stdout], attempts)
+              attempts += 1
+              run = attempt(timeout_error, &process)
+            end
+            run
+          ensure
+            latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
+            failure = run ? run[:failure] : "error"
+            calls&.<<(envelope_record(run && run[:stdout], failure:, latency_ms:, attempts:))
+          end
+        end
 
+        # One process: what it printed, and how it failed. The stdout is parsed only for stop codes,
+        # numbers and the rate limit.
+        #
+        # @api private
+        def self.attempt(timeout_error)
           stdout = nil
-          sink = ->(out) { stdout = out }
-          measure(calls, timeout_error:, body: -> { block.call(sink) }) do |failure, latency_ms|
-            envelope_record(stdout, failure:, latency_ms:)
-          end
+          result = yield(->(out) { stdout = out })
+          { stdout: stdout || result, result: }
+        rescue timeout_error => e
+          { stdout:, failure: "timeout", error: e }
+        rescue StandardError => e
+          { stdout:, failure: "error", error: e }
         end
 
         # For Adapters::OmniAI. The block returns an OmniAI::Chat::Response; its verbatim provider
@@ -61,18 +89,18 @@ module OmniAI
         # "unreported". A failed call keeps +failure+ unless its stdout is an is_error envelope, so
         # a timeout, or a non-zero exit without one, records "timeout" / "error"; stdout that is
         # not a JSON object records "error".
-        def self.envelope_record(stdout, failure:, latency_ms:)
+        def self.envelope_record(stdout, failure:, latency_ms:, attempts: 1)
           envelope = envelope(stdout)
-          return record(failure || "error", latency_ms) if envelope.nil?
-          return record(failure, latency_ms) if failure && !envelope["is_error"]
+          return record(failure || "error", latency_ms, attempts) if envelope.nil?
+          return record(failure, latency_ms, attempts) if failure && !envelope["is_error"]
 
-          result = record(finish_reason(envelope), latency_ms)
+          result = record(finish_reason(envelope), latency_ms, attempts)
           DURATION_KEYS.each { |key| result[key] = envelope[key] if envelope[key].is_a?(Numeric) }
           result
         end
 
-        def self.record(finish_reason, latency_ms)
-          { "finish_reason" => finish_reason, "latency_ms" => latency_ms, "attempts" => 1 }
+        def self.record(finish_reason, latency_ms, attempts = 1)
+          { "finish_reason" => finish_reason, "latency_ms" => latency_ms, "attempts" => attempts }
         end
 
         # Strings only, so an odd envelope can't put a nested object into the record.

data/lib/omniai/evals/adapters/claude_cli/verdict.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "json"
+
+module OmniAI
+  module Evals
+    module Adapters
+      class ClaudeCLI
+        # Reads the verdict out of a `claude -p` process's stdout. Error messages stay PHI-free: byte
+        # counts only, never the output.
+        module Verdict
+          # Parses the `claude -p --output-format json` envelope ({ is_error, result })
+          # and returns the verdict Hash carried in `result`. The error message never
+          # includes the raw output -- it holds PHI (verbatim source quotes).
+          def self.parse_output(stdout)
+            envelope = JSON.parse(stdout)
+            raise JudgeError, "claude reported an error result" if envelope["is_error"]
+
+            parse_verdict(envelope.fetch("result", "").to_s)
+          rescue JSON::ParserError
+            raise JudgeError, "unparseable judge output (#{stdout.bytesize} bytes)"
+          end
+
+          # The judge is told to emit bare JSON, but LLM output is nondeterministic -- the
+          # same input can come back ```-fenced one run and wrapped in prose the next.
+          # Strip fences first; if that still isn't valid JSON, fall back to the outermost
+          # { ... } span. Raises (PHI-free) only when no JSON object is recoverable.
+          def self.parse_verdict(result)
+            cleaned = result.strip.gsub(/\A```(?:json)?\s*/m, "").gsub(/\s*```\z/m, "").strip
+            JSON.parse(cleaned)
+          rescue JSON::ParserError
+            span = cleaned[/\{.*\}/m]
+            raise JudgeError, "unparseable judge output (#{result.bytesize} bytes)" if span.nil?
+
+            # A non-nil span can still be invalid JSON (e.g. prose braces). Rescue here too
+            # so it surfaces as a PHI-free JudgeError, never a raw JSON::ParserError leaking
+            # the (possibly PHI-bearing) content.
+            begin
+              JSON.parse(span)
+            rescue JSON::ParserError
+              raise JudgeError, "unparseable judge output (#{result.bytesize} bytes)"
+            end
+          end
+
+          # Reads the schema-constrained object from a `claude -p --json-schema` envelope's
+          # `structured_output` field (constrained decoding populates it). The free-form
+          # `result` still holds the model's prose transcription -- its chain-of-thought --
+          # but the deterministic artifact is `structured_output`. An absent field means the
+          # model transcribed but never emitted the constrained object (refusal / max_tokens):
+          # a COUNTED JudgeError, never a silent nil. Error messages stay PHI-free (byte counts
+          # only) -- the envelope can echo verbatim source quotes.
+          def self.parse_structured(stdout)
+            envelope = JSON.parse(stdout)
+            raise JudgeError, "claude reported an error result" if envelope["is_error"]
+
+            verdict = envelope["structured_output"]
+            raise JudgeError, "no structured_output in judge response (#{stdout.bytesize} bytes)" unless
+              verdict.is_a?(Hash)
+
+            verdict
+          rescue JSON::ParserError
+            raise JudgeError, "unparseable judge output (#{stdout.bytesize} bytes)"
+          end
+
+          private_class_method :parse_verdict
+        end
+      end
+    end
+  end
+end

data/lib/omniai/evals/adapters/claude_cli.rb

@@ -5,6 +5,8 @@ require "open3"
 
 require_relative "../../evals"
 require_relative "call_record"
+require_relative "rate_limit"
+require_relative "claude_cli/verdict"
 
 module OmniAI
   module Evals
@@ -41,6 +43,12 @@ module OmniAI
         class TimeoutError < JudgeError
         end
 
+        # A rate limit outlasted its attempts, so this adapter starts no new call. It is a JudgeError so
+        # a rescue that counts a failed call keeps the run's finished work. How the run ends is the
+        # caller's: rescue this, or read rate_limit.spent?, to stop and resume later.
+        class RateLimitSpent < JudgeError
+        end
+
         # The child's environment is built FROM SCRATCH, never inherited. ANTHROPIC_API_KEY is
         # exported machine-wide on the operator's box, and a bare Open3.popen3(*args) hands it
         # to `claude -p`, where it takes auth precedence over the claude.ai login -- so runs
@@ -74,6 +82,7 @@ module OmniAI
           SSL_CERT_FILE NODE_EXTRA_CA_CERTS HTTP_PROXY HTTPS_PROXY NO_PROXY
         ].freeze
         BILLING_MODES = %i[subscription api].freeze
+        RATE_LIMIT_SPENT = "rate limit spent, no new call started"
 
         # A page legitimately takes minutes (the model re-transcribes it), so the default
         # deadline is generous; a caller's timeout: overrides it. Its job is only to cap a
@@ -88,7 +97,7 @@ module OmniAI
         ISOLATION_FLAGS = ["--tools", "", "--strict-mcp-config", "--no-session-persistence",
                            "--setting-sources", "",].freeze
 
-        attr_reader :billing
+        attr_reader :billing, :rate_limit
 
         # Billing is a RUN-level choice, so it is bound once at construction and the bound
         # adapter is handed to the existing injection points -- no `billing:` parameter has
@@ -101,11 +110,15 @@ module OmniAI
         #   there is no default, so a caller cannot fall into metered billing by omission.
         def initialize(billing:)
           @billing = billing
+          @rate_limit = RateLimit.new
           child_env # validates the mode (and, for :api, the key) up front
           freeze
         end
 
-        # Runs one `claude -p` judge under the billing mode bound at construction.
+        # Runs one `claude -p` judge under the billing mode bound at construction. A process that comes
+        # back rate limited is waited out and run again (see RateLimit). Only this method retries:
+        # run_with_timeout runs one process, so a subclass calling it directly keeps its own policy.
+        # A spent adapter stays spent, so a new run uses a new adapter.
         #
         # @param system [String] the judge's system prompt -- static and PHI-free.
         # @param packet [String] the PHI-bearing user content; sent on STDIN only.
@@ -120,6 +133,7 @@ module OmniAI
         # @param calls [#<<, nil] an optional collector. Each call that reaches the subprocess
         #   pushes exactly one CallRecord however it ends, before any error is raised.
         # @return [Hash] the parsed verdict.
+        # @raise [RateLimitSpent] when the limit is spent, this call's or an earlier one's.
         # @raise [JudgeError] when the claude call errors or returns unparseable output.
         def call(system:, packet:, model: nil, schema: nil, timeout: JUDGE_TIMEOUT, calls: nil)
           # model stays effectively required -- there is no sensible default for a judge
@@ -133,11 +147,15 @@ module OmniAI
           args = ["claude", "-p", "--system-prompt", system, *ISOLATION_FLAGS, "--model", model,
                   "--output-format", "json",]
           args += ["--json-schema", JSON.generate(schema)] if schema
-          stdout = CallRecord.capture(calls, timeout_error: TimeoutError) do |on_stdout|
+          raise RateLimitSpent, RATE_LIMIT_SPENT if @rate_limit.spent?
+
+          run = CallRecord.capture(calls, timeout_error: TimeoutError, rate_limit: @rate_limit) do |on_stdout|
             run_with_timeout(args, packet, timeout:, env: child_env, on_stdout:)
           end
+          raise RateLimitSpent, RATE_LIMIT_SPENT if RateLimit.limited?(run[:stdout])
+          raise run[:error] if run[:error]
 
-          schema ? parse_structured(stdout) : parse_output(stdout)
+          schema ? Verdict.parse_structured(run[:result]) : Verdict.parse_output(run[:result])
         end
 
         # The child environment for the bound billing mode -- built from the allowlist, so no
@@ -211,59 +229,6 @@ module OmniAI
         rescue Errno::ESRCH
           # already exited between the timeout check and the signal
         end
-
-        # Parses the `claude -p --output-format json` envelope ({ is_error, result })
-        # and returns the verdict Hash carried in `result`. The error message never
-        # includes the raw output -- it holds PHI (verbatim source quotes).
-        def parse_output(stdout)
-          envelope = JSON.parse(stdout)
-          raise JudgeError, "claude reported an error result" if envelope["is_error"]
-
-          parse_verdict(envelope.fetch("result", "").to_s)
-        rescue JSON::ParserError
-          raise JudgeError, "unparseable judge output (#{stdout.bytesize} bytes)"
-        end
-
-        # The judge is told to emit bare JSON, but LLM output is nondeterministic -- the
-        # same input can come back ```-fenced one run and wrapped in prose the next.
-        # Strip fences first; if that still isn't valid JSON, fall back to the outermost
-        # { ... } span. Raises (PHI-free) only when no JSON object is recoverable.
-        def parse_verdict(result)
-          cleaned = result.strip.gsub(/\A```(?:json)?\s*/m, "").gsub(/\s*```\z/m, "").strip
-          JSON.parse(cleaned)
-        rescue JSON::ParserError
-          span = cleaned[/\{.*\}/m]
-          raise JudgeError, "unparseable judge output (#{result.bytesize} bytes)" if span.nil?
-
-          # A non-nil span can still be invalid JSON (e.g. prose braces). Rescue here too
-          # so it surfaces as a PHI-free JudgeError, never a raw JSON::ParserError leaking
-          # the (possibly PHI-bearing) content.
-          begin
-            JSON.parse(span)
-          rescue JSON::ParserError
-            raise JudgeError, "unparseable judge output (#{result.bytesize} bytes)"
-          end
-        end
-
-        # Reads the schema-constrained object from a `claude -p --json-schema` envelope's
-        # `structured_output` field (constrained decoding populates it). The free-form
-        # `result` still holds the model's prose transcription -- its chain-of-thought --
-        # but the deterministic artifact is `structured_output`. An absent field means the
-        # model transcribed but never emitted the constrained object (refusal / max_tokens):
-        # a COUNTED JudgeError, never a silent nil. Error messages stay PHI-free (byte counts
-        # only) -- the envelope can echo verbatim source quotes.
-        def parse_structured(stdout)
-          envelope = JSON.parse(stdout)
-          raise JudgeError, "claude reported an error result" if envelope["is_error"]
-
-          verdict = envelope["structured_output"]
-          raise JudgeError, "no structured_output in judge response (#{stdout.bytesize} bytes)" unless
-            verdict.is_a?(Hash)
-
-          verdict
-        rescue JSON::ParserError
-          raise JudgeError, "unparseable judge output (#{stdout.bytesize} bytes)"
-        end
       end
     end
   end

data/lib/omniai/evals/adapters/rate_limit.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative "call_record"
+
+module OmniAI
+  module Evals
+    module Adapters
+      # One run's `claude -p` rate limit, shared by every worker calling the adapter that holds it. A limited process is
+      # waited out and run again; one still limited after ATTEMPTS spends the limit, which starts no new call and wakes
+      # every call still waiting. State lives here rather than at class level, so a long-lived process isn't stopped for
+      # good by one run's limit.
+      class RateLimit
+        # The api_error_status of a limit that outlasted Claude Code's own retries: rate limited, overloaded.
+        STATUSES = [429, 529].freeze
+        # Processes per call.
+        ATTEMPTS = 5
+        # Seconds before the second process, doubling each try, plus up to this much jitter.
+        WAIT = 30
+
+        # @param stdout [String, nil] a process's stdout
+        # @return [Boolean]
+        def self.limited?(stdout)
+          envelope = CallRecord.envelope(stdout)
+          !envelope.nil? && STATUSES.include?(envelope["api_error_status"])
+        end
+
+        # @param attempts [Integer] processes run so far
+        # @return [Float] seconds to wait before the next
+        def self.delay(attempts)
+          (WAIT * (2**(attempts - 1))) + (rand * WAIT)
+        end
+
+        def initialize
+          @lock = Mutex.new
+          @changed = ConditionVariable.new
+          @spent = false
+        end
+
+        # @return [Boolean]
+        def spent?
+          @lock.synchronize { @spent }
+        end
+
+        # Spends the limit from outside too: a caller whose other calls share the account can stop this adapter's.
+        def spend!
+          @lock.synchronize do
+            @spent = true
+            @changed.broadcast
+          end
+        end
+
+        # @param stdout [String, nil] the last process's stdout
+        # @param attempts [Integer] processes run so far
+        # @return [Boolean] whether to run another, after waiting when it does
+        def run_again?(stdout, attempts)
+          return false unless self.class.limited?(stdout)
+
+          spend! if attempts >= ATTEMPTS
+          wait(self.class.delay(attempts))
+        end
+
+        # @param seconds [Numeric]
+        # @return [Boolean] true once the time is up; false, at once, when the limit is spent
+        def wait(seconds)
+          deadline = clock + seconds
+          @lock.synchronize do
+            until @spent
+              left = deadline - clock
+              return true unless left.positive?
+
+              @changed.wait(@lock, left)
+            end
+            false
+          end
+        end
+
+      private
+
+        def clock
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+      end
+    end
+  end
+end

data/lib/omniai/evals/version.rb

@@ -2,6 +2,6 @@
 
 module OmniAI
   module Evals
-    VERSION = "0.1.0"
+    VERSION = "0.2.0"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omniai-evals
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Dean Singh
@@ -38,7 +38,9 @@ files:
 - lib/omniai/evals.rb
 - lib/omniai/evals/adapters/call_record.rb
 - lib/omniai/evals/adapters/claude_cli.rb
+- lib/omniai/evals/adapters/claude_cli/verdict.rb
 - lib/omniai/evals/adapters/omniai.rb
+- lib/omniai/evals/adapters/rate_limit.rb
 - lib/omniai/evals/assertions.rb
 - lib/omniai/evals/assertions/absence.rb
 - lib/omniai/evals/assertions/abstention.rb