bitfab 0.65.7 → 0.67.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 84d81613e344f01a011f28ac437476d22e73867eb7ab2df04579075730b1c73b
-  data.tar.gz: d3bb4d9f79d587dca9ee2ef2e1ee36a41fbeb59da7cb69960bf9fdbe75e8d663
+  metadata.gz: 455ac9a2dfa11b39c17550b3f86f6ffa87f3ea0a9684fb032722184a705da7e0
+  data.tar.gz: 11c0f2491a21515f3fc75f4f4f8eb98f544b7e6d5d3c9d1c7fc09e0728787b16
 SHA512:
-  metadata.gz: 18ceec3ef05433309efff835c6bcf050789439e9d0424f9919972a70dd5d5b358eb07641fbed220d36b52e92f503d719dee044ba18003fb1371d77d0dba2e8e3
-  data.tar.gz: a6743709c41b9b69597e43e5c1336af5038bf9e5c56d5e3b247051e12fb4527d9732d79bb98e3587f90aedfbc59f7e23ebeae6c0d02e1bfad712f37d6789dd00
+  metadata.gz: d1f41b10072dd2b9b52f2a91ef5265a91805d2aa8129a05a30a303f6f55831934c8206eca6fd215ac938a526f499e9bc2b38a2e74476c8aa06e6c675e14d9b7f
+  data.tar.gz: d1e5d1ac8216e5505b5d5da628d6914349fe7ee6018fa55616b20a5ba3ca66e647484f4db176ee85feac28c476f702e831bfcf9ae3fa5ff5084d9d0570584ade

data/lib/bitfab/client.rb

@@ -37,7 +37,7 @@ module Bitfab
 
     attr_reader :service_url
 
-    attr_reader :datasets, :categories, :organization_members, :traces, :spans, :labels, :graders, :experiments
+    attr_reader :datasets, :categories, :organization_members, :traces, :spans, :verdicts, :graders, :experiments
 
     # @deprecated Use {#categories}. Returns the same object.
     def assertion_categories
@@ -45,6 +45,12 @@ module Bitfab
       @categories
     end
 
+    # @deprecated Use {#verdicts}. Returns the same object.
+    def labels
+      Bitfab.warn_once("labels-deprecated", "labels is deprecated; use verdicts instead.")
+      @verdicts
+    end
+
     def initialize(api_key: nil, service_url: nil, capture_enabled: nil, enabled: nil, strict: false, simulation_plan: true, timeout: 120, trace_across_threads: nil, env_vars: nil, baml_client: nil, db_snapshot: nil)
       @db_snapshot = DbSnapshot.normalize_config(db_snapshot)
       @env_vars = env_vars || ENV.to_h
@@ -65,7 +71,7 @@ module Bitfab
       @organization_members = OrganizationMembers.new(@http_client)
       @traces = Traces.new(@http_client)
       @spans = Spans.new(@http_client)
-      @labels = Labels.new(@http_client)
+      @verdicts = Verdicts.new(@http_client)
       @graders = Graders.new(@http_client)
       @experiments = Experiments.new(@http_client)
       # Mock overrides registered via register_mock_override, applied to every

data/lib/bitfab/graders.rb

@@ -2,15 +2,69 @@
 
 require "uri"
 
+require_relative "verdicts"
+require_relative "warn_once"
+
 module Bitfab
+  # Reads and writes grader verdicts: a grader's pass or fail on a whole trace, with its reason.
   class Graders
-    GRADER_LABELS_PATH = "/api/sdk/graderLabels"
+    GRADER_VERDICTS_PATH = "/api/sdk/graderVerdicts"
+    # @deprecated Use {GRADER_VERDICTS_PATH}.
+    GRADER_LABELS_PATH = GRADER_VERDICTS_PATH
+    GRADER_VERDICT_VALUES = ["pass", "fail"].freeze
+    LABEL_UNSET = Object.new.freeze
+
+    private_constant :GRADER_VERDICT_VALUES, :LABEL_UNSET
 
     def initialize(http_client)
       @http_client = http_client
     end
 
+    # Reads grader verdicts for +trace_ids:+, for one +grader_id:+, or both. Each verdict carries
+    # +verdict+ ("pass", "fail", or nil), +reason+, +failureDiagnostic+, +confidence+, and +source+.
+    # Raises when neither selector is given.
+    def get_verdicts(trace_ids: nil, grader_id: nil, limit: nil)
+      read(trace_ids:, grader_id:, limit:)["verdicts"]
+    end
+
+    # Saves an external grader's verdict on one trace. +verdict:+ is "pass" or "fail". The deprecated
+    # +label:+ (true or false) is still accepted in place of +verdict:+.
+    def save_verdict(grader_id:, trace_id:, verdict: nil, reason: nil, failure_diagnostic: nil, confidence: nil,
+      label: LABEL_UNSET)
+      save_verdict_all([verdict_update(grader_id:, trace_id:, verdict:, reason:, failure_diagnostic:, confidence:,
+        label:)]).first
+    end
+
+    # Saves several external grader verdict entries (+graderId+, +traceId+, +verdict+, and optional
+    # +reason+, +failureDiagnostic+, +confidence+) in one request.
+    def save_verdict_all(verdicts)
+      write(verdicts)["verdicts"]
+    end
+
+    # @deprecated Use {#get_verdicts}. Returns the rows in their old shape, with +label+ as true or false.
     def get_labels(trace_ids: nil, grader_id: nil, limit: nil)
+      Bitfab.warn_once("grader-get-labels-deprecated", "graders.get_labels is deprecated; use graders.get_verdicts instead.")
+      read(trace_ids:, grader_id:, limit:)["labels"]
+    end
+
+    # @deprecated Use {#save_verdict}.
+    def save_label(grader_id:, trace_id:, label:, reason: nil, failure_diagnostic: nil, confidence: nil)
+      Bitfab.warn_once("grader-save-label-deprecated", "graders.save_label is deprecated; use graders.save_verdict instead.")
+      update = verdict_update(grader_id:, trace_id:, verdict: VerdictInput.legacy_verdict(label), reason:,
+        failure_diagnostic:, confidence:, label: LABEL_UNSET)
+      write([update])["labels"].first
+    end
+
+    # @deprecated Use {#save_verdict_all}.
+    def save_label_all(labels)
+      Bitfab.warn_once("grader-save-label-all-deprecated",
+        "graders.save_label_all is deprecated; use graders.save_verdict_all instead.")
+      write(labels.map { |entry| legacy_entry(entry) })["labels"]
+    end
+
+    private
+
+    def read(trace_ids:, grader_id:, limit:)
       if (trace_ids.nil? || trace_ids.empty?) && grader_id.nil?
         raise ArgumentError, "Pass trace_ids, grader_id, or both: there is nothing to read otherwise."
       end
@@ -19,19 +73,29 @@ module Bitfab
       params["traceIds"] = trace_ids.join(",") unless trace_ids.nil? || trace_ids.empty?
       params["graderId"] = grader_id unless grader_id.nil?
       params["limit"] = limit unless limit.nil?
-      @http_client.get("#{GRADER_LABELS_PATH}?#{URI.encode_www_form(params)}")["labels"]
+      @http_client.get("#{GRADER_VERDICTS_PATH}?#{URI.encode_www_form(params)}")
     end
 
-    def save_label(grader_id:, trace_id:, label:, reason: nil, failure_diagnostic: nil, confidence: nil)
-      update = {"graderId" => grader_id, "traceId" => trace_id, "label" => label}
+    def write(verdicts)
+      verdicts = verdicts.map { |entry| VerdictInput.normalize_entry(entry, GRADER_VERDICT_VALUES) }
+      @http_client.request(GRADER_VERDICTS_PATH, {"verdicts" => verdicts})
+    end
+
+    def legacy_entry(entry)
+      label_key = [:label, "label"].find { |key| entry.key?(key) }
+      verdict_present = entry.key?(:verdict) || entry.key?("verdict")
+      return entry if label_key.nil? || verdict_present
+
+      entry.except(label_key).merge("verdict" => VerdictInput.legacy_verdict(entry[label_key]))
+    end
+
+    def verdict_update(grader_id:, trace_id:, verdict:, reason:, failure_diagnostic:, confidence:, label:)
+      verdict = VerdictInput.resolve_verdict(verdict, label, GRADER_VERDICT_VALUES, LABEL_UNSET)
+      update = {"graderId" => grader_id, "traceId" => trace_id, "verdict" => verdict}
       update["reason"] = reason unless reason.nil?
       update["failureDiagnostic"] = failure_diagnostic unless failure_diagnostic.nil?
       update["confidence"] = confidence unless confidence.nil?
-      save_label_all([update]).first
-    end
-
-    def save_label_all(labels)
-      @http_client.request(GRADER_LABELS_PATH, {"labels" => labels})["labels"]
+      update
     end
   end
 end

data/lib/bitfab/jev.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Bitfab
+  module Jev
+    PROVIDER = "typesafe"
+    SYSTEM_ONE = :system_one
+    WRAPPER_SCOPE_KEY = :bitfab_jev_wrapper_scope
+
+    module_function
+
+    def jev_shaped?(reply)
+      return false if reply.nil? || reply.is_a?(String) || reply.is_a?(Numeric) || reply.is_a?(Symbol)
+
+      usage = field(reply, :usage)
+      field(reply, :model).is_a?(String) && !usage.nil? && !usage.is_a?(String) && !usage.is_a?(Numeric)
+    rescue
+      false
+    end
+
+    def auto_context(reply, duration_ms)
+      jev_shaped?(reply) ? context(reply, duration_ms) : nil
+    rescue
+      nil
+    end
+
+    def mark_recorded(reply)
+      Thread.current[WRAPPER_SCOPE_KEY]&.push(reply)
+    end
+
+    def watch_recorded
+      previous = Thread.current[WRAPPER_SCOPE_KEY]
+      scope = []
+      Thread.current[WRAPPER_SCOPE_KEY] = scope
+      reply = yield
+      [reply, !scope.empty?]
+    ensure
+      Thread.current[WRAPPER_SCOPE_KEY] = previous
+      previous&.concat(scope) if scope
+    end
+
+    def record_node_call(method_name)
+      return yield unless method_name.to_sym == SYSTEM_ONE
+
+      span = Bitfab.current_span
+      started_at = monotonic_ms
+      reply, recorded_inside = watch_recorded { yield }
+      begin
+        entry = recorded_inside ? nil : auto_context(reply, monotonic_ms - started_at)
+        if entry
+          span.add_context(entry)
+          mark_recorded(reply)
+          Subtree.active_sessions.each(&:jev_recorded_below!)
+        end
+      rescue
+        nil
+      end
+      reply
+    end
+
+    def field(value, name)
+      if value.is_a?(Hash)
+        value.key?(name) ? value[name] : value[name.to_s]
+      elsif value.respond_to?(name)
+        value.public_send(name)
+      end
+    end
+
+    def context(reply, duration_ms)
+      return nil if reply.nil?
+
+      usage = field(reply, :usage)
+      model = field(reply, :model)
+      input_tokens = usage.nil? ? nil : field(usage, :input_tokens)
+      output_tokens = usage.nil? ? nil : field(usage, :output_tokens)
+      entry = {provider: PROVIDER}
+      entry[:model] = model if model.is_a?(String)
+      entry[:inputTokens] = input_tokens if input_tokens.is_a?(Numeric)
+      entry[:outputTokens] = output_tokens if output_tokens.is_a?(Numeric)
+      entry[:durationMs] = duration_ms
+      entry
+    end
+
+    def record(span, reply, duration_ms)
+      entry = context(reply, duration_ms)
+      span.add_context(entry) if entry
+    rescue
+      nil
+    end
+
+    def monotonic_ms
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+    end
+
+    class Wrapper
+      def initialize(callable, on_reply)
+        @callable = callable
+        @on_reply = on_reply
+      end
+
+      # Call the wrapped Jev method with the given arguments and return its
+      # reply unchanged. Errors from the call are raised as-is and record nothing.
+      def call(*args, **kwargs, &block)
+        span = Bitfab.current_span
+        started_at = Jev.monotonic_ms
+        reply, recorded = Jev.watch_recorded { @callable.call(*args, **kwargs, &block) }
+        Jev.record(span, reply, Jev.monotonic_ms - started_at) unless recorded
+        begin
+          @on_reply&.call(reply)
+        rescue
+          nil
+        end
+        reply
+      end
+
+      # A Proc that calls this wrapper, so it can be passed with `&`.
+      def to_proc
+        method(:call).to_proc
+      end
+    end
+  end
+
+  class Client
+    # Record the model, token usage, and duration of a Jev (TypeSafe System One)
+    # call on the span that is current when the call starts. Opens no span of
+    # its own and never raises from recording.
+    #
+    # @example
+    #   ask_jev = Bitfab.client.wrap_jev(jev.method(:system_one))
+    #   reply = ask_jev.call(state:, questions:)
+    #
+    # @param callable [#call, nil] calls Jev and returns its reply; pass a block instead to wrap the block
+    # @param on_reply [#call, nil] called with each reply after it is recorded; its errors are ignored
+    # @return [Bitfab::Jev::Wrapper] responds to +call+ with the same arguments as the wrapped callable
+    def wrap_jev(callable = nil, on_reply: nil, &block)
+      raise ArgumentError, "wrap_jev takes a callable or a block, not both." if callable && block
+      target = callable || block
+      raise ArgumentError, "wrap_jev requires a callable that calls Jev (e.g., jev.method(:system_one))." unless target.respond_to?(:call)
+
+      Jev::Wrapper.new(target, on_reply)
+    end
+  end
+
+  class BitfabFunction
+    # Jev wrappers record onto the currently active span under any key.
+    def wrap_jev(*args, **kwargs, &block)
+      @client.wrap_jev(*args, **kwargs, &block)
+    end
+  end
+end

data/lib/bitfab/labels.rb

@@ -1,143 +1,3 @@
 # frozen_string_literal: true
 
-require "uri"
-
-module Bitfab
-  class Labels
-    LABELS_PATH = "/api/sdk/traces/labels"
-    HUMAN_LABELS_PATH = "#{LABELS_PATH}/human"
-    EVIDENCE_UNSET = Object.new.freeze
-
-    private_constant :EVIDENCE_UNSET
-
-    def initialize(http_client)
-      @http_client = http_client
-    end
-
-    def save(label:, annotation:, assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil,
-      confidence: nil, experiment_id: nil, evidence: EVIDENCE_UNSET, test_run_id: nil)
-      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
-      update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
-      update["label"] = label
-      update["annotation"] = annotation
-      update["confidence"] = confidence unless confidence.nil?
-      update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
-      save_all([update], experiment_id:).first
-    end
-
-    def save_all(labels, experiment_id: nil, test_run_id: nil)
-      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
-      require_assertion_ids(labels)
-      payload = {"labels" => labels}
-      unless experiment_id.nil?
-        payload["experimentId"] = experiment_id
-        payload["testRunId"] = experiment_id
-      end
-      @http_client.request(LABELS_PATH, payload)["labels"]
-    end
-
-    def skip(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
-      test_run_id: nil, annotation: nil)
-      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
-      update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("skip" => true)
-      update["annotation"] = annotation unless annotation.nil?
-      save_all([update], experiment_id:).first
-    end
-
-    # Records that the evidence does not show the part of the run this assertion checks, so there was
-    # nothing to judge. Counted as a fail in every pass rate, but stored and read back as an abstention,
-    # apart from a fail. +annotation:+ says what the evidence would have had to show.
-    def abstain(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
-      test_run_id: nil, annotation: nil)
-      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
-      update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("abstain" => true)
-      update["annotation"] = annotation unless annotation.nil?
-      save_all([update], experiment_id:).first
-    end
-
-    def archive(trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil, assertion_id: nil, test_run_id: nil)
-      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
-      update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
-      save_all([update.merge("archive" => true)], experiment_id:).first
-    end
-
-    def save_human(label:, annotation:, trace_id:, assertion_id:, confidence: nil, evidence: EVIDENCE_UNSET)
-      update = {"traceId" => trace_id, "assertionId" => assertion_id, "label" => label, "annotation" => annotation}
-      update["confidence"] = confidence unless confidence.nil?
-      update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
-      save_human_all([update]).first
-    end
-
-    def save_human_all(labels)
-      require_human_assertion_ids(labels)
-      @http_client.request(HUMAN_LABELS_PATH, {"labels" => labels})["labels"]
-    end
-
-    def get(trace_id)
-      get_all([trace_id]).first
-    end
-
-    def get_all(trace_ids)
-      return [] if trace_ids.empty?
-
-      query = URI.encode_www_form("traceIds" => trace_ids.join(","))
-      @http_client.get("#{LABELS_PATH}?#{query}")["labels"]
-    end
-
-    def generate_label_evidence(trace_id, assertion_id)
-      encoded_trace_id = URI.encode_www_form_component(trace_id).gsub("+", "%20")
-      encoded_assertion_id = URI.encode_www_form_component(assertion_id).gsub("+", "%20")
-      @http_client.get(
-        "/api/sdk/traces/#{encoded_trace_id}/assertions/#{encoded_assertion_id}/evidence"
-      )["evidence"]
-    end
-
-    private
-
-    def require_assertion_ids(labels)
-      unscoped = labels.reject do |entry|
-        entry_value(entry, "archive") == true || !blank?(entry_value(entry, "assertionId"))
-      end
-      return if unscoped.empty?
-
-      targets = unscoped.map { |entry| entry_value(entry, "traceId") || entry_value(entry, "originalTraceId") }
-      raise ArgumentError, "Agent verdicts must name an assertion: #{targets.join(", ")}. " \
-        "Pass the assertion_id of the assertion each verdict scores to labels.save, save_all, skip, and abstain. " \
-        "Only archive may omit it, to clear an old whole-trace verdict."
-    end
-
-    def require_human_assertion_ids(labels)
-      unscoped = labels.select { |entry| blank?(entry_value(entry, "assertionId")) }
-      return if unscoped.empty?
-
-      targets = unscoped.map { |entry| entry_value(entry, "traceId") }
-      raise ArgumentError, "Human verdicts must name an assertion: #{targets.join(", ")}. " \
-        "Pass the assertion_id of the assertion each verdict scores to labels.save_human and save_human_all."
-    end
-
-    def entry_value(entry, key)
-      entry.key?(key) ? entry[key] : entry[key.to_sym]
-    end
-
-    def blank?(value)
-      value.nil? || value.to_s.empty?
-    end
-
-    def label_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
-      if trace_id.nil? == original_trace_id.nil?
-        raise ArgumentError, "Pass exactly one of trace_id or original_trace_id. " \
-          "Use original_trace_id for a replay verdict, with the experiment_id it ran under."
-      end
-
-      update = {}
-      update["assertionId"] = assertion_id unless assertion_id.nil?
-      if trace_id.nil?
-        update["originalTraceId"] = original_trace_id
-        update["attempt"] = attempt unless attempt.nil?
-      else
-        update["traceId"] = trace_id
-      end
-      update
-    end
-  end
-end
+require_relative "verdicts"

data/lib/bitfab/replay.rb

@@ -549,7 +549,7 @@ module Bitfab
           warn "Bitfab: server has no persisted trace for #{missing.length} of " \
             "#{completed_count} completed replay item(s) " \
             "(experiment #{experiment_id}). Their replay token usage is " \
-            "unavailable and they cannot be labeled."
+            "unavailable and they cannot be given a verdict."
         end
       end
 

data/lib/bitfab/subtree.rb

@@ -37,6 +37,7 @@ module Bitfab
       :name,
       :excluded,
       :passthrough,
+      :jev,
       keyword_init: true
     )
 
@@ -55,6 +56,8 @@ module Bitfab
       :declared_node,
       :contexts,
       :prompt,
+      :jev_started_at,
+      :jev_recorded_below,
       keyword_init: true
     )
 
@@ -506,7 +509,7 @@ module Bitfab
 
         return if event.event == :call && skip_boundary_call?(event)
 
-        resolved_path = owned_path(event.path)
+        resolved_path = traced_path(event)
         return unless resolved_path
 
         case event.event
@@ -523,6 +526,12 @@ module Bitfab
         @exclude[method_name.to_s] == true
       end
 
+      def jev_recorded_below!
+        @stacks[Fiber.current.object_id]&.each do |entry|
+          entry.jev_recorded_below = true if entry.is_a?(OpenSpan)
+        end
+      end
+
       def reserve_node
         stack = current_stack(Fiber.current.object_id)
         return true if captured_depth(stack) < @max_depth
@@ -684,7 +693,9 @@ module Bitfab
           args:,
           kwargs:,
           started_at: now,
-          span_data: without_content ? {"content_off_by_simulation_plan" => true} : nil
+          span_data: without_content ? {"content_off_by_simulation_plan" => true} : nil,
+          span_type: metadata.jev ? "llm" : nil,
+          jev_started_at: metadata.jev ? Jev.monotonic_ms : nil
         )
       end
 
@@ -708,6 +719,7 @@ module Bitfab
         return unless entry.is_a?(OpenSpan) && matching_identity?(entry.identity, event, resolved_path)
 
         stack.pop
+        record_jev_reply(entry, event.return_value) if entry.jev_started_at
         emit(entry, result: event.return_value)
         parent = stack.last
         parent.error ||= entry.error if entry.error && parent.is_a?(OpenSpan)
@@ -715,6 +727,17 @@ module Bitfab
         @drained = true if @exhausted && captured_calls_drained?
       end
 
+      def record_jev_reply(entry, reply)
+        return if entry.error || entry.jev_recorded_below
+
+        context = Jev.auto_context(reply, Jev.monotonic_ms - entry.jev_started_at)
+        return unless context
+
+        entry.contexts = [context]
+        Jev.mark_recorded(reply)
+        jev_recorded_below!
+      end
+
       def record_error(event)
         entry = @stacks[Fiber.current.object_id]&.last
         return unless entry.is_a?(OpenSpan)
@@ -849,7 +872,8 @@ module Bitfab
           lineno: event.lineno,
           name: span_name(event, method_id),
           excluded: @exclude[method_id.to_s],
-          passthrough: passthrough_wrapper?(event.parameters)
+          passthrough: passthrough_wrapper?(event.parameters),
+          jev: method_id == Jev::SYSTEM_ONE
         ).freeze
       end
 
@@ -862,6 +886,16 @@ module Bitfab
         event.callee_id || event.method_id
       end
 
+      def traced_path(event)
+        owned_path(event.path) || jev_path(event)
+      end
+
+      def jev_path(event)
+        return nil unless called_method_id(event) == Jev::SYSTEM_ONE && !sdk_frame?(event.path)
+
+        Subtree.realpath(event.path)
+      end
+
       def owned_path(path)
         @owned_paths.fetch(path) do
           resolved = Subtree.realpath(path)
@@ -945,7 +979,7 @@ module Bitfab
         return unless event.event == :call
         return if skip_boundary_call?(event)
 
-        resolved_path = owned_path(event.path)
+        resolved_path = traced_path(event)
         count_dropped_call(event, resolved_path) if resolved_path
       rescue Exception # rubocop:disable Lint/RescueException
         nil

data/lib/bitfab/traceable.rb

@@ -173,7 +173,9 @@ module Bitfab
               explicit_span_receiver: self, explicit_span_method_name: method_name,
               experiment_id:, finalize:, surface: :trace, result_recorded_elsewhere: call.recorded_elsewhere?, declared_node: true,
               on_start: call.method(:start), on_complete: call.method(:complete),
-              around_body: call.method(:run)) { super(*args, **kwargs, &block) }
+              around_body: call.method(:run)) do
+                Jev.record_node_call(method_name) { super(*args, **kwargs, &block) }
+              end
           end
         end
       end

data/lib/bitfab/verdicts.rb

@@ -0,0 +1,276 @@
+# frozen_string_literal: true
+
+require "uri"
+
+require_relative "warn_once"
+
+module Bitfab
+  module VerdictInput
+    JUDGED_VERDICTS = ["pass", "fail"].freeze
+
+    module_function
+
+    def resolve_verdict(verdict, label, allowed, unset)
+      unless label.equal?(unset)
+        raise ArgumentError, "Pass verdict: or the deprecated label:, not both." unless verdict.nil?
+
+        Bitfab.warn_once("label-keyword-deprecated", "label: is deprecated; pass verdict: \"pass\" or \"fail\" instead.")
+        verdict = legacy_verdict(label)
+      end
+      check_verdict(verdict, allowed)
+    end
+
+    def require_annotation(verdict, annotation)
+      return unless JUDGED_VERDICTS.include?(verdict)
+      return unless annotation.nil? || annotation.to_s.empty?
+
+      raise ArgumentError, "annotation: is required for a #{verdict.inspect} verdict: say why the assertion " \
+        "#{(verdict == "pass") ? "passed" : "failed"}. Only an \"abstained\" verdict may omit it."
+    end
+
+    def normalize_entry(entry, allowed)
+      label_key = [:label, "label"].find { |key| entry.key?(key) }
+      abstain_key = [:abstain, "abstain"].find { |key| entry.key?(key) }
+      return entry if label_key.nil? && abstain_key.nil?
+
+      verdict_key = [:verdict, "verdict"].find { |key| entry.key?(key) }
+      unless verdict_key.nil?
+        raise ArgumentError, "A verdict entry carries verdict and the deprecated #{label_key || abstain_key}, not both."
+      end
+
+      if !label_key.nil? && entry[abstain_key] == true
+        raise ArgumentError, "A verdict entry carries both the deprecated label and abstain. " \
+          "Send verdict: \"pass\", \"fail\", or \"abstained\" instead."
+      end
+
+      normalized = entry.except(label_key, abstain_key)
+      if label_key.nil?
+        Bitfab.warn_once("abstain-entry-deprecated", "abstain: true in a verdict entry is deprecated; " \
+          "send verdict: \"abstained\" instead.")
+        return normalized if entry[abstain_key] != true && action_entry?(entry)
+
+        if entry[abstain_key] != true
+          raise ArgumentError, "A verdict entry with abstain: #{entry[abstain_key].inspect} has no verdict. " \
+            "Send verdict: \"pass\", \"fail\", or \"abstained\" instead of the deprecated abstain key."
+        end
+
+        normalized["verdict"] = check_verdict("abstained", allowed)
+      else
+        Bitfab.warn_once("label-entry-deprecated", "label in a verdict entry is deprecated; " \
+          "send verdict: \"pass\" or \"fail\" instead.")
+        normalized["verdict"] = check_verdict(legacy_verdict(entry[label_key]), allowed)
+      end
+      normalized
+    end
+
+    def action_entry?(entry)
+      ["skip", "archive"].any? { |key| entry[key] == true || entry[key.to_sym] == true }
+    end
+
+    def legacy_verdict(label)
+      return "pass" if label == true
+      return "fail" if label == false
+
+      raise ArgumentError, "The deprecated label must be true or false, got #{label.inspect}."
+    end
+
+    def refuse_abstained_extras(entry)
+      return unless [entry["verdict"], entry[:verdict]].include?("abstained")
+
+      extra = ["confidence", "evidence"].find { |key| entry.key?(key) || entry.key?(key.to_sym) }
+      return if extra.nil?
+
+      raise ArgumentError, "An \"abstained\" verdict carries no #{extra}, since nothing was found to judge."
+    end
+
+    def check_verdict(verdict, allowed)
+      return verdict if allowed.include?(verdict)
+
+      raise ArgumentError, "verdict must be #{allowed.map(&:inspect).join(" or ")}, got #{verdict.inspect}."
+    end
+  end
+
+  private_constant :VerdictInput
+
+  # Reads and writes assertion verdicts on traces: an agent's or a person's pass, fail, or abstention
+  # on one assertion, with the reason behind it.
+  class Verdicts
+    VERDICTS_PATH = "/api/sdk/traces/verdicts"
+    HUMAN_VERDICTS_PATH = "#{VERDICTS_PATH}/human"
+    # @deprecated Use {VERDICTS_PATH}.
+    LABELS_PATH = VERDICTS_PATH
+    # @deprecated Use {HUMAN_VERDICTS_PATH}.
+    HUMAN_LABELS_PATH = HUMAN_VERDICTS_PATH
+    AGENT_VERDICTS = ["pass", "fail", "abstained"].freeze
+    HUMAN_VERDICTS = ["pass", "fail"].freeze
+    EVIDENCE_UNSET = Object.new.freeze
+    LABEL_UNSET = Object.new.freeze
+
+    private_constant :EVIDENCE_UNSET, :LABEL_UNSET
+
+    def initialize(http_client)
+      @http_client = http_client
+    end
+
+    # Saves an agent's verdict on one assertion of a trace. +verdict:+ is "pass", "fail", or "abstained",
+    # and +annotation:+ is the reason (required for pass and fail). Target a trace by +trace_id:+, or a
+    # replay by +original_trace_id:+ and +attempt:+ with the +experiment_id:+ it ran under. The deprecated
+    # +label:+ (true or false) is still accepted in place of +verdict:+.
+    def save(assertion_id:, annotation: nil, verdict: nil, trace_id: nil, original_trace_id: nil, attempt: nil,
+      confidence: nil, experiment_id: nil, evidence: EVIDENCE_UNSET, test_run_id: nil, label: LABEL_UNSET)
+      verdict = VerdictInput.resolve_verdict(verdict, label, AGENT_VERDICTS, LABEL_UNSET)
+      VerdictInput.require_annotation(verdict, annotation)
+      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
+      update = verdict_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
+      update["verdict"] = verdict
+      update["annotation"] = annotation unless annotation.nil?
+      update["confidence"] = confidence unless confidence.nil?
+      update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
+      save_all([update], experiment_id:).first
+    end
+
+    # Saves several agent verdict entries in one request. Each entry is a hash with the wire keys
+    # (+traceId+ or +originalTraceId+ and +attempt+, +assertionId+, and one of +verdict+, +skip+, or +archive+).
+    def save_all(verdicts, experiment_id: nil, test_run_id: nil)
+      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
+      verdicts = verdicts.map { |entry| VerdictInput.normalize_entry(entry, AGENT_VERDICTS) }
+      verdicts.each { |entry| VerdictInput.refuse_abstained_extras(entry) }
+      require_assertion_ids(verdicts)
+      payload = {"verdicts" => verdicts}
+      unless experiment_id.nil?
+        payload["experimentId"] = experiment_id
+        payload["testRunId"] = experiment_id
+      end
+      @http_client.request(VERDICTS_PATH, payload)["verdicts"]
+    end
+
+    # Withholds a verdict on one assertion without recording a pass or a fail, with an optional reason.
+    def skip(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
+      test_run_id: nil, annotation: nil)
+      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
+      update = verdict_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("skip" => true)
+      update["annotation"] = annotation unless annotation.nil?
+      save_all([update], experiment_id:).first
+    end
+
+    # Records that the evidence does not show the part of the run this assertion checks, so there was
+    # nothing to judge. Counted as a fail in every pass rate, but stored and read back as an abstention,
+    # apart from a fail. +annotation:+ says what the evidence would have had to show. Shorthand for
+    # +save(verdict: "abstained")+.
+    def abstain(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
+      test_run_id: nil, annotation: nil)
+      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
+      update = verdict_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("verdict" => "abstained")
+      update["annotation"] = annotation unless annotation.nil?
+      save_all([update], experiment_id:).first
+    end
+
+    # Archives the active verdict on one assertion, or an old whole-trace verdict when +assertion_id:+ is omitted.
+    def archive(trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil, assertion_id: nil, test_run_id: nil)
+      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
+      update = verdict_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
+      save_all([update.merge("archive" => true)], experiment_id:).first
+    end
+
+    # Saves a person's verdict on one assertion of a trace. +verdict:+ is "pass" or "fail", since a person
+    # never abstains, and +annotation:+ is the reason, which is required. The deprecated +label:+ (true or
+    # false) is still accepted in place of +verdict:+.
+    def save_human(trace_id:, assertion_id:, annotation: nil, verdict: nil, confidence: nil, evidence: EVIDENCE_UNSET,
+      label: LABEL_UNSET)
+      verdict = VerdictInput.resolve_verdict(verdict, label, HUMAN_VERDICTS, LABEL_UNSET)
+      VerdictInput.require_annotation(verdict, annotation)
+      update = {"traceId" => trace_id, "assertionId" => assertion_id, "verdict" => verdict, "annotation" => annotation}
+      update["confidence"] = confidence unless confidence.nil?
+      update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
+      save_human_all([update]).first
+    end
+
+    # Saves several of a person's verdict entries in one request.
+    def save_human_all(verdicts)
+      verdicts = verdicts.map { |entry| VerdictInput.normalize_entry(entry, HUMAN_VERDICTS) }
+      require_human_assertion_ids(verdicts)
+      @http_client.request(HUMAN_VERDICTS_PATH, {"verdicts" => verdicts})["verdicts"].map do |row|
+        row.key?("label") ? row : row.merge("label" => row["verdict"] == "pass")
+      end
+    end
+
+    # Reads one trace's verdicts, or nil when the trace has none.
+    def get(trace_id)
+      get_all([trace_id]).first
+    end
+
+    # Reads the verdicts of several traces.
+    def get_all(trace_ids)
+      return [] if trace_ids.empty?
+
+      query = URI.encode_www_form("traceIds" => trace_ids.join(","))
+      @http_client.get("#{VERDICTS_PATH}?#{query}")["verdicts"]
+    end
+
+    # Generates span-by-span evidence for one assertion on a trace, to send with its verdict.
+    def generate_evidence(trace_id, assertion_id)
+      encoded_trace_id = URI.encode_www_form_component(trace_id).gsub("+", "%20")
+      encoded_assertion_id = URI.encode_www_form_component(assertion_id).gsub("+", "%20")
+      @http_client.get(
+        "/api/sdk/traces/#{encoded_trace_id}/assertions/#{encoded_assertion_id}/evidence"
+      )["evidence"]
+    end
+
+    # @deprecated Use {#generate_evidence}.
+    def generate_label_evidence(trace_id, assertion_id)
+      Bitfab.warn_once("generate-label-evidence-deprecated",
+        "generate_label_evidence is deprecated; use verdicts.generate_evidence instead.")
+      generate_evidence(trace_id, assertion_id)
+    end
+
+    private
+
+    def require_assertion_ids(verdicts)
+      unscoped = verdicts.reject do |entry|
+        entry_value(entry, "archive") == true || !blank?(entry_value(entry, "assertionId"))
+      end
+      return if unscoped.empty?
+
+      targets = unscoped.map { |entry| entry_value(entry, "traceId") || entry_value(entry, "originalTraceId") }
+      raise ArgumentError, "Agent verdicts must name an assertion: #{targets.join(", ")}. " \
+        "Pass the assertion_id of the assertion each verdict scores to verdicts.save, save_all, skip, and abstain. " \
+        "Only archive may omit it, to clear an old whole-trace verdict."
+    end
+
+    def require_human_assertion_ids(verdicts)
+      unscoped = verdicts.select { |entry| blank?(entry_value(entry, "assertionId")) }
+      return if unscoped.empty?
+
+      targets = unscoped.map { |entry| entry_value(entry, "traceId") }
+      raise ArgumentError, "Human verdicts must name an assertion: #{targets.join(", ")}. " \
+        "Pass the assertion_id of the assertion each verdict scores to verdicts.save_human and save_human_all."
+    end
+
+    def entry_value(entry, key)
+      entry.key?(key) ? entry[key] : entry[key.to_sym]
+    end
+
+    def blank?(value)
+      value.nil? || value.to_s.empty?
+    end
+
+    def verdict_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
+      if trace_id.nil? == original_trace_id.nil?
+        raise ArgumentError, "Pass exactly one of trace_id or original_trace_id. " \
+          "Use original_trace_id for a replay verdict, with the experiment_id it ran under."
+      end
+
+      update = {}
+      update["assertionId"] = assertion_id unless assertion_id.nil?
+      if trace_id.nil?
+        update["originalTraceId"] = original_trace_id
+        update["attempt"] = attempt unless attempt.nil?
+      else
+        update["traceId"] = trace_id
+      end
+      update
+    end
+  end
+
+  Labels = Verdicts
+end

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.65.7"
+  VERSION = "0.67.0"
 end

data/lib/bitfab.rb

@@ -21,7 +21,7 @@ require_relative "bitfab/traces"
 require_relative "bitfab/spans"
 require_relative "bitfab/assertions"
 require_relative "bitfab/scenarios"
-require_relative "bitfab/labels"
+require_relative "bitfab/verdicts"
 require_relative "bitfab/graders"
 require_relative "bitfab/experiments"
 require_relative "bitfab/mock_override"
@@ -30,6 +30,7 @@ require_relative "bitfab/replay_branch"
 require_relative "bitfab/client"
 require_relative "bitfab/seed"
 require_relative "bitfab/baml"
+require_relative "bitfab/jev"
 require_relative "bitfab/traceable"
 require_relative "bitfab/replay_registry"
 require_relative "bitfab/seed_cli"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.65.7
+  version: 0.67.0
 platform: ruby
 authors:
 - Harvest Team
@@ -175,6 +175,7 @@ files:
 - lib/bitfab/git_state.rb
 - lib/bitfab/graders.rb
 - lib/bitfab/http_client.rb
+- lib/bitfab/jev.rb
 - lib/bitfab/labels.rb
 - lib/bitfab/mock_override.rb
 - lib/bitfab/organization_members.rb
@@ -205,6 +206,7 @@ files:
 - lib/bitfab/traceable.rb
 - lib/bitfab/traces.rb
 - lib/bitfab/transport.rb
+- lib/bitfab/verdicts.rb
 - lib/bitfab/version.rb
 - lib/bitfab/warn_once.rb
 homepage: https://bitfab.ai