bitfab 0.43.0 → 0.51.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32715330094ba79b22d0d629581b3d3ffb22dbfffc3ac6024545b7f841b6d7a5
-  data.tar.gz: 76a4c5cb83b5850596d2d54227402fc86320854d5d1925dd1861588ac0cd4f04
+  metadata.gz: 5ece2db3c44be909ec097e539479143d0c4138663809c26a05a968076e681066
+  data.tar.gz: 6d68e352c7e484586147d1700998de7936fe904c5d04cabfe07df80e302104b1
 SHA512:
-  metadata.gz: f2a5f30d4298a65e358cd2bd313c2ebbe443740f757f0f60e07f3fa4aa272e7f9e90b6d3a0b2d9184039bfaba563df1441e1692a7cbdb0dea0ad24aaf42c5aa3
-  data.tar.gz: 143e9f6be9ba689edfd12ffd4cc30efda985cafa7198e56ff68ea6369ff6615b849e508ee4624ac719489d64288b212974c70055df25b22f157c3d563fdc149c
+  metadata.gz: bbb4750e80b63864d612592e1bebaac664463059e2d139fe7d556f68ce73b46868001daa54391d1a4562767a2e7e16355fdb8eefc89c64ace8f864dffffce86b
+  data.tar.gz: 02d731c682b787b8233a1c9a43135fc8b6d17d5959806ff65bec3be9979dbda95d1bfb6d01373b427405a5fb9fccd976c87706d8d488ea7733fe70ab0f8fa15b

data/lib/bitfab/client.rb

@@ -93,11 +93,11 @@ module Bitfab
     # @param method_name [Symbol] the method to replay
     # @param trace_function_key [String] the trace function key for this method
     # @param limit [Integer, nil] maximum number of traces to replay (default: 5;
-    #   maximum: 5,000). Ignored when trace_ids or dataset_id is passed because
+    #   maximum: 5,000). Ignored when trace_ids or a dataset is passed because
     #   either source already determines how many traces replay. Supplying
     #   trace_ids also emits a warning.
     # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100),
-    #   mutually exclusive with dataset_id
+    #   mutually exclusive with dataset_id and dataset_ids
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -110,6 +110,9 @@ module Bitfab
     #   Omit to capture automatically, or pass nil to suppress file capture.
     # @param experiment_group_id [String, nil] optional UUID grouping multiple
     #   replay runs into a single experiment batch
+    # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
+    #   replay runs against, for benchmarking one method against several corpora
+    #   in a single run; mutually exclusive with dataset_id
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
     #   runs against, mutually exclusive with trace_ids and stored on the
     #   resulting experiment for durable attribution
@@ -158,7 +161,7 @@ module Bitfab
     #   queued work from in-flight work. A raising callback never crashes the run.
     # @return [Hash] with :items, :test_run_id, :test_run_url
     def replay(receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, max_concurrency: 10,
-      name: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, grader_ids: nil, mock: "marked",
+      name: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked",
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil)
       Replay.run(
         self,
@@ -173,6 +176,7 @@ module Bitfab
         code_change_files:,
         experiment_group_id:,
         dataset_id:,
+        dataset_ids:,
         grader_ids:,
         mock:,
         adapt_inputs:,

data/lib/bitfab/http_client.rb

@@ -173,14 +173,17 @@ module Bitfab
     #   replay runs into a single experiment batch
     # @param name [String, nil] optional display name for the resulting
     #   experiment/test run
-    # @param dataset_id [String, nil] optional UUID of the dataset this replay
-    #   runs against, stored on the resulting experiment for durable attribution
+    # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
+    #   replay runs against. The run replays the union of their traces, graded by
+    #   the union of their graders, and is attributed to every one of them. A
+    #   single id travels as datasetId so a new SDK keeps working against a
+    #   server that predates datasetIds.
     # @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
-      code_change_files: nil, experiment_group_id: nil, name: nil, include_db_branch_lease: false, dataset_id: nil,
+      code_change_files: nil, experiment_group_id: nil, name: nil, include_db_branch_lease: false, dataset_ids: nil,
       grader_ids: nil, db_branch_settings: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
@@ -195,7 +198,13 @@ module Bitfab
       payload["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?
       payload["includeDbBranchLease"] = true if include_db_branch_lease
       payload["lazyDbBranchLease"] = true if include_db_branch_lease
-      payload["datasetId"] = dataset_id unless dataset_id.nil?
+      unless dataset_ids.nil?
+        if dataset_ids.length == 1
+          payload["datasetId"] = dataset_ids.first
+        else
+          payload["datasetIds"] = dataset_ids
+        end
+      end
       payload["graderIds"] = grader_ids unless grader_ids.nil?
       payload["dbBranchSettings"] = db_branch_settings unless db_branch_settings.nil?
 

data/lib/bitfab/replay.rb

@@ -187,7 +187,7 @@ module Bitfab
     # @param method_name [Symbol] the method to replay
     # @param trace_function_key [String] the trace function key for this method
     # @param limit [Integer, nil] maximum number of traces to replay (default: 5).
-    #   Ignored when trace_ids or dataset_id is passed because either source
+    #   Ignored when trace_ids or a dataset is passed because either source
     #   already determines how many traces replay. Supplying trace_ids also
     #   emits a warning.
     # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100)
@@ -205,7 +205,13 @@ module Bitfab
     # @param experiment_group_id [String, nil] optional UUID grouping multiple
     #   replay runs into a single experiment batch
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
-    #   runs against, stored on the resulting experiment for durable attribution
+    #   runs against, stored on the resulting experiment for durable attribution;
+    #   mutually exclusive with dataset_ids
+    # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
+    #   replay runs against, for benchmarking one method against several corpora
+    #   in a single run; mutually exclusive with dataset_id. The run replays the
+    #   union of their traces, graded by the union of their graders, and is
+    #   attributed to every one of them
     # @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
@@ -248,7 +254,7 @@ module Bitfab
     # @return [Hash] with :items, :test_run_id, :test_run_url
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil,
       max_concurrency: 10, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil,
-      dataset_id: nil, grader_ids: nil, mock: "marked",
+      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked",
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil)
       unless MOCK_STRATEGIES.include?(mock.to_s)
         raise ArgumentError, "Invalid mock strategy '#{mock}'. Must be one of: #{MOCK_STRATEGIES.join(", ")}"
@@ -260,9 +266,10 @@ module Bitfab
           raise ArgumentError, "trace_ids supports at most 100 trace IDs per replay (got #{trace_ids.length})."
         end
       end
-      if trace_ids && dataset_id
+      resolved_dataset_ids = resolve_dataset_ids(dataset_id, dataset_ids)
+      if trace_ids && resolved_dataset_ids
         raise ArgumentError,
-          "trace_ids and dataset_id select different replay sources and cannot be used together."
+          "trace_ids and dataset_ids select different replay sources and cannot be used together."
       end
       if limit && trace_ids
         warn "Bitfab: limit is ignored when trace_ids is passed: the explicit trace ID list already " \
@@ -321,7 +328,7 @@ module Bitfab
         code_change_files:,
         experiment_group_id:,
         include_db_branch_lease:,
-        dataset_id:,
+        dataset_ids: resolved_dataset_ids,
         grader_ids:,
         db_branch_settings: resolved_db_branch_settings
       )
@@ -369,6 +376,8 @@ module Bitfab
       # run's tokens (span-aggregated server-side), used below to fill each
       # item's :tokens.
       replay_tokens = complete_response&.dig("tokens") || {}
+      replay_outlines = complete_response&.dig("traceOutlines") || {}
+      original_outlines = complete_response&.dig("originalTraceOutlines") || {}
 
       # trace_id_map maps each item's client-side correlation id (:_sdk_trace_id,
       # which tagged that item's spans during the run) to the server's trace row
@@ -392,7 +401,11 @@ module Bitfab
             completed_count += 1
             missing << local_id if mapped.nil?
           end
-          item[:tokens] = normalize_tokens(replay_tokens[mapped]) if mapped
+          if mapped
+            item[:tokens] = normalize_tokens(replay_tokens[mapped])
+            item[:trace_outline] = replay_outlines[mapped]
+          end
+          item[:original_trace_outline] = original_outlines[item[:original_trace_id]] if item[:original_trace_id]
         end
         # ALL completed items missing: systemic (the replayed method is not
         # traced, or uploads are wholesale broken). Raise; the run's traces
@@ -466,6 +479,19 @@ module Bitfab
     CC_MAX_FILE_BYTES = 500_000
     CC_MAX_TOTAL_BYTES = 2_000_000
 
+    def resolve_dataset_ids(dataset_id, dataset_ids)
+      if dataset_id && dataset_ids
+        raise ArgumentError,
+          "dataset_id and dataset_ids are the same selector: pass one dataset through dataset_id, " \
+          "or several through dataset_ids."
+      end
+      return [dataset_id] if dataset_id
+      return nil if dataset_ids.nil?
+      raise ArgumentError, "dataset_ids must contain at least one dataset ID." if dataset_ids.empty?
+
+      dataset_ids.uniq
+    end
+
     # Whether the caller asked for database branching. +true+ and a Hash both
     # turn it on; +false+ and +nil+ leave it off. Kept separate from
     # +db_branch_settings+ because +db_branch: true+ enables branching while
@@ -778,7 +804,9 @@ module Bitfab
                 tokens: result[:tokens],
                 model: result[:model],
                 db_snapshot_ref: result[:db_snapshot_ref],
-                db_branch_timings: result[:db_branch_timings]
+                db_branch_timings: result[:db_branch_timings],
+                trace_outline: result[:trace_outline],
+                original_trace_outline: result[:original_trace_outline]
               }
             })
           rescue => e
@@ -965,7 +993,9 @@ module Bitfab
         source_trace_id: original_trace_id,
         source_span_id: original_span_id,
         db_snapshot_ref:,
-        db_branch_timings:
+        db_branch_timings:,
+        trace_outline: nil,
+        original_trace_outline: nil
       }
     ensure
       release_db_branch_lease(http_client, lease) if lease
@@ -1265,7 +1295,9 @@ module Bitfab
         source_trace_id: source_bitfab_trace_id,
         source_span_id: input_source_span_id,
         db_snapshot_ref:,
-        db_branch_timings:
+        db_branch_timings:,
+        trace_outline: nil,
+        original_trace_outline: nil
       }
     end
   end

data/lib/bitfab/replay_registry.rb

@@ -20,7 +20,7 @@ module Bitfab
   class ReplayRegistry
     OPTION_NAMES = %i[
       limit trace_ids name max_concurrency code_change_description
-      code_change_files experiment_group_id dataset_id grader_ids mock
+      code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
       mock_override adapt_inputs db_branch
     ].freeze
 
@@ -97,29 +97,34 @@ module Bitfab
         options.merge!(dynamic_options)
       end
 
-      if options[:trace_ids] && options[:dataset_id]
+      selected_dataset_ids = registry_dataset_ids(options)
+      options.delete(:dataset_id)
+      options.delete(:dataset_ids)
+      options[:dataset_ids] = selected_dataset_ids if selected_dataset_ids
+
+      if options[:trace_ids] && selected_dataset_ids
         raise ArgumentError,
-          "Replay registry options trace_ids and dataset_id select different sources and cannot be used together."
+          "Replay registry options trace_ids and dataset_ids select different sources and cannot be used together."
       end
 
       if args[:trace_ids]
         options.delete(:limit)
-        options.delete(:dataset_id)
+        options.delete(:dataset_ids)
         options[:trace_ids] = args[:trace_ids]
-      elsif args[:dataset_id]
+      elsif args[:dataset_ids]
         options.delete(:trace_ids)
         options.delete(:limit)
       elsif args[:limit]
         options.delete(:trace_ids)
-        options.delete(:dataset_id)
+        options.delete(:dataset_ids)
         options[:limit] = args[:limit]
-      elsif options[:trace_ids] || options[:dataset_id]
+      elsif options[:trace_ids] || selected_dataset_ids
         options.delete(:limit)
       else
         options[:limit] = options.fetch(:limit, 10)
       end
 
-      %i[name max_concurrency experiment_group_id dataset_id grader_ids mock].each do |key|
+      %i[name max_concurrency experiment_group_id dataset_ids grader_ids mock].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -145,8 +150,7 @@ module Bitfab
       options[:on_item_start] = reporter
       options[:on_item_finish] = reporter
 
-      count = options[:trace_ids]&.length || options[:limit] || "dataset"
-      stderr.puts "[replay] Replaying #{count} traces from \"#{registration.trace_function_key}\"..."
+      stderr.puts "[replay] Replaying #{describe_selection(options)} from \"#{registration.trace_function_key}\"..."
 
       result = begin
         registration.client.replay(
@@ -186,7 +190,7 @@ module Bitfab
         end
         options.on("--code-change PATH") { |value| args[:code_change] = value }
         options.on("--experiment-group-id UUID") { |value| args[:experiment_group_id] = value }
-        options.on("--dataset-id UUID") { |value| args[:dataset_id] = value }
+        options.on("--dataset-ids IDS", "--dataset-id IDS") { |value| args[:dataset_ids] = comma_separated("--dataset-ids", value) }
         options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
         options.on("--mock STRATEGY", %w[none all marked]) { |value| args[:mock] = value }
         options.on("--db-branch") { args[:db_branch] = true }
@@ -196,9 +200,9 @@ module Bitfab
         options.on("--param NAME=VALUE") { |value| (args[:param] ||= []) << value }
       end
       parser.parse!(argv)
-      if args[:trace_ids] && args[:dataset_id]
+      if args[:trace_ids] && args[:dataset_ids]
         raise OptionParser::InvalidArgument,
-          "--trace-ids and --dataset-id select different replay sources and cannot be used together"
+          "--trace-ids and --dataset-ids select different replay sources and cannot be used together"
       end
       if args[:code_change] && args[:no_code_change]
         raise OptionParser::InvalidArgument, "--code-change and --no-code-change cannot be used together"
@@ -210,6 +214,26 @@ module Bitfab
       args.merge(pipeline:)
     end
 
+    def registry_dataset_ids(options)
+      dataset_id = options[:dataset_id]
+      dataset_ids = options[:dataset_ids]
+      if dataset_id && dataset_ids
+        raise ArgumentError,
+          "Replay registry options dataset_id and dataset_ids are the same selector: set one of them."
+      end
+      return [dataset_id] if dataset_id
+
+      dataset_ids
+    end
+
+    def describe_selection(options)
+      return "#{options[:trace_ids].length} traces" if options[:trace_ids]
+      return "#{options[:limit]} traces" if options[:limit]
+
+      dataset_count = options[:dataset_ids]&.length || 1
+      (dataset_count > 1) ? "#{dataset_count} datasets' traces" : "dataset traces"
+    end
+
     def positive_integer(flag, value)
       raise OptionParser::InvalidArgument, "#{flag} must be a positive integer" if value < 1
 

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.43.0"
+  VERSION = "0.51.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.43.0
+  version: 0.51.0
 platform: ruby
 authors:
 - Harvest Team