bitfab 0.43.1 → 0.51.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d1073eb267b5c91b7a845d3abac599939b8838741bfa252b688f0dbe803276c5
-  data.tar.gz: 118f34e8a91a8a0cec3e4071098744fada2879b948577d8bebae275ed1771402
+  metadata.gz: 3dd6fa16ae3ff5f26d94f6b86f9fa702129cfb502e3f94b293e383d107d1c0de
+  data.tar.gz: 220cdb686e1a5522fdcd9579e897f60b0896360583b49615006f0489723c6d21
 SHA512:
-  metadata.gz: 34f05d59438374486058802390023be43c61d0a103b30bec88ee431623dc8201253398d08e0a80600c6d0bf8d7f8861b6078748bfe1bdf8b5f4e0b92d2bb9960
-  data.tar.gz: 389a6a04ecbf88da123f48a806854f00e51b9f2cf8dc45ea25b698050f288829bfb81bf7e262e82fdaad80a0b0c2ac583f8c42e5d43e244d3259f544bd03e708
+  metadata.gz: cf75d4072ac82e17312f40f86c1a0e54c3a60a019818695507a5d53cfc1bfb15db6e705ad6a971c6e77abd67b4b752e180457d483b5f42becc52167323784dd9
+  data.tar.gz: 8857e35745377b128479948ec4622ef2223f3876015f03e161ac80e43f61ec1c1556cb4d4839dd174c35afba9b3e1fbfcbd0c4da69766b80cd593cd8caf96c07

data/lib/bitfab/client.rb

@@ -93,11 +93,14 @@ module Bitfab
     # @param method_name [Symbol] the method to replay
     # @param trace_function_key [String] the trace function key for this method
     # @param limit [Integer, nil] maximum number of traces to replay (default: 5;
-    #   maximum: 5,000). Ignored when trace_ids or dataset_id is passed because
+    #   maximum: 5,000). Ignored when trace_ids or a dataset is passed because
     #   either source already determines how many traces replay. Supplying
-    #   trace_ids also emits a warning.
-    # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100),
-    #   mutually exclusive with dataset_id
+    #   trace_ids also emits a warning. To replay part of a dataset selection,
+    #   name the members to run in trace_ids rather than bounding it here.
+    # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100).
+    #   Passed alongside dataset_id or dataset_ids it pins which members of that
+    #   selection replay, and the server rejects any ID none of those datasets
+    #   contains
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -110,9 +113,14 @@ module Bitfab
     #   Omit to capture automatically, or pass nil to suppress file capture.
     # @param experiment_group_id [String, nil] optional UUID grouping multiple
     #   replay runs into a single experiment batch
+    # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
+    #   replay runs against, for benchmarking one method against several corpora
+    #   in a single run; mutually exclusive with dataset_id. Passing trace_ids
+    #   alongside it narrows the run to those members
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
-    #   runs against, mutually exclusive with trace_ids and stored on the
-    #   resulting experiment for durable attribution
+    #   runs against, mutually exclusive with dataset_ids and stored on the
+    #   resulting experiment for durable attribution. Alone it replays the
+    #   dataset's full membership, and with trace_ids only those members
     # @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
@@ -158,7 +166,7 @@ module Bitfab
     #   queued work from in-flight work. A raising callback never crashes the run.
     # @return [Hash] with :items, :test_run_id, :test_run_url
     def replay(receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, max_concurrency: 10,
-      name: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, grader_ids: nil, mock: "marked",
+      name: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked",
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil)
       Replay.run(
         self,
@@ -173,6 +181,7 @@ module Bitfab
         code_change_files:,
         experiment_group_id:,
         dataset_id:,
+        dataset_ids:,
         grader_ids:,
         mock:,
         adapt_inputs:,

data/lib/bitfab/http_client.rb

@@ -173,14 +173,17 @@ module Bitfab
     #   replay runs into a single experiment batch
     # @param name [String, nil] optional display name for the resulting
     #   experiment/test run
-    # @param dataset_id [String, nil] optional UUID of the dataset this replay
-    #   runs against, stored on the resulting experiment for durable attribution
+    # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
+    #   replay runs against. The run replays the union of their traces, graded by
+    #   the union of their graders, and is attributed to every one of them. A
+    #   single id travels as datasetId so a new SDK keeps working against a
+    #   server that predates datasetIds.
     # @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
-      code_change_files: nil, experiment_group_id: nil, name: nil, include_db_branch_lease: false, dataset_id: nil,
+      code_change_files: nil, experiment_group_id: nil, name: nil, include_db_branch_lease: false, dataset_ids: nil,
       grader_ids: nil, db_branch_settings: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
@@ -195,7 +198,13 @@ module Bitfab
       payload["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?
       payload["includeDbBranchLease"] = true if include_db_branch_lease
       payload["lazyDbBranchLease"] = true if include_db_branch_lease
-      payload["datasetId"] = dataset_id unless dataset_id.nil?
+      unless dataset_ids.nil?
+        if dataset_ids.length == 1
+          payload["datasetId"] = dataset_ids.first
+        else
+          payload["datasetIds"] = dataset_ids
+        end
+      end
       payload["graderIds"] = grader_ids unless grader_ids.nil?
       payload["dbBranchSettings"] = db_branch_settings unless db_branch_settings.nil?
 

data/lib/bitfab/replay.rb

@@ -187,7 +187,7 @@ module Bitfab
     # @param method_name [Symbol] the method to replay
     # @param trace_function_key [String] the trace function key for this method
     # @param limit [Integer, nil] maximum number of traces to replay (default: 5).
-    #   Ignored when trace_ids or dataset_id is passed because either source
+    #   Ignored when trace_ids or a dataset is passed because either source
     #   already determines how many traces replay. Supplying trace_ids also
     #   emits a warning.
     # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100)
@@ -205,7 +205,15 @@ module Bitfab
     # @param experiment_group_id [String, nil] optional UUID grouping multiple
     #   replay runs into a single experiment batch
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
-    #   runs against, stored on the resulting experiment for durable attribution
+    #   runs against, stored on the resulting experiment for durable attribution;
+    #   mutually exclusive with dataset_ids. Alone it replays the dataset's full
+    #   membership, and with trace_ids it replays only those members
+    # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
+    #   replay runs against, for benchmarking one method against several corpora
+    #   in a single run; mutually exclusive with dataset_id. The run replays the
+    #   union of their traces, graded by the union of their graders, and is
+    #   attributed to every one of them. Passing trace_ids alongside it narrows
+    #   the run to those members
     # @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
@@ -248,7 +256,7 @@ module Bitfab
     # @return [Hash] with :items, :test_run_id, :test_run_url
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil,
       max_concurrency: 10, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil,
-      dataset_id: nil, grader_ids: nil, mock: "marked",
+      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked",
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil)
       unless MOCK_STRATEGIES.include?(mock.to_s)
         raise ArgumentError, "Invalid mock strategy '#{mock}'. Must be one of: #{MOCK_STRATEGIES.join(", ")}"
@@ -260,10 +268,7 @@ module Bitfab
           raise ArgumentError, "trace_ids supports at most 100 trace IDs per replay (got #{trace_ids.length})."
         end
       end
-      if trace_ids && dataset_id
-        raise ArgumentError,
-          "trace_ids and dataset_id select different replay sources and cannot be used together."
-      end
+      resolved_dataset_ids = resolve_dataset_ids(dataset_id, dataset_ids)
       if limit && trace_ids
         warn "Bitfab: limit is ignored when trace_ids is passed: the explicit trace ID list already " \
           "determines how many traces replay."
@@ -321,7 +326,7 @@ module Bitfab
         code_change_files:,
         experiment_group_id:,
         include_db_branch_lease:,
-        dataset_id:,
+        dataset_ids: resolved_dataset_ids,
         grader_ids:,
         db_branch_settings: resolved_db_branch_settings
       )
@@ -472,6 +477,19 @@ module Bitfab
     CC_MAX_FILE_BYTES = 500_000
     CC_MAX_TOTAL_BYTES = 2_000_000
 
+    def resolve_dataset_ids(dataset_id, dataset_ids)
+      if dataset_id && dataset_ids
+        raise ArgumentError,
+          "dataset_id and dataset_ids are the same selector: pass one dataset through dataset_id, " \
+          "or several through dataset_ids."
+      end
+      return [dataset_id] if dataset_id
+      return nil if dataset_ids.nil?
+      raise ArgumentError, "dataset_ids must contain at least one dataset ID." if dataset_ids.empty?
+
+      dataset_ids.uniq
+    end
+
     # Whether the caller asked for database branching. +true+ and a Hash both
     # turn it on; +false+ and +nil+ leave it off. Kept separate from
     # +db_branch_settings+ because +db_branch: true+ enables branching while

data/lib/bitfab/replay_registry.rb

@@ -20,7 +20,7 @@ module Bitfab
   class ReplayRegistry
     OPTION_NAMES = %i[
       limit trace_ids name max_concurrency code_change_description
-      code_change_files experiment_group_id dataset_id grader_ids mock
+      code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
       mock_override adapt_inputs db_branch
     ].freeze
 
@@ -81,8 +81,26 @@ module Bitfab
 
   # Argument parsing and output for the SDK-owned replay command.
   module ReplayCli
+    MAX_PINNED_TRACE_IDS = 100
+
     module_function
 
+    def pinned_dataset_members(client, dataset_ids, limit)
+      members = dataset_ids.flat_map { |dataset_id|
+        client.datasets.list_traces(dataset_id).fetch("traceIds")
+      }.uniq.sort
+      return nil if limit >= members.length
+
+      if limit > MAX_PINNED_TRACE_IDS
+        raise ArgumentError,
+          "Bounding a dataset replay to #{limit} names one trace ID per replayed item, and " \
+            "a replay accepts at most #{MAX_PINNED_TRACE_IDS}. Drop the limit to replay " \
+            "every selected dataset in full."
+      end
+
+      members.first(limit)
+    end
+
     def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
       args = parse(registry, argv.dup)
       registration = registry.fetch(args.fetch(:pipeline))
@@ -97,29 +115,39 @@ module Bitfab
         options.merge!(dynamic_options)
       end
 
-      if options[:trace_ids] && options[:dataset_id]
+      selected_dataset_ids = registry_dataset_ids(options)
+      options.delete(:dataset_id)
+      options.delete(:dataset_ids)
+      options[:dataset_ids] = selected_dataset_ids if selected_dataset_ids
+
+      if options[:trace_ids] && selected_dataset_ids
         raise ArgumentError,
-          "Replay registry options trace_ids and dataset_id select different sources and cannot be used together."
+          "Replay registry options trace_ids and dataset_ids select different sources and cannot be used together."
       end
 
       if args[:trace_ids]
-        options.delete(:limit)
-        options.delete(:dataset_id)
+        options.delete(:dataset_ids)
         options[:trace_ids] = args[:trace_ids]
-      elsif args[:dataset_id]
-        options.delete(:trace_ids)
-        options.delete(:limit)
-      elsif args[:limit]
+      elsif args[:dataset_ids]
         options.delete(:trace_ids)
-        options.delete(:dataset_id)
-        options[:limit] = args[:limit]
-      elsif options[:trace_ids] || options[:dataset_id]
-        options.delete(:limit)
+      end
+
+      bound = args[:limit] || options[:limit]
+      options.delete(:limit)
+      bounded_dataset_ids = args[:dataset_ids] || options[:dataset_ids]
+
+      if options[:trace_ids]
+        options[:trace_ids] = options[:trace_ids].first(bound) if bound
+      elsif bounded_dataset_ids
+        if bound
+          pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound)
+          options[:trace_ids] = pinned if pinned
+        end
       else
-        options[:limit] = options.fetch(:limit, 10)
+        options[:limit] = bound || 10
       end
 
-      %i[name max_concurrency experiment_group_id dataset_id grader_ids mock].each do |key|
+      %i[name max_concurrency experiment_group_id dataset_ids grader_ids mock].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -145,8 +173,7 @@ module Bitfab
       options[:on_item_start] = reporter
       options[:on_item_finish] = reporter
 
-      count = options[:trace_ids]&.length || options[:limit] || "dataset"
-      stderr.puts "[replay] Replaying #{count} traces from \"#{registration.trace_function_key}\"..."
+      stderr.puts "[replay] Replaying #{describe_selection(options)} from \"#{registration.trace_function_key}\"..."
 
       result = begin
         registration.client.replay(
@@ -175,7 +202,10 @@ module Bitfab
       args = {}
       parser = OptionParser.new do |options|
         options.banner = "Usage: bitfab-replay <#{registry.names.join("|")}> [options]"
-        options.on("--limit N", Integer) { |value| args[:limit] = positive_integer("--limit", value) }
+        options.on("--limit N", Integer,
+          "How many traces to replay. It bounds a trace ID or dataset selection instead of replacing it") do |value|
+          args[:limit] = positive_integer("--limit", value)
+        end
         options.on("--trace-ids IDS") { |value| args[:trace_ids] = comma_separated("--trace-ids", value) }
         options.on("--name NAME") { |value| args[:name] = value }
         options.on("--concurrency N", Integer) do |value|
@@ -186,7 +216,7 @@ module Bitfab
         end
         options.on("--code-change PATH") { |value| args[:code_change] = value }
         options.on("--experiment-group-id UUID") { |value| args[:experiment_group_id] = value }
-        options.on("--dataset-id UUID") { |value| args[:dataset_id] = value }
+        options.on("--dataset-ids IDS", "--dataset-id IDS") { |value| args[:dataset_ids] = comma_separated("--dataset-ids", value) }
         options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
         options.on("--mock STRATEGY", %w[none all marked]) { |value| args[:mock] = value }
         options.on("--db-branch") { args[:db_branch] = true }
@@ -196,9 +226,9 @@ module Bitfab
         options.on("--param NAME=VALUE") { |value| (args[:param] ||= []) << value }
       end
       parser.parse!(argv)
-      if args[:trace_ids] && args[:dataset_id]
+      if args[:trace_ids] && args[:dataset_ids]
         raise OptionParser::InvalidArgument,
-          "--trace-ids and --dataset-id select different replay sources and cannot be used together"
+          "--trace-ids and --dataset-ids select different replay sources and cannot be used together"
       end
       if args[:code_change] && args[:no_code_change]
         raise OptionParser::InvalidArgument, "--code-change and --no-code-change cannot be used together"
@@ -210,6 +240,26 @@ module Bitfab
       args.merge(pipeline:)
     end
 
+    def registry_dataset_ids(options)
+      dataset_id = options[:dataset_id]
+      dataset_ids = options[:dataset_ids]
+      if dataset_id && dataset_ids
+        raise ArgumentError,
+          "Replay registry options dataset_id and dataset_ids are the same selector: set one of them."
+      end
+      return [dataset_id] if dataset_id
+
+      dataset_ids
+    end
+
+    def describe_selection(options)
+      return "#{options[:trace_ids].length} traces" if options[:trace_ids]
+      return "#{options[:limit]} traces" if options[:limit]
+
+      dataset_count = options[:dataset_ids]&.length || 1
+      (dataset_count > 1) ? "#{dataset_count} datasets' traces" : "dataset traces"
+    end
+
     def positive_integer(flag, value)
       raise OptionParser::InvalidArgument, "#{flag} must be a positive integer" if value < 1
 

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.43.1"
+  VERSION = "0.51.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.43.1
+  version: 0.51.1
 platform: ruby
 authors:
 - Harvest Team