bitfab 0.51.0 → 0.51.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ece2db3c44be909ec097e539479143d0c4138663809c26a05a968076e681066
-  data.tar.gz: 6d68e352c7e484586147d1700998de7936fe904c5d04cabfe07df80e302104b1
+  metadata.gz: 3dd6fa16ae3ff5f26d94f6b86f9fa702129cfb502e3f94b293e383d107d1c0de
+  data.tar.gz: 220cdb686e1a5522fdcd9579e897f60b0896360583b49615006f0489723c6d21
 SHA512:
-  metadata.gz: bbb4750e80b63864d612592e1bebaac664463059e2d139fe7d556f68ce73b46868001daa54391d1a4562767a2e7e16355fdb8eefc89c64ace8f864dffffce86b
-  data.tar.gz: 02d731c682b787b8233a1c9a43135fc8b6d17d5959806ff65bec3be9979dbda95d1bfb6d01373b427405a5fb9fccd976c87706d8d488ea7733fe70ab0f8fa15b
+  metadata.gz: cf75d4072ac82e17312f40f86c1a0e54c3a60a019818695507a5d53cfc1bfb15db6e705ad6a971c6e77abd67b4b752e180457d483b5f42becc52167323784dd9
+  data.tar.gz: 8857e35745377b128479948ec4622ef2223f3876015f03e161ac80e43f61ec1c1556cb4d4839dd174c35afba9b3e1fbfcbd0c4da69766b80cd593cd8caf96c07

data/lib/bitfab/client.rb

@@ -95,9 +95,12 @@ module Bitfab
     # @param limit [Integer, nil] maximum number of traces to replay (default: 5;
     #   maximum: 5,000). Ignored when trace_ids or a dataset is passed because
     #   either source already determines how many traces replay. Supplying
-    #   trace_ids also emits a warning.
-    # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100),
-    #   mutually exclusive with dataset_id and dataset_ids
+    #   trace_ids also emits a warning. To replay part of a dataset selection,
+    #   name the members to run in trace_ids rather than bounding it here.
+    # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100).
+    #   Passed alongside dataset_id or dataset_ids it pins which members of that
+    #   selection replay, and the server rejects any ID none of those datasets
+    #   contains
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -112,10 +115,12 @@ module Bitfab
     #   replay runs into a single experiment batch
     # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
     #   replay runs against, for benchmarking one method against several corpora
-    #   in a single run; mutually exclusive with dataset_id
+    #   in a single run; mutually exclusive with dataset_id. Passing trace_ids
+    #   alongside it narrows the run to those members
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
-    #   runs against, mutually exclusive with trace_ids and stored on the
-    #   resulting experiment for durable attribution
+    #   runs against, mutually exclusive with dataset_ids and stored on the
+    #   resulting experiment for durable attribution. Alone it replays the
+    #   dataset's full membership, and with trace_ids only those members
     # @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the

data/lib/bitfab/replay.rb

@@ -206,12 +206,14 @@ module Bitfab
     #   replay runs into a single experiment batch
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
     #   runs against, stored on the resulting experiment for durable attribution;
-    #   mutually exclusive with dataset_ids
+    #   mutually exclusive with dataset_ids. Alone it replays the dataset's full
+    #   membership, and with trace_ids it replays only those members
     # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
     #   replay runs against, for benchmarking one method against several corpora
     #   in a single run; mutually exclusive with dataset_id. The run replays the
     #   union of their traces, graded by the union of their graders, and is
-    #   attributed to every one of them
+    #   attributed to every one of them. Passing trace_ids alongside it narrows
+    #   the run to those members
     # @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
@@ -267,10 +269,6 @@ module Bitfab
         end
       end
       resolved_dataset_ids = resolve_dataset_ids(dataset_id, dataset_ids)
-      if trace_ids && resolved_dataset_ids
-        raise ArgumentError,
-          "trace_ids and dataset_ids select different replay sources and cannot be used together."
-      end
       if limit && trace_ids
         warn "Bitfab: limit is ignored when trace_ids is passed: the explicit trace ID list already " \
           "determines how many traces replay."

data/lib/bitfab/replay_registry.rb

@@ -81,8 +81,26 @@ module Bitfab
 
   # Argument parsing and output for the SDK-owned replay command.
   module ReplayCli
+    MAX_PINNED_TRACE_IDS = 100
+
     module_function
 
+    def pinned_dataset_members(client, dataset_ids, limit)
+      members = dataset_ids.flat_map { |dataset_id|
+        client.datasets.list_traces(dataset_id).fetch("traceIds")
+      }.uniq.sort
+      return nil if limit >= members.length
+
+      if limit > MAX_PINNED_TRACE_IDS
+        raise ArgumentError,
+          "Bounding a dataset replay to #{limit} names one trace ID per replayed item, and " \
+            "a replay accepts at most #{MAX_PINNED_TRACE_IDS}. Drop the limit to replay " \
+            "every selected dataset in full."
+      end
+
+      members.first(limit)
+    end
+
     def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
       args = parse(registry, argv.dup)
       registration = registry.fetch(args.fetch(:pipeline))
@@ -108,20 +126,25 @@ module Bitfab
       end
 
       if args[:trace_ids]
-        options.delete(:limit)
         options.delete(:dataset_ids)
         options[:trace_ids] = args[:trace_ids]
       elsif args[:dataset_ids]
         options.delete(:trace_ids)
-        options.delete(:limit)
-      elsif args[:limit]
-        options.delete(:trace_ids)
-        options.delete(:dataset_ids)
-        options[:limit] = args[:limit]
-      elsif options[:trace_ids] || selected_dataset_ids
-        options.delete(:limit)
+      end
+
+      bound = args[:limit] || options[:limit]
+      options.delete(:limit)
+      bounded_dataset_ids = args[:dataset_ids] || options[:dataset_ids]
+
+      if options[:trace_ids]
+        options[:trace_ids] = options[:trace_ids].first(bound) if bound
+      elsif bounded_dataset_ids
+        if bound
+          pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound)
+          options[:trace_ids] = pinned if pinned
+        end
       else
-        options[:limit] = options.fetch(:limit, 10)
+        options[:limit] = bound || 10
       end
 
       %i[name max_concurrency experiment_group_id dataset_ids grader_ids mock].each do |key|
@@ -179,7 +202,10 @@ module Bitfab
       args = {}
       parser = OptionParser.new do |options|
         options.banner = "Usage: bitfab-replay <#{registry.names.join("|")}> [options]"
-        options.on("--limit N", Integer) { |value| args[:limit] = positive_integer("--limit", value) }
+        options.on("--limit N", Integer,
+          "How many traces to replay. It bounds a trace ID or dataset selection instead of replacing it") do |value|
+          args[:limit] = positive_integer("--limit", value)
+        end
         options.on("--trace-ids IDS") { |value| args[:trace_ids] = comma_separated("--trace-ids", value) }
         options.on("--name NAME") { |value| args[:name] = value }
         options.on("--concurrency N", Integer) do |value|

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.51.0"
+  VERSION = "0.51.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.51.0
+  version: 0.51.1
 platform: ruby
 authors:
 - Harvest Team