bitfab 0.61.1 → 0.61.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49768c67671b66ea8066cfa078dbba13ac3548b83b9a0abaa628ab864a61b99f
-  data.tar.gz: 76e711a41258fa421a69cd6a0475a793698f3bdd001f60d0104ec47a5096e8da
+  metadata.gz: c3ee5133d6797e0d931936274c5d05bea38532e5879f8a55567a5417cabdd04d
+  data.tar.gz: 57666b69b7277a81a3715a3f386d8660d47d6568ef027adc7a4a10fafc02f6c2
 SHA512:
-  metadata.gz: bdef7ec071b9740bd8183056b68b84e9ab2c8004f0918678c604e9e35d6426144891bd5df0e476a59f8198e1f17459a4e0e48c08e3eab78061b119146e38b3f1
-  data.tar.gz: d441d62917aaa84b3dfbc88f6544e1630645fc5a595a7a3cf9b3e111bdbf62b43059b971ecb9f51001d45c8f28ecf7322afc07537307231fab71736f3852c80a
+  metadata.gz: 8a7d3b6f1f05402242f92004139803d1246be8407c3e71a8246331e8cb2d49ed811f0e54005843549490ae385e2bec32be681392241c2a866d13d502339e2e51
+  data.tar.gz: 3e2a653d2de7a0d4e64424f3a40ada97e5922968e7862701241efb41a2e3a679e715feb1899ac0e7ace5144bbb4161269b52f2bf5f66e53eef0e9d854397bcf0

data/lib/bitfab/client.rb

@@ -120,6 +120,13 @@ module Bitfab
     #   Passed alongside dataset_id or dataset_ids it pins which members of that
     #   selection replay, and the server rejects any ID none of those datasets
     #   contains
+    # @param name [String, nil] What this run is testing, in a few words, such
+    #   as 'baseline' or 'shorter system prompt'. Bitfab records the commit,
+    #   branch, tree state, datasets, and who ran it with every experiment, so
+    #   do not repeat them here.
+    # @param notes [String, nil] Run conditions Bitfab cannot see on its own,
+    #   such as an environment override or a forced feature flag. Kept on the
+    #   experiment next to its name (max 10,000 characters).
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -190,7 +197,7 @@ module Bitfab
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
-      name: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
+      name: nil, notes: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       Replay.run(
@@ -201,6 +208,7 @@ module Bitfab
         limit:,
         trace_ids:,
         name:,
+        notes:,
         concurrency:,
         max_concurrency:,
         attempts:,

data/lib/bitfab/http_client.rb

@@ -261,7 +261,7 @@ module Bitfab
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
-      code_change_files: nil, experiment_group_id: nil, name: nil, include_db_branch_lease: false, dataset_ids: nil,
+      code_change_files: nil, experiment_group_id: nil, name: nil, notes: nil, include_db_branch_lease: false, dataset_ids: nil,
       grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, judge_assertions: false, include_original_metadata: false, experimental_selective_replay: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
@@ -271,6 +271,7 @@ module Bitfab
       payload["limit"] = limit unless limit.nil?
       payload["traceIds"] = trace_ids if trace_ids
       payload["name"] = name unless name.nil?
+      payload["notes"] = notes unless notes.nil?
       payload["codeChangeDescription"] = code_change_description unless code_change_description.nil?
       payload["codeChangeFiles"] = normalize_code_change_files(code_change_files) unless code_change_files.nil?
       payload["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?

data/lib/bitfab/replay.rb

@@ -205,7 +205,13 @@ module Bitfab
     #   already determines how many traces replay. Supplying trace_ids also
     #   emits a warning.
     # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100)
-    # @param name [String, nil] optional display name for the resulting experiment
+    # @param name [String, nil] What this run is testing, in a few words, such
+    #   as 'baseline' or 'shorter system prompt'. Bitfab records the commit,
+    #   branch, tree state, datasets, and who ran it with every experiment, so
+    #   do not repeat them here.
+    # @param notes [String, nil] Run conditions Bitfab cannot see on its own,
+    #   such as an environment override or a forced feature flag. Kept on the
+    #   experiment next to its name (max 10,000 characters).
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -274,7 +280,7 @@ module Bitfab
     #   crashes the run.
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
-    def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil,
+    def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil,
       concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil,
       dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
@@ -371,6 +377,7 @@ module Bitfab
         effective_limit,
         trace_ids:,
         name:,
+        notes:,
         code_change_description:,
         code_change_files:,
         experiment_group_id:,

data/lib/bitfab/replay_registry.rb

@@ -19,7 +19,7 @@ module Bitfab
   # Project-owned registry of production replay roots.
   class ReplayRegistry
     OPTION_NAMES = %i[
-      limit trace_ids name concurrency max_concurrency code_change_description
+      limit trace_ids name notes concurrency max_concurrency code_change_description
       code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
       mock_override adapt_inputs db_branch attempts only_with_assertions judge_assertions dry_run on_item_finish
     ].freeze
@@ -186,7 +186,7 @@ module Bitfab
         options[:limit] = bound || 10
       end
 
-      %i[name max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions dry_run].each do |key|
+      %i[name notes max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -284,7 +284,13 @@ module Bitfab
         options.on("--seed PATH", "--cases PATH") { |value| args[:seed] = value }
         options.on("--run") { args[:run] = true }
         options.on("--from-trace IDS") { |value| args[:from_trace] = comma_separated("--from-trace", value) }
-        options.on("--name NAME") { |value| args[:name] = value }
+        options.on("--name NAME",
+          "What this run is testing, in a few words, such as 'baseline' or 'shorter system prompt'. " \
+            "Bitfab records the commit, branch, tree state, datasets, and who ran it with every experiment, " \
+            "so do not repeat them here.") { |value| args[:name] = value }
+        options.on("--notes TEXT",
+          "Run conditions Bitfab cannot see on its own, such as an environment override or a forced feature flag. " \
+            "Kept on the experiment next to its name.") { |value| args[:notes] = value }
         options.on("--concurrency N", Integer) do |value|
           args[:max_concurrency] = positive_integer("--concurrency", value)
         end

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.61.1"
+  VERSION = "0.61.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.61.1
+  version: 0.61.2
 platform: ruby
 authors:
 - Harvest Team