bitfab 0.61.1 → 0.62.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49768c67671b66ea8066cfa078dbba13ac3548b83b9a0abaa628ab864a61b99f
-  data.tar.gz: 76e711a41258fa421a69cd6a0475a793698f3bdd001f60d0104ec47a5096e8da
+  metadata.gz: 4a04ce4c6087bca4125aeb03947b00facce1f8b1f8da0ab35973c110ef1f40ae
+  data.tar.gz: c66fec4e4390bbdfa35f1a12a010bc070400309b874c2531f48930729754712f
 SHA512:
-  metadata.gz: bdef7ec071b9740bd8183056b68b84e9ab2c8004f0918678c604e9e35d6426144891bd5df0e476a59f8198e1f17459a4e0e48c08e3eab78061b119146e38b3f1
-  data.tar.gz: d441d62917aaa84b3dfbc88f6544e1630645fc5a595a7a3cf9b3e111bdbf62b43059b971ecb9f51001d45c8f28ecf7322afc07537307231fab71736f3852c80a
+  metadata.gz: 5c0d3bf2b9f187c8e0ec6e9ce99cf28b81cf992377e4621e9636d77cd79a027a15f857e3572ba0bf8ebf5a4912cbd4fd8da8cd652ab1d2ad5f77e43073ba8c6e
+  data.tar.gz: 64eef7e076828f8ab95c4b53f0f2c95061b142819d8e705ac59417a6b2df95da6ed6f4e5100627881e8de8582f769a6cf557e99b9a3ad3185eeb2e1e9f247ece

data/lib/bitfab/client.rb

@@ -120,6 +120,13 @@ module Bitfab
     #   Passed alongside dataset_id or dataset_ids it pins which members of that
     #   selection replay, and the server rejects any ID none of those datasets
     #   contains
+    # @param name [String, nil] What this run is testing, in a few words, such
+    #   as 'baseline' or 'shorter system prompt'. Bitfab records the commit,
+    #   branch, tree state, datasets, and who ran it with every experiment, so
+    #   do not repeat them here.
+    # @param notes [String, nil] Run conditions Bitfab cannot see on its own,
+    #   such as an environment override or a forced feature flag. Kept on the
+    #   experiment next to its name (max 10,000 characters).
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -190,7 +197,7 @@ module Bitfab
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
-      name: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
+      name: nil, notes: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       Replay.run(
@@ -201,6 +208,7 @@ module Bitfab
         limit:,
         trace_ids:,
         name:,
+        notes:,
         concurrency:,
         max_concurrency:,
         attempts:,

data/lib/bitfab/http_client.rb

@@ -261,7 +261,7 @@ module Bitfab
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
-      code_change_files: nil, experiment_group_id: nil, name: nil, include_db_branch_lease: false, dataset_ids: nil,
+      code_change_files: nil, experiment_group_id: nil, name: nil, notes: nil, include_db_branch_lease: false, dataset_ids: nil,
       grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, judge_assertions: false, include_original_metadata: false, experimental_selective_replay: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
@@ -271,6 +271,7 @@ module Bitfab
       payload["limit"] = limit unless limit.nil?
       payload["traceIds"] = trace_ids if trace_ids
       payload["name"] = name unless name.nil?
+      payload["notes"] = notes unless notes.nil?
       payload["codeChangeDescription"] = code_change_description unless code_change_description.nil?
       payload["codeChangeFiles"] = normalize_code_change_files(code_change_files) unless code_change_files.nil?
       payload["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?

data/lib/bitfab/replay.rb

@@ -205,7 +205,13 @@ module Bitfab
     #   already determines how many traces replay. Supplying trace_ids also
     #   emits a warning.
     # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100)
-    # @param name [String, nil] optional display name for the resulting experiment
+    # @param name [String, nil] What this run is testing, in a few words, such
+    #   as 'baseline' or 'shorter system prompt'. Bitfab records the commit,
+    #   branch, tree state, datasets, and who ran it with every experiment, so
+    #   do not repeat them here.
+    # @param notes [String, nil] Run conditions Bitfab cannot see on its own,
+    #   such as an environment override or a forced feature flag. Kept on the
+    #   experiment next to its name (max 10,000 characters).
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -274,7 +280,7 @@ module Bitfab
     #   crashes the run.
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
-    def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil,
+    def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil,
       concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil,
       dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
@@ -371,6 +377,7 @@ module Bitfab
         effective_limit,
         trace_ids:,
         name:,
+        notes:,
         code_change_description:,
         code_change_files:,
         experiment_group_id:,
@@ -458,8 +465,6 @@ module Bitfab
       # run's tokens (span-aggregated server-side), used below to fill each
       # item's :tokens.
       replay_tokens = complete_response&.dig("tokens") || {}
-      replay_outlines = complete_response&.dig("traceOutlines") || {}
-      original_outlines = complete_response&.dig("originalTraceOutlines") || {}
 
       # trace_id_map maps each item's client-side correlation id (:_sdk_trace_id,
       # which tagged that item's spans during the run) to the server's trace row
@@ -483,11 +488,7 @@ module Bitfab
             completed_count += 1
             missing << local_id if mapped.nil?
           end
-          if mapped
-            item[:tokens] = normalize_tokens(replay_tokens[mapped])
-            item[:trace_outline] = replay_outlines[mapped]
-          end
-          item[:original_trace_outline] = original_outlines[item[:original_trace_id]] if item[:original_trace_id]
+          item[:tokens] = normalize_tokens(replay_tokens[mapped]) if mapped
         end
         # ALL completed items missing: systemic (the replayed method is not
         # traced, or uploads are wholesale broken). Raise; the run's traces
@@ -880,9 +881,7 @@ module Bitfab
                 tokens: result[:tokens],
                 model: result[:model],
                 db_snapshot_ref: result[:db_snapshot_ref],
-                db_branch_timings: result[:db_branch_timings],
-                trace_outline: result[:trace_outline],
-                original_trace_outline: result[:original_trace_outline]
+                db_branch_timings: result[:db_branch_timings]
               }
             })
           rescue => e
@@ -1084,9 +1083,7 @@ module Bitfab
         source_trace_id: original_trace_id,
         source_span_id: original_span_id,
         db_snapshot_ref:,
-        db_branch_timings:,
-        trace_outline: nil,
-        original_trace_outline: nil
+        db_branch_timings:
       }
     ensure
       release_db_branch_lease(http_client, lease) if lease
@@ -1396,9 +1393,7 @@ module Bitfab
         source_trace_id: source_bitfab_trace_id,
         source_span_id: input_source_span_id,
         db_snapshot_ref:,
-        db_branch_timings:,
-        trace_outline: nil,
-        original_trace_outline: nil
+        db_branch_timings:
       }
     end
   end

data/lib/bitfab/replay_registry.rb

@@ -19,7 +19,7 @@ module Bitfab
   # Project-owned registry of production replay roots.
   class ReplayRegistry
     OPTION_NAMES = %i[
-      limit trace_ids name concurrency max_concurrency code_change_description
+      limit trace_ids name notes concurrency max_concurrency code_change_description
       code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
       mock_override adapt_inputs db_branch attempts only_with_assertions judge_assertions dry_run on_item_finish
     ].freeze
@@ -186,7 +186,7 @@ module Bitfab
         options[:limit] = bound || 10
       end
 
-      %i[name max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions dry_run].each do |key|
+      %i[name notes max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -284,7 +284,13 @@ module Bitfab
         options.on("--seed PATH", "--cases PATH") { |value| args[:seed] = value }
         options.on("--run") { args[:run] = true }
         options.on("--from-trace IDS") { |value| args[:from_trace] = comma_separated("--from-trace", value) }
-        options.on("--name NAME") { |value| args[:name] = value }
+        options.on("--name NAME",
+          "What this run is testing, in a few words, such as 'baseline' or 'shorter system prompt'. " \
+            "Bitfab records the commit, branch, tree state, datasets, and who ran it with every experiment, " \
+            "so do not repeat them here.") { |value| args[:name] = value }
+        options.on("--notes TEXT",
+          "Run conditions Bitfab cannot see on its own, such as an environment override or a forced feature flag. " \
+            "Kept on the experiment next to its name.") { |value| args[:notes] = value }
         options.on("--concurrency N", Integer) do |value|
           args[:max_concurrency] = positive_integer("--concurrency", value)
         end

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.61.1"
+  VERSION = "0.62.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.61.1
+  version: 0.62.0
 platform: ruby
 authors:
 - Harvest Team