bitfab 0.20.1 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd571926bd40dc39f337e87f9977bfb0fe3219c21532dc1cf7be382c175a4559
-  data.tar.gz: 17ff724ae1a42f97fa6eae82c97404de8dc9cef738a18674d76da48ba9ea762d
+  metadata.gz: 7536f774b491c131b04a135a41c5f753c7c507615e54aa675c314cff785b08df
+  data.tar.gz: 47e7804161bc40563f5aca99147ef43d2af1af6d73d22f6ad3ae3402ea4b51f4
 SHA512:
-  metadata.gz: 012d3433d2d59200c812557a95b4f2b7cb8d93d7885d9495f59b7e21d89a892ffcc3c0bb83200f4c9f324320d5a1153f4c59bd2facac2dc1924174a7f184d81f
-  data.tar.gz: 69aaffc897238560ebecf6fedbd81509eda58fd082126107585a913cc74a025456a32111356902251a0c5b3439ff2b84ef7ea5aac4325a10e531c93a5d460334
+  metadata.gz: 4aed7d077e6aa0ca89e8130b526578276591e60b41ae10670dc14db4eaca6221c30481dcad84acb8d7dfd7e549b2b92aa2c5a750efa36c4df85224b01360fe19
+  data.tar.gz: d73a3933eceba9c705fdab9f8b38f9f46766e026e59cdf61ee0ca2ea49a01ec4e5029fcbc87161a0a16602a2365ef3eeddba1acff27e659bf8fcf69b08d85138

data/lib/bitfab/client.rb

@@ -59,14 +59,21 @@ module Bitfab
     #   onto the method's current signature when its shape changed after the
     #   traces were captured. Receives (args, kwargs, ctx) where ctx is
     #   { trace_id:, source_span_id: }, and returns [new_args, new_kwargs].
+    # @param on_progress [#call, nil] optional callback invoked once per item as
+    #   it finishes, with a running-totals hash { completed:, total:, succeeded:,
+    #   errored: }. Use it to render replay progress (e.g. a terminal progress
+    #   bar). Replay does not know pass/fail yet, so the totals only distinguish
+    #   items whose method ran (:succeeded) from items that raised (:errored).
+    #   A raising callback never crashes the run.
     # @return [Hash] with :items, :test_run_id, :test_run_url
     def replay(receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, max_concurrency: 10,
       code_change_description: nil, code_change_files: nil, experiment_group_id: nil, dataset_id: nil, mock: "none",
-      adapt_inputs: nil, environment: nil)
+      adapt_inputs: nil, environment: nil, on_progress: nil)
       Replay.run(
         self, receiver, method_name,
         trace_function_key:, limit:, trace_ids:, max_concurrency:,
-        code_change_description:, code_change_files:, experiment_group_id:, dataset_id:, mock:, adapt_inputs:, environment:
+        code_change_description:, code_change_files:, experiment_group_id:, dataset_id:, mock:, adapt_inputs:, environment:,
+        on_progress:
       )
     end
 

data/lib/bitfab/replay.rb

@@ -100,10 +100,14 @@ module Bitfab
     #   { trace_id:, source_span_id: }, and returns [new_args, new_kwargs]. Runs
     #   per item inside the same rescue as the method, so a raising adapter sets
     #   that item's :error rather than crashing the run.
+    # @param on_progress [#call, nil] optional callback invoked once per item as
+    #   it finishes, with a running-totals hash { completed:, total:, succeeded:,
+    #   errored: }. Use it to render replay progress (e.g. a terminal progress
+    #   bar). A raising callback never crashes the run.
     # @return [Hash] with :items, :test_run_id, :test_run_url
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, max_concurrency: 10,
       code_change_description: nil, code_change_files: nil, experiment_group_id: nil, dataset_id: nil, mock: "none",
-      adapt_inputs: nil, environment: nil)
+      adapt_inputs: nil, environment: nil, on_progress: nil)
       unless MOCK_STRATEGIES.include?(mock.to_s)
         raise ArgumentError, "Invalid mock strategy '#{mock}'. Must be one of: #{MOCK_STRATEGIES.join(", ")}"
       end
@@ -157,7 +161,7 @@ module Bitfab
 
       result_items = if server_items.any?
         process_items(http_client, server_items, receiver, method_name, test_run_id, max_concurrency, mock.to_s,
-          adapt_inputs, include_db_branch_lease)
+          adapt_inputs, include_db_branch_lease, on_progress:)
       else
         []
       end
@@ -170,6 +174,10 @@ module Bitfab
       # loudly.
       complete_response = http_client.complete_replay(test_run_id)
       trace_id_map = complete_response&.dig("traceIds")
+      # Per-replay-trace token usage keyed by server trace id: the REPLAYED
+      # run's tokens (span-aggregated server-side), used below to fill each
+      # item's :tokens.
+      replay_tokens = complete_response&.dig("tokens") || {}
 
       if trace_id_map.nil?
         # Older servers don't return the mapping. Preserve the legacy
@@ -202,6 +210,9 @@ module Bitfab
             completed_count += 1
             missing << item[:trace_id] if mapped.nil?
           end
+          # Pull this item's replayed-run tokens by its server trace id, before
+          # :trace_id is overwritten with that id below.
+          item[:tokens] = normalize_tokens(replay_tokens[mapped]) if mapped
           item[:trace_id] = mapped
         end
         if missing.any?
@@ -230,13 +241,39 @@ module Bitfab
 
     # Process all replay items, optionally in parallel using threads.
     def process_items(http_client, server_items, receiver, method_name, test_run_id, max_concurrency, mock_strategy,
-      adapt_inputs = nil, include_db_branch_lease = false)
+      adapt_inputs = nil, include_db_branch_lease = false, on_progress: nil)
       concurrency = max_concurrency || server_items.length
 
+      # Reports running totals once per item as it settles. In the parallel
+      # path it runs from worker threads, so the mutex both makes the counter
+      # updates safe and serializes the user's callback (never called
+      # concurrently). A raising callback is swallowed: progress UI must never
+      # crash the run.
+      total = server_items.length
+      progress_mutex = Mutex.new
+      completed = 0
+      succeeded = 0
+      errored = 0
+      report = lambda do |result|
+        return unless on_progress
+
+        progress_mutex.synchronize do
+          completed += 1
+          result[:error].nil? ? (succeeded += 1) : (errored += 1)
+          begin
+            on_progress.call({completed:, total:, succeeded:, errored:})
+          rescue => e
+            warn "Bitfab: replay on_progress callback raised: #{e.message}"
+          end
+        end
+      end
+
       if concurrency <= 1
         server_items.map do |item|
-          process_single_item(http_client, item, receiver, method_name, test_run_id, mock_strategy, adapt_inputs,
-            include_db_branch_lease)
+          result = process_single_item(http_client, item, receiver, method_name, test_run_id, mock_strategy,
+            adapt_inputs, include_db_branch_lease)
+          report.call(result)
+          result
         end
       else
         results_mutex = Mutex.new
@@ -253,6 +290,7 @@ module Bitfab
               result = process_single_item(http_client, item, receiver, method_name, test_run_id, mock_strategy,
                 adapt_inputs, include_db_branch_lease)
               results_mutex.synchronize { results[idx] = result }
+              report.call(result)
             end
           end
         end
@@ -386,27 +424,34 @@ module Bitfab
       }
     end
 
-    # Pull durationMs / tokens / model from the start-replay server item.
-    # Normalizes to symbol-keyed tokens hash and nil-safe defaults so older
-    # servers without these fields still produce a consistent shape.
+    # Pull durationMs / model from the start-replay server item. Nil-safe
+    # defaults so older servers without these fields still produce a consistent
+    # shape. Tokens are intentionally NOT read from the start item (it carries
+    # the ORIGINAL trace's tokens); the replayed run's tokens are filled in by
+    # run() from the complete-replay response once spans are aggregated
+    # server-side, and stay nil here and on older servers.
     def extract_server_item_metrics(server_item)
-      raw_tokens = server_item["tokens"]
-      tokens = if raw_tokens.is_a?(Hash)
-        {
-          input: raw_tokens["input"],
-          output: raw_tokens["output"],
-          cached: raw_tokens["cached"],
-          total: raw_tokens["total"]
-        }
-      end
-
       {
         duration_ms: server_item["durationMs"],
-        tokens:,
+        tokens: nil,
         model: server_item["model"]
       }
     end
 
+    # Normalize a complete-replay tokens hash (string-keyed JSON) into the
+    # symbol-keyed shape the replay item exposes. Nil when the server reported
+    # no token data for this trace.
+    def normalize_tokens(raw_tokens)
+      return nil unless raw_tokens.is_a?(Hash)
+
+      {
+        input: raw_tokens["input"],
+        output: raw_tokens["output"],
+        cached: raw_tokens["cached"],
+        total: raw_tokens["total"]
+      }
+    end
+
     # Execute a single replay item: deserialize inputs, call method with replay context.
     def execute_item(item, receiver, method_name, test_run_id, input_source_span_id = nil, metrics = {},
       input_source_trace_id: nil, mock_strategy: "none", mock_tree: nil, adapt_inputs: nil, adapt_ctx: nil,

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.20.1"
+  VERSION = "0.21.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.20.1
+  version: 0.21.0
 platform: ruby
 authors:
 - Harvest Team