bitfab 0.23.2 → 0.23.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a66c02d783c0a8acbe6e22b82394fcf7805df0bbec4ebf35eff5246a9c086cc
-  data.tar.gz: cd9f879e95a0b77ede2176db482ab925bcafb3f28d0a31388af3db0e64e99510
+  metadata.gz: 4c6103564370cdb20d6ed221a57c1cd29f8e410cd23ad3218b4eb0bda3ff3bd1
+  data.tar.gz: aded540c60d09f0838e529fc7616659fc6d64ad5dbcaf12baf396b164d741f7d
 SHA512:
-  metadata.gz: a8cda8f967f41bdd4d11367a961ea6c2bf567d9fad4b5f963d6894f0274e30f358c324b00eff8a34efee77e07550a1330d5f632095c29ea378d6ce4b95970623
-  data.tar.gz: d82f7d930e2b99b3201beadfa01c9919584b96439096d3d2825be48ef5685b25286f9ca97e18ddfd373c039e85134fc49a0e4ed01271334ed0162a91effc1f8a
+  metadata.gz: 4ee3df19ee7101db92bea5305f61c67cc92d1bb5cdf4999cde05f31ee15b7e5090d5f474a35abb5814ca799a8849cea1cc3eda55064ec72bd2ac601c251622a0
+  data.tar.gz: 6729607659c50822d1799e86dfc3219cf7cfa5d47a8d9e82d5e8904918f3378a4d98b8f6a0bf9435ed8f34b147d96b061730424cbd2ce0a132d82e3a10c3f415

data/README.md

@@ -291,17 +291,18 @@ fn.wrap(OpenAI::Client, :embeddings, name: "Embed", type: "llm")
 
 ### Replay with Mock Strategies
 
-Replay reruns historical traces through your code so you can compare outputs after an iteration. By default every child span runs real code - fine for offline traces, but expensive when children make paid LLM/API calls. Three strategies control whether child spans return their historical output instead of executing:
+Replay reruns historical traces through your code so you can compare outputs after an iteration. By default, spans tagged with `mock_on_replay: true` return their historical output and other child spans run real code. Three strategies control whether child spans return their historical output instead of executing:
 
 ```ruby
-# "none" (default): everything runs real code
+# "marked" (default): only spans tagged with `mock_on_replay: true` return historical output
+client.replay(pipeline, :process, trace_function_key: "my-fn")
+client.replay(pipeline, :process, trace_function_key: "my-fn", mock: "marked")
+
+# "none": everything runs real code
 client.replay(pipeline, :process, trace_function_key: "my-fn", mock: "none")
 
 # "all": every child span returns its historical output
 client.replay(pipeline, :process, trace_function_key: "my-fn", mock: "all")
-
-# "marked": only spans tagged with `mock_on_replay: true` return historical output
-client.replay(pipeline, :process, trace_function_key: "my-fn", mock: "marked")
 ```
 
 Tag the spans you want mocked at definition time:
@@ -330,7 +331,7 @@ class Pipeline
 end
 ```
 
-Use `mock: "marked"` when you want to iterate on `process`'s logic without paying for the LLM call each run. Use `mock: "all"` for the cheapest possible replay (every child span returns its recorded output).
+Use the default `mock: "marked"` behavior when you want to iterate on `process`'s logic without paying for the LLM call each run. Use `mock: "all"` for the cheapest possible replay (every child span returns its recorded output).
 
 ### Error Handling
 

data/lib/bitfab/client.rb

@@ -72,9 +72,9 @@ module Bitfab
     #   replay runs into a single experiment batch
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
     #   runs against, stored on the resulting experiment for durable attribution
-    # @param mock [String] mock strategy for child spans: "none" (default),
-    #   "all", or "marked". "all" mocks every child span; "marked" only mocks
-    #   spans declared with mock_on_replay: true.
+    # @param mock [String] mock strategy for child spans: "marked" (default),
+    #   "none", or "all". "marked" only mocks spans declared with
+    #   mock_on_replay: true; "all" mocks every child span.
     # @param adapt_inputs [#call, nil] optional hook to reshape recorded inputs
     #   onto the method's current signature when its shape changed after the
     #   traces were captured. Receives (args, kwargs, ctx) where ctx is
@@ -87,7 +87,7 @@ module Bitfab
     #   A raising callback never crashes the run.
     # @return [Hash] with :items, :test_run_id, :test_run_url
     def replay(receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, max_concurrency: 10,
-      name: nil, code_change_description: nil, code_change_files: nil, experiment_group_id: nil, dataset_id: nil, mock: "none",
+      name: nil, code_change_description: nil, code_change_files: nil, experiment_group_id: nil, dataset_id: nil, mock: "marked",
       adapt_inputs: nil, environment: nil, on_progress: nil)
       Replay.run(
         self, receiver, method_name,
@@ -673,7 +673,7 @@ module Bitfab
     # @param method_name [Symbol] the method to wrap
     # @param name [String, nil] explicit span name (defaults to method name)
     # @param type [String] span type
-    # @param mock_on_replay [Boolean] mark this span for the "marked" mock strategy
+    # @param mock_on_replay [Boolean] mark this span for the default "marked" mock strategy
     def wrap(klass, method_name, name: nil, type: "custom", mock_on_replay: false)
       Bitfab::Traceable.wrap(
         klass, method_name,

data/lib/bitfab/replay.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "fileutils"
+require "json"
+
 require_relative "constants"
 require_relative "serialize"
 require_relative "traceable"
@@ -7,10 +10,10 @@ require_relative "traceable"
 module Bitfab
   # Replay mock strategies. Mirrors the Python and TypeScript SDKs.
   #
-  # - "none"   : every child span runs real code (default)
-  # - "all"    : every child span returns its historical output
   # - "marked" : only spans declared with mock_on_replay: true return historical
-  #              output; everything else runs real code
+  #              output; everything else runs real code (default)
+  # - "none"   : every child span runs real code
+  # - "all"    : every child span returns its historical output
   MOCK_STRATEGIES = %w[none all marked].freeze
 
   # Thread-local replay context management.
@@ -40,7 +43,7 @@ module Bitfab
       ctx[:pending_persistence] = pending_persistence if pending_persistence
       if mock_tree
         ctx[:mock_tree] = mock_tree
-        ctx[:mock_strategy] = mock_strategy || "none"
+        ctx[:mock_strategy] = mock_strategy || "marked"
         ctx[:call_counters] = {}
       end
       # The per-trace DB branch (resolved server-side) and the Bitfab trace ID
@@ -92,9 +95,9 @@ module Bitfab
     #   replay runs into a single experiment batch
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
     #   runs against, stored on the resulting experiment for durable attribution
-    # @param mock [String] mock strategy for child spans: "none" (default),
-    #   "all", or "marked". "all" mocks every child span; "marked" only mocks
-    #   spans declared with mock_on_replay: true.
+    # @param mock [String] mock strategy for child spans: "marked" (default),
+    #   "none", or "all". "marked" only mocks spans declared with
+    #   mock_on_replay: true; "all" mocks every child span.
     # @param adapt_inputs [#call, nil] optional hook to reshape recorded inputs
     #   onto the method's current signature when its shape changed after the
     #   traces were captured. Receives (args, kwargs, ctx) where ctx is
@@ -112,7 +115,7 @@ module Bitfab
     # @return [Hash] with :items, :test_run_id, :test_run_url
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil,
       max_concurrency: 10, code_change_description: nil, code_change_files: nil, experiment_group_id: nil,
-      dataset_id: nil, mock: "none",
+      dataset_id: nil, mock: "marked",
       adapt_inputs: nil, environment: nil, on_progress: nil)
       unless MOCK_STRATEGIES.include?(mock.to_s)
         raise ArgumentError, "Invalid mock strategy '#{mock}'. Must be one of: #{MOCK_STRATEGIES.join(", ")}"
@@ -239,11 +242,26 @@ module Bitfab
         end
       end
 
-      {
+      replay_result = {
         items: result_items,
         test_run_id:,
         test_run_url: "#{client.service_url}#{test_run_url}"
       }
+      write_replay_result_file(replay_result)
+      replay_result
+    end
+
+    def write_replay_result_file(result)
+      result_path = ENV["BITFAB_REPLAY_RESULT_PATH"]
+      return if result_path.nil? || result_path.empty?
+
+      begin
+        dir = File.dirname(result_path)
+        FileUtils.mkdir_p(dir) unless dir.nil? || dir.empty? || dir == "."
+        File.write(result_path, "#{JSON.pretty_generate(result)}\n")
+      rescue => e
+        warn "Bitfab: failed to write replay result to BITFAB_REPLAY_RESULT_PATH (#{result_path}): #{e.message}"
+      end
     end
 
     # Process all replay items, optionally in parallel using threads.
@@ -349,8 +367,14 @@ module Bitfab
 
       mock_tree = nil
       if mock_strategy == "all" || mock_strategy == "marked"
-        tree = http_client.get_span_tree(server_item["externalSpanId"])
-        mock_tree = build_mock_tree(tree["root"] || {})
+        begin
+          tree = http_client.get_span_tree(server_item["externalSpanId"])
+          mock_tree = build_mock_tree(tree["root"] || {})
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          raise if e.is_a?(SystemExit) || e.is_a?(SignalException)
+          raise if mock_strategy == "all"
+          mock_tree = nil
+        end
       end
 
       adapt_ctx = {trace_id: server_item["traceId"], source_span_id: server_item["externalSpanId"]}
@@ -481,7 +505,7 @@ module Bitfab
 
     # Execute a single replay item: deserialize inputs, call method with replay context.
     def execute_item(item, receiver, method_name, test_run_id, input_source_span_id = nil, metrics = {},
-      input_source_trace_id: nil, mock_strategy: "none", mock_tree: nil, adapt_inputs: nil, adapt_ctx: nil,
+      input_source_trace_id: nil, mock_strategy: "marked", mock_tree: nil, adapt_inputs: nil, adapt_ctx: nil,
       db_branch_lease: nil, source_bitfab_trace_id: nil, db_snapshot_ref: nil)
       args, kwargs = Serialize.deserialize_inputs(item)
 

data/lib/bitfab/traceable.rb

@@ -38,8 +38,8 @@ module Bitfab
     # @param trace_function_key [String] the trace function key
     # @param name [String, nil] explicit span name (defaults to method name)
     # @param type [String] span type: llm, agent, function, guardrail, handoff, custom
-    # @param mock_on_replay [Boolean] mark this span for the "marked" mock strategy.
-    #   When true, `client.replay(... mock: "marked")` returns this span's
+    # @param mock_on_replay [Boolean] mark this span for the default "marked" mock strategy.
+    #   When true, `client.replay(...)` returns this span's
     #   historical output instead of executing the wrapped method.
     # @param client [Bitfab::Client, nil] route spans through this specific client
     #   instead of the global `Bitfab.client`. When nil (default), the wrapper
@@ -132,8 +132,8 @@ module Bitfab
       # @param trace_function_key [String, nil] trace function key (overrides class-level bitfab_function)
       # @param name [String, nil] explicit span name (defaults to method name)
       # @param type [String] span type: llm, agent, function, guardrail, handoff, custom
-      # @param mock_on_replay [Boolean] mark this span for the "marked" mock strategy.
-      #   When true, `client.replay(... mock: "marked")` returns this span's
+      # @param mock_on_replay [Boolean] mark this span for the default "marked" mock strategy.
+      #   When true, `client.replay(...)` returns this span's
       #   historical output instead of executing the wrapped method.
       def bitfab_span(method_name, trace_function_key: nil, name: nil, type: "custom", mock_on_replay: false)
         trace_function_key ||= @bitfab_function_key

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.23.2"
+  VERSION = "0.23.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.23.2
+  version: 0.23.4
 platform: ruby
 authors:
 - Harvest Team