bitfab 0.61.0 → 0.61.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a1a8eab8b1b1b24bdb550f90e3d7cdfde97644a6fcd4301855a12664ac34870
-  data.tar.gz: b1acc5424fa7900996bb20d403f1374d77efca40a9267dc9d986831992fe8439
+  metadata.gz: c3ee5133d6797e0d931936274c5d05bea38532e5879f8a55567a5417cabdd04d
+  data.tar.gz: 57666b69b7277a81a3715a3f386d8660d47d6568ef027adc7a4a10fafc02f6c2
 SHA512:
-  metadata.gz: e0acbbd123458aff8ac1385c256da5a29bafef8e48a948a5e3de9916e97566d700ffa693695d572cb232e979d5b24b3f8664bd02fa1a3e1659ec01f2ddda80a4
-  data.tar.gz: 8714de8dd405dcf831e1f229cb12e9262d21092eac2b9c320804b752f80348e4ba38a613512856ad1b9aa79db8f52d3368eab3777e425b678a3cc05e9c6d54ee
+  metadata.gz: 8a7d3b6f1f05402242f92004139803d1246be8407c3e71a8246331e8cb2d49ed811f0e54005843549490ae385e2bec32be681392241c2a866d13d502339e2e51
+  data.tar.gz: 3e2a653d2de7a0d4e64424f3a40ada97e5922968e7862701241efb41a2e3a679e715feb1899ac0e7ace5144bbb4161269b52f2bf5f66e53eef0e9d854397bcf0

data/lib/bitfab/client.rb

@@ -120,6 +120,13 @@ module Bitfab
     #   Passed alongside dataset_id or dataset_ids it pins which members of that
     #   selection replay, and the server rejects any ID none of those datasets
     #   contains
+    # @param name [String, nil] What this run is testing, in a few words, such
+    #   as 'baseline' or 'shorter system prompt'. Bitfab records the commit,
+    #   branch, tree state, datasets, and who ran it with every experiment, so
+    #   do not repeat them here.
+    # @param notes [String, nil] Run conditions Bitfab cannot see on its own,
+    #   such as an environment override or a forced feature flag. Kept on the
+    #   experiment next to its name (max 10,000 characters).
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -190,9 +197,9 @@ module Bitfab
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
-      name: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
+      name: nil, notes: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
-      on_experiment_start: nil)
+      on_experiment_start: nil, experimental_selective_replay: nil)
       Replay.run(
         self,
         receiver,
@@ -201,6 +208,7 @@ module Bitfab
         limit:,
         trace_ids:,
         name:,
+        notes:,
         concurrency:,
         max_concurrency:,
         attempts:,
@@ -220,7 +228,8 @@ module Bitfab
         on_item_start:,
         on_item_finish:,
         on_progress:,
-        on_experiment_start:
+        on_experiment_start:,
+        experimental_selective_replay:
       )
     end
 
@@ -337,7 +346,7 @@ module Bitfab
     # Configure an existing method only when it is discovered beneath a
     # bitfab_trace root owned by this client.
     def node(klass, method_name, name: nil, type: "custom", capture: true,
-      experiment_id: nil, mock_on_replay: nil, finalize: nil, test_run_id: nil)
+      experiment_id: nil, mock_on_replay: nil, replay_reusable: false, finalize: nil, test_run_id: nil)
       experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
       Bitfab::Traceable.node(
         klass,
@@ -347,6 +356,7 @@ module Bitfab
         capture:,
         experiment_id:,
         mock_on_replay:,
+        replay_reusable:,
         finalize:,
         client: self
       )
@@ -383,12 +393,14 @@ module Bitfab
     # Execute a block inside a span context, sending trace data on completion.
     # Called by Traceable, not intended for direct use.
     def execute_span(trace_function_key:, span_name:, span_type:, function_name:, args:, kwargs:,
-      capture_when: "always", mock_on_replay: false, independent_root: false,
+      capture_when: "always", mock_on_replay: false, replay_reusable: false, independent_root: false,
       explicit_span_receiver: nil, explicit_span_method_name: nil, experiment_id: nil, finalize: nil,
       surface: nil, on_start: nil, on_complete: nil, around_body: nil, span_data: {}, managed_root: false,
-      result_recorded_elsewhere: false, declared_node: false)
+      result_recorded_elsewhere: false, declared_node: false, input_block: false)
       return yield unless should_record?
 
+      selective_input = !input_block && (replay_reusable || mock_on_replay) && SelectiveReplay.inputs(args, kwargs)
+
       @http_client.refresh_simulation_plan
 
       resolved_capture_when = capture_when.to_s
@@ -465,7 +477,20 @@ module Bitfab
         # line up with `build_mock_tree`'s sequential numbering for the same
         # (key, name) pair. Different (key, name) pairs have independent
         # counters: they cannot shift each other.
-        call_index = advance_mock_counter(replay_ctx, trace_function_key, span_name, is_root_span:)
+        selective = replay_ctx&.dig(:selective_replay)
+        if selective
+          mocked, output = selective.enter(trace_function_key:, span_name:, span_id:, parent_span_id:,
+            args:, kwargs:, safety_mock: mock_on_replay, reuse_allowed: replay_reusable, can_reuse_output: finalize.nil? && !input_block)
+          if mocked
+            send_mocked_span(trace_function_key:, trace_id:, span_id:, parent_span_id:, span_name:, span_type:,
+              function_name:, args:, kwargs:, mocked_output: output, started_at:,
+              experiment_id: resolved_experiment_id, input_source_span_id: resolved_input_source_span_id,
+              mock_source: "recorded", instrumentation: (surface == :trace) ? "trace" : "span", declared_node:)
+            on_complete&.call(result: output, error: nil, ended_at: Bitfab.now_iso_timestamp)
+            return output
+          end
+        end
+        call_index = selective ? nil : advance_mock_counter(replay_ctx, trace_function_key, span_name, is_root_span:)
         if call_index
           mock_result = check_mock_replay(
             replay_ctx,
@@ -561,6 +586,14 @@ module Bitfab
 
           on_complete&.call(result: recorded_result, error: recorded_error, ended_at:)
 
+          span_data = span_data.merge("replay_reusable" => replay_reusable)
+          output_json = finalize.nil? && !result.is_a?(Enumerator) && SelectiveReplay.fingerprint(final_result)
+          if selective_input && output_json && recorded_error.nil? && !content_off
+            recorded_args, recorded_kwargs = JSON.parse(selective_input)
+            span_data = span_data.merge("replay_json_safe" => true,
+              "replay_recording" => {"input" => recorded_args, "kwargs" => recorded_kwargs, "output" => JSON.parse(output_json)})
+          end
+
           send_span(
             span_data: content_off ? span_data.merge("content_off_by_simulation_plan" => true) : span_data,
             instrumentation: recorded_instrumentation,
@@ -1246,7 +1279,7 @@ module Bitfab
     #   values warn once and default to "always"
     # @param mock_on_replay [Boolean] mark this span for the default "marked"
     #   mock strategy. A missing selected occurrence fails closed.
-    def wrap(klass, method_name, name: nil, type: "custom", capture_when: "always", mock_on_replay: false, finalize: nil, experiment_id: nil, test_run_id: nil)
+    def wrap(klass, method_name, name: nil, type: "custom", capture_when: "always", mock_on_replay: false, replay_reusable: false, finalize: nil, experiment_id: nil, test_run_id: nil)
       experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
       Bitfab::Traceable.wrap(
         klass,
@@ -1256,6 +1289,7 @@ module Bitfab
         type:,
         capture_when:,
         mock_on_replay:,
+        replay_reusable:,
         finalize:,
         experiment_id:,
         client: @client

data/lib/bitfab/http_client.rb

@@ -261,8 +261,8 @@ module Bitfab
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
-      code_change_files: nil, experiment_group_id: nil, name: nil, include_db_branch_lease: false, dataset_ids: nil,
-      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, judge_assertions: false, include_original_metadata: false)
+      code_change_files: nil, experiment_group_id: nil, name: nil, notes: nil, include_db_branch_lease: false, dataset_ids: nil,
+      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, judge_assertions: false, include_original_metadata: false, experimental_selective_replay: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
       }
@@ -271,6 +271,7 @@ module Bitfab
       payload["limit"] = limit unless limit.nil?
       payload["traceIds"] = trace_ids if trace_ids
       payload["name"] = name unless name.nil?
+      payload["notes"] = notes unless notes.nil?
       payload["codeChangeDescription"] = code_change_description unless code_change_description.nil?
       payload["codeChangeFiles"] = normalize_code_change_files(code_change_files) unless code_change_files.nil?
       payload["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?
@@ -288,6 +289,7 @@ module Bitfab
       payload["onlyWithAssertions"] = true if only_with_assertions
       payload["judgeAssertions"] = true if judge_assertions
       payload["includeOriginalMetadata"] = true if include_original_metadata
+      payload["experimentalSelectiveReplay"] = experimental_selective_replay if experimental_selective_replay
       payload["dbBranchSettings"] = db_branch_settings unless db_branch_settings.nil?
       git_state = GitState.resolved
       payload["git"] = git_state if git_state

data/lib/bitfab/replay.rb

@@ -9,6 +9,7 @@ require_relative "constants"
 require_relative "git_command"
 require_relative "mock_override"
 require_relative "serialize"
+require_relative "selective_replay"
 require_relative "traceable"
 require_relative "transport"
 
@@ -85,10 +86,11 @@ module Bitfab
     # thread. Child threads and processes intentionally do not inherit it.
     def with_context(experiment_id:, input_source_span_id: nil, input_source_trace_id: nil, trace_id: nil,
       mock_tree: nil, mock_strategy: nil, mock_overrides: nil, fetch_span_output: nil,
-      db_branch_lease: nil, db_branch_timings: nil, source_bitfab_trace_id: nil, replay_attempt: nil)
+      db_branch_lease: nil, db_branch_timings: nil, source_bitfab_trace_id: nil, replay_attempt: nil, selective_replay: nil)
       previous = Thread.current.thread_variable_get(REPLAY_CONTEXT_KEY)
       ctx = {
         replay_attempt:,
+        selective_replay:,
         experiment_id:,
         input_source_span_id:,
         input_source_trace_id:,
@@ -203,7 +205,13 @@ module Bitfab
     #   already determines how many traces replay. Supplying trace_ids also
     #   emits a warning.
     # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100)
-    # @param name [String, nil] optional display name for the resulting experiment
+    # @param name [String, nil] What this run is testing, in a few words, such
+    #   as 'baseline' or 'shorter system prompt'. Bitfab records the commit,
+    #   branch, tree state, datasets, and who ran it with every experiment, so
+    #   do not repeat them here.
+    # @param notes [String, nil] Run conditions Bitfab cannot see on its own,
+    #   such as an environment override or a forced feature flag. Kept on the
+    #   experiment next to its name (max 10,000 characters).
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -272,11 +280,11 @@ module Bitfab
     #   crashes the run.
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
-    def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil,
+    def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil,
       concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil,
       dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
-      on_experiment_start: nil)
+      on_experiment_start: nil, experimental_selective_replay: nil)
       concurrency_options = ReplayConcurrency.resolve(concurrency, attempts, max_concurrency)
       attempts = concurrency_options.attempts
       max_concurrency = concurrency_options.max_concurrency
@@ -337,6 +345,12 @@ module Bitfab
       # both take precedence over the base mock strategy.
       registered_overrides = client.instance_variable_get(:@mock_overrides) || []
       resolved_overrides = MockOverride.normalize(mock_override) + registered_overrides
+      if experimental_selective_replay
+        SelectiveReplay.wire_options(experimental_selective_replay)
+        unless mock.to_s == "marked" && resolved_overrides.empty? && concurrency_options.primitive != "process"
+          raise ArgumentError, "experimental_selective_replay requires marked mocks, no overrides, and in-process replay"
+        end
+      end
 
       # limit is meaningless with explicit trace_ids (the ID list determines
       # the count), so it's omitted from the request entirely.
@@ -363,6 +377,7 @@ module Bitfab
         effective_limit,
         trace_ids:,
         name:,
+        notes:,
         code_change_description:,
         code_change_files:,
         experiment_group_id:,
@@ -373,7 +388,8 @@ module Bitfab
         only_with_assertions:,
         judge_assertions:,
         include_original_metadata: !adapt_inputs.nil?,
-        db_branch_settings: resolved_db_branch_settings
+        db_branch_settings: resolved_db_branch_settings,
+        **(experimental_selective_replay ? {experimental_selective_replay: SelectiveReplay.wire_options(experimental_selective_replay)} : {})
       )
       experiment_id = replay_data["experimentId"] || replay_data["testRunId"]
       experiment_url = replay_data["experimentUrl"] || replay_data["testRunUrl"]
@@ -400,7 +416,7 @@ module Bitfab
         process_items(http_client, server_items, receiver, method_name, experiment_id, max_concurrency, mock.to_s,
           adapt_inputs, include_db_branch_lease, resolved_db_branch_settings, on_item_start:,
           on_item_finish: item_finish_callback,
-          mock_overrides: resolved_overrides, dry_run:, process_executor:)
+          mock_overrides: resolved_overrides, dry_run:, process_executor:, experimental_selective_replay:)
       else
         []
       end
@@ -776,7 +792,7 @@ module Bitfab
     # Process all replay items, optionally in parallel using threads.
     def process_items(http_client, server_items, receiver, method_name, experiment_id, max_concurrency, mock_strategy,
       adapt_inputs = nil, include_db_branch_lease = false, db_branch_settings = nil, on_item_start: nil, on_item_finish: nil,
-      mock_overrides: [], dry_run: false, process_executor: nil)
+      mock_overrides: [], dry_run: false, process_executor: nil, experimental_selective_replay: nil)
       concurrency = max_concurrency || server_items.length
 
       # Lifecycle callbacks run from worker threads in the parallel path, so the
@@ -887,7 +903,7 @@ module Bitfab
           process_executor.call(item, experiment_id, mock_strategy, include_db_branch_lease, db_branch_settings, dry_run:)
         else
           process_single_item(http_client, item, receiver, method_name, experiment_id, mock_strategy,
-            adapt_inputs, include_db_branch_lease, db_branch_settings, mock_overrides:, dry_run:)
+            adapt_inputs, include_db_branch_lease, db_branch_settings, mock_overrides:, dry_run:, experimental_selective_replay:)
         end
       end
 
@@ -930,7 +946,7 @@ module Bitfab
     # than propagated, so one bad trace never aborts the whole replay run
     # (mirrors the TypeScript and Python SDKs' per-item rescue).
     def process_single_item(http_client, server_item, receiver, method_name, experiment_id, mock_strategy,
-      adapt_inputs = nil, include_db_branch_lease = false, db_branch_settings = nil, mock_overrides: [], dry_run: false)
+      adapt_inputs = nil, include_db_branch_lease = false, db_branch_settings = nil, mock_overrides: [], dry_run: false, experimental_selective_replay: nil)
       metrics = extract_server_item_metrics(server_item)
       attempt = server_item["attempt"] || 0
       # The ORIGINAL (historical) trace/span this item replays. Canonical
@@ -945,6 +961,7 @@ module Bitfab
       # snapshot ref arrive without a lease, so the app uses its normal DB
       # path. Unsafe calls on that path still require replay mocking.
       lease = include_db_branch_lease ? server_item["dbBranchLease"] : nil
+      selective_replay = experimental_selective_replay && SelectiveReplay.new(server_item["selectiveReplayPlan"], experimental_selective_replay)
       # A resolve that was ATTEMPTED and failed is different: the caller asked
       # for a branch, so running their method against live data would produce a
       # result that looks valid and is not. Fail the item instead, loudly.
@@ -994,7 +1011,7 @@ module Bitfab
       overrides_present = !mock_overrides.nil? && !mock_overrides.empty?
       include_outputs = mock_strategy == "all"
       mock_tree = nil
-      if !dry_run && (mock_strategy == "all" || mock_strategy == "marked" || overrides_present)
+      if !selective_replay && !dry_run && (mock_strategy == "all" || mock_strategy == "marked" || overrides_present)
         begin
           tree = http_client.get_span_tree(
             original_span_id,
@@ -1035,6 +1052,7 @@ module Bitfab
         span["id"],
         metrics,
         input_source_trace_id: span["externalTraceId"],
+        selective_replay:,
         mock_strategy:,
         mock_tree:,
         mock_overrides:,
@@ -1282,7 +1300,7 @@ module Bitfab
     def execute_item(item, receiver, method_name, experiment_id, input_source_span_id = nil, metrics = {},
       input_source_trace_id: nil, mock_strategy: "marked", mock_tree: nil, mock_overrides: nil,
       fetch_span_output: nil, adapt_inputs: nil, adapt_ctx: nil, db_branch_lease: nil, db_branch_timings: nil,
-      source_bitfab_trace_id: nil, db_snapshot_ref: nil, http_client: nil, attempt: 0, dry_run: false)
+      source_bitfab_trace_id: nil, db_snapshot_ref: nil, http_client: nil, attempt: 0, dry_run: false, selective_replay: nil)
       args, kwargs = Serialize.deserialize_inputs(item)
 
       fn_result = nil
@@ -1303,6 +1321,7 @@ module Bitfab
       begin
         ReplayContext.with_context(
           replay_attempt: attempt,
+          selective_replay:,
           experiment_id:,
           input_source_span_id:,
           input_source_trace_id:,
@@ -1338,6 +1357,7 @@ module Bitfab
                 receiver.send(method_name, *args, **kwargs)
               end
               fn_result = fn_result.to_a if fn_result.is_a?(Enumerator)
+              selective_replay&.assert_no_safety_conflict!
               replay_duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - replay_started) * 1000).round
             rescue => e
               # The method ran and raised, so it still has a duration.
@@ -1357,6 +1377,7 @@ module Bitfab
 
       item_error = fn_error || replay_error
       {
+        **(selective_replay ? {selective_replay: selective_replay.report} : {}),
         attempt:,
         ingestion_type: metrics[:ingestion_type],
         input: kwargs.empty? ? args : [*args, kwargs],

data/lib/bitfab/replay_registry.rb

@@ -19,7 +19,7 @@ module Bitfab
   # Project-owned registry of production replay roots.
   class ReplayRegistry
     OPTION_NAMES = %i[
-      limit trace_ids name concurrency max_concurrency code_change_description
+      limit trace_ids name notes concurrency max_concurrency code_change_description
       code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
       mock_override adapt_inputs db_branch attempts only_with_assertions judge_assertions dry_run on_item_finish
     ].freeze
@@ -186,7 +186,7 @@ module Bitfab
         options[:limit] = bound || 10
       end
 
-      %i[name max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions dry_run].each do |key|
+      %i[name notes max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -284,7 +284,13 @@ module Bitfab
         options.on("--seed PATH", "--cases PATH") { |value| args[:seed] = value }
         options.on("--run") { args[:run] = true }
         options.on("--from-trace IDS") { |value| args[:from_trace] = comma_separated("--from-trace", value) }
-        options.on("--name NAME") { |value| args[:name] = value }
+        options.on("--name NAME",
+          "What this run is testing, in a few words, such as 'baseline' or 'shorter system prompt'. " \
+            "Bitfab records the commit, branch, tree state, datasets, and who ran it with every experiment, " \
+            "so do not repeat them here.") { |value| args[:name] = value }
+        options.on("--notes TEXT",
+          "Run conditions Bitfab cannot see on its own, such as an environment override or a forced feature flag. " \
+            "Kept on the experiment next to its name.") { |value| args[:notes] = value }
         options.on("--concurrency N", Integer) do |value|
           args[:max_concurrency] = positive_integer("--concurrency", value)
         end

data/lib/bitfab/selective_replay.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Bitfab
+  class SelectiveReplay
+    def self.wire_options(options)
+      valid = options.is_a?(Hash) && options.keys == [:must_run] && options[:must_run].is_a?(Array) &&
+        options[:must_run].length.between?(1, 100) && options[:must_run].all? do |node|
+          node.is_a?(Hash) && node.keys.sort == [:span_name, :trace_function_key] &&
+            node.values.all? { |value| value.is_a?(String) && value.length.between?(1, 500) }
+        end
+      raise ArgumentError, "experimental_selective_replay accepts only 1–100 must_run identities" unless valid
+
+      {"mustRun" => options[:must_run].map { |node| {"traceFunctionKey" => node[:trace_function_key], "spanName" => node[:span_name]} }}
+    end
+
+    # Strict JSON avoids silently changing Ruby classes, symbol keys, numeric
+    # types, aliases, or insertion order when a recorded output is returned.
+    def self.fingerprint(value)
+      seen = Set.new
+      validate = lambda do |item|
+        case item
+        when NilClass, TrueClass, FalseClass
+          true
+        when String
+          item.instance_of?(String) && seen.add?(item.object_id) && !item.start_with?("<unserializable")
+        when Integer
+          item.abs <= (2**53 - 1)
+        when Float
+          item.finite? && item != item.to_i
+        when Array, Hash
+          next false unless item.instance_of?(Array) || item.instance_of?(Hash)
+          next false unless seen.add?(item.object_id)
+
+          if item.is_a?(Array)
+            item.all? { |child| validate.call(child) }
+          else
+            item.default.nil? && item.default_proc.nil? && !item.compare_by_identity? && item.all? do |key, child|
+              key.instance_of?(String) && !key.match?(/\A(?:0|[1-9][0-9]*)\z/) && validate.call(child)
+            end
+          end
+        else
+          false
+        end
+      end
+      validate.call(value) ? JSON.generate(value) : nil
+    rescue StandardError, SystemStackError
+      nil
+    end
+
+    def self.inputs(args, kwargs)
+      return nil unless kwargs.keys.all? { |key| key.is_a?(Symbol) }
+
+      fingerprint([args, kwargs.transform_keys(&:to_s)])
+    end
+
+    def initialize(plan, options)
+      expected = self.class.wire_options(options)
+      unless plan.is_a?(Hash) && plan["version"] == 4 && plan["nodes"].is_a?(Array) &&
+          plan["nodes"].length.between?(1, 500) && plan["assertions"].is_a?(Array) &&
+          plan["unresolvedMustRun"].is_a?(Array)
+        raise ArgumentError, "Server did not return a compatible selective replay plan"
+      end
+      raise ArgumentError, "Selective replay plan does not match must_run" unless plan["options"] == expected
+
+      @plan = JSON.parse(JSON.generate(plan))
+      @required = expected["mustRun"].map { |node| key(node) }.to_set
+      @nodes = {}
+      @children = Hash.new { |hash, id| hash[id] = [] }
+      @recordings = {}
+      @parents = {}
+      @consumed = Hash.new { |hash, id| hash[id] = Set.new }
+      @required_scopes = Set.new
+      @assertion_scopes = Set.new
+      @assertion_names = Set.new
+      @unresolved_assertions = []
+      @observed = Hash.new(0)
+      @reached = Set.new
+      @decisions = []
+      @mutex = Mutex.new
+      @plan["nodes"].each do |node|
+        valid_node = node.is_a?(Hash) && node["id"].is_a?(String) && !@nodes.key?(node["id"]) &&
+          node["spanName"].is_a?(String) && node["traceFunctionKey"].is_a?(String) &&
+          [true, false].include?(node["mustRun"]) && [true, false].include?(node["reusable"]) &&
+          node.key?("parentId") && (node["parentId"].nil? || node["parentId"].is_a?(String))
+        raise ArgumentError, "Invalid selective replay node" unless valid_node
+        @nodes[node["id"]] = node
+        @children[node["parentId"]] << node unless node["parentId"].nil?
+        recording = node["recording"]
+        if recording.is_a?(Hash) && recording.key?("output")
+          @recordings[node["id"]] = [self.class.fingerprint([recording["input"], recording.fetch("kwargs", {})]), self.class.fingerprint(recording["output"])]
+        end
+      end
+      root = @nodes[@plan["rootId"]]
+      raise ArgumentError, "Selective replay requires a live recorded root" unless root && root["parentId"].nil? && root["mustRun"]
+
+      @full_trace = false
+      ids = Set.new
+      @plan["assertions"].each do |assertion|
+        unless assertion.is_a?(Hash) && assertion["id"].is_a?(String) && !assertion["id"].empty? && ids.add?(assertion["id"]) && assertion.key?("target")
+          raise ArgumentError, "Invalid selective replay assertion"
+        end
+        target = assertion["target"]
+        if target.nil?
+          @full_trace = true
+          next
+        end
+        raise ArgumentError, "Invalid selective replay assertion target" unless target.is_a?(Hash) && %w[output span].include?(target["kind"])
+        next if target["kind"] == "output"
+
+        occurrence = target.fetch("occurrence", "first")
+        valid_target = target["name"].is_a?(String) && !target["name"].empty? &&
+          (%w[first last].include?(occurrence) || (occurrence.is_a?(Integer) && occurrence >= 0))
+        raise ArgumentError, "Invalid selective replay assertion target" unless valid_target
+        @assertion_names.add(target["name"])
+        count = @nodes.values.count { |node| node["spanName"] == target["name"] }
+        @unresolved_assertions << assertion["id"] if count <= (occurrence.is_a?(Integer) ? occurrence : 0)
+      end
+      @full_trace ||= !@unresolved_assertions.empty?
+      unresolved = @required - @nodes.values.map { |node| key(node) }.to_set
+      unless unresolved == @plan["unresolvedMustRun"].map { |node| key(node) }.to_set
+        raise ArgumentError, "Selective replay plan has incomplete must_run coverage"
+      end
+      @nodes.each_value do |node|
+        ancestors = Set.new([node["id"]])
+        required = @required.include?(key(node)) || !unresolved.empty?
+        asserted = @full_trace || @assertion_names.include?(node["spanName"])
+        parent_id = node["parentId"]
+        until parent_id.nil?
+          parent = @nodes[parent_id]
+          unless parent && ancestors.add?(parent_id)
+            raise ArgumentError, "Invalid selective replay ancestry"
+          end
+          if node["mustRun"] && !parent["mustRun"]
+            raise ArgumentError, "Invalid selective replay ancestry"
+          end
+          required ||= @required.include?(key(parent))
+          asserted ||= @assertion_names.include?(parent["spanName"])
+          parent_id = parent["parentId"]
+        end
+        if !ancestors.include?(@plan["rootId"]) || ((required || asserted) && !node["mustRun"])
+          raise ArgumentError, "Selective replay plan could hide required code or assertion evidence"
+        end
+      end
+    end
+
+    def enter(trace_function_key:, span_name:, span_id:, parent_span_id:, args:, kwargs:, safety_mock:, reuse_allowed:, can_reuse_output: true)
+      @mutex.synchronize do
+        identity = [trace_function_key, span_name]
+        input = self.class.inputs(args, kwargs)
+        node = if parent_span_id.nil?
+          @nodes[@plan["rootId"]]
+        else
+          candidates = @children[@parents[parent_span_id]].select { |candidate| key(candidate) == identity && !@consumed[parent_span_id].include?(candidate["id"]) }
+          exact = candidates.select { |candidate| input && @recordings.dig(candidate["id"], 0) == input }
+          match = if exact.length == 1
+            exact.first
+          else
+            ((exact.empty? && candidates.length == 1) ? candidates.first : nil)
+          end
+          (match ? [match] : candidates).each { |candidate| @consumed[parent_span_id].add(candidate["id"]) }
+          match
+        end
+        @parents[span_id] = node&.dig("id")
+        required = @required.include?(identity) || @required_scopes.include?(parent_span_id)
+        @required_scopes.add(span_id) if required
+        asserted = @full_trace || @assertion_names.include?(span_name) || @assertion_scopes.include?(parent_span_id)
+        @assertion_scopes.add(span_id) if asserted
+        must_run = parent_span_id.nil? || required || asserted || node&.dig("mustRun") || !@plan["unresolvedMustRun"].empty?
+        recorded_input, recorded_output = @recordings[node&.dig("id")]
+        reusable = input && input == recorded_input && recorded_output && can_reuse_output
+        reason = if required
+          "must-run"
+        elsif asserted
+          "assertion"
+        elsif !@plan["unresolvedMustRun"].empty?
+          "unresolved-must-run"
+        elsif !node
+          "unmatched-or-ambiguous-call"
+        elsif !reusable
+          "input-drift-or-missing-recording"
+        else
+          node["reason"]
+        end
+        decision = {trace_function_key:, span_name:, original_span_id: node&.dig("id"), reason:}
+        if safety_mock && (must_run || !reusable)
+          @decisions << decision.merge(action: "blocked", reason: "safety-conflict:#{reason}")
+          @failure = RuntimeError.new("Selective replay cannot execute or safely reuse safety-mocked node #{span_name} (#{reason})")
+          raise @failure
+        end
+        if !must_run && reusable && (safety_mock || (node["reusable"] && reuse_allowed))
+          @decisions << decision.merge(action: "mock", reason: safety_mock ? "safety-mock" : "unchanged-inputs")
+          return [true, JSON.parse(recorded_output)]
+        end
+        @decisions << decision.merge(action: "run")
+        @observed[span_name] += 1
+        @reached.add(identity) if @required.include?(identity)
+        [false, nil]
+      end
+    end
+
+    def assert_no_safety_conflict!
+      raise @failure if @failure
+    end
+
+    def report
+      @mutex.synchronize do
+        {
+          decisions: @decisions.map(&:dup),
+          must_run_not_reached: @plan["options"]["mustRun"].reject { |node| @reached.include?(key(node)) }.map { |node| identity(node) },
+          unresolved_must_run: @plan["unresolvedMustRun"].map { |node| identity(node) },
+          assertion_ids: @plan["assertions"].map { |assertion| assertion["id"] },
+          unresolved_assertions: @unresolved_assertions.dup,
+          assertion_targets_not_reached: @plan["assertions"].filter_map do |assertion|
+            target = assertion["target"]
+            if target && target["kind"] == "span" && @observed[target["name"]] <= (target["occurrence"].is_a?(Integer) ? target["occurrence"] : 0)
+              assertion["id"]
+            end
+          end
+        }
+      end
+    end
+
+    private
+
+    def key(node)
+      [node.fetch("traceFunctionKey"), node.fetch("spanName")]
+    end
+
+    def identity(node)
+      {trace_function_key: node["traceFunctionKey"], span_name: node["spanName"]}
+    end
+  end
+end

data/lib/bitfab/simulation_plan.rb

@@ -2,7 +2,7 @@
 
 module Bitfab
   class SimulationPlan
-    CONTENT_KEYS = %w[input input_meta output output_meta input_serialized output_serialized prompt].freeze
+    CONTENT_KEYS = %w[input input_meta output output_meta input_serialized output_serialized replay_recording prompt].freeze
     FRAMEWORKS = %w[openai-agents langgraph claude-agent-sdk vercel-ai].freeze
     READ_TIMEOUT = 5.0
     REFRESH_SECONDS = 60.0

data/lib/bitfab/subtree.rb

@@ -667,10 +667,15 @@ module Bitfab
           active_parent[:span_id]
         end
         subtree_parent_span_id = stack.reverse_each.find { |entry| entry.span_id }&.span_id
+        parent_span_id = subtree_parent_span_id || @fiber_parents.dig(fiber_id, :span_id) || active_parent_span_id || @root_span_id
+        ReplayContext.current&.dig(:selective_replay)&.enter(
+          trace_function_key: @trace_function_key, span_name: metadata.name, span_id:, parent_span_id:,
+          args:, kwargs:, safety_mock: false, reuse_allowed: false, can_reuse_output: false
+        )
         stack << OpenSpan.new(
           identity: metadata.identity,
           span_id:,
-          parent_span_id: subtree_parent_span_id || @fiber_parents.dig(fiber_id, :span_id) || active_parent_span_id || @root_span_id,
+          parent_span_id:,
           name: metadata.name,
           args:,
           kwargs:,

data/lib/bitfab/traceable.rb

@@ -53,7 +53,7 @@ module Bitfab
     #   preserve the bound client through the fluent wrapper, matching Python's
     #   `BitfabFunction.span()` and TypeScript's `BitfabFunction.withSpan()`.
     def self.wrap(klass, method_name, trace_function_key:, name: nil, type: "custom",
-      capture_when: "always", mock_on_replay: false, finalize: nil, experiment_id: nil, client: nil, test_run_id: nil)
+      capture_when: "always", mock_on_replay: false, replay_reusable: false, finalize: nil, experiment_id: nil, client: nil, test_run_id: nil)
       experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
       method_name_str = method_name.to_s
       bound_client = client
@@ -67,7 +67,7 @@ module Bitfab
             trace_function_key:,
             span_name: name || Subtree.qualified_name(self, method_name),
             span_type: type, function_name: method_name_str, args:, kwargs:,
-            capture_when:, mock_on_replay:, finalize:, experiment_id:, surface: :span,
+            capture_when:, mock_on_replay:, replay_reusable:, finalize:, experiment_id:, surface: :span, input_block: !block.nil?,
             explicit_span_receiver: self, explicit_span_method_name: method_name) do
             super(*args, **kwargs, &block)
           end
@@ -135,7 +135,7 @@ module Bitfab
     # Configure an existing method only when it is discovered beneath a
     # bitfab_trace root. Outside subtree capture the method runs unchanged.
     def self.node(klass, method_name, name: nil, type: "custom", capture: true,
-      experiment_id: nil, mock_on_replay: nil, finalize: nil, client: nil, test_run_id: nil)
+      experiment_id: nil, mock_on_replay: nil, replay_reusable: false, finalize: nil, client: nil, test_run_id: nil)
       experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
       validate_node_options!(capture:, mock_on_replay:)
       method_name_str = method_name.to_s
@@ -164,7 +164,7 @@ module Bitfab
             call.owner.client.send(:execute_span,
               trace_function_key: call.owner.trace_function_key,
               span_name:, span_type: type, function_name: method_name_str,
-              args:, kwargs:, capture_when: "nested", mock_on_replay: policy,
+              args:, kwargs:, capture_when: "nested", mock_on_replay: policy, replay_reusable:, input_block: !block.nil?,
               explicit_span_receiver: self, explicit_span_method_name: method_name,
               experiment_id:, finalize:, surface: :trace, result_recorded_elsewhere: call.recorded_elsewhere?, declared_node: true,
               on_start: call.method(:start), on_complete: call.method(:complete),
@@ -267,7 +267,7 @@ module Bitfab
       #   When true, `client.replay(...)` returns this span's
       #   historical output instead of executing the wrapped method.
       def bitfab_span(method_name, trace_function_key: nil, name: nil, type: "custom",
-        capture_when: "always", mock_on_replay: false, finalize: nil, experiment_id: nil, test_run_id: nil)
+        capture_when: "always", mock_on_replay: false, replay_reusable: false, finalize: nil, experiment_id: nil, test_run_id: nil)
         experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
         trace_function_key ||= @bitfab_function_key
         unless trace_function_key
@@ -278,7 +278,7 @@ module Bitfab
         # If the method already exists (inline or after-method style), wrap it immediately
         if method_defined?(method_name) || private_method_defined?(method_name)
           _bitfab_wrap_method(
-            method_name, trace_function_key:, name:, type:, capture_when:, mock_on_replay:, finalize:, experiment_id:
+            method_name, trace_function_key:, name:, type:, capture_when:, mock_on_replay:, replay_reusable:, finalize:, experiment_id:
           )
         else
           # Method doesn't exist yet (before-method style) - register for method_added hook
@@ -289,6 +289,7 @@ module Bitfab
             type:,
             capture_when:,
             mock_on_replay:,
+            replay_reusable:,
             finalize:,
             experiment_id:
           }
@@ -328,10 +329,10 @@ module Bitfab
       # EXPERIMENTAL. Configure one method discovered beneath bitfab_trace.
       # The node inherits the active trace key and never traces by itself.
       def bitfab_node(method_name, name: nil, type: "custom", capture: true,
-        experiment_id: nil, mock_on_replay: nil, finalize: nil, test_run_id: nil)
+        experiment_id: nil, mock_on_replay: nil, replay_reusable: false, finalize: nil, test_run_id: nil)
         experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
         Traceable.send(:validate_node_options!, capture:, mock_on_replay:)
-        config = {name:, type:, capture:, experiment_id:, mock_on_replay:, finalize:}
+        config = {name:, type:, capture:, experiment_id:, mock_on_replay:, replay_reusable:, finalize:}
         if method_defined?(method_name) || private_method_defined?(method_name)
           _bitfab_node_method(method_name, **config)
         else
@@ -365,7 +366,7 @@ module Bitfab
       end
 
       def _bitfab_node_method(method_name, name: nil, type: "custom", capture: true,
-        experiment_id: nil, mock_on_replay: nil, finalize: nil)
+        experiment_id: nil, mock_on_replay: nil, replay_reusable: false, finalize: nil)
         Traceable.node(
           self,
           method_name,
@@ -374,6 +375,7 @@ module Bitfab
           capture:,
           experiment_id:,
           mock_on_replay:,
+          replay_reusable:,
           finalize:,
           client: nil
         )

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.61.0"
+  VERSION = "0.61.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.61.0
+  version: 0.61.2
 platform: ruby
 authors:
 - Harvest Team
@@ -188,6 +188,7 @@ files:
 - lib/bitfab/seed.rb
 - lib/bitfab/seed_cli.rb
 - lib/bitfab/seed_context.rb
+- lib/bitfab/selective_replay.rb
 - lib/bitfab/serialize.rb
 - lib/bitfab/simulation_plan.rb
 - lib/bitfab/span_context.rb