bitfab 0.57.7 → 0.58.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b92a57be2a0c05885f0f84f757ede302ab14aed53ad422c85ee2781591a39786
-  data.tar.gz: ddb9b050e2f66e9591c17be161c2ebd25d395b625704baee80e396801f5a9e25
+  metadata.gz: 49c5627d0b0880adf5971614d2b017bf97a1886313776b757996d1bdab1d1f92
+  data.tar.gz: cce67f5a022711dc884d9b391d2daa4547b5fc11fdff1ab159c7a5b82fb404bf
 SHA512:
-  metadata.gz: 0542a3b4dcb64f183ce35f7c9f16ead26e858fa1fefb7889115842e552b4855842ba175bea19893912ec37547708403d115259a201b2e02a1e6cb5ffa84f79f6
-  data.tar.gz: a530cd08bc1cdbc42467ea640c8f64e3035569c6723032626284bd6397744a21f5149ef0bc2d32af50aab1b1bbb856e8ba7510d1eb3640824de4f6f4e4472c43
+  metadata.gz: 28f813f00802eda3f55615bf5acba0264d6e9a80130117522436eefb943be3fbd17753289a6a2f7e6295ed9772380221a89a9c3e8477041b974f5d3f4cde917f
+  data.tar.gz: a15a81f27f2ef548b854a7b53cdd32a813583b31066a0cba8a40aa364c1bdf9e771f9534bd0aab8b5355bc146fbff18ec0a1cfb9a417634e28ab9691e6cdc6c7

data/README.md

@@ -235,7 +235,7 @@ class OrderPipeline
 end
 ```
 
-Capture uses `TracePoint` only during the root call. It skips Ruby/runtime code, dependency gems, Bitfab SDK internals, blocks, lambdas, and forwarding wrappers. When the root belongs to an installed application gem, that gem is the first-party boundary; neighboring dependency gems remain excluded. It follows recursion, modules, inheritance, `define_method`, and same-thread child Fibers; aliases are recorded and excluded by the name that was invoked. Child Fibers created during capture inherit node policies, trace identity, parenting, and depth limits while retaining separate call stacks. Pre-existing Fibers, `Fiber.new(storage: nil)`, and child threads do not inherit subtree context. When an inner capture ends, child Fibers retain any still-active outer capture; inherited context expires once all enclosing captures end. While a root remains active, hitting a span limit stops new capture but allows already-open spans in every Fiber to finish. `max_depth:` (default `30`), `max_captured_subtree_spans:` (default `512`), `max_spans:` (default `2048`), `exclude:`, and `include_wrappers:` control the subtree. `max_spans:` is a strict cap on the total number of spans one trace root sends. It counts the root span itself, discovered calls, `bitfab_node` calls, nested trace roots, and spans whose capture is on or off, including spans sent without inputs and outputs because the sim plan could not be read. Nothing past it is sent. It must be at least 1, and a lower value raises `ArgumentError`. `max_captured_subtree_spans:` caps discovered calls sent with their inputs and outputs. It does not count `bitfab_node` calls, nested trace roots, or spans the loaded sim plan turned capture off for, so those keep recording past it up to `max_spans:`. Spans sent without inputs and outputs only because the sim plan could not be read still count toward it. Past it, further discovered calls with capture on record nothing. It is never higher than `max_spans:`. Once the plan has loaded, the SDK never builds the inputs or outputs of a span whose capture it turned off. The client starts reading the sim plan when it is built. A `bitfab_trace` root that starts while that first read is still running waits for it, for at most five seconds. Once the first read has finished, whether it succeeded or failed, no later root waits. Nested roots and the calls beneath a root never wait, and nothing waits when the plan is disabled. If that first read fails, spans other than each trace's root span are sent without inputs and outputs until a read succeeds, and they still count toward both limits. Automatically captured calls are not replay mock targets. Declare a call with `bitfab_node` when replay must substitute it. The configured node records once in each enclosing trace. Mixing explicit spans with subtree tracing raises `Bitfab::MixedTracingError`.
+Capture uses `TracePoint` only during the root call. It skips Ruby/runtime code, dependency gems, Bitfab SDK internals, blocks, lambdas, and forwarding wrappers. When the root belongs to an installed application gem, that gem is the first-party boundary; neighboring dependency gems remain excluded. It follows recursion, modules, inheritance, `define_method`, and same-thread child Fibers; aliases are recorded and excluded by the name that was invoked. Child Fibers created during capture inherit node policies, trace identity, parenting, and depth limits while retaining separate call stacks. Pre-existing Fibers, `Fiber.new(storage: nil)`, and child threads do not inherit subtree context. When an inner capture ends, child Fibers retain any still-active outer capture; inherited context expires once all enclosing captures end. While a root remains active, hitting a span limit stops new capture but allows already-open spans in every Fiber to finish. `max_depth:` (default `30`), `max_captured_subtree_spans:` (default `500`), `exclude:`, and `include_wrappers:` control the subtree. There is no cap on the total number of spans one trace root sends. `max_spans:` was removed, and passing it raises `ArgumentError`. `max_captured_subtree_spans:` caps discovered calls sent with their inputs and outputs. It does not count `bitfab_node` calls, nested trace roots, or spans the loaded sim plan turned capture off for, so those keep recording past it. Spans sent without inputs and outputs only because the sim plan could not be read still count toward it. Past it, further discovered calls with capture on record nothing. Once the plan has loaded, the SDK does not send a span whose capture it turned off, and that span's children move up to the nearest visible parent. The trace's root span, a framework integration span, and a span that recorded an error are still sent, and the SDK never builds their inputs or outputs. A `bitfab_node` call and a nested trace root ignore the plan and always keep their inputs and outputs. The client starts reading the sim plan when it is built. A `bitfab_trace` root that starts while that first read is still running waits for it, for at most five seconds. Once the first read has finished, whether it succeeded or failed, no later root waits. Nested roots and the calls beneath a root never wait, and nothing waits when the plan is disabled. If that first read fails, spans other than each trace's root span, `bitfab_node` calls, and nested trace roots are sent without inputs and outputs until a read succeeds, and they still count toward `max_captured_subtree_spans:`. Automatically captured calls are not replay mock targets. Declare a call with `bitfab_node` when replay must substitute it. The configured node records once in each enclosing trace. Mixing explicit spans with subtree tracing raises `Bitfab::MixedTracingError`.
 
 When capture ends, already-open automatic calls in suspended Fibers are emitted once with a nil output and `Subtree capture ended before the call returned` as the error, unless a real error was already observed. Their inputs and parenting are retained. The SDK does not resume Fibers, and later resumption cannot add spans to the ended capture. If hot reload moves the root to an unresolved source, the last valid first-party boundary and its exclusions remain in use.
 

data/lib/bitfab/client.rb

@@ -376,7 +376,7 @@ module Bitfab
       capture_when: "always", mock_on_replay: false, independent_root: false,
       explicit_span_receiver: nil, explicit_span_method_name: nil, test_run_id: nil, finalize: nil,
       surface: nil, on_start: nil, on_complete: nil, around_body: nil, span_data: {}, managed_root: false,
-      result_recorded_elsewhere: false)
+      result_recorded_elsewhere: false, declared_node: false)
       return yield unless should_record?
 
       @http_client.refresh_simulation_plan
@@ -477,7 +477,8 @@ module Bitfab
               test_run_id: resolved_test_run_id,
               input_source_span_id: resolved_input_source_span_id,
               mock_source:,
-              instrumentation: (surface == :trace) ? "trace" : "span"
+              instrumentation: (surface == :trace) ? "trace" : "span",
+              declared_node:
             )
             on_complete&.call(result: mocked_output, error: nil, ended_at: Bitfab.now_iso_timestamp)
             return mocked_output
@@ -509,6 +510,8 @@ module Bitfab
       @http_client.trace_completion.start(trace_id, span_id)
       recording_override = managed_root ? {} : nil
       content_withheld = lambda do
+        next false if declared_node
+
         instrumentation = recording_override&.dig(:instrumentation) || ((surface == :trace) ? "trace" : "span")
         content_off_by_simulation_plan?(
           trace_id:, trace_function_key:, span_name: recording_override&.dig(:name) || span_name, instrumentation:
@@ -544,6 +547,7 @@ module Bitfab
           send_span(
             span_data: content_off ? span_data.merge("content_off_by_simulation_plan" => true) : span_data,
             instrumentation: recorded_instrumentation,
+            declared_node:,
             trace_function_key:,
             trace_id:,
             span_id:,
@@ -921,7 +925,7 @@ module Bitfab
     def send_span(trace_function_key:, trace_id:, span_id:, parent_span_id:,
       span_name:, span_type:, function_name:, contexts:, prompt:, args:, kwargs:, result:, error:,
       started_at:, ended_at:, test_run_id: nil, input_source_span_id: nil, mocked: false,
-      mock_target: nil, mock_source: nil, span_data: {}, instrumentation: "span")
+      mock_target: nil, mock_source: nil, span_data: {}, instrumentation: "span", declared_node: false)
       extra_span_data = span_data
       # If drop() was called on this trace, suppress the span PAYLOAD upload for
       # every span that completes after the flag was set. The trace completion
@@ -933,9 +937,9 @@ module Bitfab
       # Human-readable JSON (input/output fields). The *_with_report variants
       # also report what could not be faithfully captured so a lossy span is
       # marked non-replayable below instead of shipping as if it round-trips.
-      content_off = extra_span_data["content_off_by_simulation_plan"] == true ||
+      content_off = !declared_node && (extra_span_data["content_off_by_simulation_plan"] == true ||
         content_off_by_simulation_plan?(trace_id:, trace_function_key:, span_name:, instrumentation:) ||
-        (!parent_span_id.nil? && simulation_plan_unavailable?(instrumentation:))
+        (!parent_span_id.nil? && simulation_plan_unavailable?(instrumentation:)))
       if content_off
         dropped = []
         span_data = {"name" => span_name, "type" => span_type, "function_name" => function_name}
@@ -1011,7 +1015,7 @@ module Bitfab
         }]
       end
 
-      @http_client.send_external_span(payload) # Returns the background thread
+      @http_client.send_external_span(payload, declared_node)
     end
 
     # Advance the per-(key, name) call counter when this invocation is a
@@ -1141,7 +1145,7 @@ module Bitfab
     # output as the result and no error.
     def send_mocked_span(trace_function_key:, trace_id:, span_id:, parent_span_id:,
       span_name:, span_type:, function_name:, args:, kwargs:, mocked_output:,
-      started_at:, test_run_id:, input_source_span_id:, mock_source:, instrumentation: "span")
+      started_at:, test_run_id:, input_source_span_id:, mock_source:, instrumentation: "span", declared_node: false)
       ended_at = Bitfab.now_iso_timestamp
       send_span(
         trace_function_key:,
@@ -1163,6 +1167,7 @@ module Bitfab
         input_source_span_id:,
         mocked: true,
         instrumentation:,
+        declared_node:,
         # Both paths short-circuit the call, so the target is always the
         # output; only where the value came from differs.
         mock_target: "output",
@@ -1223,7 +1228,7 @@ module Bitfab
 
     # Experimentally trace an existing method and its first-party call subtree.
     def trace(klass, method_name, name: nil, type: "custom",
-      max_depth: Subtree::DEFAULT_MAX_DEPTH, max_spans: Subtree::DEFAULT_MAX_SPANS,
+      max_depth: Subtree::DEFAULT_MAX_DEPTH,
       max_captured_subtree_spans: Subtree::DEFAULT_MAX_CAPTURED_SUBTREE_SPANS,
       exclude: [], include_wrappers: false, mock_on_replay_default: false)
       Bitfab::Traceable.trace(
@@ -1232,7 +1237,6 @@ module Bitfab
         name:,
         type:,
         max_depth:,
-        max_spans:,
         max_captured_subtree_spans:,
         exclude:,
         include_wrappers:,

data/lib/bitfab/http_client.rb

@@ -188,7 +188,7 @@ module Bitfab
     end
 
     # Queue an external span on this client's trace transport (fire-and-forget).
-    def send_external_span(payload)
+    def send_external_span(payload, declared_node = false)
       if (data = payload.dig("rawSpan", "span_data")).is_a?(Hash) && !data.key?("runtime")
         payload = payload.merge("rawSpan" => payload["rawSpan"].merge("span_data" => data.merge("runtime" => ThreadPropagation.runtime)))
       end
@@ -204,7 +204,11 @@ module Bitfab
           recorded_meta("external_span", record, ref)
         )
       }
-      @simulation_plan ? @simulation_plan.send_span(payload, &submit) : submit.call(payload)
+      discard = ->(record) {
+        ref = carrier_ref(record)
+        @trace_completion.discard(ref.trace_id, ref.span_id) if ref&.span_id
+      }
+      @simulation_plan ? @simulation_plan.send_span(payload, declared_node:, discard:, &submit) : submit.call(payload)
     end
 
     def send_internal_trace(function_id, payload)

data/lib/bitfab/otel.rb

@@ -24,7 +24,7 @@ module Bitfab
     EXPORT_CONCURRENCY_ENV = "BITFAB_OTEL_EXPORT_CONCURRENCY"
     MAX_EXPORT_REQUEST_BYTES = 3_000_000
     MAX_DECOMPRESSED_REQUEST_BYTES = 8_000_000
-    MAX_QUEUE_SIZE = 8_192
+    MAX_QUEUE_SIZE = 16_384
     DIRECT_MAX_EXPORT_BATCH_SIZE = 512
     DIRECT_MAX_REQUEST_BATCH_SIZE = 128
     DEFAULT_EXPORT_CONCURRENCY = 32
@@ -331,17 +331,17 @@ module Bitfab
         if e.oversized
           warn_oversized(batch.spans.size)
         else
-          Bitfab.warn_always("failed to export an OpenTelemetry span batch: #{e.message}")
+          Bitfab.warn_delivery_problem("failed to export an OpenTelemetry span batch: #{e.message}")
         end
         false
       rescue => e
-        Bitfab.warn_always("failed to export an OpenTelemetry span batch: #{e.message}")
+        Bitfab.warn_delivery_problem("failed to export an OpenTelemetry span batch: #{e.message}")
         false
       end
 
       def warn_oversized(span_count)
         subject = (span_count == 1) ? "a single OpenTelemetry span" : "an OpenTelemetry span batch"
-        Bitfab.warn_always("#{subject} exceeded the ingestion request limit and could not be exported")
+        Bitfab.warn_delivery_problem("#{subject} exceeded the ingestion request limit and could not be exported")
       end
 
       def send_with_retries(request)

data/lib/bitfab/replay.rb

@@ -1179,7 +1179,8 @@ module Bitfab
     # repeat itself. Acks cannot settle a request that timed out client-side
     # after the server committed, so anything short of fully delivered falls
     # through to the server poll, which stays the authority there.
-    def wait_for_replay_persistence(http_client, test_run_id, sdk_trace_ids)
+    def wait_for_replay_persistence(http_client, test_run_id, sdk_trace_ids, timeout: nil)
+      timeout ||= PERSISTENCE_TIMEOUT_SECONDS
       trace_ids = sdk_trace_ids.compact
 
       # Checked before flushing: a run whose traces never closed submitted
@@ -1194,7 +1195,7 @@ module Bitfab
       # timeout can fire while requests are still in flight and the server goes
       # on to persist every one of them. Failing here failed runs whose data had
       # fully landed.
-      flushed = Bitfab.flush_traces(timeout: PERSISTENCE_TIMEOUT_SECONDS)
+      flushed = Bitfab.flush_traces(timeout:)
 
       raw_deliveries = http_client.take_trace_deliveries(trace_ids)
       # Server-assigned traces.id per replay trace, read back off the ingest
@@ -1206,7 +1207,7 @@ module Bitfab
       expected_span_counts = deliveries.transform_values(&:span_count)
       return read_back_trace_ids if expected_span_counts.empty? || deliveries.each_value.all?(&:delivered)
 
-      deadline = Otel.monotonic_now + PERSISTENCE_TIMEOUT_SECONDS
+      deadline = Otel.monotonic_now + timeout
       missing = expected_span_counts.keys
 
       while missing.any?

data/lib/bitfab/replay_concurrency.rb

@@ -3,9 +3,9 @@
 module Bitfab
   class ReplayConcurrency
     UNSET = Object.new.freeze
-    attr_reader :attempts, :max_concurrency, :primitive, :memory_throttle, :on_item_finish_in_child_process, :child_timeout
+    attr_reader :attempts, :max_concurrency, :primitive, :memory_throttle, :on_item_finish_in_child_process, :child_timeout, :child_delivery_timeout
 
-    def initialize(attempts: 1, max_concurrency: UNSET, primitive: "async", memory_throttle: UNSET, on_item_finish_in_child_process: nil, child_timeout: 2400)
+    def initialize(attempts: 1, max_concurrency: UNSET, primitive: "async", memory_throttle: UNSET, on_item_finish_in_child_process: nil, child_timeout: 2400, child_delivery_timeout: UNSET)
       raise ArgumentError, "primitive must be async or process" unless %w[async process].include?(primitive)
       raise ArgumentError, "attempts must be an integer from 1 to 100" unless attempts.is_a?(Integer) && attempts.between?(1, 100)
       max_concurrency = (primitive == "process") ? 4 : 10 if max_concurrency.equal?(UNSET)
@@ -24,11 +24,31 @@ module Bitfab
       end
       raise ArgumentError, "child_timeout must be a positive finite number" unless child_timeout.is_a?(Numeric) && child_timeout.finite? && child_timeout.positive?
       @child_timeout = child_timeout
+      @child_delivery_timeout = resolve_child_delivery_timeout(child_delivery_timeout, primitive)
       @attempts, @max_concurrency, @primitive = attempts, max_concurrency, primitive
       @memory_throttle, @on_item_finish_in_child_process = memory_throttle, on_item_finish_in_child_process
       freeze
     end
 
+    private
+
+    def resolve_child_delivery_timeout(timeout, primitive)
+      if timeout.equal?(UNSET)
+        return (primitive == "process") ? Replay::PERSISTENCE_TIMEOUT_SECONDS : nil
+      end
+      return nil if timeout.nil? && primitive != "process"
+      unless timeout.is_a?(Numeric) && timeout.finite? && timeout.positive?
+        raise ArgumentError, "child_delivery_timeout must be a positive number of seconds (got #{timeout.inspect})"
+      end
+      if primitive != "process"
+        raise ArgumentError, "child_delivery_timeout bounds how long a child process waits for its spans to reach Bitfab, " \
+          "and only primitive: process creates one. Under primitive: async the run's own delivery wait applies."
+      end
+      timeout
+    end
+
+    public
+
     def self.resolve(concurrency, attempts, max_concurrency)
       if concurrency
         raise ArgumentError, "concurrency must be a ReplayConcurrency" unless concurrency.is_a?(self)

data/lib/bitfab/replay_processes.rb

@@ -42,37 +42,57 @@ module Bitfab
       end
       http = client.instance_variable_get(:@http_client)
       overrides = MockOverride.normalize(options[:mock_override]) + (client.instance_variable_get(:@mock_overrides) || [])
-      result = Replay.process_single_item(http, assignment.fetch("item"), receiver, method_name,
-        assignment.fetch("test_run_id"), assignment.fetch("mock"), options[:adapt_inputs],
-        assignment.fetch("include_db_branch_lease"), assignment["db_branch_settings"],
-        mock_overrides: overrides, dry_run: assignment.fetch("dry_run"))
-      unless assignment["dry_run"]
-        begin
-          mapped = Replay.wait_for_replay_persistence(http, assignment.fetch("test_run_id"), [result[:_sdk_trace_id]])
-          result[:trace_id] = mapped[result[:_sdk_trace_id]] || result[:trace_id]
-          if result[:_sdk_trace_id] && result[:trace_id].nil?
-            status = http.get_replay_status(assignment.fetch("test_run_id"), {result[:_sdk_trace_id] => 1})
-            result[:trace_id] = status.fetch("traceIds", {})[result[:_sdk_trace_id]]
+      delivery_timeout = options[:concurrency]&.child_delivery_timeout || Replay::PERSISTENCE_TIMEOUT_SECONDS
+      result = nil
+      Bitfab.collect_delivery_problems do
+        result = Replay.process_single_item(http, assignment.fetch("item"), receiver, method_name,
+          assignment.fetch("test_run_id"), assignment.fetch("mock"), options[:adapt_inputs],
+          assignment.fetch("include_db_branch_lease"), assignment["db_branch_settings"],
+          mock_overrides: overrides, dry_run: assignment.fetch("dry_run"))
+        unless assignment["dry_run"]
+          begin
+            mapped = Replay.wait_for_replay_persistence(http, assignment.fetch("test_run_id"), [result[:_sdk_trace_id]], timeout: delivery_timeout)
+            result[:trace_id] = mapped[result[:_sdk_trace_id]] || result[:trace_id]
+            if result[:_sdk_trace_id] && result[:trace_id].nil?
+              status = http.get_replay_status(assignment.fetch("test_run_id"), {result[:_sdk_trace_id] => 1})
+              result[:trace_id] = status.fetch("traceIds", {})[result[:_sdk_trace_id]]
+            end
+            raise "Child replay trace persistence could not be confirmed" if result[:error].nil? && result[:trace_id].nil?
+          rescue => error
+            result[:replay_error] = error
+            result[:error] = error.message
           end
-          raise "Child replay trace persistence could not be confirmed" if result[:error].nil? && result[:trace_id].nil?
-        rescue => error
-          result[:replay_error] = error
-          result[:error] = error.message
         end
-      end
-      write_result(assignment.fetch("result_path"), result)
-      hook = options[:concurrency]&.on_item_finish_in_child_process
-      if hook && !assignment["dry_run"]
-        begin
-          hook.call({test_run_id: assignment.fetch("test_run_id"), item: Replay.public_replay_items([result]).first})
-        rescue => error
-          stderr.puts "[replay] child grading hook failed for #{result[:original_trace_id]} attempt #{result[:attempt]}: #{error}"
+        write_result(assignment.fetch("result_path"), result)
+        hook = options[:concurrency]&.on_item_finish_in_child_process
+        if hook && !assignment["dry_run"]
+          begin
+            hook.call({test_run_id: assignment.fetch("test_run_id"), item: Replay.public_replay_items([result]).first})
+          rescue => error
+            stderr.puts "[replay] child grading hook failed for #{result[:original_trace_id]} attempt #{result[:attempt]}: #{error}"
+          end
         end
+        unless client.close(timeout: delivery_timeout) || assignment["dry_run"]
+          Bitfab.count_delivery_problem("delivery was not confirmed within child_delivery_timeout (#{delivery_timeout}s). " \
+            "Spans that failed or were still sending when the child exited may never reach Bitfab.")
+        end
+        write_delivery_problems(assignment["delivery_problems_path"], result, Bitfab.delivery_problem_counts, stderr)
       end
-      client.close(timeout: 5)
       result
     end
 
+    def write_delivery_problems(path, result, problems, stderr)
+      return if problems.empty?
+      label = "#{result[:original_trace_id]}##{result[:attempt]}"
+      report = problems.map do |message, count|
+        "[replay] span delivery problem for #{label}: #{message}#{" (x#{count})" if count > 1}\n"
+      end.join
+      stderr.write(report)
+      File.write(path, report, mode: "w", perm: 0o600) if path
+    rescue SystemCallError
+      nil
+    end
+
     def write_result(path, result)
       encoded = begin
         Marshal.dump(result)
@@ -116,7 +136,8 @@ module Bitfab
         result_path = File.join(@directory, "#{index}.result")
         assignment_path = File.join(@directory, "#{index}.json")
         stdout_path, stderr_path = File.join(@directory, "#{index}.out"), File.join(@directory, "#{index}.err")
-        assignment = {pipeline: @args.fetch(:pipeline), params: @params, item:, test_run_id:, mock:, include_db_branch_lease:, db_branch_settings:, dry_run:, result_path:}
+        delivery_problems_path = File.join(@directory, "#{index}.delivery")
+        assignment = {pipeline: @args.fetch(:pipeline), params: @params, item:, test_run_id:, mock:, include_db_branch_lease:, db_branch_settings:, dry_run:, result_path:, delivery_problems_path:}
         File.write(assignment_path, JSON.generate(assignment), mode: "w", perm: 0o600)
         executable = File.expand_path("../../exe/bitfab-replay", __dir__)
         pid = @mutex.synchronize do
@@ -154,6 +175,8 @@ module Bitfab
         end
         child_errors = read_tail(stderr_path)
         @mutex.synchronize { @stderr.write(child_errors) } unless child_errors.empty?
+        delivery_problems = File.file?(delivery_problems_path) ? File.read(delivery_problems_path) : ""
+        @mutex.synchronize { @stderr.write(delivery_problems) } unless delivery_problems.empty? || child_errors.include?(delivery_problems)
         unless File.file?(result_path)
           output_tail = read_tail(stdout_path)
           @mutex.synchronize { @stderr.write(output_tail) } unless output_tail.empty?

data/lib/bitfab/replay_registry.rb

@@ -220,7 +220,8 @@ module Bitfab
                            end,
           memory_throttle: args.fetch(:memory_throttle, configured ? configured.memory_throttle : ReplayConcurrency::UNSET),
           on_item_finish_in_child_process: configured&.on_item_finish_in_child_process,
-          child_timeout: configured&.child_timeout || 2400
+          child_timeout: configured&.child_timeout || 2400,
+          child_delivery_timeout: configured&.child_delivery_timeout || ReplayConcurrency::UNSET
         )
         ReplayProcesses.current&.configure(registry, options[:concurrency]) if options[:concurrency].primitive == "process"
       end

data/lib/bitfab/simulation_plan.rb

@@ -69,7 +69,7 @@ module Bitfab
       @mutex.synchronize { start_reader(0) if @thread.nil? && now >= @refresh_after }
     end
 
-    def send_span(payload, &submit)
+    def send_span(payload, declared_node: false, discard: nil, &submit)
       refresh
       data = payload.dig("rawSpan", "span_data")
       key = payload["rootTraceFunctionKey"] || payload["traceFunctionKey"]
@@ -77,7 +77,7 @@ module Bitfab
         submit.call(payload)
         return
       end
-      entry = [payload, key, submit]
+      entry = [payload, key, submit, discard, declared_node]
       ready = @mutex.synchronize { reading? ? hold(entry) : entry }
       deliver(*ready) if ready
     end
@@ -164,6 +164,7 @@ module Bitfab
       names = @mutex.synchronize { @content_off&.fetch(key, nil) }
       data = payload.dig("rawSpan", "span_data")
       return payload unless data.is_a?(Hash) && names&.include?(data["name"])
+      return nil unless root?(payload) || data["error"]
 
       without_content(payload, data)
     end
@@ -196,12 +197,13 @@ module Bitfab
       @held.shift if @held.length > MAX_HELD
     end
 
-    def deliver(payload, key, submit)
-      submit.call(prepare(payload, key))
+    def deliver(payload, key, submit, discard = nil, declared_node = false)
+      record = declared_node ? payload : prepare(payload, key)
+      record ? submit.call(record) : discard&.call(payload)
     end
 
-    def deliver_safely(payload, key, submit)
-      deliver(payload, key, submit)
+    def deliver_safely(payload, key, submit, discard = nil, declared_node = false)
+      deliver(payload, key, submit, discard, declared_node)
     rescue => error
       Bitfab.warn_once("sim-plan-held-span-dropped", "Held record could not be sent: #{error}")
     end

data/lib/bitfab/subtree.rb

@@ -10,9 +10,8 @@ module Bitfab
 
   module Subtree
     DEFAULT_MAX_DEPTH = 30
-    DEFAULT_MAX_SPANS = 2048
-    DEFAULT_MAX_CAPTURED_SUBTREE_SPANS = 512
-    LIMIT_ORDER = %i[max_depth max_spans max_captured_subtree_spans].freeze
+    DEFAULT_MAX_CAPTURED_SUBTREE_SPANS = 500
+    LIMIT_ORDER = %i[max_depth max_captured_subtree_spans].freeze
     SESSION_KEY = :bitfab_subtree_session
     DISPATCHER_KEY = :bitfab_subtree_dispatcher
     FIBER_CONTEXT_KEY = :bitfab_subtree_fiber_context
@@ -53,6 +52,7 @@ module Bitfab
       :span_type,
       :test_run_id,
       :span_data,
+      :declared_node,
       keyword_init: true
     )
 
@@ -175,7 +175,7 @@ module Bitfab
       CaptureConfig.new(root: nil, root_path: nil, exclude: {}.freeze).freeze
     end
 
-    def capture(client:, trace_function_key:, config:, max_depth:, max_spans:, include_wrappers:,
+    def capture(client:, trace_function_key:, config:, max_depth:, include_wrappers:,
       root_receiver:, root_method_name:, max_captured_subtree_spans: DEFAULT_MAX_CAPTURED_SUBTREE_SPANS, mock_on_replay_default: false)
       return yield unless config.root
 
@@ -193,7 +193,6 @@ module Bitfab
           input_source_span_id: replay&.dig(:input_source_span_id),
           config:,
           max_depth:,
-          max_spans:,
           max_captured_subtree_spans:,
           include_wrappers:,
           mock_on_replay_default:,
@@ -418,7 +417,7 @@ module Bitfab
       end
 
       def initialize(client:, trace_function_key:, trace_id:, root_span_id:, test_run_id:,
-        input_source_span_id:, config:, max_depth:, max_spans:, include_wrappers:,
+        input_source_span_id:, config:, max_depth:, include_wrappers:,
         root_receiver:, root_method_name:, max_captured_subtree_spans: DEFAULT_MAX_CAPTURED_SUBTREE_SPANS, mock_on_replay_default: false)
         @mock_on_replay_default = mock_on_replay_default
         @client = client
@@ -433,8 +432,7 @@ module Bitfab
         @root_method_name = root_method_name.to_sym
         @skip_root_call = true
         @max_depth = max_depth
-        @max_spans = max_spans
-        @max_captured_subtree_spans = [max_captured_subtree_spans, max_spans].min
+        @max_captured_subtree_spans = max_captured_subtree_spans
         @exclude = config.exclude
         @include_wrappers = include_wrappers
         @stacks = {}
@@ -443,7 +441,6 @@ module Bitfab
         @suppressed_depths = {}
         @owned_paths = {}
         @method_metadata = Hash.new { |owners, owner| owners[owner] = {} }
-        @spans_used = 1
         @captured_subtree_count = 0
         @limits_fired = Set.new
         @dropped_spans = 0
@@ -524,14 +521,9 @@ module Bitfab
 
       def reserve_node
         stack = current_stack(Fiber.current.object_id)
-        if captured_depth(stack) >= @max_depth
-          drop(:max_depth)
-          return false
-        end
-        return true if take_slot(discovered: false)
+        return true if captured_depth(stack) < @max_depth
 
-        drop(:max_spans)
-        @exhausted = true
+        drop(:max_depth)
         false
       end
 
@@ -547,7 +539,7 @@ module Bitfab
           span_id: SecureRandom.uuid,
           parent_span_id: current_parent_span_id,
           name:, span_type: type, args:, kwargs:, test_run_id:, started_at: now,
-          span_data: {}
+          span_data: {}, declared_node: true
         )
         @configured_entries[entry.object_id] = entry
         entry
@@ -652,9 +644,9 @@ module Bitfab
           return
         end
         content_off = content_off?(metadata.name)
-        if (limit = limit_reached(discovered: true, content_off:))
-          drop(limit)
-          stop_discovery(fiber_id) unless limit == :max_captured_subtree_spans && content_off_names?
+        if !content_off && @captured_subtree_count >= @max_captured_subtree_spans
+          drop(:max_captured_subtree_spans)
+          stop_discovery(fiber_id) unless content_off_names?
           return
         end
 
@@ -665,7 +657,7 @@ module Bitfab
           return
         end
 
-        count_span(discovered: true, content_off:)
+        @captured_subtree_count += 1 unless content_off
         span_id = SecureRandom.uuid
         without_content = content_off || plan_unavailable?
         args, kwargs = without_content ? [[], {}] : inputs(event)
@@ -741,6 +733,7 @@ module Bitfab
           span_name: entry.name,
           span_type: entry.span_type || "function",
           instrumentation: "trace",
+          declared_node: entry.declared_node == true,
           span_data: entry.span_data || {},
           function_name: entry.name,
           contexts: nil,
@@ -890,24 +883,6 @@ module Bitfab
         @suppressed_depths[fiber_id] = 1
       end
 
-      def limit_reached(discovered:, content_off: false)
-        return :max_spans if @spans_used >= @max_spans
-
-        :max_captured_subtree_spans if discovered && !content_off && @captured_subtree_count >= @max_captured_subtree_spans
-      end
-
-      def take_slot(discovered:)
-        return false if limit_reached(discovered:)
-
-        count_span(discovered:, content_off: false)
-        true
-      end
-
-      def count_span(discovered:, content_off:)
-        @spans_used += 1
-        @captured_subtree_count += 1 if discovered && !content_off
-      end
-
       def stop_discovery(fiber_id)
         @exhausted = true
         if captured_calls_drained?
@@ -968,7 +943,7 @@ module Bitfab
         Bitfab.warn_once(
           "subtree-truncated:#{@trace_function_key}",
           "'#{@trace_function_key}' hit a subtree capture limit (max_depth=#{@max_depth}, " \
-          "max_captured_subtree_spans=#{@max_captured_subtree_spans}, max_spans=#{@max_spans}) and its trace is incomplete. " \
+          "max_captured_subtree_spans=#{@max_captured_subtree_spans}) and its trace is incomplete. " \
           "Raise the limits, add bitfab_node to the calls that matter, or narrow what is traced with exclude: [...]."
         )
       end

data/lib/bitfab/trace_completion.rb

@@ -31,6 +31,10 @@ module Bitfab
       end
     end
 
+    def discard(trace_id, span_id)
+      synchronize { @traces[trace_id]&.dig(:span_ids)&.delete(span_id) }
+    end
+
     def finish(trace_id, span_id)
       ready = synchronize do
         entry = @traces[trace_id]

data/lib/bitfab/traceable.rb

@@ -82,10 +82,9 @@ module Bitfab
     # The root records with the requested type; first-party Ruby methods called
     # beneath it record automatically as function spans.
     def self.trace(klass, method_name, trace_function_key:, name: nil, type: "custom",
-      max_depth: Subtree::DEFAULT_MAX_DEPTH, max_spans: Subtree::DEFAULT_MAX_SPANS,
+      max_depth: Subtree::DEFAULT_MAX_DEPTH,
       max_captured_subtree_spans: Subtree::DEFAULT_MAX_CAPTURED_SUBTREE_SPANS,
       exclude: [], include_wrappers: false, mock_on_replay_default: false, client: nil)
-      validate_trace_options!(max_spans:)
       capture_config = Subtree.prepare(root_source: source_of(klass, method_name), exclude:)
       method_name_str = method_name.to_s
       bound_client = client
@@ -106,7 +105,7 @@ module Bitfab
             SpanContext.current[:surface] = :trace
             next Subtree.capture(
               client: target_client, trace_function_key:, config: capture_config,
-              max_depth:, max_spans:, max_captured_subtree_spans:, include_wrappers:, mock_on_replay_default:,
+              max_depth:, max_captured_subtree_spans:, include_wrappers:, mock_on_replay_default:,
               root_receiver: self, root_method_name: method_name
             ) { super(*args, **kwargs, &block) }
           end
@@ -120,7 +119,7 @@ module Bitfab
             around_body: call.method(:run)) do
             Subtree.capture(
               client: target_client, trace_function_key:, config: capture_config,
-              max_depth:, max_spans:, max_captured_subtree_spans:, include_wrappers:, mock_on_replay_default:,
+              max_depth:, max_captured_subtree_spans:, include_wrappers:, mock_on_replay_default:,
               root_receiver: self, root_method_name: method_name
             ) { super(*args, **kwargs, &block) }
           end
@@ -165,7 +164,7 @@ module Bitfab
               span_name:, span_type: type, function_name: method_name_str,
               args:, kwargs:, capture_when: "nested", mock_on_replay: policy,
               explicit_span_receiver: self, explicit_span_method_name: method_name,
-              test_run_id:, finalize:, surface: :trace, result_recorded_elsewhere: call.recorded_elsewhere?,
+              test_run_id:, finalize:, surface: :trace, result_recorded_elsewhere: call.recorded_elsewhere?, declared_node: true,
               on_start: call.method(:start), on_complete: call.method(:complete),
               around_body: call.method(:run)) { super(*args, **kwargs, &block) }
           end
@@ -193,14 +192,6 @@ module Bitfab
     end
     private_class_method :preserve_visibility
 
-    def self.validate_trace_options!(max_spans:)
-      return if max_spans.is_a?(Integer) && max_spans >= 1
-
-      raise ArgumentError,
-        "bitfab_trace max_spans must be an integer of at least 1 because the root span takes one slot, got #{max_spans.inspect}."
-    end
-    private_class_method :validate_trace_options!
-
     def self.validate_node_options!(capture:, mock_on_replay:)
       return unless !capture && mock_on_replay
 
@@ -304,10 +295,9 @@ module Bitfab
       # EXPERIMENTAL. Trace a method and every first-party Ruby method it calls.
       # Descendants require no Bitfab declarations and record as function spans.
       def bitfab_trace(method_name, trace_function_key: nil, name: nil, type: "custom",
-        max_depth: Subtree::DEFAULT_MAX_DEPTH, max_spans: Subtree::DEFAULT_MAX_SPANS,
+        max_depth: Subtree::DEFAULT_MAX_DEPTH,
         max_captured_subtree_spans: Subtree::DEFAULT_MAX_CAPTURED_SUBTREE_SPANS,
         exclude: [], include_wrappers: false, mock_on_replay_default: false)
-        Traceable.send(:validate_trace_options!, max_spans:)
         trace_function_key ||= @bitfab_function_key
         unless trace_function_key
           raise "No trace function key provided. Pass `trace_function_key:` to `bitfab_trace` " \
@@ -319,7 +309,6 @@ module Bitfab
           name:,
           type:,
           max_depth:,
-          max_spans:,
           max_captured_subtree_spans:,
           exclude:,
           include_wrappers:,

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.57.7"
+  VERSION = "0.58.0"
 end

data/lib/bitfab/warn_once.rb

@@ -12,6 +12,8 @@ module Bitfab
   # warns once, not just the first one seen.
   @warn_once_seen = Set.new
   @warn_once_mutex = Mutex.new
+  @delivery_problems = nil
+  @delivery_problems_mutex = Mutex.new
 
   class << self
     def warn_once(key, message)
@@ -37,6 +39,27 @@ module Bitfab
       # Logging must never crash the host app (e.g. a closed $stderr).
     end
 
+    def warn_delivery_problem(message)
+      count_delivery_problem(message)
+      warn_always(message)
+    end
+
+    def count_delivery_problem(message)
+      @delivery_problems_mutex.synchronize { @delivery_problems[message] += 1 if @delivery_problems }
+    end
+
+    def delivery_problem_counts
+      @delivery_problems_mutex.synchronize { @delivery_problems ? @delivery_problems.dup : {} }
+    end
+
+    def collect_delivery_problems
+      problems = Hash.new(0)
+      @delivery_problems_mutex.synchronize { @delivery_problems = problems }
+      yield problems
+    ensure
+      @delivery_problems_mutex.synchronize { @delivery_problems = nil }
+    end
+
     # Test-only: clear the dedup set so a warning can fire again.
     def _reset_warn_once
       @warn_once_mutex.synchronize { @warn_once_seen.clear }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.57.7
+  version: 0.58.0
 platform: ruby
 authors:
 - Harvest Team