bitfab 0.75.0 → 0.75.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6976d07abea8e8c456185b0f5a76bbe861148e2a10baa1c3a65ac1a694b6993d
-  data.tar.gz: 072af5081f4c15e748478dbd058a8cc4c249804b345dcc70ed47b3e47f3c5830
+  metadata.gz: 5b80995a3476aa0fd9dd0eb926924081189bd6dc85efbdf9fe5ad0a07f375dae
+  data.tar.gz: 409f0f9b9837f142aed5233a6307fb30740f5409955dd0d74af8c9d36e27ebc6
 SHA512:
-  metadata.gz: 5189232282c64908bada35335eac364e16bb6fcadfe9b677cdc5b3330735f59dbdebc6ff71fe4a2984ad1a5c8386b45ade841710c0b2ec1ac0d26c0d07c7d44f
-  data.tar.gz: 90b5fd0acb0d266e6f2e7d2b6d04dc4a08c66d3e1b25ce46f51425e5020cf372167c40ba141cf05c9f7e4f2348cc9a106f1888e9a5b792bbbe9a4f08d2bb0b80
+  metadata.gz: fe3ae22d0d2da90db167cf4d968cf2fe71bcaba899fa3393ab822a8cc7682961c0fcb8aa9bf842e0630cca46827fbfed433813e1a1b2f0bf191f3766bee708c8
+  data.tar.gz: 4490e80fdac9fbb0288177e158926e46e5c17f16889d11ce1b241579c98616bf144250a59def158ce771831edfaba80c046f9d2d78604b3079d582f495eb86c1

data/lib/bitfab/client.rb

@@ -722,7 +722,8 @@ module Bitfab
             explicit_span_receiver:,
             explicit_span_method_name: explicit_span_method_name&.to_sym,
             surface:, recording_override:, content_withheld:,
-            sends_on_completion: sends_on_completion
+            sends_on_completion: sends_on_completion,
+            trace_function_key:
           ) do
             invoke = -> { around_body ? around_body.call { yield } : yield }
             result = if explicit_span_receiver
@@ -742,7 +743,7 @@ module Bitfab
           end
         end
       rescue => e
-        error = e.message
+        error = ErrorChain.describe(e)
         complete.call(result, error)
         raise
       end
@@ -769,7 +770,8 @@ module Bitfab
           surface:,
           around_body:,
           explicit_span_receiver:,
-          explicit_span_method_name:
+          explicit_span_method_name:,
+          trace_function_key:
         )
       end
 
@@ -812,7 +814,7 @@ module Bitfab
     end
 
     def root_trace_function_key(trace_id, trace_function_key)
-      TraceState.get(trace_id)&.dig(:trace_function_key) || trace_function_key
+      TraceState.trace_function_key(trace_id) || trace_function_key
     end
 
     def wait_for_simulation_plan
@@ -902,8 +904,10 @@ module Bitfab
     # span output. The span is sent exactly once: when iteration finishes,
     # raises, or the wrapper is `.close`d.
     def wrap_enumerator(source, trace_id:, span_id:, complete:, explicit_span_receiver: nil,
-      explicit_span_method_name: nil, surface: nil, around_body: nil)
+      explicit_span_method_name: nil, surface: nil, around_body: nil, trace_function_key: nil)
       span_entry = {trace_id:, span_id:, surface:}
+      SpanContext.record_trace_function_keys(span_entry, trace_function_key)
+      trace_function_keys = span_entry[:trace_function_keys]
       if explicit_span_receiver
         span_entry[:explicit_span_receiver] = explicit_span_receiver
         span_entry[:explicit_span_method_name] = explicit_span_method_name&.to_sym
@@ -916,7 +920,7 @@ module Bitfab
         # the right parent. The fiber bridge carries the same parent into
         # Enumerator.new / enum_for source fibers on this thread.
         ReplayContext.with_span(trace_id:, span_id:) do
-          SpanContext.with_fiber_bridge(trace_id:, span_id:, surface:) do
+          SpanContext.with_fiber_bridge(trace_id:, span_id:, surface:, trace_function_key:, trace_function_keys:) do
             SpanContext.stack.push(span_entry)
             begin
               iterate = lambda do
@@ -940,7 +944,7 @@ module Bitfab
               end
               complete.call(yielded, nil)
             rescue => e
-              complete.call(yielded, e.message)
+              complete.call(yielded, ErrorChain.describe(e))
               raise
             ensure
               SpanContext.stack.pop

data/lib/bitfab/error_chain.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Bitfab
+  # Reads the chain of causes below an exception. A database driver error
+  # usually reaches a traced method wrapped one or two levels down, so
+  # recording only the top message makes a connection timeout read as a broken
+  # query. Spans and replay results carry the deepest cause too.
+  module ErrorChain
+    # How many causes below an exception are followed.
+    MAX_DEPTH = 10
+    CODE_METHODS = %i[code sqlstate errno].freeze
+
+    module_function
+
+    # The causes below `error`, nearest first, stopping at the depth limit or
+    # at a cause already seen.
+    def causes(error)
+      found = []
+      seen = {error.object_id => true}
+      current = error
+      while found.length < MAX_DEPTH
+        cause = current.cause
+        break if cause.nil? || seen[cause.object_id]
+
+        found << cause
+        seen[cause.object_id] = true
+        current = cause
+      end
+      found
+    end
+
+    # The exception's message, followed by the deepest cause's message when it
+    # wraps another exception, e.g. `Failed query: ... (caused by: connection
+    # timeout)`. The cause is left off when the message already contains it.
+    def describe(error)
+      message = error.message
+      deepest = causes(error).last
+      return message if deepest.nil?
+
+      deepest_message = deepest.message.to_s.strip
+      return message if deepest_message.empty? || message.include?(deepest_message)
+
+      "#{message} (caused by: #{deepest_message})"
+    end
+
+    # An exception as a JSON record: type, message, backtrace, the driver's
+    # code when it has one, and its causes nested under `cause` down to the
+    # depth limit. The block adds fields a specific exception type carries.
+    def serialize(error, &extra)
+      serialize_at(error, 1, {}, &extra)
+    end
+
+    # `seen` is created fresh by `serialize` for each top-level call, so one
+    # call never skips a cause another call already recorded.
+    def serialize_at(error, depth, seen, &extra)
+      seen[error.object_id] = true
+      serialized = {type: error.class.name, message: error.message, backtrace: error.backtrace}
+      code = error_code(error)
+      serialized[:code] = code unless code.nil?
+      serialized.merge!(extra.call(error)) if extra
+      cause = error.cause
+      if cause && !seen[cause.object_id] && depth < MAX_DEPTH
+        serialized[:cause] = serialize_at(cause, depth + 1, seen, &extra)
+      end
+      serialized
+    end
+    private_class_method :serialize_at
+
+    def error_code(error)
+      CODE_METHODS.each do |name|
+        next unless error.respond_to?(name)
+
+        value = begin
+          error.public_send(name)
+        rescue
+          nil
+        end
+        return value if value.is_a?(String) || value.is_a?(Integer)
+      end
+      nil
+    end
+  end
+end

data/lib/bitfab/replay.rb

@@ -160,13 +160,9 @@ module Bitfab
     def json_safe(value)
       case value
       when Exception
-        serialized = {type: value.class.name, message: value.message, backtrace: value.backtrace}
-        if value.is_a?(DbBranchReplayError)
-          serialized[:code] = value.code
-          serialized[:original_trace_id] = value.original_trace_id
-          serialized[:cause] = json_safe(value.cause) if value.cause
+        ErrorChain.serialize(value) do |error|
+          error.is_a?(DbBranchReplayError) ? {original_trace_id: error.original_trace_id} : {}
         end
-        serialized
       when Hash
         value.to_h { |key, child| [key, json_safe(child)] }
       when Array
@@ -177,7 +173,7 @@ module Bitfab
     end
 
     def replay_item_error_message(error)
-      return error.message unless error.is_a?(DbBranchReplayError)
+      return ErrorChain.describe(error) unless error.is_a?(DbBranchReplayError)
 
       "Replay requested a database branch for trace #{error.original_trace_id} but it " \
         "could not be resolved (#{error.code}): #{error.message}. The method was not run, " \
@@ -1528,7 +1524,7 @@ module Bitfab
         input: kwargs.empty? ? args : [*args, kwargs],
         result: fn_result,
         original_output: item["output"],
-        error: item_error&.message,
+        error: item_error && ErrorChain.describe(item_error),
         trace_error: fn_error,
         replay_error:,
         duration_ms: replay_duration_ms,

data/lib/bitfab/span_context.rb

@@ -19,6 +19,16 @@ module Bitfab
       @span_state[:trace_id]
     end
 
+    # The keys of every trace function running at this point, outermost first,
+    # each listed once. A trace function called inside another one comes after
+    # it, so a cost recorded here can be counted toward each of them. Empty
+    # outside a trace.
+    def trace_function_keys
+      span_keys = @span_state[:trace_function_keys] || []
+      root_keys = Subtree.active_sessions.reverse.map(&:trace_function_key).uniq
+      (root_keys - span_keys) + span_keys
+    end
+
     # Add a context entry to this span.
     # The entire hash is pushed as a single entry in the contexts array.
     # Context entries are accumulated - multiple calls add to the list.
@@ -53,6 +63,15 @@ module Bitfab
       @trace_id = trace_id
     end
 
+    # The key of the trace function the current trace is stored under (the one
+    # that started it). A bitfab_span or get_function call inside another trace
+    # function joins that trace, so it reports the outer key. A bitfab_trace
+    # root inside another root starts its own trace, so it reports its own key.
+    # Empty string outside a trace.
+    def trace_function_key
+      TraceState.trace_function_key(@trace_id) || ""
+    end
+
     # Set the session ID for this trace.
     # Session ID is used to group traces from the same user session.
     # This is stored as a database column.
@@ -121,7 +140,8 @@ module Bitfab
   end
 
   # Thread-local span stack for tracking nested spans.
-  # Each entry is a Hash with :trace_id and :span_id keys.
+  # Each entry is a Hash with :trace_id and :span_id keys, plus the span's own
+  # :trace_function_key and the :trace_function_keys running at it.
   module SpanContext
     STACK_KEY = :__bitfab_span_stack
     FIBER_BRIDGE_STACK_KEY = :__bitfab_span_fiber_bridge_stack
@@ -139,8 +159,9 @@ module Bitfab
     # Enumerator.new and enum_for run their source body in another Fiber on
     # the same thread. Bridge only the active Enumerator parent across that
     # boundary; ordinary fiber-local span stacks remain isolated.
-    def with_fiber_bridge(trace_id:, span_id:, surface: nil)
+    def with_fiber_bridge(trace_id:, span_id:, surface: nil, trace_function_key: nil, trace_function_keys: nil)
       entry = {trace_id:, span_id:}
+      record_trace_function_keys(entry, trace_function_key, trace_function_keys)
       entry[:surface] = surface if surface
       fiber_bridge_stack.push(entry)
       yield
@@ -150,8 +171,9 @@ module Bitfab
 
     # Execute a block with a new span pushed onto the stack.
     # The span is automatically popped when the block completes.
-    def with_span(trace_id:, span_id:, explicit_span_receiver: nil, explicit_span_method_name: nil, surface: nil, recording_override: nil, content_withheld: nil, sends_on_completion: nil)
+    def with_span(trace_id:, span_id:, explicit_span_receiver: nil, explicit_span_method_name: nil, surface: nil, recording_override: nil, content_withheld: nil, sends_on_completion: nil, trace_function_key: nil)
       entry = {trace_id:, span_id:}
+      record_trace_function_keys(entry, trace_function_key)
       entry[:sends_on_completion] = sends_on_completion if sends_on_completion
       entry[:surface] = surface if surface
       entry[:recording_override] = recording_override if recording_override
@@ -166,6 +188,22 @@ module Bitfab
       stack.pop
     end
 
+    # The trace function keys running at this point with the given key added
+    # at the end, each listed once. A nil key returns the running keys as is.
+    def trace_function_keys_with(trace_function_key)
+      running = current&.dig(:trace_function_keys) || []
+      trace_function_key ? running | [trace_function_key] : running
+    end
+
+    # Store a span's own trace function key and the keys running at that span
+    # on its stack entry, computing the running keys from the current span
+    # when they are not given. Nothing is stored when no key is running.
+    def record_trace_function_keys(entry, trace_function_key, trace_function_keys = nil)
+      entry[:trace_function_key] = trace_function_key if trace_function_key
+      keys = trace_function_keys || trace_function_keys_with(trace_function_key)
+      entry[:trace_function_keys] = keys unless keys.empty?
+    end
+
     def fiber_bridge_stack
       current_thread = Thread.current
       current_thread.thread_variable_get(FIBER_BRIDGE_STACK_KEY) || begin
@@ -208,6 +246,10 @@ module Bitfab
       end
     end
 
+    def trace_function_key(trace_id)
+      get(trace_id)&.dig(:trace_function_key)
+    end
+
     def delete(trace_id)
       @states_mutex.synchronize { @states.delete(trace_id) }
     end

data/lib/bitfab/subtree.rb

@@ -740,7 +740,7 @@ module Bitfab
         return unless entry.is_a?(OpenSpan)
 
         exception = event.raised_exception
-        entry.error = "#{exception.class}: #{exception.message}"
+        entry.error = "#{exception.class}: #{ErrorChain.describe(exception)}"
       end
 
       def clear_rescued_error(event)

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.75.0"
+  VERSION = "0.75.2"
 end

data/lib/bitfab.rb

@@ -5,6 +5,7 @@ require "json"
 require_relative "bitfab/version"
 require_relative "bitfab/constants"
 require_relative "bitfab/warn_once"
+require_relative "bitfab/error_chain"
 require_relative "bitfab/experiment_id"
 require_relative "bitfab/payload_budget"
 require_relative "bitfab/serialize"
@@ -49,6 +50,10 @@ module Bitfab
       ""
     end
 
+    def trace_function_keys
+      []
+    end
+
     def add_context(_context)
       # No-op
     end
@@ -61,6 +66,10 @@ module Bitfab
   # No-op trace handle returned when outside a span context.
   # All methods do nothing, preventing crashes when called outside traced code.
   class NoOpCurrentTrace
+    def trace_function_key
+      ""
+    end
+
     def set_session_id(_session_id)
       # No-op
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.75.0
+  version: 0.75.2
 platform: ruby
 authors:
 - Harvest Team
@@ -183,6 +183,7 @@ files:
 - lib/bitfab/db_snapshot.rb
 - lib/bitfab/detached_trace.rb
 - lib/bitfab/env.rb
+- lib/bitfab/error_chain.rb
 - lib/bitfab/experiment_id.rb
 - lib/bitfab/experiments.rb
 - lib/bitfab/git_command.rb