bitfab 0.67.3 → 0.68.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d1b6f82e669d8a99caa8d5b2e94a9057456b4246d739cb80bade802d9bc4214
-  data.tar.gz: c429087a28e28243c339127692797f069dbec86a7c39c61547ee5510612710a3
+  metadata.gz: c05530ec639d246b474f368fae48df4ed2230e4275272f2d903c9ef81bc96b6b
+  data.tar.gz: 1019089f79f07cafb3b0e8be5604717ed32a1a894f28a7ae6b26d71cf05c1889
 SHA512:
-  metadata.gz: da254947e50206457558b7d897e47fa3fe9c53ce2a700299cec3eec4cd0e2546b5d363a33fc3fa12cd9a1c27cddf53dccf48b08281c81437156f86e7c543741e
-  data.tar.gz: 9d34b5617f75d6d5bcfbae28ced5eda7df437d4854e442d1f661a2053989ac5189cb98feeb0fc99224e0dcf70cc5b17348e892b6718bf486c27c804f14c3ca09
+  metadata.gz: 551641d9eb87d4c98d16d17cf5f6dcfe3f795f84a8fe1c155ba14d5db41bb5e53b04229ce0424ecac59430f854e79790c1460fb9fbc7ba10a2e0cb2bf739acd4
+  data.tar.gz: 28c329c02e65d57a75af5ae12895be94bc525dfa6ef01e2203a7485e3998a5d6c0e04209e2e78694f6085777bdad2009dba36c6d8ee4b7e75bb95bf8d98e6f0f

data/lib/bitfab/client.rb

@@ -1039,6 +1039,7 @@ module Bitfab
       content_off = !declared_node && (extra_span_data["content_off_by_simulation_plan"] == true ||
         content_off_by_simulation_plan?(trace_id:, trace_function_key:, span_name:, instrumentation:) ||
         (!parent_span_id.nil? && simulation_plan_unavailable?(instrumentation:)))
+      snapshot = nil
       if content_off
         dropped = []
         span_data = {"name" => span_name, "type" => span_type, "function_name" => function_name}
@@ -1049,10 +1050,6 @@ module Bitfab
         human_output, output_dropped = Serialize.serialize_value_with_report(result)
         dropped = input_dropped + output_dropped
 
-        raw_input = (args.length == 1 && kwargs.empty?) ? args[0] : {args:, kwargs:}
-        marshalled_input = Serialize.marshal_value(raw_input)
-        marshalled_output = Serialize.marshal_value(result)
-
         span_data = {
           "name" => span_name,
           "type" => span_type,
@@ -1061,8 +1058,13 @@ module Bitfab
           "function_name" => function_name
         }
         span_data.merge!(extra_span_data)
-        span_data["input_serialized"] = marshalled_input if marshalled_input
-        span_data["output_serialized"] = marshalled_output if marshalled_output
+        snapshot = SpanMeta::Snapshot.capture(args, kwargs, result)
+        unless snapshot
+          input_meta = SpanMeta.build_input_meta(args, kwargs, human_inputs)
+          output_meta = SpanMeta.build_output_meta(result, human_output)
+          span_data["input_meta"] = input_meta if input_meta
+          span_data["output_meta"] = output_meta if output_meta
+        end
       end
       if error
         span_data["error"] = error
@@ -1120,7 +1122,7 @@ module Bitfab
         }]
       end
 
-      @http_client.send_external_span(payload, declared_node)
+      @http_client.send_external_span(payload, declared_node, snapshot)
     end
 
     # Advance the per-(key, name) call counter when this invocation is a

data/lib/bitfab/experiments.rb

@@ -45,6 +45,15 @@ module Bitfab
       @http_client.get("#{PATH}/rollups?#{query}")["rollups"]
     end
 
+    # Reads one run's written summary. "status" is "pending" while the run has not finished or its verdicts are
+    # still arriving; the summary is written once every verdict is in. "unavailable" means the run ended failed or
+    # interrupted and no summary will be written. "ready" carries the summary, compared against the first finished
+    # run in the same experiment group for the same traced function, or against the run's original traces when
+    # there is none.
+    def get_summary(id)
+      @http_client.get("#{PATH}/#{encode(id)}/summary")
+    end
+
     private
 
     def iso8601(value)

data/lib/bitfab/http_client.rb

@@ -193,7 +193,7 @@ module Bitfab
     end
 
     # Queue an external span on this client's trace transport (fire-and-forget).
-    def send_external_span(payload, declared_node = false)
+    def send_external_span(payload, declared_node = false, snapshot = nil)
       if (data = payload.dig("rawSpan", "span_data")).is_a?(Hash) && !data.key?("runtime")
         payload = payload.merge("rawSpan" => payload["rawSpan"].merge("span_data" => data.merge("runtime" => ThreadPropagation.runtime)))
       end
@@ -203,11 +203,16 @@ module Bitfab
       submit = ->(record) {
         ref = carrier_ref(record)
         @trace_completion.record(ref.trace_id, ref.span_id) if ref&.span_id
-        trace_transport&.submit(
-          "external_span",
-          record.merge("sdkVersion" => VERSION),
-          recorded_meta("external_span", record, ref)
-        )
+        transport = trace_transport
+        carrier = record.merge("sdkVersion" => VERSION)
+        meta = recorded_meta("external_span", record, ref)
+        if snapshot.nil? || !snapshot.content?(carrier)
+          transport&.submit("external_span", carrier, meta)
+        elsif transport.respond_to?(:submit_with_snapshot)
+          transport.submit_with_snapshot("external_span", carrier, meta, snapshot)
+        else
+          transport&.submit("external_span", snapshot.apply(carrier), meta)
+        end
       }
       discard = ->(record) {
         ref = carrier_ref(record)

data/lib/bitfab/otel.rb

@@ -492,6 +492,94 @@ module Bitfab
     # Owns one private TracerProvider + BatchSpanProcessor per client. The
     # provider is never installed globally, so the SDK cannot disturb an
     # application's own OpenTelemetry setup.
+    class SpanWorker
+      MAX_PENDING = 256
+      MAX_PENDING_BYTES = 64 * 1024 * 1024
+
+      def initialize
+        reset
+      end
+
+      def enqueue(bytes = 0, &job)
+        reset_after_fork
+        queued = @mutex.synchronize do
+          next false if @queue.closed? || @pending >= MAX_PENDING
+          next false if @pending.positive? && @pending_bytes + bytes > MAX_PENDING_BYTES
+
+          @pending += 1
+          @pending_bytes += bytes
+          @queue << [job, bytes]
+          @thread = Thread.new { work } unless @thread&.alive?
+          true
+        end
+        run(job) unless queued
+      end
+
+      def after_pending(&job)
+        reset_after_fork
+        return run(job) if @mutex.synchronize { @pending.zero? }
+
+        enqueue(&job)
+      end
+
+      def drain(timeout)
+        reset_after_fork
+        deadline = Otel.monotonic_now + [timeout, 0].max
+        @mutex.synchronize do
+          while @pending.positive?
+            remaining = deadline - Otel.monotonic_now
+            return false unless remaining.positive?
+
+            @idle.wait(@mutex, remaining)
+          end
+          true
+        end
+      end
+
+      def stop
+        @queue.close
+      end
+
+      private
+
+      def reset
+        @owner_pid = Process.pid
+        @mutex = Mutex.new
+        @idle = ConditionVariable.new
+        @queue = Thread::Queue.new
+        @pending = 0
+        @pending_bytes = 0
+        @thread = nil
+      end
+
+      def reset_after_fork
+        reset unless @owner_pid == Process.pid
+      end
+
+      def work
+        while (entry = @queue.pop)
+          job, bytes = entry
+          run(job)
+          @mutex.synchronize do
+            @pending -= 1
+            @pending_bytes -= bytes
+            @idle.broadcast if @pending.zero?
+          end
+        end
+      end
+
+      def run(job)
+        job.call
+      rescue => e
+        Bitfab.warn_once(
+          "span-worker-failed",
+          "failed to finish a span in the background (further occurrences " \
+          "suppressed): #{e.message}"
+        )
+        nil
+      end
+    end
+
     class BatchTransport
       attr_reader :owner_pid
 
@@ -510,57 +598,40 @@ module Bitfab
         @owner_pid = Process.pid
         @state_mutex = Mutex.new
         @flush_mutex = Mutex.new
+        @span_worker = SpanWorker.new
         @closed = false
         create_pipeline
         Otel.register_transport(self)
       end
 
       def submit(operation, payload, meta = nil)
-        meta ||= CarrierMeta.new
+        meta = stamped(meta)
         # Encoding is the expensive part of a submit and needs no mutual
         # exclusion, so it stays outside the lock.
-        name = meta.name || "bitfab.#{operation}"
-        encoded_payload = Serialize.safe_generate(
-          payload,
-          max_carrier_bytes: PayloadBudget::MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES
-        )
-        encoded_payload.instance_variable_set(CARRIER_REF_IVAR, meta.ref) if meta.ref
-        started_at = meta.started_at || Time.now
-        ended_at = meta.ended_at || Time.now
-        errored = meta.errored
-
-        # Pipeline construction and the closed check are serialized: concurrent
-        # first submits in a forked child would otherwise each build a pipeline,
-        # orphaning one batch worker along with whatever it had queued.
-        @state_mutex.synchronize do
-          return warn_closed if @closed
-
-          ensure_process
-          tracer = @tracer
-          return warn_closed if tracer.nil?
+        encoded_payload = encode_payload(payload, meta)
+        @span_worker.after_pending { emit(operation, encoded_payload, meta) }
+        nil
+      rescue => e
+        warn_submit_failed(e)
+      end
 
-          span = tracer.start_root_span(
-            name,
-            attributes: {
-              OPERATION_ATTRIBUTE => operation,
-              PAYLOAD_ATTRIBUTE => encoded_payload
-            },
-            start_timestamp: started_at
-          )
-          span.status = OpenTelemetry::Trace::Status.error if errored
-          span.finish(end_timestamp: ended_at)
+      def submit_with_snapshot(operation, payload, meta, snapshot)
+        meta = stamped(meta)
+        encoded_payload = encode_payload(payload, meta)
+        @span_worker.enqueue(encoded_payload.bytesize + snapshot.bytesize) do
+          completed = complete_with_snapshot(encoded_payload, snapshot)
+          emit(operation, completed ? encode_payload(completed, meta) : encoded_payload, meta)
         end
         nil
       rescue => e
-        Bitfab.warn_once(
-          "otel-submit-failed",
-          "failed to queue an OpenTelemetry span (further occurrences " \
-          "suppressed): #{e.message}"
-        )
-        nil
+        warn_submit_failed(e)
       end
 
       def flush(timeout = 30.0)
+        deadline = Otel.monotonic_now + [timeout, 0].max
+        return false unless @span_worker.drain(timeout)
+
+        timeout = [deadline - Otel.monotonic_now, 0].max
         processor, delivery_tracker = @state_mutex.synchronize do
           ensure_process
           [@processor, @delivery_tracker]
@@ -591,8 +662,10 @@ module Bitfab
 
       def shutdown(timeout = 30.0)
         deadline = Otel.monotonic_now + [timeout, 0].max
+        drained = @span_worker.drain(timeout)
         @state_mutex.synchronize { @closed = true }
-        flushed = flush([deadline - Otel.monotonic_now, 0].max)
+        @span_worker.stop
+        flushed = flush([deadline - Otel.monotonic_now, 0].max) && drained
 
         processor = @state_mutex.synchronize do
           current = @processor
@@ -611,6 +684,68 @@ module Bitfab
 
       private
 
+      def stamped(meta)
+        meta = (meta || CarrierMeta.new).dup
+        now = Time.now
+        meta.started_at ||= now
+        meta.ended_at ||= now
+        meta
+      end
+
+      def encode_payload(payload, meta)
+        encoded = Serialize.safe_generate(
+          payload,
+          max_carrier_bytes: PayloadBudget::MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES
+        )
+        encoded.instance_variable_set(CARRIER_REF_IVAR, meta.ref) if meta.ref
+        encoded
+      end
+
+      def complete_with_snapshot(encoded_payload, snapshot)
+        snapshot.apply(JSON.parse(encoded_payload))
+      rescue StandardError, SystemStackError => e
+        Bitfab.warn_once(
+          "span-meta-snapshot-failed",
+          "could not record replay type details for a span; it will replay " \
+          "from its readable value (further occurrences suppressed): #{e.message}"
+        )
+        nil
+      end
+
+      # Pipeline construction and the closed check are serialized: concurrent
+      # first submits in a forked child would otherwise each build a pipeline,
+      # orphaning one batch worker along with whatever it had queued.
+      def emit(operation, encoded_payload, meta)
+        @state_mutex.synchronize do
+          return warn_closed if @closed
+
+          ensure_process
+          tracer = @tracer
+          return warn_closed if tracer.nil?
+
+          span = tracer.start_root_span(
+            meta.name || "bitfab.#{operation}",
+            attributes: {
+              OPERATION_ATTRIBUTE => operation,
+              PAYLOAD_ATTRIBUTE => encoded_payload
+            },
+            start_timestamp: meta.started_at
+          )
+          span.status = OpenTelemetry::Trace::Status.error if meta.errored
+          span.finish(end_timestamp: meta.ended_at)
+        end
+        nil
+      end
+
+      def warn_submit_failed(error)
+        Bitfab.warn_once(
+          "otel-submit-failed",
+          "failed to queue an OpenTelemetry span (further occurrences " \
+          "suppressed): #{error.message}"
+        )
+        nil
+      end
+
       def finish_shutdown(succeeded)
         Otel.unregister_transport(self)
         succeeded

data/lib/bitfab/replay.rb

@@ -1278,8 +1278,6 @@ module Bitfab
 
         span = http_client.get_external_span(external_span_id, replay_view: true)
         span_data = (span["rawData"] || {})["span_data"] || {}
-        # Prefer the Ruby Marshal payload (output_serialized) written by this
-        # SDK; fall back to another SDK's output_meta, then the raw JSON output.
         meta = span_data["output_serialized"]
         meta = span_data["output_meta"] if meta.nil? || meta == ""
         cache[external_span_id] = Serialize.deserialize_output(span_data["output"], meta)
@@ -1294,6 +1292,7 @@ module Bitfab
       {
         "input" => span_data["input"],
         "output" => span_data["output"],
+        "inputMeta" => span_data["input_meta"],
         "inputSerialized" => span_data["input_serialized"],
         "outputSerialized" => span_data["output_serialized"]
       }

data/lib/bitfab/seed.rb

@@ -99,7 +99,7 @@ module Bitfab
       unless source.fetch("traceFunctionKey") == trace_function_key
         raise ArgumentError, "Trace #{trace_id} belongs to '#{source["traceFunctionKey"]}', not '#{trace_function_key}'."
       end
-      args, kwargs = Serialize.deserialize_inputs({"input" => source["input"], "input_serialized" => source["inputSerialized"]})
+      args, kwargs = Serialize.deserialize_inputs({"input" => source["input"], "inputSerialized" => source["inputSerialized"], "inputMeta" => source["inputMeta"]})
       run_trace_id = seed_trace(trace_function_key, callable, args:, kwargs:,
         metadata: source["metadata"], session_id: source["sessionId"], name: source["name"])
       adopted = @http_client.request(endpoint, {runTraceId: run_trace_id})

data/lib/bitfab/serialize.rb

@@ -5,6 +5,7 @@ require "json"
 require "time"
 
 require_relative "payload_budget"
+require_relative "span_meta"
 
 module Bitfab
   module Serialize
@@ -196,35 +197,19 @@ module Bitfab
       Marshal.load(Base64.strict_decode64(encoded)) # rubocop:disable Security/MarshalLoad
     end
 
-    # Deserialize a recorded span output. Prefers the Ruby-side Marshal+Base64
-    # `meta` when present (type-preserving); falls back to the raw JSON output
-    # when meta is absent or a shape Ruby cannot reconstruct (e.g. the
-    # superjson/jsonpickle meta another SDK's span may carry). Shared by the
-    # inline mock-tree path and the lazy per-span fetch so both deserialize
-    # identically.
-    #
-    # @param raw_output [Object] the human-readable JSON output
-    # @param meta [String, nil] the Marshal+Base64 metadata, if any
-    # @return [Object] the reconstructed output, or raw_output on fallback
     def deserialize_output(raw_output, meta)
-      if meta.is_a?(String) && !meta.empty?
-        begin
-          return unmarshal_value(meta)
-        rescue
-          # Fall through to the raw JSON output
-        end
-      end
+      return SpanMeta.restore_output(raw_output, meta) if meta.is_a?(Hash)
+      return raw_output unless meta.is_a?(String) && !meta.empty?
+
+      unmarshal_value(meta)
+    rescue
       raw_output
     end
 
-    # Deserialize replay inputs from a span's data into [args, kwargs].
-    #
-    # Prefers Marshal-serialized `inputSerialized` for type preservation,
-    # falls back to the raw `input` field.
-    #
-    # @param item [Hash] with "inputSerialized" and/or "input" keys
-    # @return [Array(Array, Hash)] positional args and keyword args
     def deserialize_inputs(item)
+      restored = SpanMeta.restore_inputs(item["input"], item["inputMeta"])
+      return restored if restored
+
       input_serialized = item["inputSerialized"]
       raw_input = item["input"]
 
@@ -236,7 +221,7 @@ module Bitfab
           end
           return deserialized.nil? ? [[], {}] : [[deserialized], {}]
         rescue
-          # Fall through to raw_input
+          nil
         end
       end
 

data/lib/bitfab/span_meta.rb

@@ -0,0 +1,635 @@
+# frozen_string_literal: true
+
+require "date"
+require "json"
+require "time"
+
+module Bitfab
+  module SpanMeta
+    MAX_NODES = 1_000_000
+    PLAIN_ENCODINGS = [Encoding::UTF_8, Encoding::US_ASCII].freeze
+
+    class NeedsCopy < StandardError; end
+
+    class GiveUp < StandardError; end
+
+    class Cycle < StandardError
+      attr_reader :target
+
+      def initialize(target)
+        @target = target
+        super("value refers back to #{target.inspect}")
+      end
+    end
+
+    SpanType = Struct.new(:name, :matches, :describe, :rebuild)
+
+    PathTrie = Struct.new(:children, :annotation, :copy, :same, :store, :symbol)
+
+    extend self
+
+    def build_input_meta(args, kwargs, readable_input)
+      meta = {}
+      meta["kwarg_names"] = kwargs.keys.map { |name| Serialize.safe_to_s(name) } unless kwargs.empty?
+      root = kwargs.empty? ? args.dup : args + [kwargs]
+      symbol_paths = kwargs.empty? ? [] : [[args.length.to_s]]
+      meta.merge!(value_meta(root, readable_input, symbol_paths))
+      meta.empty? ? nil : meta
+    end
+
+    def build_output_meta(value, readable_output)
+      meta = value_meta(value, readable_output, [])
+      meta.empty? ? nil : meta
+    end
+
+    def restore_inputs(raw_input, input_meta)
+      return nil unless input_meta.is_a?(Hash) && raw_input.is_a?(Array)
+
+      kwargs_path = input_meta["kwarg_names"]&.any? ? [(raw_input.length - 1).to_s] : nil
+      root = restore(raw_input, input_meta, kwargs_path ? [kwargs_path] : [])
+      return nil unless root.is_a?(Array)
+
+      args = root.dup
+      return [args, {}] unless kwargs_path
+
+      kwargs = args.pop
+      kwargs.is_a?(Hash) ? [args, kwargs] : nil
+    rescue
+      nil
+    end
+
+    def restore_output(raw_output, output_meta)
+      return raw_output unless output_meta.is_a?(Hash)
+
+      restore(raw_output, output_meta, [])
+    rescue
+      raw_output
+    end
+
+    def value_meta(value, readable, symbol_paths)
+      recorder = Recorder.new(symbol_paths)
+      recorder.walk(value, readable, [])
+      meta = recorder.meta
+      raise GiveUp if meta.key?("v") && !restores_exactly?(value, readable, meta, recorder.lossy, symbol_paths)
+
+      meta
+    rescue StandardError, SystemStackError
+      copy = Serialize.marshal_value(value)
+      copy ? {"copy" => copy} : {}
+    end
+
+    def restores_exactly?(value, readable, meta, lossy, symbol_paths)
+      restored = restore(JSON.parse(JSON.generate(readable)), JSON.parse(JSON.generate(meta)), symbol_paths)
+      Exact.new(lossy).same?(value, restored, [])
+    rescue StandardError, SystemStackError
+      false
+    end
+
+    def tracks_identity?(value)
+      !(value.nil? || true.equal?(value) || false.equal?(value) || value.is_a?(Numeric) ||
+        value.is_a?(Symbol) || value.is_a?(Module) || value.frozen? || value.is_a?(String) || value.is_a?(Time) || value.is_a?(Date))
+    end
+
+    def require!(condition)
+      raise NeedsCopy unless condition
+    end
+
+    def class_name_for(klass)
+      name = klass.name
+      raise NeedsCopy if name.nil? || name.empty?
+
+      name
+    end
+
+    def symbol_keys_note(hash, symbol_count, symbol_default)
+      kind = if symbol_count.zero?
+        "string"
+      elsif symbol_count == hash.size
+        "symbol"
+      else
+        hash.keys.select { |key| key.is_a?(Symbol) }.map(&:name)
+      end
+      default = symbol_default ? "symbol" : "string"
+      (kind == default) ? nil : kind
+    end
+
+    def plain_string?(value, readable)
+      return false unless readable.is_a?(String) && readable == value && value.instance_variables.empty?
+
+      case value.encoding
+      when Encoding::UTF_8 then value.valid_encoding?
+      when Encoding::US_ASCII then value.ascii_only?
+      else false
+      end
+    end
+
+    class Snapshot
+      def self.capture(args, kwargs, result)
+        new(Marshal.dump([args, kwargs, result]))
+      rescue StandardError, SystemStackError
+        nil
+      end
+
+      def initialize(bytes)
+        @bytes = bytes
+      end
+
+      def bytesize
+        @bytes.bytesize
+      end
+
+      def content?(payload)
+        data = payload.dig("rawSpan", "span_data")
+        data.is_a?(Hash) && !data["content_off_by_simulation_plan"] && (data.key?("input") || data.key?("output"))
+      end
+
+      def apply(payload)
+        return payload unless content?(payload)
+
+        data = payload["rawSpan"]["span_data"]
+        args, kwargs, result = Marshal.load(@bytes) # rubocop:disable Security/MarshalLoad
+        input_meta = SpanMeta.build_input_meta(args, kwargs, data["input"]) if data.key?("input")
+        output_meta = SpanMeta.build_output_meta(result, data["output"]) if data.key?("output")
+        data["input_meta"] = input_meta if input_meta
+        data["output_meta"] = output_meta if output_meta
+        payload
+      end
+    end
+
+    class Node
+      def initialize(recorder, path)
+        @recorder = recorder
+        @path = path
+      end
+
+      def walk(child, readable, key)
+        @recorder.walk(child, readable, @path + [key.to_s])
+      end
+
+      def walk_items(items, readable)
+        SpanMeta.require!(readable.is_a?(Array) && readable.length == items.length)
+        items.each_with_index { |item, index| walk(item, readable[index], index) }
+      end
+
+      def symbol_keys_default?
+        @recorder.symbol_path?(@path)
+      end
+    end
+
+    class Recorder
+      attr_reader :lossy
+
+      def initialize(symbol_paths)
+        @symbol_paths = symbol_paths
+        @values = {}
+        @same = []
+        @copies = {}
+        @seen = {}.compare_by_identity
+        @dropped = []
+        @lossy = []
+        @nodes = 0
+      end
+
+      def symbol_path?(path)
+        @symbol_paths.include?(path)
+      end
+
+      def meta
+        meta = {}
+        meta["values"] = SpanMeta.values_tree(@values) unless @values.empty?
+        meta["referentialEqualities"] = SpanMeta.equalities_record(@same) unless @same.empty?
+        meta["v"] = 1 unless meta.empty?
+        if @copies.values.sum(&:bytesize) > Serialize::MAX_SERIALIZED_BYTES
+          @lossy.concat(@copies.keys)
+        elsif @copies.key?([])
+          meta["copy"] = @copies[[]]
+        elsif !@copies.empty?
+          meta["copies"] = @copies.transform_keys { |path| SpanMeta.stringify_path(path) }
+        end
+        meta
+      end
+
+      def walk(value, readable, path)
+        @nodes += 1
+        raise GiveUp if @nodes > MAX_NODES
+
+        values_mark = @values.size
+        same_mark = @same.size
+        copies_mark = @copies.size
+        lossy_mark = @lossy.size
+        if SpanMeta.tracks_identity?(value)
+          first = @seen[value]
+          if first && !dropped?(first)
+            raise Cycle.new(first) if path.first(first.length) == first
+
+            @same << [path, first]
+            return
+          end
+          @seen[value] = path
+        end
+        annotate(value, readable, path)
+      rescue GiveUp
+        raise
+      rescue Cycle => cycle
+        raise unless cycle.target == path
+
+        copy_here(value, path, values_mark, same_mark, copies_mark, lossy_mark)
+      rescue StandardError, SystemStackError
+        copy_here(value, path, values_mark, same_mark, copies_mark, lossy_mark)
+      end
+
+      private
+
+      def copy_here(value, path, values_mark, same_mark, copies_mark, lossy_mark)
+        drop_after(values_mark, same_mark, copies_mark, lossy_mark)
+        @dropped << path
+        copy = Serialize.marshal_value(value)
+        copy ? @copies[path] = copy : @lossy << path
+      end
+
+      def drop_after(values_size, same_size, copies_size, lossy_size)
+        @values.keys.drop(values_size).each { |key| @values.delete(key) } if @values.size > values_size
+        @same.slice!(same_size..) if @same.size > same_size
+        @copies.keys.drop(copies_size).each { |key| @copies.delete(key) } if @copies.size > copies_size
+        @lossy.slice!(lossy_size..) if @lossy.size > lossy_size
+      end
+
+      def dropped?(path)
+        @dropped.any? { |prefix| path.first(prefix.length) == prefix }
+      end
+
+      def annotate(value, readable, path)
+        case value
+        when nil, true, false
+          SpanMeta.require!(readable.equal?(value))
+        when Integer
+          SpanMeta.require!(readable.is_a?(Integer) && readable == value)
+        when Float
+          SpanMeta.require!(value.finite? && readable.is_a?(Float) && readable == value)
+        else
+          return if value.instance_of?(String) && SpanMeta.plain_string?(value, readable)
+          return Node.new(self, path).walk_items(value, readable) if value.instance_of?(Array)
+
+          span_type = value.instance_of?(Hash) ? SPAN_TYPES_BY_NAME["rb:hash"] : SPAN_TYPES.find { |entry| entry.matches.call(value) }
+          raise NeedsCopy unless span_type
+
+          params = span_type.describe.call(value, readable, Node.new(self, path))
+          @values[path] = ["custom", span_type.name] + (params.empty? ? [] : [params]) if params
+        end
+      end
+    end
+
+    def describe_symbol(value, readable, _node)
+      require!(readable == value.to_s)
+      {}
+    end
+
+    def describe_string(value, readable, _node)
+      require!(readable.is_a?(String) && readable == value && value.dup.force_encoding(Encoding::UTF_8).valid_encoding?)
+      require!(value.instance_variables.empty?)
+      params = {}
+      params["class"] = class_name_for(value.class) unless value.instance_of?(String)
+      params["encoding"] = value.encoding.name unless PLAIN_ENCODINGS.include?(value.encoding)
+      params.empty? ? nil : params
+    end
+
+    def describe_array(value, readable, node)
+      node.walk_items(value, readable)
+      value.instance_of?(Array) ? nil : {"class" => class_name_for(value.class)}
+    end
+
+    def describe_hash(value, readable, node)
+      require!(value.default.nil? && value.default_proc.nil? && !value.compare_by_identity?)
+      require!(readable.is_a?(Hash) && readable.size == value.size)
+      readable_keys = readable.keys
+      symbol_count = 0
+      value.each_with_index do |(key, item), index|
+        name = case key
+        when String then key
+        when Symbol
+          symbol_count += 1
+          key.name
+        else raise NeedsCopy
+        end
+        require!(readable_keys[index] == name)
+        node.walk(item, readable[name], name)
+      end
+      params = {}
+      params["class"] = class_name_for(value.class) unless value.instance_of?(Hash)
+      keys = symbol_keys_note(value, symbol_count, node.symbol_keys_default?)
+      params["keys"] = keys if keys
+      params.empty? ? nil : params
+    end
+
+    def describe_set(value, readable, node)
+      require!(value.instance_of?(Set))
+      node.walk_items(value.to_a, readable)
+      {}
+    end
+
+    def describe_time(value, readable, _node)
+      require!(value.instance_of?(Time) && readable == value.iso8601(3))
+      {"nsec" => value.nsec}
+    end
+
+    def describe_date(value, readable, _node)
+      require!(value.instance_of?(Date) && readable == value.to_s)
+      {}
+    end
+
+    def describe_bigdecimal(value, readable, _node)
+      require!(readable == value.to_s)
+      {}
+    end
+
+    def describe_struct(value, readable, node)
+      members = value.members
+      require!(readable.is_a?(Hash) && readable.keys == members.map(&:to_s))
+      members.each { |member| node.walk(value[member], readable[member.to_s], member) }
+      {"class" => class_name_for(value.class)}
+    end
+
+    def rebuild_array(built, params)
+      Array.instance_method(:replace).bind_call(resolve(params["class"]).allocate, built)
+    end
+
+    def rebuild_string(built, params)
+      text = params["class"] ? String.instance_method(:replace).bind_call(resolve(params["class"]).allocate, built) : built.dup
+      params["encoding"] ? text.force_encoding(params["encoding"]) : text
+    end
+
+    def rebuild_hash(built, params)
+      keys = params["keys"]
+      hash = case keys
+      when "symbol" then built.transform_keys(&:to_sym)
+      when Array then built.transform_keys { |key| keys.include?(key) ? key.to_sym : key }
+      else built
+      end
+      return hash unless params["class"]
+
+      Hash.instance_method(:replace).bind_call(resolve(params["class"]).allocate, hash)
+    end
+
+    def rebuild_time(built, params)
+      base = begin
+        Time.new(built)
+      rescue ArgumentError
+        Time.iso8601(built)
+      end
+      zone = built.end_with?("Z") ? "UTC" : base.strftime("%:z")
+      Time.at(base.to_i, params["nsec"], :nsec, in: zone)
+    end
+
+    def rebuild_struct(built, params)
+      klass = resolve(params["class"])
+      struct = klass.allocate
+      klass.members.each { |member| struct[member] = built.fetch(member.to_s) }
+      struct
+    end
+
+    def resolve(name)
+      Object.const_get(name)
+    end
+
+    def stringify_path(path)
+      path.map { |segment| escape_segment(segment) }.join(".")
+    end
+
+    def escape_segment(segment)
+      return segment unless segment.include?(".") || segment.include?("\\")
+
+      segment.gsub("\\") { "\\\\" }.gsub(".") { "\\." }
+    end
+
+    def parse_path(text)
+      return (text.empty? ? [""] : text.split(".", -1)) unless text.include?("\\")
+
+      segments = []
+      segment = +""
+      index = 0
+      while index < text.length
+        char = text[index]
+        if char == "\\"
+          escaped = text[index + 1]
+          raise ArgumentError, "invalid path #{text.inspect}" unless escaped == "\\" || escaped == "."
+
+          segment << escaped
+          index += 2
+          next
+        end
+        if char == "."
+          segments << segment
+          segment = +""
+        else
+          segment << char
+        end
+        index += 1
+      end
+      segments << segment
+    end
+
+    def values_tree(annotations)
+      root = {}
+      annotations.each_key do |path|
+        node = path.reduce(root) { |parent, segment| parent[segment] ||= {} }
+        node[:path] = path
+      end
+      tree_for = nil
+      collect = lambda do |node, prefix, out|
+        node.each do |segment, child|
+          next unless segment.is_a?(String)
+
+          relative = prefix + [segment]
+          if child.key?(:path)
+            out[stringify_path(relative)] = tree_for.call(child)
+          else
+            collect.call(child, relative, out)
+          end
+        end
+        out
+      end
+      tree_for = lambda do |node|
+        kids = collect.call(node, [], {})
+        kids.empty? ? [annotations[node[:path]]] : [annotations[node[:path]], kids]
+      end
+      root.key?(:path) ? tree_for.call(root) : collect.call(root, [], {})
+    end
+
+    def flatten_values(tree, origin = [], out = {})
+      case tree
+      when Hash
+        tree.each { |key, child| flatten_values(child, origin + parse_path(key), out) }
+      when Array
+        unless tree.empty?
+          flatten_values(tree[1], origin, out) if tree[1].is_a?(Hash)
+          out[origin] = tree[0]
+        end
+      end
+      out
+    end
+
+    def equalities_record(same)
+      same.each_with_object({}) do |(path, first), record|
+        (record[stringify_path(first)] ||= []) << stringify_path(path)
+      end
+    end
+
+    def parse_equalities(annotations)
+      annotations = annotations[1] if annotations.is_a?(Array)
+      return {} unless annotations.is_a?(Hash)
+
+      annotations.each_with_object({}) do |(first, paths), out|
+        next unless paths.is_a?(Array)
+
+        paths.each { |path| out[parse_path(path)] = parse_path(first) if path.is_a?(String) }
+      end
+    end
+
+    def restore_trie(meta, symbol_paths)
+      root = PathTrie.new({})
+      insert = lambda do |path|
+        path.reduce(root) { |node, segment| node.children[segment] ||= PathTrie.new({}) }
+      end
+      flatten_values(meta["values"]).each { |path, annotation| insert.call(path).annotation = annotation }
+      parse_equalities(meta["referentialEqualities"]).each do |path, target|
+        insert.call(path).same = target
+        insert.call(target).store = target
+      end
+      meta["copies"].each { |path, copy| insert.call(parse_path(path)).copy = [copy] } if meta["copies"].is_a?(Hash)
+      symbol_paths.each { |path| insert.call(path).symbol = true }
+      root
+    end
+
+    def restore(readable, meta, symbol_paths)
+      return unmarshal_or_keep(meta["copy"], readable) if meta.key?("copy")
+
+      trie = restore_trie(meta, symbol_paths)
+      return readable if trie.children.empty? && trie.annotation.nil? && !trie.symbol
+
+      built = {}
+      build = lambda do |node, trie_node|
+        return node if trie_node.nil?
+
+        result = if trie_node.copy
+          unmarshal_or_keep(trie_node.copy.first, node)
+        elsif trie_node.same
+          built.fetch(trie_node.same, node)
+        else
+          children = if trie_node.children.empty?
+            node
+          else
+            case node
+            when Array then node.each_with_index.map { |item, index| build.call(item, trie_node.children[index.to_s]) }
+            when Hash then node.to_h { |key, item| [key, build.call(item, trie_node.children[key])] }
+            else node
+            end
+          end
+          apply_or_keep(annotation_for(trie_node.annotation, children, trie_node.symbol), children)
+        end
+        built[trie_node.store] = result if trie_node.store
+        result
+      end
+      build.call(readable, trie)
+    end
+
+    def annotation_for(annotation, children, symbol_default)
+      return annotation unless children.is_a?(Hash)
+      return annotation if annotation.nil? && !symbol_default
+
+      annotation ||= ["custom", "rb:hash"]
+      return annotation unless annotation[1] == "rb:hash"
+
+      params = annotation[2].is_a?(Hash) ? annotation[2] : {}
+      return annotation if params.key?("keys")
+
+      ["custom", "rb:hash", params.merge("keys" => symbol_default ? "symbol" : "string")]
+    end
+
+    def apply_or_keep(annotation, built)
+      return built unless annotation.is_a?(Array) && annotation[0] == "custom"
+
+      span_type = SPAN_TYPES_BY_NAME[annotation[1]]
+      return built unless span_type
+
+      span_type.rebuild.call(built, annotation[2].is_a?(Hash) ? annotation[2] : {})
+    rescue
+      built
+    end
+
+    def unmarshal_or_keep(encoded, readable)
+      return readable unless encoded.is_a?(String) && !encoded.empty?
+
+      Serialize.unmarshal_value(encoded)
+    rescue
+      readable
+    end
+
+    SPAN_TYPES = [
+      SpanType.new("rb:symbol", ->(value) { value.is_a?(Symbol) }, method(:describe_symbol), ->(built, _) { built.to_sym }),
+      SpanType.new("rb:string", ->(value) { value.is_a?(String) }, method(:describe_string), method(:rebuild_string)),
+      SpanType.new("rb:array", ->(value) { value.is_a?(Array) }, method(:describe_array), method(:rebuild_array)),
+      SpanType.new("rb:hash", ->(value) { value.is_a?(Hash) }, method(:describe_hash), method(:rebuild_hash)),
+      SpanType.new("rb:set", ->(value) { value.is_a?(Set) }, method(:describe_set), ->(built, _) { Set.new(built) }),
+      SpanType.new("rb:time", ->(value) { value.is_a?(Time) }, method(:describe_time), method(:rebuild_time)),
+      SpanType.new("rb:date", ->(value) { value.is_a?(Date) }, method(:describe_date), ->(built, _) { Date.iso8601(built) }),
+      SpanType.new("rb:bigdecimal", ->(value) { defined?(::BigDecimal) && value.is_a?(::BigDecimal) }, method(:describe_bigdecimal), ->(built, _) { BigDecimal(built) }),
+      SpanType.new("rb:struct", ->(value) { value.is_a?(Struct) }, method(:describe_struct), method(:rebuild_struct))
+    ].freeze
+
+    SPAN_TYPES_BY_NAME = SPAN_TYPES.to_h { |span_type| [span_type.name, span_type] }.freeze
+
+    class Exact
+      def initialize(lossy)
+        @lossy = lossy
+        @tracks_paths = !lossy.empty?
+        @forward = {}.compare_by_identity
+        @backward = {}.compare_by_identity
+      end
+
+      def same?(a, b, path)
+        return true if @tracks_paths && @lossy.include?(path)
+        return false unless a.instance_of?(b.class)
+
+        if SpanMeta.tracks_identity?(a)
+          return false unless @forward.fetch(a, b).equal?(b) && @backward.fetch(b, a).equal?(a)
+          return true if @forward.key?(a)
+
+          @forward[a] = b
+          @backward[b] = a
+        end
+        same_value?(a, b, path) && same_ivars?(a, b, path)
+      end
+
+      private
+
+      def same_value?(a, b, path)
+        case a
+        when Float then a == b || (a.nan? && b.nan?)
+        when String then a == b && (a.encoding == b.encoding || (a.ascii_only? && PLAIN_ENCODINGS.include?(a.encoding) && PLAIN_ENCODINGS.include?(b.encoding)))
+        when Array then a.length == b.length && a.each_with_index.all? { |item, index| same?(item, b[index], child(path, index)) }
+        when Hash then same_hash?(a, b, path)
+        when Time then a == b && a.utc_offset == b.utc_offset && a.utc? == b.utc? && a.zone == b.zone
+        when Struct then a.members.all? { |member| same?(a[member], b[member], child(path, member)) }
+        else
+          a.method(:==).owner == BasicObject || a == b
+        end
+      end
+
+      def same_hash?(a, b, path)
+        a.keys == b.keys && a.default == b.default && a.default_proc == b.default_proc &&
+          a.compare_by_identity? == b.compare_by_identity? &&
+          a.all? { |key, item| same?(item, b[key], child(path, key)) }
+      end
+
+      def same_ivars?(a, b, path)
+        names = a.instance_variables
+        names == b.instance_variables &&
+          names.all? { |name| same?(a.instance_variable_get(name), b.instance_variable_get(name), child(path, name)) }
+      end
+
+      def child(path, segment)
+        @tracks_paths ? path + [segment.to_s] : path
+      end
+    end
+  end
+end

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.67.3"
+  VERSION = "0.68.0"
 end

data/lib/bitfab.rb

@@ -8,6 +8,7 @@ require_relative "bitfab/warn_once"
 require_relative "bitfab/experiment_id"
 require_relative "bitfab/payload_budget"
 require_relative "bitfab/serialize"
+require_relative "bitfab/span_meta"
 require_relative "bitfab/db_snapshot"
 require_relative "bitfab/git_command"
 require_relative "bitfab/commit_ref"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.67.3
+  version: 0.68.0
 platform: ruby
 authors:
 - Harvest Team
@@ -211,6 +211,7 @@ files:
 - lib/bitfab/serialize.rb
 - lib/bitfab/simulation_plan.rb
 - lib/bitfab/span_context.rb
+- lib/bitfab/span_meta.rb
 - lib/bitfab/span_origin.rb
 - lib/bitfab/spans.rb
 - lib/bitfab/subtree.rb