bitfab 0.36.4 → 0.36.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5578e8fdcdd81610b4e65fa94a5e9be88339e816742f6ba7fea81b53867eedf5
-  data.tar.gz: 308d35969ac643d1ddcfb76c169a6108906c958275b392f7d159ea647b0bf63c
+  metadata.gz: 49b714ee6d468fc34c0c6009802c5736d33f03275f06d7d7148635b5d70b7caa
+  data.tar.gz: 3816843ddea8722c0222cfc0bed5c65ea05f5abb5ff764233e5ab302d609f2d2
 SHA512:
-  metadata.gz: 2c0511909223595a7792e780d0c77b8c0f5ef2d239e4b2a5dda7946a63fbe4801893ae02ba3392b74520ad9d82844e519d3319a900f22c30bef4857e616c0420
-  data.tar.gz: 0ae83e2f731bc2f6fa9f564b6fe8754b7fff82c5ae03a914067cca6fb75f4ddb10e935bba8976de6f0dabe7025812e49d9392e2ace61b5202c02d80b21fe00aa
+  metadata.gz: 1f5504e7d7622ccf55145b9124ee0d8ca42f774bdaea720a92382af42148bbf16afb72708d673fb359b37a9840364a770e4a1a87003f016aff2ebd1a59a52b38
+  data.tar.gz: db5fbc39adf358e640ae315348eb15ceb5798c67747d092eaaf37651cf9a89542ae9c4d8899a44ebfd4425ba623598e9e5af382dec11f119a917d52cf9ccb0dd

data/lib/bitfab/compress.rb

@@ -12,18 +12,29 @@ module Bitfab
     # batches) ride uncompressed.
     MIN_COMPRESSED_BYTES = 8_192
 
+    PreparedRequest = Struct.new(:body, :content_encoding, :raw_bytes, :wire_bytes)
+
     module_function
 
     # Returns the body to send and the Content-Encoding it carries, or nil when
     # the body is sent as-is. Compression is best-effort: any failure sends the
     # original body rather than dropping the span.
     def encode_request_body(body)
-      return [body, nil] if ENV[DISABLE_COMPRESSION_ENV]
-      return [body, nil] if body.bytesize < MIN_COMPRESSED_BYTES
+      prepared = prepare_request_body(body)
+      [prepared.body, prepared.content_encoding]
+    end
+
+    def prepare_request_body(body)
+      size = body.bytesize
+      return PreparedRequest.new(body, nil, size, size) if ENV[DISABLE_COMPRESSION_ENV]
+      return PreparedRequest.new(body, nil, size, size) if size < MIN_COMPRESSED_BYTES
+
+      compressed = Zlib.gzip(body)
+      return PreparedRequest.new(body, nil, size, size) if compressed.bytesize >= size
 
-      [Zlib.gzip(body), "gzip"]
+      PreparedRequest.new(compressed, "gzip", size, compressed.bytesize)
     rescue
-      [body, nil]
+      PreparedRequest.new(body, nil, body.bytesize, body.bytesize)
     end
   end
 end

data/lib/bitfab/http_client.rb

@@ -58,6 +58,16 @@ module Bitfab
     # POST an already-encoded body. The span transport encodes its own batches,
     # so routing them back through #request would encode the same data twice.
     def send_encoded(endpoint, body, timeout: nil, max_retries: 1, retry_delay: 0.1)
+      send_prepared(
+        endpoint,
+        Compress.prepare_request_body(body),
+        timeout:,
+        max_retries:,
+        retry_delay:
+      )
+    end
+
+    def send_prepared(endpoint, prepared, timeout: nil, max_retries: 1, retry_delay: 0.1)
       uri = URI("#{@service_url}#{endpoint}")
       request_timeout = timeout || @timeout
 
@@ -70,9 +80,8 @@ module Bitfab
         http.read_timeout = request_timeout
 
         req = Net::HTTP::Post.new(uri.path, headers)
-        encoded_body, content_encoding = Compress.encode_request_body(body)
-        req["Content-Encoding"] = content_encoding if content_encoding
-        req.body = encoded_body
+        req["Content-Encoding"] = prepared.content_encoding if prepared.content_encoding
+        req.body = prepared.body
 
         response = http.request(req)
 
@@ -279,8 +288,8 @@ module Bitfab
       end
     end
 
-    def send_transport_request(endpoint, body, timeout)
-      send_encoded(endpoint, body, timeout:, max_retries: 1)
+    def send_transport_request(endpoint, request, timeout)
+      send_prepared(endpoint, request, timeout:, max_retries: 1)
     end
 
     # Normalize each entry to a hash with stable string keys, accepting either

data/lib/bitfab/otel.rb

@@ -5,6 +5,7 @@ require "net/http"
 require "time"
 require "opentelemetry/sdk"
 
+require_relative "compress"
 require_relative "serialize"
 require_relative "version"
 require_relative "warn_once"
@@ -22,6 +23,7 @@ module Bitfab
     MAX_REQUEST_BYTES_ENV = "BITFAB_OTEL_MAX_REQUEST_BYTES"
     EXPORT_CONCURRENCY_ENV = "BITFAB_OTEL_EXPORT_CONCURRENCY"
     MAX_EXPORT_REQUEST_BYTES = 3_000_000
+    MAX_DECOMPRESSED_REQUEST_BYTES = 8_000_000
     MAX_QUEUE_SIZE = 8_192
     DIRECT_MAX_EXPORT_BATCH_SIZE = 512
     DIRECT_MAX_REQUEST_BATCH_SIZE = 8
@@ -314,13 +316,35 @@ module Bitfab
       end
 
       def send_batch(envelope, batch)
-        if batch.size > @max_request_bytes
-          warn_oversized(batch.spans.size)
-          return false
-        end
+        request_spans = batch.spans
+        request_raw_bytes = batch.size
+        already_trimmed = false
+        loop do
+          if request_raw_bytes <= MAX_DECOMPRESSED_REQUEST_BYTES
+            prepared = Compress.prepare_request_body(Encoder.encode_request(envelope, request_spans))
+            if prepared.wire_bytes <= @max_request_bytes
+              send_with_retries(prepared)
+              return true
+            end
+          end
 
-        send_with_retries(Encoder.encode_request(envelope, batch.spans))
-        true
+          if batch.spans.size != 1
+            warn_oversized(batch.spans.size)
+            return false
+          end
+          if already_trimmed
+            warn_oversized(1)
+            return false
+          end
+          trimmed = Encoder.trim_span(batch.spans.first)
+          if trimmed.nil?
+            warn_oversized(1)
+            return false
+          end
+          request_spans = [trimmed]
+          request_raw_bytes = envelope.size + trimmed.size
+          already_trimmed = true
+        end
       rescue PayloadTooLargeError
         warn_oversized(batch.spans.size)
         false
@@ -343,11 +367,11 @@ module Bitfab
         )
       end
 
-      def send_with_retries(body)
+      def send_with_retries(request)
         attempt = 0
         begin
           attempt += 1
-          response = @direct_sender.call(OTLP_TRACES_ENDPOINT, body, EXPORT_TIMEOUT_MILLIS / 1000.0)
+          response = @direct_sender.call(OTLP_TRACES_ENDPOINT, request, EXPORT_TIMEOUT_MILLIS / 1000.0)
           check_partial_success(response)
         rescue PartialSuccessError
           raise
@@ -456,7 +480,10 @@ module Bitfab
         # Encoding is the expensive part of a submit and needs no mutual
         # exclusion, so it stays outside the lock.
         name = Encoder.span_name(operation, payload)
-        encoded_payload = Serialize.safe_generate(payload)
+        encoded_payload = Serialize.safe_generate(
+          payload,
+          max_carrier_bytes: PayloadBudget::MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES
+        )
         started_at = Encoder.timestamp(payload, "started_at")
         ended_at = Encoder.timestamp(payload, "ended_at")
         errored = Encoder.error?(payload)
@@ -616,6 +643,20 @@ module Bitfab
           EncodedSpan.new(encoded, encoded.bytesize)
         end
 
+        def trim_span(span)
+          carrier = JSON.parse(span.encoded)
+          attribute = carrier.fetch("attributes").find { |entry| entry["key"] == PAYLOAD_ATTRIBUTE }
+          return nil if attribute.nil?
+
+          value = attribute.fetch("value")
+          payload = JSON.parse(value.fetch("stringValue"))
+          value["stringValue"] = Serialize.safe_generate(payload)
+          encoded = JSON.generate(carrier)
+          EncodedSpan.new(encoded, encoded.bytesize)
+        rescue KeyError, JSON::ParserError, TypeError
+          nil
+        end
+
         def request_envelope(first)
           scope = first.instrumentation_scope
           resource = JSON.generate({"attributes" => attributes(first.resource.attribute_enumerator.to_h)})

data/lib/bitfab/payload_budget.rb

@@ -19,10 +19,12 @@ module Bitfab
   # on a body sized exactly to a 2.4 MB cap, prose produced a 2.4 MB carrier but
   # backslash-dense content produced 4.8 MB, which the exporter dropped.
   #
-  # 2.8 MB leaves room beneath the 3 MB request ceiling for the span and request
-  # envelopes wrapped around the attribute.
+  # The normal 2.8 MB fallback leaves room beneath the 3 MB wire target. Trace
+  # transport may first preserve a carrier up to 7.8 MB when its single-span
+  # request compresses below that target and stays under the 8 MB raw ceiling.
   module PayloadBudget
     MAX_SPAN_CARRIER_BYTES = 2_800_000
+    MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES = 7_800_000
 
     # Span fields that identify the span rather than carry user data. Trimming
     # one would leave a span that no longer says what it is, so they stay
@@ -47,23 +49,23 @@ module Bitfab
     # can at most double it, so anything under half the budget always fits and
     # anything past the budget never does. Ordinary spans settle on the first
     # comparison and never pay for the scan.
-    def fits_carrier_budget?(body)
+    def fits_carrier_budget?(body, max_bytes = MAX_SPAN_CARRIER_BYTES)
       size = body.bytesize
-      return true if size * 2 + 2 <= MAX_SPAN_CARRIER_BYTES
-      return false if size + 2 > MAX_SPAN_CARRIER_BYTES
+      return true if size * 2 + 2 <= max_bytes
+      return false if size + 2 > max_bytes
 
-      carrier_byte_length(body) <= MAX_SPAN_CARRIER_BYTES
+      carrier_byte_length(body) <= max_bytes
     end
 
     # Return a body within the budget, plus the fields that had to be stubbed.
-    def enforce(payload, body, &encode)
-      return [body, []] if fits_carrier_budget?(body)
+    def enforce(payload, body, max_bytes = MAX_SPAN_CARRIER_BYTES, &encode)
+      return [body, []] if fits_carrier_budget?(body, max_bytes)
       return [body, []] unless payload.is_a?(Hash)
 
-      trimmed_payload, trimmed = trim(payload, &encode)
+      trimmed_payload, trimmed = trim(payload, max_bytes, &encode)
       return [body, []] if trimmed_payload.nil?
 
-      mark_trimmed(trimmed_payload, trimmed)
+      mark_trimmed(trimmed_payload, trimmed, max_bytes)
       [encode.call(trimmed_payload), trimmed]
     rescue
       [body, []]
@@ -73,7 +75,7 @@ module Bitfab
     # Returns [trimmed_payload, trimmed_keys], or nil when nothing could be
     # trimmed: the caller then ships the oversized body and lets the exporter
     # report the drop, which still beats silently emptying a span.
-    def trim(payload, &encode)
+    def trim(payload, max_bytes = MAX_SPAN_CARRIER_BYTES, &encode)
       copy, containers = clone_trimmable(payload)
       candidates = collect_candidates(containers)
       return nil if candidates.empty?
@@ -83,7 +85,7 @@ module Bitfab
         container[key] = "<unserializable: too_large_#{size}_bytes>"
         trimmed << key
         body = encode.call(copy)
-        return [copy, trimmed] if fits_carrier_budget?(body)
+        return [copy, trimmed] if fits_carrier_budget?(body, max_bytes)
       end
       nil
     end
@@ -135,7 +137,7 @@ module Bitfab
 
     # Record the trim in the payload's own errors, which is what the server
     # reads to flag a trace as incomplete.
-    def mark_trimmed(payload, trimmed)
+    def mark_trimmed(payload, trimmed, max_bytes)
       key = payload.key?(:errors) ? :errors : "errors"
       existing = payload[key]
       errors = existing.is_a?(Array) ? existing.dup : []
@@ -143,7 +145,7 @@ module Bitfab
         "source" => "sdk",
         "step" => "payload_budget",
         "error" => "trimmed oversized field(s) to fit the " \
-          "#{MAX_SPAN_CARRIER_BYTES}-byte span carrier budget: " \
+          "#{max_bytes}-byte span carrier budget: " \
           "#{trimmed.uniq.join(", ")}"
       }
       payload[key] = errors

data/lib/bitfab/serialize.rb

@@ -14,7 +14,7 @@ module Bitfab
     # further. It is deliberately the same number as the whole-span budget: one
     # legitimately large value may use the entire budget, and PayloadBudget is
     # what enforces the total once every field is in.
-    MAX_SERIALIZED_BYTES = PayloadBudget::MAX_SPAN_CARRIER_BYTES
+    MAX_SERIALIZED_BYTES = PayloadBudget::MAX_COMPRESSIBLE_SPAN_CARRIER_BYTES
 
     # Recursion guard for cyclic graphs and pathologically nested structures.
     MAX_SERIALIZE_DEPTH = 16
@@ -260,18 +260,20 @@ module Bitfab
     # raises and stubs strays) instead of letting JSON.generate raise and drop
     # the whole span/trace silently. A degraded payload warns loudly so the
     # trace isn't quietly left incomplete or not replayable.
-    def safe_generate(payload)
+    def safe_generate(payload, max_carrier_bytes: PayloadBudget::MAX_SPAN_CARRIER_BYTES)
       # Budget the value that was actually encoded, not the caller's original: a
       # cyclic or otherwise non-encodable field cannot be sized (JSON.generate
       # raises on it), so on the original graph the biggest field is skipped as
       # a trim candidate and the oversized body ships anyway. The sanitized copy
       # has those values already replaced with stubs, so every field is sizeable.
       body, encoded = encode_unbounded(payload)
-      body, trimmed = PayloadBudget.enforce(encoded, body) { |value| encode_unbounded(value).first }
+      body, trimmed = PayloadBudget.enforce(encoded, body, max_carrier_bytes) do |value|
+        encode_unbounded(value).first
+      end
       if trimmed.any?
         Bitfab.warn_once(
           "payload-over-budget",
-          "a span payload exceeded the #{PayloadBudget::MAX_SPAN_CARRIER_BYTES}-byte carrier budget; its " \
+          "a span payload exceeded the #{max_carrier_bytes}-byte carrier budget; its " \
           "largest field(s) (#{trimmed.uniq.join(", ")}) were replaced with placeholders " \
           "so the span still ships. The span is incomplete and may not be replayable."
         )

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.36.4"
+  VERSION = "0.36.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.36.4
+  version: 0.36.5
 platform: ruby
 authors:
 - Harvest Team