bitfab 0.33.6 → 0.34.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d72d477d9b3533bbba990387aeaf342774ec7e3271e11e51fc297a7de9534bb
-  data.tar.gz: 624832fc2f8b1e526370a6c07b75fcb36dc5ffe62871ed888392e3f3d9eeb5ca
+  metadata.gz: 897fb65140b86333bb5012746e570c785560c7865cd79da1e872e4ec567fada2
+  data.tar.gz: 26b89a73668a0f86746a349c8f57fe39317f069dbab04dd8731ff91950d24a49
 SHA512:
-  metadata.gz: d8e17a56b93c6ab2ab59e5ba5f5bb264f68329f19f7c1801c10e750609be96b3ae2be5dca4117aeae6e41f2d62ae5be757188c6a85bc98548290f1afe6b0aef1
-  data.tar.gz: 7de5823eb5e244c4576723234d7ad6812fded976cd8f22837585a563ded544906d0f2c91114eb748d971056563fe2301a17ce937473538d8c3e507870a5e9929
+  metadata.gz: cb837f5a94d68391c0b25ac1bc073cba636816270abd623b825b59a2a3d8f387ac67a49aaef93aebcc3201ad24dbe4ef9f4d4738955d5db5387181eb1f94ee68
+  data.tar.gz: 0b609be82dca49043083bf966b52b01466f8252abe15b243e4d2ac6a3ea06d5238fb5fdf7b88e7a2c1be22fa4c36b06425baa2261a62ffed6967b5e8c86a8150

data/README.md

@@ -20,7 +20,6 @@ gem install bitfab
 
 - Ruby >= 3.4
 - Runtime dependencies: `base64`, `opentelemetry-sdk` (`>= 1.2, < 2`, verified against 1.2 through 1.13, so an application already pinning its own OpenTelemetry version does not have to move)
-- Optional: `opentelemetry-exporter-otlp`, only to deliver through an OpenTelemetry Collector. Use a version compatible with your `opentelemetry-sdk`; if the gem cannot load, the SDK warns once and keeps delivering directly to Bitfab
 
 ## Quick Start
 
@@ -391,41 +390,17 @@ worker, export scheduling, flush, and shutdown lifecycle. Pipelines are created
 trace send and are never installed globally, so an unused or disabled client starts no OTel worker
 and the SDK does not replace an application's own OpenTelemetry setup.
 
-By default, batches are sent directly to Bitfab as OTLP/JSON. To send through a local OTel Collector
-instead, add the `opentelemetry-exporter-otlp` gem and set `BITFAB_OTEL_EXPORTER_ENDPOINT` to the
-Collector's OTLP/HTTP base URL (for example, `http://localhost:4318`). The SDK appends `/v1/traces`
-and sends protobuf through OTel's official exporter. Production and replay traffic share the same
-pipeline. Before finalizing a replay, the SDK flushes OTel and polls Bitfab's replay-status API
-until every expected trace completion and span count is persisted, so Collector acceptance is never
-mistaken for Bitfab persistence.
-
-Direct requests carry at most eight carriers each and up to 32 run concurrently; set
-`BITFAB_OTEL_EXPORT_CONCURRENCY` to an integer from `1` through `64` to tune that. Direct JSON and
-Collector protobuf requests are partitioned at roughly 3 MB; set `BITFAB_OTEL_MAX_REQUEST_BYTES` to
-a positive integer no greater than `3000000` for a Collector or proxy with a stricter limit.
-Invalid values warn once and fall back to the defaults. If Bitfab accepts only part of a batch, it
-returns the standard OTLP `partialSuccess` response and the SDK warns with the rejected-span count
-and reason.
-
-```yaml
-receivers:
-  otlp:
-    protocols:
-      http:
-
-exporters:
-  otlphttp/bitfab:
-    endpoint: https://bitfab.ai/api/sdk/otel
-    encoding: json
-    headers:
-      Authorization: Bearer ${env:BITFAB_API_KEY}
-
-service:
-  pipelines:
-    traces:
-      receivers: [otlp]
-      exporters: [otlphttp/bitfab]
-```
+Requests carry at most eight carriers each and up to 32 run concurrently; set
+`BITFAB_OTEL_EXPORT_CONCURRENCY` to an integer from `1` through `64` to tune that. Requests are
+partitioned at roughly 3 MB; set `BITFAB_OTEL_MAX_REQUEST_BYTES` to a positive integer no greater
+than `3000000` for a proxy with a stricter limit. Invalid values warn once and fall back to the
+defaults. Each carrier is encoded once and the request body is assembled from those encodings, so a
+batch is never re-encoded to measure its size. If Bitfab accepts only part of a batch, it returns
+the standard OTLP `partialSuccess` response and the SDK warns with the rejected-span count and
+reason.
+
+Before finalizing a replay, the SDK flushes OTel and polls Bitfab's replay-status API until every
+expected trace completion and span count is persisted.
 
 See the [OpenTelemetry Transport Architecture](https://docs.bitfab.ai/otel-architecture) for the
 full component ownership, carrier format, replay barrier, batching, and lifecycle design.

data/lib/bitfab/http_client.rb

@@ -51,6 +51,12 @@ module Bitfab
     # Make a POST request to the Bitfab API.
     # Returns parsed JSON response hash.
     def request(endpoint, payload, timeout: nil, max_retries: 1, retry_delay: 0.1)
+      send_encoded(endpoint, Serialize.safe_generate(payload), timeout:, max_retries:, retry_delay:)
+    end
+
+    # POST an already-encoded body. The span transport encodes its own batches,
+    # so routing them back through #request would encode the same data twice.
+    def send_encoded(endpoint, body, timeout: nil, max_retries: 1, retry_delay: 0.1)
       uri = URI("#{@service_url}#{endpoint}")
       request_timeout = timeout || @timeout
 
@@ -63,7 +69,7 @@ module Bitfab
         http.read_timeout = request_timeout
 
         req = Net::HTTP::Post.new(uri.path, headers)
-        req.body = Serialize.safe_generate(payload)
+        req.body = body
 
         response = http.request(req)
 
@@ -260,14 +266,13 @@ module Bitfab
         end
 
         @transport ||= Transport.create_trace_transport(
-          api_key: @api_key,
           direct_sender: method(:send_transport_request)
         )
       end
     end
 
-    def send_transport_request(endpoint, payload, timeout)
-      request(endpoint, payload, timeout:, max_retries: 1)
+    def send_transport_request(endpoint, body, timeout)
+      send_encoded(endpoint, body, timeout:, max_retries: 1)
     end
 
     # Normalize each entry to a hash with stable string keys, accepting either

data/lib/bitfab/otel.rb

@@ -13,20 +13,17 @@ module Bitfab
   # OpenTelemetry delivery for Bitfab spans and trace completions.
   #
   # Bitfab payloads ride as OTLP span attributes (bitfab.operation +
-  # bitfab.payload) so a batch can travel either straight to Bitfab as
-  # OTLP/JSON or through a customer's OTel Collector as protobuf, without the
+  # bitfab.payload), so a batch travels to Bitfab as OTLP/JSON without the
   # payload itself changing shape.
   module Otel
     OPERATION_ATTRIBUTE = "bitfab.operation"
     PAYLOAD_ATTRIBUTE = "bitfab.payload"
     OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces"
-    COLLECTOR_ENDPOINT_ENV = "BITFAB_OTEL_EXPORTER_ENDPOINT"
     MAX_REQUEST_BYTES_ENV = "BITFAB_OTEL_MAX_REQUEST_BYTES"
     EXPORT_CONCURRENCY_ENV = "BITFAB_OTEL_EXPORT_CONCURRENCY"
     MAX_EXPORT_REQUEST_BYTES = 3_000_000
     MAX_QUEUE_SIZE = 8_192
     DIRECT_MAX_EXPORT_BATCH_SIZE = 512
-    COLLECTOR_MAX_EXPORT_BATCH_SIZE = 32
     DIRECT_MAX_REQUEST_BATCH_SIZE = 8
     DEFAULT_EXPORT_CONCURRENCY = 32
     MAX_EXPORT_CONCURRENCY = 64
@@ -35,9 +32,25 @@ module Bitfab
     EXPORT_RETRIES = 3
     RETRY_DELAY_SECONDS = 0.1
 
+    # The comma that joins adjacent spans in the request's span list.
+    SPAN_SEPARATOR_BYTES = 1
+
     SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
     FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
 
+    # A span encoded exactly as it goes on the wire, carrying its byte count.
+    # Encoding once and remembering the size is what keeps request packing
+    # linear: sizing a candidate batch by re-encoding the whole request
+    # re-escapes every carrier's bitfab.payload string on every span considered.
+    EncodedSpan = Struct.new(:encoded, :size)
+
+    # The invariant head and tail of an OTLP request for one export window. Key
+    # order matches what JSON.generate emits for the equivalent Hash, so a body
+    # assembled by concatenation is byte-identical to encoding that Hash.
+    RequestEnvelope = Struct.new(:head, :tail, :size)
+
+    RequestBatch = Struct.new(:spans, :size)
+
     SPAN_KINDS = {
       internal: 1,
       server: 2,
@@ -82,11 +95,9 @@ module Bitfab
         DEFAULT_EXPORT_CONCURRENCY
       end
 
-      def create_transport(api_key:, direct_sender:)
+      def create_transport(direct_sender:)
         BatchTransport.new(
-          api_key:,
           direct_sender:,
-          collector_endpoint: collector_endpoint_from_env,
           export_concurrency: export_concurrency_from_env,
           max_request_bytes: max_request_bytes_from_env
         )
@@ -177,11 +188,6 @@ module Bitfab
 
       private
 
-      def collector_endpoint_from_env
-        endpoint = ENV[COLLECTOR_ENDPOINT_ENV]
-        (endpoint.nil? || endpoint.empty?) ? nil : endpoint
-      end
-
       def record_span_submission(source_trace_id, payload)
         raw_span = payload["rawSpan"]
         span_id = raw_span["id"] if raw_span.is_a?(Hash)
@@ -237,7 +243,8 @@ module Bitfab
         return SUCCESS if spans.empty?
 
         begin
-          encoded = spans.map { |span| Encoder.span_to_otlp(span) }
+          encoded = spans.map { |span| Encoder.encode_span(span) }
+          envelope = Encoder.request_envelope(spans.first)
         rescue => e
           Bitfab.warn_once(
             "otel-encode-failed",
@@ -246,8 +253,8 @@ module Bitfab
           return FAILURE
         end
 
-        batches = build_request_batches(spans.first, encoded)
-        results = send_batches(spans.first, batches)
+        batches = build_request_batches(envelope, encoded)
+        results = send_batches(envelope, batches)
         results.all? ? SUCCESS : FAILURE
       end
 
@@ -261,7 +268,7 @@ module Bitfab
 
       private
 
-      def send_batches(first, batches)
+      def send_batches(envelope, batches)
         results = Array.new(batches.size, false)
         next_index = 0
         index_mutex = Mutex.new
@@ -277,7 +284,7 @@ module Bitfab
               end
               break if index >= batches.size
 
-              results[index] = send_batch(first, batches[index])
+              results[index] = send_batch(envelope, batches[index])
             end
           end
         end
@@ -285,40 +292,37 @@ module Bitfab
         results
       end
 
-      def build_request_batches(first, encoded)
+      def build_request_batches(envelope, encoded)
         batches = []
         current = []
+        size = envelope.size
 
         encoded.each do |span|
-          if current.size >= @max_request_batch_size
-            batches << current
+          addition = span.size + (current.empty? ? 0 : SPAN_SEPARATOR_BYTES)
+          if !current.empty? && (current.size >= @max_request_batch_size || size + addition > @max_request_bytes)
+            batches << RequestBatch.new(current, size)
             current = []
+            size = envelope.size
+            addition = span.size
           end
-
-          candidate = current + [span]
-          if !current.empty? && Encoder.encoded_size(Encoder.build_request(first, candidate)) > @max_request_bytes
-            batches << current
-            current = [span]
-          else
-            current = candidate
-          end
+          current << span
+          size += addition
         end
 
-        batches << current unless current.empty?
+        batches << RequestBatch.new(current, size) unless current.empty?
         batches
       end
 
-      def send_batch(first, spans)
-        payload = Encoder.build_request(first, spans)
-        if Encoder.encoded_size(payload) > @max_request_bytes
-          warn_oversized(spans.size)
+      def send_batch(envelope, batch)
+        if batch.size > @max_request_bytes
+          warn_oversized(batch.spans.size)
           return false
         end
 
-        send_with_retries(payload)
+        send_with_retries(Encoder.encode_request(envelope, batch.spans))
         true
       rescue PayloadTooLargeError
-        warn_oversized(spans.size)
+        warn_oversized(batch.spans.size)
         false
       rescue PartialSuccessError
         false
@@ -339,11 +343,11 @@ module Bitfab
         )
       end
 
-      def send_with_retries(payload)
+      def send_with_retries(body)
         attempt = 0
         begin
           attempt += 1
-          response = @direct_sender.call(OTLP_TRACES_ENDPOINT, payload, EXPORT_TIMEOUT_MILLIS / 1000.0)
+          response = @direct_sender.call(OTLP_TRACES_ENDPOINT, body, EXPORT_TIMEOUT_MILLIS / 1000.0)
           check_partial_success(response)
         rescue PartialSuccessError
           raise
@@ -386,72 +390,6 @@ module Bitfab
       end
     end
 
-    # Splits an export into requests no larger than the configured byte target
-    # before handing them to the wrapped Collector exporter.
-    class SizeLimitedExporter
-      def initialize(exporter, max_request_bytes)
-        @exporter = exporter
-        @max_request_bytes = max_request_bytes
-        # The OTLP gem exposes no public way to measure an encoded request, so
-        # requests are partitioned by count alone when its encoder is missing.
-        @measurable = exporter.respond_to?(:encode, true)
-      end
-
-      def export(spans, timeout: nil)
-        batches = partition(spans.to_a)
-        batches.reduce(SUCCESS) do |result, batch|
-          exported = @exporter.export(batch, timeout:)
-          (exported == SUCCESS) ? result : FAILURE
-        end
-      end
-
-      def force_flush(timeout: nil)
-        @exporter.force_flush(timeout:)
-      end
-
-      def shutdown(timeout: nil)
-        @exporter.shutdown(timeout:)
-      end
-
-      private
-
-      def partition(spans)
-        return [spans] if !@measurable || spans.empty?
-
-        batches = []
-        current = []
-        current_size = 0
-
-        spans.each do |span|
-          size = span_bytes(span)
-          return [spans] if size.nil?
-
-          if !current.empty? && current_size + size > @max_request_bytes
-            batches << current
-            current = []
-            current_size = 0
-          end
-          current << span
-          current_size += size
-        end
-
-        batches << current unless current.empty?
-        batches
-      end
-
-      # The OTLP gem's encoder returns a serialized String on current versions
-      # and a protobuf message on older ones, and returns nil when it cannot
-      # encode a span at all. Anything unmeasurable falls back to count-based
-      # partitioning rather than failing the export from here.
-      def span_bytes(span)
-        encoded = @exporter.send(:encode, [span])
-        return encoded.bytesize if encoded.respond_to?(:bytesize)
-        return encoded.to_proto.bytesize if encoded.respond_to?(:to_proto)
-
-        nil
-      end
-    end
-
     # Records failed exports so a flush can report delivery, not just drain.
     class DeliveryTrackingExporter
       def initialize(exporter)
@@ -494,15 +432,13 @@ module Bitfab
     class BatchTransport
       attr_reader :owner_pid
 
-      def initialize(api_key:, direct_sender:, collector_endpoint: nil, max_export_batch_size: nil,
+      def initialize(direct_sender:, max_export_batch_size: nil,
         max_request_batch_size: DIRECT_MAX_REQUEST_BATCH_SIZE, max_queue_size: MAX_QUEUE_SIZE,
         export_concurrency: DEFAULT_EXPORT_CONCURRENCY, max_request_bytes: nil)
         raise ArgumentError, "max_request_batch_size must be a positive integer" unless max_request_batch_size.positive?
 
-        @api_key = api_key
         @direct_sender = direct_sender
-        @collector_endpoint = collector_endpoint
-        @max_export_batch_size = max_export_batch_size || default_export_batch_size
+        @max_export_batch_size = max_export_batch_size || DIRECT_MAX_EXPORT_BATCH_SIZE
         @max_request_batch_size = max_request_batch_size
         @max_queue_size = max_queue_size
         @export_concurrency = export_concurrency
@@ -620,10 +556,6 @@ module Bitfab
         nil
       end
 
-      def default_export_batch_size
-        @collector_endpoint.nil? ? DIRECT_MAX_EXPORT_BATCH_SIZE : COLLECTOR_MAX_EXPORT_BATCH_SIZE
-      end
-
       # A forked child inherits the parent's provider and processor but none of
       # its threads, so the pipeline is rebuilt on the first submit after a fork.
       # Callers hold @state_mutex, so exactly one thread rebuilds.
@@ -667,27 +599,6 @@ module Bitfab
       end
 
       def create_exporter
-        return direct_exporter if @collector_endpoint.nil?
-
-        require_relative "otel_collector"
-        SizeLimitedExporter.new(
-          CollectorExporter.new(
-            api_key: @api_key,
-            endpoint: CollectorExporter.traces_endpoint(@collector_endpoint),
-            timeout: EXPORT_TIMEOUT_MILLIS / 1000.0
-          ),
-          @max_request_bytes
-        )
-      rescue LoadError
-        Bitfab.warn_once(
-          "otel-collector-gem-missing",
-          "#{COLLECTOR_ENDPOINT_ENV} is set but the opentelemetry-exporter-otlp " \
-          "gem is not installed; sending spans directly to Bitfab instead"
-        )
-        direct_exporter
-      end
-
-      def direct_exporter
         DirectExporter.new(
           direct_sender: @direct_sender,
           max_request_bytes: @max_request_bytes,
@@ -700,21 +611,22 @@ module Bitfab
     # Turns finished carrier spans into the OTLP/JSON shapes Bitfab ingests.
     module Encoder
       class << self
-        def build_request(first, spans)
+        def encode_span(span)
+          encoded = JSON.generate(span_to_otlp(span))
+          EncodedSpan.new(encoded, encoded.bytesize)
+        end
+
+        def request_envelope(first)
           scope = first.instrumentation_scope
-          {
-            "resourceSpans" => [{
-              "resource" => {"attributes" => attributes(first.resource.attribute_enumerator.to_h)},
-              "scopeSpans" => [{
-                "scope" => {"name" => scope.name, "version" => scope.version || ""},
-                "spans" => spans
-              }]
-            }]
-          }
+          resource = JSON.generate({"attributes" => attributes(first.resource.attribute_enumerator.to_h)})
+          scope_json = JSON.generate({"name" => scope.name, "version" => scope.version || ""})
+          head = %({"resourceSpans":[{"resource":#{resource},"scopeSpans":[{"scope":#{scope_json},"spans":[)
+          tail = "]}]}]}"
+          RequestEnvelope.new(head, tail, (head + tail).bytesize)
         end
 
-        def encoded_size(value)
-          JSON.generate(value).bytesize
+        def encode_request(envelope, spans)
+          envelope.head + spans.map(&:encoded).join(",") + envelope.tail
         end
 
         def span_to_otlp(span)

data/lib/bitfab/transport.rb

@@ -7,8 +7,8 @@ module Bitfab
   # mechanism, so callers never depend on OpenTelemetry directly.
   module Transport
     class << self
-      def create_trace_transport(api_key:, direct_sender:)
-        Otel.create_transport(api_key:, direct_sender:)
+      def create_trace_transport(direct_sender:)
+        Otel.create_transport(direct_sender:)
       end
 
       def flush_trace_transports(timeout = 30.0)

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.33.6"
+  VERSION = "0.34.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.33.6
+  version: 0.34.0
 platform: ruby
 authors:
 - Harvest Team
@@ -43,20 +43,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-- !ruby/object:Gem::Dependency
-  name: opentelemetry-exporter-otlp
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.30'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.30'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -157,7 +143,6 @@ files:
 - lib/bitfab/http_client.rb
 - lib/bitfab/mock_override.rb
 - lib/bitfab/otel.rb
-- lib/bitfab/otel_collector.rb
 - lib/bitfab/replay.rb
 - lib/bitfab/replay_branch.rb
 - lib/bitfab/serialize.rb

data/lib/bitfab/otel_collector.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-require "opentelemetry-exporter-otlp"
-
-module Bitfab
-  module Otel
-    # OTLP/HTTP protobuf delivery to a customer's Collector.
-    class CollectorExporter < OpenTelemetry::Exporter::OTLP::Exporter
-      def self.traces_endpoint(endpoint)
-        base = endpoint.chomp("/")
-        base.end_with?("/v1/traces") ? base : "#{base}/v1/traces"
-      end
-
-      def initialize(api_key:, **options)
-        super(**options)
-        @api_key = api_key
-      end
-
-      # The OTLP exporter reads headers once at construction, while a Bitfab key
-      # is resolved at request time (an env var may load after the client is built).
-      def export(span_data, timeout: nil)
-        key = @api_key.respond_to?(:call) ? @api_key.call : @api_key
-        @headers = @headers.merge("Authorization" => "Bearer #{key}")
-        super
-      end
-    end
-  end
-end