datadog 2.17.0 → 2.37.0

data/lib/datadog/symbol_database/file_hash.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'digest/sha1'
+require_relative '../di/fatal_exceptions'
+
+module Datadog
+  module SymbolDatabase
+    # Computes Git-style SHA-1 hashes of Ruby source files for backend commit inference.
+    #
+    # Uses Git's blob hash algorithm: SHA1("blob <size>\0<content>")
+    # Hashes enable the backend to correlate runtime code with Git repository history,
+    # identifying which commit is actually deployed.
+    #
+    # Called by: Extractor (when building MODULE scopes)
+    # Stores result in: Scope's language_specifics[:file_hash]
+    # Returns: 40-character hex string or nil if file unreadable
+    #
+    # @api private
+    module FileHash
+      module_function
+
+      # Compute Git-style SHA-1 hash of a file.
+      # Uses Git's blob hash algorithm: SHA1("blob <size>\0<content>")
+      # Returns nil on any error (file not found, permission denied, etc.)
+      #
+      # @param file_path [String] Path to the file
+      # @param logger [#debug] Logger for error reporting
+      # @return [String, nil] 40-character hex-encoded SHA-1 hash, or nil if error
+      def compute(file_path, logger:)
+        return nil unless file_path
+        return nil unless File.exist?(file_path)
+
+        content = File.read(file_path, mode: 'rb')
+        size = content.bytesize
+        git_blob = "blob #{size}\0#{content}"
+
+        # SHA-1 is required here to match Git's blob hash format for commit inference.
+        # This is not a security vulnerability - we're computing file content hashes
+        # to match against Git objects, not using SHA-1 for authentication/integrity.
+        Digest::SHA1.hexdigest(git_blob)  # nosemgrep: ruby.lang.security.weak-hashes-sha1.weak-hashes-sha1
+      rescue Exception => e # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(e)
+        logger.debug { "symdb: file hash failed for #{file_path}: #{e.class}: #{e.message}" }
+        nil
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/logger.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Datadog
+  module SymbolDatabase
+    # Logger facade that adds a config-gated +trace+ method.
+    #
+    # Wraps any logger (customer-provided or default) and delegates
+    # standard methods. The +trace+ method is a sub-debug level that
+    # is a no-op unless DD_TRACE_DEBUG is set, avoiding overhead for
+    # high-frequency log sites (per-module filtering, dedup checks).
+    #
+    # @api private
+    class Logger
+      extend Forwardable
+
+      # @param settings [Configuration::Settings] Tracer settings (reads trace_logging flag)
+      # @param target [::Logger] Underlying logger to delegate to
+      def initialize(settings, target)
+        @settings = settings
+        @target = target
+      end
+
+      attr_reader :settings
+
+      # Only debug and warn are delegated by design — symbol database
+      # extraction logs only at debug (high-volume diagnostics) and warn
+      # (user-actionable problems). Adding info/error would invite
+      # log-level drift; explicit additions can be made if needed.
+      def_delegators :@target, :debug, :warn
+
+      # Log at trace level (sub-debug). No-op unless DD_TRACE_DEBUG is set.
+      # @yield Block that returns the log message string
+      # @return [void]
+      def trace(&block)
+        if settings.symbol_database.internal.trace_logging
+          debug(&block)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/remote.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require_relative '../di/fatal_exceptions'
+
+module Datadog
+  module SymbolDatabase
+    # Provides remote configuration integration for symbol database.
+    #
+    # Responsibilities:
+    # - Registers with Core::Remote as a receiver for LIVE_DEBUGGING_SYMBOL_DB product
+    # - Processes remote config changes (insert/update/delete)
+    # - Calls Component.start_upload when upload_symbols: true
+    # - Calls Component.stop_upload when config deleted or upload_symbols: false
+    #
+    # Flow:
+    # 1. Remote config system calls receiver with repository and changes
+    # 2. For each change, process_change called
+    # 3. parse_config extracts upload_symbols flag
+    # 4. enable_upload or disable_upload called on component
+    #
+    # Created by: Symbol database initialization
+    # Accessed by: Core::Remote system when configurations change
+    # Requires: Component must exist (accessed via Datadog.send(:components).symbol_database)
+    #
+    # @api private
+    module Remote
+      PRODUCT = 'LIVE_DEBUGGING_SYMBOL_DB'
+
+      class << self
+        # Declare products this receiver handles.
+        # @return [Array<String>] Product names
+        def products
+          [PRODUCT]
+        end
+
+        # Declare capabilities for this receiver.
+        # @return [Array] Capabilities (none for symbol database)
+        def capabilities
+          []
+        end
+
+        # Create receivers for remote configuration.
+        # @return [Array<Receiver>] Array of receivers
+        def receivers(_telemetry)
+          receiver do |repository, changes|
+            telemetry = lookup_telemetry
+            component = begin
+              Datadog.send(:components, allow_initialization: false)&.symbol_database
+            rescue Exception => e # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(e)
+              Datadog.logger.debug { "symdb: failed to look up component in RC receiver: #{e.class}: #{e.message}" }
+              telemetry&.report(e, description: 'symdb: failed to look up component in RC receiver')
+              nil
+            end
+
+            if component
+              changes.each do |change|
+                process_change(component, change, telemetry)
+              end
+            end
+          end
+        end
+
+        # Create a single receiver for the product.
+        # @param products [Array<String>] Product names to match
+        # @return [Array<Receiver>] Receiver array
+        def receiver(products = [PRODUCT], &block)
+          matcher = Core::Remote::Dispatcher::Matcher::Product.new(products)
+          [Core::Remote::Dispatcher::Receiver.new(matcher, &block)]
+        end
+
+        private
+
+        # Look up the telemetry component for error reporting. Returns nil if the
+        # component tree isn't built yet (very early boot) or the lookup raises.
+        # `allow_initialization: false` avoids triggering component-tree construction
+        # from inside an RC receiver callback.
+        # @return [Core::Telemetry::Component, nil]
+        # @api private
+        def lookup_telemetry
+          Datadog.send(:components, allow_initialization: false)&.telemetry
+        rescue Exception => e # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(e)
+          nil
+        end
+
+        # Process a single configuration change.
+        # @param component [Component] Symbol database component
+        # @param change [Change] Configuration change (:insert, :update, :delete)
+        # @param telemetry [Core::Telemetry::Component, nil] Telemetry for error reporting
+        # @return [void]
+        # @api private
+        def process_change(component, change, telemetry)
+          case change.type
+          when :insert
+            # @type var change: ::Datadog::Core::Remote::Configuration::Repository::Change::Inserted
+            enable_upload(component, change.content)
+            change.content.applied
+          when :update
+            # @type var change: ::Datadog::Core::Remote::Configuration::Repository::Change::Updated
+            disable_upload(component)
+            enable_upload(component, change.content)
+            change.content.applied
+          when :delete
+            # @type var change: ::Datadog::Core::Remote::Configuration::Repository::Change::Deleted
+            disable_upload(component)
+            change.previous&.applied
+          else
+            component.logger.debug { "symdb: unrecognized change type: #{change.type}" }
+            # Steep cannot narrow `change.content` from a respond_to? check — it sees
+            # the Repository::Change union type where `Deleted` lacks `content`.
+            change.content.errored("Unrecognized change type: #{change.type}") if change.respond_to?(:content) # steep:ignore NoMethod
+          end
+        rescue Exception => e # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(e)
+          component.logger.debug { "symdb: error processing remote config change: #{e.class}: #{e.message}" }
+          telemetry&.report(e, description: 'symdb: error processing remote config change')
+          # Rescue runs regardless of which branch raised — Steep cannot narrow the
+          # union type from a respond_to? check.
+          content_obj = change.respond_to?(:content) ? change.content : change.previous # steep:ignore NoMethod
+          content_obj&.errored(e.to_s)
+        end
+
+        # Enable upload if config has upload_symbols: true.
+        # @param component [Component] Symbol database component
+        # @param content [Content] Remote config content
+        # @return [void]
+        # @api private
+        def enable_upload(component, content)
+          config = parse_config(content, component.logger)
+
+          unless config
+            return
+          end
+
+          if config['upload_symbols']
+            component.logger.debug { "symdb: upload enabled via remote config" }
+            component.start_upload
+          else
+            component.logger.debug { "symdb: upload disabled in config" }
+          end
+        end
+
+        # Disable upload.
+        # @param component [Component] Symbol database component
+        # @return [void]
+        # @api private
+        def disable_upload(component)
+          component.logger.debug { "symdb: upload disabled via remote config" }
+          component.stop_upload
+        end
+
+        # Parse and validate remote config content.
+        # @param content [Content] Remote config content
+        # @param logger [SymbolDatabase::Logger] Logger for invalid-config diagnostics
+        # @return [Hash, nil] Parsed config or nil if invalid
+        # @api private
+        #
+        # JSON::ParserError is intentionally NOT rescued here — it propagates to
+        # process_change's rescue, which logs and reports to telemetry. Catching
+        # it locally would swallow the error from telemetry observability.
+        def parse_config(content, logger)
+          config = JSON.parse(content.data)
+
+          unless config.is_a?(Hash)
+            logger.debug { "symdb: invalid config format: expected Hash, got #{config.class}" }
+            return nil
+          end
+
+          unless config.key?('upload_symbols')
+            logger.debug { "symdb: missing 'upload_symbols' key in config" }
+            return nil
+          end
+
+          config
+        end
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/scope.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Datadog
+  module SymbolDatabase
+    # Represents a scope in the hierarchical symbol structure (FILE → MODULE/CLASS → METHOD).
+    #
+    # Scopes form a tree structure representing Ruby code organization. Each scope contains:
+    # - Metadata: name, source file, line range, scope type (MODULE/CLASS/METHOD/etc.)
+    # - Symbols: Variables, constants, parameters defined in this scope
+    # - Nested scopes: Child scopes (e.g., methods within a class)
+    #
+    # Created by: Extractor (during symbol extraction)
+    # Used by: ScopeBatcher (batching), ServiceVersion (wrapping for upload)
+    # Serialized to: JSON via to_h/to_json for upload to agent
+    #
+    # @api private
+    class Scope
+      attr_reader :scope_type, :name, :source_file, :start_line, :end_line,
+        # Ranges of executable lines [{start:, end:}]. Three states:
+        # - nil: not computed (source unreadable, native/C-extension method)
+        # - []: computed but no executable lines found (comments/whitespace only)
+        # - non-empty: computed, contains executable line ranges
+        # nil and [] both serialize as has_injectible_lines: false on METHOD
+        # scopes. Key is absent on non-METHOD scopes. The wire format key
+        # name keeps the historical spelling +injectible+ for backend
+        # compatibility; the Ruby identifier is +targetable_lines+.
+        :targetable_lines,
+        :language_specifics, :symbols, :scopes
+
+      # Initialize a new Scope
+      # @param scope_type [String] Type of scope (FILE, MODULE, CLASS, METHOD)
+      # @param name [String, nil] Name of the scope (class name, method name, etc.)
+      # @param source_file [String, nil] Path to source file
+      # @param start_line [Integer, nil] Starting line number (UNKNOWN_MIN_LINE for unknown)
+      # @param end_line [Integer, nil] Ending line number (UNKNOWN_MAX_LINE for entire file)
+      # @param targetable_lines [Array<Hash>, nil] Ranges of executable lines [{start:, end:}]
+      # @param language_specifics [Hash, nil] Ruby-specific metadata
+      # @param symbols [Array<Symbol>, nil] Symbols defined in this scope
+      # @param scopes [Array<Scope>, nil] Nested child scopes
+      def initialize(
+        scope_type:,
+        name: nil,
+        source_file: nil,
+        start_line: nil,
+        end_line: nil,
+        targetable_lines: nil,
+        language_specifics: nil,
+        symbols: nil,
+        scopes: nil
+      )
+        @scope_type = scope_type
+        @name = name
+        @source_file = source_file
+        @start_line = start_line
+        @end_line = end_line
+        @targetable_lines = targetable_lines
+        @language_specifics = language_specifics || {}
+        @symbols = symbols || []
+        @scopes = scopes || []
+      end
+
+      # @return [Boolean] true when targetable_lines is non-nil and non-empty
+      def targetable_lines?
+        !targetable_lines.nil? && !targetable_lines.empty?
+      end
+
+      # Convert scope to Hash for JSON serialization.
+      # Removes nil values to reduce payload size.
+      # @return [Hash] Scope as hash with symbol keys
+      def to_h
+        h = {
+          scope_type: scope_type,
+          name: name,
+          source_file: source_file,
+          start_line: start_line,
+          end_line: end_line,
+          language_specifics: language_specifics.empty? ? nil : language_specifics,
+          symbols: symbols.empty? ? nil : symbols.map(&:to_h),
+          scopes: scopes.empty? ? nil : scopes.map(&:to_h),
+        }
+        h.compact!
+        # Targetable lines only on METHOD scopes (per spec — not on CLASS/MODULE/FILE).
+        # Always emit has_injectible_lines (even when false) on METHOD scopes.
+        # Wire format keeps the historical spelling +injectible+; Ruby identifier
+        # is +targetable_lines+.
+        if scope_type == 'METHOD'
+          h[:has_injectible_lines] = targetable_lines? # steep:ignore ArgumentTypeMismatch
+          h[:injectible_lines] = targetable_lines if targetable_lines && !targetable_lines.empty?
+        end
+        h
+      end
+
+      # Serialize scope to JSON.
+      # @return [String] JSON string representation
+      def to_json(_state = nil)
+        JSON.generate(to_h)
+      end
+    end
+  end
+end

data/lib/datadog/symbol_database/scope_batcher.rb

@@ -0,0 +1,292 @@
+# frozen_string_literal: true
+
+require 'set'
+require_relative '../di/fatal_exceptions'
+
+module Datadog
+  module SymbolDatabase
+    # Batches extracted scopes and triggers uploads at appropriate times.
+    #
+    # Implements two upload triggers:
+    # 1. Size-based: Immediate upload when 400 scopes collected (MAX_SCOPES)
+    # 2. Time-based: Upload after 1 second of inactivity (debounce timer, not periodic)
+    #
+    # Also provides:
+    # - Deduplication: Tracks uploaded module names to prevent re-uploads
+    # - File limiting: Stops after 10,000 files to prevent runaway extraction
+    # - Thread safety: Mutex-protected state for concurrent access
+    #
+    # Timer implementation: A single long-lived thread waits on a ConditionVariable
+    # with a timeout. Each add_scope signals the CV to reset the deadline. When the
+    # timeout expires without a signal, the timer fires and flushes the batch.
+    # This avoids creating/destroying a thread per add_scope call.
+    #
+    # Flow: Extractor → add_scope → (batch or timer) → Uploader
+    # Created by: Component (during initialization)
+    # Calls: Uploader.upload_scopes when batch full or timer fires
+    #
+    # @api private
+    class ScopeBatcher
+      # Maximum scopes per batch before triggering immediate upload.
+      # This matches the batch size used in Java and Python tracers to ensure
+      # consistent upload behavior across languages.
+      MAX_SCOPES = 400
+      INACTIVITY_TIMEOUT = 1.0  # seconds
+      # Maximum unique files to track before stopping extraction.
+      # This prevents runaway memory usage in applications with very large
+      # numbers of loaded classes (e.g., heavily modularized Rails apps).
+      MAX_FILES = 10_000
+      # Seconds to wait for the timer thread to exit when joining during
+      # shutdown or reset. Bounded so a misbehaving thread cannot hang the
+      # caller indefinitely.
+      TIMER_JOIN_TIMEOUT = 5
+
+      # Initialize batching context.
+      # @param uploader [Uploader] Uploader instance for triggering uploads
+      # @param logger [Logger] Logger for diagnostics
+      # @param on_upload [Proc, nil] Optional callback called after upload (for testing)
+      # @param timer_enabled [Boolean] Enable async timer (default true, false for tests)
+      def initialize(uploader, logger:, on_upload: nil, timer_enabled: true)
+        @uploader = uploader
+        @logger = logger
+        @on_upload = on_upload
+        @timer_enabled = timer_enabled
+        @scopes = []
+        @mutex = Mutex.new
+        @file_count = 0
+        @uploaded_modules = Set.new
+
+        # Timer state: single long-lived thread + ConditionVariable for debounce.
+        # @timer_signaled is set to true on each add_scope and cleared by the timer
+        # thread after waking. This flag is needed because ConditionVariable#wait
+        # does not distinguish signal vs timeout on Ruby < 3.2 (returns self in both
+        # cases). The flag gives a portable way to detect whether the wakeup was a
+        # signal (reset deadline) or a timeout (fire the timer).
+        @timer_cv = ConditionVariable.new
+        @timer_thread = nil
+        @timer_stopped = false
+        @timer_signaled = false
+      end
+
+      # Add a scope to the batch.
+      # Triggers immediate upload if batch reaches 400 scopes.
+      # Resets inactivity timer if batch not full.
+      # @param scope [Scope] The scope to add
+      # @return [void]
+      def add_scope(scope)
+        # @type var scopes_to_upload: ::Array[Scope]?
+        scopes_to_upload = nil
+
+        @mutex.synchronize do
+          # Check file limit (counts only unique accepted files; duplicates are
+          # filtered by the dedup check below and do not consume the budget).
+          if @file_count >= MAX_FILES
+            @logger.debug { "symdb: file limit (#{MAX_FILES}) reached, ignoring scope: #{scope.name}" }
+            return
+          end
+
+          # Check if already uploaded — duplicates do not count toward MAX_FILES
+          # so a re-extraction scenario does not exhaust the budget for unique scopes.
+          if @uploaded_modules.include?(scope.name)
+            @logger.trace { "symdb: skipping #{scope.name}: already uploaded" }
+            return
+          end
+
+          # Marked uploaded before perform_upload runs. This is intentional:
+          # symdb extraction is a one-shot operation per process — extract_all
+          # walks ObjectSpace once and never revisits a module within the same
+          # process. The Set deduplicates within a single extraction run, not
+          # across upload attempts. Failed uploads are not retried; symbols
+          # from a failed batch are lost until the next process restart, by
+          # design (matches Python and Go; Java retries via OkHttp, .NET via
+          # exponential backoff — Ruby does neither).
+          @uploaded_modules.add(scope.name)
+          @file_count += 1
+
+          # Add the scope
+          @scopes << scope
+
+          # Check if batch size reached (AFTER adding)
+          if @scopes.size >= MAX_SCOPES
+            # Prepare for upload (clear within mutex)
+            scopes_to_upload = @scopes.dup
+            @scopes.clear
+          end
+
+          # Signal the timer thread to reset its inactivity deadline.
+          # If batch was full, this is harmless — the timer will just
+          # re-check and find an empty batch if it fires.
+          ensure_timer_running
+          @timer_signaled = true
+          @timer_cv.signal
+        end
+
+        # Upload outside mutex (if batch was full)
+        perform_upload(scopes_to_upload) if scopes_to_upload
+      rescue Exception => e # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(e)
+        @logger.debug { "symdb: failed to add scope: #{e.class}: #{e.message}" }
+        # Don't propagate, continue operation
+      end
+
+      # Force upload of current batch immediately.
+      # @return [void]
+      def flush
+        # @type var scopes_to_upload: ::Array[Scope]?
+        scopes_to_upload = nil
+
+        @mutex.synchronize do
+          return if @scopes.empty?
+
+          scopes_to_upload = @scopes.dup
+          @scopes.clear
+        end
+
+        perform_upload(scopes_to_upload)
+      end
+
+      # Shutdown and upload remaining scopes.
+      # @return [void]
+      def shutdown
+        # @type var scopes_to_upload: ::Array[Scope]?
+        scopes_to_upload = nil
+        # @type var thread_to_join: ::Thread?
+        thread_to_join = nil
+
+        @mutex.synchronize do
+          @timer_stopped = true
+          @timer_cv.signal  # Wake the timer thread so it exits
+
+          # Capture the timer thread under the mutex so a concurrent add_scope
+          # cannot create a new thread that we'd accidentally orphan when we
+          # nil the field below.
+          thread_to_join = @timer_thread
+          @timer_thread = nil
+
+          scopes_to_upload = @scopes.dup
+          @scopes.clear
+        end
+
+        # Join the timer thread outside the mutex.
+        # The thread checks @timer_stopped and exits when signaled.
+        thread_to_join&.join(TIMER_JOIN_TIMEOUT)
+
+        # Upload outside mutex
+        perform_upload(scopes_to_upload) unless scopes_to_upload.nil? || scopes_to_upload.empty?
+      end
+
+      # Check if scopes are pending upload.
+      # @return [Boolean] true if scopes waiting in batch
+      def scopes_pending?
+        @mutex.synchronize { @scopes.any? }
+      end
+
+      # Get current batch size.
+      # @return [Integer] Number of scopes in current batch
+      def size
+        @mutex.synchronize { @scopes.size }
+      end
+
+      private
+
+      # Reset state. Private so production code cannot accidentally invoke it;
+      # tests call via +send(:reset)+.
+      # @return [void]
+      def reset
+        # @type var thread_to_join: ::Thread?
+        thread_to_join = nil
+
+        @mutex.synchronize do
+          @scopes.clear
+          @timer_stopped = true
+          @timer_cv.signal
+          @file_count = 0
+          @uploaded_modules.clear
+
+          # Capture under the mutex (see shutdown for rationale).
+          thread_to_join = @timer_thread
+          @timer_thread = nil
+        end
+
+        thread_to_join&.join(TIMER_JOIN_TIMEOUT)
+
+        # Allow timer to be restarted after reset
+        @mutex.synchronize do
+          @timer_stopped = false
+          @timer_signaled = false
+        end
+      end
+
+      # Start the timer thread if not already running.
+      # Must be called from within @mutex.synchronize.
+      # @return [void]
+      def ensure_timer_running
+        return unless @timer_enabled
+        return if @timer_thread&.alive?
+
+        @timer_stopped = false
+        @timer_signaled = false
+
+        @timer_thread = Thread.new do
+          timer_loop
+        end
+      end
+
+      # Timer thread main loop. Waits on the ConditionVariable with a timeout.
+      # Each signal resets the deadline (debounce). When the wait times out
+      # (no signal within INACTIVITY_TIMEOUT), the batch is flushed.
+      #
+      # Uses @timer_signaled flag instead of ConditionVariable#wait return value
+      # because Ruby < 3.2 returns self for both signal and timeout (no way to
+      # distinguish). The flag is set by add_scope before signaling, and cleared
+      # by the timer thread after waking.
+      # @return [void]
+      def timer_loop
+        loop do
+          should_flush = false
+
+          @mutex.synchronize do
+            return if @timer_stopped
+
+            @timer_signaled = false
+            @timer_cv.wait(@mutex, INACTIVITY_TIMEOUT)
+
+            return if @timer_stopped
+
+            if @timer_signaled
+              # Woke up because add_scope signaled — loop back to re-wait with
+              # a fresh timeout. This implements the debounce: the timeout resets
+              # on every scope addition.
+              next # steep:ignore BreakTypeMismatch
+            end
+
+            # Timed out (no signal within INACTIVITY_TIMEOUT). If there are
+            # scopes pending, flush them. Otherwise, loop back and wait again.
+            should_flush = !@scopes.empty?
+          end
+
+          if should_flush
+            flush
+          end
+        end
+      rescue Exception => e # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(e)
+        @logger.debug { "symdb: timer thread error: #{e.class}: #{e.message}" }
+      end
+
+      # Perform upload via uploader.
+      # @param scopes [Array<Scope>] Scopes to upload
+      # @return [void]
+      def perform_upload(scopes)
+        return if scopes.nil? || scopes.empty?
+
+        @uploader.upload_scopes(scopes)
+        @on_upload&.call(scopes)  # Notify tests after upload
+      rescue Exception => e # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(e)
+        @logger.debug { "symdb: upload failed: #{e.class}: #{e.message}" }
+        # Don't propagate, uploader handles retries
+      end
+    end
+  end
+end