appsignal 4.10.4-java → 5.0.0-java

data/lib/appsignal/integrations/que.rb

@@ -2,21 +2,184 @@
 
 module Appsignal
   module Integrations
+    # @!visibility private
+    #
+    # Reads and writes W3C trace context the way OpenTelemetry's Que
+    # instrumentation does: as `"key:value"` strings in the job's tags array
+    # (the only carrier Que's enqueue API exposes). Collector mode only.
+    module QueTraceContext
+      module_function
+
+      # Que has no header map, so context rides in the tags array. OTel writes
+      # each header as a `"key:value"` tag; mirror that exact format.
+      module TagSetter
+        def self.set(carrier, key, value)
+          carrier << "#{key}:#{value}"
+        end
+      end
+
+      # Que rejects jobs with too many or too-long tags, so injected context
+      # must stay within these or the enqueue would raise. Read the limits from
+      # Que when available, with the documented defaults as a fallback.
+      MAX_TAGS_COUNT =
+        defined?(::Que::Job::MAXIMUM_TAGS_COUNT) ? ::Que::Job::MAXIMUM_TAGS_COUNT : 5
+      MAX_TAG_LENGTH =
+        defined?(::Que::Job::MAXIMUM_TAG_LENGTH) ? ::Que::Job::MAXIMUM_TAG_LENGTH : 100
+
+      # Marks a job as one of a batch enqueued by `bulk_enqueue`, so the worker
+      # can tell the two enqueue paths apart. Que itself records nothing that
+      # distinguishes them: both insert paths write the same columns, and the
+      # job's `data` only ever holds its tags. So the enqueue side has to say so.
+      #
+      # This deliberately contains no colon. The trace context rides in the same
+      # tags array as `"key:value"` strings, so a reader that splits tags on the
+      # colon to rebuild the carrier drops this tag on its own. That keeps it out
+      # of the carrier for us and for OpenTelemetry's own Que instrumentation.
+      BULK_TAG = "appsignal.bulk_enqueue"
+
+      # Que only has tags from version 1.0 on, and they are the only carrier its
+      # enqueue API exposes. On Que 0.x there is nowhere to put the trace context
+      # that survives to the worker, so propagation is skipped there. Writing the
+      # context into the job's arguments instead would change the arguments the
+      # job is called with, which breaks the job.
+      TAGS_SUPPORTED = defined?(::Que::Job::MAXIMUM_TAGS_COUNT)
+
+      # Read the incoming context off the job's tags. Splits each `"key:value"`
+      # tag on the first colon back into a carrier hash, then extracts. Returns
+      # an `OpenTelemetry::Context`, or `nil` outside collector mode and on Que
+      # versions without tags.
+      def extract(tags)
+        return unless TAGS_SUPPORTED
+
+        Appsignal::OpenTelemetry.if_started do
+          carrier = Array(tags)
+            .map { |tag| tag.split(":", 2) }
+            .select { |pair| pair.size == 2 }
+            .to_h
+          ::OpenTelemetry.propagation.extract(carrier)
+        end
+      end
+
+      # Returns the tags array to enqueue the job with. In collector mode that is
+      # a copy with the current context injected, kept only if it still fits
+      # Que's limits: propagation is skipped rather than break the enqueue.
+      # Outside collector mode, and on Que versions without tags, the tags are
+      # returned unchanged.
+      #
+      # Pass `bulk` for a `bulk_enqueue` batch, which adds the bulk marker too.
+      # Both are kept or dropped together, because a batch missing its marker
+      # would parent every job in it to the one producer span.
+      def inject(tags, bulk: false)
+        original = Array(tags)
+        return original unless TAGS_SUPPORTED
+
+        injected = Appsignal::OpenTelemetry.if_started do
+          copy = original.dup
+          ::OpenTelemetry.propagation.inject(copy, :setter => TagSetter)
+          # The marker has nothing to link back to without a trace context, so
+          # only add it when the context was actually injected. The propagator
+          # writes nothing when there is no valid span to propagate.
+          copy << BULK_TAG if bulk && copy.length > original.length
+          copy
+        end
+        return original if injected.nil? || !within_limits?(injected)
+
+        injected
+      end
+
+      # Whether the job being performed was enqueued as part of a batch, which
+      # the enqueue side records by adding `BULK_TAG` to the job's tags.
+      def bulk?(tags)
+        Array(tags).include?(BULK_TAG)
+      end
+
+      # The class the Active Job adapter enqueues, with the serialized job data
+      # as its only argument. Que records the name in the job's `job_class`, and
+      # the name is fixed, because it is stored in every job record the adapter
+      # has ever written.
+      ACTIVE_JOB_WRAPPER = "ActiveJob::QueueAdapters::QueAdapter::JobWrapper"
+
+      # The trace context to continue: the Active Job layer when this is an
+      # Active Job job, the job's own tags otherwise. See `Appsignal::OpenTelemetry.extract_active_job_context`
+      # for why that layer wins.
+      # Que's tags are a carrier with a hard limit: five per job, shared with
+      # whatever the user puts there.
+      def extract_context(job_data, tags)
+        Appsignal::OpenTelemetry.extract_active_job_context(job_data) || extract(tags)
+      end
+
+      # The serialized Active Job job data inside a Que job, or nil when this is
+      # not an Active Job job.
+      #
+      # Que reads a job's arguments out of JSONB with symbol keys and only turns
+      # them back into strings on the way into Active Job, so they are
+      # stringified here too. Only the outer keys need it: the headers below them
+      # are an array of pairs of strings, which Que leaves alone.
+      def active_job_data(attrs)
+        return unless attrs[:job_class] == ACTIVE_JOB_WRAPPER
+
+        job_data = Array(attrs[:args]).first
+        job_data.transform_keys(&:to_s) if job_data.is_a?(Hash)
+      end
+
+      def within_limits?(tags)
+        tags.length <= MAX_TAGS_COUNT && tags.all? { |tag| tag.length <= MAX_TAG_LENGTH }
+      end
+    end
+
     # @!visibility private
     module QuePlugin
       def _run(*args)
+        local_attrs = respond_to?(:que_attrs) ? que_attrs : attrs
+        tags = local_attrs.dig(:data, :tags)
+
+        job_data = QueTraceContext.active_job_data(local_attrs)
+        # A Que batch says so with a tag, and an Active Job batch says so in the
+        # job data, so both have to be asked. See
+        # `Appsignal::OpenTelemetry.active_job_relationship` for why a batch
+        # links back rather than parenting under the span that enqueued it.
+        relationship =
+          if QueTraceContext.bulk?(tags)
+            :link
+          else
+            Appsignal::OpenTelemetry.active_job_relationship(job_data)
+          end
+
+        # Read the incoming trace context off the job's tags so the transaction
+        # links back to the enqueuer. No-op outside collector mode.
         transaction =
-          Appsignal::Transaction.create(Appsignal::Transaction::BACKGROUND_JOB)
+          Appsignal::Transaction.create(
+            Appsignal::Transaction::BACKGROUND_JOB,
+            :opentelemetry_context => QueTraceContext.extract_context(job_data, tags),
+            :opentelemetry_scope => ["appsignal-ruby/que", Appsignal::VERSION],
+            :opentelemetry_kind => :consumer,
+            :opentelemetry_relationship => relationship
+          )
+        # Describes this span as a job being performed. The messaging system is
+        # what the trace timeline reads to recognize background job work, and
+        # `que` is the value OpenTelemetry's own Que instrumentation uses.
+        transaction.add_opentelemetry_attributes(
+          Appsignal::OpenTelemetry::Messaging
+            .perform_attributes("que", :destination => local_attrs[:queue])
+        )
 
         begin
-          Appsignal.instrument("perform_job.que") { super }
+          Appsignal.instrument(
+            "perform_job.que",
+            :opentelemetry_scope => ["appsignal-ruby/que", Appsignal::VERSION]
+          ) do
+            Appsignal::Transaction.current.add_opentelemetry_attributes(
+              Appsignal::OpenTelemetry::Messaging
+                .perform_attributes("que", :destination => local_attrs[:queue])
+            )
+            super
+          end
         rescue Exception => error
           transaction.set_error(error)
           raise error
         ensure
-          local_attrs = respond_to?(:que_attrs) ? que_attrs : attrs
           transaction.set_action_if_nil("#{local_attrs[:job_class]}#run")
-          transaction.add_params_if_nil do
+          transaction.add_function_parameters_if_nil do
             {
               :arguments => local_attrs[:args]
             }.tap do |hash|
@@ -37,36 +200,91 @@ module Appsignal
 
     # @!visibility private
     #
-    # Prepended to `Que::Job`'s singleton so it records each enqueue as an
-    # `enqueue.que` event under the active transaction. Like all AppSignal
-    # events, it only records when there's an active transaction (e.g. enqueuing
-    # from within a web request or another job); otherwise it's a transparent
-    # pass-through.
+    # Prepended to `Que::Job`'s singleton so it wraps enqueues. Records the
+    # enqueue as an AppSignal event (a producer span in collector mode), and in
+    # collector mode writes the current trace context onto the job's tags so the
+    # job that later performs links back to it. Like all AppSignal events, the
+    # enqueue only records when there's an active transaction; otherwise it's a
+    # transparent pass-through.
     module QueClientPlugin
-      # The keyword arguments are captured into a single `**kwargs` hash, rather
-      # than declaring a `job_options:` keyword with a default, so that an
-      # implicit `super` forwards the original call unchanged. Declaring
-      # `job_options: {}` would bind that default even when the caller did not
-      # pass it, and `super` would then forward it to Que. On Que 0.14, whose
-      # `enqueue` takes only positional arguments, that extra keyword ends up
-      # persisted as an additional job argument.
-      def enqueue(*_args, **kwargs)
-        job_options = kwargs[:job_options] || {}
-
-        # Inside a `bulk_enqueue` block the batch is recorded once by the
-        # `bulk_enqueue` wrapper, so each inner enqueue is a pass-through to
-        # avoid recording an event per job.
+      # The keyword arguments are captured into a single `kwargs` hash, rather
+      # than declaring a `job_options:` keyword with a default, so that a
+      # keyword the caller did not pass is never forwarded to Que. See
+      # `forward_job_options` for why that matters.
+      def enqueue(*args, **kwargs)
+        # Inside a `bulk_enqueue` block the per-job enqueue must stay a
+        # pass-through: tags come from `bulk_enqueue`'s own `job_options` (Que
+        # raises if an inner enqueue passes them), and the batch's event and
+        # propagation are recorded once by the `bulk_enqueue` wrapper.
         return super if Thread.current[:appsignal_que_bulk_enqueue]
 
-        # Under Active Job the enqueue is already recorded as an
-        # `enqueue.active_job` event, so skip recording it again here.
-        return super if Appsignal::Transaction.current? &&
-          Appsignal::Transaction.current.job_enqueue_events_suppressed?
+        job_options = kwargs[:job_options] || {}
 
         # Resolve the job class the way Que does: an explicit `:job_class`, else
         # the class `enqueue` was called on.
         title = "enqueue #{job_options[:job_class] || name} job"
-        Appsignal.instrument("enqueue.que", title) { super }
+        record_enqueue(job_options, "enqueue.que", title) do |merged|
+          super(*args, **forward_job_options(kwargs, merged))
+        end
+      end
+
+      private
+
+      # Builds the keyword arguments to enqueue the job with. Passes
+      # `job_options` on to Que only when the caller passed it, or when the
+      # trace context was injected into it. Adding a `job_options` keyword to a
+      # call that did not have one breaks Que 0.x: it does not recognise the
+      # keyword, so it stores it as an extra job argument, and the job then
+      # fails because it is called with one argument too many.
+      def forward_job_options(kwargs, job_options)
+        return kwargs unless kwargs.key?(:job_options) || job_options.any?
+
+        kwargs.merge(:job_options => job_options)
+      end
+
+      # Records the enqueue as a producer event and, in collector mode, injects
+      # the current trace context into the job's tags so the job that later
+      # performs links back. Yields the (possibly tag-augmented) `job_options` to
+      # do the actual enqueue.
+      def record_enqueue(job_options, event_name, title, bulk: false)
+        # When enqueue instrumentation is disabled, drop the trace context along
+        # with the event, so yield the job options untouched. Without an enqueue
+        # event there is no producer span, so the context we would write is that
+        # of whatever span is current, such as the surrounding web request. The
+        # job that performs later would then link back to a span that is not a
+        # producer.
+        if Appsignal.config && !Appsignal.config[:enable_job_enqueue_instrumentation]
+          return yield job_options
+        end
+
+        # Under Active Job the enqueue is already recorded as an
+        # `enqueue.active_job` event, so skip recording it again here. The trace
+        # context is still injected so the performed job links back.
+        if Appsignal::Transaction.current? &&
+            Appsignal::Transaction.current.job_enqueue_events_suppressed?
+          return yield job_options_with_context(job_options, :bulk => bulk)
+        end
+
+        Appsignal.instrument(
+          event_name,
+          title,
+          :opentelemetry_kind => :producer,
+          :opentelemetry_scope => ["appsignal-ruby/que", Appsignal::VERSION]
+        ) do
+          Appsignal::Transaction.current.add_opentelemetry_attributes(
+            Appsignal::OpenTelemetry::Messaging
+              .enqueue_attributes("que", :destination => job_options[:queue])
+          )
+          yield job_options_with_context(job_options, :bulk => bulk)
+        end
+      end
+
+      # In collector mode, injects the current trace context into a copy of the
+      # job's tags and returns the tag-augmented `job_options`; a no-op that
+      # returns `job_options` unchanged outside collector mode.
+      def job_options_with_context(job_options, bulk: false)
+        tags = QueTraceContext.inject(job_options[:tags], :bulk => bulk)
+        tags.empty? ? job_options : job_options.merge(:tags => tags)
       end
     end
 
@@ -74,22 +292,23 @@ module Appsignal
     #
     # `bulk_enqueue` exists only on Que 2+, so this lives in its own module that
     # the hook prepends only when Que has the method -- otherwise we'd define a
-    # `bulk_enqueue` on Que versions that have none. The whole batch records a
-    # single `bulk_enqueue.que` event; the inner enqueues are pass-throughs.
+    # `bulk_enqueue` on Que versions that have none. The whole batch shares one
+    # `job_options`, so it records a single `bulk_enqueue.que` producer event and
+    # the inner enqueues are pass-throughs.
     module QueBulkClientPlugin
-      def bulk_enqueue(*_args, job_options: {}, **_rest)
-        # Under Active Job the enqueue is already recorded as an
-        # `enqueue.active_job` event, so skip recording it again here.
-        return super if Appsignal::Transaction.current? &&
-          Appsignal::Transaction.current.job_enqueue_events_suppressed?
-
-        Appsignal.instrument("bulk_enqueue.que", bulk_enqueue_title(job_options)) do
+      def bulk_enqueue(job_options: {}, **rest, &block)
+        record_enqueue(
+          job_options,
+          "bulk_enqueue.que",
+          bulk_enqueue_title(job_options),
+          :bulk => true
+        ) do |merged|
           # Flag the batch so the enqueues this block triggers pass through
           # without recording, without reading Que's internal bulk state.
           was_bulk = Thread.current[:appsignal_que_bulk_enqueue]
           Thread.current[:appsignal_que_bulk_enqueue] = true
           begin
-            super
+            super(:job_options => merged, **rest, &block)
           ensure
             Thread.current[:appsignal_que_bulk_enqueue] = was_bulk
           end

data/lib/appsignal/integrations/railtie.rb

@@ -98,7 +98,10 @@ module Appsignal
           is_rails_runner = source == "application.runner.railties"
           namespace, action_name, tags, custom_data = context_for(context.dup)
 
-          Appsignal.report_error(error) do |transaction|
+          Appsignal.report_error(
+            error,
+            :opentelemetry_scope => ["appsignal-ruby/rails", Appsignal::VERSION]
+          ) do |transaction|
             if namespace
               transaction.set_namespace(namespace)
             elsif is_rails_runner

data/lib/appsignal/integrations/rake.rb

@@ -22,7 +22,10 @@ module Appsignal
           end
 
         begin
-          Appsignal.instrument "task.rake" do
+          Appsignal.instrument(
+            "task.rake",
+            :opentelemetry_scope => ["appsignal-ruby/rake", Appsignal::VERSION]
+          ) do
             super
           end
         rescue Exception => error
@@ -38,7 +41,7 @@ module Appsignal
             params, _ = args
             params = params.to_hash if params.respond_to?(:to_hash)
             transaction.set_action(name)
-            transaction.add_params_if_nil(params)
+            transaction.add_request_payload_if_nil(params)
             transaction.complete
           end
         end
@@ -47,7 +50,10 @@ module Appsignal
       private
 
       def _appsignal_create_transaction
-        Appsignal::Transaction.create("rake")
+        Appsignal::Transaction.create(
+          "rake",
+          :opentelemetry_scope => ["appsignal-ruby/rake", Appsignal::VERSION]
+        )
       end
     end
 

data/lib/appsignal/integrations/redis.rb

@@ -11,8 +11,29 @@ module Appsignal
           else
             "#{command[0]}#{" ?" * (command.size - 1)}"
           end
+        operation_name = command[0].to_s
 
-        Appsignal.instrument "query.redis", id, sanitized_command do
+        Appsignal.instrument(
+          "query.redis",
+          id,
+          sanitized_command,
+          :opentelemetry_kind => :client,
+          :opentelemetry_scope => ["appsignal-ruby/redis", Appsignal::VERSION]
+        ) do
+          # Names the datastore this span talks to, which is what the trace
+          # timeline reads to recognize a cache call, along with the command and
+          # the database it ran against.
+          #
+          # The command stays in the event body rather than moving to
+          # `db.query.text`: it is sanitized here already, and the collector
+          # sanitizes `db.query.text` again for Redis, which would mangle it.
+          Appsignal::Transaction.current.add_opentelemetry_attributes(
+            {
+              "db.system.name" => "redis",
+              "db.operation.name" => (operation_name unless operation_name.empty?),
+              "db.namespace" => (db.to_s if respond_to?(:db) && db)
+            }.compact
+          )
           super
         end
       end

data/lib/appsignal/integrations/redis_client.rb

@@ -11,8 +11,29 @@ module Appsignal
           else
             "#{command[0]}#{" ?" * (command.size - 1)}"
           end
+        operation_name = command[0].to_s
 
-        Appsignal.instrument "query.redis", @config.id, sanitized_command do
+        Appsignal.instrument(
+          "query.redis",
+          @config.id,
+          sanitized_command,
+          :opentelemetry_kind => :client,
+          :opentelemetry_scope => ["appsignal-ruby/redis_client", Appsignal::VERSION]
+        ) do
+          # Names the datastore this span talks to, which is what the trace
+          # timeline reads to recognize a cache call, along with the command and
+          # the database it ran against.
+          #
+          # The command stays in the event body rather than moving to
+          # `db.query.text`: it is sanitized here already, and the collector
+          # sanitizes `db.query.text` again for Redis, which would mangle it.
+          Appsignal::Transaction.current.add_opentelemetry_attributes(
+            {
+              "db.system.name" => "redis",
+              "db.operation.name" => (operation_name unless operation_name.empty?),
+              "db.namespace" => (@config.db.to_s if @config.respond_to?(:db) && @config.db)
+            }.compact
+          )
           super
         end
       end

data/lib/appsignal/integrations/resque.rb

@@ -5,9 +5,33 @@ module Appsignal
     # @!visibility private
     module ResqueIntegration
       def perform
-        transaction = Appsignal::Transaction.create(Appsignal::Transaction::BACKGROUND_JOB)
+        # Read trace context off the job so the transaction links back to the
+        # enqueuer. No-op outside collector mode.
+        job_data = ResqueHelpers.active_job_data(payload)
+        transaction = Appsignal::Transaction.create(
+          Appsignal::Transaction::BACKGROUND_JOB,
+          :opentelemetry_context => ResqueHelpers.extract_context(payload, job_data),
+          :opentelemetry_scope => ["appsignal-ruby/resque", Appsignal::VERSION],
+          :opentelemetry_kind => :consumer,
+          :opentelemetry_relationship =>
+            Appsignal::OpenTelemetry.active_job_relationship(job_data)
+        )
+        # Describes this span as a job being performed. The messaging system is
+        # what the trace timeline reads to recognize background job work, and
+        # `resque` is the value OpenTelemetry's own Resque instrumentation uses.
+        transaction.add_opentelemetry_attributes(
+          Appsignal::OpenTelemetry::Messaging
+            .perform_attributes("resque", :destination => queue)
+        )
 
-        Appsignal.instrument "perform.resque" do
+        Appsignal.instrument(
+          "perform.resque",
+          :opentelemetry_scope => ["appsignal-ruby/resque", Appsignal::VERSION]
+        ) do
+          Appsignal::Transaction.current.add_opentelemetry_attributes(
+            Appsignal::OpenTelemetry::Messaging
+              .perform_attributes("resque", :destination => queue)
+          )
           super
         end
       rescue Exception => exception
@@ -16,7 +40,7 @@ module Appsignal
       ensure
         if transaction
           transaction.set_action_if_nil("#{payload["class"]}#perform")
-          transaction.add_params_if_nil { ResqueHelpers.arguments(payload) }
+          transaction.add_function_parameters_if_nil { ResqueHelpers.arguments(payload) }
           transaction.add_tags("queue" => queue)
 
           Appsignal::Transaction.complete_current!
@@ -25,8 +49,10 @@ module Appsignal
       end
     end
 
-    # Wraps `Resque.push` to record an `enqueue.resque` event so the enqueue
-    # shows up under the active transaction.
+    # Wraps `Resque.push` to record an `enqueue.resque` event so the
+    # enqueue shows up under the active transaction (both modes), and in
+    # collector mode writes the trace context onto the job hash so the job that
+    # later performs links back to it.
     #
     # Like all AppSignal events, this only records when there's an active
     # transaction (e.g. enqueuing from within a web request or another job).
@@ -34,26 +60,70 @@ module Appsignal
     #
     # @!visibility private
     module ResquePushIntegration
-      def push(_queue, item)
+      def push(queue, item)
+        # When enqueue instrumentation is disabled, drop the trace context along
+        # with the event. Without an enqueue event there is no producer span, so
+        # the context we would write is that of whatever span is current, such as
+        # the surrounding web request. The job that performs later would then
+        # link back to a span that is not a producer.
+        return super if Appsignal.config && !Appsignal.config[:enable_job_enqueue_instrumentation]
+
         # Under Active Job the enqueue is already recorded as an
-        # `enqueue.active_job` event, so skip recording it again here.
-        return super if Appsignal::Transaction.current? &&
-          Appsignal::Transaction.current.job_enqueue_events_suppressed?
+        # `enqueue.active_job` event, so skip recording it again here. The trace
+        # context is still injected so the performed job links back.
+        if Appsignal::Transaction.current? &&
+            Appsignal::Transaction.current.job_enqueue_events_suppressed?
+          Appsignal::OpenTelemetry.inject_context(item)
+          return super
+        end
 
-        Appsignal.instrument("enqueue.resque", "enqueue #{item["class"]} job") { super }
+        Appsignal.instrument(
+          "enqueue.resque",
+          "enqueue #{item["class"]} job",
+          :opentelemetry_kind => :producer,
+          :opentelemetry_scope => ["appsignal-ruby/resque", Appsignal::VERSION]
+        ) do
+          Appsignal::Transaction.current.add_opentelemetry_attributes(
+            Appsignal::OpenTelemetry::Messaging
+              .enqueue_attributes("resque", :destination => queue)
+          )
+          Appsignal::OpenTelemetry.inject_context(item)
+          super
+        end
       end
     end
 
     # @!visibility private
     class ResqueHelpers
+      # The class the Active Job adapter enqueues, with the serialized job data
+      # as its only argument.
+      ACTIVE_JOB_WRAPPER = "ActiveJob::QueueAdapters::ResqueAdapter::JobWrapper"
+
       def self.arguments(payload)
         case payload["class"]
-        when "ActiveJob::QueueAdapters::ResqueAdapter::JobWrapper"
+        when ACTIVE_JOB_WRAPPER
           nil # Set in the ActiveJob integration
         else
           payload["args"]
         end
       end
+
+      # The serialized Active Job job data inside a Resque job, or nil when this
+      # is not an Active Job job.
+      def self.active_job_data(payload)
+        return unless payload["class"] == ACTIVE_JOB_WRAPPER
+
+        job_data = payload["args"]&.first
+        job_data if job_data.is_a?(Hash)
+      end
+
+      # The trace context to continue: the Active Job layer when this is an
+      # Active Job job, the Resque job itself otherwise. See `Appsignal::OpenTelemetry.extract_active_job_context`
+      # for why that layer wins.
+      def self.extract_context(payload, job_data)
+        Appsignal::OpenTelemetry.extract_active_job_context(job_data) ||
+          Appsignal::OpenTelemetry.extract_job_context(payload)
+      end
     end
   end
 end