newrelic-infinite_tracing 6.11.0.365

data/lib/infinite_tracing/connection.rb

@@ -0,0 +1,171 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# frozen_string_literal: true
+
+# The connection class manages the channel and connection to the gRPC server.
+#
+# Calls to the gRPC server are blocked until the agent connects to the collector
+# and obtains a license_key and agent_run_token from the server side configuration.
+#
+# If the agent is instructed to reconnect by the collector, that event triggers
+# server_source_configuration_added, which this connection is subscribed to and will
+# also notify the client to restart and re-establish its bi-directional streaming
+# with the gRPC server.
+#
+# NOTE: Connection is implemented as a Singleton and it also only ever expects *one*
+# client instance by design.
+module NewRelic::Agent
+  module InfiniteTracing
+    class Connection
+
+
+      # listens for server side configurations added to the agent.  When a new config is
+      # added, we have a new agent run token and need to restart the client's RPC stream
+      # with the new metadata information.
+      NewRelic::Agent.agent.events.subscribe(:server_source_configuration_added) do
+        begin
+          Connection.instance.notify_agent_started
+        rescue => error
+          NewRelic::Agent.logger.error \
+            "Error during notify :server_source_configuration_added event",
+            error
+        end
+      end
+
+      class << self
+
+        def instance
+          @@instance ||= new
+        end
+
+        def reset
+          @@instance = new
+        end
+
+        # RPC calls will pass the calling client instance in.  We track this
+        # so we're able to signal the client to restart when connectivity to the
+        # server is disrupted.
+        def record_spans client, enumerator, exponential_backoff
+          instance.record_spans client, enumerator, exponential_backoff
+        end
+
+        # RPC calls will pass the calling client instance in.  We track this
+        # so we're able to signal the client to restart when connectivity to the
+        # server is disrupted.
+        def record_span_batches client, enumerator, exponential_backoff
+          instance.record_span_batch client, enumerator, exponential_backoff
+        end
+
+        def metadata
+          instance.metadata
+        end
+      end
+
+      # We attempt to connect and record spans with reconnection backoff in order to deal with
+      # unavailable errors coming from the stub being created and record_span call
+      def record_spans client, enumerator, exponential_backoff
+          @active_clients[client] = client
+          with_reconnection_backoff(exponential_backoff) { rpc.record_span enumerator, metadata: metadata }
+      end
+
+      # RPC calls will pass the calling client instance in.  We track this
+      # so we're able to signal the client to restart when connectivity to the
+      # server is disrupted.
+      def record_span_batches client, enumerator, exponential_backoff
+        @active_clients[client] = client
+        with_reconnection_backoff(exponential_backoff) { rpc.record_span_batch enumerator, metadata: metadata }
+      end
+
+      # Acquires the new channel stub for the RPC calls.
+      # We attempt to connect and record spans with reconnection backoff in order to deal with
+      # unavailable errors coming from the stub being created and record_span call
+      def rpc
+        @rpc ||= Channel.new.stub
+      end
+
+      # The metadata for the RPC calls is a blocking call waiting for the Agent to
+      # connect and receive the server side configuration, which contains the license_key
+      # as well as the agent_id (agent_run_token).
+      def metadata
+        return @metadata if @metadata
+
+        @lock.synchronize do
+          @agent_started.wait(@lock) if !@agent_connected
+
+          @metadata = {
+            "license_key" => license_key,
+            "agent_run_token" => agent_id
+          }
+        end
+      end
+
+      # Initializes rpc so we can get a Channel and Stub (connect to gRPC server)
+      # Initializes metadata so we use newest values in establishing channel
+      # Sets the agent_connected flag and signals the agent started so any
+      # waiting locks (rpc calls ahead of the agent connecting) can proceed.
+      def notify_agent_started
+        @lock.synchronize do
+          @rpc = nil
+          @metadata = nil
+          @agent_connected = true
+          @agent_started.signal
+        end
+        @active_clients.each_value(&:restart)
+      end
+
+      private
+
+      # prepares the connection to wait for the agent to connect and have an
+      # agent_run_token ready for metadata on rpc calls.
+      def initialize
+        @active_clients = {}
+        @rpc = nil
+        @metadata = nil
+        @connection_attempts = 0
+        @agent_connected = NewRelic::Agent.agent.connected?
+        @agent_started = ConditionVariable.new
+        @lock = Mutex.new
+      end
+
+      # The agent run token, which is only available after a server source configuration has
+      # been added to the agent's config stack.
+      def agent_id
+        NewRelic::Agent.agent.service.agent_id.to_s
+      end
+
+      def license_key
+        NewRelic::Agent.config[:license_key]
+      end
+
+      # Continues retrying the connection at backoff intervals until a successful connection is made
+      def with_reconnection_backoff exponential_backoff=true, &block
+        @connection_attempts = 0
+        begin
+          yield
+        rescue => exception
+          retry_connection_period = retry_connection_period(exponential_backoff)
+          ::NewRelic::Agent.logger.error "Error establishing connection with infinite tracing service:", exception
+          ::NewRelic::Agent.logger.info "Will re-attempt infinte tracing connection in #{retry_connection_period} seconds"
+          sleep retry_connection_period
+          note_connect_failure
+          retry
+        end
+      end
+
+      def retry_connection_period exponential_backoff=true
+        if exponential_backoff
+          NewRelic::CONNECT_RETRY_PERIODS[@connection_attempts] || NewRelic::MAX_RETRY_PERIOD
+        else
+          NewRelic::MIN_RETRY_PERIOD
+        end
+      end
+
+      # broken out to help for testing
+      def note_connect_failure
+        @connection_attempts += 1
+      end
+
+    end
+  end
+end

data/lib/infinite_tracing/constants.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic::Agent
+  module InfiniteTracing
+    module Constants
+      SUPPORTABILITY_PREFIX   = "Supportability/InfiniteTracing/Span"
+
+      SPANS_SEEN_METRIC       = "#{SUPPORTABILITY_PREFIX}/Seen"
+      SPANS_SENT_METRIC       = "#{SUPPORTABILITY_PREFIX}/Sent"
+      QUEUE_DUMPED_METRIC     = "#{SUPPORTABILITY_PREFIX}/AgentQueueDumped"
+      RESPONSE_ERROR_METRIC   = "#{SUPPORTABILITY_PREFIX}/Response/Error"
+
+      GRPC_ERROR_NAME_METRIC  = "#{SUPPORTABILITY_PREFIX}/gRPC/%s"
+      GRPC_OTHER_ERROR_METRIC = GRPC_ERROR_NAME_METRIC % "Other"
+    end
+  end
+end

data/lib/infinite_tracing/proto.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# frozen_string_literal: true
+
+require 'grpc'
+require 'google/protobuf'
+
+require_relative 'proto/infinite_tracing_pb'
+require_relative 'proto/infinite_tracing_services_pb'
+
+# Mapping gRPC namespaced classes into New Relic's
+module NewRelic::Agent::InfiniteTracing
+  Span = Com::Newrelic::Trace::V1::Span
+  SpanBatch = Com::Newrelic::Trace::V1::SpanBatch
+  AttributeValue = Com::Newrelic::Trace::V1::AttributeValue
+  RecordStatus = Com::Newrelic::Trace::V1::RecordStatus
+end

data/lib/infinite_tracing/proto/infinite_tracing_pb.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# frozen_string_literal: true
+
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: infinite_tracing.proto
+
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "com.newrelic.trace.v1.SpanBatch" do
+    repeated :spans, :message, 1, "com.newrelic.trace.v1.Span"
+  end
+  add_message "com.newrelic.trace.v1.Span" do
+    optional :trace_id, :string, 1
+    map :intrinsics, :string, :message, 2, "com.newrelic.trace.v1.AttributeValue"
+    map :user_attributes, :string, :message, 3, "com.newrelic.trace.v1.AttributeValue"
+    map :agent_attributes, :string, :message, 4, "com.newrelic.trace.v1.AttributeValue"
+  end
+  add_message "com.newrelic.trace.v1.AttributeValue" do
+    oneof :value do
+      optional :string_value, :string, 1
+      optional :bool_value, :bool, 2
+      optional :int_value, :int64, 3
+      optional :double_value, :double, 4
+    end
+  end
+  add_message "com.newrelic.trace.v1.RecordStatus" do
+    optional :messages_seen, :uint64, 1
+  end
+end
+
+module Com
+  module Newrelic
+    module Trace
+      module V1
+        SpanBatch = Google::Protobuf::DescriptorPool.generated_pool.lookup("com.newrelic.trace.v1.SpanBatch").msgclass
+        Span = Google::Protobuf::DescriptorPool.generated_pool.lookup("com.newrelic.trace.v1.Span").msgclass
+        AttributeValue = Google::Protobuf::DescriptorPool.generated_pool.lookup("com.newrelic.trace.v1.AttributeValue").msgclass
+        RecordStatus = Google::Protobuf::DescriptorPool.generated_pool.lookup("com.newrelic.trace.v1.RecordStatus").msgclass
+      end
+    end
+  end
+end

data/lib/infinite_tracing/proto/infinite_tracing_services_pb.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# frozen_string_literal: true
+
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: infinite_tracing.proto for package 'com.newrelic.trace.v1'
+# Original file comments:
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+#
+
+
+module Com
+  module Newrelic
+    module Trace
+      module V1
+        module IngestService
+          class Service
+
+            include GRPC::GenericService
+
+            self.marshal_class_method = :encode
+            self.unmarshal_class_method = :decode
+            self.service_name = 'com.newrelic.trace.v1.IngestService'
+
+            # Accepts a stream of Span messages, and returns an irregular stream of
+            # RecordStatus messages.
+            rpc :RecordSpan, stream(Span), stream(RecordStatus)
+            # Accepts a stream of SpanBatch messages, and returns an irregular
+            # stream of RecordStatus messages. This endpoint can be used to improve
+            # throughput when Span messages are small
+            rpc :RecordSpanBatch, stream(SpanBatch), stream(RecordStatus)
+          end
+
+          Stub = Service.rpc_stub_class
+        end
+      end
+    end
+  end
+end

data/lib/infinite_tracing/record_status_handler.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic::Agent
+  module InfiniteTracing
+    class RecordStatusHandler
+      def initialize client, enumerator
+        @client = client
+        @enumerator = enumerator
+        @messages_seen = nil
+        @lock = Mutex.new
+        @lock.synchronize { @worker = start_handler }
+      end
+
+      def messages_seen
+        @messages_seen ? @messages_seen.messages_seen : 0
+      end
+
+      def start_handler
+        Worker.new self.class.name do
+          begin
+            @enumerator.each do |response|
+              break if response.nil? || response.is_a?(Exception)
+              @lock.synchronize do
+                @messages_seen = response
+                NewRelic::Agent.logger.debug "gRPC Infinite Tracer Observer saw #{messages_seen} messages"
+              end
+            end
+          rescue => error
+            @lock.synchronize { @client.handle_error error }
+          end
+        end
+      rescue => error
+        NewRelic::Agent.logger.error "gRPC Worker Error", error
+      end
+
+      def stop
+        return if @worker.nil?
+        @lock.synchronize do
+          NewRelic::Agent.logger.debug "gRPC Stopping Response Handler"
+          @worker.stop
+          @worker = nil
+        end
+      end
+    end
+  end
+end

data/lib/infinite_tracing/streaming_buffer.rb

@@ -0,0 +1,162 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# frozen_string_literal: true
+
+# The StreamingBuffer class provides an Enumerator to the standard Ruby Queue
+# class.  The enumerator is blocking while the queue is empty.
+module NewRelic::Agent
+  module InfiniteTracing
+
+    BATCH_SIZE = 100
+
+    class StreamingBuffer
+      include Constants
+      include Enumerable
+      extend Forwardable
+      def_delegators :@queue, :empty?, :num_waiting, :push
+      
+      DEFAULT_QUEUE_SIZE = 10_000
+      FLUSH_DELAY        = 0.005
+      MAX_FLUSH_WAIT     = 3 # three seconds
+      
+      attr_reader :queue
+
+      def initialize max_size = DEFAULT_QUEUE_SIZE
+        @max_size = max_size
+        @lock = Mutex.new
+        @queue = Queue.new
+        @batch = Array.new
+      end
+
+      # Dumps the contents of this streaming buffer onto 
+      # the given buffer and closes the queue
+      def transfer new_buffer
+        @lock.synchronize do
+          until @queue.empty? do new_buffer.push @queue.pop end
+          @queue.close
+        end
+      end
+
+      # Pushes the segment given onto the queue.
+      #
+      # If the queue is at capacity, it is dumped and a
+      # supportability metric is recorded for the event.
+      #
+      # When a restart signal is received, the queue is
+      # locked with a mutex, blocking the push until
+      # the queue has restarted.
+      def << segment
+        @lock.synchronize do
+          clear_queue if @queue.size >= @max_size
+          NewRelic::Agent.increment_metric SPANS_SEEN_METRIC
+          @queue.push segment
+        end
+      end
+
+      # Drops all segments from the queue and records a
+      # supportability metric for the event.
+      def clear_queue
+        @queue.clear
+        NewRelic::Agent.increment_metric QUEUE_DUMPED_METRIC
+      end
+
+      # # Waits for the queue to be fully consumed or for the
+      # # waiting consumers to release.
+      # def flush_queue
+      #   @queue.num_waiting.times { @queue.push nil }
+      #   close_queue
+      #   until @queue.empty? do sleep(FLUSH_DELAY) end
+      # end
+
+      # Waits for the queue to be fully consumed or for the
+      # waiting consumers to release.
+      def flush_queue
+        @queue.num_waiting.times { @queue.push nil }
+        close_queue
+
+        # Logs if we're throwing away spans because nothing's 
+        # waiting to take them off the queue.
+        if @queue.num_waiting == 0 && !@queue.empty?
+          NewRelic::Agent.logger.warn "Discarding #{@queue.size} segments on Streaming Buffer"
+          return
+        end
+
+        # Only wait a short while for queue to flush
+        cutoff = Time.now + MAX_FLUSH_WAIT
+        until @queue.empty? || Time.now >= cutoff do sleep(FLUSH_DELAY) end
+      end
+
+      def close_queue
+        @lock.synchronize { @queue.close }
+      end
+
+      # Returns the blocking enumerator that will pop
+      # items off the queue while any items are present
+      # If +nil+ is popped, the queue is closing.
+      #
+      # The segment is transformed into a serializable 
+      # span here so processing is taking place within
+      # the gRPC call's thread rather than in the main 
+      # application thread.
+      def enumerator
+        return enum_for(:enumerator) unless block_given?
+        loop do
+          if segment = @queue.pop(false)
+            NewRelic::Agent.increment_metric SPANS_SENT_METRIC
+            yield transform(segment)
+
+          else
+            raise ClosedQueueError
+          end
+        end
+      end
+
+      # Returns the blocking enumerator that will pop
+      # items off the queue while any items are present
+      # 
+      # yielding is deferred until batch_size spans is 
+      # reached.
+      #
+      # If +nil+ is popped, the queue is closing. A 
+      # final yield on non-empty batch is fired.
+      #
+      # The segment is transformed into a serializable 
+      # span here so processing is taking place within
+      # the gRPC call's thread rather than in the main 
+      # application thread.
+      def batch_enumerator
+        return enum_for(:enumerator) unless block_given?
+        loop do
+          if proc_or_segment = @queue.pop(false)
+            NewRelic::Agent.increment_metric SPANS_SENT_METRIC
+            @batch << transform(proc_or_segment)
+            if @batch.size >= BATCH_SIZE
+              yield SpanBatch.new(spans: @batch)
+              @batch.clear
+            end
+
+          else
+            yield SpanBatch.new(spans: @batch) unless @batch.empty?
+            raise ClosedQueueError
+          end
+        end
+      end
+
+      private
+
+      def span_event proc_or_segment
+        if proc_or_segment.is_a?(Proc)
+          proc_or_segment.call 
+        else
+          SpanEventPrimitive.for_segment(proc_or_segment)
+        end
+      end
+
+      def transform proc_or_segment
+        Span.new Transformer.transform(span_event proc_or_segment)
+      end
+
+    end
+  end
+end