atatus 1.0.0

data/lib/atatus/agent.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+require 'atatus/error'
+
+require 'atatus/context_builder'
+require 'atatus/error_builder'
+require 'atatus/stacktrace_builder'
+require 'atatus/collector/base'
+
+require 'atatus/central_config'
+# require 'atatus/transport/base'
+require 'atatus/metrics'
+
+require 'atatus/spies'
+
+module Atatus
+  # rubocop:disable Metrics/ClassLength
+  # @api private
+  class Agent
+    include Logging
+
+    LOCK = Mutex.new
+
+    # life cycle
+
+    def self.instance # rubocop:disable Style/TrivialAccessors
+      @instance
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def self.start(config)
+      return @instance if @instance
+
+      config = Config.new(config) unless config.is_a?(Config)
+
+      LOCK.synchronize do
+        return @instance if @instance
+
+        unless config.active?
+          config.logger.debug format(
+            "%sAgent disabled with `active: false'",
+            Logging::PREFIX
+          )
+          return
+        end
+
+        @instance = new(config).start
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def self.stop
+      LOCK.synchronize do
+        return unless @instance
+
+        @instance.stop
+        @instance = nil
+      end
+    end
+
+    def self.running?
+      !!@instance
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def initialize(config)
+      @config = config
+
+      @stacktrace_builder = StacktraceBuilder.new(config)
+      @context_builder = ContextBuilder.new(config)
+      @error_builder = ErrorBuilder.new(self)
+
+      @central_config = CentralConfig.new(config)
+      # @transport = Transport::Base.new(config)
+      @instrumenter = Instrumenter.new(
+        config,
+        stacktrace_builder: stacktrace_builder
+      ) { |event| enqueue event }
+      @metrics = Metrics.new(config) { |event| enqueue event }
+      @collector = Collector::Base.new(config)
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    attr_reader(
+      :central_config,
+      :config,
+      :context_builder,
+      :error_builder,
+      :instrumenter,
+      :metrics,
+      :stacktrace_builder,
+      # :transport,
+      :collector
+    )
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def start
+      unless config.disable_start_message
+        info '[%s] Starting atatus ruby agent', VERSION
+      end
+
+      if @config.license_key.nil? || @config.app_name.nil?
+        if @config.license_key.nil? && @config.app_name.nil?
+          error 'Atatus configuration license_key and app_name are missing'
+        elsif @config.license_key.nil?
+          error 'Atatus configuration license_key is missing'
+        elsif @config.app_name.nil?
+          error 'Atatus configuration app_name is missing'
+        end
+      end
+
+      collector.start
+      central_config.start
+      # transport.start
+      instrumenter.start
+      metrics.start
+
+      config.enabled_instrumentations.each do |lib|
+        debug "Requiring spy: #{lib}"
+        require "atatus/spies/#{lib}"
+      end
+
+      self
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+    def stop
+      debug 'Stopping agent'
+
+      central_config.stop
+      metrics.stop
+      instrumenter.stop
+      # transport.stop
+
+      self
+    end
+
+    at_exit do
+      stop
+    end
+
+    # transport
+
+    def enqueue(obj)
+      # transport.submit obj
+      case obj
+      when Atatus::Transaction
+        collector.add_txn(obj)
+      when Atatus::Span
+        collector.add_span(obj)
+      when Atatus::Error
+        collector.add_error(obj)
+      when Atatus::Metricset
+        collector.add_metrics(obj)
+      end
+    end
+
+    # instrumentation
+
+    def current_transaction
+      instrumenter.current_transaction
+    end
+
+    def current_span
+      instrumenter.current_span
+    end
+
+    def start_transaction(
+      name = nil,
+      type = nil,
+      context: nil,
+      trace_context: nil
+    )
+      instrumenter.start_transaction(
+        name,
+        type,
+        context: context,
+        trace_context: trace_context
+      )
+    end
+
+    def end_transaction(result = nil)
+      instrumenter.end_transaction(result)
+    end
+
+    # rubocop:disable Metrics/ParameterLists
+    def start_span(
+      name = nil,
+      type = nil,
+      subtype: nil,
+      action: nil,
+      backtrace: nil,
+      context: nil,
+      trace_context: nil
+    )
+      instrumenter.start_span(
+        name,
+        type,
+        subtype: subtype,
+        action: action,
+        backtrace: backtrace,
+        context: context,
+        trace_context: trace_context
+      )
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    def end_span
+      instrumenter.end_span
+    end
+
+    def set_label(key, value)
+      instrumenter.set_label(key, value)
+    end
+
+    def set_custom_context(context)
+      instrumenter.set_custom_context(context)
+    end
+
+    def set_user(user)
+      instrumenter.set_user(user)
+    end
+
+    def build_context(rack_env:, for_type:)
+      @context_builder.build(rack_env: rack_env, for_type: for_type)
+    end
+
+    # errors
+
+    def report(exception, context: nil, handled: true)
+      return if config.filter_exception_types.include?(exception.class.to_s)
+
+      error = @error_builder.build_exception(
+        exception,
+        context: context,
+        handled: handled
+      )
+      enqueue error
+      error.id
+    end
+
+    def report_message(message, context: nil, backtrace: nil, **attrs)
+      error = @error_builder.build_log(
+        message,
+        context: context,
+        backtrace: backtrace,
+        **attrs
+      )
+      enqueue error
+      error.id
+    end
+
+    # filters
+
+    def add_filter(key, callback)
+      # transport.add_filter(key, callback)
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/atatus/central_config.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require 'atatus/central_config/cache_control'
+
+module Atatus
+  # @api private
+  class CentralConfig
+    include Logging
+
+    # @api private
+    class ResponseError < InternalError
+      def initialize(response)
+        @response = response
+      end
+
+      attr_reader :response
+    end
+    class ClientError < ResponseError; end
+    class ServerError < ResponseError; end
+
+    DEFAULT_MAX_AGE = 300
+
+    def initialize(config)
+      @config = config
+      @modified_options = {}
+      @service_info = {
+        'service.name': config.service_name,
+        'service.environment': config.environment
+      }.to_json
+    end
+
+    attr_reader :config
+    attr_reader :scheduled_task, :promise # for specs
+
+    def start
+      return unless config.central_config?
+
+      fetch_and_apply_config
+    end
+
+    def fetch_and_apply_config
+      @promise =
+        Concurrent::Promise
+        .execute(&method(:fetch_config))
+        .on_success(&method(:handle_success))
+        .rescue(&method(:handle_error))
+    end
+
+    def stop
+      @scheduled_task&.cancel
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def fetch_config
+      resp = perform_request
+
+      case resp.status
+      when 200..299
+        resp
+      when 300..399
+        resp
+      when 400..499
+        raise ClientError, resp
+      when 500..599
+        raise ServerError, resp
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def assign(update)
+      # For each updated option, store the original value,
+      # unless already stored
+      update.each_key do |key|
+        @modified_options[key] ||= config.get(key.to_sym)&.value
+      end
+
+      # If the new update doesn't set a previously modified option,
+      # revert it to the original
+      @modified_options.each_key do |key|
+        next if update.key?(key)
+        update[key] = @modified_options.delete(key)
+      end
+
+      config.assign(update)
+    end
+
+    private
+
+    # rubocop:disable Metrics/MethodLength
+    def handle_success(resp)
+      if resp.status == 304
+        info 'Received 304 Not Modified'
+      else
+        update = JSON.parse(resp.body.to_s)
+        assign(update)
+
+        info 'Updated config from Kibana'
+      end
+
+      schedule_next_fetch(resp)
+
+      true
+    rescue Exception => e
+      error 'Failed to apply remote config, %s', e.inspect
+      debug { e.backtrace.join('\n') }
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def handle_error(error)
+      error(
+        'Failed fetching config: %s, trying again in %d seconds',
+        error.response.body, DEFAULT_MAX_AGE
+      )
+
+      assign({})
+
+      schedule_next_fetch(error.response)
+    end
+
+    def perform_request
+      Http.post(
+        config.server_url + '/agent/v1/config/',
+        body: @service_info,
+        headers: { etag: 1, content_type: 'application/json' }
+      )
+    end
+
+    def schedule_next_fetch(resp)
+      seconds =
+        if (cache_header = resp.headers['Cache-Control'])
+          CacheControl.new(cache_header).max_age
+        else
+          DEFAULT_MAX_AGE
+        end
+
+      @scheduled_task =
+        Concurrent::ScheduledTask
+        .execute(seconds, &method(:fetch_and_apply_config))
+    end
+  end
+end

data/lib/atatus/central_config/cache_control.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Atatus
+  class CentralConfig
+    # @api private
+    class CacheControl
+      def initialize(value)
+        @header = value
+        parse!(value)
+      end
+
+      attr_reader(
+        :must_revalidate,
+        :no_cache,
+        :no_store,
+        :no_transform,
+        :public,
+        :private,
+        :proxy_revalidate,
+        :max_age,
+        :s_maxage
+      )
+
+      private
+
+      def parse!(value)
+        value.split(',').each do |token|
+          k, v = token.split('=').map(&:strip)
+          instance_variable_set(:"@#{k.gsub('-', '_')}", v ? v.to_i : true)
+        end
+      end
+    end
+  end
+end

data/lib/atatus/collector/base.rb

@@ -0,0 +1,329 @@
+require 'atatus'
+require 'atatus/transaction'
+require 'atatus/config'
+# require 'atatus/logger'
+require 'atatus/collector/transport'
+require 'json'
+
+module Atatus
+  module Collector
+    class Layer
+      attr_reader :count, :min, :max, :total, :type, :kind, :id, :pid
+      attr_writer :id, :pid
+
+      def initialize(type, kind, duration)
+        @type = type
+        @kind = kind
+        @count = 1
+        @min = duration
+        @max = duration
+        @total = duration
+      end
+
+      def aggregate!(duration)
+        @count += 1
+        @min = duration if duration < @min
+        @max = duration if duration > @max
+        @total += duration
+      end
+
+      def self.span_kind(value)
+        return KINDS.fetch(value, value)
+      end
+
+      def self.span_type(value)
+        return TYPES.fetch(value, value)
+      end
+
+      private
+      KINDS = {
+        'db' => 'Database',
+        'cache' => 'Database',
+        'ext' => 'Remote',
+        'websocket' => 'Remote',
+        'template' => 'Template'
+      }.freeze
+
+      TYPES = {
+        'mysql' => 'MySQL',
+        'mysql2' => 'MySQL',
+        'postgresql' => 'Postgres',
+        'mssql' => 'MS SQL',
+        'mongodb' => 'MongoDB',
+        'redis' => 'Redis',
+        'graphql' => 'GraphQL',
+        'elasticsearch' => 'Elasticsearch',
+        'cassandra' => 'Cassandra',
+        'http' => 'External Requests',
+        'http2' => 'External Requests',
+        'http_rb' => 'External Requests',
+        'net_http' => 'External Requests'
+      }.freeze
+    end
+
+    class Txn < Layer
+      attr_reader :spans
+
+      def initialize(type, kind, duration)
+        super(type, kind, duration)
+        @spans = Hash.new {}
+      end
+
+      def add_span(span_name, type, kind, duration)
+        if !@spans.has_key?(span_name)
+          @spans[key] = Layer.new(type, kind, duration)
+        else
+          @spans[key].aggregate! duration
+        end
+      end
+    end
+
+    class Base
+      include Logging
+
+      def initialize(config)
+        # info 'Initializing Collector'
+        @config = config
+        @spans = Hash.new {|h,k| h[k]=[]}
+        @running = false
+        uri = URI("https://apm-rx.atatus.com/")
+        @transport = Atatus::BaseTransport.new(config, uri.host, uri.port)
+        @txn_lock = Mutex.new
+        @txn_aggs = Hash.new {}
+        @trace_aggs = Array.new()
+        @error_metrics = Hash.new {}
+        @error_requests = Array.new()
+
+        @error_lock = Mutex.new
+        @error_aggs = Array.new()
+
+        @metrics_lock = Mutex.new
+        @metrics = Array.new()
+        @collect_counter = 0
+      end
+
+      attr_reader :config
+
+      def pid_str
+        format('[PID:%s]', Process.pid)
+      end
+
+      def start
+        debug '%s: Starting Collector', pid_str
+
+        ensure_worker_running
+      end
+
+      def add_error(error)
+        ensure_worker_running
+
+        @error_lock.synchronize do
+          if @error_aggs.length < 20
+            @error_aggs.push(error)
+          else
+            i = rand(20)
+            @error_aggs[i] = error
+          end
+        end
+        # puts error.to_yaml
+      end
+
+      def add_metrics(metricset)
+        ensure_worker_running
+        metric = {}
+        if metricset.samples.key?(:'system.memory.total')
+          metric[:'system.cpu.total.norm.pct'] = metricset.samples[:'system.cpu.total.norm.pct']
+          metric[:'system.memory.actual.free'] = metricset.samples[:'system.memory.actual.free']
+          metric[:'system.memory.total'] = metricset.samples[:'system.memory.total']
+          metric[:'system.process.cpu.total.norm.pct'] = metricset.samples[:'system.process.cpu.total.norm.pct']
+          metric[:'system.process.memory.size'] = metricset.samples[:'system.process.memory.size']
+          metric[:'system.process.memory.rss.bytes'] = metricset.samples[:'system.process.memory.rss.bytes']
+        end
+
+        if metricset.samples.key?(:'ruby.gc.count')
+          metric[:'ruby.gc.count'] = metricset.samples[:'ruby.gc.count']
+          metric[:'ruby.threads'] = metricset.samples[:'ruby.threads']
+          metric[:'ruby.heap.slots.live'] = metricset.samples[:'ruby.heap.slots.live']
+          metric[:'ruby.heap.slots.free'] = metricset.samples[:'ruby.heap.slots.free']
+          metric[:'ruby.heap.allocations.total'] = metricset.samples[:'ruby.heap.allocations.total']
+        end
+
+        @metrics_lock.synchronize do
+          @metrics << metric
+        end
+        # puts metric.to_yaml
+      end
+
+      def add_span(span)
+        ensure_worker_running
+
+        @spans[span.transaction_id] << span if span.transaction_id
+        # puts span.to_yaml
+      end
+
+      def add_txn(txn)
+        ensure_worker_running
+
+        # puts txn.to_yaml
+        txn_name = txn.name
+        return unless txn_name
+        @txn_lock.synchronize do
+          if !@txn_aggs.has_key?(txn_name)
+            @txn_aggs[txn_name] = Txn.new(@config.framework_name, "Ruby", txn.duration)
+            @txn_aggs[txn_name].id = txn.id
+            @txn_aggs[txn_name].pid = txn.id
+          else
+            @txn_aggs[txn_name].aggregate! txn.duration
+          end
+
+          spans_present = false
+          if @spans.has_key?(txn.id)
+            @spans[txn.id].each do |span|
+              span_name = span.name
+              if !@txn_aggs[txn_name].spans.has_key?(span_name)
+                kind = Layer.span_kind(span.type)
+                type = Layer.span_type(span.subtype)
+                @txn_aggs[txn_name].spans[span_name] = Layer.new(type, kind, span.duration)
+                @txn_aggs[txn_name].spans[span_name].id = span.id
+                @txn_aggs[txn_name].spans[span_name].pid = span.transaction_id
+              else
+                @txn_aggs[txn_name].spans[span_name].aggregate! span.duration
+              end
+            end
+            spans_present = true
+          end
+
+          if spans_present
+            if Util.ms(txn.duration) >= @config.trace_threshold
+              trace_txn = txn
+              trace_txn.spans = @spans[txn.id]
+
+              if @trace_aggs.length < 5
+                @trace_aggs.push(trace_txn)
+              else
+                i = rand(5)
+                @trace_aggs[i] = trace_txn
+              end
+            end
+          end
+
+          if spans_present
+            @spans.delete(txn.id)
+          end
+
+          status_code = txn.context.response.status_code
+
+          if status_code >= 400 && status_code != 404
+            if !@error_metrics.has_key?(txn_name)
+              @error_metrics[txn_name] = {status_code => 1}
+            else
+              if !@error_metrics[txn_name].has_key?(status_code)
+                @error_metrics[txn_name][status_code] = 1
+              else
+                @error_metrics[txn_name][status_code] += 1
+              end
+            end
+
+            # puts txn.context.to_yaml
+            if @error_requests.length < 20
+              @error_requests.push({'name' => txn_name, 'context' => txn.context})
+            else
+              i = rand(20)
+              @error_requests[i] = {'name' => txn_name, 'context' => txn.context}
+            end
+          end
+
+        end
+      end
+
+      private
+      def ensure_worker_running
+        return if worker_active?
+        @running = true
+        @worker = Thread.new() do
+          debug '%s: Starting Collector Worker', pid_str
+
+          while @running
+            start_time = Time.now
+            sleep(60)
+            collect start_time
+          end
+        end
+
+        at_exit do
+          debug '%s: Waiting for Collector Worker to exit', pid_str
+          @running = false
+          @worker.run
+          @worker.join(10)
+        end
+      end
+
+      def worker_active?
+        @worker && @worker.alive?
+      end
+
+      def collect(start_time)
+        # puts @config.to_yaml
+        # return
+        if @config.license_key.nil? || @config.app_name.nil?
+          if @config.license_key.nil? && @config.app_name.nil?
+            error '%s: Atatus configuration license_key and app_name are missing', pid_str
+            return
+          elsif @config.license_key.nil?
+            error '%s: Atatus configuration license_key is missing', pid_str
+            return
+          elsif @config.app_name.nil?
+            error '%s: Atatus configuration app_name is missing', pid_str
+            return
+          end
+        end
+
+        if @collect_counter % 30 == 0
+          @transport.hostinfo(start_time)
+          @collect_counter = 0
+        end
+        @collect_counter += 1
+
+        end_time = Time.now
+        debug '%s: Collecting transactions', pid_str
+
+        txn_data = nil
+        trace_data = nil
+        error_data = nil
+        error_metrics_data = nil
+        error_requests_data = nil
+        metrics_data = nil
+
+        @txn_lock.synchronize do
+          txn_data = @txn_aggs
+          @txn_aggs = Hash.new {}
+
+          trace_data = @trace_aggs
+          @trace_aggs = Array.new()
+
+          error_metrics_data = @error_metrics
+          @error_metrics = Hash.new {}
+
+          error_requests_data = @error_requests
+          @error_requests = Array.new()
+        end
+
+        @error_lock.synchronize do
+          error_data = @error_aggs
+          @error_aggs = Array.new()
+        end
+        
+        @metrics_lock.synchronize do
+          metrics_data = @metrics
+          @metrics = Array.new()
+        end
+
+        @transport.txn(start_time, end_time, txn_data) unless txn_data.empty?
+        @transport.trace(start_time, end_time, trace_data) unless trace_data.empty?
+        @transport.error_metrics(start_time, end_time, error_metrics_data, error_requests_data) unless error_metrics_data.empty?
+        @transport.errors(start_time, end_time, error_data) unless error_data.empty?
+        @transport.metrics(start_time, end_time, metrics_data) unless metrics_data.empty?
+      end
+    end
+  end
+end