skylight-core 2.0.0.beta1

data/lib/skylight/core/railtie.rb

@@ -0,0 +1,166 @@
+require 'skylight/core'
+require 'rails'
+
+module Skylight::Core
+  # @api private
+  module Railtie
+    extend ActiveSupport::Concern
+
+    included do
+      def self.root_key; :skylight end
+      def self.config_class; Config end
+      def self.middleware_class; Middleware end
+      def self.gem_name; 'Skylight' end
+      def self.log_file_name; 'skylight' end
+      def self.namespace; Skylight end
+      def self.version; Skylight::Core::VERSION end
+    end
+
+  private
+
+    def log_prefix
+      "[#{self.class.gem_name.upcase}] [#{self.class.version}]"
+    end
+
+    def development_warning
+      "#{log_prefix} Running #{self.class.gem_name} in development mode. No data will be reported until you deploy your app."
+    end
+
+    def run_initializer(app)
+      # Load probes even when agent is inactive to catch probe related bugs sooner
+      load_probes
+
+      config = load_skylight_config(app)
+
+      if activate?
+        if config
+          begin
+            if self.class.namespace.start!(config)
+              set_middleware_position(app, config)
+              Rails.logger.info "#{log_prefix} #{self.class.gem_name} agent enabled"
+            else
+              Rails.logger.info "#{log_prefix} Unable to start, see the #{self.class.gem_name} logs for more details"
+            end
+          rescue ConfigError => e
+            Rails.logger.error "#{log_prefix} #{e.message}; disabling #{self.class.gem_name} agent"
+          end
+        end
+      elsif Rails.env.development?
+        # FIXME: The CLI isn't part of core so we should change this message
+        unless config.user_config.disable_dev_warning?
+          log_warning config, development_warning
+        end
+      elsif !Rails.env.test?
+        unless config.user_config.disable_env_warning?
+          log_warning config, "#{log_prefix} You are running in the #{Rails.env} environment but haven't added it to config.#{self.class.root_key}.environments, so no data will be sent to skylight.io."
+        end
+      end
+    end
+
+    def log_warning(config, msg)
+      if config
+        config.alert_logger.warn(msg)
+      else
+        Rails.logger.warn(msg)
+      end
+    end
+
+    def existent_paths(paths)
+      paths.respond_to?(:existent) ? paths.existent : paths.select { |f| File.exists?(f) }
+    end
+
+    def load_skylight_config(app)
+      path = config_path(app)
+      path = nil unless File.exist?(path)
+
+      unless tmp = app.config.paths['tmp'].first
+        Rails.logger.error "#{log_prefix} tmp directory missing from rails configuration"
+        return nil
+      end
+
+      config = self.class.config_class.load(file: path, environment: Rails.env.to_s)
+      config[:root] = Rails.root
+
+      configure_logging(config, app)
+
+      config[:'daemon.sockdir_path'] ||= tmp
+      config[:'normalizers.render.view_paths'] = existent_paths(app.config.paths["app/views"]) + [Rails.root.to_s]
+      config
+    end
+
+    def configure_logging(config, app)
+      if logger = skylight_config(app).logger
+        config.logger = logger
+      else
+        # Configure the log file destination
+        if log_file = skylight_config(app).log_file
+          config['log_file'] = log_file
+        elsif !config.key?('log_file') && !config.on_heroku?
+          config['log_file'] = File.join(Rails.root, "log/#{self.class.log_file_name}.log")
+        end
+
+        # Configure the log level
+        if level = skylight_config(app).log_level
+          config['log_level'] = level
+        elsif !config.key?('log_level')
+          if level = app.config.log_level
+            config['log_level'] = level
+          end
+        end
+      end
+    end
+
+    def config_path(app)
+      File.expand_path(skylight_config.config_path, app.root)
+    end
+
+    def environments
+      Array(skylight_config.environments).map { |e| e && e.to_s }.compact
+    end
+
+    def activate?
+      key = "#{self.class.config_class.native_env_prefix}ENABLED"
+      if ENV.key?(key)
+        ENV[key] !~ /^false$/i
+      else
+        environments.include?(Rails.env.to_s)
+      end
+    end
+
+    def load_probes
+      probes = skylight_config.probes || []
+      Probes.probe(*probes)
+    end
+
+    def middleware_position
+      skylight_config.middleware_position.is_a?(Hash) ? skylight_config.middleware_position.symbolize_keys : skylight_config.middleware_position
+    end
+
+    def insert_middleware(app, config)
+      if middleware_position.has_key?(:after)
+        app.middleware.insert_after(middleware_position[:after], self.class.middleware_class, config: config)
+      elsif middleware_position.has_key?(:before)
+        app.middleware.insert_before(middleware_position[:before], self.class.middleware_class, config: config)
+      else
+        raise "The middleware position you have set is invalid. Please be sure `config.#{self.class.root_key}.middleware_position` is set up correctly."
+      end
+    end
+
+    def set_middleware_position(app, config)
+      if middleware_position.is_a?(Integer)
+        app.middleware.insert middleware_position, self.class.middleware_class, config: config
+      elsif middleware_position.is_a?(Hash) && middleware_position.keys.count == 1
+        insert_middleware(app, config)
+      elsif middleware_position.nil?
+        app.middleware.insert 0, self.class.middleware_class, config: config
+      else
+        raise "The middleware position you have set is invalid. Please be sure `config.#{self.class.root_key}.middleware_position` is set up correctly."
+      end
+    end
+
+    def skylight_config(target=self)
+      target.config.send(self.class.root_key)
+    end
+
+  end
+end

data/lib/skylight/core/subscriber.rb

@@ -0,0 +1,124 @@
+module Skylight::Core
+  # @api private
+  class Subscriber
+    include Util::Logging
+
+    attr_reader :config
+
+    def initialize(config, instrumenter)
+      @config       = config
+      @subscriber   = nil
+      @normalizers  = Normalizers.build(config)
+      @instrumenter = instrumenter
+    end
+
+    def register!
+      unregister! if @subscriber
+      pattern = ArrayPattern.new(@normalizers.keys)
+      @subscriber = ActiveSupport::Notifications.subscribe pattern, self
+    end
+
+    def unregister!
+      ActiveSupport::Notifications.unsubscribe @subscriber
+      @subscriber = nil
+    end
+
+    class ArrayPattern
+
+      def initialize(keys)
+        @keys = Set.new keys
+      end
+
+      def ===(item)
+        @keys.include?(item)
+      end
+
+    end
+
+    #
+    #
+    # ===== ActiveSupport::Notifications API
+    #
+    #
+
+    class Notification
+      attr_reader :name, :span
+
+      def initialize(name, span)
+        @name, @span = name, span
+      end
+    end
+
+    def start(name, id, payload)
+      return if @instrumenter.disabled?
+      return unless trace = @instrumenter.current_trace
+
+      result = normalize(trace, name, payload)
+
+      unless result == :skip
+        case result.size
+        when 4
+          cat, title, desc, meta = result
+        when 3
+          cat, title, desc = result
+        else
+          raise "Invalid normalizer result: #{result.inspect}"
+        end
+
+        span = trace.instrument(cat, title, desc, meta)
+      end
+
+      trace.notifications << Notification.new(name, span)
+    rescue Exception => e
+      error "Subscriber#start error; msg=%s", e.message
+      debug "trace=%s", trace.inspect
+      debug "in:  name=%s", name.inspect
+      debug "in:  payload=%s", payload.inspect
+      debug "out: cat=%s, title=%s, desc=%s", cat.inspect, name.inspect, desc.inspect
+      t { e.backtrace.join("\n") }
+      nil
+    end
+
+    def finish(name, id, payload)
+      return if @instrumenter.disabled?
+      return unless trace = @instrumenter.current_trace
+
+      while curr = trace.notifications.pop
+        if curr.name == name
+          begin
+            normalize_after(trace, curr.span, name, payload)
+          ensure
+            meta = { }
+            meta[:exception] = payload[:exception] if payload[:exception]
+            meta[:exception_object] = payload[:exception_object] if payload[:exception_object]
+            trace.done(curr.span, meta) if curr.span
+          end
+          return
+        end
+      end
+
+    rescue Exception => e
+      error "Subscriber#finish error; msg=%s", e.message
+      debug "trace=%s", trace.inspect
+      debug "in:  name=%s", name.inspect
+      debug "in:  payload=%s", payload.inspect
+      t { e.backtrace.join("\n") }
+      nil
+    end
+
+    def publish(name, *args)
+      # Ignored for now because nothing in rails uses it
+    end
+
+  private
+
+    def normalize(*args)
+      @normalizers.normalize(*args)
+    end
+
+    def normalize_after(*args)
+      @normalizers.normalize_after(*args)
+    end
+
+  end
+end

data/lib/skylight/core/test.rb

@@ -0,0 +1,98 @@
+module Skylight
+  module Core
+    module Test
+      module Mocking
+
+        def mock!(config_opts={}, &callback)
+          config_opts[:mock_submission] ||= callback || proc {}
+          config = config_class.load(config_opts)
+
+          mock_instrumenter_klass = Class.new(instrumenter_class) do
+            def self.trace_class
+              @trace_class ||= Class.new(super) do
+                def self.native_new(start, _uuid, endpoint, meta)
+                  inst = allocate
+                  inst.instance_variable_set(:@start, start)
+                  inst.instance_variable_set(:@endpoint, endpoint)
+                  inst.instance_variable_set(:@starting_endpoint, endpoint)
+                  inst.instance_variable_set(:@meta, meta)
+                  inst
+                end
+
+                attr_reader :endpoint, :starting_endpoint, :meta
+
+                def mock_spans
+                  @mock_spans ||= []
+                end
+
+                def native_get_started_at
+                  @start
+                end
+
+                def native_set_endpoint(endpoint)
+                  @endpoint = endpoint
+                end
+
+                def native_start_span(time, cat)
+                  span = {
+                    start: time,
+                    cat: cat
+                  }
+                  mock_spans << span
+                  # Return integer like the native method does
+                  mock_spans.index(span)
+                end
+
+                def native_span_set_title(sp, title)
+                  mock_spans[sp][:title] = title
+
+                end
+
+                def native_span_set_description(sp, desc)
+                  mock_spans[sp][:desc] = desc
+                end
+
+                def native_span_set_meta(sp, meta)
+                  mock_spans[sp][:meta] = meta
+                end
+
+                def native_span_set_exception(sp, exception_object, exception)
+                  mock_spans[sp][:exception_object] = exception_object
+                  mock_spans[sp][:exception] = exception
+                end
+
+                def native_stop_span(span, time)
+                  span = mock_spans[span]
+                  span[:duration] = time - span[:start]
+                  nil
+                end
+              end
+            end
+
+            def self.native_new(*)
+              allocate
+            end
+
+            def native_start
+              true
+            end
+
+            def native_submit_trace(trace)
+              config[:mock_submission].call(trace)
+            end
+
+            def native_stop
+            end
+
+            def limited_description(description)
+              description
+            end
+          end
+
+          @instrumenter = mock_instrumenter_klass.new(config).start!
+        end
+
+      end
+    end
+  end
+end

data/lib/skylight/core/trace.rb

@@ -0,0 +1,190 @@
+module Skylight::Core
+  class Trace
+    GC_CAT = 'noise.gc'.freeze
+
+    include Util::Logging
+
+    attr_reader :instrumenter, :endpoint, :notifications, :meta
+
+    def self.new(instrumenter, endpoint, start, cat, title=nil, desc=nil, meta=nil)
+      inst = native_new(normalize_time(start), "TODO", endpoint, meta)
+      inst.send(:initialize, instrumenter, cat, title, desc, meta)
+      inst.endpoint = endpoint
+      inst
+    end
+
+    # TODO: Move this into native
+    def self.normalize_time(time)
+      # At least one customer has extensions that cause integer division to produce rationals.
+      # Since the native code expects an integer, we force it again.
+      (time.to_i / 100_000).to_i
+    end
+
+    def initialize(instrumenter, cat, title, desc, meta)
+      raise ArgumentError, 'instrumenter is required' unless instrumenter
+
+      @instrumenter = instrumenter
+      @submitted = false
+      @broken = false
+
+      @notifications = []
+
+      @spans = []
+
+      # create the root node
+      @root = start(native_get_started_at, cat, title, desc, meta, normalize: false)
+
+      # Also store meta for later access
+      @meta = meta
+
+      @gc = config.gc.track unless ENV.key?("SKYLIGHT_DISABLE_GC_TRACKING")
+    end
+
+    def endpoint=(value)
+      @endpoint = value
+      native_set_endpoint(value)
+      value
+    end
+
+    def config
+      @instrumenter.config
+    end
+
+    def record(cat, title=nil, desc=nil)
+      return if @broken
+
+      title.freeze if title.is_a?(String)
+      desc.freeze  if desc.is_a?(String)
+
+      desc = @instrumenter.limited_description(desc)
+
+      time = Util::Clock.nanos - gc_time
+
+      stop(start(time, cat, title, desc, nil), time)
+
+      nil
+    rescue => e
+      error "failed to record span; msg=%s", e.message
+      @broken = true
+      nil
+    end
+
+    def instrument(cat, title=nil, desc=nil, meta=nil)
+      return if @broken
+      t { "instrument: #{cat}, #{title}" }
+
+      title.freeze if title.is_a?(String)
+      desc.freeze  if desc.is_a?(String)
+
+      original_desc = desc
+      now           = Util::Clock.nanos
+      desc          = @instrumenter.limited_description(desc)
+
+      if desc == Instrumenter::TOO_MANY_UNIQUES
+        debug "[SKYLIGHT] [#{Skylight::Core::VERSION}] A payload description produced <too many uniques>"
+        debug "original desc=%s", original_desc
+        debug "cat=%s, title=%s, desc=%s", cat, title, desc
+      end
+
+      start(now - gc_time, cat, title, desc, meta)
+    rescue => e
+      error "failed to instrument span; msg=%s", e.message
+      @broken = true
+      nil
+    end
+
+    def span_correlation_header(span)
+      native_span_get_correlation_header(span)
+    end
+
+    def done(span, meta=nil)
+      return unless span
+      return if @broken
+
+      if meta && (meta[:exception_object] || meta[:exception])
+        native_span_set_exception(span, meta[:exception_object], meta[:exception])
+      end
+
+      stop(span, Util::Clock.nanos - gc_time)
+    rescue => e
+      error "failed to close span; msg=%s", e.message
+      @broken = true
+      nil
+    end
+
+    def release
+      return unless @instrumenter.current_trace == self
+      @instrumenter.current_trace = nil
+    end
+
+    def traced
+      time = gc_time
+      now = Util::Clock.nanos
+
+      if time > 0
+        t { fmt "tracking GC time; duration=%d", time }
+        stop(start(now - time, GC_CAT, nil, nil, nil), now)
+      end
+
+      stop(@root, now)
+    end
+
+    def submit
+      return if @broken
+
+      t { "submitting trace" }
+
+      if @submitted
+        t { "already submitted" }
+        return
+      end
+
+      release
+      @submitted = true
+
+      traced
+
+      @instrumenter.process(self)
+    rescue Exception => e
+      error e.message
+      t { e.backtrace.join("\n") }
+    end
+
+  private
+
+    def start(time, cat, title, desc, meta, opts={})
+      time = self.class.normalize_time(time) unless opts[:normalize] == false
+
+      sp = native_start_span(time, cat.to_s)
+      native_span_set_title(sp, title.to_s) if title
+      native_span_set_description(sp, desc.to_s) if desc
+      native_span_set_meta(sp, meta) if meta
+
+      @spans << sp
+      t { "started span: #{sp} - #{cat}, #{title}" }
+
+      sp
+    end
+
+    def stop(span, time)
+      t { "stopping span: #{span}" }
+
+      expected = @spans.pop
+      unless span == expected
+        error "invalid span nesting"
+        # TODO: Actually log span title here
+        t { "expected=#{expected}, actual=#{span}" }
+      end
+
+      time = self.class.normalize_time(time)
+      native_stop_span(span, time)
+      nil
+    end
+
+    def gc_time
+      return 0 unless @gc
+      @gc.update
+      @gc.time
+    end
+  end
+end

data/lib/skylight/core/user_config.rb

@@ -0,0 +1,61 @@
+require 'yaml'
+require 'skylight/core/errors'
+
+module Skylight::Core
+  class UserConfig
+
+    attr_accessor :disable_dev_warning, :disable_env_warning
+
+    def initialize(config)
+      @config = config
+      reload
+    end
+
+    def file_path
+      return @file_path if @file_path
+
+      config_path = @config[:user_config_path] || begin
+        require "etc"
+        home_dir = ENV['HOME'] || Etc.getpwuid.dir || (ENV["USER"] && File.expand_path("~#{ENV['USER']}"))
+        if home_dir
+          File.join(home_dir, ".skylight")
+        else
+          raise ConfigError, "The Skylight `user_config_path` must be defined since the home directory cannot be inferred"
+        end
+      end
+
+      @file_path = File.expand_path(config_path)
+    end
+
+    def disable_dev_warning?
+      disable_dev_warning
+    end
+
+    def disable_env_warning?
+      disable_env_warning
+    end
+
+    def reload
+      config = File.exist?(file_path) ? YAML.load_file(file_path) : false
+      return unless config
+
+      self.disable_dev_warning = !!config['disable_dev_warning']
+      self.disable_env_warning = !!config['disable_env_warning']
+    end
+
+    def save
+      FileUtils.mkdir_p(File.dirname(file_path))
+      File.open(file_path, 'w') do |f|
+        f.puts YAML.dump(to_hash)
+      end
+    end
+
+    def to_hash
+      {
+        'disable_dev_warning' => disable_dev_warning,
+        'disable_env_warning' => disable_env_warning
+      }
+    end
+
+  end
+end

data/lib/skylight/core/util/allocation_free.rb

@@ -0,0 +1,26 @@
+module Skylight::Core
+  module Util
+    # Helpers to reduce memory allocation
+    module AllocationFree
+
+      # Find an item in an array without allocation.
+      #
+      # @param array [Array] the array to search
+      # @yield a block called against each item until a match is found
+      # @yieldparam item an item from the array
+      # @yieldreturn [Boolean] whether `item` matches the criteria
+      # return the found item or nil, if nothing found
+      def array_find(array)
+        i = 0
+
+        while i < array.size
+          item = array[i]
+          return item if yield item
+          i += 1
+        end
+
+        nil
+      end
+    end
+  end
+end