skylight 0.0.16 → 0.1.0.alpha1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 82650226a057fc3b8b8595ac4fc760edaf56c656
+  data.tar.gz: f50d0c8f8e3d22d915664760f7eba80aece2fbae
+SHA512:
+  metadata.gz: 99a7f5225e962df3c946c6440454a1892c1401d19feebee8cb3698a048c4f3a26980d2db77e99092f55023020ec6f85b88146ffd0aa799c03e4b3f1880c969c6
+  data.tar.gz: 0b40b0d1f2c08ee5872f6891f7b889900e96ec70beb6c938fe2016c35e07c8271e901f7b6d1e5c1919418a963698027fddbad7e2fe63d78ef2c13953da7b85d6

data/README.md

@@ -1,20 +1,4 @@
-# Direwolf Agent
+# Skylight Ruby Agent
 
-Collect instrumentation from a rails application and send it off to the
-direwolf servers.
-
-## Traces
-
-An conceptual overview of a trace.
-
-  * Trace ID (UUID)
-  * Tiers
-    * Name
-    * Spans (max 256 unique per tier)
-      * Category (multi-level, example: cache.redis)
-      * Description
-      * Annotations
-
-## TODO
-
-  * Cap the max number of spans per trace (2048k?)
+Instrument your ruby application and send the data to the Skylight
+servers.

data/bin/skylight

@@ -1,5 +1,3 @@
 #!/usr/bin/env ruby
-
 require "skylight/cli"
-
 Skylight::CLI.start(ARGV)

data/lib/skylight.rb

@@ -1,11 +1,49 @@
-require 'thread'
+require 'rbconfig'
 require 'socket'
-require 'openssl'
-require 'net/http'
-require 'active_support/notifications'
+require 'skylight/version'
 
 module Skylight
-  class Error < RuntimeError; end
+  TRACE_ENV_KEY      = 'SK_ENABLE_TRACE_LOGS'.freeze
+  STANDALONE_ENV_KEY = 'SK_STANDALONE'.freeze
+  STANDALONE_ENV_VAL = 'server'.freeze
+
+  def self.daemon?
+    ENV[STANDALONE_ENV_KEY] == STANDALONE_ENV_VAL
+  end
+
+  unless daemon?
+    require 'active_support/notifications'
+    require 'skylight/compat' # Require after AS::N
+  end
+
+  autoload :Api,          'skylight/api'
+  autoload :CLI,          'skylight/cli'
+  autoload :Config,       'skylight/config'
+  autoload :GC,           'skylight/gc'
+  autoload :Instrumenter, 'skylight/instrumenter'
+  autoload :Messages,     'skylight/messages'
+  autoload :Normalizers,  'skylight/normalizers'
+  autoload :Subscriber,   'skylight/subscriber'
+  autoload :Worker,       'skylight/worker'
+
+  module Util
+    autoload :Clock,         'skylight/util/clock'
+    autoload :Gzip,          'skylight/util/gzip'
+    autoload :HTTP,          'skylight/util/http'
+    autoload :Logging,       'skylight/util/logging'
+    autoload :Queue,         'skylight/util/queue'
+    autoload :Task,          'skylight/util/task'
+    autoload :UniformSample, 'skylight/util/uniform_sample'
+  end
+
+  # ==== Vendor ====
+  autoload :Beefcake, 'skylight/vendor/beefcake'
+
+  # ==== Exceptions ====
+  class IpcProtoError    < RuntimeError; end
+  class WorkerStateError < RuntimeError; end
+  class ConfigError      < RuntimeError; end
+  class TraceError       < RuntimeError; end
 
   TIERS = %w(
     app
@@ -14,42 +52,14 @@ module Skylight
     noise
     other)
 
-  # First require all util files
-  require 'skylight/util/atomic'
-  require 'skylight/util/bytes'
-  require 'skylight/util/clock'
-  require 'skylight/util/ewma'
-  require 'skylight/util/gzip'
-  require 'skylight/util/http'
-  require 'skylight/util/queue'
-  require 'skylight/util/uniform_sample'
-  require 'skylight/util/uuid'
-
-  # Then require the rest
-  require 'skylight/sanity_checker'
-  require 'skylight/compat'
-  require 'skylight/config'
-  require 'skylight/instrumenter'
-  require 'skylight/middleware'
-  require 'skylight/normalize'
-  require 'skylight/json_proto'
-  require 'skylight/subscriber'
-  require 'skylight/trace'
-  require 'skylight/worker'
-
-  # Some helper functions
-  def self.current_endpoint=(name)
-    if t = Trace.current
-      t.endpoint = name
-    end
-
-    name
-  rescue
-    nil
-  ensure
-  end
-end
+  RUBYBIN = File.join(
+    RbConfig::CONFIG['bindir'],
+    "#{RbConfig::CONFIG['ruby_install_name']}#{RbConfig::CONFIG['EXEEXT']}")
+
+  # Called by the standalone agent
+  Worker::Server.boot if daemon?
 
-if defined?(Rails)
-  require 'skylight/railtie'
+  if defined?(Rails)
+    require 'skylight/railtie'
+  end
 end

data/lib/skylight/api.rb

@@ -0,0 +1,34 @@
+module Skylight
+  class Api
+    include Util::Logging
+
+    attr_reader :config, :http
+
+    def initialize(config, service = :accounts)
+      @config = config
+      @http   = Util::HTTP.new(config, service)
+    end
+
+    def authentication
+      @http.authentication
+    end
+
+    def authentication=(token)
+      @http.authentication = token
+    end
+
+    def login(email, password)
+      res = http.get('/me', 'X-Email' => email, 'X-Password' => password)
+
+      if res && res.success?
+        res.get('me.authentication_token')
+      end
+    end
+
+    def create_app(name)
+      res = @http.post('/apps', { app: { name: name }})
+      res if res.success?
+    end
+
+  end
+end

data/lib/skylight/cli.rb

@@ -1,92 +1,103 @@
-# Use vendored thor / highline
-$:.unshift File.expand_path('../vendor', __FILE__)
+$:.unshift File.expand_path('../vendor/cli', __FILE__)
 
-require "skylight"
-require "thor"
-require "highline"
-require "active_support/inflector"
+require 'skylight'
+require 'thor'
+require 'yaml'
+require 'highline'
+require 'active_support/inflector'
 
 module Skylight
   class CLI < Thor
+
     desc "setup", "Sets up a new app"
     def setup
-      if !SanityChecker.new.smoke_test(File.expand_path("config/skylight.yml"))
+      if File.exist?(config_path)
         say "Your app is already on Skylight. http://www.skylight.io", :green
         return
       end
 
-      token = load_credentials
-      config = http_config
-
-      response = Util::HTTP.new(config).create_app(token, app_name)["app"]
-
-      config.app_id = response["id"]
-      config.authentication_token = response["token"]
+      api.authentication = load_credentials
 
-      config.yaml_file = File.expand_path("config/skylight.yml")
+      unless res = api.create_app(app_name)
+        say "Could not create the application", :red
+        return
+      end
 
-      config.save
+      config[:application]    = res.get('app.id')
+      config[:authentication] = res.get('app.token')
+      config.write(config_path)
 
       say "Congratulations. Your application is on Skylight! http://www.skylight.io", :green
+    rescue Interrupt
     end
 
   private
-    def user_settings
-      File.expand_path("~/.skylight")
-    end
 
-    def http_config
-      @http_config ||= Config.new do |c|
-        c.host = "www.skylight.io"
-        c.port = 443
-        c.ssl = true
-        c.deflate = false
-      end
+    def app_name
+      @app_name ||=
+        begin
+          if File.exist?("config/application.rb")
+            require "./config/application"
+            Rails.application.class.name.split("::").first.underscore.humanize
+          else
+            File.basename(File.expand_path('.')).humanize
+          end
+        end
     end
 
     def load_credentials
-      if credentials?
-        return YAML.load_file(user_settings)["token"]
-      end
+      load_credentials_from_file || login
+    end
 
-      token = nil
+    def login
+      10.times do
+        email    = highline.ask("Email: ")
+        password = highline.ask("Password: ") { |q| q.echo = "*" }
 
-      loop do
-        h = HighLine.new
-        email = h.ask("Email: ")
-        password = h.ask("Password: ") { |q| q.echo = "*" }
+        if token = api.login(email, password)
+          # Write the token
+          FileUtils.mkdir_p(File.dirname(credentials_path))
+          File.open(credentials_path, 'w') do |f|
+            f.puts YAML.dump('token' => token)
+          end
 
-        response = Util::HTTP.new(http_config).auth(email, password)
-        if response["authenticated"] == false
-          say "Sorry. That email and password was invalid. Please try again", :red
-          puts
-        else
-          token = response["token"]
-          break
+          return token
         end
-      end
 
-      File.open(user_settings, "w") do |file|
-        file.puts YAML.dump("token" => token)
+        say "Sorry. That email and password was invalid. Please try again", :red
+        puts
       end
 
-      token
+      say "Could not login", :red
+      return
     end
 
-    def credentials?
-      !SanityChecker.new.user_credentials(user_settings)
+    def load_credentials_from_file
+      return unless File.exist?(credentials_path)
+      return unless yaml = YAML.load_file(credentials_path)
+      yaml['token']
     end
 
-    def app_name
-      @app_name ||=
-        begin
-          if File.exist?("config/application.rb")
-            require "./config/application"
-            Rails.application.class.name.split("::").first.underscore.humanize
-          else
-            File.basename(File.expand_path('.')).humanize
-          end
-        end
+    def config_path
+      File.expand_path('config/skylight.yml')
+    end
+
+    def credentials_path
+      return nil unless ENV['HOME']
+      File.expand_path(".skylight", ENV['HOME'])
+    end
+
+    def api
+      @api ||= Api.new(config)
     end
+
+    def highline
+      @highline ||= HighLine.new
+    end
+
+    def config
+      @config ||= Config.new
+    end
+
   end
 end

data/lib/skylight/compat.rb

@@ -1,9 +1,23 @@
 module Skylight
-  # Catch-up out of date software
-  #
-  # TODO: Add smarter detection
-  if true
-    require 'skylight/compat/notifications'
+  # Ensure the version of AS:N being used is recent enough
+  begin
+    # Attempt to reference an internal class
+    ActiveSupport::Notifications::Fanout::Subscribers
+  rescue NameError
+    # If the class is missing, require our vendored AS::N
+    require 'skylight/vendor/active_support/notifications'
   end
+end
 
+if defined?(ActiveSupport::Notifications::Fanout::Subscribers::Evented)
+  # Handle early RCs of rails 4.0
+  class ActiveSupport::Notifications::Fanout::Subscribers::Evented
+    unless method_defined?(:publish)
+      def publish(name, *args)
+        if @delegate.respond_to?(:publish)
+          @delegate.publish name, *args
+        end
+      end
+    end
+  end
 end

data/lib/skylight/config.rb

@@ -1,147 +1,268 @@
 require 'yaml'
-require 'logger'
 require 'fileutils'
+require 'logger'
 
 module Skylight
   class Config
-    class Normalizer < Struct.new(:view_paths)
-    end
 
-    # Stub out the GC Profiler in cases when it is not requested
-    GC_PROFILER_STUB = Struct.new(
-      :enable,
-      :disable,
-      :clear,
-      :total_time).
-      new(nil, nil, nil, 0)
-
-    class << self
-      def load_from_yaml(path, env=ENV)
-        new do |config|
-          data = YAML.load_file(path)
-
-          data.each do |key, value|
-            apply_config(config, key, value)
-          end
+    # Map environment variable keys with Skylight configuration keys
+    ENV_TO_KEY = {
+      'SK_LOG'                 => :'log',
+      'SK_LOG_LEVEL'           => :'log_level',
+      'SK_APPLICATION'         => :'application',
+      'SK_AUTHENTICATION'      => :'authentication',
+      'SK_AGENT_INTERVAL'      => :'agent.interval',
+      'SK_AGENT_KEEPALIVE'     => :'agent.keepalive',
+      'SK_AGENT_SAMPLE_SIZE'   => :'agent.sample',
+      'SK_AGENT_SOCKFILE_PATH' => :'agent.sockfile_path',
+      'SK_REPORT_HOST'         => :'report.host',
+      'SK_REPORT_PORT'         => :'report.port',
+      'SK_REPORT_SSL'          => :'report.ssl',
+      'SK_REPORT_DEFLATE'      => :'report.deflate',
+      'SK_ACCOUNTS_HOST'       => :'accounts.host',
+      'SK_ACCOUNTS_PORT'       => :'accounts.port',
+      'SK_ACCOUNTS_SSL'        => :'accounts.ssl',
+      'SK_ACCOUNTS_DEFLATE'    => :'accounts.deflate' }
+
+    # Default values for Skylight configuration keys
+    DEFAULTS = {
+      :'log'              => '-'.freeze,
+      :'log_level'        => 'INFO'.freeze,
+      :'agent.keepalive'  => 60,
+      :'agent.interval'   => 5,
+      :'agent.sample'     => 200,
+      :'report.host'      => 'agent.skylight.io'.freeze,
+      :'report.port'      => 443,
+      :'report.ssl'       => true,
+      :'report.deflate'   => true,
+      :'accounts.host'    => 'www.skylight.io'.freeze,
+      :'accounts.port'    => 443,
+      :'accounts.ssl'     => true,
+      :'accounts.deflate' => false }.freeze
+
+    REQUIRED = {
+      :'authentication' => "authentication token",
+      :'report.host'    => "skylight remote host",
+      :'report.port'    => "skylight remote port" }
+
+    VALIDATORS = {
+      :'agent.interval' => lambda { |v, c| Integer === v && v > 0 }
+    }
 
-          config.yaml_file = path
+    def self.load(path = nil, environment = nil, env = ENV)
+      attrs   = {}
+      version = nil
 
-          apply_env(config, env)
+      if path
+        begin
+          attrs = YAML.load_file(path)
+        rescue Exception => e
+          raise ConfigError, "could not load config file; msg=#{e.message}"
         end
+
+        version = File.mtime(path).to_i
+      end
+
+      if env
+        attrs[:priority] = remap_env(env)
       end
 
-      def load_from_env(opts={}, env=ENV)
-        new(opts) do |config|
-          apply_env(config, env)
+      new(environment, attrs)
+    end
+
+    def self.load_from_env(env = ENV)
+      self.load(nil, nil, env)
+    end
+
+    def self.remap_env(env)
+      ret = {}
+
+      env.each do |k, val|
+        if key = ENV_TO_KEY[k]
+          ret[key] =
+            case val
+            when /^false$/i      then false
+            when /^true$/i       then true
+            when /^(nil|null)$/i then nil
+            when /^\d+$/         then val.to_i
+            when /^\d+\.\d+$/    then val.to_f
+            else val
+            end
         end
+      end if env
+
+      ret
+    end
+
+    attr_reader :environment
+
+    def initialize(*args)
+      attrs = {}
+
+      if Hash === args.last
+        attrs = args.pop.dup
       end
 
-    private
+      @values   = {}
+      @priority = {}
+      @regexp   = nil
+
+      p = attrs.delete(:priority)
 
-      def apply_env(config, env)
-        env.each do |key, value|
-          name = normalize_env(key)
-          apply_config(config, name, value) if name
+      if @environment = args[0]
+        @regexp = /^#{Regexp.escape(@environment)}\.(.+)$/
+      end
+
+      attrs.each do |k, v|
+        self[k] = v
+      end
+
+      if p
+        p.each do |k, v|
+          @priority[k.to_sym] = v
         end
       end
+    end
 
-      def normalize_env(key)
-        match = key.match(/^SKYLIGHT_(\w+)$/)
-        match && match[1].downcase
+    def validate!
+      REQUIRED.each do |k, v|
+        unless get(k)
+          raise ConfigError, "#{v} required"
+        end
       end
 
-      def apply_config(config, key, value)
-        config.send("#{key}=", value) if config.respond_to?("#{key}=")
+      true
+    end
+
+    def get(key, default = nil, &blk)
+      key = key.to_sym
+
+      return @priority[key] if @priority.key?(key)
+      return @values[key]   if @values.key?(key)
+      return DEFAULTS[key]  if DEFAULTS.key?(key)
+
+      if default
+        return default
+      elsif blk
+        return blk.call(key)
       end
+
+      nil
     end
 
-    def initialize(attrs = {})
-      # Set defaults
-      @ssl      = true
-      @deflate  = true
-      @host     = "agent.skylight.io"
-      @port     = 443
-      @interval = 5
-      @protocol = JsonProto.new(self)
-      @enable_gc_profiler   = true
-      @max_pending_traces   = 500
-      @samples_per_interval = 100
+    alias [] get
+
+    def set(key, val, scope = nil)
+      if scope
+        key = [scope, key].join('.')
+      end
 
-      @logger = Logger.new(STDOUT)
-      @logger.level = Logger::INFO
+      if Hash === val
+        val.each do |k, v|
+          set(k, v, key)
+        end
+      else
+        k = key.to_sym
 
-      @normalizer = Normalizer.new
+        if validator = VALIDATORS[k]
+          unless validator.call(val, self)
+            raise ConfigError, "invalid value for #{k} (#{val})"
+          end
+        end
 
-      attrs.each do |k, v|
-        if respond_to?("#{k}=")
-          send("#{k}=", v)
+        if @regexp && key =~ @regexp
+          @priority[$1.to_sym] = val
         end
+
+        @values[k] = val
       end
+    end
 
-      yield self if block_given?
+    alias []= set
 
-      @http ||= Util::HTTP.new(self)
+    def to_env
+      ret = {}
 
-      unless @gc_profiler
-        if enable_gc_profiler && GC::Profiler.respond_to?(:enable)
-          @gc_profiler = GC::Profiler
-        else
-          @gc_profiler = GC_PROFILER_STUB
+      ENV_TO_KEY.each do |k, v|
+        if (c = get(v)) != DEFAULTS[v]
+          ret[k] = cast_for_env(c)
         end
       end
+
+      ret
     end
 
-    attr_accessor :yaml_file,
-      :authentication_token,
-      :app_id,
-      :ssl,
-      :deflate,
-      :host,
-      :port,
-      :http,
-      :samples_per_interval,
-      :interval,
-      :max_pending_traces,
-      :normalizer,
-      :protocol,
-      :logger,
-      :enable_gc_profiler,
-      :gc_profiler
-
-    alias_method :ssl?,     :ssl
-    alias_method :deflate?, :deflate
-
-    def protocol=(val)
-      if val.is_a?(String) || val.is_a?(Symbol)
-        class_name = val.to_s.capitalize+"Proto"
-        val = Skylight.const_get(class_name).new(self)
+    #
+    #
+    # ===== Writing =====
+    #
+    #
+
+    def write(path)
+      FileUtils.mkdir_p(File.dirname(path))
+
+      File.open(path, 'w') do |f|
+        f.puts <<-YAML
+---
+# The Skylight ID for the application.
+application: #{self[:application]}
+
+# The authentication token for the application.
+authentication: #{self[:authentication]}
+        YAML
       end
+    end
+
+    #
+    #
+    # ===== Helpers =====
+    #
+    #
 
-      @protocol = val
+    def worker
+      Worker::Builder.new(self)
     end
 
-    def log_level
-      logger && logger.level
+    def gc
+      GC.new(get('gc.profiler', ::GC::Profiler))
     end
 
-    def log_level=(level)
-      if logger
-        if level.is_a?(String) || level.is_a?(Symbol)
-          level = Logger.const_get(level.to_s.upcase)
+    def logger
+      @logger ||=
+        begin
+          out = get(:'log')
+          out = STDOUT if out == '-'
+
+          unless IO === out
+            FileUtils.mkdir_p(File.dirname(out))
+          end
+
+          l = Logger.new(out)
+          l.level =
+            case get(:'log_level')
+            when /^debug$/i then Logger::DEBUG
+            when /^info$/i  then Logger::INFO
+            when /^warn$/i  then Logger::WARN
+            when /^error$/i then Logger::ERROR
+            end
+          l
         end
-        logger.level = level
-      end
     end
 
-    def save(filename=yaml_file)
-      FileUtils.mkdir_p File.dirname(filename)
+    def logger=(logger)
+      @logger = logger
+    end
+
+  private
 
-      File.open(filename, "w") do |file|
-        config = {}
-        config["authentication_token"] = authentication_token if authentication_token
-        config["app_id"] = app_id if app_id
-        file.puts YAML.dump(config)
+    def cast_for_env(v)
+      case v
+      when true  then 'true'
+      when false then 'false'
+      when nil   then 'nil'
+      else v.to_s
       end
     end
+
   end
 end