skylight 0.3.7 → 0.3.8

data/lib/skylight/version.rb

@@ -1,4 +1,4 @@
 module Skylight
-  VERSION = '0.3.7'
+  VERSION = '0.3.8'
 end
 

data/lib/skylight/worker.rb

@@ -5,12 +5,14 @@ module Skylight
     CHUNK_SIZE = 16 * 1024
 
     # === Modules
-    autoload :Builder,    'skylight/worker/builder'
-    autoload :Collector,  'skylight/worker/collector'
-    autoload :Connection, 'skylight/worker/connection'
-    autoload :Embedded,   'skylight/worker/embedded'
-    autoload :Server,     'skylight/worker/server'
-    autoload :Standalone, 'skylight/worker/standalone'
+    autoload :Builder,         'skylight/worker/builder'
+    autoload :Collector,       'skylight/worker/collector'
+    autoload :Connection,      'skylight/worker/connection'
+    autoload :ConnectionSet,   'skylight/worker/connection_set'
+    autoload :Embedded,        'skylight/worker/embedded'
+    autoload :MetricsReporter, 'skylight/worker/metrics_reporter'
+    autoload :Server,          'skylight/worker/server'
+    autoload :Standalone,      'skylight/worker/standalone'
 
   end
 end

data/lib/skylight/worker/builder.rb

@@ -19,7 +19,7 @@ module Skylight
         case s
         when 'embedded'
           trace "building embedded worker"
-          Embedded.new(Collector.new(config))
+          Embedded.new(Collector.build(config))
         when 'standalone'
           trace "building standalone worker"
 

data/lib/skylight/worker/collector.rb

@@ -11,23 +11,39 @@ module Skylight
 
       include Util::Logging
 
-      attr_reader :config
+      attr_reader :config, :metrics_reporter
 
-      def initialize(config)
+      def initialize(config, metrics_reporter = nil)
         super(1000, 0.25)
 
-        @config      = config
-        @size        = config[:'agent.sample']
-        @batch       = nil
-        @interval    = config[:'agent.interval']
-        @refresh_at  = 0
-        @http_auth   = Util::HTTP.new(config, :accounts)
+        @config = config
+        @size = config[:'agent.sample']
+        @batch = nil
+        @interval = config[:'agent.interval']
+        @refresh_at = 0
+        @http_auth = Util::HTTP.new(config, :accounts)
         @http_report = nil
-        # @http_report = Util::HTTP.new(config, :report)
+        @report_meter = Metrics::Meter.new
+        @report_success_meter = Metrics::Meter.new
+        @metrics_reporter = metrics_reporter
+
+        @metrics_reporter.register("collector.report-rate", @report_meter)
+        @metrics_reporter.register("collector.report-success-rate", @report_success_meter)
 
         t { fmt "starting collector; interval=%d; size=%d", @interval, @size }
       end
 
+      def self.build(config)
+        new(config, MetricsReporter.new(config))
+      end
+
+      def prepare
+        if @metrics_reporter
+          @metrics_reporter.register("worker.collector.queue-depth", queue_depth_metric)
+          @metrics_reporter.spawn
+        end
+      end
+
       def handle(msg, now = Util::Clock.absolute_secs)
         @batch ||= new_batch(now)
 
@@ -61,15 +77,23 @@ module Skylight
         true
       end
 
-      def send_status(status)
-        post_data(:status, status)
+      def send_http_exception(http, response)
+        send_exception(response.exception, additional_info: {
+          host: http.host,
+          port: http.port,
+          path: response.request.path,
+          method: response.request.method
+        })
       end
 
-      def send_exception(exception)
-        data = {class_name: exception.class.name}
+      def send_exception(exception, data={})
+        data = { class_name: exception.class.name,
+                 agent_info: @metrics_reporter.build_report }.merge(data)
+
         if Exception === exception
           data.merge!(message: exception.message, backtrace: exception.backtrace)
         end
+
         post_data(:exception, data, false)
       end
 
@@ -84,7 +108,12 @@ module Skylight
           warn "#{type} wasn't sent successfully; status=%s", res.status
         end
 
-        send_exception(res.exception) if notify && res.exception
+        if res.exception
+          send_http_exception(@http_auth, res) if notify
+          false
+        else
+          true
+        end
       rescue Exception => e
         error "exception; msg=%s; class=%s", e.message, e.class
         t { e.backtrace.join("\n") }
@@ -109,6 +138,10 @@ module Skylight
         end
 
         @batch = nil
+      ensure
+        if @metrics_reporter
+          @metrics_reporter.shutdown
+        end
       end
 
       def flush(batch)
@@ -116,8 +149,16 @@ module Skylight
 
         debug "flushing batch; size=%d", batch.sample.count
 
+        @report_meter.mark
+
         res = @http_report.post(ENDPOINT, batch.encode, CONTENT_TYPE => SKYLIGHT_V2)
-        send_exception(res.exception) if res.exception
+
+        if res.exception
+          send_http_exception(@http_report, res)
+        else
+          @report_success_meter.mark
+        end
+
         nil
       end
 
@@ -125,7 +166,7 @@ module Skylight
         res = @http_auth.get("/agent/authenticate?hostname=#{escape(config[:'hostname'])}")
 
         if res.exception
-          send_exception(res.exception)
+          send_http_exception(@http_auth, res)
           return
         end
 

data/lib/skylight/worker/connection.rb

@@ -1,14 +1,18 @@
 module Skylight
   module Worker
+    # Represents the IPC client connection
     class Connection
       FRAME_HDR_LEN = 8
 
-      attr_reader :sock
+      attr_reader :sock, :throughput
 
       def initialize(sock)
         @sock = sock
         @len  = nil
         @buf  = ""
+
+        # Metrics
+        @throughput = Metrics::Meter.new
       end
 
       def read
@@ -17,6 +21,7 @@ module Skylight
         end
 
         if chunk = read_sock
+
           @buf << chunk
 
           if !@len && @buf.bytesize >= FRAME_HDR_LEN
@@ -27,6 +32,10 @@ module Skylight
         end
       end
 
+      def cleanup
+        # Any cleanup code here
+      end
+
     private
 
       def read_len
@@ -66,7 +75,10 @@ module Skylight
       end
 
       def read_sock
-        @sock.read_nonblock(CHUNK_SIZE)
+        ret = @sock.read_nonblock(CHUNK_SIZE)
+        # Track the throughput
+        @throughput.mark(ret.bytesize) if ret
+        ret
       rescue Errno::EAGAIN, Errno::EWOULDBLOCK
       end
 

data/lib/skylight/worker/connection_set.rb

@@ -0,0 +1,56 @@
+require 'thread'
+
+module Skylight
+  module Worker
+    class ConnectionSet
+      attr_reader :open_connections, :throughput
+
+      def initialize
+        @connections = {}
+        @lock = Mutex.new
+
+        # Metrics
+        @open_connections = build_open_connections_metric
+        @throughput = build_throughput_metric
+      end
+
+      def add(sock)
+        conn = Connection.new(sock)
+        @lock.synchronize { @connections[sock] = conn }
+        conn
+      end
+
+      def socks
+        @lock.synchronize { @connections.keys }
+      end
+
+      def [](sock)
+        @lock.synchronize do
+          @connections[sock]
+        end
+      end
+
+      def cleanup(sock)
+        if conn = @lock.synchronize { @connections.delete(sock) }
+          conn.cleanup
+          sock.close rescue nil
+        end
+      end
+
+    private
+
+      def build_open_connections_metric
+        lambda do
+          @lock.synchronize { @connections.length }
+        end
+      end
+
+      def build_throughput_metric
+        lambda do
+          conns = @lock.synchronize { @connections.values }
+          conns.map { |c| c.throughput.rate.to_i }
+        end
+      end
+    end
+  end
+end

data/lib/skylight/worker/metrics_reporter.rb

@@ -0,0 +1,103 @@
+require 'thread'
+require 'rbconfig'
+
+module Skylight
+  module Worker
+    class MetricsReporter < Util::Task
+
+      include Util::Logging
+
+      attr_reader :config
+
+      def initialize(config)
+        super(1000, 0.25)
+
+        @metrics = {}
+        @config = config
+        @interval = config[:'metrics.report_interval']
+        @lock = Mutex.new
+        @next_report_at = nil
+        @http_auth = Util::HTTP.new(config, :accounts)
+      end
+
+      # A metric responds to #call and returns metric info
+      def register(name, metric)
+        @lock.synchronize { @metrics[name] = metric }
+      end
+
+      def unregister(name)
+        @lock.synchronize { @metrics.delete(name) }
+      end
+
+      # msg is always nil, but we can use the Task abstraction anyway
+      def handle(msg, now = Util::Clock.absolute_secs)
+        # Initially set the next report at
+        unless @next_report_at
+          update_next_report_at(now)
+          return true
+        end
+
+        if now < @next_report_at
+          # Nothing to do
+          return true
+        end
+
+        update_next_report_at(now)
+        post_report
+
+        true
+      end
+
+      def build_report
+        report = {
+          "hostname"           => config[:'hostname'],
+          "host.info"          => RbConfig::CONFIG['arch'],
+          "ruby.version"       => RUBY_VERSION,
+          "ruby.engine"        => RUBY_ENGINE,
+          "skylight.version"   => Skylight::VERSION
+        }
+
+        metric_names.each do |name|
+          # Since we are operating in a concurrent environment, it is possible
+          # that the metric for the current name is unregistered before we
+          # access it here.
+          unless m = metric(name)
+            next
+          end
+
+          report[name] = m.call
+        end
+
+        report
+      end
+
+      def post_report
+        report = build_report
+
+        # Send the report
+        t { fmt "reporting internal metrics; payload=%s", report.inspect }
+
+        res = @http_auth.post("/agent/metrics", report: report)
+
+        unless res.success?
+          warn "internal metrics report failed; status=%s", res.status
+        end
+      end
+
+    private
+
+      def metric_names
+        @lock.synchronize { @metrics.keys }
+      end
+
+      def metric(name)
+        @lock.synchronize { @metrics[name] }
+      end
+
+      def update_next_report_at(now)
+        @next_report_at = now + @interval
+      end
+
+    end
+  end
+end

data/lib/skylight/worker/server.rb

@@ -17,7 +17,6 @@ module Skylight
         :keepalive,
         :lockfile_path,
         :sockfile_path,
-        :status_interval,
         :last_status_update,
         :max_memory
 
@@ -26,20 +25,23 @@ module Skylight
           raise ArgumentError, "lockfile and unix domain server socket are required"
         end
 
-        @pid           = Process.pid
-        @run           = true
-        @tick          = 1
-        @socks         = []
-        @config        = config
-        @server        = srv
-        @lockfile      = lockfile
-        @collector     = Collector.new(config)
-        @keepalive     = @config[:'agent.keepalive']
-        @connections   = {}
+        @pid = Process.pid
+        @run = true
+        @tick = 1
+        @socks = []
+        @config = config
+        @server = srv
+        @lockfile = lockfile
+        @collector = Collector.build(config)
+        @metrics_reporter = @collector.metrics_reporter
+        @keepalive = @config[:'agent.keepalive']
+        @connections = ConnectionSet.new
         @lockfile_path = lockfile_path
         @sockfile_path = @config[:'agent.sockfile_path']
-        @status_interval = 60
-        @max_memory      = @config[:'agent.max_memory']
+        @process_mem_gauge = Metrics::ProcessMemGauge.new
+        @process_cpu_gauge = Metrics::ProcessCpuGauge.new
+        @max_memory = @config[:'agent.max_memory']
+        @booted_at = Util::Clock.absolute_secs
       end
 
       # Called from skylight.rb on require
@@ -122,9 +124,21 @@ module Skylight
     private
 
       def init
+        # TODO: Not super ideal to always iterate here even if debug mode isn't
+        # enabled, but it's not super perf critical. We will fix when we revamp
+        # logging
+        debug "initializing server; config=%s", config.to_env
+
         trap('TERM') { @run = false }
         trap('INT')  { @run = false }
 
+        # Register metrics
+        @metrics_reporter.register("worker.memory", @process_mem_gauge)
+        @metrics_reporter.register("worker.cpu", @process_cpu_gauge)
+        @metrics_reporter.register("worker.uptime", lambda { Util::Clock.absolute_secs - @booted_at })
+        @metrics_reporter.register("worker.ipc.open-connections", @connections.open_connections)
+        @metrics_reporter.register("worker.ipc.throughput", @connections.throughput)
+
         info "starting skylight daemon"
         @collector.spawn
       end
@@ -190,9 +204,8 @@ module Skylight
               sanity_check
             end
 
-            if status_interval < now - last_status_update
-              last_status_update = now
-              status_check
+            if @process_mem_gauge.call > max_memory
+              raise WorkerStateError, "Memory limit exceeded: #{memory_usage} (max: #{max_memory})"
             end
           end
 
@@ -213,6 +226,9 @@ module Skylight
         end while @run
 
         true # Successful return
+      ensure
+        # Send a final metrics report
+        @metrics_reporter.post_report
       end
 
       # Handles an incoming message. Will be instances from
@@ -254,7 +270,7 @@ module Skylight
       def connect(sock)
         trace "client accepted"
         @socks << sock
-        @connections[sock] = Connection.new(sock)
+        @connections.add(sock)
       end
 
       def cleanup
@@ -272,16 +288,15 @@ module Skylight
       end
 
       def clients_close
-        @connections.keys.each do |sock|
+        @connections.socks.each do |sock|
           client_close(sock)
         end
       end
 
       def client_close(sock)
         trace "closing client connection; fd=%d", sock.fileno
-        @connections.delete(sock)
+        @connections.cleanup(sock)
         @socks.delete(sock)
-        sock.close rescue nil
       end
 
       def sockfile
@@ -315,22 +330,6 @@ module Skylight
           raise WorkerStateError, "sockfile gone"
         end
       end
-
-      def status_check
-        memory_usage = get_memory_usage
-
-        @collector.send_status(memory: memory_usage, max_memory: max_memory)
-
-        if memory_usage > max_memory
-          raise WorkerStateError, "Memory limit exceeded: #{memory_usage} (max: #{max_memory})"
-        end
-      end
-
-      def get_memory_usage
-        `ps -o rss= -p #{Process.pid}`.to_i / 1024
-      rescue Errno::ENOENT, Errno::EINTR
-        0
-      end
     end
   end
 end