skylight 0.3.21 → 0.4.0.alpha1

data/lib/skylight/worker/collector.rb

@@ -1,274 +0,0 @@
-require 'uri'
-
-module Skylight
-  module Worker
-    class Collector < Util::Task
-      include URI::Escape
-
-      ENDPOINT     = '/report'.freeze
-      CONTENT_TYPE = 'content-type'.freeze
-      SKYLIGHT_V2  = 'application/x-skylight-report-v2'.freeze
-
-      include Util::Logging
-
-      attr_reader :config, :metrics_reporter
-
-      def initialize(config, metrics_reporter = nil)
-        super(1000, 0.25)
-
-        @config = config
-        @size = config[:'agent.sample']
-        @batch = nil
-        @interval = config[:'agent.interval']
-        @refresh_at = 0
-        @http_auth = Util::HTTP.new(config, :accounts)
-        @http_report = nil
-        @report_meter = Metrics::Meter.new
-        @report_success_meter = Metrics::Meter.new
-        @metrics_reporter = metrics_reporter
-
-        @metrics_reporter.register("collector.report-rate", @report_meter)
-        @metrics_reporter.register("collector.report-success-rate", @report_success_meter)
-
-        t { fmt "starting collector; interval=%d; size=%d", @interval, @size }
-      end
-
-      def self.build(config)
-        new(config, MetricsReporter.new(config))
-      end
-
-      def prepare
-        if @metrics_reporter
-          @metrics_reporter.register("worker.collector.queue-depth", queue_depth_metric)
-          @metrics_reporter.spawn
-        end
-      end
-
-      def handle(msg, now = Util::Clock.absolute_secs)
-        @batch ||= new_batch(now)
-
-        if should_refresh_token?(now)
-          refresh_report_token(now)
-        end
-
-        if @batch.should_flush?(now)
-          if has_report_token?(now)
-            flush(@batch)
-          else
-            warn "do not have valid session token -- dropping"
-            return true
-          end
-
-          @batch = new_batch(now)
-        end
-
-        return true unless msg
-
-        case msg
-        when Messages::TraceEnvelope
-          t { fmt "collector received trace" }
-          @batch.push(msg)
-        when Error
-          send_error(msg)
-        else
-          debug "Received unknown message; class=%s", msg.class.to_s
-        end
-
-        true
-      end
-
-      def send_http_exception(http, response)
-        send_exception(response.exception, additional_info: {
-          host: http.host,
-          port: http.port,
-          path: response.request.path,
-          method: response.request.method
-        })
-      end
-
-      def send_exception(exception, data={})
-        data = { class_name: exception.class.name,
-                 agent_info: @metrics_reporter.build_report }.merge(data)
-
-        if Exception === exception
-          data.merge!(message: exception.message, backtrace: exception.backtrace)
-        end
-
-        post_data(:exception, data, false)
-      end
-
-    private
-
-      def post_data(type, data, notify = true)
-        t { "posting data (#{type}): #{data.inspect}" }
-
-        res = @http_auth.post("/agent/#{type}?hostname=#{escape(config[:'hostname'])}", data)
-
-        unless res.success?
-          warn "#{type} wasn't sent successfully; status=%s", res.status
-        end
-
-        if res.exception
-          send_http_exception(@http_auth, res) if notify
-          false
-        else
-          true
-        end
-      rescue Exception => e
-        error "exception; msg=%s; class=%s", e.message, e.class
-        t { e.backtrace.join("\n") }
-      end
-
-      def send_error(msg)
-        details = msg.details ? JSON.parse(msg.details) : nil
-        post_data(:error, type: msg.type, description: msg.description, details: details)
-      end
-
-      def finish
-        t { fmt "collector finishing up" }
-
-        now = Util::Clock.absolute_secs
-
-        if should_refresh_token?(now)
-          refresh_report_token(now)
-        end
-
-        if @batch && has_report_token?(now)
-          flush(@batch)
-        end
-
-        @batch = nil
-      ensure
-        if @metrics_reporter
-          @metrics_reporter.shutdown
-        end
-      end
-
-      def flush(batch)
-        return if batch.empty?
-
-        debug "flushing batch; size=%d", batch.sample.count
-
-        @report_meter.mark
-
-        res = @http_report.post(ENDPOINT, batch.encode, CONTENT_TYPE => SKYLIGHT_V2)
-
-        if res.exception
-          send_http_exception(@http_report, res)
-        else
-          @report_success_meter.mark
-        end
-
-        nil
-      end
-
-      def refresh_report_token(now)
-        res = @http_auth.get("/agent/authenticate?hostname=#{escape(config[:'hostname'])}")
-
-        if res.exception
-          send_http_exception(@http_auth, res)
-          return
-        end
-
-        unless res.success?
-          if (400..499).include? res.status
-            warn "token request rejected; status=%s", res.status
-            @http_report = nil
-          end
-
-          warn "could not fetch report session token; status=%s", res.status
-          return
-        end
-
-        session = res.body['session']
-        tok, expires_at = session['token'], session['expires_at'] if session
-
-        if tok && expires_at
-          if expires_at <= now
-            error "token is expired: token=%s; expires_at=%s", tok, expires_at
-            return
-          end
-
-          # 30 minute buffer or split the difference
-          @refresh_at  = expires_at - now > 3600 ?
-                            now + ((expires_at - now) / 2) :
-                            expires_at - 1800
-
-          @http_report = Util::HTTP.new(config, :report)
-          @http_report.authentication = tok
-        else
-          if @http_report
-            @refresh_at = now + 60
-          end
-          warn "server did not return a session token"
-        end
-      rescue Exception => e
-        error "exception; msg=%s; class=%s", e.message, e.class
-        t { e.backtrace.join("\n") }
-      end
-
-      def should_refresh_token?(now)
-        now >= @refresh_at
-      end
-
-      def has_report_token?(now)
-        return true if config.ignore_token?
-        return unless @http_report
-        now < @refresh_at + (3600 * 3 - 660)
-      end
-
-      def new_batch(now)
-        Batch.new(config, @size, round(now), @interval)
-      end
-
-      def round(time)
-        (time.to_i / @interval) * @interval
-      end
-
-      class Batch
-        include Util::Logging
-
-        attr_reader :config, :from, :counts, :sample, :flush_at
-
-        def initialize(config, size, from, interval)
-          @config   = config
-          @from     = from
-          @flush_at = from + interval
-          @sample   = Util::UniformSample.new(size)
-          @counts   = Hash.new(0)
-        end
-
-        def should_flush?(now)
-          return true if @config.constant_flush?
-          now >= @flush_at
-        end
-
-        def empty?
-          @sample.empty?
-        end
-
-        def push(trace)
-          # Count it
-          @counts[trace.endpoint_name] += 1
-          # Push the trace into the sample
-          @sample << trace
-        end
-
-        def encode
-          batch = Skylight::Batch.native_new(from, config[:hostname])
-
-          sample.each do |trace|
-            batch.native_move_in(trace.data)
-          end
-
-          @counts.each do |endpoint_name,count|
-            batch.native_set_endpoint_count(endpoint_name, count)
-          end
-
-          batch.native_serialize
-        end
-      end
-
-    end
-  end
-end

data/lib/skylight/worker/connection.rb

@@ -1,87 +0,0 @@
-module Skylight
-  module Worker
-    # Represents the IPC client connection
-    class Connection
-      FRAME_HDR_LEN = 8
-
-      attr_reader :sock, :throughput
-
-      def initialize(sock)
-        @sock = sock
-        @len  = nil
-        @buf  = ""
-
-        # Metrics
-        @throughput = Metrics::Meter.new
-      end
-
-      def read
-        if msg = maybe_read_message
-          return msg
-        end
-
-        if chunk = read_sock
-
-          @buf << chunk
-
-          if !@len && @buf.bytesize >= FRAME_HDR_LEN
-            @len = read_len
-          end
-
-          maybe_read_message
-        end
-      end
-
-      def cleanup
-        # Any cleanup code here
-      end
-
-    private
-
-      def read_len
-        if len = @buf[4, 4]
-          len.unpack("L")[0]
-        end
-      end
-
-      def read_message_id
-        if win = @buf[0, 4]
-          win.unpack("L")[0]
-        end
-      end
-
-      def maybe_read_message
-        if @len && @buf.bytesize >= @len + FRAME_HDR_LEN
-          mid   = read_message_id
-          klass = Messages::ID_TO_KLASS.fetch(mid) do
-            raise IpcProtoError, "unknown message `#{mid}`"
-          end
-          data  = @buf[FRAME_HDR_LEN, @len]
-          @buf  = @buf[(FRAME_HDR_LEN + @len)..-1] || ""
-
-          if @buf.bytesize >= FRAME_HDR_LEN
-            @len = read_len
-          else
-            @len = nil
-          end
-
-          begin
-            return klass.deserialize(data)
-          rescue Exception => e
-            # reraise protobuf decoding exceptions
-            raise IpcProtoError, e.message
-          end
-        end
-      end
-
-      def read_sock
-        ret = @sock.read_nonblock(CHUNK_SIZE)
-        # Track the throughput
-        @throughput.mark(ret.bytesize) if ret
-        ret
-      rescue Errno::EAGAIN, Errno::EWOULDBLOCK
-      end
-
-    end
-  end
-end

data/lib/skylight/worker/connection_set.rb

@@ -1,56 +0,0 @@
-require 'thread'
-
-module Skylight
-  module Worker
-    class ConnectionSet
-      attr_reader :open_connections, :throughput
-
-      def initialize
-        @connections = {}
-        @lock = Mutex.new
-
-        # Metrics
-        @open_connections = build_open_connections_metric
-        @throughput = build_throughput_metric
-      end
-
-      def add(sock)
-        conn = Connection.new(sock)
-        @lock.synchronize { @connections[sock] = conn }
-        conn
-      end
-
-      def socks
-        @lock.synchronize { @connections.keys }
-      end
-
-      def [](sock)
-        @lock.synchronize do
-          @connections[sock]
-        end
-      end
-
-      def cleanup(sock)
-        if conn = @lock.synchronize { @connections.delete(sock) }
-          conn.cleanup
-          sock.close rescue nil
-        end
-      end
-
-    private
-
-      def build_open_connections_metric
-        lambda do
-          @lock.synchronize { @connections.length }
-        end
-      end
-
-      def build_throughput_metric
-        lambda do
-          conns = @lock.synchronize { @connections.values }
-          conns.map { |c| c.throughput.rate.to_i }
-        end
-      end
-    end
-  end
-end

data/lib/skylight/worker/embedded.rb

@@ -1,24 +0,0 @@
-module Skylight
-  module Worker
-    class Embedded
-      def initialize(collector)
-        @collector = collector
-      end
-
-      def spawn
-        @collector.spawn
-      end
-
-      def shutdown
-        @collector.shutdown
-      end
-
-      def submit(msg)
-        decoder = Messages::ID_TO_KLASS.fetch(Messages::KLASS_TO_ID.fetch(msg.class))
-        msg = decoder.deserialize(msg.serialize)
-
-        @collector.submit(msg)
-      end
-    end
-  end
-end

data/lib/skylight/worker/metrics_reporter.rb

@@ -1,104 +0,0 @@
-require 'thread'
-require 'rbconfig'
-
-module Skylight
-  module Worker
-    class MetricsReporter < Util::Task
-
-      include Util::Logging
-
-      attr_reader :config
-
-      def initialize(config)
-        super(1000, 0.25)
-
-        @metrics = {}
-        @config = config
-        @interval = config[:'metrics.report_interval']
-        @lock = Mutex.new
-        @next_report_at = nil
-        @http_auth = Util::HTTP.new(config, :accounts)
-      end
-
-      # A metric responds to #call and returns metric info
-      def register(name, metric)
-        @lock.synchronize { @metrics[name] = metric }
-      end
-
-      def unregister(name)
-        @lock.synchronize { @metrics.delete(name) }
-      end
-
-      # msg is always nil, but we can use the Task abstraction anyway
-      def handle(msg, now = Util::Clock.absolute_secs)
-        # Initially set the next report at
-        unless @next_report_at
-          update_next_report_at(now)
-          return true
-        end
-
-        if now < @next_report_at
-          # Nothing to do
-          return true
-        end
-
-        update_next_report_at(now)
-        post_report
-
-        true
-      end
-
-      def build_report
-        report = {
-          "hostname"           => config[:'hostname'],
-          "host.info"          => RbConfig::CONFIG['arch'],
-          "ruby.version"       => "#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}",
-          "ruby.engine"        => RUBY_ENGINE,
-          "rails.version"      => defined?(Rails) ? Rails.version : nil,
-          "skylight.version"   => Skylight::VERSION
-        }
-
-        metric_names.each do |name|
-          # Since we are operating in a concurrent environment, it is possible
-          # that the metric for the current name is unregistered before we
-          # access it here.
-          unless m = metric(name)
-            next
-          end
-
-          report[name] = m.call
-        end
-
-        report
-      end
-
-      def post_report
-        report = build_report
-
-        # Send the report
-        t { fmt "reporting internal metrics; payload=%s", report.inspect }
-
-        res = @http_auth.post("/agent/metrics", report: report)
-
-        unless res.success?
-          warn "internal metrics report failed; status=%s", res.status
-        end
-      end
-
-    private
-
-      def metric_names
-        @lock.synchronize { @metrics.keys }
-      end
-
-      def metric(name)
-        @lock.synchronize { @metrics[name] }
-      end
-
-      def update_next_report_at(now)
-        @next_report_at = now + @interval
-      end
-
-    end
-  end
-end