skylight 0.3.21 → 0.4.0.alpha1

data/lib/skylight/worker/server.rb

@@ -1,336 +0,0 @@
-require 'socket'
-
-module Skylight
-  module Worker
-    class Server
-      LOCKFILE_PATH      = 'SKYLIGHT_LOCKFILE_PATH'.freeze
-      LOCKFILE_ENV_KEY   = 'SKYLIGHT_LOCKFILE_FD'.freeze
-      UDS_SRV_FD_KEY     = 'SKYLIGHT_UDS_FD'.freeze
-      KEEPALIVE_KEY      = 'SKYLIGHT_KEEPALIVE'.freeze
-
-      include Util::Logging
-
-      attr_reader \
-        :pid,
-        :tick,
-        :config,
-        :keepalive,
-        :lockfile_path,
-        :sockfile_path,
-        :last_status_update,
-        :max_memory
-
-      def initialize(config, lockfile, srv, lockfile_path)
-        unless lockfile && srv
-          raise ArgumentError, "lockfile and unix domain server socket are required"
-        end
-
-        @pid = Process.pid
-        @run = true
-        @tick = 1
-        @socks = []
-        @config = config
-        @server = srv
-        @lockfile = lockfile
-        @collector = Collector.build(config)
-        @metrics_reporter = @collector.metrics_reporter
-        @keepalive = @config[:'agent.keepalive']
-        @connections = ConnectionSet.new
-        @lockfile_path = lockfile_path
-        @sockfile_path = @config[:'agent.sockfile_path']
-        @process_mem_gauge = Metrics::ProcessMemGauge.new
-        @process_cpu_gauge = Metrics::ProcessCpuGauge.new
-        @max_memory = @config[:'agent.max_memory']
-        @booted_at = Util::Clock.absolute_secs
-      end
-
-      # Called from skylight.rb on require
-      def self.boot
-        fail = lambda do |msg|
-          STDERR.puts msg
-          exit 1
-        end
-
-        config = Config.load_from_env
-
-        unless fd = ENV[LOCKFILE_ENV_KEY]
-          fail.call "missing lockfile FD"
-        end
-
-        unless fd =~ /^\d+$/
-          fail.call "invalid lockfile FD"
-        end
-
-        begin
-          lockfile = IO.open(fd.to_i)
-        rescue Exception => e
-          fail.call "invalid lockfile FD: #{e.message}"
-        end
-
-        unless lockfile_path = ENV[LOCKFILE_PATH]
-          fail.call "missing lockfile path"
-        end
-
-        unless config[:'agent.sockfile_path']
-          fail.call "missing sockfile path"
-        end
-
-        srv = nil
-        if fd = ENV[UDS_SRV_FD_KEY]
-          srv = UNIXServer.for_fd(fd.to_i)
-        end
-
-        server = new(
-          config,
-          lockfile,
-          srv,
-          lockfile_path)
-
-        server.run
-      end
-
-      def self.exec(cmd, config, lockfile, srv, lockfile_path)
-        env = config.to_env
-        env.merge!(
-          STANDALONE_ENV_KEY => STANDALONE_ENV_VAL,
-          LOCKFILE_PATH      => lockfile_path,
-          LOCKFILE_ENV_KEY   => lockfile.fileno.to_s)
-
-        if srv
-          env[UDS_SRV_FD_KEY] = srv.fileno.to_s
-        end
-
-        opts = {}
-        args = [env] + cmd + [opts]
-
-        unless RUBY_VERSION < '1.9'
-          [lockfile, srv].each do |io|
-            next unless io
-            fd = io.fileno.to_i
-            opts[fd] = fd
-          end
-        end
-
-        Kernel.exec(*args)
-      end
-
-      def run
-        init
-        work
-      ensure
-        cleanup
-      end
-
-    private
-
-      def init
-        # TODO: Not super ideal to always iterate here even if debug mode isn't
-        # enabled, but it's not super perf critical. We will fix when we revamp
-        # logging
-        debug "initializing server; config=%s", config.to_env
-
-        trap('TERM') { @run = false }
-        trap('INT')  { @run = false }
-
-        # Register metrics
-        @metrics_reporter.register("worker.memory", @process_mem_gauge)
-        @metrics_reporter.register("worker.cpu", @process_cpu_gauge)
-        @metrics_reporter.register("worker.uptime", lambda { Util::Clock.absolute_secs - @booted_at })
-        @metrics_reporter.register("worker.ipc.open-connections", @connections.open_connections)
-        @metrics_reporter.register("worker.ipc.throughput", @connections.throughput)
-
-        info "starting skylight daemon"
-        @collector.spawn
-      end
-
-      def work
-        t { "server working" }
-        @socks << @server
-
-        now = Time.now.to_i
-        next_sanity_check_at = now + tick
-        had_client_at = now
-        last_status_update = now
-
-        trace "starting IO loop"
-        begin
-          # Wait for something to do
-          r, _, _ = IO.select(@socks, [], [], tick)
-
-          if r
-            r.each do |sock|
-              if sock == @server
-                # If the server socket, accept
-                # the incoming connection
-                if client = accept
-                  connect(client)
-                end
-              else
-                # Client socket, lookup the associated connection
-                # state machine.
-                unless conn = @connections[sock]
-                  # No associated connection, weird.. bail
-                  client_close(sock)
-                  next
-                end
-
-                begin
-                  # Pop em while we got em
-                  while msg = conn.read
-                    handle(msg)
-                  end
-                rescue SystemCallError, EOFError
-                  client_close(sock)
-                rescue IpcProtoError => e
-                  error "Server#work - IPC protocol exception: %s", e.message
-                  client_close(sock)
-                end
-              end
-            end
-          end
-
-          now = Time.now.to_i
-
-          if @socks.length > 1
-            had_client_at = now
-          end
-
-          if keepalive < now - had_client_at
-            info "no clients for #{keepalive} sec - shutting down"
-            @run = false
-          else
-            if next_sanity_check_at <= now
-              next_sanity_check_at = now + tick
-              sanity_check
-            end
-
-            memory_usage = @process_mem_gauge.call
-            if memory_usage > max_memory
-              raise WorkerStateError, "Memory limit exceeded: #{memory_usage} (max: #{max_memory})"
-            end
-          end
-
-        rescue SignalException => e
-          error "Did not handle: #{e.class}"
-          @run = false
-        rescue WorkerStateError => e
-          info "#{e.message} - shutting down"
-          @run = false
-        rescue Exception => e
-          error "Loop exception: %s (%s)\n%s", e.message, e.class, e.backtrace.join("\n")
-          @collector.send_exception(e)
-          return false
-        rescue Object => o
-          error "Unknown object thrown: `%s`", o.to_s
-          @collector.send_exception(o)
-          return false
-        end while @run
-
-        true # Successful return
-      ensure
-        # Send a final metrics report
-        @metrics_reporter.post_report
-      end
-
-      # Handles an incoming message. Will be instances from
-      # the Messages namespace
-      def handle(msg)
-        case msg
-        when nil
-          return
-        when Hello
-          if msg.newer?
-            info "newer version of agent deployed - restarting; curr=%s; new=%s", VERSION, msg.version
-            reload(msg)
-          end
-        when Messages::TraceEnvelope, Error
-          t { "received message" }
-          @collector.submit(msg)
-        when :unknown
-          debug "received unknown message"
-        else
-          debug "recieved: %s", msg
-        end
-      end
-
-      def reload(hello)
-        # Close all client connections
-        trace "closing all client connections"
-        clients_close
-
-        # Re-exec the process
-        trace "re-exec"
-        Server.exec(hello.cmd, @config, @lockfile, @server, lockfile_path)
-      end
-
-      def accept
-        @server.accept_nonblock
-      rescue Errno::EWOULDBLOCK, Errno::EAGAIN, Errno::ECONNABORTED
-      end
-
-      def connect(sock)
-        trace "client accepted"
-        @socks << sock
-        @connections.add(sock)
-      end
-
-      def cleanup
-        t { "server cleaning up" }
-        # The lockfile is not deleted. There is no way to atomically ensure
-        # that we are deleting the lockfile for the current process.
-        cleanup_curr_sockfile
-        close
-        @lockfile.close
-      end
-
-      def close
-        @server.close if @server
-        clients_close
-      end
-
-      def clients_close
-        @connections.socks.each do |sock|
-          client_close(sock)
-        end
-      end
-
-      def client_close(sock)
-        trace "closing client connection; fd=%d", sock.fileno
-        @connections.cleanup(sock)
-        @socks.delete(sock)
-      end
-
-      def sockfile
-        "#{sockfile_path}/skylight-#{pid}.sock"
-      end
-
-      def sockfile?
-        File.exist?(sockfile)
-      end
-
-      def cleanup_curr_sockfile
-        File.unlink(sockfile) rescue nil
-      end
-
-      def sanity_check
-        if !File.exist?(lockfile_path)
-          raise WorkerStateError, "lockfile gone"
-        end
-
-        pid = File.read(lockfile_path) rescue nil
-
-        unless pid
-          raise WorkerStateError, "could not read lockfile"
-        end
-
-        unless pid == Process.pid.to_s
-          raise WorkerStateError, "lockfile points to different process"
-        end
-
-        unless sockfile?
-          raise WorkerStateError, "sockfile gone"
-        end
-      end
-    end
-  end
-end

data/lib/skylight/worker/standalone.rb

@@ -1,421 +0,0 @@
-require 'socket'
-require 'thread'
-require 'fileutils'
-require 'rbconfig'
-
-# TODO: Handle cool-off
-module Skylight
-  module Worker
-    # Handle to the agent subprocess. Manages creation, communication, and
-    # shutdown. Lazily spawns a thread that handles writing messages to the
-    # unix domain socket
-    #
-    class Standalone
-      include Util::Logging
-
-      # Locates skylight_native so that it can be included in the standalone agent startup command
-      def self.locate_skylight_native
-        $LOADED_FEATURES.find do |feature|
-          return feature if feature =~ /skylight_native\.#{RbConfig::CONFIG['DLEXT']}/
-        end
-      end
-
-      def self.build_subprocess_cmd
-        # Native extension location
-        native_path = locate_skylight_native
-        native_dir = native_path ? File.dirname(native_path) : nil
-
-        paths = [
-          File.expand_path('../../..', __FILE__), # Ruby code root
-          native_dir
-        ].uniq.compact
-
-        ret = [ RUBYBIN ]
-        paths.each { |path| ret << "-I" << path }
-        ret << File.expand_path('../../../skylight.rb', __FILE__) # The agent startup script
-        ret
-      end
-
-      # Used to start the standalone agent as well as included in the hello message
-      SUBPROCESS_CMD = build_subprocess_cmd
-
-      # Used to handle starting the thread
-      LOCK = Mutex.new
-
-      attr_reader \
-        :pid,
-        :config,
-        :lockfile,
-        :keepalive,
-        :max_spawns,
-        :spawn_window,
-        :sockfile_path
-
-      def initialize(config, lockfile, server)
-        @pid  = nil
-        @sock = nil
-
-        unless config && lockfile && server
-          raise ArgumentError, "all arguments are required"
-        end
-
-        @me = Process.pid
-        @config = config
-        @spawns = []
-        @server = server
-        @lockfile = lockfile
-        @keepalive = config[:'agent.keepalive']
-        @sockfile_path = config[:'agent.sockfile_path']
-
-        # Should be configurable
-        @max_spawns = 3
-        @spawn_window = 5 * 60
-
-        # Writer background processor will accept messages and write them to
-        # the IPC socket
-        @writer = build_queue
-      end
-
-      def spawn(*args)
-        return if @pid
-
-        if __spawn(*args)
-          @writer.spawn
-          true
-        end
-      end
-
-      def submit(msg)
-        unless msg.respond_to?(:encode) || msg.respond_to?(:native_serialize)
-          raise ArgumentError, "message not encodable"
-        end
-
-        unless @pid
-          t { "no pid, can't submit: #{msg.inspect}" }
-          return
-        end
-
-        if @me != Process.pid
-          handle_fork
-        end
-
-        @writer.submit(msg, @me)
-      end
-
-      # Shutdown any side task threads. Let the agent process die on it's own.
-      def shutdown
-        # TODO: implement
-        @writer.submit(:SHUTDOWN)
-        @writer.shutdown
-      end
-
-      # Shutdown any side task threads as well as the agent process
-      def shutdown_all
-        # TODO: implement
-        shutdown
-      end
-
-    private
-
-      def __spawn(timeout = 10)
-        if timeout < 2
-          raise ArgumentError, "at least 2 seconds required"
-        end
-
-        start = Time.now
-
-        if @spawns.length >= @max_spawns
-          if @spawn_window >= (start - @spawns.first)
-            trace "too many spawns in window"
-            return false
-          end
-
-          @spawns.unshift
-        end
-
-        @spawns << start
-
-        check_permissions
-
-        lockf = File.open lockfile, File::RDWR | File::CREAT
-
-        spawn_worker(lockf)
-
-        while timeout >= (Time.now - start)
-          if pid = read_lockfile
-            if sockfile?(pid)
-              if sock = connect(pid)
-                trace "connected to unix socket; pid=%s", pid
-                write_msg(sock, build_hello)
-                @sock = sock
-                @pid  = pid
-                return true
-              end
-            end
-          end
-
-          sleep 0.1
-        end
-
-        trace "failed to spawn worker"
-        return false
-
-      ensure
-        lockf.close rescue nil if lockf
-      end
-
-      def repair
-        @sock.close rescue nil if @sock
-
-        t { "repairing socket" }
-
-        # Attempt to reconnect to the currently known agent PID. If the agent
-        # is still healthy but is simply reloading itself, this should work
-        # just fine.
-        if sock = connect(@pid)
-          t { "reconnected to worker" }
-          @sock = sock
-          # TODO: Should HELLO be sent again?
-          return true
-        end
-
-        debug "failed to reconnect -- attempting worker respawn"
-
-        # Attempt to respawn the agent process
-        unless __spawn
-          debug "could not respawn -- shutting down"
-
-          @pid  = nil
-          @sock = nil
-          return false
-        end
-
-        true
-      end
-
-      def writer_tick(msg)
-        if :SHUTDOWN == msg
-          trace "shuting down agent connection"
-          @sock.close if @sock
-          @pid = nil
-
-          return false
-        elsif msg
-          return handle(msg)
-        else
-          begin
-            @sock.read_nonblock(1)
-          rescue Errno::EWOULDBLOCK, Errno::EAGAIN, Errno::EINTR
-          rescue Exception => e
-            trace "bad socket: #{e}"
-            unless repair
-              raise WorkerStateError, "could not repair connection to agent"
-            end
-          end
-
-          return true
-        end
-      rescue WorkerStateError => e
-        error "skylight shutting down: %s", e.message
-        return false
-      end
-
-      def handle(msg)
-        2.times do
-          unless sock = @sock
-            return false unless repair
-            sock = @sock
-          end
-
-          if write_msg(sock, msg)
-            return true
-          end
-
-          @sock = nil
-          sock.close rescue nil
-
-          unless repair
-            return false
-          end
-        end
-
-        debug "could not handle message; msg=%s", msg.class
-
-        false
-      end
-
-      def write_msg(sock, msg)
-        t { "writing a #{msg.class} on the wire" }
-        id = Messages::KLASS_TO_ID.fetch(msg.class)
-        buf = msg.serialize
-
-        frame = [ id, buf.bytesize ].pack("LL")
-
-        write(sock, frame) && write(sock, buf)
-      end
-
-      SOCK_TIMEOUT_VAL = [ 0, 0.01 * 1_000_000 ].pack("l_2")
-
-      # TODO: Handle configuring the socket with proper timeouts
-      def connect(pid)
-        sock = UNIXSocket.new(sockfile(pid)) rescue nil
-        if sock
-          sock.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, SOCK_TIMEOUT_VAL
-          sock
-        end
-      end
-
-      def write(sock, msg, timeout = 5)
-        msg = msg.to_s
-        cnt = 10
-
-        begin
-          while true
-            res = sock.write_nonblock(msg)
-
-            if res == msg.bytesize
-              return true
-            elsif res > 0
-              msg = msg.byteslice(res..-1)
-              cnt = 10
-            else
-              if 0 <= (cnt -= 1)
-                t { "write failed -- max attempts" }
-                return false
-              end
-            end
-          end
-        rescue Errno::EAGAIN, Errno::EWOULDBLOCK
-          _, socks, = IO.select([], [sock], [], timeout)
-          unless socks == [sock]
-            t { "write timed out" }
-            return false
-          end
-          retry
-        rescue Errno::EINTR
-          raise
-        rescue SystemCallError => e
-          t { fmt "write failed; err=%s", e.class }
-          return false
-        end
-      end
-
-      # Spawn the worker process.
-      def spawn_worker(f)
-        pid = fork do
-          # Note: By default, Ruby will finalize C objects inside the fork. Because those C objects
-          # are shared with the parent, this can cause database connections to disconnect in the
-          # parent process. We need to double-fork for proper semantics, so we disable the GC and
-          # exit! to avoid finalizing shared handles.
-          #
-          # We should continue to look for alternate solutions, and to determine whether there is
-          # still a possible race between the fork and the GC disabling.
-          ::GC.disable
-          ::Process.setsid
-          exit! if fork
-
-          # Acquire exclusive file lock, exit otherwise
-          unless f.flock(File::LOCK_EX | File::LOCK_NB)
-            exit! 1
-          end
-
-          f.truncate(0)
-
-          # Lock acquired, cleanup old sock files
-          Dir["#{sockfile_path}/skylight-*.sock"].each do |sf|
-            File.unlink(sf) rescue nil
-          end
-
-          pid = Process.pid.to_s
-
-          # Write the pid
-          f.write(pid)
-          f.flush
-
-          sf = sockfile(pid)
-          File.unlink(sf) rescue nil
-
-          t { fmt "opening a new socket; %s", sf }
-          srv = UNIXServer.new(sf)
-
-          unless ENV[TRACE_ENV_KEY]
-            null = File.open "/dev/null", File::RDWR
-            STDIN.reopen null
-            STDOUT.reopen null
-            STDERR.reopen null
-          end
-
-          # Cleanup the ENV
-          ENV['RUBYOPT'] = nil
-
-          @server.exec(SUBPROCESS_CMD, @config, f, srv, lockfile)
-        end
-
-        Process.detach(pid)
-      end
-
-      # If the process was forked, create a new queue and restart the worker
-      def handle_fork
-        LOCK.synchronize do
-          if @me != Process.pid
-            trace "process forked; recovering"
-            # Update the current process ID
-            @me = Process.pid
-
-            # Deal w/ the inherited socket
-            @sock.close rescue nil if @sock
-            @sock = nil
-
-            @writer = build_queue
-            @writer.spawn
-          end
-        end
-      end
-
-      def check_permissions
-        lockfile_root = File.dirname(lockfile)
-
-        FileUtils.mkdir_p lockfile_root
-        FileUtils.mkdir_p sockfile_path
-
-        if File.exist?(lockfile)
-          if !FileTest.writable?(lockfile)
-            raise WorkerStateError, "`#{lockfile}` not writable. Please set agent.lockfile or agent.sockfile_path in your config to a writable path."
-          end
-        else
-          if !FileTest.writable?(lockfile_root)
-            raise WorkerStateError, "`#{lockfile_root}` not writable. Please set agent.lockfile or agent.sockfile_path in your config to a writable path."
-          end
-        end
-
-        unless FileTest.writable?(sockfile_path)
-          raise WorkerStateError, "`#{sockfile_path}` not writable. Please set agent.sockfile_path in your config to a writable path."
-        end
-      end
-
-      def build_hello
-        Messages::Hello.build(VERSION, SUBPROCESS_CMD)
-      end
-
-      def build_queue
-        Util::Task.new(100, 1) { |m| writer_tick(m) }
-      end
-
-      def read_lockfile
-        pid = File.read(lockfile) rescue nil
-        if pid =~ /^\d+$/
-          pid.to_i
-        end
-      end
-
-      def sockfile(pid)
-        "#{sockfile_path}/skylight-#{pid}.sock"
-      end
-
-      def sockfile?(pid)
-        File.exist?(sockfile(pid))
-      end
-
-    end
-  end
-end