spider-gazelle 1.2.0 → 2.0.0

data/lib/spider-gazelle/gazelle/request.rb

@@ -0,0 +1,155 @@
+require 'stringio'
+require 'rack'            # Ruby webserver abstraction
+
+module SpiderGazelle
+    class Request < ::Libuv::Q::DeferredPromise
+        RACK_VERSION = 'rack.version'.freeze
+        RACK_ERRORS = 'rack.errors'.freeze
+        RACK_MULTITHREAD = "rack.multithread".freeze
+        RACK_MULTIPROCESS = "rack.multiprocess".freeze
+        RACK_RUN_ONCE = "rack.run_once".freeze
+        SCRIPT_NAME = "SCRIPT_NAME".freeze
+        EMPTY = ''.freeze
+        SERVER_PROTOCOL = "SERVER_PROTOCOL".freeze
+        HTTP_11 = "HTTP/1.1".freeze
+        SERVER_SOFTWARE = "SERVER_SOFTWARE".freeze
+        GATEWAY_INTERFACE = "GATEWAY_INTERFACE".freeze
+        CGI_VER = "CGI/1.2".freeze
+        SERVER = "SpiderGazelle".freeze
+        LOCALHOST = 'localhost'.freeze
+
+
+        # TODO:: Add HTTP headers to the env and capitalise them and prefix them with HTTP_
+        #   convert - signs to underscores
+        PROTO_ENV = {
+            RACK_VERSION => ::Rack::VERSION,   # Should be an array of integers
+            RACK_ERRORS => $stderr,            # An error stream that supports: puts, write and flush
+            RACK_MULTITHREAD => true,          # can the app be simultaneously invoked by another thread?
+            RACK_MULTIPROCESS => true,        # will the app be simultaneously be invoked in a separate process?
+            RACK_RUN_ONCE => false,            # this isn't CGI so will always be false
+
+            SCRIPT_NAME => ENV['SCRIPT_NAME'.freeze] || EMPTY,   #  The virtual path of the app base (empty if root)
+            SERVER_PROTOCOL => HTTP_11,
+
+            GATEWAY_INTERFACE => CGI_VER,
+            SERVER_SOFTWARE   => SERVER
+        }
+
+        attr_accessor :env, :url, :header, :body, :keep_alive, :upgrade
+        attr_reader :hijacked, :defer, :is_async
+
+
+        SERVER_PORT = "SERVER_PORT".freeze
+        REMOTE_ADDR = "REMOTE_ADDR".freeze
+        RACK_URL_SCHEME = "rack.url_scheme".freeze
+        ASYNC = "async.callback".freeze
+
+        def initialize(thread, app, port, remote_ip, scheme, async_callback)
+            super(thread, thread.defer)
+
+            @app = app
+            @body = ''
+            @header = ''
+            @url = ''
+            @env = PROTO_ENV.dup
+            @env[SERVER_PORT] = port
+            @env[REMOTE_ADDR] = remote_ip
+            @env[RACK_URL_SCHEME] = scheme
+            @env[ASYNC] = async_callback
+        end
+
+
+        CONTENT_LENGTH = "CONTENT_LENGTH".freeze
+        HTTP_CONTENT_LENGTH = "HTTP_CONTENT_LENGTH".freeze
+        CONTENT_TYPE = "CONTENT_TYPE".freeze
+        HTTP_CONTENT_TYPE = "HTTP_CONTENT_TYPE".freeze
+        DEFAULT_TYPE = "text/plain".freeze
+        REQUEST_URI= "REQUEST_URI".freeze
+        ASCII_8BIT = "ASCII-8BIT".freeze
+        RACK_INPUT = "rack.input".freeze
+        PATH_INFO = "PATH_INFO".freeze
+        REQUEST_PATH = "REQUEST_PATH".freeze
+        QUERY_STRING = "QUERY_STRING".freeze
+        HTTP_HOST = "HTTP_HOST".freeze
+        COLON = ":".freeze
+        SERVER_NAME = "SERVER_NAME".freeze
+        # Hijacking IO is supported
+        HIJACK_P = "rack.hijack?".freeze
+        # Callback for indicating that this socket will be hijacked
+        HIJACK = "rack.hijack".freeze
+        # The object for performing IO on after hijack is called
+        HIJACK_IO = "rack.hijack_io".freeze
+        QUESTION_MARK = "?".freeze
+        
+        HTTP_UPGRADE = 'HTTP_UPGRADE'.freeze
+        USE_HTTP2 = 'h2c'.freeze
+
+
+        def execute!
+            @env[CONTENT_LENGTH] = @env.delete(HTTP_CONTENT_LENGTH) || @body.length
+            @env[CONTENT_TYPE] = @env.delete(HTTP_CONTENT_TYPE) || DEFAULT_TYPE
+            @env[REQUEST_URI] = @url.freeze
+
+            # For Rack::Lint on 1.9, ensure that the encoding is always for spec
+            @body.force_encoding(ASCII_8BIT) if @body.respond_to?(:force_encoding)
+            @env[RACK_INPUT] = StringIO.new @body
+
+            # Break the request into its components
+            query_start  = @url.index QUESTION_MARK
+            if query_start
+                path = @url[0...query_start].freeze
+                @env[PATH_INFO] = path
+                @env[REQUEST_PATH] = path
+                @env[QUERY_STRING] = @url[query_start + 1..-1].freeze
+            else
+                @env[PATH_INFO] = @url
+                @env[REQUEST_PATH] = @url
+                @env[QUERY_STRING] = EMPTY
+            end
+
+            # Grab the host name from the request
+            if host = @env[HTTP_HOST]
+                if colon = host.index(COLON)
+                    @env[SERVER_NAME] = host[0, colon]
+                    @env[SERVER_PORT] = host[colon+1, host.bytesize]
+                else
+                    @env[SERVER_NAME] = host
+                end
+            else
+                @env[SERVER_NAME] = LOCALHOST
+            end
+
+            # Provide hijack options if this is an upgrade request
+            if @upgrade == true
+                if @env[HTTP_UPGRADE] == USE_HTTP2
+                    # TODO:: implement the upgrade process here
+                else
+                    @env[HIJACK_P] = true
+                    @env[HIJACK] = method :hijack
+                end
+            end
+
+            # Execute the request
+            # NOTE:: Catch was overloaded by Promise so this does the 
+            resp = ruby_catch(:async) { @app.call @env }
+            if resp.nil? || resp[0] == -1
+                @is_async = true
+
+                # close the body for deferred responses
+                unless resp.nil?
+                    body = resp[2]
+                    body.close if body.respond_to?(:close)
+                end
+            end
+            resp
+        end
+
+        protected
+
+        def hijack
+            @hijacked = @loop.defer
+            @env.delete HIJACK # don't want to hold a reference to this request object
+            @env[HIJACK_IO] = @hijacked.promise
+        end
+    end
+end

data/lib/spider-gazelle/logger.rb

@@ -0,0 +1,122 @@
+require 'libuv'
+
+module SpiderGazelle
+    class Logger
+        include Singleton
+        attr_reader :level, :thread, :pipe
+        attr_accessor :formatter
+
+
+        LEVEL = {
+            debug: 0,
+            info: 1,
+            warn: 2,
+            error: 3,
+            fatal: 4
+        }.freeze
+        DEFAULT_LEVEL = 1
+        LEVELS = LEVEL.keys.freeze
+
+
+        def initialize
+            @thread = ::Libuv::Loop.default
+            @level = DEFAULT_LEVEL
+            @write = method(:server_write)
+        end
+
+
+        def self.log(data)
+            Logger.instance.server_write(data)
+        end
+
+
+        def set_client(uv_io)
+            @pipe = uv_io
+            @write = method(:client_write)
+        end
+
+        def level=(level)
+            @level = LEVEL[level] || level
+        end
+
+        def verbose!(enabled = true)
+            @verbose = enabled
+        end
+
+        def debug(msg = nil)
+            if @level <= 0
+                msg = yield if block_given?
+                log(:debug, msg)
+            end
+        end
+
+        def info(msg = nil)
+            if @level <= 1
+                msg = yield if block_given?
+                log(:info, msg)
+            end
+        end
+
+        def warn(msg = nil)
+            if @level <= 2
+                msg = yield if block_given?
+                log(:warn, msg)
+            end
+        end
+
+        def error(msg = nil)
+            if @level <= 3
+                msg = yield if block_given?
+                log(:error, msg)
+            end
+        end
+
+        def fatal(msg = nil)
+            if @level <= 4
+                msg = yield if block_given?
+                log(:fatal, msg)
+            end
+        end
+
+        def verbose(msg = nil)
+            if @verbose
+                msg = yield if block_given?
+                @write.call ">> #{msg}\n"
+            end
+        end
+
+        def print_error(e, msg = '', trace = nil)
+            msg << ":\n" unless msg.empty?
+            msg << "#{e.message}\n"
+            backtrace = e.backtrace if e.respond_to?(:backtrace)
+            if backtrace
+                msg << "#{backtrace.join("\n")}\n"
+            elsif trace.nil?
+                trace = caller
+            end
+            msg << "Caller backtrace:\n#{trace.join("\n")}\n" if trace
+            error(msg)
+        end
+
+        # NOTE:: should only be called on reactor thread
+        def server_write(msg)
+            STDOUT.write msg
+        end
+
+
+        protected
+
+
+        def log(level, msg)
+            output = "[#{level}] #{msg}\n"
+            @thread.schedule do
+                @write.call output
+            end
+        end
+
+        # NOTE:: should only be called on reactor thread
+        def client_write(msg)
+            @pipe.write "\x02Logger log #{msg}\x03"
+        end
+    end
+end

data/lib/spider-gazelle/options.rb

@@ -0,0 +1,213 @@
+require 'set'
+require 'optparse'
+require 'spider-gazelle/logger'
+
+
+module SpiderGazelle
+    module Options
+        DEFAULTS = {
+            host: "0.0.0.0",
+            port: 3000,
+            verbose: false,
+            tls: false,
+            backlog: 5000,
+            rackup: "#{Dir.pwd}/config.ru",
+            mode: :thread,
+            app_mode: :thread_pool,
+            isolate: true
+        }.freeze
+
+
+        # Options that can't be used when more than one set of options is being applied
+        APP_OPTIONS = [:port, :host, :verbose, :debug, :environment, :rackup, :mode, :backlog, :count, :name, :loglevel].freeze
+        MUTUALLY_EXCLUSIVE = {
+
+            # Only :password is valid when this option is present
+            update: APP_OPTIONS
+
+        }.freeze
+
+
+        def self.parse(args)
+            options = {}
+
+            parser = OptionParser.new do |opts|
+                # ================
+                # STANDARD OPTIONS
+                # ================
+                opts.on "-p", "--port PORT", Integer, "Define what port TCP port to bind to (default: 3000)" do |arg|
+                    options[:port] = arg
+                end
+
+                opts.on "-h", "--host ADDRESS", "bind to address (default: 0.0.0.0)" do |arg|
+                    options[:host] = arg
+                end
+
+                opts.on "-v", "--verbose", "loud output" do
+                    options[:verbose] = true
+                end
+
+                opts.on "-d", "--debug", "debugging mode with lowered security and manual processes" do
+                    options[:debug] = true
+                end
+
+                opts.on "-e", "--environment ENVIRONMENT", "The environment to run the Rack app on (default: development)" do |arg|
+                    options[:environment] = arg
+                end
+
+                opts.on "-r", "--rackup FILE", "Load Rack config from this file (default: config.ru)" do |arg|
+                    options[:rackup] = arg
+                end
+
+                opts.on "-m", "--mode MODE", MODES, "Either process, thread or no_ipc (default: process)" do |arg|
+                    options[:mode] = arg
+                end
+
+                opts.on "-a", "--app-mode MODE", APP_MODE, "How should requests be processed (default: thread_pool)" do |arg|
+                    options[:host] = arg
+                end
+
+                opts.on "-b", "--backlog BACKLOG", Integer, "Number of pending connections allowed (default: 5000)" do |arg|
+                    options[:backlog] = arg
+                end
+
+
+                # =================
+                # TLS Configuration
+                # =================
+                opts.on "-t", "--use-tls PRIVATE_KEY_FILE", "Enables TLS on the port specified using the provided private key in PEM format" do |arg|
+                    options[:tls] = true
+                    options[:private_key] = arg
+                end
+
+                opts.on "-tc", "--tls-chain-file CERT_CHAIN_FILE", "The certificate chain to provide clients" do |arg|
+                    options[:cert_chain] = arg
+                end
+
+                opts.on "-ts", "--tls-ciphers CIPHER_LIST", "A list of Ciphers that the server will accept" do |arg|
+                    options[:ciphers] = arg
+                end
+
+                opts.on "-tv", "--tls-verify-peer", "Do we want to verify the client connections? (default: false)" do
+                    options[:verify_peer] = true
+                end
+
+
+                # ========================
+                # CHILD PROCESS INDICATORS
+                # ========================
+                opts.on "-g", "--gazelle PASSWORD", 'For internal use only' do |arg|
+                    options[:gazelle] = arg
+                end
+
+                opts.on "-f", "--file IPC", 'For internal use only' do |arg|
+                    options[:gazelle_ipc] = arg
+                end
+
+
+                opts.on "-s", "--spider PASSWORD", 'For internal use only' do |arg|
+                    options[:spider] = arg
+                end
+
+
+                opts.on "-i", "--interactive-mode", 'Loads a multi-process version of spider-gazelle that can live update your app' do
+                    options[:isolate] = false
+                end
+
+                opts.on "-c", "--count NUMBER", Integer, "Number of gazelle processes to launch (default: number of CPU cores)" do |arg|
+                    options[:count] = arg
+                end
+
+
+                # ==================
+                # SIGNALLING OPTIONS
+                # ==================
+                opts.on "-u", "--update", "Live migrates to a new process without dropping existing connections" do |arg|
+                    options[:update] = true
+                end
+
+                opts.on "-up", "--update-password PASSWORD", "Sets a password for performing updates" do |arg|
+                    options[:password] = arg
+                end
+
+                opts.on "-l", "--loglevel LEVEL", Logger::LEVELS, "Sets the log level" do |arg|
+                    options[:loglevel] = arg
+                end
+            end
+
+            parser.banner = "sg <options> <rackup file>"
+            parser.on_tail "-h", "--help", "Show help" do
+                puts parser
+                exit 1
+            end
+            parser.parse!(args)
+
+            # Check for rackup file
+            if args.last =~ /\.ru$/
+                options[:rackup] = args.last
+            end
+
+            # Unless this is a signal then we want to include the default options
+            unless options[:update]
+                options = DEFAULTS.merge(options)
+
+                unless File.exist? options[:rackup]
+                    abort "No rackup found at #{options[:rackup]}"
+                end
+
+                options[:environment] ||= ENV['RACK_ENV'] || 'development'
+                ENV['RACK_ENV'] = options[:environment]
+
+                # isolation and process mode don't mix
+                options[:isolate] = false if options[:mode] == :process
+            end
+
+            options
+        end
+
+        def self.sanitize(args)
+            # Use "\0" as this character won't be used in the command
+            cmdline = args.join("\0")
+            components = cmdline.split("\0--", -1)
+
+            # Ensure there is at least one component
+            # (This will occur when no options are provided)
+            components << '' if components.empty?
+
+            # Parse the commandline options
+            options = []
+            components.each do |app_opts|
+                options << parse(app_opts.split(/\0+/))
+            end
+
+            # Check for any invalid requests
+            exclusive = Set.new(MUTUALLY_EXCLUSIVE.keys)
+
+            if options.length > 1
+
+                # Some options can only be used by themselves
+                options.each do |opt|
+                    keys = Set.new(opt.keys)
+
+                    if exclusive.intersect? keys
+                        invalid = exclusive & keys
+
+                        abort "The requested actions can only be used in isolation: #{invalid.to_a}"
+                    end
+                end
+
+                # Ensure there are no conflicting ports
+                ports = [options[0][:port]]
+                options[1..-1].each do |opt|
+                    # If there is a clash we'll increment the port by 1
+                    while ports.include? opt[:port]
+                        opt[:port] += 1
+                    end
+                    ports << opt[:port]
+                end
+            end
+
+            options
+        end
+    end
+end