spider-gazelle 1.2.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca80be46b162d37bbcb510ba2522bbfd0ea6460d
-  data.tar.gz: fcec86de21cd802686919983912c130a34762ad4
+  metadata.gz: 883af96ffb1054dc17b7f623ca7a6a1b6afe48cf
+  data.tar.gz: 9dda42f38e4cb802d4c6a11632438c16c2868cde
 SHA512:
-  metadata.gz: a3be697d78d1e77e23017ba627b4fd33cc29ecb3bf45d51081daf1321c7956e5e95e521be8253ee2c0572b62af922c9c84a948f6746bc4e8c0bd3c755c68f164
-  data.tar.gz: 84045ad878fdced61b7c8e4739e099291c52cdcd53d1fc0916cbe802b5a7e76f1a5fc33b0db52ede7f4194864e1dca319887afe38c65368185cb6d0ae81d8edc
+  metadata.gz: 848ba774b73a281f074505d8a0c92e8742e78cc6608ed441e283497f26fdfda6b3f992547b2df4eae4960cb3649e93ca135ffdf0173e7b97e22c67f193e3f8ab
+  data.tar.gz: 170c737ed7502cbff903f1ef8d04f8050d0632471d50077384125f284608b5362fe3585e8c823faa30973d21f0fb55901154c579c3c861ebd724d32190f0b2d6

data/bin/sg

@@ -1,67 +1,4 @@
 #!/usr/bin/env ruby
 
-
 require 'spider-gazelle'
-require 'optparse'
-
-
-options = {
-    Host: "0.0.0.0",
-    Port: 3000,
-    environment: ENV['RACK_ENV'] || 'development',
-    rackup: "#{Dir.pwd}/config.ru",
-    Verbose: false
-}
-
-parser = OptionParser.new do |opts|
-    opts.on "-p", "--port PORT", Integer, "Define what port TCP port to bind to (default: 3000)" do |arg|
-        options[:Port] = arg
-    end
-
-    opts.on "-a", "--address HOST", "bind to HOST address (default: 0.0.0.0)" do |arg|
-        options[:Host] = arg
-    end
-
-    opts.on "-v", "--verbose", "loud output" do
-        options[:Verbose] = true
-    end
-
-    opts.on "-e", "--environment ENVIRONMENT", "The environment to run the Rack app on (default: development)" do |arg|
-        options[:environment] = arg
-    end
-
-    opts.on "-r", "--rackup FILE", "Load Rack config from this file (default: config.ru)" do |arg|
-        options[:rackup] = arg
-    end
-
-    opts.on "-l", "--log FILE", "Location of the servers log file (default: logs/server.log)" do |arg|
-        ENV['SG_LOG'] = arg
-    end
-
-    opts.on "-m", "--mode MODE", "Either thread, process or no_ipc (default: thread)" do |arg|
-        ENV['SG_MODE'] = arg
-    end
-end
-
-parser.banner = "sg <options> <rackup file>"
-parser.on_tail "-h", "--help", "Show help" do
-    puts parser
-    exit 1
-end
-
-parser.parse!(ARGV)
-
-if ARGV.last =~ /\.ru$/
-    options[:rackup] = ARGV.shift
-end
-
-unless File.exists?(options[:rackup])
-    abort "No rackup found at #{options[:rackup]}"
-end
-
-# Force process mode on Windows (sockets over pipes not working at the moment)
-ENV['SG_MODE'] = 'no_ipc' if ::FFI::Platform.windows?
-
-::SpiderGazelle::Spider.run options[:rackup], options
-
-puts "\nSpider-Gazelle leaps through the veldt"
+SpiderGazelle::LaunchControl.instance.exec(ARGV)

data/lib/rack/handler/spider-gazelle.rb

@@ -1,37 +1,28 @@
 require "rack/handler"
 require "spider-gazelle"
-require "spider-gazelle/const"
 
 module Rack
-  module Handler
-    module SpiderGazelle
-      DEFAULT_OPTIONS = {
-        :Host => "0.0.0.0",
-        :Port => 8080,
-        :Verbose => false
-      }
+    module Handler
+        module SpiderGazelle
+            def self.run(app, options = {})
+                
+                # Replace the rackup with app
+                options = ::SpiderGazelle::Options::DEFAULTS.merge(options)
+                options.delete(:rackup)
+                options[:app] = app
 
-      def self.run(app, options = {})
-        options = DEFAULT_OPTIONS.merge(options)
+                # Can't pass an object over a pipe
+                options[:isolate] = true
+                options[:mode] = :thread if options[:mode] == :process
 
-        if options[:Verbose]
-          app = Rack::CommonLogger.new(app, STDOUT)
-        end
+                # Ensure the environment is set
+                options[:environment] ||= ENV['RACK_ENV'] || 'development'
+                ENV['RACK_ENV'] = options[:environment]
 
-        if options[:environment]
-          ENV["RACK_ENV"] = options[:environment].to_s
+                ::SpiderGazelle::LaunchControl.instance.launch([options])
+            end
         end
 
-        ::SpiderGazelle::Spider.run app, options
-      end
-
-      def self.valid_options
-        { "Host=HOST"       => "Hostname to listen on (default: 0.0.0.0)",
-          "Port=PORT"       => "Port to listen on (default: 8080)",
-          "Quiet"           => "Don't report each request" }
-      end
+        register :"spider-gazelle", SpiderGazelle
     end
-
-    register :"spider-gazelle", SpiderGazelle
-  end
 end

data/lib/rack/lock_patch.rb

@@ -4,36 +4,36 @@ require 'rack/body_proxy'
 require 'rack/lock' # ensure this loads first
 
 module Rack
-  # Rack::Lock locks every request inside a mutex, so that every request
-  # will effectively be executed synchronously.
-  class Lock
-    # FLAG = 'rack.multithread'.freeze # defined in rack/lock
-    RACK_MULTITHREAD ||= FLAG
+    # Rack::Lock locks every request inside a mutex, so that every request
+    # will effectively be executed synchronously.
+    class Lock
+        # FLAG = 'rack.multithread'.freeze # defined in rack/lock
+        RACK_MULTITHREAD ||= FLAG
 
-    def initialize(app, mutex = Mutex.new)
-      @app, @mutex = app, mutex
-      @sig = ConditionVariable.new
-      @count = 0
-    end
+        def initialize(app, mutex = Mutex.new)
+            @app, @mutex = app, mutex
+            @sig = ConditionVariable.new
+            @count = 0
+        end
 
-    def call(env)
-      @mutex.lock
-      @count += 1
-      @sig.wait(@mutex) if @count > 1
-      response = @app.call(env.merge(RACK_MULTITHREAD => false))
-      returned = response << BodyProxy.new(response.pop) {
-        @mutex.synchronize { unlock }
-      }
-    ensure
-      unlock unless returned
-      @mutex.unlock
-    end
+        def call(env)
+            @mutex.lock
+            @count += 1
+            @sig.wait(@mutex) if @count > 1
+            response = @app.call(env.merge(RACK_MULTITHREAD => false))
+            returned = response << BodyProxy.new(response.pop) {
+                @mutex.synchronize { unlock }
+            }
+        ensure
+            unlock unless returned
+            @mutex.unlock
+        end
 
-    private
+        private
 
-    def unlock
-      @count -= 1
-      @sig.signal
+        def unlock
+            @count -= 1
+            @sig.signal
+        end
     end
-  end
 end

data/lib/spider-gazelle.rb

@@ -1,22 +1,171 @@
-require "http-parser"     # C based, fast, http parser
-require "libuv"           # Ruby Libuv FFI wrapper
-require "rack"            # Ruby webserver abstraction
-require "rack/lock_patch" # Serialize execution in development mode
+require 'thread'
+require 'singleton'
 
-require "spider-gazelle/request"        # Holds request information and handles request processing
-require "spider-gazelle/connection"     # Holds connection information and handles request pipelining
-require "spider-gazelle/gazelle"        # Processes data received from connections
 
-require "spider-gazelle/app_store"      # Holds references to the loaded rack applications
-require "spider-gazelle/binding"        # Holds a reference to a bound port and associated rack application
-require "spider-gazelle/spider"         # Accepts connections and offloads them to gazelles
+require 'spider-gazelle/options'
+require 'spider-gazelle/logger'
+require 'spider-gazelle/reactor'
+require 'spider-gazelle/signaller'
 
-# Reactor aware websocket implementation
-require "spider-gazelle/upgrades/websocket"
 
 module SpiderGazelle
-  # Delegate pipe used for passing sockets to the gazelles
-  DELEGATE_PIPE = ENV['SG_DELEGATE_PIPE'] || "/tmp/spider-gazelle.delegate"
-  # Signal pipe used to pass control signals
-  SIGNAL_PIPE = ENV['SG_SIGNAL_PIPE'] || "/tmp/spider-gazelle.signal"
+    VERSION = '2.0.0'.freeze
+    EXEC_NAME = 'sg'.freeze
+    INTERNAL_PIPE_BACKLOG = 128
+
+    # Signaller is used to communicate:
+    # * command line requests
+    # * Startup and shutdown requests 
+    # * Live updates (bindings passed by this pipe)
+    SIGNAL_SERVER = '/tmp/sg-signaller.pipe'.freeze
+
+    # Spider server is used to
+    # * Track gazelles
+    # * Signal shutdown as required
+    # * Pass sockets
+    SPIDER_SERVER = '/tmp/sg-spider.pipe.'.freeze
+
+    MODES = [:process, :thread, :no_ipc].freeze
+    APP_MODE = [:thread_pool, :fiber_pool, :libuv, :eventmachine, :celluloid]
+
+
+    class LaunchControl
+        include Singleton
+
+
+        attr_reader :password, :args
+
+
+        def exec(args)
+            options = SpiderGazelle::Options.sanitize(args)
+            @args = args
+            launch(options)
+        end
+
+        def launch(options)
+            # Enable verbose messages if requested
+            Logger.instance.verbose! if options[0][:verbose]
+
+            # Start the Libuv Event Loop
+            reactor = ::SpiderGazelle::Reactor.instance
+            reactor.run do
+
+                # Check if SG is already running
+                signaller = ::SpiderGazelle::Signaller.instance
+
+                if options[0][:isolate]
+                    # This ensures this process will load the spider code
+                    options[0][:spider] = true
+                    boot(true, signaller, options)
+                else
+                    signaller.check.then do |running|
+                        boot(running, signaller, options)
+                    end
+                end
+            end
+        end
+
+        def shutdown
+            reactor = Reactor.instance
+            reactor.thread.schedule do
+                reactor.shutdown
+            end
+        end
+
+
+        # ---------------------------------------
+        # SPIDER LAUNCH CONTROL
+        # ---------------------------------------
+        def launch_spider(args)
+            require 'securerandom'
+            require 'thread'
+
+            @password ||= SecureRandom.hex
+            #cmd = "#{EXEC_NAME} -s #{@password} #{Shellwords.join(args)}"
+            cmd = [EXEC_NAME, '-s', @password] + args
+
+            Thread.new do
+                result = system(*cmd)
+
+                # TODO:: We need to detect a failed load
+                # This is a little more tricky as spiders
+                # may come and go without this process exiting
+            end
+        end
+
+        # This is called when a spider process starts
+        def start_spider(signaller, logger, options)
+            logger.set_client signaller.pipe unless options[0][:isolate]
+
+            require 'spider-gazelle/spider'
+            Spider.instance.run!(options)
+        end
+
+
+
+        # ---------------------------------------
+        # GAZELLE LAUNCH CONTROL
+        # ---------------------------------------
+        def start_gazelle(signaller, logger, options)
+            logger.set_client signaller.pipe
+
+            require 'spider-gazelle/gazelle'
+            ::SpiderGazelle::Gazelle.new(logger.thread, :process).run!(options)
+        end
+
+
+        # ---------------------------------------
+        # TTY SIGNALLING CONTROL
+        # ---------------------------------------
+        def signal_master(reactor, signaller, logger, options)
+            # This is a signal request
+            promise = signaller.request(options)
+
+            promise.then do |result|
+                logger.info "signal recieved #{result}"
+            end
+            promise.catch do |error|
+                logger.info "there was an error #{error}"
+            end
+            promise.finally do
+                reactor.shutdown
+            end
+        end
+
+
+        protected
+
+
+        def boot(running, signaller, options)
+            logger = ::SpiderGazelle::Logger.instance
+
+            begin
+                # What do we want to do?
+                master = options[0]
+
+                if running
+                    if master[:spider]
+                        logger.verbose "Starting Spider".freeze
+                        start_spider(signaller, logger, options)
+                    elsif master[:gazelle]
+                        logger.verbose "Starting Gazelle".freeze
+                        start_gazelle(signaller, logger, options)
+                    else
+                        logger.verbose "Sending signal to SG Master".freeze
+                        signal_master(reactor, signaller, logger, options)
+                    end
+
+                elsif master[:debug]
+                    logger.verbose "SG is now running in debug mode".freeze
+                else
+                    logger.verbose "SG was not running, launching Spider".freeze
+                    launch_spider(@args)
+                end
+            rescue => e
+                logger.verbose "Error performing requested operation".freeze
+                logger.print_error(e)
+                shutdown
+            end
+        end
+    end
 end

data/lib/spider-gazelle/gazelle.rb

@@ -1,162 +1,179 @@
-require 'spider-gazelle/const'
-require 'set'
+require "rack"            # Ruby webserver abstraction
+require "rack/lock_patch" # Serialize execution in development mode
+require 'spider-gazelle/gazelle/app_store'
+require 'spider-gazelle/gazelle/http1'
 
-module SpiderGazelle
-  class Gazelle
-    include Const
 
-    attr_reader :parser_cache, :connections, :logger
+# Reactor aware websocket implementation
+require "spider-gazelle/upgrades/websocket"
 
-    def set_instance_type(inst)
-      inst.type = :request
-    end
 
-    def initialize(loop, logger, mode)
-      @gazelle = loop
-      # Set of active connections on this thread
-      @connections = Set.new
-      # Stale parser objects cached for reuse
-      @parser_cache = []
+module SpiderGazelle
+    class Gazelle
+        SPACE = ' '.freeze
+
+        def initialize(thread, type)
+            raise ArgumentError, "type must be one of #{MODES}" unless MODES.include?(type)
+            
+            @type = type
+            @logger = Logger.instance
+            @thread = thread
+
+            @http1_cache = []
+            @http2_cache = []
+            @return_http1 = method(:return_http1)
+            @return_http2 = method(:return_http2)
+            @parser_count = 0
+
+            @on_progress   = method(:on_progress)
+            @set_protocol  = method(:set_protocol)
+
+            # Register the gazelle with the signaller so we can shutdown elegantly
+            if @type == :process
+                Signaller.instance.gazelle = self
+            end
+        end
 
-      @mode = mode
-      @logger = logger
-      @app_cache = {}
-      @connection_queue = ::Libuv::Q::ResolvedPromise.new @gazelle, true
+        def run!(options)
+            @options = options
+            @logger.verbose { "Gazelle: #{@type} Pid: #{Process.pid} started" }
 
-      # A single parser instance for processing requests for each gazelle
-      @parser = ::HttpParser::Parser.new self
-      @set_instance_type = method :set_instance_type
+            connect_to_spider unless @type == :no_ipc
 
-      # Single progress callback for each gazelle
-      @on_progress = method :on_progress
-    end
+            load_required_applications
+            self
+        end
 
-    def run
-      @gazelle.run do |logger|
-        logger.progress do |level, errorid, error|
-          begin
-            msg = "Gazelle log: #{level}: #{errorid}\n#{error.message}\n#{error.backtrace.join("\n") if error.backtrace}\n"
-            @logger.error msg
-          rescue Exception
-            puts 'error in gazelle logger'
-          end
+        def new_app(options)
+            # TODO:: load this app into all of the gazelles dynamically
         end
 
-        unless @mode == :no_ipc
-          # A pipe used to forward connections to different threads
-          @socket_server = @gazelle.pipe true
-          @socket_server.connect(DELEGATE_PIPE) do
-            @socket_server.progress &method(:new_connection)
-            @socket_server.start_read
-          end
-
-          # A pipe used to signal various control commands (shutdown, etc)
-          @signal_server = @gazelle.pipe
-          @signal_server.connect(SIGNAL_PIPE) do
-            @signal_server.progress &method(:process_signal)
-            @signal_server.start_read
-          end
+        def new_connection(data, binding)
+            socket = @pipe.check_pending
+            return if socket.nil?
+            process_connection(socket, data.to_i)
         end
-      end
-    end
 
-    # HTTP Parser callbacks:
-    def on_message_begin(parser)
-      @connection.start_parsing
-    end
+        def shutdown(finished = nil)
+            # Wait for the requests to finish (give them 15 seconds)
+            # TODO::
+
+            @logger.verbose { "Gazelle: #{@type} Pid: #{Process.pid} shutting down" }
+
+            # Then stop the current thread if we are in threaded mode
+            if @type == :thread
+                # In threaded mode the gazelle has the power
+                @thread.stop
+            else
+                # Both no_ipc and process need to know when the requests
+                # have completed to shutdown
+                finished.resolve(true)
+            end
+        end
 
-    def on_url(parser, url)
-      @connection.parsing.url << url
-    end
 
-    def on_header_field(parser, header)
-      req = @connection.parsing
-      req.header.frozen? ? req.header = header : req.header << header
-    end
+        protected
+
+
+        def connect_to_spider
+            @pipe = @thread.pipe :ipc
+            @pipe.connect(@options[0][:gazelle_ipc]) do |client|
+                client.progress method(:new_connection)
+                client.start_read
+
+                authenticate
+            end
 
-    def on_header_value(parser, value)
-      req = @connection.parsing
-      if req.header.frozen?
-        req.env[req.header] << value
-      else
-        header = req.header
-        header.upcase!
-        header.gsub!(DASH, UNDERSCORE)
-        header.prepend(HTTP_META)
-        header.freeze
-        if req.env[header]
-          req.env[header] << COMMA
-          req.env[header] << value
-        else
-          req.env[header] = value
+            @pipe.catch do |error|
+                @logger.print_error(error)
+            end
+
+            @pipe.finally do
+                if @type == :process
+                    Reactor.instance.shutdown
+                else
+                    # Threaded mode
+                    shutdown
+                end
+            end
         end
-      end
-    end
 
-    def on_headers_complete(parser)
-      @connection.parsing.env[REQUEST_METHOD] = @connection.state.http_method.to_s
-    end
+        def authenticate
+            @pipe.write "#{@options[0][:gazelle]} #{@type}"
+        end
 
-    def on_body(parser, data)
-      @connection.parsing.body << data
-    end
+        def load_required_applications
+            @options.each do |app|
+                if app[:rackup]
+                    AppStore.load(app[:rackup], app)
+                elsif app[:app]
+                    AppStore.add(app[:app], app)
+                end
+            end
+        end
 
-    def on_message_complete(parser)
-      @connection.finished_parsing
-    end
 
-    def discard(connection)
-      @connections.delete(connection)
-      state = connection.state
-      state.reset!
-      @parser_cache << state
-    end
+        # ---------------------
+        # Connection Management
+        # ---------------------
+        def process_connection(socket, app_id)
+            # Put application details in the socket storage as we negotiate protocols
+            details = AppStore.get(app_id)
+            socket.storage = details
+            tls = details[-1]
+
+            # Hook up the socket and kick off TLS if required
+            if tls
+                socket.on_handshake @set_protocol
+                socket.start_tls(tls)
+            else
+                set_protocol(socket, :http1)
+            end
+
+            socket.start_read
+            socket.enable_nodelay
+        end
 
-    protected
+        def on_progress(data, socket)
+            # Storage contains the parser for this connection
+            parser = socket.storage
+            parser.parse(data)
+        end
 
-    def on_progress(data, socket)
-      # Keep track of which connection we are processing for the callbacks
-      @connection = socket.storage
+        def set_protocol(socket, version)
+            app, app_mode, port, tls = socket.storage
 
-      # Check for errors during the parsing of the request
-      @connection.parsing_error if @parser.parse(@connection.state, data)
-    end
+            parser = if version == :h2
+                @http2_cache.pop || new_http2_parser
+            else
+                @http1_cache.pop || new_http1_parser
+            end
 
-    def new_connection(data, socket)
-      # When in no IPC mode @socket_server is nil and the socket parameter is the socket
-      # If socket server exists then we are expecting sockets over the pipe
-      if @socket_server
-        socket = @socket_server.check_pending
-        return if socket.nil?
-      end
-
-      # Data == "TLS_indicator Port APP_ID"
-      tls, port, app_id = data.split(SPACE, 3)
-      app = @app_cache[app_id.to_sym] ||= AppStore.get(app_id)
-      inst = @parser_cache.pop || ::HttpParser::Parser.new_instance(&@set_instance_type)
-
-      # process any data coming from the socket
-      socket.progress @on_progress
-      # TODO:: Allow some globals for supplying the certs
-      #  --> We could store these in the AppStore
-      socket.start_tls(:server => true) if tls == USE_TLS
-
-      # Keep track of the connection
-      connection = Connection.new self, @gazelle, socket, port, inst, app, @connection_queue
-      @connections.add connection
-      # This allows us to re-use the one proc for parsing
-      socket.storage = connection
-
-      socket.start_read
-    end
+            parser.load(socket, port, app, app_mode, tls)
+            socket.progress @on_progress
+            socket.storage = parser
+        end
 
-    def process_signal(data, pipe)
-      shutdown if data == KILL_GAZELLE
-    end
 
-    def shutdown
-      # TODO:: do this nicely. Need to signal the connections to close
-      @gazelle.stop
+        def new_http1_parser
+            @h1_parser_obj ||= Http1::Callbacks.new
+
+            @parser_count += 1
+            Http1.new(@return_http1, @h1_parser_obj, @thread, @logger)
+        end
+
+        def return_http1(parser)
+            @http1_cache.push parser
+        end
+
+        def new_http2_parser
+            raise NotImplementedError.new 'TODO:: Create HTTP2 parser class'
+            @parser_count += 1
+            Http2.new(@return_http2)
+        end
+
+        def return_http2(parser)
+            @http2_cache << parser
+        end
     end
-  end
 end