spider-gazelle 1.2.0 → 2.0.0

data/spider-gazelle.gemspec

@@ -1,8 +1,8 @@
 # -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
 
-require 'spider-gazelle/const'
-version = SpiderGazelle::Const::VERSION
+require 'spider-gazelle'
+version = SpiderGazelle::VERSION
 
 Gem::Specification.new do |s|
     s.name        = "spider-gazelle"
@@ -20,11 +20,11 @@ Gem::Specification.new do |s|
 
     s.add_dependency 'rake'
     s.add_dependency 'http-parser'          # Ruby FFI bindings for https://github.com/joyent/http-parser
-    s.add_dependency 'libuv', '>= 2.0.0'    # Ruby FFI bindings for https://github.com/libuv/libuv
+    s.add_dependency 'libuv',  '>= 2.0.5'   # Ruby FFI bindings for https://github.com/libuv/libuv
+    s.add_dependency 'uv-rays','>= 1.2.0'   # Provides buffering tools
     s.add_dependency 'rack', '>= 1.0.0'     # Ruby web server interface
     s.add_dependency 'websocket-driver'     # Websocket parser
-    s.add_dependency 'thread_safe'          # Thread safe hashes
-    s.add_dependency 'radix'                # Converts numbers to the unicode representation
+    s.add_dependency 'http-2'               # HTTP2 parsing and response management
 
     s.add_development_dependency 'rspec'    # Testing framework
     s.add_development_dependency 'yard'     # Comment based documentation generation
@@ -34,7 +34,7 @@ Gem::Specification.new do |s|
     s.extra_rdoc_files = ["README.md"]
 
     s.bindir = 'bin'
-    s.executables = ['sg']
+    s.executables = [SpiderGazelle::EXEC_NAME]
 
     s.require_paths = ["lib"]
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spider-gazelle
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Stephen von Takach
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-22 00:00:00.000000000 Z
+date: 2015-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -44,44 +44,44 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.5
 - !ruby/object:Gem::Dependency
-  name: rack
+  name: uv-rays
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
-  name: websocket-driver
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
-  name: thread_safe
+  name: websocket-driver
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,7 +95,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: radix
+  name: http-2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -155,14 +155,19 @@ files:
 - lib/rack/handler/spider-gazelle.rb
 - lib/rack/lock_patch.rb
 - lib/spider-gazelle.rb
-- lib/spider-gazelle/app_store.rb
-- lib/spider-gazelle/binding.rb
-- lib/spider-gazelle/connection.rb
-- lib/spider-gazelle/const.rb
 - lib/spider-gazelle/gazelle.rb
-- lib/spider-gazelle/request.rb
+- lib/spider-gazelle/gazelle/app_store.rb
+- lib/spider-gazelle/gazelle/http1.rb
+- lib/spider-gazelle/gazelle/request.rb
+- lib/spider-gazelle/logger.rb
+- lib/spider-gazelle/options.rb
+- lib/spider-gazelle/reactor.rb
+- lib/spider-gazelle/signaller.rb
+- lib/spider-gazelle/signaller/signal_parser.rb
 - lib/spider-gazelle/spider.rb
+- lib/spider-gazelle/spider/binding.rb
 - lib/spider-gazelle/upgrades/websocket.rb
+- spec/http1_spec.rb
 - spec/rack_lock_spec.rb
 - spider-gazelle.gemspec
 homepage: https://github.com/cotag/spider-gazelle
@@ -190,4 +195,6 @@ signing_key:
 specification_version: 4
 summary: A fast, parallel and concurrent web server for ruby
 test_files:
+- spec/http1_spec.rb
 - spec/rack_lock_spec.rb
+has_rdoc: 

data/lib/spider-gazelle/app_store.rb

@@ -1,64 +0,0 @@
-require 'thread'
-require 'radix/base'
-
-module SpiderGazelle
-  module AppStore
-    # Basic compression using UTF (more efficient for ID's stored as strings)
-    B65 = ::Radix::Base.new(::Radix::BASE::B62 + ['-', '_', '~'])
-    B10 = ::Radix::Base.new(10)
-
-    @mutex = Mutex.new
-    @apps = ThreadSafe::Cache.new
-    @loaded = ThreadSafe::Cache.new
-    @count = 0
-
-    # Load an app and assign it an ID
-    def self.load(app, options={})
-      is_rack_app = !app.is_a?(String)
-      app_key = is_rack_app ? app.class.name.to_sym : app.to_sym
-      id = @loaded[app_key]
-
-      if id.nil?
-        app, options = ::Rack::Builder.parse_file(app) unless is_rack_app
-
-        count = 0
-        @mutex.synchronize { count = @count += 1 }
-        id = Radix.convert(count, B10, B65).to_sym
-        @apps[id] = app
-        @loaded[app_key] = id
-      end
-
-      id
-    end
-
-    # Manually load an app
-    def self.add(app)
-      id = @loaded[app.__id__]
-
-      if id.nil?
-        count = 0
-        @mutex.synchronize { count = @count += 1 }
-        id = Radix.convert(count, B10, B65).to_sym
-        @apps[id] = app
-        @loaded[app.__id__] = id
-      end
-
-      id
-    end
-
-    # Lookup an application
-    def self.lookup(app)
-      if app.is_a?(String) || app.is_a?(Symbol)
-        @apps[@loaded[app.to_sym]]
-      else
-        @apps[@loaded[app.__id__]]
-      end
-    end
-
-    # Get an app using the id directly
-    def self.get(id)
-      id = id.to_sym if id.is_a?(String)
-      @apps[id]
-    end
-  end
-end

data/lib/spider-gazelle/binding.rb

@@ -1,53 +0,0 @@
-require 'spider-gazelle/const'
-require 'set'
-require 'thread'
-
-module SpiderGazelle
-  class Binding
-    include Const
-
-    attr_reader :app_id
-
-    def initialize(loop, delegate, app_id, options = {})
-      @app_id = app_id
-      @options = options
-      @loop = loop
-      @delegate = delegate
-      @tls = @options[:tls] || false
-      @port = @options[:Port] || (@tls ? PORT_443 : PORT_80)
-      @optimize = @options[:optimize_for_latency] || true
-
-      # Connection management functions
-      @accept_connection = method :accept_connection
-    end
-
-    # Bind the application to the selected port
-    def bind
-      # Bind the socket
-      @tcp = @loop.tcp
-      @tcp.bind @options[:Host], @port, @accept_connection
-      @tcp.listen @options[:backlog]
-
-      # Delegate errors
-      @tcp.catch { |e| @loop.log(:error, 'application bind failed', e) }
-      @tcp
-    end
-
-    # Close the bindings
-    def unbind
-      # close unless we've never been bound
-      @tcp.close unless @tcp.nil?
-      @tcp
-    end
-
-    protected
-
-    # Once the connection is accepted we disable Nagles Algorithm
-    # This improves performance as we are using vectored or scatter/gather IO
-    # Then the spider delegates to the gazelle loops
-    def accept_connection(client)
-      client.enable_nodelay if @optimize == true
-      @delegate.call client, @tls, @port, @app_id
-    end
-  end
-end

data/lib/spider-gazelle/connection.rb

@@ -1,371 +0,0 @@
-require 'spider-gazelle/const'
-require 'digest/md5'
-require 'stringio'
-
-module SpiderGazelle
-  class Connection
-    include Const
-
-    Hijack = Struct.new :socket, :env
-
-    def self.on_progress(data, socket); end
-    DUMMY_PROGRESS = self.method :on_progress
-
-    # For Gazelle
-    attr_reader :state, :parsing
-    # For Request
-    attr_reader :tls, :port, :loop, :socket, :async_callback
-
-    def initialize(gazelle, loop, socket, port, state, app, queue)
-      # A single parser instance per-connection (supports pipelining)
-      @state = state
-      @pending = []
-
-      # Work callback for thread pool processing
-      @request = nil
-      @work = method :work
-
-      # Called after the work on the thread pool is complete
-      @send_response = method :send_response
-      @send_error = method :send_error
-
-      # Used to chain promises (ensures requests are processed in order)
-      @process_next = method :process_next
-      # Keep track of work queue head to prevent unintentional GC
-      @current_worker = queue
-      # Start queue with an existing resolved promise (::Libuv::Q::ResolvedPromise.new(@loop, true))
-      @queue_worker = queue
-
-      # Socket for writing the response
-      @socket = socket
-      @app = app
-      @port = port
-      @tls = @socket.tls?
-      @loop = loop
-      @gazelle = gazelle
-      @async_callback = method :deferred_callback
-
-      # Remove connection if the socket closes
-      socket.finally &method(:unlink)
-    end
-
-    # Lazy eval the IP
-    def remote_ip
-      @remote_ip ||= @socket.peername[0]
-    end
-
-    # Creates a new request state object
-    def start_parsing
-      @parsing = Request.new self, @app
-    end
-
-    # Chains the work in a promise queue
-    def finished_parsing
-      if !@state.keep_alive?
-        @parsing.keep_alive = false
-        # We don't want to do any more work than we need to
-        @socket.stop_read
-      end
-
-      @parsing.upgrade = @state.upgrade?
-      @pending.push @parsing
-      @queue_worker = @queue_worker.then @process_next
-    end
-
-    # The parser encountered an error
-    def parsing_error
-      # Grab the error
-      send_error @state.error
-
-      # We no longer care for any further requests from this client
-      # however we will finish processing any valid pipelined requests before shutting down
-      @socket.stop_read
-      @queue_worker = @queue_worker.then do
-        @socket.write ERROR_400_RESPONSE
-        @socket.shutdown
-      end
-    end
-
-    # Schedule send
-    def response(data)
-      @loop.schedule
-    end
-
-    protected
-
-    ##
-    # State handlers
-
-    # Called when an error occurs at any point while responding
-    def send_error(reason)
-      # Close the socket as this is fatal (file read error, gazelle error etc)
-      @socket.close
-
-      # Log the error in a worker thread
-      @loop.work do
-        msg = "connection error: #{reason.message}\n#{reason.backtrace.join("\n") if reason.backtrace}\n"
-        @gazelle.logger.error msg
-      end
-    end
-
-    # We use promise chaining to move the requests forward
-    # This provides an elegant way to handle persistent and pipelined connections
-    def process_next(result)
-      @request = @pending.shift
-      @current_worker = @loop.work @work
-      # Resolves the promise with a promise
-      @current_worker.then @send_response, @send_error
-    end
-
-    # Returns the response as the result of the work
-    # We support the unofficial rack async api (multi-call version for chunked responses)
-    def work
-      begin
-        @request.execute!
-      rescue => e
-        @gazelle.logger.error "framework error: #{e.message}\n#{e.backtrace.join("\n") if e.backtrace}\n"
-        @request.keep_alive = false
-        [500, {}, EMPTY_RESPONSE]
-      end
-    end
-
-    # Unlinks the connection from the rack app
-    # This occurs when requested and when the socket closes
-    def unlink
-      if @gazelle
-        # Unlink the progress callback (prevent funny business)
-        @socket.progress &DUMMY_PROGRESS
-        @gazelle.discard self
-        @gazelle = nil
-        @state = nil
-      end
-    end
-
-    ##
-    # Core response handlers
-
-    def send_response(result)
-      # As we have come back from another thread the socket may have closed
-      # This check is an optimisation, the call to write and shutdown would fail safely
-
-      if @request.hijacked
-        # Unlink the management of the socket
-        unlink
-
-        # Pass the hijack response to the captor using the promise. This forwards the socket and
-        # environment as well as moving continued execution onto the event loop.
-        @request.hijacked.resolve Hijack.new(@socket, @request.env)
-      elsif @socket.closed
-        unless result.nil? || @request.deferred
-          body = result[2]
-          body.close if body.respond_to?(:close)
-        end
-      else
-        if @request.deferred
-          # Wait for the response using this promise
-          promise = @request.deferred.promise
-
-          # Process any responses that might have made it here first
-          if @deferred_responses
-            @deferred_responses.each &method(:respond_with)
-            @deferred_responses = nil
-          end
-
-          return promise
-        elsif result
-          # clear any cached responses just in case
-          # could be set by error in the rack application
-          @deferred_responses = nil if @deferred_responses
-
-          status, headers, body = result
-
-          send_body = @request.env[REQUEST_METHOD] != HEAD
-
-          # If a file, stream the body in a non-blocking fashion
-          if body.respond_to? :to_path
-            file = @loop.file body.to_path, File::RDONLY
-
-            # Send the body in parallel without blocking the next request in dev
-            # Also if this is a head request we still want the body closed
-            body.close if body.respond_to?(:close)
-
-            file.progress do
-              statprom = file.stat
-              statprom.then do |stats|
-                headers[ETAG] = ::Digest::MD5.hexdigest "#{stats[:st_mtim][:tv_sec]}#{body.to_path}"
-
-                if headers[CONTENT_LENGTH2]
-                  type = :raw
-                else
-                  type = :http
-                  headers[TRANSFER_ENCODING] = CHUNKED
-                end
-
-                write_headers status, headers
-
-                if send_body
-                  # File is open and available for reading
-                  file.send_file(@socket, type).finally do
-                    file.close
-                    @socket.shutdown if @request.keep_alive == false
-                  end
-                else
-                  file.close
-                  @socket.shutdown if @request.keep_alive == false
-                end
-              end
-
-              # Ensure the file is closed if there is an error
-              statprom.catch do |reason|
-                file.close
-                @loop.work do
-                  msg = "connection error: #{reason.message}\n#{reason.backtrace.join("\n") if reason.backtrace}\n"
-                  @gazelle.logger.error msg
-                end
-
-                send_response [500, {}, EMPTY_RESPONSE]
-              end
-            end
-
-            return file
-          else
-            # Optimize the response
-            begin
-              if body.size < 2
-                headers[CONTENT_LENGTH2] = body.size == 1 ? body[0].bytesize : ZERO
-              end
-            rescue # just in case
-            end
-
-            if send_body
-              write_response status, headers, body
-            else
-              body.close if body.respond_to?(:close)
-              write_headers status, headers
-              @socket.shutdown if @request.keep_alive == false
-            end
-          end
-        end
-      end
-
-      # continue processing (don't wait for write to complete)
-      # if the write fails it will close the socket
-      nil
-    end
-
-    def write_response(status, headers, body)
-      if headers[CONTENT_LENGTH2]
-        headers[CONTENT_LENGTH2] = headers[CONTENT_LENGTH2].to_s
-        write_headers status, headers
-
-        # Stream the response (pass directly into @socket.write)
-        body.each &@socket.method(:write)
-
-        if @request.deferred
-          @request.deferred.resolve true
-          # Prevent data being sent after completed
-          @request.deferred = nil
-        end
-
-        @socket.shutdown if @request.keep_alive == false
-      else
-        headers[TRANSFER_ENCODING] = CHUNKED
-        write_headers status, headers
-
-        # Stream the response
-        @write_chunk ||= method :write_chunk
-        body.each &@write_chunk
-
-        if @request.deferred.nil?
-          @socket.write CLOSE_CHUNKED
-          @socket.shutdown if @request.keep_alive == false
-        else
-          @async_state = :chunked
-        end
-      end
-
-      body.close if body.respond_to?(:close)
-    end
-    
-    def add_header(header, key, value)
-      header << key
-      header << COLON_SPACE
-      header << value
-      header << LINE_END
-    end
-
-    def write_headers(status, headers)
-      headers[CONNECTION] = CLOSE if @request.keep_alive == false
-
-      header = "HTTP/1.1 #{status} #{fetch_code(status)}\r\n"
-      headers.each do |key, value|
-        next if key.start_with? RACK
-        value.to_s.split(NEWLINE).each {|val| add_header(header, key, val)}
-      end
-      header << LINE_END
-      @socket.write header
-    end
-
-    def write_chunk(part)
-      chunk = part.bytesize.to_s(HEX_SIZE_CHUNKED_RESPONSE) << LINE_END << part << LINE_END
-      @socket.write chunk
-    end
-
-    def fetch_code(status)
-      HTTP_STATUS_CODES.fetch(status, &HTTP_STATUS_DEFAULT)
-    end
-
-    ##
-    # Async response functions
-
-    # Callback from a response that was marked async
-    def deferred_callback(data)
-      @loop.next_tick { callback(data) }
-    end
-
-    # Process a response that was marked as async. Save the data if the request hasn't responded yet
-    def callback(data)
-      begin
-        if @request.deferred && @deferred_responses.nil?
-          respond_with data
-        else
-          @deferred_responses ||= []
-          @deferred_responses << data
-        end
-      rescue Exception => e
-        # This provides the same level of protection that the regular responses provide
-        send_error e
-      end
-    end
-
-    # Process the async request in the same way as Mizuno
-    # See: http://polycrystal.org/2012/04/15/asynchronous_responses_in_rack.html
-    def respond_with(data)
-      status, headers, body = data
-
-      if @async_state.nil?
-        # Respond with the headers here
-        write_response status, headers, body
-      elsif body.empty?
-        body.close if body.respond_to?(:close)
-
-        @socket.write CLOSE_CHUNK
-        @socket.shutdown if @request.keep_alive == false
-
-        # Complete the request here
-        deferred = @request.deferred
-        # Prevent data being sent after completed
-        @request.deferred = nil
-        @async_state = nil
-        deferred.resolve true
-      else
-        # Send the chunks provided
-        @write_chunk ||= method :write_chunk
-        body.each &@write_chunk
-        body.close if body.respond_to?(:close)
-      end
-
-      nil
-    end
-  end
-end