spider-gazelle 0.1.6 → 0.1.7

data/lib/spider-gazelle/const.rb

@@ -0,0 +1,198 @@
+# Thanks to Puma https://github.com/puma/puma/blob/master/lib/puma/const.rb
+require "rack"
+
+module SpiderGazelle
+  # UnsupportedOption = Class.new(RuntimeError)
+
+  # Every standard HTTP code mapped to the appropriate message.  These are
+  # used so frequently that they are placed directly in SpiderGazelle for easy
+  # access rather than SpiderGazelle::Const itself.
+  HTTP_STATUS_CODES = Rack::Utils::HTTP_STATUS_CODES
+
+  # # For some HTTP status codes the client only expects headers.
+  # STATUS_WITH_NO_ENTITY_BODY = Hash[Rack::Utils::STATUS_WITH_NO_ENTITY_BODY.map { |s|
+  #   [s, true]
+  # }]
+
+  # Based on http://rack.rubyforge.org/doc/SPEC.html
+  # Frequently used constants when constructing requests or responses.  Many times
+  # the constant just refers to a string with the same contents.  Using these constants
+  # gave about a 3% to 10% performance improvement over using the strings directly.
+  #
+  # The constants are frozen because Hash#[]= when called with a String key dups
+  # the String UNLESS the String is frozen. This saves us therefore 2 object
+  # allocations when creating the env hash later.
+  #
+  # While SpiderGazelle does try to emulate the CGI/1.2 protocol, it does not use the REMOTE_IDENT,
+  # REMOTE_USER, or REMOTE_HOST parameters since those are either a security problem or
+  # too taxing on performance.
+  module Const
+    SPIDER_GAZELLE_VERSION = VERSION = "0.1.7".freeze
+    # CODE_NAME = "Earl of Sandwich Partition"
+    SERVER = "SpiderGazelle".freeze
+
+    # FAST_TRACK_KA_TIMEOUT = 0.2
+
+    # # The default number of seconds for another request within a persistent
+    # # session.
+    # PERSISTENT_TIMEOUT = 20
+
+    # # The default number of seconds to wait until we get the first data
+    # # for the request
+    # FIRST_DATA_TIMEOUT = 30
+
+    # # How long to wait when getting some write blocking on the socket when
+    # # sending data back
+    # WRITE_TIMEOUT = 10
+
+    # DATE = "Date".freeze
+
+    SCRIPT_NAME = "SCRIPT_NAME".freeze
+
+    # The original URI requested by the client.
+    REQUEST_URI= "REQUEST_URI".freeze
+    REQUEST_PATH = "REQUEST_PATH".freeze
+
+    PATH_INFO = "PATH_INFO".freeze
+
+    # SPIDER_GAZELLE_TMP_BASE = "spider-gazelle".freeze
+
+    # # Indicate that we couldn"t parse the request
+    ERROR_400_RESPONSE = "HTTP/1.1 400 Bad Request\r\n\r\n"
+
+    # The standard empty 404 response for bad requests.  Use Error4040Handler for custom stuff.
+    ERROR_404_RESPONSE = "HTTP/1.1 404 Not Found\r\nConnection: close\r\nServer: #{SERVER} #{SPIDER_GAZELLE_VERSION}\r\n\r\nNOT FOUND".freeze
+
+    # The standard empty 408 response for requests that timed out.
+    ERROR_408_RESPONSE = "HTTP/1.1 408 Request Timeout\r\nConnection: close\r\nServer:  #{SERVER} #{SPIDER_GAZELLE_VERSION}\r\n\r\n".freeze
+
+    # Indicate that there was an internal error, obviously.
+    ERROR_500_RESPONSE = "HTTP/1.1 500 Internal Server Error\r\n\r\n"
+
+    # A common header for indicating the server is too busy.  Not used yet.
+    ERROR_503_RESPONSE = "HTTP/1.1 503 Service Unavailable\r\n\r\nBUSY".freeze
+
+    # # The basic max request size we"ll try to read.
+    # CHUNK_SIZE = 16 * 1024
+    HEX_SIZE_CHUNKED_RESPONSE = 16
+    INTERNAL_PIPE_BACKLOG = 16
+
+    # # This is the maximum header that is allowed before a client is booted.  The parser detects
+    # # this, but we"d also like to do this as well.
+    # MAX_HEADER = 1024 * (80 + 32)
+
+    # # Maximum request body size before it is moved out of memory and into a tempfile for reading.
+    # MAX_BODY = MAX_HEADER
+
+    # # A frozen format for this is about 15% faster
+    # STATUS_FORMAT = "HTTP/1.1 %d %s\r\nConnection: close\r\n".freeze
+
+    CONTENT_TYPE = "CONTENT_TYPE".freeze
+    HTTP_CONTENT_TYPE = "HTTP_CONTENT_TYPE".freeze
+    DEFAULT_TYPE = "text/plain".freeze
+
+    # LAST_MODIFIED = "Last-Modified".freeze
+    # ETAG = "ETag".freeze
+    # SLASH = "/".freeze
+    REQUEST_METHOD = "REQUEST_METHOD".freeze
+    # GET = "GET".freeze
+    HEAD = "HEAD".freeze
+    # # ETag is based on the apache standard of hex mtime-size-inode (inode is 0 on win32)
+    # ETAG_FORMAT = "\"%x-%x-%x\"".freeze
+    LINE_END = "\r\n".freeze
+    REMOTE_ADDR = "REMOTE_ADDR".freeze
+    # HTTP_X_FORWARDED_FOR = "HTTP_X_FORWARDED_FOR".freeze
+    # HTTP_IF_MODIFIED_SINCE = "HTTP_IF_MODIFIED_SINCE".freeze
+    # HTTP_IF_NONE_MATCH = "HTTP_IF_NONE_MATCH".freeze
+    # REDIRECT = "HTTP/1.1 302 Found\r\nLocation: %s\r\nConnection: close\r\n\r\n".freeze
+    # HOST = "HOST".freeze
+
+    HTTP_META = "HTTP_".freeze
+    # Portion of the request following a "?" (empty if none)
+    QUERY_STRING = "QUERY_STRING".freeze
+    # Required although HTTP_HOST takes priority if set
+    SERVER_NAME = "SERVER_NAME".freeze
+    # Required (set in spider.rb init)
+    SERVER_PORT = "SERVER_PORT".freeze
+    HTTP_HOST = "HTTP_HOST".freeze
+    PORT_80 = "80".freeze
+    PORT_443 = "443".freeze
+    LOCALHOST = "localhost".freeze
+
+    HTTP_STATUS_DEFAULT = proc { "CUSTOM" }
+    SERVER_PROTOCOL = "SERVER_PROTOCOL".freeze
+    HTTP_11 = "HTTP/1.1".freeze
+    # HTTP_10 = "HTTP/1.0".freeze
+
+    SERVER_SOFTWARE = "SERVER_SOFTWARE".freeze
+    GATEWAY_INTERFACE = "GATEWAY_INTERFACE".freeze
+    CGI_VER = "CGI/1.2".freeze
+
+    # STOP_COMMAND = "?".freeze
+    # HALT_COMMAND = "!".freeze
+    # RESTART_COMMAND = "R".freeze
+
+    RACK = "rack".freeze
+    RACK_VERSION = "rack.version".freeze
+    RACK_ERRORS = "rack.errors".freeze
+    RACK_MULTITHREAD = "rack.multithread".freeze
+    RACK_MULTIPROCESS = "rack.multiprocess".freeze
+    RACK_RUN_ONCE = "rack.run_once".freeze
+
+    # An IO like object containing all the request body
+    RACK_INPUT = "rack.input".freeze
+    # http or https
+    RACK_URL_SCHEME = "rack.url_scheme".freeze
+    # RACK_AFTER_REPLY = "rack.after_reply".freeze
+    # SPIDER_GAZELLE_SOCKET = "spider-gazelle.socket".freeze
+    # SPIDER_GAZELLE_CONFIG = "spider-gazelle.config".freeze
+
+    HTTP = "http".freeze
+    HTTPS = "https".freeze
+
+    # HTTPS_KEY = "HTTPS".freeze
+
+    # HTTP_VERSION = "HTTP_VERSION".freeze
+    # HTTP_CONNECTION = "HTTP_CONNECTION".freeze
+
+    # HTTP_11_200 = "HTTP/1.1 200 OK\r\n".freeze
+    # HTTP_10_200 = "HTTP/1.0 200 OK\r\n".freeze
+
+    CLOSE = "close".freeze
+    KEEP_ALIVE = "Keep-Alive".freeze
+
+    CONTENT_LENGTH = "CONTENT_LENGTH".freeze
+    HTTP_CONTENT_LENGTH = "HTTP_CONTENT_LENGTH".freeze
+    CONTENT_LENGTH2 = "Content-Length".freeze
+    # CONTENT_LENGTH_S = "Content-Length: ".freeze
+    TRANSFER_ENCODING = "Transfer-Encoding".freeze
+
+    CONNECTION = "Connection".freeze
+    # CONNECTION_CLOSE = "Connection: close\r\n".freeze
+    # CONNECTION_KEEP_ALIVE = "Connection: Keep-Alive\r\n".freeze
+
+    CHUNKED = "chunked".freeze
+    # TRANSFER_ENCODING_CHUNKED = "Transfer-Encoding: chunked\r\n".freeze
+    CLOSE_CHUNKED = "0\r\n\r\n".freeze
+
+    COMMA = ", ".freeze
+    COLON = ": ".freeze
+    NEWLINE = "\n".freeze
+    EMPTY = "".freeze
+
+    ZERO = "0".freeze
+
+    # Hijacking IO is supported
+    HIJACK_P = "rack.hijack?".freeze
+    # Callback for indicating that this socket will be hijacked
+    HIJACK = "rack.hijack".freeze
+    # The object for performing IO on after hijack is called
+    HIJACK_IO = "rack.hijack_io".freeze
+
+    ASYNC = "async.callback".freeze
+
+    USE_TLS = 'T'.freeze
+    NO_TLS = 'F'.freeze
+    KILL_GAZELLE = 'k'.freeze
+  end
+end

data/lib/spider-gazelle/gazelle.rb

@@ -1,172 +1,154 @@
+require 'spider-gazelle/const'
 require 'set'
 
-
 module SpiderGazelle
-    class Gazelle
-
-
-        HTTP_META = 'HTTP_'.freeze
-        REQUEST_METHOD = 'REQUEST_METHOD'.freeze    # GET, POST, etc
-        COMMA = ', '.freeze
-
+  class Gazelle
+    include Const
 
-        attr_reader :parser_cache, :connections, :logger
+    attr_reader :parser_cache, :connections, :logger
 
+    def set_instance_type(inst)
+      inst.type = :request
+    end
 
-        def set_instance_type(inst)
-            inst.type = :request
-        end
-        
-
+    def initialize(loop, logger, mode)
+      @gazelle = loop
+      # Set of active connections on this thread
+      @connections = Set.new
+      # Stale parser objects cached for reuse
+      @parser_cache = []
 
-        def initialize(loop, logger, mode)
-            @gazelle = loop
-            @connections = Set.new      # Set of active connections on this thread
-            @parser_cache = []      	# Stale parser objects cached for reuse
+      @mode = mode
+      @logger = logger
+      @app_cache = {}
+      @connection_queue = ::Libuv::Q::ResolvedPromise.new @gazelle, true
 
-            @mode = mode
-            @logger = logger
-            @app_cache = {}
-            @connection_queue = ::Libuv::Q::ResolvedPromise.new(@gazelle, true)
+      # A single parser instance for processing requests for each gazelle
+      @parser = ::HttpParser::Parser.new self
+      @set_instance_type = method :set_instance_type
 
-            # A single parser instance for processing requests for each gazelle
-            @parser = ::HttpParser::Parser.new(self)
-            @set_instance_type = method(:set_instance_type)
+      # Single progress callback for each gazelle
+      @on_progress = method :on_progress
+    end
 
-            # Single progress callback for each gazelle
-            @on_progress = method(:on_progress)
+    def run
+      @gazelle.run do |logger|
+        logger.progress do |level, errorid, error|
+          begin
+            msg = "Gazelle log: #{level}: #{errorid}\n#{error.message}\n#{error.backtrace.join("\n") if error.backtrace}\n"
+            @logger.error msg
+          rescue Exception
+            puts 'error in gazelle logger'
+          end
         end
 
-        def run
-            @gazelle.run do |logger|
-                logger.progress do |level, errorid, error|
-                    begin
-                        msg = "Gazelle log: #{level}: #{errorid}\n#{error.message}\n#{error.backtrace.join("\n") if error.backtrace}\n"
-                        @logger.error msg
-                        puts msg
-                    rescue Exception
-                        p 'error in gazelle logger'
-                    end
-                end
-
-                unless @mode == :no_ipc
-                    # A pipe used to forward connections to different threads
-                    @socket_server = @gazelle.pipe(true)
-                    @socket_server.connect(DELEGATE_PIPE) do
-                        @socket_server.progress &method(:new_connection)
-                        @socket_server.start_read2
-                    end
-
-                    # A pipe used to signal various control commands (shutdown, etc)
-                    @signal_server = @gazelle.pipe
-                    @signal_server.connect(SIGNAL_PIPE) do
-                        @signal_server.progress &method(:process_signal)
-                        @signal_server.start_read
-                    end
-                end
-            end
+        unless @mode == :no_ipc
+          # A pipe used to forward connections to different threads
+          @socket_server = @gazelle.pipe true
+          @socket_server.connect(DELEGATE_PIPE) do
+            @socket_server.progress &method(:new_connection)
+            @socket_server.start_read2
+          end
+
+          # A pipe used to signal various control commands (shutdown, etc)
+          @signal_server = @gazelle.pipe
+          @signal_server.connect(SIGNAL_PIPE) do
+            @signal_server.progress &method(:process_signal)
+            @signal_server.start_read
+          end
         end
+      end
+    end
 
+    # HTTP Parser callbacks:
+    def on_message_begin(parser)
+      @connection.start_parsing
+    end
 
-        # HTTP Parser callbacks:
-        def on_message_begin(parser)
-            @connection.start_parsing
-        end
+    def on_url(parser, url)
+      @connection.parsing.url << url
+    end
 
-        def on_url(parser, url)
-            @connection.parsing.url << url
-        end
+    def on_header_field(parser, header)
+      req = @connection.parsing
+      req.header.frozen? ? req.header = header : req.header << header
+    end
 
-        def on_header_field(parser, header)
-            req = @connection.parsing
-            if req.header.frozen?
-                req.header = header
-            else
-                req.header << header
-            end
+    def on_header_value(parser, value)
+      req = @connection.parsing
+      if req.header.frozen?
+        req.env[req.header] << value
+      else
+        header = req.header
+        header.upcase!
+        header.gsub!('-', '_')
+        header.prepend(HTTP_META)
+        header.freeze
+        if req.env[header]
+          req.env[header] << COMMA
+          req.env[header] << value
+        else
+          req.env[header] = value
         end
+      end
+    end
 
-        def on_header_value(parser, value)
-            req = @connection.parsing
-            if req.header.frozen?
-                req.env[req.header] << value
-            else
-                header = req.header
-                header.upcase!
-                header.gsub!('-', '_')
-                header.prepend(HTTP_META)
-                header.freeze
-                if req.env[header]
-                    req.env[header] << COMMA
-                    req.env[header] << value
-                else
-                    req.env[header] = value
-                end
-            end
-        end
+    def on_headers_complete(parser)
+      @connection.parsing.env[REQUEST_METHOD] = @connection.state.http_method.to_s
+    end
 
-        def on_headers_complete(parser)
-            @connection.parsing.env[REQUEST_METHOD] = @connection.state.http_method.to_s
-        end
+    def on_body(parser, data)
+      @connection.parsing.body << data
+    end
 
-        def on_body(parser, data)
-            @connection.parsing.body << data
-        end
+    def on_message_complete(parser)
+      @connection.finished_parsing
+    end
 
-        def on_message_complete(parser)
-            @connection.finished_parsing
-        end
+    def discard(connection)
+      @connections.delete(connection)
+      state = connection.state
+      state.reset!
+      @parser_cache << state
+    end
 
-        def discard(connection)
-            @connections.delete(connection)
-            @parser_cache << connection.state
-        end
+    protected
 
+    def on_progress(data, socket)
+      # Keep track of which connection we are processing for the callbacks
+      @connection = socket.storage
 
-        protected
+      # Check for errors during the parsing of the request
+      @connection.parsing_error if @parser.parse(@connection.state, data)
+    end
 
+    def new_connection(data, socket)
+      # Data == "TLS_indicator Port APP_ID"
+      tls, port, app_id = data.split(' ', 3)
+      app = @app_cache[app_id.to_sym] ||= AppStore.get(app_id)
+      inst = @parser_cache.pop || ::HttpParser::Parser.new_instance(&@set_instance_type)
 
-        def on_progress(data, socket)
-            # Keep track of which connection we are processing for the callbacks
-            @connection = socket.storage
+      # process any data coming from the socket
+      socket.progress @on_progress
+      # TODO:: Allow some globals for supplying the certs
+      socket.start_tls(:server => true) if tls == 'T'
 
-            # Check for errors during the parsing of the request
-            if @parser.parse(@connection.state, data)
-                @connection.parsing_error
-            end
-        end
+      # Keep track of the connection
+      connection = Connection.new self, @gazelle, socket, port, inst, app, @connection_queue
+      @connections.add connection
+      # This allows us to re-use the one proc for parsing
+      socket.storage = connection
 
-        def new_connection(data, socket)
-            # Data == "TLS_indicator Port APP_ID"
-            tls, port, app_id = data.split(' ', 3)
-            app = @app_cache[app_id.to_sym] ||= AppStore.get(app_id)
-            inst = @parser_cache.pop || ::HttpParser::Parser.new_instance(&@set_instance_type)
-
-            # process any data coming from the socket
-            socket.progress @on_progress
-            if tls == 'T'
-                # TODO:: Allow some globals for supplying the certs
-                socket.start_tls(:server => true)
-            end
-
-            # Keep track of the connection
-            connection = Connection.new self, @gazelle, socket, port, inst, app, @connection_queue
-            @connections.add connection
-            socket.storage = connection     # This allows us to re-use the one proc for parsing
-
-            socket.start_read
-        end
+      socket.start_read
+    end
 
-        def process_signal(data, pipe)
-            if data == Spider::KILL_GAZELLE
-                shutdown
-            end
-        end
+    def process_signal(data, pipe)
+      shutdown if data == KILL_GAZELLE
+    end
 
-        def shutdown
-            # TODO:: do this nicely
-            # Need to signal the connections to close
-            @gazelle.stop
-        end
+    def shutdown
+      # TODO:: do this nicely. Need to signal the connections to close
+      @gazelle.stop
     end
+  end
 end