swiftiply 0.6.1.1 → 1.0.0

data/src/swiftcore/Swiftiply/http_recognizer.rb

@@ -0,0 +1,267 @@
+# Encoding:ascii-8bit
+
+require 'cgi'
+require "swiftcore/Swiftiply/constants"
+
+module Swiftcore
+  module Swiftiply
+    class NotImplemented < Exception; end
+
+    # This module implements the HTTP handling code. I call it a recognizer,
+    # and not a parser because it does not parse HTTP. It is much simpler than
+    # that, being designed only to recognize certain useful bits very quickly.
+
+    class HttpRecognizer < EventMachine::Connection
+
+      attr_accessor :create_time, :last_action_time, :uri, :unparsed_uri, :associate, :name, :redeployable, :data_pos, :data_len, :peer_ip, :connection_header, :keepalive, :header_data
+
+      Crn = "\r\n".freeze
+      Crnrn = "\r\n\r\n".freeze
+      C_blank = ''.freeze
+      C_percent = '%'.freeze
+      Cunknown_host = 'unknown host'.freeze
+      C404Header = "HTTP/1.1 404 Not Found\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n"
+      C400Header = "HTTP/1.1 400 Bad Request\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n"
+
+      def self.proxy_bag_class_is klass
+        const_set(:ProxyBag, klass)
+      end
+
+      def self.init_class_variables
+        @count_404 = 0
+        @count_400 = 0
+      end
+
+      def self.increment_404_count
+        @count_404 += 1
+      end
+      
+      def self.increment_400_count
+        @count_400 += 1
+      end
+
+      # Initialize the @data array, which is the temporary storage for blocks
+      # of data received from the web browser client, then invoke the superclass
+      # initialization.
+
+      def initialize *args
+        @data = Deque.new
+        @data_pos = 0
+        @connection_header = C_empty
+        @header_missing_pieces = @name = @uri = @unparsed_uri = @http_version = @request_method = @none_match = @done_parsing = @header_data = nil
+        @keepalive = true
+        @klass = self.class
+        super
+      end
+
+      def reset_state
+        @data.clear
+        @data_pos = 0
+        @connection_header = C_empty
+        @header_missing_pieces = @name = @uri = @unparsed_uri = @http_version = @request_method = @none_match = @done_parsing = @header_data = nil
+        @keepalive = true
+      end
+      
+      # States:
+      # uri
+      # name
+      # \r\n\r\n
+      #   If-None-Match
+      # Done Parsing
+      def receive_data data
+        if @done_parsing
+          @data.unshift data
+          push
+        else
+          unless @uri
+            # It's amazing how, when writing the code, the brain can be in a zone
+            # where line noise like this regexp makes perfect sense, and is clear
+            # as day; one looks at it and it reads like a sentence.  Then, one
+            # comes back to it later, and looks at it when the brain is in a
+            # different zone, and 'lo!  It looks like line noise again.
+            #
+            # data =~ /^(\w+) +(?:\w+:\/\/([^\/]+))?([^ \?]+)\S* +HTTP\/(\d\.\d)/
+            #
+            # In case it looks like line noise to you, dear reader, too:            
+            #
+            # 1) Match and save the first set of word characters.
+            #
+            #    Followed by one or more spaces.
+            #
+            #    Match but do not save the word characters followed by ://
+            #
+            #    2) Match and save one or more characters that are not a slash
+            #
+            #    And allow this whole thing to match 1 or 0 times.
+            #
+            # 3) Match and save one or more characters that are not a question
+            #    mark or a space.
+            #
+            #    Match zero or more non-whitespace characters, followed by one
+            #    or more spaces, followed by "HTTP/".
+            #
+            # 4) Match and save a digit dot digit.
+            #
+            # Thus, this pattern will match both the standard:
+            #   GET /bar HTTP/1.1
+            # style request, as well as the valid (for a proxy) but less common:
+            #   GET http://foo/bar HTTP/1.0
+            #
+            # If the match fails, then this is a bad request, and an appropriate
+            # response will be returned.
+            #
+            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec5.1.2
+            #
+            if data =~ /^(\w+) +(?:\w+:\/\/([^ \/]+))?(([^ \?\#]*)\S*) +HTTP\/(\d\.\d)/
+              @request_method = $1
+              @unparsed_uri = $3
+              @uri = $4
+              @http_version = $5
+              if $2
+                @name = $2.intern
+                @uri = C_slash if @uri.empty?
+                # Rewrite the request to get rid of the http://foo portion.
+                
+                data.sub!(/^\w+ +\w+:\/\/[^ \/]+([^ \?]*)/,"#{@request_method} #{@uri}")
+              end
+              @uri = @uri.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n) {[$1.delete(C_percent)].pack('H*')} if @uri.include?(C_percent)
+            else
+              send_400_response
+              return
+            end
+          end
+          unless @name
+            if data =~ /^Host: *([^\r\0:]+)/
+              @name = $1.intern
+            end
+          end
+          if @header_missing_pieces
+            # Hopefully this doesn't happen often.
+            d = @data.to_s << data
+          else
+            d = data
+          end
+          if d.include?(Crnrn)
+            @name = ProxyBag.default_name unless ProxyBag.incoming_mapping(@name)
+            if d =~ /If-None-Match: *([^\r]+)/
+              @none_match = $1
+            end
+            @header_data = d.scan(/Cookie:.*/).collect {|c| c =~ /: ([^\r]*)/; $1}
+            @done_parsing = true
+
+            # Keep-Alive works differently on HTTP 1.0 versus HTTP 1.1
+            # HTTP 1.0 was not written to support Keep-Alive initially; it was
+            # bolted on.  Thus, for an HTTP 1.0 client to indicate that it
+            # wants to initiate a Keep-Alive session, it must send a header:
+            #
+            # Connection: Keep-Alive
+            #
+            # Then, when the server sends the response, it must likewise add:
+            #
+            # Connection: Keep-Alive
+            #
+            # to the response.
+            #
+            # For HTTP 1.1, Keep-Alive is assumed.  If a client does not want
+            # Keep-Alive, then it must send the following header:
+            #
+            # Connection: close
+            #
+            # Likewise, if the server does not want to keep the connection
+            # alive, it must send the same header:
+            #
+            # Connection: close
+            #
+            # to the client.
+            
+            if @name
+              unless ProxyBag.keepalive(@name) == false
+                if @http_version == C1_0
+                  if data =~ /Connection: Keep-Alive/i
+                    # Nonstandard HTTP 1.0 situation; apply keepalive header.
+                    @connection_header = CConnection_KeepAlive
+                  else
+                    # Standard HTTP 1.0 situation; connection will be closed.
+                    @keepalive = false
+                    @connection_header = CConnection_close
+                  end
+                else # The connection is an HTTP 1.1 connection.
+                  if data =~ /Connection: [Cc]lose/
+                    # Nonstandard HTTP 1.1 situation; connection will be closed.
+                    @keepalive = false
+                  end
+                end
+              end
+              
+              # THIS IS BROKEN; the interaction of @data, data, and add_frontend_client needs to be revised
+              ProxyBag.add_frontend_client(self,@data,data)
+            else
+              send_404_response
+            end           
+          else
+            @data.unshift data
+            @header_missing_pieces = true
+          end
+        end
+      end
+      
+      # Hardcoded 400 response that is sent if the request is malformed.
+
+      def send_400_response
+        ip = Socket::unpack_sockaddr_in(get_peername).last rescue Cunknown_host
+        error = "The request received on #{ProxyBag.now.asctime} from #{ip} was malformed and could not be serviced."
+        send_data "#{C400Header}Bad Request\n\n#{error}"
+        ProxyBag.logger.log(Cinfo,"Bad Request -- #{error}")
+        close_connection_after_writing
+        increment_400_count
+      end
+
+      # Hardcoded 404 response.  This is sent if a request can't be matched to
+      # any defined incoming section.
+
+      def send_404_response
+        ip = Socket::unpack_sockaddr_in(get_peername).last rescue Cunknown_host
+        error = CGI.escapeHTML "The request (#{ @uri } --> #{@name}), received on #{ProxyBag.now.asctime} from #{ip} did not match any resource know to this server."
+        send_data "#{C404Header}Resource not found.\n\n#{error}"
+        ProxyBag.logger.log(Cinfo,"Resource not found -- #{error}")
+        close_connection_after_writing
+        increment_404_count
+      end
+  
+      # The push method pushes data from the HttpRecognizer to whatever
+      # entity is responsible for handling it. You MUST override this with
+      # something useful.
+
+      def push
+        raise NotImplemented
+      end
+
+      # The connection with the web browser client has been closed, so the
+      # object must be removed from the ProxyBag's queue if it is has not
+      # been associated with a backend.  If it has already been associated
+      # with a backend, then it will not be in the queue and need not be
+      # removed.
+
+      def unbind
+        ProxyBag.remove_client(self) unless @associate
+      end
+
+      def request_method; @request_method; end
+      def http_version; @http_version; end
+      def none_match; @none_match; end
+
+      def setup_for_redeployment
+        @data_pos = 0
+      end
+
+      def increment_404_count
+        @klass.increment_404_count
+      end
+      
+      def increment_400_count
+        @klass.increment_400_count
+      end
+      
+    end
+  end
+end

data/src/swiftcore/Swiftiply/loggers/Analogger.rb

@@ -0,0 +1,21 @@
+# The Swiftiply logging support is really written with Analogger in mind, since
+# the ideal logging solution should provide a minimal performance impact.
+
+require 'swiftcore/Analogger/Client'
+
+module Swiftcore
+	module Swiftiply
+		module Loggers
+			class Analogger
+				def self.new(params)
+					lp = []
+					lp << params['service'] || 'swiftiply'
+					lp << params['host'] || '127.0.0.1'
+					lp << (params['port'] && params['port'].to_i) || 6766
+					lp << params['key']
+					::Swiftcore::Analogger::Client.new(*lp)
+				end
+			end
+		end
+	end
+end

data/src/swiftcore/Swiftiply/loggers/stderror.rb

@@ -0,0 +1,13 @@
+module Swiftcore
+	module Swiftiply
+		module Loggers
+			class Stderror
+				def initialize(*args);end
+				
+				def log(severity, msg)
+					$stderr.puts "#{Time.now.asctime}:#{severity}:#{msg}"
+				end
+			end
+		end
+	end
+end

data/src/swiftcore/Swiftiply/mocklog.rb

@@ -0,0 +1,10 @@
+# This is just a mock log to provide a bit bucket logging location, if needed.
+
+module Swiftcore
+	module Swiftiply
+		class MockLog
+			def log(*args)
+			end
+		end
+	end
+end

data/src/swiftcore/Swiftiply/proxy.rb

@@ -0,0 +1,15 @@
+# Super class for all proxy implementations.
+module Swiftcore
+	module Swiftiply
+		class Proxy
+			def self.config(conf, new_conf)
+				# Process Proxy config; determine which specific proxy implementation to load and pass config control into.
+				conf[Cproxy] ||= Ckeepalive if conf[Ckeepalive]
+				require "swiftcore/Swiftiply/proxy_backends/#{conf[Cproxy]}"
+				klass = Swiftcore::Swiftiply::get_const_from_name(conf[Cproxy].upcase, ::Swiftcore::Swiftiply::Proxies)
+				# Need error handling here!
+				klass.config(conf, new_conf)
+			end
+		end
+	end
+end

data/src/swiftcore/Swiftiply/proxy_backends/keepalive.rb

@@ -0,0 +1,286 @@
+# Encoding:ascii-8bit
+
+require 'swiftcore/Swiftiply/config'
+# Standard style proxy.
+module Swiftcore
+  module Swiftiply
+    module Proxies
+      class Keepalive < EventMachine::Connection
+
+        def self.is_a_server?
+          true
+        end
+
+        # While the configuration parsing now lives with the protocol, a lot of the code for it is generic. Those generic pieces need to live back up in Swiftcore::Swiftiply or maybe Swiftcore::Swiftiply::Config.
+        def self.config(m,new_config)
+          #require 'swiftcore/Swiftiply/backend_protocol'
+          # keepalive requests are standard Swiftiply requests.
+
+          # The hash of the "outgoing" config section.  It is used to
+          # uniquely identify a section.
+          owners = m[Cincoming].sort.join('|')
+          hash = Digest::SHA256.hexdigest(owners).intern
+          config_data = {:hash => hash, :owners => owners}
+
+          Config.configure_logging(m, config_data)
+          file_cache = Config.configure_file_cache(m, config_data)
+          dynamic_request_cache = Config.configure_dynamic_request_cache(m, config_data)
+          etag_cache = Config.configure_etag_cache(m, config_data)
+
+          # For each incoming entry, do setup.
+          new_config[Cincoming] = {}
+          m[Cincoming].each do |p_|
+            configure_one({ :p_                     => p_,
+                            :m                     => m,
+                            :new_config            => new_config,
+                            :config_data           => config_data,
+                            :file_cache            => file_cache,
+                            :dynamic_request_cache => dynamic_request_cache,
+                            :etag_cache            => etag_cache })
+          end
+        end
+
+        def self.configure_one(args = {})
+          new_config = args[:new_config]
+          config_data = args[:config_data]
+          p_ = args[:p_]
+          file_cache = args[:file_cache]
+          dynamic_request_cache = args[:dynamic_request_cache]
+          etag_cache = args[:etag_cache]
+          m = args[:m]
+          
+          ProxyBag.logger.log(Cinfo,"Configuring incoming #{p_}") if Swiftcore::Swiftiply::log_level > 1
+          p = p_.intern
+
+          Config.setup_caches(new_config, config_data.merge({:p => p, :file_cache => file_cache, :dynamic_request_cache => dynamic_request_cache, :etag_cache => etag_cache}))
+
+          ProxyBag.add_backup_mapping(m[Cbackup].intern,p) if m.has_key?(Cbackup)
+          Config.configure_docroot(m, p)
+          config_data[:permit_xsendfile] = Config.configure_sendfileroot(m, p)
+          Config.configure_xforwardedfor(m, p)
+          Config.configure_redeployable(m, p)
+          Config.configure_key(m, p, config_data)
+          Config.configure_staticmask(m, p)
+          Config.configure_cache_extensions(m,p)
+          Config.configure_cluster_manager(m,p)
+          Config.configure_backends(Coutgoing, { :config => m,
+                                                 :p => p,
+                                                 :config_data => config_data,
+                                                 :new_config => new_config,
+                                                 :self => self,
+                                                 :directory_class => ::Swiftcore::Deque,
+                                                 :directory_args => []})
+          Config.stop_unused_servers(new_config)
+          Config.set_server_queue(config_data, ::Swiftcore::Deque, [])
+        end
+
+        # Swiftcore::Swiftiply::Proxies::Keepalive is the EventMachine::Connection
+        # subclass that handles the communications between Swiftiply and a
+        # persistently connected Swiftiply client process.
+
+        attr_accessor :associate, :id
+
+        C0rnrn = "0\r\n\r\n".freeze
+        Crnrn = "\r\n\r\n".freeze
+
+        def initialize *args
+          @name = self.class.bname
+          @permit_xsendfile = self.class.xsendfile
+          @enable_sendfile_404 = self.class.enable_sendfile_404
+          super
+        end
+
+        def name
+          @name
+        end
+
+        # Call setup() and add the backend to the ProxyBag queue.
+
+        def post_init
+          setup
+          @initialized = nil
+          ProxyBag.add_server self
+        end
+
+        # Setup the initial variables for receiving headers and content.
+
+        def setup
+          @headers = ''
+          @headers_completed = @dont_send_data = false
+          #@content_length = nil
+          @content_sent = 0
+          @filter = self.class.filter
+        end
+
+        # Receive data from the backend process.  Headers are parsed from
+        # the rest of the content.  If a Content-Length header is present,
+        # that is used to determine how much data to expect.  Otherwise,
+        # if 'Transfer-encoding: chunked' is present, assume chunked
+        # encoding.  Otherwise be paranoid; something isn't the way we like
+        # it to be.
+
+        def receive_data data
+          unless @initialized
+            # preamble = data.slice!(0..24)
+            preamble = data[0..24]
+            data = data[25..-1] || C_empty
+            keylen = preamble[23..24].to_i(16)
+            keylen = 0 if keylen < 0
+            key = keylen > 0 ? data.slice!(0..(keylen - 1)) : C_empty
+            #if preamble[0..10] == Cswiftclient and key == ProxyBag.get_key(@name)
+            if preamble.index(Cswiftclient) == 0 and key == ProxyBag.get_key(@name)
+              @id = preamble[11..22]
+              ProxyBag.add_id(self,@id)
+              @initialized = true
+            else
+              # The worker that connected did not present the proper authentication,
+              # so something is fishy; time to cut bait.
+              close_connection
+              return
+            end
+          end
+
+          unless @headers_completed
+            if data.include?(Crnrn)
+              @headers_completed = true
+              h,data = data.split(/\r\n\r\n/,2)
+              #@headers << h << Crnrn
+              if @headers.length > 0
+                @headers << h
+              else
+                @headers = h
+              end
+
+              if @headers =~ /Content-[Ll]ength: *([^\r]+)/
+                @content_length = $1.to_i
+              elsif @headers =~ /Transfer-encoding:\s*chunked/
+                @content_length = nil
+              else
+                @content_length = 0
+              end
+
+              if @permit_xsendfile && @headers =~ /X-[Ss]endfile: *([^\r]+)/
+                @associate.uri = $1
+                if ProxyBag.serve_static_file(@associate,ProxyBag.get_sendfileroot(@associate.name))
+                  @dont_send_data = true
+                else
+                  if @enable_sendfile_404
+                    msg = "#{@associate.uri} could not be found."
+                    @associate.send_data "HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\nContent-Type: text/html\r\nContent-Length: #{msg.length}\r\n\r\n#{msg}"
+                    @associate.close_connection_after_writing
+                    @dont_send_data = true
+                  else
+                    @associate.send_data (@headers + Crnrn)
+                  end
+                end
+              else
+                @associate.send_data (@headers + Crnrn)
+              end
+
+              # If keepalive is turned on, the assumption is that it will stay
+              # on, unless the headers being returned indicate that the connection
+              # should be closed.
+              # So, check for a 'Connection: Closed' header.
+              if keepalive = @associate.keepalive
+                keepalive = false if @headers =~ /Connection: [Cc]lose/
+                if @associate_http_version == C1_0
+                  keepalive = false unless @headers == /Connection: Keep-Alive/i
+                end
+              end
+            else
+              @headers << data
+            end
+          end
+
+          if @headers_completed
+            @associate.send_data data unless @dont_send_data
+            @content_sent += data.length
+            if ( @content_length and @content_sent >= @content_length ) or data[-6..-1] == C0rnrn or @associate.request_method == CHEAD
+              # If @dont_send_data is set, then the connection is going to be closed elsewhere.
+              unless @dont_send_data
+                # Check to see if keepalive is enabled.
+                if keepalive
+                  @associate.reset_state
+                  ProxyBag.remove_client(self) unless @associate
+                else
+                  @associate.close_connection_after_writing
+                end
+              end
+              @associate = @headers_completed = @dont_send_data = nil
+              @headers = ''
+              #@headers_completed = false
+              #@content_length = nil
+              @content_sent = 0
+              #setup
+              ProxyBag.add_server self
+            end
+          end
+        # TODO: Log these errors!
+        rescue Exception => e
+          puts "Kaboom: #{e} -- #{e.backtrace.inspect}"
+          @associate.close_connection_after_writing if @associate
+          @associate = nil
+          setup
+          ProxyBag.add_server self
+        end
+
+        # This is called when the backend disconnects from the proxy.
+        # If the backend is currently associated with a web browser client,
+        # that connection will be closed.  Otherwise, the backend will be
+        # removed from the ProxyBag's backend queue.
+
+        def unbind
+          if @associate
+            if !@associate.redeployable or @content_length
+              @associate.close_connection_after_writing
+            else
+              @associate.associate = nil
+              @associate.setup_for_redeployment
+              ProxyBag.rebind_frontend_client(@associate)
+            end
+          else
+            ProxyBag.remove_server(self)
+          end
+          ProxyBag.remove_id(self)
+        end
+
+        def self.bname=(val)
+          @bname = val
+        end
+
+        def self.bname
+          @bname
+        end
+
+        def self.xsendfile=(val)
+          @xsendfile = val
+        end
+
+        def self.xsendfile
+          @xsendfile
+        end
+
+        def self.enable_sendfile_404=(val)
+          @enable_sendfile_404 = val
+        end
+
+        def self.enable_sendfile_404
+          @enable_sendfile_404
+        end
+
+        def self.filter=(val)
+          @filter = val
+        end
+
+        def self.filter
+          @filter
+        end
+
+        def filter
+          @filter
+        end
+
+      end
+    end
+  end
+end