mongrel_esi 0.4.0

data/lib/esi/cache.rb

@@ -0,0 +1,207 @@
+require 'digest/sha1'
+require 'thread'
+
+require 'esi/logger'
+
+##
+# Provide classes with an efficient method for dynamically changing from
+# multiple threaded to single threaded when the class is created.
+# rather then requiring multiple object types just change the object behavior
+
+module Synchronizable
+  def synchronize_methods( *methods_names )
+    methods_names.flatten.each do|name|
+      base_name = name
+      if name.to_s.match(/!$/)
+        suffix = "!"
+        base_name = name.to_s.gsub(/!$/,'')
+      end
+      unlocked_name = "#{base_name}_unlocked#{suffix}".to_sym
+      locked_name = "#{base_name}_locked#{suffix}".to_sym
+
+      self.class.send :alias_method, unlocked_name, name
+
+			next unless @semaphore
+      
+      self.class.send(:define_method, locked_name) do
+        @semaphore.synchronize do
+          send unlocked_name, args
+        end
+      end
+    end
+  end
+end
+
+module ESI
+
+  # cache interface
+  #
+  #  all caches store Fragments
+  #
+  #  get - uri,params => Fragment
+  #
+  #  cached? - uri,params => boolean
+  # 
+  #  put - uri,params, max_age, body => nil
+  #
+  #  sweep! - tells the cache to expire anything that needs expiring
+  # 
+  #  keys - is an optional method for iterating over all keys, may not be exposed by all cache backends
+  #
+  #  delete - key remove the key from the cache
+  
+  # Each object in the cache has a ttl
+  class Fragment
+    attr_accessor :body, :max_age, :uri
+    alias read_body body
+
+    def initialize(uri,max_age,body)
+      @uri = uri
+      max_age.gsub!(/\+.*$/,'') if max_age.class == String
+      @max_age = Time.now.to_i + max_age.to_i
+      @body = body
+    end
+
+    def valid?
+      (Time.now.to_i < @max_age)
+    end
+
+  end
+  
+
+  # Base Cache object
+  class Cache
+		include Synchronizable
+
+		def initialize( options = OpenStruct.new({}) )
+      options = OpenStruct.new(options) if options.is_a?(Hash)
+      if( options.locked )
+        @semaphore = Mutex.new
+      end
+			synchronize_methods :get, :put, :put, :sweep!, :keys, :delete
+      # TODO: load the cache_headers config
+		end
+
+  protected
+    def cache_key( uri, params )
+      http_x_requested_with = params['HTTP_X_REQUESTED_WITH'] || params["X-Requested-With"]
+      key = Digest::SHA1.hexdigest("#{uri}-#{http_x_requested_with}")
+      key
+    end
+    
+  end
+  
+  #
+  # A memcache cache store.  Uses the memcached ruby client, see => http://seattlerb.rubyforge.org/memcache-client/
+  # and http://www.danga.com/memcached/
+  #
+  # There are few issues to consider about providing a memcached backed.
+  #
+  # First, there's no good way to iterate over all the keys within memcached and doing so is problematic beyound the impelmentation
+  # more details can be found here => http://lists.danga.com/pipermail/memcached/2007-February/003610.html
+  #
+  # Okay, so now what?  Well we still have options:
+  #  - We could try Tugela Cache => http://meta.wikimedia.org/wiki/Tugela_Cache
+  #  - This looks promising => http://www.marcworrell.com/article-500-en.htmlhttp://www.marcworrell.com/article-500-en.html
+  #  - We could also store the keys in the ruby process or alternatively look in INVALIDATION-WITH-MEMCACHED
+  #
+  # We can't support advanced selector with memcached backend
+  class MemcachedCache < Cache
+    require 'rubygems'
+    gem 'memcache-client' if respond_to? :gem
+    require 'memcache'
+    include ESI::Log
+
+    def initialize( options )
+      super
+      # TODO: OpenStruct.marshal_dump !!!
+      options = { :servers => options.servers, 
+                  :debug => options.debug,
+                  :namespace => options.namespace,
+                  :readonly => options.readonly }
+      puts "Using memcache with options => #{options.inspect}"
+      @cache = MemCache.new options
+      puts "Using memcache servers at #{options['servers'].inspect}"
+      @cache.servers = options[:servers]
+    end
+
+    def cached?( uri, params )
+      fragment = get(uri,params)
+      fragment and fragment.valid?
+    rescue Object => e # never raise an exception from this method
+      false
+    end
+
+    def get( uri, params )
+      @cache.get(cache_key(uri,params))
+    end
+ 
+    def put( uri, params, max_age, body )
+      fragment = Fragment.new(uri,max_age,body)
+      key = cache_key(uri,params)
+      @cache.add( key, fragment, fragment.max_age )
+    rescue Object => e
+			log_error e.message
+    end
+
+		# run through the cache and dump anything that has expired
+		def sweep!
+      # TODO: not really a memcached equivalent??
+		end
+
+    def keys(&block)
+      # TODO: can't implement this method directly with memcached, unless we introduce another key
+      # and increase the data, but it is possible...
+    end
+
+    def delete( key )
+      @cache.delete(key)
+    end
+  end
+
+  #
+  # A ruby thread safe cache, stores cached fragments in the current ruby process memory
+  #
+  # A hash table indexed by cache_key of Fragments.
+  # the cache is made thread safe if the external invalidator is active otherwise the Mutex is a no op
+  class RubyCache < Cache
+
+    def initialize( options = {} )
+      super
+      @cache = {}
+    end
+
+    def cached?( uri, params )
+      key = cache_key(uri,params)
+      fragment = @cache[key]
+      fragment and fragment.valid?
+    end
+
+    def get( uri, params )
+      @cache[cache_key(uri,params)]
+    end
+ 
+    def put( uri, params, max_age, body )
+      key = cache_key(uri,params)
+      @cache[key] = Fragment.new(uri,max_age,body)
+      sweep_unlocked!
+    end
+
+		# run through the cache and dump anything that has expired
+    def sweep!
+      @cache.reject! {|k,v| !v.valid? }
+    end
+
+    def keys(&block)
+      @cache.each do|key,data|
+        yield key, data
+      end
+    end
+
+    def delete( key )
+      @cache.delete(key)
+    end
+
+  end
+
+end

data/lib/esi/config.rb

@@ -0,0 +1,154 @@
+require 'ostruct'
+
+module ESI
+  class Config
+
+    attr_reader :config
+    
+    def initialize(options)
+      @config = options
+    end
+
+    # access configuration values
+    def [](key)
+      @config[key]
+    end
+
+    def enable_esi_processor?( headers )
+      # check for surrogate control configuration
+      # check for matching content type
+      # if both are set it surrogate takes presendence 
+      use_esi = false
+      allowed_content_types = @config[:allowed_content_types]
+
+      if allowed_content_types and headers["Content-Type"] and allowed_content_types.respond_to?(:detect)
+        use_esi = allowed_content_types.detect do |type|
+          headers["Content-Type"].match( type )
+        end
+        use_esi = true if use_esi
+      end
+
+      if @config[:enable_for_surrogate_only]
+        use_esi = headers["Surrogate-Control"] and /ESI/.match(headers["Surrogate-Control"])
+      end
+
+      use_esi
+    end
+
+    class CacheConfig
+      attr_reader :options
+      def initialize
+        @memcached = false
+        @options = OpenStruct.new({}) 
+      end
+
+      def memcached
+        @memcached = true
+        yield @options
+      end
+
+      def memcached?
+        @memcached
+      end
+
+      def locked
+        !@memcached
+      end
+
+      def ttl=( ttl )
+        @options.ttl = ttl
+      end
+
+    end
+ 
+    # returns the cache object as given in the config/esi.yml
+    # cache: key, or defaults to ruby as in uses this process
+    # the options allowed are ruby and memcache
+    def cache
+      if block_given?
+        # allow this method to be called in config scripts
+        cache_options = CacheConfig.new
+        yield cache_options
+        if cache_options.memcached?
+          @config[:cache] = 'memcached'
+        end
+        @config[:cache_options] = cache_options.options
+      else
+        cache_type = @config[:cache]
+        options = @config[:cache_options]
+        # always return the same cache object, per process
+        $cache ||= case cache_type
+        when 'ruby'
+          ESI::RubyCache.new(options)
+        when 'memcached'
+          ESI::MemcachedCache.new(options)
+        else
+          raise "Unsupported cache"
+        end
+      end
+    end
+
+    def esi
+      options = OpenStruct.new({})
+      yield options
+      @config[:allowed_content_types] = options.allowed_content_types if options.allowed_content_types
+      @config[:enable_for_surrogate_only] = options.enable_for_surrogate_only if options.enable_for_surrogate_only
+      @config[:chunk_size] = options.chunk_size if options.chunk_size
+      @config[:max_depth] = options.max_depth if options.max_depth
+    end
+
+    def router
+      Router.new( @config[:routing] )
+    end
+ 
+    class ConfigRouter
+      attr_accessor :servers
+      attr_reader :routes
+
+      def initialize
+        @routes = []
+        @servers = []
+      end
+
+      def match( expr )
+        yield self
+        @routes << { :host => @servers.first.split(':').first, 
+                     :port => @servers.last.split(':').last,
+                     :match_url => expr }
+      end
+
+      def default
+        yield self
+        @routes << { :host => @servers.first.split(':').first, 
+                     :port => @servers.last.split(':').last,
+                     :match_url => 'default' }
+      end
+
+    end
+
+    def routes
+      config_router = ConfigRouter.new
+      yield config_router
+      @config[:routing] = config_router.routes
+    end
+
+    # return true/false depending on the value of 
+    # invalidator: on/off
+    # within in config/esi.yml
+    def start_invalidator?
+      @config[:invalidator]
+    end
+
+    def self.define( listeners )
+      listeners.each do|host,server|
+        esi_handlers = server.classifier.handler_map.select do|uri,handler|
+          handler.first.class == ESI::Dispatcher
+        end
+        config = esi_handlers.first.last.first.config
+        yield config
+      end
+    end
+
+  end    
+end
+

data/lib/esi/dispatcher.rb

@@ -0,0 +1,27 @@
+require 'esi/handler'
+require 'mongrel'
+
+module ESI
+
+  class Dispatcher < Mongrel::HttpHandler
+    attr_reader :config, :router
+
+    Thread.abort_on_exception = false
+
+    def initialize( options )
+      super()
+      @config = ESI::Config.new( options )
+      @router = nil
+      if @config.start_invalidator?
+        ESI::Invalidator.start( self )
+      end
+    end
+
+    def process(request, response)
+      @router = @config.router
+      handler = ESI::Handler.new(self)
+      handler.process(request, response)
+    end
+
+  end
+end

data/lib/esi/handler.rb

@@ -0,0 +1,236 @@
+require 'uri'
+require 'timeout'
+require 'net/http'
+
+require 'esi/logger'
+require 'esi/cache'
+require 'esi/config'
+require 'esi/router'
+require 'esi/esi'
+require 'esi/tag/base'
+require 'esi/tag/include'
+require 'esi/tag/invalidate'
+require 'esi/tag/attempt'
+require 'esi/tag/except'
+require 'esi/tag/try'
+
+module ESI
+
+  class Handler
+    attr_reader :config
+    include ESI::Log
+
+    def initialize(dispatcher)
+      @config = dispatcher.config
+      @router = dispatcher.router
+    end
+
+    def process(request, response)
+ 
+      start = Time.now
+      status = 200
+      url = @router.url_for(request.params["REQUEST_URI"])
+
+      params = http_params(request.params)
+
+      proxy_error = nil
+
+
+      log_debug "#{request.params["REQUEST_METHOD"]} => #{url}"
+      chunk_count = 0
+			uri = URI.parse(url)
+
+			path_with_query = uri.query ? "#{uri.path}?#{uri.query}" :  uri.path
+
+			proxy_request = (request.params["REQUEST_METHOD"] == "POST") ?
+													Net::HTTP::Post.new( path_with_query, params ) :
+													Net::HTTP::Get.new( path_with_query, params )
+
+			proxy_connection = Net::HTTP.start(uri.host, uri.port)
+
+			# open the conneciton up so we can start to stream the connection
+			proxy_connection.request(proxy_request,request.body.read) do|proxy_response|
+
+				status = read_status( proxy_response )
+
+				copy_headers( response.header, proxy_response ) unless status >= 500
+
+				if status >= 500 or (status < 400 and status >= 300) or !@config.enable_esi_processor?( proxy_response )
+					response.start(status, true) do|head,out|
+
+						if status >= 500
+							# TODO: only report this if configured to expose it
+							out << proxy_error
+						end
+
+						# proxy the 500 response
+						proxy_response.read_body do|fragment|
+							out << fragment
+						end
+
+					end
+				else
+          # NOTE: It's very important that surrogate control headers are set to parse only if the 
+          # page has esi:include tags.  Because of the nature of Transfer-Encoding: chunked if we keep
+          # everything in memory until we reach an esi tag.  Then we load the tag into memory and send the next
+          # chunk and so on.  This means that the density of tags to markup will result in more or less of the document
+          # being stored in memory.  A way we can get around this and attempt to keep a fixed size of the document in 
+          # memory at all time is by setting a buffer size of say 1024.  Then no matter what we'll always chunk the document
+          # by 1024 or some other size chunk.
+					begin
+						# Use the ESI Parser
+
+						response.header["Transfer-Encoding"] = "chunked"
+						# this is the important part, rather then send the whole document back we send in chunks
+						# each fragment is roughly in it's own chunk, this does mean we require http 1.1, chunk size is still a limit
+						header = Mongrel::Const::STATUS_FORMAT % [status, Mongrel::HTTP_STATUS_CODES[status]]
+						header.gsub!(/Connection: close\r\n/,'')
+						response.header.out.rewind
+						header << response.header.out.read + Mongrel::Const::LINE_END
+						header.gsub!(/Status:.*?\r\n/,'')
+						response.write( header )
+
+						#print header
+ 
+            parser = ESI::CParser.new
+            chunk_size = @config[:chunk_size] || 4096
+            max_depth = @config[:max_depth] || 3
+            buffer = "" # when buffer reaches chunk_size write to the response socket
+
+            # handle start tags
+            parser.start_tag_handler do|tag_name, attrs|
+              tag = ESI::Tag::Base.create( @router,
+                                           request.params,
+                                           params,
+                                           tag_name.gsub(/esi:/,''),
+                                           attrs,
+                                           @config.cache )
+              # set the tag depth
+              tag.depth = parser.depth if tag.respond_to?(:depth=)
+              tag.max_depth = max_depth if tag.respond_to?(:max_depth=)
+
+              if parser.esi_tag
+                parser.esi_tag.add_child(tag)
+              else
+                parser.esi_tag = tag
+              end
+            end
+
+            # handle end tags
+            parser.end_tag_handler do|tag_name|
+              #ct = Time.now
+              if parser.esi_tag.name == tag_name.gsub(/esi:/,'')
+                parser.esi_tag.close(parser.output)
+                parser.esi_tag = nil
+              else
+                parser.esi_tag.close_child(parser.output,tag_name)
+              end
+              #puts "\t[#{tag_name}] Time to close: #{Time.now - ct}"
+            end
+ 
+            # handle data streaming
+            parser.output_handler do|chars|
+              buffer << chars
+              if buffer.size >= chunk_size
+                #print buffer
+                send_chunk( response, buffer )
+                chunk_count += 1
+                buffer = ""
+              end
+            end
+
+            #t = Time.now
+            proxy_response.read_body do|data|
+              begin
+                #pt = Time.now
+                parser.process data
+                #puts "Time in process: #{Time.now - pt}"
+              rescue => e
+                puts e.message
+                puts e.backtrace.join("\n")
+                raise e
+              end
+            end
+            #puts "Response Time: #{Time.now - t}"
+
+            parser.finish 
+            parser = nil
+
+            if buffer.size > 0
+              #print buffer
+							send_chunk( response, buffer )
+              chunk_count += 1
+            end
+
+					rescue => e
+            status, error = error_response(e,url)
+						response.write( error )
+					end
+					response.write( "0\r\n\r\n" )
+					response.done = true
+				end
+			end # end request
+
+		rescue => e
+			status = error_response(e,url).first
+		ensure
+
+      log_request "\nCompleted => #{url}, #{Time.now - start} seconds with status #{status} and #{chunk_count} chunks\n"
+
+    end
+
+protected
+
+    def send_chunk( response, buffer )
+      # send a new chunk
+      size = buffer.size
+      chunk_header = "#{"%x" % size}" + Mongrel::Const::LINE_END
+      #puts chunk_header.inspect
+      response.write( chunk_header )  # write the chunk size
+      #puts buffer.inspect
+      response.write( buffer + Mongrel::Const::LINE_END )  # write the chunk
+    end
+
+    def error_response(e,url)
+      status = 500
+      error = "<h1>Internal Server Errror</h1><h4>Failed while requesting => '#{url}'</h4>\n<pre>#{e.message}#{e.backtrace.join("\n")}</pre>"
+      log_error e.backtrace.join("\n")
+      log_error error
+      [status, "<html><body>#{error}</body></html>"]
+    end
+
+    def read_status(response)
+      Net::HTTPResponse::CODE_TO_OBJ.select { |k,v| v == response.class }.first[0].to_i rescue 500
+    end
+
+    def http_params(params)
+      updated_params = {}
+      params.each do|k,v|
+        k = k.split('_').collect { |t| t.capitalize }.join('-')
+        if k[0,5] =='Http-'
+          k[0,5] = ''
+          updated_params[k] = v
+        end
+      end
+      updated_params
+    end
+
+    def copy_headers(head,response)
+      response.to_hash.each do |k,v|
+        # for Set-Cookie we need to split on ,
+        # some edge cases with , since things like expires might be a date with , in them.
+        k = k.split(/-/).map{|s| s.capitalize }.join('-')
+        if k == "Set-Cookie"
+          v.each do|cookie|
+            head["Set-Cookie"] = cookie.strip # mongrel is case sensitive about handling duplicates
+          end
+        else
+          head[k] = v unless k == "Content-Length" or k == "Surrogate-Control" or k == "Server"
+        end
+      end
+      head["Server"] = "MongrelESI 0.4"
+    end
+
+  end # Handler
+
+end # ESI