RubyGems - mongrel_esi - Versions diffs - 0.4.0 - Mend

mongrel_esi 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (73) hide show

data/COPYING +53 -0
data/LICENSE +471 -0
data/README +186 -0
data/Rakefile +141 -0
data/bin/mongrel_esi +271 -0
data/ext/esi/common.rl +41 -0
data/ext/esi/esi_parser.c +387 -0
data/ext/esi/extconf.rb +6 -0
data/ext/esi/machine.rb +499 -0
data/ext/esi/parser.c +1675 -0
data/ext/esi/parser.h +113 -0
data/ext/esi/parser.rb +49 -0
data/ext/esi/parser.rl +398 -0
data/ext/esi/ruby_esi.rl +135 -0
data/ext/esi/run-test.rb +3 -0
data/ext/esi/test/common.rl +41 -0
data/ext/esi/test/parser.c +1676 -0
data/ext/esi/test/parser.h +113 -0
data/ext/esi/test/parser.rl +398 -0
data/ext/esi/test/test.c +373 -0
data/ext/esi/test1.rb +56 -0
data/ext/esi/test2.rb +45 -0
data/lib/esi/cache.rb +207 -0
data/lib/esi/config.rb +154 -0
data/lib/esi/dispatcher.rb +27 -0
data/lib/esi/handler.rb +236 -0
data/lib/esi/invalidator.rb +40 -0
data/lib/esi/logger.rb +46 -0
data/lib/esi/router.rb +84 -0
data/lib/esi/tag/attempt.rb +6 -0
data/lib/esi/tag/base.rb +85 -0
data/lib/esi/tag/except.rb +24 -0
data/lib/esi/tag/include.rb +190 -0
data/lib/esi/tag/invalidate.rb +54 -0
data/lib/esi/tag/try.rb +40 -0
data/lib/multi_dirhandler.rb +70 -0
data/setup.rb +1585 -0
data/test/integration/basic_test.rb +39 -0
data/test/integration/cache_test.rb +37 -0
data/test/integration/docs/content/500.html +16 -0
data/test/integration/docs/content/500_with_failover.html +16 -0
data/test/integration/docs/content/500_with_failover_to_alt.html +8 -0
data/test/integration/docs/content/ajax_test_page.html +15 -0
data/test/integration/docs/content/cookie_variable.html +3 -0
data/test/integration/docs/content/foo.html +15 -0
data/test/integration/docs/content/include_in_include.html +15 -0
data/test/integration/docs/content/malformed_transforms.html +16 -0
data/test/integration/docs/content/malformed_transforms.html-correct +11 -0
data/test/integration/docs/content/static-failover.html +20 -0
data/test/integration/docs/content/test2.html +1 -0
data/test/integration/docs/content/test3.html +17 -0
data/test/integration/docs/esi_invalidate.html +6 -0
data/test/integration/docs/esi_mixed_content.html +15 -0
data/test/integration/docs/esi_test_content.html +27 -0
data/test/integration/docs/index.html +688 -0
data/test/integration/docs/test1.html +1 -0
data/test/integration/docs/test3.html +9 -0
data/test/integration/docs/test_failover.html +1 -0
data/test/integration/handler_test.rb +270 -0
data/test/integration/help.rb +234 -0
data/test/net/get_test.rb +197 -0
data/test/net/net_helper.rb +16 -0
data/test/net/server_test.rb +249 -0
data/test/unit/base_tag_test.rb +44 -0
data/test/unit/esi-sample.html +56 -0
data/test/unit/help.rb +77 -0
data/test/unit/include_request_test.rb +69 -0
data/test/unit/include_tag_test.rb +14 -0
data/test/unit/parser_test.rb +478 -0
data/test/unit/router_test.rb +34 -0
data/test/unit/sample.html +21 -0
data/tools/rakehelp.rb +119 -0
metadata +182 -0

data/lib/esi/invalidator.rb ADDED Viewed

@@ -0,0 +1,40 @@
+# this is a separate thread that runs on port 4001
+# when requests come to this they must authenticate
+# and be of type POST
+# the requests are processed as described by: http://www.w3.org/TR/esi-invp
+require 'webrick'
+module ESI
+  module Invalidator
+    def self.start( cache )
+      Thread.new( cache ) do|cache|
+        s = WEBrick::HTTPServer.new( :Port => 4001 )
+        s.mount_proc("/invalidate"){|req, res|
+          res.body = "<html>invalidate posted objects</html>"
+          res['Content-Type'] = "text/html"
+        }
+        s.mount_proc("/status"){|req, res|
+          res.body = "<html><body><h1>Cached objects</h1>"
+          res.body << "<ul>"
+          cache.keys do|key,data|
+            res.body << "<li>#{key}</li>"
+          end
+          res.body << "</ul>"
+          res.body << "</body>"
+          res.body << "</html>"
+          res['Content-Type'] = "text/html"
+        }
+        # XXX: this doesn't chain so ends up removing the mongrel trap locking the server up
+        #trap("INT"){ s.shutdown }
+        s.start
+      end
+    end
+  end
+end

data/lib/esi/logger.rb ADDED Viewed

@@ -0,0 +1,46 @@
+require 'logger'
+module ESI
+  module Log
+    if ENV["test"] == "true"
+      $logger ||= Logger.new("log/test.log")
+      $logger.instance_eval do
+        def puts( msg )
+          debug( msg )
+        end
+        def print( msg )
+          debug( msg )
+        end
+      end
+    end
+    def log( io, msg )
+      io.puts msg
+    end
+    def msg( io, msg )
+      io.print msg
+    end
+    def log_request( msg )
+      msg( $logger || STDERR, msg )
+    end
+    def log_debug( msg )
+      log( $logger || STDERR, msg )
+    end
+    def log_error( msg )
+      log( $logger || STDERR, msg )
+    end
+    def log_info( msg )
+      log( $logger || STDERR, msg )
+    end
+    def log_warn( msg )
+      log( $logger || STDERR, msg )
+    end
+  end
+end

data/lib/esi/router.rb ADDED Viewed

@@ -0,0 +1,84 @@
+module ESI
+  # ESI Server is a reverse proxy caching server.  It will forward all requests to app servers
+  # by processing a config_file that specifies routes:
+  #
+  # content:
+  #   host: 127.0.0.1
+  #   port: 3000
+  #   match_url: ^\/(content|samples|extra).*
+  # default:
+  #   host: 127.0.0.1
+  #   port: 3001
+  #
+  # This sample configuration will route all urls starting with /content, /samples, or /extras
+  # to the server running at 127.0.0.1 on port 3000
+  # everything else that matches the .* will be routed to the server running on port 3001
+  # optionally the caching duration can be specificied explicity for each host, this will be the default per host
+  # if the esi:include tag does not specify otherwise
+  #
+  # default:
+  #   host: 127.0.01
+  #   port: 3001
+  #   cache_ttl: 300
+  #
+  # This example will cache all requests for 300 seconds, by default.
+  #
+  # To create a router load either from memory or file the above YAML
+  #
+  #   router = ESI::Router.new( YAML.load_file('config.yml') )
+  #
+  # or from memory
+  #
+  #   router = ESI::Router.new( YAML.load(config_str) )
+  #
+  class Router
+    # config is a routing table as defined above
+    def initialize( routes )
+      @hosts = []
+      @default = nil
+      routes.each do|cfg|
+        raise "Configuration error missing host for #{cfg.inspect}" if !cfg[:host]
+        raise "Configuration error missing port for #{cfg.inspect}" if !cfg[:port]
+        raise "Configuration error missing match_url for #{cfg.inspect}" if !cfg[:match_url]
+        if cfg[:match_url] == 'default'
+          @default = cfg
+        else
+          raise "Configuration error missing match_url for #{cfg.inspect}" if !cfg[:match_url]
+          @hosts << cfg
+        end
+      end
+      @default = {:host => '127.0.0.1', :port => '3000'} unless @default
+    end
+    # return a uri given a request_uri
+    def url_for( request_uri )
+      return request_uri if request_uri.match(/http:\/\//)
+      # locate the first entry to match the given uri
+      config = @hosts.find do |cfg|
+        Regexp.new(cfg[:match_url]).match(request_uri)
+      end
+      config = @default unless config
+      # build a url given a valid config or abort 404 not found
+      "http://" + (config[:host] + ":" + (config[:port] || "").to_s + "/" + request_uri).squeeze("/")
+    end
+    def cache_ttl( request_uri )
+      return 600 if request_uri.match(/http:\/\//) # default cache for absolute URLs
+      # locate the first entry to match the given uri
+      config = @hosts.find do |cfg|
+        Regexp.new(cfg[:match_url]).match(request_uri)
+      end
+      config = @default unless config
+      (config[:cache_ttl] || 600).to_i
+    end
+  end
+end

data/lib/esi/tag/attempt.rb ADDED Viewed

@@ -0,0 +1,6 @@
+module ESI
+  module Tag
+    class Attempt < Base
+    end
+  end
+end

data/lib/esi/tag/base.rb ADDED Viewed

@@ -0,0 +1,85 @@
+require 'esi/logger'
+module ESI
+  module Tag
+    class Base
+      Validate = ['try','attempt','except','include','invalidate']
+      attr_reader :attributes, :children, :name, :closed, :cache
+      include ESI::Log
+      extend ESI::Log
+      def initialize(router,headers,http_params,name,attrs,cache)
+        @router = router
+        @headers = headers
+        @http_params = http_params
+        @attributes = attrs
+        @cache = cache
+        @children = []
+        @name = name
+        @unclosed = nil
+        @closed = false
+      end
+      def self.create(router,headers,http_params,tag_name,attrs,cache)
+        tag_name.gsub!(/esi:/,'')
+        raise "Unsupport ESI tag: #{tag_name}" unless Validate.include?(tag_name)
+        eval(tag_name.capitalize).new(router,headers,http_params,tag_name,attrs,cache)
+      rescue => e
+        log_debug "Failed while creating tag: #{tag_name}, with error: #{e.message}"
+        raise e
+      end
+      def buffer( output, inner_html )
+        if output.respond_to?("<<")
+          output << inner_html
+        else
+          output.call inner_html
+        end
+      end
+      def close( output, options = {} )
+        @closed = true
+      end
+      def close_child( output, name )
+        name = name.gsub(/esi:/,'')
+        if @unclosed and @unclosed.name == name
+          @unclosed.close( output )
+          @unclosed = nil
+        end
+      end
+      def add_child(tag)
+        if @unclosed
+          @unclosed.add_child(tag)
+        else
+          @unclosed = tag
+          @children << tag
+        end
+      end
+      # :startdoc:
+      # scans the fragment url, replacing occurrances of $(VAR{ ..  with the given value read from
+      # the HTTP Request object
+      # :enddoc:
+      def prepare_url_vars(url)
+        # scan url  for $(VAR{
+        # using the regex look for url vars that get set via http params
+        # for each var extract the value and store it in the operations array
+        operations = []
+        url.scan(/\$\([0-9a-zA-Z_{}]+\)/x) do|var|
+          # extract the var name
+          name = var.gsub(/\$\(/,'').gsub(/\{.*$/,'')
+          key = var.gsub(/^.*\{/,'').gsub(/\}.*$/,'')
+          value = Mongrel::HttpRequest.query_parse(@headers[name])[key]
+          operations << {:sub => var, :with => (value||"")}
+        end
+        # apply each operation to the url
+        operations.each { |op| url.gsub!(op[:sub],op[:with]) }
+        url
+      end
+    end
+  end
+end

data/lib/esi/tag/except.rb ADDED Viewed

@@ -0,0 +1,24 @@
+module ESI
+  module Tag
+    class Except < Base
+      attr_reader :buffers
+      def initialize(uri,headers,http_params,name,attrs,cache)
+        super
+        @buffers = [] # buffer output since this may only appear if the attempt block fails first
+        @buffer_index = 0
+        @buffers[@buffer_index] = ""
+      end
+      def add_child(tag)
+        super(tag)
+        @buffer_index += 1
+        @buffers[@buffer_index] = ""
+      end
+      def buffer( output, inner_html )
+        @buffers[@buffer_index] << inner_html
+      end
+    end
+  end
+end

data/lib/esi/tag/include.rb ADDED Viewed

@@ -0,0 +1,190 @@
+require 'uri'
+require 'net/http'
+module ESI
+  module Tag
+    #
+    #
+    #  ir = IncludeRequest.new( {'header1'=>'value1'} )
+    #
+    #  ir.request( '/fragment' ) do|status,response|
+    #     if status
+    #         response.read_body do|str|
+    #         end
+    #     else
+    #         # error case
+    #     end
+    #  end
+    #
+    class IncludeRequest
+      class Error
+        attr_reader :message, :response
+        def initialize(msg,response)
+          @message = msg
+          @response = response
+        end
+      end
+      attr_reader :exception, :overflow_index # TODO
+      def initialize(forward_headers)
+        @headers = forward_headers
+      end
+      def request(uri, timeout = 1, alt_failover=nil, follow_limit=3)
+        uri = URI.parse(uri) if uri.is_a?(String)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.read_timeout = timeout
+        http.get2( uri.to_s, @headers ) do|response|
+          case response
+          when Net::HTTPSuccess
+            yield true, response, uri
+          when Net::HTTPRedirection
+            ir = IncludeRequest.new(@headers)
+            ir.request(URI.parse(response['location']), timeout, alt_failover, follow_limit - 1) do|s,r|
+              yield s, r, URI.parse(response['location'])
+            end
+          else
+            if alt_failover
+              ir = IncludeRequest.new(@headers)
+              ir.request(alt_failover, timeout, nil, follow_limit) do|s,r|
+                yield s, r, URI.parse(alt_failover)
+              end
+            else
+              yield false, Error.new("Failed to request fragment: #{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}", response), uri
+            end
+          end
+        end
+      rescue Timeout::Error => e
+        yield false, Error.new("Failed to request fragment: #{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}, timeout error: #{e.message}", nil), uri
+      end
+    end
+    class Include < Base
+      attr_accessor :depth, :max_depth
+      def initialize(uri,headers,http_params,name,attrs,cache)
+        super
+        @depth = 0
+        @max_depth = 3
+      end
+      def parse_fragment?
+        @depth <= @max_depth
+      end
+      def close( output, options = {} )
+        super(output)
+        raise_on_error = options[:raise] || false
+        ir = IncludeRequest.new(@http_params)
+        src = @router.url_for(prepare_url_vars(@attributes["src"]))
+        alt = @attributes['alt']
+        alt = @router.url_for(prepare_url_vars(alt)) if alt
+        parser = nil
+        if parse_fragment?
+          parser = ESI::CParser.new
+          parser.output = output
+          parser.depth = (@depth+1)
+          # NOTE: really bad things happen if we attempt to copy the closure from the main parser
+          # in esi/handler.rb
+          # handle start tags
+          parser.start_tag_handler do|tag_name, attrs|
+            tag = ESI::Tag::Base.create( @router,
+                                         @headers,
+                                         @http_params,
+                                         tag_name.gsub(/esi:/,''),
+                                         attrs,
+                                         @cache )
+            # set the tag depth
+            tag.depth = (@depth+1) if tag.respond_to?(:depth=)
+            tag.max_depth = @max_depth if tag.respond_to?(:max_depth=)
+            if parser.esi_tag
+              parser.esi_tag.add_child(tag)
+            else
+              parser.esi_tag = tag
+            end
+          end
+          # handle end tags
+          parser.end_tag_handler do|tag_name|
+            if parser.esi_tag.name == tag_name.gsub(/esi:/,'')
+              parser.esi_tag.close(parser.output)
+              parser.esi_tag = nil
+            else
+              parser.esi_tag.close_child(parser.output,tag_name)
+            end
+          end
+        end
+        if @cache.cached?( src, @http_params )
+          cached_fragment = @cache.get( src, @headers ).body
+          log_request "C"
+          if parse_fragment?
+            parser.process cached_fragment
+            parser.finish
+          else
+            output << cached_fragment
+          end
+        else
+          ir.request(src, @attributes['timeout'].to_i, alt ) do|status,response,uri|
+            if status
+              # NOTE: it's important that we cache the unprocessed markup, because we need to
+              # reprocess the esi:include vars even for cached content, this way we can have cached content
+              # with HTTP_COOKIE vars and avoid re-requesting content
+              log_request "R"
+              cache_buffer = ""
+              response.read_body do|s|
+                cache_buffer << s
+                if parse_fragment?
+                  parser.process s
+                else
+                  output << s
+                end
+              end
+              parser.finish if parse_fragment?
+              if src != uri # these won't be equal if the fragment followed a redirect or used the alt condition
+                if uri.query
+                  request_uri = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}?#{uri.query}"
+                else
+                  request_uri =  "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
+                end
+              else
+                request_uri = src
+              end
+              @cache.put(request_uri, @http_params, 600, cache_buffer )
+            else
+              # error/ check if the include has an onerror specifier
+              return if @attributes['onerror'] == 'continue'
+              # response is an IncludeRequest::Error
+              raise response.message if raise_on_error
+              # stop processing and return the error object
+              return response
+            end
+          end
+        end
+        parser = nil
+      end
+    end
+  end
+end