RubyGems - mongrel_esi - Versions diffs - 0.4.0 - Mend

mongrel_esi 0.4.0

Files changed (73) hide show

data/COPYING +53 -0
data/LICENSE +471 -0
data/README +186 -0
data/Rakefile +141 -0
data/bin/mongrel_esi +271 -0
data/ext/esi/common.rl +41 -0
data/ext/esi/esi_parser.c +387 -0
data/ext/esi/extconf.rb +6 -0
data/ext/esi/machine.rb +499 -0
data/ext/esi/parser.c +1675 -0
data/ext/esi/parser.h +113 -0
data/ext/esi/parser.rb +49 -0
data/ext/esi/parser.rl +398 -0
data/ext/esi/ruby_esi.rl +135 -0
data/ext/esi/run-test.rb +3 -0
data/ext/esi/test/common.rl +41 -0
data/ext/esi/test/parser.c +1676 -0
data/ext/esi/test/parser.h +113 -0
data/ext/esi/test/parser.rl +398 -0
data/ext/esi/test/test.c +373 -0
data/ext/esi/test1.rb +56 -0
data/ext/esi/test2.rb +45 -0
data/lib/esi/cache.rb +207 -0
data/lib/esi/config.rb +154 -0
data/lib/esi/dispatcher.rb +27 -0
data/lib/esi/handler.rb +236 -0
data/lib/esi/invalidator.rb +40 -0
data/lib/esi/logger.rb +46 -0
data/lib/esi/router.rb +84 -0
data/lib/esi/tag/attempt.rb +6 -0
data/lib/esi/tag/base.rb +85 -0
data/lib/esi/tag/except.rb +24 -0
data/lib/esi/tag/include.rb +190 -0
data/lib/esi/tag/invalidate.rb +54 -0
data/lib/esi/tag/try.rb +40 -0
data/lib/multi_dirhandler.rb +70 -0
data/setup.rb +1585 -0
data/test/integration/basic_test.rb +39 -0
data/test/integration/cache_test.rb +37 -0
data/test/integration/docs/content/500.html +16 -0
data/test/integration/docs/content/500_with_failover.html +16 -0
data/test/integration/docs/content/500_with_failover_to_alt.html +8 -0
data/test/integration/docs/content/ajax_test_page.html +15 -0
data/test/integration/docs/content/cookie_variable.html +3 -0
data/test/integration/docs/content/foo.html +15 -0
data/test/integration/docs/content/include_in_include.html +15 -0
data/test/integration/docs/content/malformed_transforms.html +16 -0
data/test/integration/docs/content/malformed_transforms.html-correct +11 -0
data/test/integration/docs/content/static-failover.html +20 -0
data/test/integration/docs/content/test2.html +1 -0
data/test/integration/docs/content/test3.html +17 -0
data/test/integration/docs/esi_invalidate.html +6 -0
data/test/integration/docs/esi_mixed_content.html +15 -0
data/test/integration/docs/esi_test_content.html +27 -0
data/test/integration/docs/index.html +688 -0
data/test/integration/docs/test1.html +1 -0
data/test/integration/docs/test3.html +9 -0
data/test/integration/docs/test_failover.html +1 -0
data/test/integration/handler_test.rb +270 -0
data/test/integration/help.rb +234 -0
data/test/net/get_test.rb +197 -0
data/test/net/net_helper.rb +16 -0
data/test/net/server_test.rb +249 -0
data/test/unit/base_tag_test.rb +44 -0
data/test/unit/esi-sample.html +56 -0
data/test/unit/help.rb +77 -0
data/test/unit/include_request_test.rb +69 -0
data/test/unit/include_tag_test.rb +14 -0
data/test/unit/parser_test.rb +478 -0
data/test/unit/router_test.rb +34 -0
data/test/unit/sample.html +21 -0
data/tools/rakehelp.rb +119 -0
metadata +182 -0

data/lib/esi/invalidator.rb ADDED Viewed

@@ -0,0 +1,40 @@
+# this is a separate thread that runs on port 4001
+# when requests come to this they must authenticate
+# and be of type POST
+# the requests are processed as described by: http://www.w3.org/TR/esi-invp
+require 'webrick'
+module ESI
+  module Invalidator
+    def self.start( cache )
+      Thread.new( cache ) do|cache|
+        s = WEBrick::HTTPServer.new( :Port => 4001 )
+        s.mount_proc("/invalidate"){|req, res|
+          res.body = "<html>invalidate posted objects</html>"
+          res['Content-Type'] = "text/html"
+        }
+        s.mount_proc("/status"){|req, res|
+          res.body = "<html><body><h1>Cached objects</h1>"
+          res.body << "<ul>"
+          cache.keys do|key,data|
+            res.body << "<li>#{key}</li>"
+          end
+          res.body << "</ul>"
+          res.body << "</body>"
+          res.body << "</html>"
+          res['Content-Type'] = "text/html"
+        }
+        # XXX: this doesn't chain so ends up removing the mongrel trap locking the server up
+        #trap("INT"){ s.shutdown }
+        s.start
+      end
+    end
+  end
+end

data/lib/esi/logger.rb ADDED Viewed

@@ -0,0 +1,46 @@
+require 'logger'
+module ESI
+  module Log
+    if ENV["test"] == "true"
+      $logger ||= Logger.new("log/test.log")
+      $logger.instance_eval do
+        def puts( msg )
+          debug( msg )
+        end
+        def print( msg )
+          debug( msg )
+        end
+      end
+    end
+    def log( io, msg )
+      io.puts msg
+    end
+    def msg( io, msg )
+      io.print msg
+    end
+    def log_request( msg )
+      msg( $logger || STDERR, msg )
+    end
+    def log_debug( msg )
+      log( $logger || STDERR, msg )
+    end
+    def log_error( msg )
+      log( $logger || STDERR, msg )
+    end
+    def log_info( msg )
+      log( $logger || STDERR, msg )
+    end
+    def log_warn( msg )
+      log( $logger || STDERR, msg )
+    end
+  end
+end

data/lib/esi/router.rb ADDED Viewed

@@ -0,0 +1,84 @@
+module ESI
+  # ESI Server is a reverse proxy caching server.  It will forward all requests to app servers
+  # by processing a config_file that specifies routes:
+  #
+  # content:
+  #   host: 127.0.0.1
+  #   port: 3000
+  #   match_url: ^\/(content|samples|extra).*
+  # default:
+  #   host: 127.0.0.1
+  #   port: 3001
+  #
+  # This sample configuration will route all urls starting with /content, /samples, or /extras
+  # to the server running at 127.0.0.1 on port 3000
+  # everything else that matches the .* will be routed to the server running on port 3001
+  # optionally the caching duration can be specificied explicity for each host, this will be the default per host
+  # if the esi:include tag does not specify otherwise
+  #
+  # default:
+  #   host: 127.0.01
+  #   port: 3001
+  #   cache_ttl: 300
+  #
+  # This example will cache all requests for 300 seconds, by default.
+  #
+  # To create a router load either from memory or file the above YAML
+  #
+  #   router = ESI::Router.new( YAML.load_file('config.yml') )
+  #
+  # or from memory
+  #
+  #   router = ESI::Router.new( YAML.load(config_str) )
+  #
+  class Router
+    # config is a routing table as defined above
+    def initialize( routes )
+      @hosts = []
+      @default = nil
+      routes.each do|cfg|
+        raise "Configuration error missing host for #{cfg.inspect}" if !cfg[:host]
+        raise "Configuration error missing port for #{cfg.inspect}" if !cfg[:port]
+        raise "Configuration error missing match_url for #{cfg.inspect}" if !cfg[:match_url]
+        if cfg[:match_url] == 'default'
+          @default = cfg
+        else
+          raise "Configuration error missing match_url for #{cfg.inspect}" if !cfg[:match_url]
+          @hosts << cfg
+        end
+      end
+      @default = {:host => '127.0.0.1', :port => '3000'} unless @default
+    end
+    # return a uri given a request_uri
+    def url_for( request_uri )
+      return request_uri if request_uri.match(/http:\/\//)
+      # locate the first entry to match the given uri
+      config = @hosts.find do |cfg|
+        Regexp.new(cfg[:match_url]).match(request_uri)
+      end
+      config = @default unless config
+      # build a url given a valid config or abort 404 not found
+      "http://" + (config[:host] + ":" + (config[:port] || "").to_s + "/" + request_uri).squeeze("/")
+    end
+    def cache_ttl( request_uri )
+      return 600 if request_uri.match(/http:\/\//) # default cache for absolute URLs
+      # locate the first entry to match the given uri
+      config = @hosts.find do |cfg|
+        Regexp.new(cfg[:match_url]).match(request_uri)
+      end
+      config = @default unless config
+      (config[:cache_ttl] || 600).to_i
+    end
+  end
+end

data/lib/esi/tag/attempt.rb ADDED Viewed

@@ -0,0 +1,6 @@
+module ESI
+  module Tag
+    class Attempt < Base
+    end
+  end
+end

data/lib/esi/tag/base.rb ADDED Viewed

@@ -0,0 +1,85 @@
+require 'esi/logger'
+module ESI
+  module Tag
+    class Base
+      Validate = ['try','attempt','except','include','invalidate']
+      attr_reader :attributes, :children, :name, :closed, :cache
+      include ESI::Log
+      extend ESI::Log
+      def initialize(router,headers,http_params,name,attrs,cache)
+        @router = router
+        @headers = headers
+        @http_params = http_params
+        @attributes = attrs
+        @cache = cache
+        @children = []
+        @name = name
+        @unclosed = nil
+        @closed = false
+      end
+      def self.create(router,headers,http_params,tag_name,attrs,cache)
+        tag_name.gsub!(/esi:/,'')
+        raise "Unsupport ESI tag: #{tag_name}" unless Validate.include?(tag_name)
+        eval(tag_name.capitalize).new(router,headers,http_params,tag_name,attrs,cache)
+      rescue => e
+        log_debug "Failed while creating tag: #{tag_name}, with error: #{e.message}"
+        raise e
+      end
+      def buffer( output, inner_html )
+        if output.respond_to?("<<")
+          output << inner_html
+        else
+          output.call inner_html
+        end
+      end
+      def close( output, options = {} )
+        @closed = true
+      end
+      def close_child( output, name )
+        name = name.gsub(/esi:/,'')
+        if @unclosed and @unclosed.name == name
+          @unclosed.close( output )
+          @unclosed = nil
+        end
+      end
+      def add_child(tag)
+        if @unclosed
+          @unclosed.add_child(tag)
+        else
+          @unclosed = tag
+          @children << tag
+        end
+      end
+      # :startdoc:
+      # scans the fragment url, replacing occurrances of $(VAR{ ..  with the given value read from
+      # the HTTP Request object
+      # :enddoc:
+      def prepare_url_vars(url)
+        # scan url  for $(VAR{
+        # using the regex look for url vars that get set via http params
+        # for each var extract the value and store it in the operations array
+        operations = []
+        url.scan(/\$\([0-9a-zA-Z_{}]+\)/x) do|var|
+          # extract the var name
+          name = var.gsub(/\$\(/,'').gsub(/\{.*$/,'')
+          key = var.gsub(/^.*\{/,'').gsub(/\}.*$/,'')
+          value = Mongrel::HttpRequest.query_parse(@headers[name])[key]
+          operations << {:sub => var, :with => (value||"")}
+        end
+        # apply each operation to the url
+        operations.each { |op| url.gsub!(op[:sub],op[:with]) }
+        url
+      end
+    end
+  end
+end

data/lib/esi/tag/except.rb ADDED Viewed

@@ -0,0 +1,24 @@
+module ESI
+  module Tag
+    class Except < Base
+      attr_reader :buffers
+      def initialize(uri,headers,http_params,name,attrs,cache)
+        super
+        @buffers = [] # buffer output since this may only appear if the attempt block fails first
+        @buffer_index = 0
+        @buffers[@buffer_index] = ""
+      end
+      def add_child(tag)
+        super(tag)
+        @buffer_index += 1
+        @buffers[@buffer_index] = ""
+      end
+      def buffer( output, inner_html )
+        @buffers[@buffer_index] << inner_html
+      end
+    end
+  end
+end

data/lib/esi/tag/include.rb ADDED Viewed

@@ -0,0 +1,190 @@
+require 'uri'
+require 'net/http'
+module ESI
+  module Tag
+    #
+    #
+    #  ir = IncludeRequest.new( {'header1'=>'value1'} )
+    #
+    #  ir.request( '/fragment' ) do|status,response|
+    #     if status
+    #         response.read_body do|str|
+    #         end
+    #     else
+    #         # error case
+    #     end
+    #  end
+    #
+    class IncludeRequest
+      class Error
+        attr_reader :message, :response
+        def initialize(msg,response)
+          @message = msg
+          @response = response
+        end
+      end
+      attr_reader :exception, :overflow_index # TODO
+      def initialize(forward_headers)
+        @headers = forward_headers
+      end
+      def request(uri, timeout = 1, alt_failover=nil, follow_limit=3)
+        uri = URI.parse(uri) if uri.is_a?(String)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.read_timeout = timeout
+        http.get2( uri.to_s, @headers ) do|response|
+          case response
+          when Net::HTTPSuccess
+            yield true, response, uri
+          when Net::HTTPRedirection
+            ir = IncludeRequest.new(@headers)
+            ir.request(URI.parse(response['location']), timeout, alt_failover, follow_limit - 1) do|s,r|
+              yield s, r, URI.parse(response['location'])
+            end
+          else
+            if alt_failover
+              ir = IncludeRequest.new(@headers)
+              ir.request(alt_failover, timeout, nil, follow_limit) do|s,r|
+                yield s, r, URI.parse(alt_failover)
+              end
+            else
+              yield false, Error.new("Failed to request fragment: #{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}", response), uri
+            end
+          end
+        end
+      rescue Timeout::Error => e
+        yield false, Error.new("Failed to request fragment: #{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}, timeout error: #{e.message}", nil), uri
+      end
+    end
+    class Include < Base
+      attr_accessor :depth, :max_depth
+      def initialize(uri,headers,http_params,name,attrs,cache)
+        super
+        @depth = 0
+        @max_depth = 3
+      end
+      def parse_fragment?
+        @depth <= @max_depth
+      end
+      def close( output, options = {} )
+        super(output)
+        raise_on_error = options[:raise] || false
+        ir = IncludeRequest.new(@http_params)
+        src = @router.url_for(prepare_url_vars(@attributes["src"]))
+        alt = @attributes['alt']
+        alt = @router.url_for(prepare_url_vars(alt)) if alt
+        parser = nil
+        if parse_fragment?
+          parser = ESI::CParser.new
+          parser.output = output
+          parser.depth = (@depth+1)
+          # NOTE: really bad things happen if we attempt to copy the closure from the main parser
+          # in esi/handler.rb
+          # handle start tags
+          parser.start_tag_handler do|tag_name, attrs|
+            tag = ESI::Tag::Base.create( @router,
+                                         @headers,
+                                         @http_params,
+                                         tag_name.gsub(/esi:/,''),
+                                         attrs,
+                                         @cache )
+            # set the tag depth
+            tag.depth = (@depth+1) if tag.respond_to?(:depth=)
+            tag.max_depth = @max_depth if tag.respond_to?(:max_depth=)
+            if parser.esi_tag
+              parser.esi_tag.add_child(tag)
+            else
+              parser.esi_tag = tag
+            end
+          end
+          # handle end tags
+          parser.end_tag_handler do|tag_name|
+            if parser.esi_tag.name == tag_name.gsub(/esi:/,'')
+              parser.esi_tag.close(parser.output)
+              parser.esi_tag = nil
+            else
+              parser.esi_tag.close_child(parser.output,tag_name)
+            end
+          end
+        end
+        if @cache.cached?( src, @http_params )
+          cached_fragment = @cache.get( src, @headers ).body
+          log_request "C"
+          if parse_fragment?
+            parser.process cached_fragment
+            parser.finish
+          else
+            output << cached_fragment
+          end
+        else
+          ir.request(src, @attributes['timeout'].to_i, alt ) do|status,response,uri|
+            if status
+              # NOTE: it's important that we cache the unprocessed markup, because we need to
+              # reprocess the esi:include vars even for cached content, this way we can have cached content
+              # with HTTP_COOKIE vars and avoid re-requesting content
+              log_request "R"
+              cache_buffer = ""
+              response.read_body do|s|
+                cache_buffer << s
+                if parse_fragment?
+                  parser.process s
+                else
+                  output << s
+                end
+              end
+              parser.finish if parse_fragment?
+              if src != uri # these won't be equal if the fragment followed a redirect or used the alt condition
+                if uri.query
+                  request_uri = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}?#{uri.query}"
+                else
+                  request_uri =  "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
+                end
+              else
+                request_uri = src
+              end
+              @cache.put(request_uri, @http_params, 600, cache_buffer )
+            else
+              # error/ check if the include has an onerror specifier
+              return if @attributes['onerror'] == 'continue'
+              # response is an IncludeRequest::Error
+              raise response.message if raise_on_error
+              # stop processing and return the error object
+              return response
+            end
+          end
+        end
+        parser = nil
+      end
+    end
+  end
+end