mongrel_esi 0.4.1 → 0.5.0

data/ext/esi/test/test.c

@@ -54,7 +54,7 @@ static int verify_string( const char *str, const char *str_value, int line, cons
   int str_value_length = strlen( str_value ); 
 
   if( str_len != str_value_length || strcmp( str, str_value ) ){
-    printf( "Strings are not equal\n\"%s\":%d\n!=\n\"%s\":%d\nat %s:%d\n", str, str_len, str_value, str_value_length, test_name, line );
+    printf( "Strings are not equal\n\t\"%s\":%d\n!=\t\n\"%s\":%d\nat\n%s:%d\n", str, str_len, str_value, str_value_length, test_name, line );
     return 1;
   }
   return 0;
@@ -139,7 +139,6 @@ static void start_tag_handler( const void *data,
                                ESIAttribute *attributes,
                                void *user_data )
 {
-  ESIAttribute *attr = attributes;
   ++start_tag_count;
   detected_start_tags = add_detected_tag( detected_start_tags, tag_info_new( name_start, name_length, attributes ) );
 }
@@ -244,7 +243,7 @@ static void test_simple_parser_input( ESIParser *parser )
 {
   TEST_INIT
 
-  feed_data( parser, "<p>some input</p><esi:include/>some more input\nsome input<esi:include timeout='10' src='hello'/>some more input" );
+  feed_data( parser, "<p>some input</p><esi:include />some more input\nsome input<esi:include timeout='10' src='hello'/>some more input" );
 
   TEST_PREPARE_ASSERTIONS
 
@@ -304,7 +303,17 @@ static void test_sample_input( ESIParser *parser )
   ASSERT_MATCH("    <div>some content</div>", output);
   ASSERT_MATCH("<em>Support for em tags since they have an initial start sequence similar to and &lt;esi: start/end sequence</em>", output );
   ASSERT_NO_MATCH("<esi:", output);
+/*
+  TagInfo *ptr = start_tags;
+
+  while( ptr  ){ 
+    tag_info_show( ptr );
+    ptr = ptr->next;
+  }
 
+  printf( "start_tag_count: %d\n", start_tag_count );
+  printf( "end_tag_count: %d\n", end_tag_count );
+*/
   ASSERT_TRUE( start_tag_count == 13 );
   ASSERT_TRUE( end_tag_count == 13 );
 
@@ -352,7 +361,7 @@ static void test_large_file( ESIParser *parser )
 
   TEST_FINISH
 }
-
+  
 int main( int argc, char **argv )
 {
   ESIParser *parser = esi_parser_new();

data/ext/esi/test1.rb

@@ -4,13 +4,16 @@ output = ""
 
 p = ESI::CParser.new
 
+start_tags = []
+end_tags = []
+
 
 p.start_tag_handler do|tag_name, attrs|
-  puts "Start: #{tag_name} #{attrs.inspect}"
+  start_tags << {:name => tag_name, :attributes => attrs }
 end
 
 p.end_tag_handler do|tag_name|
-  puts "End: #{tag_name}"
+  end_tags << tag_name
 end
 
 p.output_handler do|data|
@@ -28,7 +31,36 @@ some inputsome more inputsome inputsome more input
 some inputsome more input<p>some input</p>some more input
 some inputsome more input</body></html>) 
 
-if( expected !=  output )
+expected_start_tags = [
+  {:name => 'esi:include', :attributes => {"src"=>"hello", "max-age"=>"600+600", "timeout"=>"1"} },
+  {:name => 'esi:include', :attributes => {"src"=>"hello", "max-age"=>"600+600", "timeout"=>"1"} },
+  {:name => 'esi:include', :attributes => {"src"=>"hello", "max-age"=>"600+600", "timeout"=>"1"} },
+  {:name => 'esi:inline',  :attributes => {"src"=>"hello", "max-age"=>"600+600", "timeout"=>"1"} },
+  {:name => 'esi:comment', :attributes => {"text"=>"hello", "src"=>"hello", "max-age"=>"600+600", "timeout"=>"1"} },
+  {:name => 'esi:include', :attributes => {"text"=>"hello", "src"=>"hello", "max-age"=>"600+600", "timeout"=>"1"} },
+  {:name => 'esi:include', :attributes => {"text"=>"hello", "src"=>"hello", "max-age"=>"600+600", "timeout"=>"1"} }
+]
+expected_end_tags =[ 'esi:include', 'esi:include', 'esi:include', 'esi:inline', 'esi:comment', 'esi:include', 'esi:include']
+
+if( start_tags.size != expected_start_tags.size and end_tags.size != expected_end_tags.size )
+  puts "Failed expected start tags: #{expected_start_tags.size}, expected end tags: #{expected_end_tags.size}"
+  puts "Actual: start tags: #{start_tags.size}, end tags: #{end_tags.size}"
+  exit(1)
+end
+
+if expected_start_tags != start_tags
+  puts "Failed expected start tags: #{expected_start_tags.inspect}"
+  puts "Actual start tags: #{start_tags.inspect}"
+  exit(1)
+end
+
+if expected_end_tags != end_tags
+  puts "Failed expected end tags: #{expected_end_tags.inspect}"
+  puts "Actual end tags: #{end_tags.inspect}"
+  exit(1)
+end
+
+if( expected != output )
   puts "Failed output was different from the expected"
   puts "Expected: #{expected}"
   puts "\n"

data/lib/esi/cache.rb

@@ -1,3 +1,5 @@
+# Copyright (c) 2008 Todd A. Fisher
+# see LICENSE
 require 'digest/sha1'
 require 'thread'
 require 'ostruct'
@@ -12,7 +14,6 @@ module ESI
 
     def initialize(uri,max_age,body)
       @uri = uri
-      max_age.gsub!(/\+.*$/,'') if max_age.class == String
       @max_age = Time.now.to_i + max_age.to_i
       @body = body
     end

data/lib/esi/config.rb

@@ -1,6 +1,43 @@
+# Copyright (c) 2008 Todd A. Fisher
+# see LICENSE
 require 'ostruct'
 
 module ESI
+  # This file provides configuration options to mongrel-esi
+  # Mongrel ESI is a proxy caching server with limited load balancing capablities
+  #
+  #  ESI::Config.define(listeners) do|config|
+  #  
+  #    # define the caching rules globally for all routes, defaults to ruby
+  #    config.cache do|c|
+  #      c.memcached do|mc|
+  #        mc.servers = ['localhost:11211']  # memcahed servers
+  #        mc.namespace = 'mesi'  # namespace for cache storage
+  #      end
+  #      c.ttl = 600 # default fragment time to live, when <esi:include does not include the max-age attribute
+  #    end
+  #
+  #    # define rules for when to enable esi processing globally for all routes
+  #    # using content type it is more flexible, but sometimes you will want to be
+  #    # explicit about when to enable esi processing. For those cases, enable_for_surrogate_only will
+  #    # require the presense of the Surrogate-Control header to contain the content="ESI/1.0" line.
+  #    # see [http://www.w3.org/TR/edge-arch Edge Arch] for details.
+  #    config.esi do|c|
+  #      c.allowed_content_types = ['text/plain', 'text/html']
+  #      #c.enable_for_surrogate_only = true # default is false
+  #    end
+  #
+  #    # define request path routing rules, these rules match against request path to select a specific server
+  #    config.routes do|s|
+  #      #s.match( /content/ ) do|r|
+  #      #  r.servers = ['127.0.0.1:4000']
+  #      #end
+  #      s.default do|r|
+  #        r.servers = ['127.0.0.1:3000']
+  #      end
+  #    end
+  #
+  #  end
   class Config
 
     attr_reader :config

data/lib/esi/dispatcher.rb

@@ -1,9 +1,8 @@
-require 'esi/handler'
-require 'mongrel'
+require 'esi/proxy'
 
 module ESI
 
-  class Dispatcher < Mongrel::HttpHandler
+  class Dispatcher < ESI::Proxy
     attr_reader :config, :router
 
     Thread.abort_on_exception = false
@@ -19,8 +18,7 @@ module ESI
 
     def process(request, response)
       @router = @config.router
-      handler = ESI::Handler.new(self)
-      handler.process(request, response)
+      super
     end
 
   end

data/lib/esi/invalidator.rb

@@ -1,3 +1,6 @@
+# Copyright (c) 2008 Todd A. Fisher
+# see LICENSE
+
 # this is a separate thread that runs on port 4001
 # when requests come to this they must authenticate
 # and be of type POST

data/lib/esi/logger.rb

@@ -35,12 +35,5 @@ module ESI
       log( $logger || STDERR, msg )
     end
 
-    def log_info( msg )
-      log( $logger || STDERR, msg )
-    end
-
-    def log_warn( msg )
-      log( $logger || STDERR, msg )
-    end
   end
 end

data/lib/esi/parser.rb

@@ -0,0 +1,70 @@
+# Copyright (c) 2008 Todd A. Fisher
+# see LICENSE
+require 'esi/esi'
+require 'esi/tag/base'
+require 'esi/tag/include'
+require 'esi/tag/invalidate'
+require 'esi/tag/attempt'
+require 'esi/tag/except'
+require 'esi/tag/try'
+
+module ESI
+  class Parser < CParser
+    attr_reader :response
+
+    def initialize( output, router, cache, max_depth )
+      super()
+      @router = router
+      @cache = cache
+      @max_depth = max_depth
+      @response = ESI::Response.new( output )
+ 
+      end_tag_handler do|tag_name|
+        if self.esi_tag.name == tag_name.gsub(/esi:/,'')
+          @response.close_active_buffer
+          tag_buffer = @response.reserve_buffer
+          tag = self.esi_tag.clone
+          thread = Thread.new(tag,tag_buffer) do|tag,buffer|
+            begin
+              tag.close(tag_buffer)
+            ensure
+              tag_buffer.close_write
+            end
+          end
+          self.esi_tag = nil
+
+          @response.wait_thread( thread )
+        else
+          self.esi_tag.close_child(self.output,tag_name)
+        end
+      end
+    end
+
+    def prepare( request_params, http_params, &block )
+      start_tag_handler do|tag_name, attrs|
+        tag = ESI::Tag::Base.create( @router,
+                                     request_params,
+                                     http_params,
+                                     tag_name.gsub(/esi:/,''),
+                                     attrs,
+                                     @cache )
+        # set the tag depth
+        tag.depth = self.depth if tag.respond_to?(:depth=)
+        tag.max_depth = @max_depth if tag.respond_to?(:max_depth=)
+
+        if self.esi_tag
+          self.esi_tag.add_child(tag)
+        else
+          self.esi_tag = tag
+        end
+      end
+      self.output=block if block_given?
+    end
+
+    def finish
+      super
+      @response.flush
+    end
+
+  end
+end

data/lib/esi/processor.rb

@@ -0,0 +1,88 @@
+# Copyright (c) 2008 Todd A. Fisher
+require 'esi/parser'
+require 'esi/response'
+
+module ESI
+  class Processor
+    attr_reader :config, :router
+
+    include ESI::Log
+
+    def initialize( config, router )
+      @config = config
+      @router = router
+      @chunk_count = 0
+      @chunk_size = @config[:chunk_size] || 4096
+      @max_depth = @config[:max_depth] || 3
+      @chunk_buffer = StringIO.new # when buffer reaches chunk_size write to the http_response socket
+      @parser = ESI::Parser.new( @chunk_buffer, @router, @config.cache, @max_depth )
+    end
+    
+    def send_head( http_response, status )
+
+      http_response.header["Transfer-Encoding"] = "chunked"
+      # this is the important part, rather then send the whole document back we send in chunks
+      # each fragment is roughly in it's own chunk, this does mean we require http 1.1, chunk size is still a limit
+      header = Mongrel::Const::STATUS_FORMAT % [status, Mongrel::HTTP_STATUS_CODES[status]]
+      header.gsub!(/Connection: close\r\n/,'')
+      http_response.header.out.rewind
+      header << http_response.header.out.read + Mongrel::Const::LINE_END
+      header.gsub!(/Status:.*?\r\n/,'')
+      http_response.write( header )
+
+      #print header
+
+    end
+
+    def send_body( http_request, params, http_response, proxy_response )
+      
+      @http_response = http_response
+
+      # prepare the parser given the raw request params and the preprocessed request params
+      @parser.prepare( http_request.params, params ) do|chars|
+        if @parser.response.active_buffer.closed_write?
+          @parser.response.open_active_buffer
+          @parser.response.send
+        end
+        @parser.response.active_buffer << chars
+        if @chunk_buffer.size > @chunk_size
+          send_chunk
+          @chunk_buffer = StringIO.new
+          @parser.response.update_output( @chunk_buffer )
+        end
+      end
+
+      #timer = Time.now
+      # feed data to the parser
+      proxy_response.read_body { |data| @parser.process data }
+      #puts "processing time: #{Time.now - timer}"
+
+      #timer = Time.now
+      # finish the request
+      @parser.finish 
+      #puts "finish time: #{Time.now - timer}"
+
+      send_chunk
+
+      @chunk_count
+    rescue => e
+      STDERR.puts "\n#{e.message}: #{e.backtrace.join("\n")}\n"
+    ensure 
+      http_response.write( "0\r\n\r\n" )
+      http_response.done = true
+    end
+
+    def send_chunk
+      @chunk_buffer.rewind
+      size = @chunk_buffer.size
+      chunk_header = "#{"%x" % size}" + Mongrel::Const::LINE_END
+      #puts chunk_header.inspect
+      @http_response.write( chunk_header )  # write the chunk size
+      #puts buffer.inspect
+      @http_response.write( @chunk_buffer.read + Mongrel::Const::LINE_END )  # write the chunk
+
+      @chunk_count += 1
+    end
+
+  end
+end

data/lib/esi/proxy.rb

@@ -0,0 +1,104 @@
+# Copyright (c) 2008 Todd A. Fisher
+# see LICENSE
+require 'uri'
+require 'timeout'
+require 'net/http'
+require 'mongrel'
+require 'esi/logger'
+require 'esi/cache'
+require 'esi/config'
+require 'esi/router'
+require 'esi/processor'
+
+module ESI
+
+  class Proxy < Mongrel::HttpHandler
+    attr_reader :config, :router
+    include ESI::Log
+
+    def process(request, response)
+ 
+      start = Time.now
+      status = 200
+      url = @router.url_for(request.params["REQUEST_URI"])
+
+      params = http_params(request.params)
+
+      log_debug "#{request.params["REQUEST_METHOD"]} => #{url}"
+      chunk_count = 0
+      uri = URI.parse(url)
+
+      path_with_query = uri.query ? "#{uri.path}?#{uri.query}" :  uri.path
+
+      proxy_request = (request.params["REQUEST_METHOD"] == "POST") ?
+                          Net::HTTP::Post.new( path_with_query, params ) :
+                          Net::HTTP::Get.new( path_with_query, params )
+
+      proxy_connection = Net::HTTP.start(uri.host, uri.port)
+
+      # open the conneciton up so we can start to stream the connection
+      proxy_connection.request(proxy_request,request.body.read) do|proxy_response|
+
+        status = read_status( proxy_response )
+
+        copy_headers( response.header, proxy_response ) unless status >= 500
+
+        if status >= 500 or !@config.enable_esi_processor?( proxy_response )
+          response.start(status, true) do|head,out|
+
+            # proxy the 500 response
+            proxy_response.read_body do|fragment|
+              out << fragment
+            end
+
+          end
+        else
+          processor = Processor.new( @config, @router )
+          processor.send_head( response, status )
+          chunk_count = processor.send_body( request, params, response, proxy_response )
+        end
+      end # end request
+
+    rescue => e
+      STDERR.puts "\n#{e.message}: error at #{e.backtrace.first} msg at #{__FILE__}:#{__LINE__}\n"
+    ensure
+      log_request "\nCompleted => #{url}, #{Time.now - start} seconds with status #{status} and #{chunk_count} chunks\n"
+    end
+
+  protected
+
+    def read_status(response)
+      Net::HTTPResponse::CODE_TO_OBJ.select { |k,v| v == response.class }.first[0].to_i rescue 500
+    end
+
+    def http_params(params)
+      updated_params = {}
+      params.each do|k,v|
+        k = k.split('_').collect { |t| t.capitalize }.join('-')
+        if k[0,5] =='Http-'
+          k[0,5] = ''
+          updated_params[k] = v
+        end
+      end
+      updated_params
+    end
+
+    def copy_headers(head,response)
+      response.to_hash.each do |k,v|
+        # for Set-Cookie we need to split on ,
+        # some edge cases with , since things like expires might be a date with , in them.
+        k = k.split(/-/).map{|s| s.capitalize }.join('-')
+        if k == "Set-Cookie"
+          v.each do|cookie|
+            head["Set-Cookie"] = cookie.strip # mongrel is case sensitive about handling duplicates
+          end
+        else
+          head[k] = v unless k == "Content-Length" or k == "Surrogate-Control" or k == "Server"
+        end
+      end
+      head["Server"] = "MongrelESI 0.4"
+    end
+
+  end # Handler
+
+end # ESI