mongrel_esi 0.4.1 → 0.5.0

data/lib/esi/response.rb

@@ -0,0 +1,106 @@
+require 'fastthread'
+require 'thread'
+require 'stringio'
+
+
+module ESI
+  class Response
+    attr_reader :active_buffer
+    attr_accessor :output
+
+    def initialize( output )
+      @lock = Mutex.new
+      @count = 0
+      @back_buffer = []
+      @output = output
+      @last_out = -1
+      @threads = []
+      @active_buffer = reserve_buffer 
+    end
+
+    def update_output(output)
+      @lock.synchronize do
+        @output = output
+      end
+    end
+
+    def close_active_buffer
+      @lock.synchronize do
+        if !@active_buffer.closed_write?
+          @active_buffer.close_write
+        end
+      end
+    end
+
+    def back_buffer_ready?
+      @lock.synchronize do
+        @back_buffer[@last_out+1..@count].size > 2
+      end
+    end
+
+    def open_active_buffer
+      @lock.synchronize do
+        if @active_buffer.closed_write?
+          begin
+            @lock.unlock
+            @active_buffer = reserve_buffer
+          ensure
+            @lock.lock
+          end
+        end
+      end
+    end
+
+    # return's new buffer
+    def reserve_buffer
+      @lock.synchronize do
+        buffer = @back_buffer[@count] = StringIO.new
+        @count += 1
+        buffer
+      end
+    end
+
+    def send
+      @lock.synchronize do
+        # roll up requests
+        until @last_out == @count
+          o = @back_buffer[@last_out+1]
+          if o.nil? or !o.closed_write?
+            #puts "#{self} buffering...#{@last_out} to #{@count} #{@back_buffer[@last_out+1..@count].map{|b| b ? b.closed_write? : 'nil' }.join(',')}"
+            break
+          end
+          o.rewind
+          buf = o.read
+          #puts "#{Thread.current} #{self} sending: #{@last_out+1} #{buf.inspect}"
+          @back_buffer[@last_out+1] = nil # clear it out, release memory
+          @output << buf
+          @last_out += 1
+        end
+      end
+    end
+
+    def wait_thread(thread)
+      @threads << thread
+    end
+
+    def flush
+      @threads.each{|t| t.join }
+      @active_buffer.close_write
+      # roll up requests
+      @last_out = 0 if @last_out == -1
+      tail_buffer = (@back_buffer[@last_out..@back_buffer.size]||[])
+      #puts "\nflushing:#{Thread.current} #{self} with #{tail_buffer.inspect}"
+      while !tail_buffer.empty?
+        o = tail_buffer.shift
+        unless o.nil?
+          o.rewind
+          buf = o.read
+          #puts "#{Thread.current} #{self} flush : #{buf.inspect}"
+          @output << buf
+        end
+        #puts "#{self} sending: #{@count-tail_buffer.size}"
+      end
+    end
+
+  end
+end

data/lib/esi/router.rb

@@ -1,3 +1,6 @@
+# Copyright (c) 2008 Todd A. Fisher
+# see LICENSE
+
 module ESI
  
   # ESI Server is a reverse proxy caching server.  It will forward all requests to app servers

data/lib/esi/tag/attempt.rb

@@ -1,3 +1,6 @@
+# Copyright (c) 2008 Todd A. Fisher
+# see LICENSE
+
 module ESI
   module Tag
     class Attempt < Base

data/lib/esi/tag/base.rb

@@ -1,3 +1,5 @@
+# Copyright (c) 2008 Todd A. Fisher
+# see LICENSE
 require 'esi/logger'
 
 module ESI
@@ -5,7 +7,7 @@ module ESI
     class Base
       Validate = ['try','attempt','except','include','invalidate']
       attr_reader :attributes, :children, :name, :closed, :cache
-      
+ 
       include ESI::Log
       extend ESI::Log
 

data/lib/esi/tag/except.rb

@@ -1,3 +1,5 @@
+# Copyright (c) 2008 Todd A. Fisher
+# see LICENSE
 module ESI
   module Tag
     class Except < Base

data/lib/esi/tag/include.rb

@@ -1,69 +1,73 @@
+# Copyright (c) 2008 Todd A. Fisher
+# see LICENSE
 require 'uri'
 require 'net/http'
+require 'esi/parser'
 
 module ESI
   module Tag
-
-    #
-    #
-    #  ir = IncludeRequest.new( {'header1'=>'value1'} )
-    #
-    #  ir.request( '/fragment' ) do|status,response|
-    #     if status 
-    #         response.read_body do|str|
-    #         end
-    #     else
-    #         # error case
-    #     end
-    #  end
-    #
-    class IncludeRequest
-      class Error
-        attr_reader :message, :response
-        def initialize(msg,response)
-          @message = msg
-          @response = response
+    class Include < Base
+      #
+      #
+      #  ir = Request.new( {'header1'=>'value1'} )
+      #
+      #  ir.request( '/fragment' ) do|status,response|
+      #     if status 
+      #         response.read_body do|str|
+      #         end
+      #     else
+      #         # error case
+      #     end
+      #  end
+      #
+      class Request
+        class Error
+          attr_reader :message, :response
+          def initialize(msg,response)
+            @message = msg
+            @response = response
+          end
         end
-      end
-      attr_reader :exception, :overflow_index # TODO
+        attr_reader :exception, :overflow_index # TODO
 
-      def initialize(forward_headers)
-        @headers = forward_headers
-      end
+        def initialize(forward_headers)
+          @headers = forward_headers
+        end
 
-      def request(uri, timeout = 1, alt_failover=nil, follow_limit=3)
-        uri = URI.parse(uri) if uri.is_a?(String)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.read_timeout = timeout
-        http.get2( uri.to_s, @headers ) do|response|
-          case response
-          when Net::HTTPSuccess
-            yield true, response, uri
-          when Net::HTTPRedirection
-            ir = IncludeRequest.new(@headers)
-            ir.request(URI.parse(response['location']), timeout, alt_failover, follow_limit - 1) do|s,r|
-              yield s, r, URI.parse(response['location'])
-            end
-          else
-            if alt_failover
-              ir = IncludeRequest.new(@headers)
-              ir.request(alt_failover, timeout, nil, follow_limit) do|s,r|
-                yield s, r, URI.parse(alt_failover)
+        def request(uri, timeout = 1, alt_failover=nil, follow_limit=3)
+          uri = URI.parse(uri) if uri.is_a?(String)
+          Net::HTTP.start(uri.host, uri.port) do|http|
+            http.read_timeout = timeout if timeout and timeout > 0
+            rp = uri.query ? "#{uri.path}?#{uri.query}" : uri.path
+            http.request_get( rp, @headers ) do|response|
+              case response
+              when Net::HTTPSuccess
+                yield true, response, uri
+              when Net::HTTPRedirection
+                ir = Request.new(@headers)
+                ir.request(URI.parse(response['location']), timeout, alt_failover, follow_limit - 1) do|s,r|
+                  yield s, r, URI.parse(response['location'])
+                end
+              else
+                if alt_failover
+                  ir = Request.new(@headers)
+                  ir.request(alt_failover, timeout, nil, follow_limit) do|s,r|
+                    yield s, r, URI.parse(alt_failover)
+                  end
+                else
+                  yield false, Error.new("Failed to request fragment: #{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}", response), uri
+                end
               end
-            else
-              yield false, Error.new("Failed to request fragment: #{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}", response), uri
             end
           end
+        rescue Timeout::Error => e
+          yield false, Error.new("Failed to request fragment: #{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}, timeout error: #{e.message}", nil), uri
         end
-      rescue Timeout::Error => e
-        yield false, Error.new("Failed to request fragment: #{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}, timeout error: #{e.message}", nil), uri
-      end
 
-    end
-
-    class Include < Base
+      end
 
       attr_accessor :depth, :max_depth
+
       def initialize(uri,headers,http_params,name,attrs,cache)
         super
         @depth = 0
@@ -76,114 +80,109 @@ module ESI
 
       def close( output, options = {} )
         super(output)
+        @output = output
 
-        raise_on_error = options[:raise] || false
-        ir = IncludeRequest.new(@http_params)
+        raise_on_error = options[:raise] || false # default to false
 
         src = @router.url_for(prepare_url_vars(@attributes["src"]))
         alt = @attributes['alt']
         alt = @router.url_for(prepare_url_vars(alt)) if alt
 
-        parser = nil
+        prepare_parser
+
+        if @cache.cached?( src, @http_params )
+          send_from_cache( src )
+        else
+          send_from_surrogate( src, alt, raise_on_error )
+        end
+
+      end
 
+    protected
+      def prepare_parser
         if parse_fragment?
-          parser = ESI::CParser.new
-          parser.output = output
-          parser.depth = (@depth+1)
-
-          # NOTE: really bad things happen if we attempt to copy the closure from the main parser
-          # in esi/handler.rb
-
-          # handle start tags
-          parser.start_tag_handler do|tag_name, attrs|
-            tag = ESI::Tag::Base.create( @router,
-                                         @headers,
-                                         @http_params,
-                                         tag_name.gsub(/esi:/,''),
-                                         attrs,
-                                         @cache )
-            # set the tag depth
-            tag.depth = (@depth+1) if tag.respond_to?(:depth=)
-            tag.max_depth = @max_depth if tag.respond_to?(:max_depth=)
-
-            if parser.esi_tag
-              parser.esi_tag.add_child(tag)
-            else
-              parser.esi_tag = tag
+          @parser = ESI::Parser.new( @output, @router, @cache, @max_depth )
+          @parser.prepare( @headers, @http_params ) do|chars|
+            if @parser.response.active_buffer.closed_write?
+              @parser.response.open_active_buffer
+              @parser.response.send
             end
+            @parser.response.active_buffer << chars
           end
-          
-          # handle end tags
-          parser.end_tag_handler do|tag_name|
-            if parser.esi_tag.name == tag_name.gsub(/esi:/,'')
-              parser.esi_tag.close(parser.output)
-              parser.esi_tag = nil
-            else
-              parser.esi_tag.close_child(parser.output,tag_name)
-            end
-          end
-
+          @parser.depth = (@depth+1) # increment the depth
         end
+      end
 
+      def send_from_cache( src )
+        cached_fragment = @cache.get( src, @headers ).body
+        log_request "C"
+        if parse_fragment?
 
-        if @cache.cached?( src, @http_params )
-          cached_fragment = @cache.get( src, @headers ).body
-          log_request "C"
-          if parse_fragment?
-
-            parser.process cached_fragment
-            parser.finish
+          @parser.process cached_fragment
+          @parser.finish
 
-          else
-            output << cached_fragment
-          end
- 
         else
+          @output << cached_fragment 
+        end
 
-          ir.request(src, @attributes['timeout'].to_i, alt ) do|status,response,uri|
-            if status
-              # NOTE: it's important that we cache the unprocessed markup, because we need to 
-              # reprocess the esi:include vars even for cached content, this way we can have cached content
-              # with HTTP_COOKIE vars and avoid re-requesting content
-              log_request "R"
-              cache_buffer = ""
-              response.read_body do|s|
-                cache_buffer << s
-                if parse_fragment?
-                  parser.process s
-                else
-                  output << s
-                end
-              end
- 
-              parser.finish if parse_fragment?
+      end
 
-              if src != uri # these won't be equal if the fragment followed a redirect or used the alt condition
-                if uri.query
-                  request_uri = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}?#{uri.query}"
-                else
-                  request_uri =  "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
-                end
+      def send_from_surrogate( src, alt, raise_on_error )
+        ir = Request.new(@http_params)
+        #timer = Time.now
+        ir.request(src, @attributes['timeout'].to_i, alt ) do|status,response,uri|
+          if status
+            # NOTE: it's important that we cache the unprocessed markup, because we need to 
+            # reprocess the esi:include vars even for cached content, this way we can have cached content
+            # with HTTP_COOKIE vars and avoid re-requesting content
+            cache_buffer = ""
+            response.read_body do|s|
+              cache_buffer << s
+              if parse_fragment?
+                @parser.process s
               else
-                request_uri = src
+                @output << s
               end
+            end
 
-              @cache.put(request_uri, @http_params, 600, cache_buffer )
+            if parse_fragment?
+              @parser.finish
+            end
 
+            if src != uri # these won't be equal if the fragment followed a redirect or used the alt condition
+              if uri.query
+                request_uri = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}?#{uri.query}"
+              else
+                request_uri =  "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
+              end
             else
-              # error/ check if the include has an onerror specifier
-              return if @attributes['onerror'] == 'continue'
-              # response is an IncludeRequest::Error
-              raise response.message if raise_on_error
-              # stop processing and return the error object
-              return response
+              request_uri = src
             end
+
+            cache_ttl = (@attributes['max-age']||600) # defaults to 600... maybe should have this be configurable?
+            if cache_ttl.respond_to?(:match) and cache_ttl.match(/\+/)
+              total=0
+              cache_ttl.split('+').each {|comp| total+=comp.to_i}
+              cache_ttl = total
+            else
+              cache_ttl = cache_ttl.to_i
+            end
+
+            @cache.put(request_uri, @http_params, cache_ttl, cache_buffer )
+            #log_request "R #{src} #{Time.now - timer}\n"
+
+          else
+            # error/ check if the include has an onerror specifier
+            return if @attributes['onerror'] == 'continue'
+            # response is a Request::Error
+            raise response.message if raise_on_error
+            # stop processing and return the error object
+            return response
           end
         end
+      end
 
-        parser = nil
 
-      end
     end
 
   end