rack-esi 0.1.2 → 0.2.0

data/.gitignore

@@ -1,21 +1,4 @@
-## MAC OS
-.DS_Store
-
-## TEXTMATE
-*.tmproj
-tmtags
-
-## EMACS
-*~
-\#*
-.\#*
-
-## VIM
-*.swp
-
-## PROJECT::GENERAL
-coverage
-rdoc
-pkg
-
-## PROJECT::SPECIFIC
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009 Florian Assmann
+Copyright (c) 2009 Florian Aßmann
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -0,0 +1,75 @@
+# rack-esi
+
+Rack-ESI is a Nokogiri based ESI middleware implementation for Rack with support for include tags, all other ESI namespaced nodes are just removed.
+
+To make this gem work you must define the [xmlns:esi](http://www.edge-delivery.org/esi/1.0) namespace in your text/html response.
+
+Note: This gem should only be used in development. For production use setup varnish or any other ESI enabled server.
+
+## Features
+
+ * threaded (in case we have slow IOs)
+ * PATH_INFO blacklisting (:skip => nil, should respond to ===)
+ * support for esi|include[alt] and esi|include[noerror] fallbacks
+
+## Dependencies
+
+ * Nokogiri
+ * Rack
+
+## Setup
+
+### w/o Gemfile
+
+    $ gem install rack-esi
+
+### w/ Gemfile
+
+    gem 'rack-esi'
+
+### rackup
+
+    use Rack::ESI, options || {}
+    run Application.new
+
+### Rails 2.3: environment.rb
+
+    config.gem 'rack-esi' # for setups w/o Gemfile
+    config.middleware.unshift Rack::ESI
+
+### Rails 3.1: application.rb
+
+    config.middleware.insert_before ActionDispatch::Static, Rack::ESI
+
+## Options
+
+ * poolsize: 4,
+   Number of worker threads. A value of 1 disables threading model.
+ * skip: nil,
+   This should be an object which responds to #===(PATH_INFO).
+ * parser: Nokogiri::XML::Document,
+   You can change this to Nokogiri::HTML::Document, but you should change the serializer, too (see below).
+ * serializer: :to_xhtml,
+   The serializer value specifies the method name which is send to the object created by the parser#parse.
+
+## TODO
+
+ * write documentation
+ * write more tests
+ * support more ESI elements
+
+## Note on Patches/Pull Requests
+ 
+ * Fork the project.
+ * Make your feature addition or bug fix.
+ * Add tests for it.
+ * Commit, do not mess with rakefile, version, or history.
+ * Send me a pull request.
+
+## Thanks
+
+tenderlove and Qerub
+
+## Copyright
+
+Copyright (c) 2009 Florian Aßmann. See LICENSE for details.

data/Rakefile

@@ -1,53 +1 @@
-require 'rubygems'
-require 'rake'
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "rack-esi"
-    gem.summary = %Q{ESI middleware implementation for Rack.}
-    gem.description = %Q{Nokogiri based ESI middleware implementation for Rack with (limited) support for include, remove and comment.}
-    gem.email = "florian.assmann@email.de"
-    gem.homepage = "http://github.com/boof/rack-esi"
-    gem.authors = ["Florian Aßmann"]
-    gem.add_development_dependency "riot", ">= 0"
-    gem.add_development_dependency "yard", ">= 0"
-    gem.add_dependency 'nokogiri', '>= 0'
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
-end
-
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/*_test.rb'
-  test.verbose = true
-end
-
-begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |test|
-    test.libs << 'test'
-    test.pattern = 'test/**/*_test.rb'
-    test.verbose = true
-  end
-rescue LoadError
-  task :rcov do
-    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-  end
-end
-
-task :test => :check_dependencies
-
-task :default => :test
-
-begin
-  require 'yard'
-  YARD::Rake::YardocTask.new
-rescue LoadError
-  task :yardoc do
-    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
-  end
-end
+require "bundler/gem_tasks"

data/lib/rack-esi.rb

@@ -1,97 +1,45 @@
 require 'rack'
 require 'nokogiri'
+require 'bundler'
+Bundler.require
 
-class Rack::ESI
-  NS = { 'esi' => 'http://www.edge-delivery.org/esi/1.0' }
-  METHODS = { 'include' => :esi_include, 'remove' => nil, 'comment' => nil }
-  CSS = METHODS.keys.map { |cmd| "esi|#{ cmd }" } * ','
+require File.expand_path('../rack-esi/processor', __FILE__)
 
-  class Error < RuntimeError
-    def initialize(status, headers, response)
-      @status, @headers, @response = status, headers, response
-    end
-    def finish
-      return [@status, @headers, backtrace]
-    end
-  end
+class Rack::ESI
 
   def initialize(app, options = {})
-    @app  = app
+    @app        = app
 
-    @paths          = options[:skip]
-    @types          = options[:only] || /^text\/(?:x|ht)ml/
-    @max_includes   = options[:includes] || 32
-    @max_recursion  = options[:depth] || 5
+    @parser     = options.fetch :parser, Nokogiri::XML::Document
+    @serializer = options.fetch :serializer, :to_xhtml
+    @skip       = options[:skip]
+    @poolsize   = options.fetch :poolsize, 4
+    @processor  = @poolsize == 1 ? Processor::Linear : Processor::Threaded
   end
 
-  def call env, counter = { :recursion => 0, :includes => 0 }
-    return @app.call(env) if skip_path? env['PATH_INFO']
-
-    status, headers, input = @app.call env.dup
-    return status, headers, input if skip_type? headers['Content-Type']
-
-    output = []
-    input.each { |body| output << compile_body(body, env, counter) }
-
-    Rack::Response.new(output, status, headers).finish
-  end
-
-  private
-
-    def with_compiled_path(env, path)
-      # TODO: should compile variables.
-      env.merge 'PATH_INFO' => path, 'REQUEST_URI' => path
-    end
+  def queue(&block)
+    unless @queue
+      @queue, @group = Queue.new, ThreadGroup.new
+      @poolsize.times { @group.add Worker.new(@queue) }
 
-    def fetch(path, env, counter)
-      call with_compiled_path(env, path), counter if path
-    rescue => e
-      return [500, {}, e.backtrace]
+      at_exit { Finisher.wait @queue }
     end
 
-    # Should I use XML::SAX::Parser?
-    def compile_body(body, env, counter)
-      document = Nokogiri.XML body
+    @queue.push block
+  end
 
-      document.css(CSS, NS).each do |node|
-        method = METHODS[node.name] and send method, node, env, counter
-        node.unlink
-      end
+  attr_reader :parser, :serializer
 
-      document.to_xhtml
-    end
+  def call(env)
+    return @app.call(env) if @skip === env['PATH_INFO']
 
-    def skip_path?(path)
-      @paths =~ path if @paths
-    end
-    def skip_type?(type)
-      @types !~ type
-    end
+    status, headers, body = @app.call env.dup
 
-    def max?(counter)
-      not counter[:includes] < @max_includes &&
-          counter[:recursion] < @max_recursion
+    if status == 200 and headers['Content-Type'] =~ /text\/html/
+      body = @processor.new(self, env).process body
     end
 
-    def esi_include(node, env, counter)
-      return if max? counter
-
-      counter[:includes] += 1
-      counter[:recursion] += 1
-
-      status, headers, response = fetch node['src'], env, counter
-      status, headers, response = fetch node['alt'], env, counter if status != 200
-
-      if status == 200
-        data = ''
-        response.each { |inc| data << inc }
-        node.before data
-      elsif node['onerror'] != 'continue'
-        raise Error.new(status, headers, response)
-      end
-
-    ensure
-      counter[:recursion] -= 1
-    end
+    return status, headers, body
+  end
 
 end

data/lib/rack-esi/processor.rb

@@ -0,0 +1,55 @@
+class Rack::ESI
+  class Processor < Struct.new(:esi, :env)
+
+    class Linear < self
+      def process_document(d)
+        d.xpath('//e:*', 'e' => NAMESPACE).each { |n| process_node n }
+      end
+    end
+    autoload :Threaded, File.expand_path('../threaded', __FILE__)
+
+    NAMESPACE = 'http://www.edge-delivery.org/esi/1.0'
+    Error = Class.new RuntimeError
+
+    def read(enumerable, buffer = '')
+      enumerable.each { |str| buffer << str }
+      buffer
+    end
+
+    def include(path)
+      # RADAR patron here?
+      esi.call env.merge('PATH_INFO' => path, 'REQUEST_URI' => path)
+    rescue => e
+      return 500, {}, []
+    end
+    def process_node(node)
+      case node.name
+      when 'include'
+        status, headers, body = include node['src']
+
+        unless status == 200 or node['alt'].nil?
+          status, headers, body = include node['alt']
+        end
+
+        if status == 200
+          node.replace read(body)
+        elsif node['onerror'] != 'continue'
+          raise Error
+        end
+      else
+        node.remove
+      end
+    end
+    def process_document(document)
+      raise NotImplementedError
+    end
+    def process(body)
+      document = esi.parser.parse read(body)
+      process_document document
+      [
+        document.send( esi.serializer )
+      ]
+    end
+
+  end
+end

data/lib/rack-esi/threaded.rb

@@ -0,0 +1,50 @@
+require 'thread'
+#require 'timeout'
+
+class Rack::ESI
+
+  class Finisher < Proc
+    def self.wait(queue)
+      finisher = new do |worker|
+        puts "Finishing #{ worker.inspect }..."
+        worker[:finish] = true
+        queue.push finisher
+      end
+
+      # cast the first stone
+      queue.push finisher
+
+      # wait at the end
+      queue.pop
+    end
+  end
+
+  class Worker < Thread
+    def initialize(queue)
+      super do
+        begin
+          queue.pop[ self ]
+        rescue => e
+          puts e
+        end until key? :finish
+      end
+    end
+  end
+
+  class Processor::Threaded < Processor
+    def process_document(document)
+      nodes = document.xpath '//e:*', 'e' => NAMESPACE
+
+      countdown, main = nodes.length, Thread.current
+      nodes.each do |node|
+        esi.queue do
+          process_node node
+          main.run if (countdown -= 1).zero?
+        end
+      end
+      # TODO prevent nesting depth bigger than poolsize
+      Thread.stop if countdown > 0 # wait for worker
+    end
+  end
+
+end

data/lib/rack-esi/version.rb

@@ -0,0 +1,5 @@
+module Rack
+  class ESI
+    VERSION = "0.2.0"
+  end
+end

data/rack-esi.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "rack-esi/version"
+
+Gem::Specification.new do |s|
+  s.name        = "rack-esi"
+  s.version     = Rack::ESI::VERSION
+  s.authors     = ["Florian Aßmann"]
+  s.email       = ["florian.assmann@email.de"]
+  s.homepage    = ""
+  s.summary     = %q{ ESI middleware implementation for Rack. }
+  s.description = <<-EOF
+Rack-ESI is a Nokogiri based ESI middleware implementation for Rack with support for include tags, all other ESI namespaced nodes are just removed.
+To make this gem work you must define the (xmlns:esi)[http://www.edge-delivery.org/esi/1.0] namespace in your text/html response.
+Note: This gem should only be used in development. For production use setup varnish or any other ESI enabled server.
+EOF
+  # s.rubyforge_project = "rack-esi"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "rack"
+  s.add_dependency "nokogiri"
+  # s.add_dependency "patron"
+  s.add_development_dependency "riot"
+end

data/test/_test.rb

@@ -0,0 +1,31 @@
+require File.expand_path('../teststrap', __FILE__)
+
+context 'Rack::ESI' do
+
+  __dirname__ = File.expand_path File.dirname(__FILE__)
+  root = Pathname.new File.join(__dirname__, 'fixtures')
+  opts = { :urls => ['/'], :root => root }
+
+  setup { ESI.new Static.new(App.new, opts), skip: /raw/, :poolsize => 1 }
+
+  context 'GET /raw.html' do
+    setup { MockRequest.new(topic).get '/raw.html' }
+    asserts('Content-Type') { topic.content_type }.equals 'text/html'
+    should('not be altered') { topic.body == root.join('raw.html').read }
+  end
+
+  context 'GET /index.html' do
+    setup { MockRequest.new(topic).get '/index.html' }
+
+    asserts('Content-Type') { topic.content_type }.equals 'text/html'
+    should('not have any ESI specific nodes') do
+      html(topic.body).
+      at('//e:*', 'e' => Rack::ESI::Processor::NAMESPACE).nil?
+    end
+    should('have meta replacement with content') do
+      not html(topic.body).
+      at("//meta[@name='replacement' and @content='content']").nil?
+    end
+  end
+
+end

data/test/rack-esi_test.rb

@@ -1,4 +1,4 @@
-require 'teststrap'
+require File.expand_path('../teststrap', __FILE__)
 
 context 'Rack::ESI' do
 
@@ -6,7 +6,7 @@ context 'Rack::ESI' do
   root = Pathname.new File.join(__dirname__, 'fixtures')
   opts = { :urls => ['/'], :root => root }
 
-  setup { ESI.new Static.new(App.new, opts), :skip => /raw/ }
+  setup { ESI.new Static.new(App.new, opts), skip: /raw/, :poolsize => 1 }
 
   context 'GET /raw.html' do
     setup { MockRequest.new(topic).get '/raw.html' }
@@ -20,7 +20,7 @@ context 'Rack::ESI' do
     asserts('Content-Type') { topic.content_type }.equals 'text/html'
     should('not have any ESI specific nodes') do
       html(topic.body).
-      at('//esi:include|//esi:remove|//esi:comment', Rack::ESI::NS).nil?
+      at('//e:*', 'e' => Rack::ESI::Processor::NAMESPACE).nil?
     end
     should('have meta replacement with content') do
       not html(topic.body).

data/test/teststrap.rb

@@ -1,8 +1,12 @@
 require 'pathname'
-require 'rubygems'
-require 'riot'
-require 'rack-esi'
 require 'rack/mock'
+require 'rack/static'
+require 'rack/file'
+
+require File.expand_path('../../lib/rack-esi', __FILE__)
+Bundler.require :development
+
+Nokogiri
 
 def html(body)
   Nokogiri.HTML(body).root

metadata

@@ -1,97 +1,111 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: rack-esi
-version: !ruby/object:Gem::Version 
-  version: 0.1.2
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
 platform: ruby
-authors: 
-- "Florian A\xC3\x9Fmann"
+authors:
+- Florian Aßmann
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2009-11-27 00:00:00 +01:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: riot
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: yard
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
-    version: 
-- !ruby/object:Gem::Dependency 
+date: 2011-09-30 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: &70252059616420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70252059616420
+- !ruby/object:Gem::Dependency
   name: nokogiri
+  requirement: &70252059615980 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
-    version: 
-description: Nokogiri based ESI middleware implementation for Rack with (limited) support for include, remove and comment.
-email: florian.assmann@email.de
-executables: []
+  prerelease: false
+  version_requirements: *70252059615980
+- !ruby/object:Gem::Dependency
+  name: riot
+  requirement: &70252059615560 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70252059615560
+description: ! 'Rack-ESI is a Nokogiri based ESI middleware implementation for Rack
+  with support for include tags, all other ESI namespaced nodes are just removed.
 
-extensions: []
+  To make this gem work you must define the (xmlns:esi)[http://www.edge-delivery.org/esi/1.0]
+  namespace in your text/html response.
 
-extra_rdoc_files: 
-- LICENSE
-- README.markdown
-files: 
+  Note: This gem should only be used in development. For production use setup varnish
+  or any other ESI enabled server.
+
+'
+email:
+- florian.assmann@email.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
 - .document
 - .gitignore
+- Gemfile
+- Gemfile.lock
 - LICENSE
-- README.markdown
+- README.md
 - Rakefile
-- VERSION
 - lib/rack-esi.rb
+- lib/rack-esi/processor.rb
+- lib/rack-esi/threaded.rb
+- lib/rack-esi/version.rb
+- rack-esi.gemspec
+- test/_test.rb
 - test/fixtures/index.html
 - test/fixtures/metatags.fragment
 - test/fixtures/raw.html
 - test/rack-esi_test.rb
 - test/teststrap.rb
-has_rdoc: true
-homepage: http://github.com/boof/rack-esi
+homepage: ''
 licenses: []
-
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
-require_paths: 
+rdoc_options: []
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.8.8
 signing_key: 
 specification_version: 3
 summary: ESI middleware implementation for Rack.
-test_files: 
+test_files:
+- test/_test.rb
+- test/fixtures/index.html
+- test/fixtures/metatags.fragment
+- test/fixtures/raw.html
 - test/rack-esi_test.rb
 - test/teststrap.rb

data/README.markdown

@@ -1,54 +0,0 @@
-# rack-esi
-
-Nokogiri based ESI middleware implementation for Rack with (limited) support
-for include, remove and comment.
-
-## Features
-
- * path blacklisting (:skip => nil, expects Regexp)
- * type whitelisting (:only => /^text\/(?:x|ht)ml/)
- * recursion limit (:depth => 5)
- * include limits (:includes => 32)
- * support for <include> alt and noerror attributes
-
-_It's for development purpose..._
-
-## Installation
-
-    gem install rack-esi
-
-## Rails Setup (environment.rb)
-
-    config.gem 'rack-esi'
-    require 'rack-esi'
-    config.middleware.insert_before config.middleware.first, Rack::ESI
-
-## TODO
-
- * write documentation
- * write more tests
- * support more ESI elements
- * switch to Nokogiri::XML::SAX::Document?
-
-## Dependencies
-
- * Nokogiri
- * Rack
-
-## Note on Patches/Pull Requests
- 
- * Fork the project.
- * Make your feature addition or bug fix.
- * Add tests for it. This is important so I don't break it in a
-  future version unintentionally.
- * Commit, do not mess with rakefile, version, or history.
-  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
- * Send me a pull request. Bonus points for topic branches.
-
-## Thanks
-
-tenderlove and Qerub
-
-## Copyright
-
-Copyright (c) 2009 Florian Assmann. See LICENSE for details.

data/VERSION

@@ -1 +0,0 @@
-0.1.2