rack-httperflog 0.1.1

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Emmanuel Oga
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,48 @@
+= rack/httperflog
+
+Generates a replay log of the actions performed by the web application.
+
+When a url is requested, it is logged to the replay log (if the response status is not an error).
+
+Then, if the response content type is html, paths for the images, scripts, stylesheets and iframes are extracted and logged as well.
+
+== Usage
+
+On your rack configuration, add the middleware providing the following parameters:
+
+  ...
+  use ReplayLogger, :ping_urls, "/log/path", "/flag/path"
+  ...
+
+ping_urls:: pass :ping_urls for making the middleware perform an http get to validate each path parsed from the html body returns a status 200. WARNING: only use if your http server can handle more than one concurrent connection (e.g.: you are using passenger or have a cluster of load balanced mongrels). Any other value avoids performing the check requests.
+log_path:: path to the log file that will contain the session log.
+flag_path:: the logging will be performed only when the file pointed by flag_path exists.
+
+== On Rails
+
+Rails::Initializer.run do |config|
+  config.gem 'rack-httperflog', :lib => "rack/httperflog"
+
+  config.middleware.use "Rack::Httperflog", :ping_urls, Rails.root.join("log", "wsess.log"), Rails.root.join("log", "wsess.record")
+end
+
+== Notes
+
+The log will not contain any newlines (which signal separate sessions on httperf's wsesslog format). If you want to generated separated logs, simple echo a new line in the session log with a command like:
+
+  echo '' >> log/path
+
+== Note on Patches/Pull Requests
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but
+   bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2009 Emmanuel Oga. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "rack-httperflog"
+    gem.summary = %Q{generates a replay log in httperfs' wsesslog format}
+    gem.description = %Q{rack middleware to generate a replay log}
+    gem.email = "EmmanuelOga@gmail.com"
+    gem.homepage = "http://github.com/EmmanuelOga/rack-httperflog"
+    gem.authors = ["Emmanuel Oga"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_dependency "nokogiri", ">= 1.3.3"
+    gem.add_dependency "rack", ">= 1.0.0"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "rack-httperflog #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/lib/rack/httperflog.rb

@@ -0,0 +1,125 @@
+require "net/http"
+require "nokogiri"
+require "rack"
+
+module Rack
+  # Httperflog is a middleware that will log the request and all the
+  # img, iframes, css and js paths founds in its body if the response is an html document in
+  # a format appropriate for using it as an httperf session log.
+  #
+  # use Httperflog :ping_urls, "log/path", "flag/path"
+  #
+  #   ping_urls :: pass :ping_urls for making the middleware perform an http get
+  #                to validate each path found inside the html body. WARNING: only use if your
+  #                http server can handle more than one concurrent connection (e.g.: you are
+  #                using passenger or have a cluster of load balanced mongrels). Any other value
+  #                avoids performing the requests.
+  #   log_path  :: path to the log file that will contain the session log.
+  #   flag_path :: the logging will be performed only when the file pointed by
+  #                flag_path exists.
+  class Httperflog < Struct.new(:app, :ping_urls, :log_path, :flag_path)
+    def call(env)
+      status, headers, response = app.call(env)
+
+      Httperflog::Logger.new(request_to_log(env), response_to_log(status, headers, response), log_file, ping_urls).perform_logging if logging_enabled?
+
+      [status, headers, response]
+    end
+
+    # Logging will be enabled if file in flag_path exists.
+    def logging_enabled?
+      ::File.file?(flag_path)
+    end
+
+    # Log file opened on append mode.
+    def log_file
+      @lof_file ||= ::File.open(log_path, "a")
+    end
+
+    # Extract the Rack::Request object needed by the logger
+    def request_to_log(env)
+      env["rack.request"].class <= Rack::Request ? env["rack.request"] : Rack::Request.new(env)
+    end
+
+    # Extract the Rack::Response object needed by the logger
+    def response_to_log(status, headers, response)
+      r = response.class <= Rack::Response ? response : Rack::Response.new(response, status, headers)
+      r.status = r.status.to_i # avoid problems with Rack::Response methods that expect the status code to be an integer.
+      r
+    end
+
+    # Extracts the lines to log from the rack request and response and logs 
+    # them to a file on log_path path.
+    #
+    # Logger.new("log/path", :ping_urls, request, response)
+    #
+    # log_path  :: path to the log file to log the lines
+    # ping_urls :: provide :ping_urls to perform an http get to each path found inside
+    #              the body of the response. Provide any other value to avoid this check.
+    # request   :: a Rack::Request instance
+    # response  :: a Rack::Response instance
+    class Logger < Struct.new(:request, :response, :file, :ping_urls)
+
+      # Perform the logging of each log line to a file
+      def perform_logging
+        each { |line| file << line }
+        file.flush
+      end
+
+      # yields each line for the log
+      def each
+        return if response.server_error? || response.client_error?
+
+        form_data = " contents=\"#{ Rack::Utils.build_query(request.params) }\"" if request.form_data? && !request.params.empty?
+
+        yield "#{ request.path_info } method=#{ request.request_method.to_s.upcase }#{ form_data }\n"
+
+        paths_to_log.each do |path|
+          yield "    #{ path }\n" if path != request.path_info && (ping_urls != :ping_urls || UrlHelper.url_status_is?("200", request.host, request.port, path))
+        end
+      end
+
+      # uses a HtmlPathsExtractor object to parse the response if it contains html, and extracts all the relevant paths.
+      def paths_to_log
+        response.content_type =~ /html/ ? HtmlPathsExtractor.new(response.body).paths : Array.new
+      end
+    end
+
+    # Extract each path linked inside the source html document
+    #
+    # HtmlPathsExtractor.new(source)
+    #
+    # source :: an html document
+    class HtmlPathsExtractor < Struct.new(:source)
+
+      # return all link, script, img and iframe paths found in the html document
+      def paths
+        links   = attributes_from_nodes("href", "link")
+        scripts = attributes_from_nodes("src",  "script")
+        imgs    = attributes_from_nodes("src",  "img")
+        iframes = attributes_from_nodes("src",  "iframe")
+        links + scripts + imgs + iframes
+      end
+
+      # parse the source html
+      def parsed_body
+        @parsed_body ||= Nokogiri(source.to_s)
+      end
+
+      # extract the attribute attribute_name from all the html nodes of type node_type.
+      # returns only non-empty attributes that do not start with http (quick way to avoid
+      # paths that point to an external host).
+      def attributes_from_nodes(attribute_name, node_type)
+        (parsed_body / node_type).map { |node| node[attribute_name].to_s.strip }.select { |path| path =~ /\S/ && path !~ /^http:\/\// }
+      end
+    end
+
+    module UrlHelper
+      # Checks if the status code returned by a GET to http://host:port/path is the one provided in the first param.
+      def url_status_is?(status, host, port, path)
+        Net::HTTP.start(host, port) {|http| http.get(path) }.code.to_s.strip == status.to_s rescue false
+      end
+      extend self
+    end
+  end
+end

data/rack-httperflog.gemspec

@@ -0,0 +1,65 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{rack-httperflog}
+  s.version = "0.1.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Emmanuel Oga"]
+  s.date = %q{2009-10-28}
+  s.description = %q{rack middleware to generate a replay log}
+  s.email = %q{EmmanuelOga@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "lib/rack/httperflog.rb",
+     "rack-httperflog.gemspec",
+     "spec/lib/html_paths_extractor_spec.rb",
+     "spec/lib/httperflog_spec.rb",
+     "spec/lib/logger_spec.rb",
+     "spec/spec.opts",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/EmmanuelOga/rack-httperflog}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{generates a replay log in httperfs' wsesslog format}
+  s.test_files = [
+    "spec/spec_helper.rb",
+     "spec/lib/html_paths_extractor_spec.rb",
+     "spec/lib/httperflog_spec.rb",
+     "spec/lib/logger_spec.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.3.3"])
+      s.add_runtime_dependency(%q<rack>, [">= 1.0.0"])
+    else
+      s.add_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_dependency(%q<nokogiri>, [">= 1.3.3"])
+      s.add_dependency(%q<rack>, [">= 1.0.0"])
+    end
+  else
+    s.add_dependency(%q<rspec>, [">= 1.2.9"])
+    s.add_dependency(%q<nokogiri>, [">= 1.3.3"])
+    s.add_dependency(%q<rack>, [">= 1.0.0"])
+  end
+end
+

data/spec/lib/html_paths_extractor_spec.rb

@@ -0,0 +1,15 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Rack::Httperflog::HtmlPathsExtractor do
+  before :each do
+    @extractor = Rack::Httperflog::HtmlPathsExtractor.new(fixture_markup)
+  end
+
+   it "should list all the known paths on the markup (w/o path of broken markup)" do
+     @extractor.paths.length.should == 4
+     @extractor.paths.should include("/some/css.css")
+     @extractor.paths.should include("/some/js.js")
+     @extractor.paths.should include("/some/img.jpg")
+     @extractor.paths.should include("/some/iframe.html")
+   end
+end

data/spec/lib/httperflog_spec.rb

@@ -0,0 +1,57 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Rack::Httperflog do
+  before :each do
+    @log = ::Tempfile.new("log")
+    @flag = ::Tempfile.new("flag")
+  end
+
+  it "only logs when the flag_path file is set" do
+    Rack::Httperflog.new(app, :pings_urls, "/some/path", "/fake/path").should_not be_logging_enabled
+    Rack::Httperflog.new(app, :pings_urls, "/some/path", @flag.path).should be_logging_enabled
+  end
+
+  it "extracts a rack request from the env" do
+    rr = Rack::Request.new("some" => "env")
+
+    r1 = Rack::Httperflog.new.request_to_log("rack.request" => rr)
+    r2 = Rack::Httperflog.new.request_to_log("some" => "env")
+
+    r1.should be_an_instance_of(Rack::Request)
+    r2.should be_an_instance_of(Rack::Request)
+
+    r1.env["some"].should == r2.env["some"]
+  end
+
+  it "extracts a rack response from the response of the app (with integer status code)" do
+    rr = Rack::Response.new("Body", "200", "Content-Length" => "4")
+
+    r1 = Rack::Httperflog.new.response_to_log("200", { "Content-Type"=>"text/html", "Content-Length" => "4" }, rr)
+    r2 = Rack::Httperflog.new.response_to_log("200", { "Content-Type"=>"text/html", "Content-Length" => "4" }, ["Body"])
+
+    r1.should be_an_instance_of(Rack::Response)
+    r2.should be_an_instance_of(Rack::Response)
+
+    r1.status.should == 200
+    r2.status.should == 200
+
+    r1.headers.should == { "Content-Type"=>"text/html", "Content-Length" => "4" }
+    r2.headers.should == { "Content-Type"=>"text/html", "Content-Length" => "4" }
+
+    r1.body.should == ["Body"]
+    r2.body.should == ["Body"]
+  end
+
+  it "returns the same status headers and response that the rack application generated" do
+    response_without_middleware = Rack::MockRequest.new(app).get("")
+    response_with_middleware = Rack::MockRequest.new(app_with_middleware).get("")
+
+    response_with_middleware.status.should  == app.call({})[0]
+    response_with_middleware.headers.should == app.call({})[1]
+    response_with_middleware.body.should    == app.call({})[2]
+
+    response_without_middleware.body.should    == response_with_middleware.body
+    response_without_middleware.status.should  == response_with_middleware.status
+    response_without_middleware.headers.should == response_with_middleware.headers
+  end
+end

data/spec/lib/logger_spec.rb

@@ -0,0 +1,49 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Rack::Httperflog::Logger do
+  before :each do
+    @log = ::Tempfile.new("log")
+  end
+
+  def request_and_response(app, method, url)
+    mock = Rack::MockRequest.new(app)
+
+    request = Rack::Request.new Rack::MockRequest.env_for(url, :method => method)
+    response = mock.send(method, url)
+
+    [request, response]
+  end
+
+  it "should log lines matching the request information (only for good requests)" do
+    req, resp = request_and_response(app(fixture_markup, "text/html", 500), "get", "/some/url")
+    Rack::Httperflog::Logger.new(req, resp, @log, :dont_ping_urls).perform_logging
+
+    req, resp = request_and_response(app, "get", "/some/url")
+    Rack::Httperflog::Logger.new(req, resp, @log, :dont_ping_urls).perform_logging
+
+    req, resp = request_and_response(app(fixture_markup), "post", "/some/url?some=params")
+    Rack::Httperflog::Logger.new(req, resp, @log, :dont_ping_urls).perform_logging
+
+    @log.flush; @log.rewind
+
+    lines = @log.read.split("\n")
+
+    lines[0].should =~ /\/some\/url method=GET/
+    lines[1].should =~ /\/some\/url method=POST contents="some=params"/
+    lines[2].should =~ /    \/some\/css.css/
+    lines[3].should =~ /    \/some\/js.js/
+    lines[4].should =~ /    \/some\/img.jpg/
+    lines[5].should =~ /    \/some\/iframe.html/
+  end
+
+  it "should use the url_exist? helper on each path if :ping_urls is set" do
+    req, resp = request_and_response(app(fixture_markup), "post", "/some/url?some=params")
+
+    Rack::Httperflog::UrlHelper.should_receive(:url_status_is?).with("200", req.host, req.port, "/some/css.css")
+    Rack::Httperflog::UrlHelper.should_receive(:url_status_is?).with("200", req.host, req.port, "/some/js.js")
+    Rack::Httperflog::UrlHelper.should_receive(:url_status_is?).with("200", req.host, req.port, "/some/img.jpg")
+    Rack::Httperflog::UrlHelper.should_receive(:url_status_is?).with("200", req.host, req.port, "/some/iframe.html")
+
+    Rack::Httperflog::Logger.new(req, resp, @log, :ping_urls).perform_logging
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,2 @@
+--color
+--backtrace

data/spec/spec_helper.rb

@@ -0,0 +1,57 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'tempfile'
+require 'rubygems'
+
+require 'rack'
+require 'rack/mock'
+require 'rack/httperflog'
+
+require 'spec'
+require 'spec/autorun'
+
+module HttperflogAppsHelper
+  def app(body = "Hello World!", content_type = "text/html", status = 200)
+    lambda do
+      [status, { "Content-Length" => body.length.to_s, "Content-Type" => content_type }, body]
+    end
+  end
+
+  def app_with_middleware
+    app, flag_path, log_path = app(), @flag.path, @log.path
+
+    Rack::Builder.app do
+      use Rack::Httperflog, :dont_ping_urls, log_path, flag_path
+      run app
+    end
+  end
+end
+
+module Fixtures
+  MARKUP = <<-EOMARKUP
+  <html>
+  <head>
+    <link href="/some/css.css" type="text/css"></link>
+    <link href="" type="text/css"></link>
+    <script src="/some/js.js" type "text/javascript></script>
+    <script src="" type "text/javascript></script>
+  </head>
+  <body>
+    <img src="/some/img.jpg" />
+    <img src="
+    " />
+    <iframe src="   "></iframe>
+    <iframe src="/some/iframe.html"></iframe>
+  </body>
+  </html>
+  EOMARKUP
+
+  def fixture_markup
+    MARKUP
+  end
+end
+
+Spec::Runner.configure do |config|
+  config.include HttperflogAppsHelper, Fixtures
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification 
+name: rack-httperflog
+version: !ruby/object:Gem::Version 
+  version: 0.1.1
+platform: ruby
+authors: 
+- Emmanuel Oga
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-28 00:00:00 -02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.9
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.3.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rack
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.0
+    version: 
+description: rack middleware to generate a replay log
+email: EmmanuelOga@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/rack/httperflog.rb
+- rack-httperflog.gemspec
+- spec/lib/html_paths_extractor_spec.rb
+- spec/lib/httperflog_spec.rb
+- spec/lib/logger_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/EmmanuelOga/rack-httperflog
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: generates a replay log in httperfs' wsesslog format
+test_files: 
+- spec/spec_helper.rb
+- spec/lib/html_paths_extractor_spec.rb
+- spec/lib/httperflog_spec.rb
+- spec/lib/logger_spec.rb