ucnv-chainsaw 0.0.0

data/lib/chainsaw/common.rb

@@ -0,0 +1,87 @@
+
+module Chainsaw
+  
+  module Util
+    
+    def is_xml_parsable?(content_type)
+      content_type =~ %r{(?:text|application)/x?(?:ht)?ml}
+    end
+    
+    def to_nth(num)
+      case num.to_i
+      when 1
+        '1st'
+      when 2
+        '2nd'
+      when 3
+        '3rd'
+      else
+        num.to_i.to_s + 'th'
+      end
+    end
+    
+    def qq(str)
+      "\"#{str}\""
+    end
+  end
+  
+  module Encoding
+    NKF_TO_LIBXML2 = {
+      NKF::ASCII   => 10, #=> 22,
+      NKF::JIS     => 19,
+      NKF::SJIS    => 20,
+      NKF::EUC     => 21,
+      NKF::UTF8    => 1,
+      NKF::UTF16   => 2,
+      NKF::UNKNOWN => 10,
+      NKF::BINARY  => 10,
+    }
+    
+    def self.guess(data)
+      nokogiri_consts = Nokogiri::XML::SAX::Parser::ENCODINGS
+      nkf_guessed = NKF.guess data
+      value = NKF_TO_LIBXML2[nkf_guessed]
+      value = NKF_TO_LIBXML2[NKF::UTF8] unless nokogiri_consts.value?(value)
+      c = nokogiri_consts.find do |k, v|
+        v == value
+      end
+      c.first
+    end
+  end
+  
+  module ErrorWrapper
+    APPLY_TESTUNIT_BACKTRACEFILTER = true
+    @@indent = '| '
+    @@tab = '  '
+    def set_backtrace(backtrace)
+      root_cause = $!
+      unless root_cause.nil?
+        backtrace = filter_backtrace(backtrace)
+        backtrace.push '............', 'root cause'
+        backtrace.push "#{@@indent}#{root_cause.class.name}: #{root_cause.to_s}"
+        backtrace.concat(filter_backtrace(root_cause.backtrace).map do |t|
+          "#{@@indent}#{@@tab}#{t}"
+        end)
+      end
+      super backtrace
+    end
+    
+    def filter_backtrace(backtrace)
+      if APPLY_TESTUNIT_BACKTRACEFILTER
+        require 'test/unit/util/backtracefilter'
+        Test::Unit::Util::BacktraceFilter.module_eval do
+          module_function :filter_backtrace
+        end
+        Test::Unit::Util::BacktraceFilter.filter_backtrace backtrace
+      else
+        backtrace
+      end
+    end
+  end
+  
+  class RequestError < StandardError
+    include Chainsaw::ErrorWrapper
+  end
+  
+end
+

data/lib/chainsaw/element.rb

@@ -0,0 +1,62 @@
+
+require 'pathname'
+
+module Chainsaw
+  module Element
+    include Chainsaw::Util
+    
+    def is_form?
+      name =~ /^form$/i
+    end
+    
+    def serialize_form(clicked_button = nil, image_x = -1, image_y = -1)
+      return nil unless is_form?
+      form = []
+      if clicked_button.nil?
+        s = self.xpath('.//input[@type="submit"]')
+        unless s.empty?
+          form.push [s.first['name'], s.first['value']]
+        end
+      else 
+        s = self.xpath(".//input[@name=#{qq clicked_button}]")
+        
+        if s.size and ['submit', 'image'].include? s.first['type']
+          form.push [s.first['name'], s.first['value']]
+          form.push [s.first['name'] + '.x', image_x] if image_x > 0
+          form.push [s.first['name'] + '.y', image_y] if image_y > 0
+        end
+      end
+      self.xpath('.//input').each do |e|
+        next unless e['name']
+        case e['type']
+        when 'text', 'password', 'hidden', nil
+          form.push [e['name'], e['value']]
+        when 'radio', 'checkbox'
+          form.push [e['name'], e['value']] unless e['checked'].nil?
+        when 'file'
+          if self['enctype'] =~ %r{^multipart/form-data}i
+            f = Pathname.new e['value'].to_s
+            form.push [e['name'], f]
+          end
+        else
+          ;
+        end
+      end
+      self.xpath('.//textarea').each do |e|
+        form.push [e['name'], e.text]
+      end
+      self.xpath('.//select').each do |e|
+        opt = e.xpath('.//option')
+        slctd = opt.reject do |o|
+          o['selected'].nil?
+        end
+        slctd = opt.first if e['multiple'].nil? and slctd.empty?
+        slctd.each {|s| form.push([e['name'], s['value']]) } unless slctd.nil?
+      end
+      form
+    end
+    
+  end
+end
+
+

data/lib/chainsaw/ext/httpclient.rb

@@ -0,0 +1,27 @@
+
+
+class HTTPClient
+  def get_r(uri, query = nil, extheader = {}, &block)
+    follow_redirect(:get, uri, query, nil, extheader, &block)
+  end
+  
+  def post_r(uri, body = nil, extheader = {}, &block)
+    follow_redirect(:post, uri, nil, body, extheader, &block)
+  end
+  
+  alias do_get_header_original do_get_header
+  def do_get_header(req, res, sess)
+    # It is not a good way, but HTTPClient cannot keep requested URIs..
+    unless res.respond_to?(:uri)
+      HTTP::Message.module_eval do
+        attr_accessor :uri
+      end
+    end
+    res.uri = req.header.request_uri
+    do_get_header_original(req, res, sess)
+  end
+end
+
+HTTP::Status::SUCCESSFUL_STATUS.push 404
+
+

data/lib/chainsaw/ext/nokogiri.rb

@@ -0,0 +1,29 @@
+
+
+module Nokogiri
+  module XML
+    class Node
+      def fix_xpath(*paths)
+        paths.map do |path|
+          if path.is_a? String
+            # fix for id() func
+            path.sub(/^id\(\s*["']([^"']*)["']\s*\)/, './/*[@id="\1"]')
+          else
+            path
+          end
+        end
+      end
+
+      alias original_search search
+      def search(*paths)
+        original_search(*fix_xpath(*paths))
+      end
+      
+      alias original_xpath xpath
+      def xpath(*paths)
+        original_xpath(*fix_xpath(*paths))
+      end
+
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,15 @@
+require 'stringio'
+require 'test/unit'
+require 'chainsaw'
+require 'open-uri'
+require File.dirname(__FILE__) + '/server.rb'
+
+unless defined? TEST_URL
+  TEST_URL = "http://#{TEST_SERVER_CONFIG[:BindAddress]}:#{TEST_SERVER_CONFIG[:Port]}/"
+
+  begin
+    open TEST_URL
+  rescue
+    raise 'Run test/server.rb before you test.'
+  end
+end

data/test/htdocs/01.html

@@ -0,0 +1,17 @@
+<html>
+  <head>
+    <title>test 01</title>
+  </head>
+  <body>
+    <div id="links">
+      <ul>
+        <li><a href="01.html">1</a></li>
+        <li><a href="02.html">2</a></li>
+        <li><a href="/03.html">3</a></li>
+        <li><a href="/cgi.rb">4</a></li>
+        <li><a href="00.html">5</a></li>
+      </ul>
+    </div>
+  </body>
+</html>
+

data/test/htdocs/02.html

@@ -0,0 +1,9 @@
+<html>
+  <head>
+    <title>test go</title>
+  </head>
+  <body>
+    <a href="00.html">not found</a>
+  </body>
+</html>
+

data/test/htdocs/03.html

@@ -0,0 +1,25 @@
+<html>
+  <head>
+    <title>test 03</title>
+  </head>
+  <body>
+    <div id="forms">
+      <form action="cgi.rb" method="get" id="get-form">
+        <input type="file" name="f" />
+        <input type="text" name="t" />
+        <input type="submit" name="s" value="go" />
+      </form>
+      <form action="cgi.rb" method="post" id="post-form">
+        <input type="text" name="t">
+        <input type="password" name="p">
+        
+        <input type="submit" name="s" value="go" />
+      </form>
+      <form action="cgi.rb" method="post" enctype="multipart/form-data" id="upload-form">
+        <input type="file" name="f" />
+        <input type="submit" name="s" value="go" />
+      </form>
+    </div>
+  </body>
+</html>
+

data/test/htdocs/04.html

@@ -0,0 +1,14 @@
+<html>
+  <head>
+    <title>test utf8</title>
+  </head>
+  <body>
+    <div id="links">
+      <ul>
+        <li><a name="ウェブ検索のサービス" href="http://www.google.co.jp/">グーグル・ジャパン</a></li>
+        <li><a name="ウェブ検索のサービス" href="http://www.yahoo.co.jp/">ヤフー・ジャパン</a></li>
+      </ul>
+    </div>
+  </body>
+</html>
+<!-- 上記の内容だけでは NKF#guess が NKF::ASCII を返すため失敗する... -->

data/test/htdocs/05.html

@@ -0,0 +1,14 @@
+<html>
+  <head>
+    <title>test sjis</title>
+  </head>
+  <body>
+    <div id="links">
+      <ul>
+        <li><a name="�E�F�u�����̃T�[�r�X" href="http://www.google.co.jp/">�O�[�O���E�W���p��</a></li>
+        <li><a name="�E�F�u�����̃T�[�r�X" href="http://www.yahoo.co.jp/">���t�[�E�W���p��</a></li>
+      </ul>
+    </div>
+  </body>
+</html>
+

data/test/htdocs/06.html

@@ -0,0 +1,14 @@
+<html>
+  <head>
+    <title>test sjis</title>
+  </head>
+  <body>
+    <div id="links">
+      <ul>
+        <li><a name="�����ָ����Υ����ӥ�" href="http://www.google.co.jp/">�������롦����ѥ�</a></li>
+        <li><a name="�����ָ����Υ����ӥ�" href="http://www.yahoo.co.jp/">��ա�������ѥ�</a></li>
+      </ul>
+    </div>
+  </body>
+</html>
+

data/test/htdocs/cgi.rb

@@ -0,0 +1,50 @@
+require 'cgi'
+require 'yaml'
+require 'base64'
+
+cgi = CGI.new
+env = Hash[ENV.collect() { |k, v| [k, v] }]
+res = {}
+
+if cgi.has_key? 'redirect'
+  l = env['REQUEST_URI'].split('cgi.rb').first + '02.html'
+  cgi.print [
+    'Status: 302 Moved Temporarily',
+    'Location: ' + l,
+    "\n"
+  ].join("\n")
+  exit 0
+end
+
+if cgi.has_key? 'auth'
+  auth = env['HTTP_AUTHORIZATION'] || ''
+  user_pass = Base64.decode64(auth.split.last || '').split(/:/, 2)
+  if user_pass.empty?
+    cgi.print [
+      'Status: 401 Unauthorized',
+      'WWW-Authenticate: Basic realm="test"',
+      "\n", 
+      'Access denied.'
+    ].join("\n")
+    exit 0
+  else
+    res['auth'] = user_pass.join(':')
+  end
+end
+
+if env['CONTENT_TYPE'] =~ %r{^multipart/form-data;}
+  upload = cgi.params['f'][0]
+  res['upload'] = {
+    'size' => upload.size,
+    'original_filename' => upload.original_filename,
+    'content_type' => upload.content_type
+  } if upload
+end
+
+res.update(
+  'params' => cgi.params,
+  'cookies' => cgi.cookies,
+  'env' => env
+)
+
+cgi.print "\n\n" + res.to_yaml

data/test/server.rb

@@ -0,0 +1,20 @@
+require 'webrick'
+
+TEST_SERVER_CONFIG = {
+  :DocumentRoot => File.join(File.expand_path(File.dirname(__FILE__)), 'htdocs'),
+  :BindAddress => '127.0.0.1',
+  :Port => 3080,
+  :CGIInterpreter => WEBrick::HTTPServlet::CGIHandler::Ruby,
+}
+
+if __FILE__ == $0
+  s = WEBrick::HTTPServer.new(TEST_SERVER_CONFIG)
+  s.mount(
+    '/cgi.rb', 
+    WEBrick::HTTPServlet::CGIHandler, 
+    TEST_SERVER_CONFIG[:DocumentRoot] + '/cgi.rb'
+  )
+  Signal.trap(:INT) { s.shutdown }
+  Signal.trap(:TERM) { exit 0 }
+  s.start
+end

data/test/test_browser.rb

@@ -0,0 +1,274 @@
+require File.dirname(__FILE__) + '/helper.rb'
+
+
+class TestBrowser < Test::Unit::TestCase
+
+  def setup
+    @text_val = 'value for text field'
+  end
+  
+  
+  def test_open
+    cs = Chainsaw.launch(TEST_URL + '01.html')
+    cs.open { |cs|
+      assert_equal cs.uri.to_s, TEST_URL + '01.html'
+      assert_instance_of Nokogiri::HTML::Document, cs.doc
+      assert_equal 200, cs.res.status
+      assert_equal  'text/html', cs.res.contenttype
+      links = cs.doc.search('//a')
+      assert_equal links.length, 5
+      cs.set_next links[1]
+    }
+    cs.open { |cs|
+      assert_equal cs.uri.to_s, TEST_URL + '02.html'
+      assert_instance_of Nokogiri::HTML::Document, cs.doc
+      links = cs.doc.search('//a')
+      cs.set_next links.last
+    }
+    cs.open { |cs|
+      assert_equal cs.uri.to_s, TEST_URL + '00.html'
+      assert_equal 404, cs.res.status
+    }
+  end
+
+  def test_back
+    d = ''
+    cs = Chainsaw.launch(TEST_URL + '01.html').
+    open { |cs|
+      d = cs.doc.to_s
+      cs.set_next cs.doc.search('//a')[1]
+    }.open.back { |cs|
+      assert_equal TEST_URL + '01.html', cs.uri.to_s
+      assert_equal d, cs.doc.to_s
+    }
+  end
+  
+  def test_open_file
+    Chainsaw.launch(TEST_URL + 'img.gif').
+    open { |cs|
+      assert_equal 200, cs.res.status
+      assert_equal 'image/gif', cs.res.contenttype
+      assert_not_nil cs.file
+      assert_instance_of Tempfile, cs.file
+      
+      original = File.stat(File.dirname(__FILE__) + '/htdocs/img.gif')
+      saved = File.stat(cs.file.path)
+      assert_equal saved.size, original.size
+    }
+  end
+  
+  def test_submit_get
+    Chainsaw.launch(TEST_URL + '03.html').
+    open { |cs|
+      assert_instance_of Nokogiri::HTML::Document, cs.doc
+      form = cs.doc.search('id("get-form")').first
+      form.xpath('//input[@name="t"]').attr('value', @text_val)
+      cs.set_next form
+    }.
+    submit { |cs|
+      x = YAML.load cs.res.content
+      assert_equal @text_val, x['params']['t'][0]
+      assert_equal 'go', x['params']['s'][0]
+    }
+  end
+  
+  def test_submit_post
+    Chainsaw.launch(TEST_URL + '03.html').
+    open { |cs|
+      assert_instance_of Nokogiri::HTML::Document, cs.doc
+      form = cs.doc.search('id("post-form")').first
+      form.xpath('//input[@name="t"]').attr('value', @text_val)
+      cs.set_next form
+    }.
+    submit { |cs|
+      x = YAML.load cs.res.content
+      assert_equal @text_val, x['params']['t'][0]
+      assert_equal 'go', x['params']['s'][0]
+    }
+  end
+  
+  
+  def test_submit_process
+    Chainsaw.launch(TEST_URL + '03.html').
+    process { |cs|
+      assert_instance_of Nokogiri::HTML::Document, cs.doc
+      form = cs.doc.search('id("post-form")').first
+      form.xpath('//input[@name="t"]').attr('value', @text_val)
+      cs.set_next form
+    }.
+    process { |cs|
+      x = YAML.load cs.res.content
+      assert_equal @text_val, x['params']['t'][0]
+      assert_equal 'go', x['params']['s'][0]
+    }
+  end
+  
+  def test_submit_file
+    file_path = File.dirname(__FILE__) + '/htdocs/01.html'
+    Chainsaw.launch(TEST_URL + '03.html').
+    open { |cs|
+      assert_instance_of Nokogiri::HTML::Document, cs.doc
+      form = cs.doc.search('id("upload-form")').first
+      form.xpath('//input[@name="f"]').attr('value', file_path)
+      cs.set_next form
+    }.
+    submit { |cs|
+      stat = File.stat(file_path)
+      x = YAML.load cs.res.content
+      data = x['upload']
+      assert_equal stat.size, data['size'].to_i
+    }
+  end
+  
+  def test_each
+    count = 0
+    Chainsaw.launch(TEST_URL + '01.html').open { |cs|
+      cs.set_next cs.doc.xpath('//a')
+    }.each { |cs|
+      count += 1
+      cs.open { |cs|
+        if count == 4
+          x = YAML.load cs.res.content
+          assert_equal TEST_URL + '01.html', x['env']['HTTP_REFERER']
+        end
+      }
+    }
+    assert_equal 5, count
+  end
+  
+  def test_each_with_index
+    count = 0
+    Chainsaw.launch(TEST_URL + '01.html').open { |cs|
+      cs.set_next cs.doc.xpath('//a')
+    }.each_with_index { |cs, index|
+      assert_equal index, count
+      count += 1
+      cs.open { |cs|
+        if count == 4
+          x = YAML.load cs.res.content
+          assert_equal TEST_URL + '01.html', x['env']['HTTP_REFERER']
+        end
+      }
+    }
+  end
+  
+  def test_referer
+    Chainsaw.launch(TEST_URL + '01.html').
+    open { |cs|
+      cs.set_next cs.doc.search('//a[contains(@href,"cgi.rb")]')[0]
+    }.
+    open { |cs|
+      x = YAML.load cs.res.content
+      assert_equal TEST_URL + '01.html', x['env']['HTTP_REFERER']
+    }
+    
+    Chainsaw.launch(TEST_URL + '01.html', {:hide_referer => true}).
+    open { |cs|
+      cs.set_next cs.doc.search('//a[contains(@href,"cgi.rb")]')[0]
+    }.
+    open { |cs|
+      x = YAML.load cs.res.content
+      assert_nil x['env']['HTTP_REFERER']
+    }
+    
+  end
+  
+=begin
+  def test_auth
+    user_pass = 'testuser:testpass'
+    Chainsaw.launch(TEST_URL + 'cgi.rb?auth').
+    set_auth(*user_pass.split(':')).
+    open {|cs|
+      x = YAML.load cs.res.content
+      assert_equal user_pass, x['auth']
+      cs.set_next cs.uri
+    }.
+    open { |cs|
+      assert_equal 200, cs.res.status
+      cs.uri
+    }
+    
+    # I want to fix this to return 401 status and content body...
+    assert_raise Chainsaw::RequestError do 
+      Chainsaw.launch(TEST_URL + 'cgi.rb?auth').
+      open { |cs|
+        assert_equal 401, cs.res.status
+      }
+    end
+    
+    assert_raise Chainsaw::RequestError do 
+      Chainsaw.launch.
+      set_auth(*user_pass.split(':')).
+      set_next(TEST_URL + 'cgi.rb?auth').
+      open { |cs|
+          assert_equal 401, cs.res.status
+      }
+    end
+  end
+=end
+  
+  def test_redirect
+    Chainsaw.launch(TEST_URL + 'cgi.rb?redirect').
+    open { |cs|
+      assert_equal TEST_URL + '02.html', cs.uri.to_s
+    }
+  end
+  
+  def test_request_failed
+    x = begin
+      Chainsaw.launch(TEST_URL).open.
+      set_next('http://localhost/').open
+    rescue => e
+      e.message
+    end
+    assert_equal 'Error occured during 2nd request; url: "http://localhost/"', x
+  end
+  
+  def test_invalid_url
+    assert_raise(TypeError) { Chainsaw.launch('ppppppppppp').open.doc }
+  end
+  
+  def test_max_history_count
+    cs = Chainsaw.launch
+    cs.max_history_count = 3
+    6.times do |i|
+      cs.set_next("#{TEST_URL}0#{i+1}.html").open
+    end
+    
+    assert_equal 3, cs.history.size
+    assert_equal TEST_URL + '06.html', cs.history.first[:location]
+  end
+  
+  def test_ignore_redirect
+    
+  end
+  
+  def test_result
+    cs = Chainsaw.launch
+    6.times do |i|
+      cs.set_next("#{TEST_URL}0#{i+1}.html").open do |cs|
+        [i, cs.uri]
+      end
+    end
+    assert_equal 6, cs.results.size
+    assert_equal 2, cs.results[2][0]
+    assert_equal TEST_URL + '03.html', cs.results[2][1].to_s
+    
+  end
+  
+  def test_aliases
+    Chainsaw.launch(TEST_URL + '03.html').> { |cs|
+      assert_instance_of Nokogiri::HTML::Document, cs.doc
+      form = cs.doc.search('id("post-form")').first
+      form.xpath('//input[@name="t"]').attr('value', @text_val)
+      cs.set_next form
+    }.> { |cs|
+      x = YAML.load cs.res.content
+      assert_equal @text_val, x['params']['t'][0]
+      assert_equal 'go', x['params']['s'][0]
+    }
+  
+  end
+  
+end
+