ucnv-chainsaw 0.0.0

data/README.rdoc

@@ -0,0 +1,42 @@
+= Chainsaw
+
+* http://github.com/ucnv/chainsaw/tree/master
+
+== Description
+
+A Ruby library for spidering web resources.
+
+== Features/Problems
+
+
+== Synopsis
+
+  Chainsaw.launch('http://example.com/').open { |cs|
+    cs.doc.css('#navi a')[2]
+  }.open { |cs|
+    form = cs.doc.xpath('//form')[0]
+    input = form.xpath('.//input[@type="text"]')[0]
+    input.set_attribute('value', 'blurblur')
+    cs.set_next form
+  }.submit { |cs|
+    puts cs.res.status
+    cs.doc.search('.//a') do |link| 
+      puts link.content
+    end
+  }
+
+== Requirements
+
+* nokogiri
+* httpclient
+
+== Installation
+
+* gem install ucnv-chainsaw
+
+== Copyright
+
+Author::    ucnv <ucnvvv at gmail.com>
+Copyright:: Copyright (c) 2009 ucnv
+License::   MIT
+

data/Rakefile

@@ -0,0 +1,149 @@
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/testtask'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/contrib/rubyforgepublisher'
+require 'rake/contrib/sshpublisher'
+require 'fileutils'
+include FileUtils
+
+NAME              = "chainsaw"
+AUTHOR            = "ucnv"
+EMAIL             = "ucnvvv at gmail.com"
+DESCRIPTION       = "A Ruby library for spidering web resources."
+RUBYFORGE_PROJECT = "chainsaw"
+HOMEPATH          = "http://github.com/ucnv/chainsaw/tree/master"
+BIN_FILES         = %w(  )
+VERS              = "0.0.1"
+
+REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil
+CLEAN.include ['**/.*.sw?', '*.gem', '.config']
+RDOC_OPTS = [
+  '--title', "#{NAME} documentation",
+  "--charset", "utf-8",
+  "--opname", "index.html",
+  "--line-numbers",
+  "--main", "README.rdoc",
+  "--inline-source",
+]
+
+task :default => [:test]
+task :package => [:clean]
+
+Rake::TestTask.new("test") do |t|
+  t.libs   << "test"
+  t.pattern = "test/**/test_*.rb"
+  t.verbose = true
+end
+
+spec = Gem::Specification.new do |s|
+  s.name              = NAME
+  s.version           = VERS
+  s.platform          = Gem::Platform::RUBY
+  s.has_rdoc          = true
+  s.extra_rdoc_files  = ["README.rdoc", "ChangeLog"]
+  s.rdoc_options     += RDOC_OPTS + ['--exclude', '^(examples|extras)/']
+  s.summary           = DESCRIPTION
+  s.description       = DESCRIPTION
+  s.author            = AUTHOR
+  s.email             = EMAIL
+  s.homepage          = HOMEPATH
+  s.executables       = BIN_FILES
+  s.rubyforge_project = RUBYFORGE_PROJECT
+  s.bindir            = "bin"
+  s.require_path      = "lib"
+  s.autorequire       = ""
+  s.test_files        = Dir["test/test_*.rb"]
+
+  s.add_dependency('nokogiri', '>=1.2.1')
+  s.add_dependency('httpclient', '>=2.1.4')
+  s.required_ruby_version = '>= 1.8.6'
+
+  s.files = %w(README.rdoc ChangeLog Rakefile) +
+    Dir.glob("{bin,doc,test,lib,templates,generator,extras,website,script}/**/*") + 
+    Dir.glob("ext/**/*.{h,c,rb}") +
+    Dir.glob("examples/**/*.rb") +
+    Dir.glob("tools/*.rb")
+
+  s.extensions = FileList["ext/**/extconf.rb"].to_a
+end
+
+Rake::GemPackageTask.new(spec) do |p|
+  p.need_tar = true
+  p.gem_spec = spec
+end
+
+task :install do
+  name = "#{NAME}-#{VERS}.gem"
+  sh %{rake package}
+  begin
+    sh %{sudo gem install pkg/#{name}}
+  rescue
+    sh %{gem install pkg/#{name}}
+  end
+end
+
+task :uninstall => [:clean] do
+  begin
+    sh %{sudo gem uninstall #{NAME}}
+  rescue
+    sh %{gem uninstall #{NAME}}
+  end
+end
+
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'html'
+  rdoc.options += RDOC_OPTS
+  rdoc.template = "resh"
+  #rdoc.template = "#{ENV['template']}.rb" if ENV['template']
+  if ENV['DOC_FILES']
+    rdoc.rdoc_files.include(ENV['DOC_FILES'].split(/,\s*/))
+  else
+    rdoc.rdoc_files.include('README.rdoc', 'ChangeLog')
+    rdoc.rdoc_files.include('lib/**/*.rb')
+    rdoc.rdoc_files.include('ext/**/*.c')
+  end
+end
+
+desc "Publish to RubyForge"
+task :rubyforge => [:rdoc, :package] do
+  require 'rubyforge'
+  Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, 'comelade').upload
+end
+
+desc 'Package and upload the release to rubyforge.'
+task :release => [:clean, :package] do |t|
+  v = ENV["VERSION"] or abort "Must supply VERSION=x.y.z"
+  abort "Versions don't match #{v} vs #{VERS}" unless v == VERS
+  pkg = "pkg/#{NAME}-#{VERS}"
+
+  rf = RubyForge.new
+  puts "Logging in"
+  rf.login
+
+  c = rf.userconfig
+#  c["release_notes"] = description if description
+#  c["release_changes"] = changes if changes
+  c["preformatted"] = true
+
+  files = [
+    "#{pkg}.tgz",
+    "#{pkg}.gem"
+  ].compact
+
+  puts "Releasing #{NAME} v. #{VERS}"
+  rf.add_release RUBYFORGE_PROJECT, NAME, VERS, *files
+end
+
+desc 'Show information about the gem.'
+task :debug_gem do
+  puts spec.to_ruby
+end
+
+desc 'Update gem spec'
+task :gemspec do
+  open("#{NAME}.gemspec", 'w').write spec.to_ruby
+end

data/examples/01_google.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'chainsaw'
+
+query = ARGV.shift
+
+Chainsaw.launch('http://google.com/').
+open { |a|
+  f = a.doc.xpath('//form[@name="f"]').first
+  q = f.xpath('//input[@name="q"]').first
+  q['value'] = query || 'Hello world'
+  a.set_next f
+}.
+submit { |a|
+  a.doc.search('//a').each do |l|
+    puts l.content
+  end
+}

data/examples/02_twitter.rb

@@ -0,0 +1,15 @@
+require 'rubygems'
+require 'chainsaw'
+
+username, password, tweet = ARGV
+
+Chainsaw.launch('https://twitter.com/home').> { |a|
+  f = a.doc.css('.signin').first
+  f.xpath('id("username_or_email")').first['value'] = username
+  f.xpath('id("session[password]")').first['value'] = password
+  a.set_next f
+}.> { |a|
+  f = a.doc.css('#doingForm').first
+  f.xpath('id("status")').first.content = tweet || "(o'-')-o fwip fwip"
+  a.set_next f
+}.>

data/lib/chainsaw.rb

@@ -0,0 +1,40 @@
+require 'rubygems'
+require 'nokogiri'
+require 'httpclient'
+require 'nkf'
+
+require 'chainsaw/ext/httpclient'
+require 'chainsaw/ext/nokogiri'
+
+require 'chainsaw/common'
+require 'chainsaw/element'
+
+require 'chainsaw/browser'
+
+
+Nokogiri::XML::Element.module_eval do 
+  include Chainsaw::Element
+end
+
+module Chainsaw
+  VERSION = '0.0.1'
+  
+  #
+  # Return a instance of the Chainsaw::Browser class.
+  def self.launch(*args)
+    args.pop! if args.last.is_a? Proc
+    cs = Chainsaw::Browser.new *args
+    yield cs if block_given?
+    cs
+  end
+end
+
+module Kernel
+  #
+  # alias for Chainsaw.launch
+  def Chainsaw(*args)
+    Chainsaw.launch *args
+  end
+  
+  module_function :Chainsaw
+end

data/lib/chainsaw/browser.rb

@@ -0,0 +1,262 @@
+
+
+module Chainsaw
+  
+  class Browser
+    include Chainsaw::Util
+    
+    DEFAULT_USER_AGENT = '' # TODO: set value
+    
+    attr_accessor :user_agent, :ignore_redirect, :hide_referer, :max_history_count, :encoding  # configurables
+    attr_accessor :request_headers, :url_base, :results
+    attr_reader :engine, :response, :history, :request_count      # session
+    
+    def initialize(location = '', options = {})
+      @user_agent        = options[:user_agent] || DEFAULT_USER_AGENT
+      @max_history_count = options[:max_history_count] || 20
+      @ignore_redirect   = options[:ignore_redirect] || false
+      @hide_referer      = options[:hide_referer] || false
+      @encoding          = options[:encoding]
+      @url_base          = options[:url_base] || ''
+      @request_headers   = options[:request_headers] || {}
+      
+      @history = []
+      @results = []
+      
+      @request_count = 0
+      
+      @engine = HTTPClient.new
+      
+      set_next(location)
+      self
+    end
+    
+    def add_header(key, value, overwrite_if_exists = true)
+      @request_headers = {} if @request_header.nil?
+      if !@request_headers.has_key?(key) or overwrite_if_exists
+        @request_headers[key] = value
+      end
+      self
+    end
+    
+    def set_next(obj)
+      @reserved = obj
+      self
+    end
+    
+    def set_auth(username, password)
+      @engine.set_auth(@reserved.to_s, username, password)
+      self
+    end
+    
+    def process(&block)
+      if @reserved.is_a?(Nokogiri::XML::Element) and @reserved.is_form?
+        begin
+          submit &block
+        rescue TypeError
+          open &block
+        end
+      else
+        open &block
+      end
+    end
+    
+    alias > process
+    
+    def open
+      uri = prepare_next
+      @response = request(:get, uri)
+      process_chain &Proc.new if block_given?
+      self
+    end
+    
+    def submit
+      submit_with(nil)
+      process_chain &Proc.new if block_given?
+      self
+    end
+    
+    def submit_with(button_name, image_x = -1, image_y = -1)
+      uri, form = prepare_next
+      raise TypeError, 'No form found.' unless form
+      unless button_name
+        s = form.xpath('.//input[@type="submit"]')
+        button_name = s.first['name'] if s.length
+      end
+      query = form.serialize_form(button_name, image_x, image_y)
+      method = (form['method'] || 'get').downcase
+      if method == 'get'
+        @response = request(:get, uri, query)
+      elsif method == 'post'
+        paths, q = query.partition do |name, value|
+          value.instance_of? Pathname
+        end
+        files = []
+        begin
+          paths.each do |name, path| 
+            f = File.open(path, 'rb')
+            q.push [name, f]
+            files.push f
+          end
+          @response = request(:post, uri, q) 
+        ensure
+          files.each { |f| f.close unless f.nil? }
+        end
+      end
+      process_chain &Proc.new if block_given?
+      self
+    end
+    
+    def back
+      require 'ostruct'
+      cleanup
+      @history.shift
+      back = @history.first
+      return self if back.nil?
+      @response = OpenStruct.new(
+        'uri' => back[:location],
+        'contenttype' => back[:content_type],
+        'content' => back[:document]
+      )
+      @reserved = nil
+      process_chain &Proc.new if block_given?
+      self
+    end
+    
+    def uri
+      return nil if @response.nil?
+      @response.uri
+    end
+    
+    def each(&block)
+      iterate(false, &block)
+    end
+    
+    def each_with_index(&block)
+      iterate(true, &block)
+    end
+    
+    def document
+      return @document unless @document.nil?
+      return nil if res.content.nil?
+      return nil unless is_xml_parsable?(res.contenttype)
+      enc = @encoding || Encoding.guess(res.content)
+      @document = Nokogiri.parse(res.content, nil, enc)
+      
+      b = @document.xpath('//base')
+      base = b.empty? ? '' : b[0]['href']
+      @history.first.update(:base => base.to_s)
+      
+      @document
+    end
+    
+    def file
+      return @file unless @file.nil?
+      return nil if @response.content.nil?
+      
+      Tempfile.open(uri.to_s.split('/').last) do |f|
+        f.binmode.write @response.content
+        @file = f
+      end
+      
+      @file
+    end
+    
+    alias res response
+    alias doc document
+    
+  private
+    
+    def process_chain(&block)
+      if block.arity == 0
+        results.push yield
+      else
+        results.push yield(self)
+      end
+    end
+    
+    def request(method, uri, query = nil)
+      raise TypeError, "Invalid URI: #{qq uri.to_s}" unless uri.kind_of?(URI::HTTP)
+      
+      call = method.to_s || 'get'
+      call += '_r' unless @ignore_redirect
+      add_header('Referer', @history.first[:location], false) if !@hide_referer and @history.first
+      @request_count += 1
+      r = begin
+        @engine.send(call, uri, query, @request_headers)
+      rescue
+        raise(
+          Chainsaw::RequestError, 
+          "Error occured during #{to_nth(@request_count)} request; url: #{qq uri.to_s}"
+        )
+      end
+      
+      # history
+      set_history uri.to_s, r.contenttype, r.content
+      r
+    end
+    
+    def iterate(with_index, &block)
+      set = prepare_next
+      unless set
+        warn 'Iteration called without Enumerable.'
+        return self 
+      end
+      
+      set.each_with_index do |e, index|
+        set_next(e)
+        with_index ? yield(self, index) : yield(self)
+        back
+      end
+      self
+    end
+    
+    def prepare_next(reserved = nil)
+      reserved ||= @reserved
+      cleanup
+      case reserved
+      when String   # expecting a url
+        return prepare_next(URI.parse reserved)
+      when URI
+        if reserved.is_a?(URI::HTTP)
+          return reserved
+        elsif @history.size > 0
+          base = @history[0][:base] || ''
+          base = @history[0][:location] if base.empty?
+          l = URI.join base, reserved
+          return prepare_next(l) if l.is_a?(URI::HTTP)
+        end
+      when Nokogiri::XML::Element
+        url = reserved['href'] || reserved['action'] || reserved['value']
+        if reserved.is_form?
+          return [prepare_next(url), reserved]
+        else
+          return prepare_next(url)
+        end
+      when Enumerable
+        return reserved
+      end
+      
+      raise TypeError, "Unexpected value is set for next before #{to_nth(@request_count+1)} request: #{@reserved.class.name}"
+    end
+    
+    def cleanup
+      @reserved = @document = @response = @file = nil
+    end
+    
+    def set_history(location, content_type, document, base = nil)
+      document = nil unless is_xml_parsable?(content_type)
+      data = {
+        :location => location, 
+        :content_type => content_type,
+        :document => document, 
+        :base => base
+      }
+      @history.unshift(data)
+      @history.slice! @max_history_count, @history.size - @max_history_count
+    end
+    
+    
+    
+  end
+end