wombat 2.1.1 → 2.1.2

data/README.md

@@ -1,7 +1,8 @@
 # Wombat
 
-[![CI Build Status](https://secure.travis-ci.org/felipecsl/wombat.png?branch=master)][travis] [![Dependency Status](https://gemnasium.com/felipecsl/wombat.png?travis)][gemnasium] [![Code Climate](https://codeclimate.com/badge.png)][codeclimate]
+[[![Gem Version](https://badge.fury.io/rb/wombat.png)](http://badge.fury.io/rb/wombat)][rubygems] [![CI Build Status](https://secure.travis-ci.org/felipecsl/wombat.png?branch=master)][travis] [![Dependency Status](https://gemnasium.com/felipecsl/wombat.png?travis)][gemnasium] [![Code Climate](https://codeclimate.com/github/felipecsl/wombat.png)][codeclimate]
 
+[rubygems]: http://rubygems.org/gems/wombat
 [travis]: http://travis-ci.org/felipecsl/wombat
 [gemnasium]: https://gemnasium.com/felipecsl/wombat
 [codeclimate]: https://codeclimate.com/github/felipecsl/wombat
@@ -25,19 +26,19 @@ Wombat.crawl do
   base_url "http://www.github.com"
   path "/"
 
-  headline "xpath=//h1"
-  subheading "css=p.subheading"
+  headline xpath: "//h1"
+  subheading css: "p.subheading"
 
-  what_is "css=.teaser h3", :list
+  what_is({ css: ".teaser h3" }, :list)
 
   links do
-    explore 'xpath=//*[@id="wrapper"]/div[1]/div/ul/li[1]/a' do |e|
+    explore xpath: '//*[@id="wrapper"]/div[1]/div/ul/li[1]/a' do |e|
       e.gsub(/Explore/, "Love")
     end
 
-    search 'css=.search'
-    features 'css=.features'
-    blog 'css=.blog'
+    search css: '.search'
+    features css: '.features'
+    blog css: '.blog'
   end
 end
 ```
@@ -62,10 +63,11 @@ end
 }
 ```
 
-### This is just a sneak peek of what Wombat can do. For the complete documentation, please check the [project Wiki](http://github.com/felipecsl/wombat/wiki).
-### [API Documentation](http://rubydoc.info/gems/wombat/2.0.0/frames)
-### [Changelog](https://github.com/felipecsl/wombat/wiki/Changelog)
+### This is just a sneak peek of what Wombat can do. For the complete documentation, please check the links below:
 
+### [Wiki](http://github.com/felipecsl/wombat/wiki).
+### [API Documentation](http://rubydoc.info/gems/wombat/2.1.1/frames)
+### [Changelog](https://github.com/felipecsl/wombat/wiki/Changelog)
 
 ## Contributing to Wombat
 

data/VERSION

@@ -1 +1 @@
-2.1.1
+2.1.2

data/examples/hashes.rb

@@ -0,0 +1,43 @@
+#coding: utf-8
+require 'wombat'
+
+data = Wombat.crawl do
+  base_url "http://www.github.com"
+  path "/"
+
+  headline xpath: "//h1"
+  subheading css: "p.subheading"
+
+  what_is({ css: ".teaser h3" }, :list)
+
+  links do
+    explore xpath: '//*[@id="wrapper"]/div[1]/div/ul/li[1]/a' do |e|
+      e.gsub(/Explore/, "Love")
+    end
+
+    search css: '.search'
+    features css: '.features'
+    blog css: '.blog'
+  end
+end
+
+pp data
+
+=begin
+pp data
+{
+ "headline"=>"Build software better, together.",
+ "subheading"=>
+  "Powerful collaboration, review, and code management for open source and private development projects.",
+ "what_is"=>
+  ["Great collaboration starts with communication.",
+   "Manage and contribute from all your devices.",
+   "The world’s largest open source community."],
+ "links"=>
+  {"explore"=>"Love GitHub",
+   "search"=>"Search",
+   "features"=>"Features",
+   "blog"=>"Blog"
+  }
+}
+=end

data/lib/wombat.rb

@@ -4,12 +4,23 @@ require 'wombat/crawler'
 
 module Wombat
 	class << self
+
+    attr_reader :proxy_args
+
 		def crawl(&block)
 			klass = Class.new
 			klass.send(:include, Wombat::Crawler)
 			klass.new.crawl(&block)
 		end
-		
+
+    def configure
+      yield self
+    end
+
+    def set_proxy(*args)
+      @proxy_args = args
+    end
+
 		alias_method :scrape, :crawl
 	end
 end

data/lib/wombat/dsl/follower.rb

@@ -5,12 +5,12 @@ module Wombat
 
       def initialize(name, selector)
         @wombat_property_selector = selector
-        
+
         super(name)
       end
 
       # So that Property::Locators::Iterator can identify this class
-      # as an iterator property.
+      # as a follow property.
       def wombat_property_format
         :follow
       end

data/lib/wombat/processing/node_selector.rb

@@ -1,12 +1,23 @@
 module Wombat
-	module Processing
-	  module NodeSelector
-	    def select_nodes(selector, namespaces = nil)
-	      return [selector.to_s] if selector.is_a? Symbol
-	      return @context.xpath selector[6..-1], namespaces if selector.start_with? "xpath="
-	      return @context.css selector[4..-1] if selector.start_with? "css="
-	      [selector]
-	    end
-	  end
+  module Processing
+    module NodeSelector
+      def select_nodes(selector, namespaces = nil)
+        return [selector.to_s] if selector.is_a? Symbol
+
+        if selector.is_a? String
+          return @context.xpath selector[6..-1], namespaces if selector.start_with? "xpath="
+          return @context.css selector[4..-1] if selector.start_with? "css="
+        end
+
+        if selector.is_a? Hash
+          method = selector.flatten.first
+          selector = selector.flatten.last
+          return @context.xpath selector, namespaces if method == :xpath
+          return @context.css selector if method == :css
+        end
+
+        [selector]
+      end
+    end
   end
 end

data/lib/wombat/processing/parser.rb

@@ -19,10 +19,7 @@ module Wombat
 
       def initialize
         @mechanize = Mechanize.new
-      end
-
-      def set_proxy(host, port)
-      	@mechanize.set_proxy host, port
+        @mechanize.set_proxy(*Wombat.proxy_args) if Wombat.proxy_args
       end
 
       def parse(metadata)

data/spec/integration/integration_spec.rb

@@ -6,7 +6,7 @@ describe 'basic crawler setup' do
     VCR.use_cassette('basic_crawler_page') do
       crawler = Class.new
       crawler.send(:include, Wombat::Crawler)
-      
+
       crawler.base_url "http://www.terra.com.br"
       crawler.path '/portal'
 
@@ -32,6 +32,36 @@ describe 'basic crawler setup' do
     end
   end
 
+  it 'should support hash based selectors' do
+    VCR.use_cassette('basic_crawler_page') do
+      crawler = Class.new
+      crawler.send(:include, Wombat::Crawler)
+
+      crawler.base_url "http://www.terra.com.br"
+      crawler.path '/portal'
+
+      crawler.search css: ".btn-search"
+      crawler.social do
+        twitter css: ".ctn-bar li.last"
+      end
+      crawler.links({css: ".ctn-links"}, :iterator) do
+        menu css: "a"
+      end
+      crawler.subheader css: "h2.ttl-dynamic" do |h|
+        h.gsub("London", "Londres")
+      end
+
+      crawler_instance = crawler.new
+
+      results = crawler_instance.crawl
+
+      results["search"].should == "Buscar"
+      results["links"].should == [{"menu"=>"Agenda"}, {"menu"=>"Brasileiro"}, {"menu"=>"Brasil"}, {"menu"=>"Bolsas"}, {"menu"=>"Cinema"}, {"menu"=>"Galerias de Fotos"}, {"menu"=>"Beleza"}, {"menu"=>"Esportes"}, {"menu"=>"Assine o RSS"}]
+      results["subheader"].should == "Londres 2012"
+      results["social"]["twitter"].should == "Verão"
+    end
+  end
+
   it 'should clear iterators between multiple runs' do
     crawler = Class.new
     crawler.send(:include, Wombat::Crawler)
@@ -56,7 +86,7 @@ describe 'basic crawler setup' do
     VCR.use_cassette('basic_crawler_page') do
       results = crawler_instance.crawl
     end
-    
+
     results["links"].should == result_hash
   end
 
@@ -123,7 +153,7 @@ describe 'basic crawler setup' do
     VCR.use_cassette('for_each_page') do
       crawler = Class.new
       crawler.send(:include, Wombat::Crawler)
-      
+
       crawler.base_url "https://www.github.com"
       crawler.path "/explore"
 
@@ -151,13 +181,13 @@ describe 'basic crawler setup' do
     VCR.use_cassette('xml_with_namespace') do
       crawler = Class.new
       crawler.send(:include, Wombat::Crawler)
-      
+
       crawler.document_format :xml
       crawler.base_url "http://ws.audioscrobbler.com"
       crawler.path "/2.0/?method=geo.getevents&location=#{URI.escape('San Francisco')}&api_key=060decb474b73437d5bbec37f527ae7b"
 
       crawler.artist "xpath=//title", :list
-      
+
       crawler.location 'xpath=//event', :iterator do
         latitude "xpath=./venue/location/geo:point/geo:lat", :text, { 'geo' => 'http://www.w3.org/2003/01/geo/wgs84_pos#' }
         longitude "xpath=./venue/location/geo:point/geo:long", :text, { 'geo' => 'http://www.w3.org/2003/01/geo/wgs84_pos#' }
@@ -168,18 +198,18 @@ describe 'basic crawler setup' do
       iterator = results['location']
 
       iterator.should == [
-        {"latitude"=>"37.807775", "longitude"=>"-122.272736"}, 
-        {"latitude"=>"37.807717", "longitude"=>"-122.270059"}, 
-        {"latitude"=>"37.869784", "longitude"=>"-122.267701"}, 
-        {"latitude"=>"37.870873", "longitude"=>"-122.269313"}, 
-        {"latitude"=>"37.782348", "longitude"=>"-122.408059"}, 
-        {"latitude"=>"37.775529", "longitude"=>"-122.437757"}, 
-        {"latitude"=>"37.771079", "longitude"=>"-122.412604"}, 
-        {"latitude"=>"37.771079", "longitude"=>"-122.412604"}, 
-        {"latitude"=>"37.784963", "longitude"=>"-122.418871"}, 
+        {"latitude"=>"37.807775", "longitude"=>"-122.272736"},
+        {"latitude"=>"37.807717", "longitude"=>"-122.270059"},
+        {"latitude"=>"37.869784", "longitude"=>"-122.267701"},
+        {"latitude"=>"37.870873", "longitude"=>"-122.269313"},
+        {"latitude"=>"37.782348", "longitude"=>"-122.408059"},
+        {"latitude"=>"37.775529", "longitude"=>"-122.437757"},
+        {"latitude"=>"37.771079", "longitude"=>"-122.412604"},
+        {"latitude"=>"37.771079", "longitude"=>"-122.412604"},
+        {"latitude"=>"37.784963", "longitude"=>"-122.418871"},
         {"latitude"=>"37.788978", "longitude"=>"-122.40664"}
       ]
-      
+
       results["artist"].should =~ ["Davka", "Digitalism (DJ Set)", "Gary Clark Jr.", "Lenny Kravitz", "Little Muddy", "Michael Schenker Group", "The Asteroids Galaxy Tour", "When Indie Attacks", "When Indie Attacks", "YOB"]
     end
   end
@@ -188,7 +218,7 @@ describe 'basic crawler setup' do
     VCR.use_cassette('follow_links') do
       crawler = Class.new
       crawler.send(:include, Wombat::Crawler)
-      
+
       crawler.base_url "https://www.github.com"
       crawler.path "/"
 
@@ -199,7 +229,7 @@ describe 'basic crawler setup' do
       crawler_instance = crawler.new
       results = crawler_instance.crawl
 
-      results.should == { 
+      results.should == {
         "github" => [
           { "heading"=>"GitHub helps people build software together." },
           { "heading"=>nil },

data/spec/wombat_spec.rb

@@ -17,9 +17,16 @@ describe Wombat do
 		Wombat.scrape
 	end
 
+	it 'should provide configuration method with block' do
+		Wombat.configure do |config|
+			config.set_proxy "10.0.0.1", 8080
+		end
+		Wombat.proxy_args.should == ["10.0.0.1", 8080]
+	end
+
 	it 'should accept regular properties (non-selectors)' do
 		VCR.use_cassette('broken_selector') do
-			lambda { 
+			lambda {
 				Wombat.crawl do
 					base_url "http://www.github.com"
 		  		path "/"

data/wombat.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = "wombat"
-  s.version = "2.1.1"
+  s.version = "2.1.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Felipe Lima"]
@@ -27,6 +27,7 @@ Gem::Specification.new do |s|
     "README.md",
     "Rakefile",
     "VERSION",
+    "examples/hashes.rb",
     "examples/iterator.rb",
     "examples/list.rb",
     "examples/no_class.rb",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wombat
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -189,6 +189,7 @@ files:
 - README.md
 - Rakefile
 - VERSION
+- examples/hashes.rb
 - examples/iterator.rb
 - examples/list.rb
 - examples/no_class.rb