wombat 0.2.5 → 0.3.0

data/Gemfile

@@ -10,6 +10,6 @@ group :development, :test do
   gem 'yard'
   gem 'jeweler'
   gem 'rspec'
-  gem 'vcr', '2.0.0.rc1'
+  gem 'vcr'
   gem 'fakeweb'
 end

data/Gemfile.lock

@@ -37,7 +37,7 @@ GEM
     unf (0.0.4)
       unf_ext
     unf_ext (0.0.4)
-    vcr (2.0.0.rc1)
+    vcr (2.0.0)
     webrobots (0.0.12)
       nokogiri (>= 1.4.4)
     yard (0.7.4)
@@ -54,5 +54,5 @@ DEPENDENCIES
   rake
   rest-client
   rspec
-  vcr (= 2.0.0.rc1)
+  vcr
   yard

data/README.md

@@ -1,13 +1,18 @@
 # Wombat
 
-[![CI Build Status](https://secure.travis-ci.org/felipecsl/wombat.png?branch=master)](http://travis-ci.org/felipecsl/wombat)
+[![CI Build Status](https://secure.travis-ci.org/felipecsl/wombat.png?branch=master)](travis) [![Dependency Status](https://gemnasium.com/felipecsl/wombat.png?travis)](gemnasium)
 
-Generic Web crawler with a DSL that parses structured data from web pages.
+[travis]: http://travis-ci.org/felipecsl/wombat
+[gemnasium]: https://gemnasium.com/felipecsl/wombat
+
+Generic Web crawler with an elegant DSL that parses structured data from web pages.
 
 ## Usage:
 
 ``gem install wombat``
 
+OBS: Requires ruby 1.9
+
 Creating a crawler:
 
 ###### Create a class that includes ``Wombat::Crawler``:
@@ -76,6 +81,11 @@ my_crawler.crawl
  * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
  * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
 
+## Contributors
+
+ * Felipe Lima ([@felipecsl](https://github.com/felipecsl))
+ * [@sigi](https://github.com/sigi)
+
 ## Copyright
 
 Copyright (c) 2012 Felipe Lima. See LICENSE.txt for further details.

data/VERSION

@@ -1 +1 @@
-0.2.5
+0.3.0

data/lib/wombat/iterator.rb

@@ -2,9 +2,33 @@ module Wombat
   class Iterator < PropertyContainer
     attr_accessor :selector
 
-    def initialize selector
+    def initialize(selector)
       @selector = selector
       super()
-    end    
+    end
+
+    def parse
+    	raise ArgumentError.new('Must provide a block to locate property values') unless block_given?
+
+    	all_properties.each do |p|
+        p.result ||= []
+        result = yield p
+        if result
+          result = p.callback ? p.callback.call(result) : result
+          p.result << result
+        end
+      end
+    end
+
+    def flatten(depth = nil)
+    	# determine the iterator length by the biggest property array that we have
+    	length = all_properties.map(&:result).sort { |a| a.length }.last.size
+
+    	Array.new.tap do |a|
+		  	length.times do |i|
+		  		a << super(i)
+		  	end
+		  end
+	  end
   end
 end

data/lib/wombat/parser.rb

@@ -13,35 +13,25 @@ module Wombat
     end
 
     def parse metadata
-      self.context = get_parser metadata
+      self.context = parser_for metadata
       original_context = self.context
 
       metadata.iterators.each do |it|
-        select_nodes(it.selector).each do |n|
-          self.context = n
-          it.all_properties.each do |p|
-            p.result ||= []
-            result = locate(p)
-            if result
-              result = p.callback ? p.callback.call(result) : result
-              p.result << result
-            end
-          end
+        select_nodes(it.selector).each do |node|
+          self.context = node
+          it.parse { |p| locate p }
         end
       end
 
       self.context = original_context
 
-      metadata.all_properties.each do |p|
-        result = locate p
-        p.result = p.callback ? p.callback.call(result) : result
-      end
+      metadata.parse { |p| locate p }
 
       metadata.flatten
     end
 
     private 
-    def get_parser metadata
+    def parser_for metadata
       url = "#{metadata[:base_url]}#{metadata[:list_page]}"
 
       if metadata[:format] == :html

data/lib/wombat/property.rb

@@ -2,7 +2,7 @@ module Wombat
   class Property
     attr_accessor :name, :selector, :format, :namespaces, :callback, :result
 
-    def initialize options
+    def initialize(options)
       @name = options[:name]
       @selector = options[:selector]
       @format = options[:format]
@@ -10,8 +10,8 @@ module Wombat
       @callback = options[:callback]
     end
 
-    def flatten
-      result
+    def flatten(depth = nil)
+      depth ? result[depth] : result
     end
   end
 end

data/lib/wombat/property_container.rb

@@ -37,14 +37,28 @@ module Wombat
       }.compact
     end
 
-    def flatten
-      Hash.new.tap do |h|
+    def parse
+      all_properties.each do |p|
+        result = yield p if block_given?
+        p.result = p.callback ? p.callback.call(result) : result 
+      end
+    end
+
+    def flatten(depth = nil)
+      properties = Hash.new.tap do |h|
         keys.map do |k|
-          if self[k].kind_of?(PropertyContainer) || self[k].kind_of?(Property)
-            h[k] = self[k].flatten
+          val = self[k]
+          if val.is_a?(PropertyContainer) || val.is_a?(Property)
+            h[k] = val.flatten depth
           end
         end
-      end.merge(iterators.inject({}) {|memo, i| memo.merge(i.flatten) })
+      end
+
+      iters = iterators.reduce({}) do |memo, i| 
+        memo.merge("iterator#{iterators.index(i)}" => i.flatten)
+      end
+
+      properties.merge iters
     end
 
     def for_each selector

data/spec/integration/integration_spec.rb

@@ -28,7 +28,7 @@ describe 'basic crawler setup' do
       results = crawler_instance.crawl
 
       results["search"].should == "Buscar"
-      results["menu"].should =~ ["Agenda", "Brasileiro", "Brasil", "Bolsas", "Cinema", "Galerias de Fotos", "Beleza", "Esportes", "Assine o RSS"]
+      results["iterator0"].should == [{"menu"=>"Agenda"}, {"menu"=>"Brasileiro"}, {"menu"=>"Brasil"}, {"menu"=>"Bolsas"}, {"menu"=>"Cinema"}, {"menu"=>"Galerias de Fotos"}, {"menu"=>"Beleza"}, {"menu"=>"Esportes"}, {"menu"=>"Assine o RSS"}]
       results["subheader"].should == "Londres 2012"
       results["social"]["twitter"].should == "Verão"
     end
@@ -43,21 +43,22 @@ describe 'basic crawler setup' do
       crawler.list_page "/explore"
 
       crawler.for_each "css=ol.ranked-repositories li" do
-        repo 'css=h3'
-        description('css=p.description') { |d| d.gsub(/for/, '') }
+        project do |p|
+          p.repo 'css=h3'
+          p.description('css=p.description') { |d| d.gsub(/for/, '') }
+        end
       end
 
       crawler_instance = crawler.new
       results = crawler_instance.crawl
 
-      results["repo"].should =~ ["jairajs89 / Touchy.js", "mcavage / node-restify", "notlion / streetview-stereographic", "twitter / bootstrap", "stolksdorf / Parallaxjs"]
-      results["description"].should =~ [
-        "node.js REST framework specifically meant  web service APIs",
-        "A simple light-weight JavaScript library  dealing with touch events",
-        "Shader Toy + Google Map + Panoramic Explorer",
-        "HTML, CSS, and JS toolkit from Twitter",
-        "a Library  Javascript that allows easy page parallaxing"
-      ]
+      results.should == { "iterator0" => [
+        { "project" => { "repo" => "jairajs89 / Touchy.js", "description" => "A simple light-weight JavaScript library  dealing with touch events" } },
+        { "project" => { "repo" => "mcavage / node-restify", "description" => "node.js REST framework specifically meant  web service APIs" } },
+        { "project" => { "repo" => "notlion / streetview-stereographic", "description" => "Shader Toy + Google Map + Panoramic Explorer" } },
+        { "project" => { "repo" => "twitter / bootstrap", "description" => "HTML, CSS, and JS toolkit from Twitter" } },
+        { "project" => { "repo" => "stolksdorf / Parallaxjs", "description" => "a Library  Javascript that allows easy page parallaxing" } }
+      ]}
     end
   end
 
@@ -79,9 +80,21 @@ describe 'basic crawler setup' do
 
       crawler_instance = crawler.new
       results = crawler_instance.crawl
+      iterator = results['iterator0']
+
+      iterator.should == [
+        {"latitude"=>"37.807775", "longitude"=>"-122.272736"}, 
+        {"latitude"=>"37.807717", "longitude"=>"-122.270059"}, 
+        {"latitude"=>"37.869784", "longitude"=>"-122.267701"}, 
+        {"latitude"=>"37.870873", "longitude"=>"-122.269313"}, 
+        {"latitude"=>"37.782348", "longitude"=>"-122.408059"}, 
+        {"latitude"=>"37.775529", "longitude"=>"-122.437757"}, 
+        {"latitude"=>"37.771079", "longitude"=>"-122.412604"}, 
+        {"latitude"=>"37.771079", "longitude"=>"-122.412604"}, 
+        {"latitude"=>"37.784963", "longitude"=>"-122.418871"}, 
+        {"latitude"=>"37.788978", "longitude"=>"-122.40664"}
+      ]
       
-      results["latitude"].should =~ ["37.807775", "37.807717", "37.869784", "37.870873", "37.782348", "37.775529", "37.771079", "37.771079", "37.784963", "37.788978"] 
-      results["longitude"].should =~ ["-122.272736", "-122.270059", "-122.267701", "-122.269313", "-122.408059", "-122.437757", "-122.412604", "-122.412604", "-122.418871", "-122.40664"]
       results["artist"].should =~ ["Davka", "Digitalism (DJ Set)", "Gary Clark Jr.", "Lenny Kravitz", "Little Muddy", "Michael Schenker Group", "The Asteroids Galaxy Tour", "When Indie Attacks", "When Indie Attacks", "YOB"]
     end
   end

data/spec/iterator_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+describe Wombat::Iterator do
+	let(:it) { Wombat::Iterator.new "it_selector" }
+
+   context 'parse' do
+	   it 'should iterate in for_each properties' do
+	    it.prop_1 "some_selector"
+	    it.prop_2 "another_selector"
+	    
+	    it['prop_1'].should_receive(:result).twice.and_return([])
+	    it['prop_2'].should_receive(:result).twice.and_return([])
+	    
+	    parser = double :parser
+	    parser.should_receive(:locate).with(it['prop_1']).twice
+	    parser.should_receive(:locate).with(it['prop_2']).twice
+
+	    it.parse { |p| parser.locate p }
+	    it.parse { |p| parser.locate p }
+	  end
+
+	  it 'should raise if no block given' do
+	  	expect{
+	  		it.parse
+  		}.to raise_error(ArgumentError)
+	  end
+	end
+
+  it 'should flatten properties to plain hash format' do
+  	it.prop_1 "some_selector"
+    it.prop_2 "another_selector"
+
+    it.parse {|p| }
+    it.parse {|p| }
+    it['prop_1'].result = ['result 1', 'result 2']
+    it['prop_2'].result = ['result 3', 'result 4']
+
+    it.flatten.should == [
+    	{ "prop_1" => "result 1", "prop_2" => "result 3" },
+    	{ "prop_1" => "result 2", "prop_2" => "result 4" }
+    ]
+  end
+end

data/spec/parser_spec.rb

@@ -72,10 +72,6 @@ describe Wombat::Parser do
     block_called.should be_true
   end
 
-  it 'should invoke callback inside for_each block' do
-    
-  end
-
   it 'should return hash with requested properties' do
     hash = double :results
     fake_parser = double :parser
@@ -88,33 +84,6 @@ describe Wombat::Parser do
     @parser.parse(@metadata).should == hash
   end
 
-  it 'should iterate in for_each properties' do
-    fake_parser = double :parser
-    fake_document = double :document
-    c1 = double :context
-    c2 = double :context
-    it = Wombat::Iterator.new "it_selector"
-    it.prop_1 "some_selector"
-    it.prop_2 "another_selector"
-    
-    @parser.should_receive(:context=).ordered
-    @metadata.should_receive(:iterators).and_return [it]
-    @metadata.should_receive(:flatten)
-    fake_document.should_receive(:parser).and_return(fake_parser)
-    it['prop_1'].should_receive(:result).exactly(2).times.and_return([])
-    it['prop_2'].should_receive(:result).exactly(2).times.and_return([])
-    @parser.mechanize.stub(:get).and_return fake_document
-    @parser.should_receive(:select_nodes).with("it_selector").and_return [c1, c2]
-    @parser.should_receive(:context=).with(c1).ordered
-    @parser.should_receive(:context=).with(c2).ordered
-    @parser.should_receive(:context=).ordered
-    @parser.should_receive(:locate).with(it['prop_1']).twice
-    @parser.should_receive(:locate).with(it['prop_2']).twice
-    @parser.stub(:locate)
-
-    @parser.parse(@metadata)
-  end
-
   it 'should not include null results in iterated block' do
     fake_parser = double :parser
     fake_document = double :document

data/spec/property_container_spec.rb

@@ -55,7 +55,7 @@ describe Wombat::PropertyContainer do
           "section" => "Lorem Ipsum"
         }
       },
-      "felipe" => ["correa", "de souza", "lima"],
+      "iterator0"=>[{"felipe"=>"correa"}, {"felipe"=>"de souza"}, {"felipe"=>"lima"}],
       "footer" => "bla bla bla"
     }
   end

data/wombat.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "wombat"
-  s.version = "0.2.5"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Felipe Lima"]
-  s.date = "2012-03-21"
+  s.date = "2012-03-25"
   s.description = "Generic Web crawler with a DSL that parses structured data from web pages"
   s.email = "felipe.lima@gmail.com"
   s.extra_rdoc_files = [
@@ -42,6 +42,7 @@ Gem::Specification.new do |s|
     "spec/crawler_spec.rb",
     "spec/helpers/sample_crawler.rb",
     "spec/integration/integration_spec.rb",
+    "spec/iterator_spec.rb",
     "spec/metadata_spec.rb",
     "spec/parser_spec.rb",
     "spec/property_container_spec.rb",
@@ -70,7 +71,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<yard>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
-      s.add_development_dependency(%q<vcr>, ["= 2.0.0.rc1"])
+      s.add_development_dependency(%q<vcr>, [">= 0"])
       s.add_development_dependency(%q<fakeweb>, [">= 0"])
     else
       s.add_dependency(%q<mechanize>, [">= 0"])
@@ -81,7 +82,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<yard>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 0"])
-      s.add_dependency(%q<vcr>, ["= 2.0.0.rc1"])
+      s.add_dependency(%q<vcr>, [">= 0"])
       s.add_dependency(%q<fakeweb>, [">= 0"])
     end
   else
@@ -93,7 +94,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<yard>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 0"])
-    s.add_dependency(%q<vcr>, ["= 2.0.0.rc1"])
+    s.add_dependency(%q<vcr>, [">= 0"])
     s.add_dependency(%q<fakeweb>, [">= 0"])
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wombat
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-21 00:00:00.000000000 Z
+date: 2012-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -144,17 +144,17 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - '='
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.rc1
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - '='
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.rc1
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fakeweb
   requirement: !ruby/object:Gem::Requirement
@@ -204,6 +204,7 @@ files:
 - spec/crawler_spec.rb
 - spec/helpers/sample_crawler.rb
 - spec/integration/integration_spec.rb
+- spec/iterator_spec.rb
 - spec/metadata_spec.rb
 - spec/parser_spec.rb
 - spec/property_container_spec.rb