wombat 0.1.5 → 0.1.6

data/lib/wombat/crawler.rb

@@ -29,12 +29,8 @@ module Wombat
         end
       end
 
-      def with_details_page
-        yield metadata if block_given?
-      end
-
-      def for_each selector
-        
+      def for_each selector, &block
+        metadata.for_each(selector).instance_eval(&block) if block
       end
 
       def follow_links selector

data/lib/wombat/iterator.rb

@@ -0,0 +1,9 @@
+module Wombat
+  class Iterator < PropertyContainer
+    attr_accessor :selector
+
+    def initialize selector
+      @selector = selector
+    end    
+  end
+end

data/lib/wombat/metadata.rb

@@ -1,8 +1,15 @@
 #coding: utf-8
 require 'wombat/property_container'
+require 'wombat/iterator'
 
 module Wombat
   class Metadata < PropertyContainer
+    attr_accessor :iterators
+
+    def initialize
+      @iterators = []
+    end
+
     def base_url url
       self[:base_url] = url
     end
@@ -10,5 +17,11 @@ module Wombat
     def list_page url
       self[:list_page] = url
     end
+
+    def for_each selector
+      Iterator.new(selector).tap do |i|
+        iterators << i
+      end
+    end
   end
 end

data/lib/wombat/parser.rb

@@ -14,10 +14,12 @@ module Wombat
     def parse metadata
       @context = @mechanize.get("#{metadata[:base_url]}#{metadata[:list_page]}").parser
 
-      locate metadata
+      props = metadata.all_properties
 
-      metadata.all_properties.each do |p|
-        p.callback.call(p.result) if p.callback
+      locate props
+
+      props.each do |p|
+        p.result = p.callback.call(p.result) if p.callback
       end
 
       metadata.flatten

data/lib/wombat/property_container.rb

@@ -22,7 +22,7 @@ module Wombat
     def all_properties
       values.flat_map { |v|
         v.kind_of?(PropertyContainer) \
-          ? v.values \
+          ? v.all_properties \
           : v.kind_of?(Property) \
             ? v \
             : nil

data/lib/wombat/property_locator.rb

@@ -2,15 +2,15 @@
 
 module Wombat
   module PropertyLocator
-    def locate metadata
-      metadata.all_properties.each do |p|
+    def locate properties
+      properties.each do |p|
         p.result = locate_property(p).first
       end
     end
 
     private 
     def locate_property property
-      result = locate_selector(property.selector, property.namespaces)
+      result = locate_selector(property.selector, property.namespaces).to_a
       result.map! {|r| r.inner_html.strip } if property.format == :html
       result.map {|r| r.kind_of?(String) ? r : r.inner_text }.map(&:strip)
     end

data/spec/crawler_spec.rb

@@ -91,4 +91,19 @@ describe Wombat::Crawler do
   it 'should not explode if no block given' do
     @crawler.event
   end
+
+  it 'should iterate on elements inside for_each block' do
+    @crawler.for_each "css=.element" do
+      title "css=.title"
+      body "css=.body"
+    end
+
+    @crawler_instance.should_receive(:parse) do |arg|
+      arg.iterators.first.selector.should == "css=.element"
+      arg.iterators.first["title"].selector.should == "css=.title"
+      arg.iterators.first["body"].selector.should == "css=.body"
+    end 
+
+    @crawler_instance.crawl
+  end
 end

data/spec/integration/integration_spec.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require 'spec_helper'
 
 describe 'basic crawler setup' do
@@ -14,12 +15,17 @@ describe 'basic crawler setup' do
         s.twitter "css=.ctn-bar li.last"
       end
 
+      crawler.subheader "css=h2.ttl-dynamic" do |h|
+        h.gsub("London", "Londres")
+      end
+
       crawler_instance = crawler.new
 
       results = crawler_instance.crawl
 
       results["search"].should == "Buscar"
-      results["social"]["twitter"].should == "Terra Magazine"
+      results["subheader"].should == "Londres 2012"
+      results["social"]["twitter"].should == "Verão"
     end
   end
 end

data/spec/metadata_spec.rb

@@ -11,4 +11,10 @@ describe Wombat::Metadata do
     @metadata.list_page "/yeah"
     @metadata.all_properties.should == [@metadata['another_property']]
   end
+
+  it 'should store iterators' do
+    @metadata.for_each("some_selector").kind_of?(Wombat::Iterator).should be_true
+    @metadata.iterators.size.should == 1
+    @metadata.iterators.first.selector.should == "some_selector"
+  end
 end

data/spec/parser_spec.rb

@@ -24,7 +24,7 @@ describe Wombat::Parser do
     fake_parser = double :parser
     fake_document.should_receive(:parser).and_return(fake_parser)
     @parser.mechanize.stub(:get).and_return fake_document
-    @parser.should_receive(:locate).with(@metadata)
+    @parser.should_receive(:locate).with(@metadata.all_properties)
     @parser.parse @metadata
   end
 
@@ -38,10 +38,11 @@ describe Wombat::Parser do
     property.stub(:result)
     fake_document.should_receive(:parser).and_return(fake_parser)
     property.should_receive(:callback).twice.and_return(block)
+    property.should_receive(:result=).with(true)
     
     @parser.mechanize.stub(:get).and_return fake_document
-    @parser.should_receive(:locate).with(@metadata)
-    @metadata.should_receive(:all_properties).and_return [property]
+    @metadata.stub(:all_properties).and_return [property]
+    @parser.should_receive(:locate).with(@metadata.all_properties)
 
     @parser.parse @metadata
 
@@ -61,10 +62,11 @@ describe Wombat::Parser do
     property.should_receive(:result).and_return("blah")
     fake_document.should_receive(:parser).and_return(fake_parser)
     property.should_receive(:callback).twice.and_return(block)
+    property.should_receive(:result=).with(true)
     
     @parser.mechanize.stub(:get).and_return fake_document
-    @parser.should_receive(:locate).with(@metadata)
-    @metadata.should_receive(:all_properties).and_return [property]
+    @metadata.stub(:all_properties).and_return [property]
+    @parser.should_receive(:locate).with(@metadata.all_properties)
 
     @parser.parse @metadata
 

data/spec/property_container_spec.rb

@@ -9,12 +9,13 @@ describe Wombat::PropertyContainer do
     @metadata["event"] = Wombat::PropertyContainer.new
     @metadata["venue"] = Wombat::PropertyContainer.new
     @metadata.another_property "/some/selector", :text
-    @metadata["event"].something "else"
+    @metadata["event"]["something"] = Wombat::PropertyContainer.new
+    @metadata["event"]["something"].else "Wohooo"
     @metadata["venue"].awesome "whooea"
     
     all_propes = @metadata.all_properties
     
-    all_propes.should =~ [@metadata["another_property"], @metadata["event"]["something"], @metadata["venue"]["awesome"]]
+    all_propes.should =~ [@metadata["another_property"], @metadata["event"]["something"]["else"], @metadata["venue"]["awesome"]]
   end
 
   it 'should be able to change properties via all_properties' do

data/spec/property_locator_spec.rb

@@ -28,7 +28,7 @@ describe Wombat::PropertyLocator do
 
     @locator_instance.stub(:context).and_return context
     
-    @locator_instance.locate @metadata
+    @locator_instance.locate @metadata.all_properties
 
     @metadata.get_property("blah").result.should == "abc"
     @metadata["event"].get_property("data1").result.should == "Something cool"
@@ -47,7 +47,7 @@ describe Wombat::PropertyLocator do
 
     @metadata["event"].another_info "xpath=/anotherData", :html
 
-    @locator_instance.locate @metadata
+    @locator_instance.locate @metadata.all_properties
 
     @metadata["event"].get_property("another_info").result.should == "some another info"
   end
@@ -59,7 +59,7 @@ describe Wombat::PropertyLocator do
     @locator_instance.stub(:context).and_return context
     @metadata["event"].description "xpath=/event/some/description", :text, "blah"
 
-    @locator_instance.locate @metadata
+    @locator_instance.locate @metadata.all_properties
 
     @metadata["event"].get_property("description").result.should == "awesome event"
   end

data/wombat.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "wombat"
-  s.version = "0.1.5"
+  s.version = "0.1.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Felipe Lima"]
-  s.date = "2012-02-06"
+  s.date = "2012-02-08"
   s.description = "Generic Web crawler with a DSL that parses structured data from web pages"
   s.email = "felipe.lima@gmail.com"
   s.extra_rdoc_files = [
@@ -30,6 +30,7 @@ Gem::Specification.new do |s|
     "fixtures/vcr_cassettes/basic_crawler_page.yml",
     "lib/wombat.rb",
     "lib/wombat/crawler.rb",
+    "lib/wombat/iterator.rb",
     "lib/wombat/metadata.rb",
     "lib/wombat/parser.rb",
     "lib/wombat/property.rb",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wombat
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-06 00:00:00.000000000 Z
+date: 2012-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &70197219823100 !ruby/object:Gem::Requirement
+  requirement: &70163534813940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70197219823100
+  version_requirements: *70163534813940
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70197219822620 !ruby/object:Gem::Requirement
+  requirement: &70163534813160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70197219822620
+  version_requirements: *70163534813160
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70197219822140 !ruby/object:Gem::Requirement
+  requirement: &70163534812680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70197219822140
+  version_requirements: *70163534812680
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70197219821660 !ruby/object:Gem::Requirement
+  requirement: &70163534812200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70197219821660
+  version_requirements: *70163534812200
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &70197219821120 !ruby/object:Gem::Requirement
+  requirement: &70163534811700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70197219821120
+  version_requirements: *70163534811700
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70197219820520 !ruby/object:Gem::Requirement
+  requirement: &70163534811200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70197219820520
+  version_requirements: *70163534811200
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70197219819920 !ruby/object:Gem::Requirement
+  requirement: &70163534810720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70197219819920
+  version_requirements: *70163534810720
 - !ruby/object:Gem::Dependency
   name: guard
-  requirement: &70197219819200 !ruby/object:Gem::Requirement
+  requirement: &70163534810240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70197219819200
+  version_requirements: *70163534810240
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &70197219818500 !ruby/object:Gem::Requirement
+  requirement: &70163535048080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70197219818500
+  version_requirements: *70163535048080
 - !ruby/object:Gem::Dependency
   name: guard-bundler
-  requirement: &70197219818020 !ruby/object:Gem::Requirement
+  requirement: &70163535047560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70197219818020
+  version_requirements: *70163535047560
 - !ruby/object:Gem::Dependency
   name: vcr
-  requirement: &70197219817400 !ruby/object:Gem::Requirement
+  requirement: &70163535047060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -131,10 +131,10 @@ dependencies:
         version: 2.0.0.rc1
   type: :development
   prerelease: false
-  version_requirements: *70197219817400
+  version_requirements: *70163535047060
 - !ruby/object:Gem::Dependency
   name: fakeweb
-  requirement: &70197219816680 !ruby/object:Gem::Requirement
+  requirement: &70163535046440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -142,7 +142,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70197219816680
+  version_requirements: *70163535046440
 description: Generic Web crawler with a DSL that parses structured data from web pages
 email: felipe.lima@gmail.com
 executables: []
@@ -164,6 +164,7 @@ files:
 - fixtures/vcr_cassettes/basic_crawler_page.yml
 - lib/wombat.rb
 - lib/wombat/crawler.rb
+- lib/wombat/iterator.rb
 - lib/wombat/metadata.rb
 - lib/wombat/parser.rb
 - lib/wombat/property.rb