wombat 0.1.2 → 0.1.3

data/lib/wombat/crawler.rb

@@ -1,49 +1,52 @@
 #coding: utf-8
-require 'wombat/properties'
 require 'wombat/metadata'
+require 'wombat/property'
 require 'wombat/parser'
 require 'active_support'
 require 'date'
 
 module Wombat
   module Crawler
+    include Parser
     extend ActiveSupport::Concern
 
     module InstanceMethods
       def crawl
-        parser.parse self.class.send(:metadata)
+        parse self.class.send(:metadata)
       end
 
       def supports_city?
       end
-
-      def parser
-        @parser ||= Parser.new
-      end
-
-      def parser= parser
-        @parser = parser
-      end
     end
 
     module ClassMethods
-      [:event, :venue, :location].each do |m|
-        define_method(m) do |&block|
-          block.call(metadata["#{m.to_s}_props".to_sym]) if block
-        end
-      end
-
       def method_missing method, *args, &block
-        metadata[method] = args.first
+        if args.empty? && block
+          metadata["#{method.to_s}"] = PropertyContainer.new unless metadata["#{method.to_s}"]
+          block.call(metadata["#{method.to_s}"])
+        else
+          metadata.send method, *args, &block
+        end
       end
 
       def with_details_page
         yield metadata if block_given?
       end
 
+      def for_each selector
+        
+      end
+
+      def follow_links selector
+
+      end
+
       def supported_cities
       end
 
+      def to_ary
+      end
+
       private
       def metadata
         @metadata ||= Metadata.new

data/lib/wombat/metadata.rb

@@ -1,24 +1,14 @@
 #coding: utf-8
-module Wombat
-  class Metadata < Hash
-    def initialize
-      self[:event_props] = Properties.new
-      self[:venue_props] = Properties.new
-      self[:location_props] = Properties.new
-    end
+require 'wombat/property_container'
 
-    [:event, :venue, :location].each do |m|
-      define_method(m) do
-        self["#{m.to_s}_props".to_sym]
-      end
+module Wombat
+  class Metadata < PropertyContainer
+    def base_url url
+      self[:base_url] = url
     end
 
-    def method_missing method, *args, &block
-      if method.to_s.end_with? '='
-        self[method] = args.first
-      else
-        self[method]
-      end
+    def list_page url
+      self[:list_page] = url
     end
   end
 end

data/lib/wombat/parser.rb

@@ -1,9 +1,9 @@
-#coding: utf-8
+ #coding: utf-8
 require 'wombat/property_locator'
 require 'mechanize'
 
 module Wombat
-  class Parser
+  module Parser
     include PropertyLocator
     attr_accessor :mechanize, :context
 
@@ -12,13 +12,15 @@ module Wombat
     end
 
     def parse metadata
-      @context = @mechanize.get("#{metadata.base_url}#{metadata.event_list_page}").parser
+      @context = @mechanize.get("#{metadata[:base_url]}#{metadata[:list_page]}").parser
 
       locate metadata
 
-      [metadata.event_props, metadata.venue_props, metadata.location_props].flat_map { |p| p.all_properties }.each do |p|
+      metadata.all_properties.each do |p|
         p.callback.call(p.result) if p.callback
       end
+
+      metadata.flatten
     end
   end
 end

data/lib/wombat/property_container.rb

@@ -0,0 +1,44 @@
+#coding: utf-8
+
+module Wombat
+  class PropertyContainer < Hash
+    def add_property property
+      self[property.name] = property
+    end
+
+    def get_property name
+      self[name]
+    end
+
+    def method_missing method, *args, &block
+      self[method.to_s] = Property.new(
+        name: method.to_s,
+        selector: args.first,
+        format: args[1],
+        namespaces: args[2],
+        callback: block)
+    end
+
+    def all_properties
+      values.flat_map { |v|
+        v.kind_of?(PropertyContainer) \
+          ? v.values \
+          : v.kind_of?(Property) \
+            ? v \
+            : nil
+      }.compact
+    end
+
+    def flatten
+      Hash.new.tap do |h|
+        keys.map do |k|
+          if self[k].kind_of?(PropertyContainer)
+            h[k] = self[k].flatten
+          elsif self[k].kind_of?(Property)
+            h[k] = self[k].result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wombat/property_locator.rb

@@ -3,7 +3,7 @@
 module Wombat
   module PropertyLocator
     def locate metadata
-      [metadata.event_props, metadata.venue_props, metadata.location_props].flat_map { |p| p.all_properties }.each do |p|
+      metadata.all_properties.each do |p|
         p.result = locate_property(p).first
       end
     end
@@ -11,8 +11,8 @@ module Wombat
     private 
     def locate_property property
       result = locate_selector(property.selector, property.namespaces)
-      result.map! {|r| r.inner_html } if property.format == :html
-      result.map {|r| r.strip }
+      result.map! {|r| r.inner_html.strip } if property.format == :html
+      result.map {|r| r.kind_of?(String) ? r : r.inner_text }.map(&:strip)
     end
 
     def locate_selector selector, namespaces = nil

data/spec/crawler_spec.rb

@@ -3,10 +3,8 @@ require 'spec_helper'
 describe Wombat::Crawler do
   before(:each) do
     @crawler = Class.new
-    @parser = Wombat::Parser.new
     @crawler.send(:include, Wombat::Crawler)
     @crawler_instance = @crawler.new
-    @crawler_instance.parser = @parser
   end
 
   it 'should call the provided block' do
@@ -34,35 +32,59 @@ describe Wombat::Crawler do
     @crawler.venue { |v| v.name "Scooba" }
     @crawler.location { |v| v.latitude -50.2323 }
 
-    @parser.should_receive(:parse) do |arg|
-      arg.event_props.get_property("title").selector.should == "Fulltronic Dezembro"
-      arg.event_props.get_property("time").selector.to_s.should == time.to_s
-      arg.venue_props.get_property("name").selector.should == "Scooba"
-      arg.location_props.get_property("latitude").selector.should == -50.2323
+    @crawler_instance.should_receive(:parse) do |arg|
+      arg["event"].get_property("title").selector.should == "Fulltronic Dezembro"
+      arg["event"].get_property("time").selector.to_s.should == time.to_s
+      arg["venue"].get_property("name").selector.should == "Scooba"
+      arg["location"].get_property("latitude").selector.should == -50.2323
     end
     
     @crawler_instance.crawl
   end
 
   it 'should isolate metadata between different instances' do
-    another_parser = Wombat::Parser.new
     another_crawler = Class.new
     another_crawler.send(:include, Wombat::Crawler)
     another_crawler_instance = another_crawler.new
-    another_crawler_instance.parser = another_parser
 
     another_crawler.event { |e| e.title 'Ibiza' }
-    another_parser.should_receive(:parse) { |arg| arg.event_props.get_property("title").selector.should == "Ibiza" }
+    another_crawler_instance.should_receive(:parse) { |arg| arg["event"].get_property("title").selector.should == "Ibiza" }
     another_crawler_instance.crawl
 
     @crawler.event { |e| e.title 'Fulltronic Dezembro' }
-    @parser.should_receive(:parse) { |arg| arg.event_props.get_property("title").selector.should == "Fulltronic Dezembro" }
+    @crawler_instance.should_receive(:parse) { |arg| arg["event"].get_property("title").selector.should == "Fulltronic Dezembro" }
     @crawler_instance.crawl
   end
 
   it 'should be able to assign arbitrary plain text metadata' do
-    @crawler.some_data "/event/list"
-    @parser.should_receive(:parse) { |arg| arg.some_data.should == "/event/list" }
+    @crawler.some_data("/event/list", :html, "geo") {|p| true }
+    
+    @crawler_instance.should_receive(:parse) do |arg|
+      prop = arg['some_data']
+      prop.name.should == "some_data"
+      prop.selector.should == "/event/list"
+      prop.format.should == :html
+      prop.namespaces.should == "geo"
+      prop.callback.should_not be_nil
+    end
+    
+    @crawler_instance.crawl
+  end
+
+  it 'should be able to specify arbitrary block structure more than once' do
+    @crawler.structure do |s|
+      s.data "xpath=/xyz"
+    end
+
+    @crawler.structure do |s|
+      s.another "css=.information"
+    end
+
+    @crawler_instance.should_receive(:parse) do |arg|
+      arg["structure"]["data"].selector.should == "xpath=/xyz"
+      arg["structure"]["another"].selector.should == "css=.information"
+    end
+
     @crawler_instance.crawl
   end
 

data/spec/helpers/sample_crawler.rb

@@ -4,14 +4,33 @@ require 'wombat'
 class SampleCrawler
   include Wombat::Crawler
 
-  event do |e|
-    e.title "Sample Event"
-    e.description "This event's description"
-    e.date DateTime.now.to_date
-  end
+  base_url "http://www.obaoba.com.br"
+  list_page "/porto-alegre/agenda"
+  
+  for_each "css=div.title-agenda" do
+    event do |e|
+      e.title("xpath=.") { |t| t.split(" | ")[1].strip }
+      e.date "xpath=//div[@class='scrollable-items']/div[@class='s-item active']//a" do |d|
+        DateTime.strptime(d, '%d/%m')
+      end
+      e.type("xpath=.") { |t| t.split(" | ").first.strip.casecmp('SHOW') == 0 ? :show : :party }
+    end
+
+    venue do |v|
+      v.name("xpath=.") { |n| name.split(" | ")[2].strip }
+    end
+
+    follow_links "xpath=.//a[1]/@href" do
+      event { |e| e.description "css=#main-node-content", :html }
+      venue do |v|
+        v.phone "css=span.tel .value"
+        v.image "xpath=//div[@id='article-image']/div/img/@src"
+      end
 
-  venue do |v|
-    v.name "Cafe de La Musique"
-    v.address "324 Dom Pedro II Street"
+      location do |l|
+        l.city "css=span.locality"
+        l.street("css=span.street-address") { |s| s.gsub(/\n/, '').gsub(/  /, '') }
+      end
+    end
   end
 end

data/spec/integration/integration_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe 'basic crawler setup' do
+  it 'should crawl page' do
+    VCR.use_cassette('basic_crawler_page') do
+      crawler = Class.new
+      crawler.send(:include, Wombat::Crawler)
+      
+      crawler.base_url "http://www.terra.com.br"
+      crawler.list_page '/portal'
+
+      crawler.search "css=.btn-search"
+      crawler.social do |s|
+        s.twitter "css=.ctn-bar li.last"
+      end
+
+      crawler_instance = crawler.new
+
+      results = crawler_instance.crawl
+
+      results["search"].should == "Buscar"
+      results["social"]["twitter"].should == "Terra Magazine"
+    end
+  end
+end

data/spec/metadata_spec.rb

@@ -1,21 +1,14 @@
 require 'spec_helper'
 
 describe Wombat::Metadata do
-  it 'should have basic structure' do
-    metadata = Wombat::Metadata.new
-
-    metadata[:event_props].class.should == Wombat::Properties
-    metadata[:venue_props].class.should == Wombat::Properties
-    metadata[:location_props].class.should == Wombat::Properties
-
-    metadata.event_props.should == metadata[:event_props]
-    metadata.venue_props.should == metadata[:venue_props]
-    metadata.location_props.should == metadata[:location_props]
+  before(:each) do
+    @metadata = Wombat::Metadata.new
   end
 
-  it 'should be able to get hash key like a method' do
-    m = Wombat::Metadata.new
-    m[:some_data] = "yeah"
-    m.some_data.should == "yeah"
+  it 'should not include non-properties in all properties list' do
+    @metadata.another_property "/some/selector", :text
+    @metadata.base_url "felipecsl.com"
+    @metadata.list_page "/yeah"
+    @metadata.all_properties.should == [@metadata['another_property']]
   end
 end

data/spec/parser_spec.rb

@@ -2,13 +2,15 @@ require 'spec_helper'
 
 describe Wombat::Parser do
   before(:each) do
-    @parser = Wombat::Parser.new
+    crawler = Class.new
+    crawler.send(:include, Wombat::Parser)
+    @parser = crawler.new
     @metadata = Wombat::Metadata.new
   end
 
   it 'should request page document with correct url' do
-    @metadata[:base_url] = "http://www.google.com"
-    @metadata[:event_list_page] = "/search"
+    @metadata.base_url "http://www.google.com"
+    @metadata.list_page "/search"
     fake_document = double :document
     fake_parser = double :parser
     fake_document.should_receive(:parser).and_return(fake_parser)
@@ -26,66 +28,20 @@ describe Wombat::Parser do
     @parser.parse @metadata
   end
 
-  it 'should invoke event callbacks' do
+  it 'should invoke metadata callbacks' do
     fake_document = double :document
     fake_parser = double :parser
     property = double :property
-    properties = double :properties
     block_called = false
     block = lambda { |p| block_called = true }
     
     property.stub(:result)
     fake_document.should_receive(:parser).and_return(fake_parser)
     property.should_receive(:callback).twice.and_return(block)
-    properties.should_receive(:all_properties).and_return [property]
     
     @parser.mechanize.stub(:get).and_return fake_document
     @parser.should_receive(:locate).with(@metadata)
-    @metadata.should_receive(:event_props).and_return properties
-
-    @parser.parse @metadata
-
-    block_called.should be_true
-  end
-
-  it 'should invoke venue callbacks' do
-    fake_document = double :document
-    fake_parser = double :parser
-    property = double :property
-    properties = double :properties
-    block_called = false
-    block = lambda { |p| block_called = true }
-    
-    property.stub(:result)
-    fake_document.should_receive(:parser).and_return(fake_parser)
-    property.should_receive(:callback).twice.and_return(block)
-    properties.should_receive(:all_properties).and_return [property]
-    
-    @parser.mechanize.stub(:get).and_return fake_document
-    @parser.should_receive(:locate).with(@metadata)
-    @metadata.should_receive(:venue_props).and_return properties
-
-    @parser.parse @metadata
-
-    block_called.should be_true
-  end
-
-  it 'should invoke location callbacks' do
-    fake_document = double :document
-    fake_parser = double :parser
-    property = double :property
-    properties = double :properties
-    block_called = false
-    block = lambda { |p| block_called = true }
-    
-    property.stub(:result)
-    fake_document.should_receive(:parser).and_return(fake_parser)
-    property.should_receive(:callback).twice.and_return(block)
-    properties.should_receive(:all_properties).and_return [property]
-    
-    @parser.mechanize.stub(:get).and_return fake_document
-    @parser.should_receive(:locate).with(@metadata)
-    @metadata.should_receive(:venue_props).and_return properties
+    @metadata.should_receive(:all_properties).and_return [property]
 
     @parser.parse @metadata
 
@@ -96,7 +52,6 @@ describe Wombat::Parser do
     fake_document = double :document
     fake_parser = double :parser
     property = double :property
-    properties = double :properties
     block_called = false
     block = lambda { |p|
       block_called = true 
@@ -106,14 +61,25 @@ describe Wombat::Parser do
     property.should_receive(:result).and_return("blah")
     fake_document.should_receive(:parser).and_return(fake_parser)
     property.should_receive(:callback).twice.and_return(block)
-    properties.should_receive(:all_properties).and_return [property]
     
     @parser.mechanize.stub(:get).and_return fake_document
     @parser.should_receive(:locate).with(@metadata)
-    @metadata.should_receive(:event_props).and_return properties
+    @metadata.should_receive(:all_properties).and_return [property]
 
     @parser.parse @metadata
 
     block_called.should be_true
   end
+
+  it 'should return array with requested properties' do
+    hash = double :results
+    fake_parser = double :parser
+    fake_document = double :document
+
+    fake_document.should_receive(:parser).and_return fake_parser
+    @parser.mechanize.stub(:get).and_return fake_document
+    @metadata.should_receive(:flatten).and_return hash
+    
+    @parser.parse(@metadata).should == hash
+  end
 end