wombat 1.0.0 → 2.0.0

data/lib/wombat/crawler.rb

@@ -1,13 +1,13 @@
 #coding: utf-8
-require 'wombat/metadata'
-require 'wombat/property'
-require 'wombat/parser'
+require 'wombat/dsl/metadata'
+require 'wombat/dsl/property'
+require 'wombat/processing/parser'
 require 'active_support'
 require 'date'
 
 module Wombat
   module Crawler
-    include Parser
+    include Processing::Parser
     extend ActiveSupport::Concern
 
     def crawl(&block)
@@ -31,33 +31,23 @@ module Wombat
       end
     end
 
+    alias_method :scrape, :crawl
+
     def method_missing(method, *args, &block)
       self.class.send method, *args, &block
     end
 
-    def for_each(selector, &block)
-      self.class.for_each selector, &block
-    end
-
     module ClassMethods
       def method_missing(method, *args, &block)
         metadata.send method, *args, &block
       end
 
-      def for_each(selector, &block)
-        metadata.for_each(selector).instance_eval(&block) if block
-      end
-
-      def follow_links(selector)
-
-      end
-
       def to_ary
       end
 
       private
       def metadata
-        @metadata ||= Metadata.new
+        @metadata ||= DSL::Metadata.new
       end
     end
   end

data/lib/wombat/dsl/follower.rb

@@ -0,0 +1,19 @@
+module Wombat
+  module DSL
+    class Follower < PropertyGroup
+      attr_accessor :wombat_property_selector
+
+      def initialize(name, selector)
+        @wombat_property_selector = selector
+        
+        super(name)
+      end
+
+      # So that Property::Locators::Iterator can identify this class
+      # as an iterator property.
+      def wombat_property_format
+        :follow
+      end
+    end
+  end
+end

data/lib/wombat/dsl/iterator.rb

@@ -0,0 +1,19 @@
+module Wombat
+  module DSL
+    class Iterator < PropertyGroup
+      attr_accessor :wombat_property_selector
+
+      def initialize(name, selector)
+        @wombat_property_selector = selector
+        
+        super(name)
+      end
+
+      # So that Property::Locators::Iterator can identify this class
+      # as an iterator property.
+      def wombat_property_format
+        :iterator
+      end
+    end
+  end
+end

data/lib/wombat/dsl/metadata.rb

@@ -0,0 +1,27 @@
+#coding: utf-8
+require 'wombat/dsl/property_group'
+require 'wombat/dsl/iterator'
+require 'wombat/dsl/follower'
+
+module Wombat
+  module DSL
+    class Metadata < PropertyGroup
+      def initialize
+        self[:document_format] = :html
+        super
+      end
+
+      def base_url(url)
+        self[:base_url] = url
+      end
+
+      def path(url)
+        self[:path] = url
+      end
+
+      def document_format(format)
+        self[:document_format] = format
+      end
+    end
+  end
+end

data/lib/wombat/dsl/property.rb

@@ -0,0 +1,27 @@
+module Wombat
+  module DSL
+    class Property
+      attr_accessor :wombat_property_name, :wombat_property_selector, :wombat_property_format, :wombat_property_namespaces, :callback
+
+      def initialize(name, *args, &block)
+        @wombat_property_name = name
+        @wombat_property_selector = args[0]
+        @wombat_property_format = args[1] || :text
+        @wombat_property_namespaces = args[2]
+        @callback = block
+      end
+
+      def selector
+        @wombat_property_selector
+      end
+
+      def namespaces
+        @wombat_property_namespaces
+      end
+
+      def format
+        @wombat_property_format
+      end
+    end
+  end
+end

data/lib/wombat/dsl/property_group.rb

@@ -0,0 +1,48 @@
+#coding: utf-8
+
+module Wombat
+  module DSL
+    class PropertyGroup < Hash
+      attr_accessor :wombat_property_name
+
+      def initialize(name = nil)
+        @wombat_property_name = name
+      end
+
+      def method_missing(method, *args, &block)
+        property_name = method.to_s
+
+        if args.empty? && block
+          # TODO: Verify if another property with same name already exists
+          # before overwriting
+          property_group = self[property_name] || PropertyGroup.new(property_name)
+          self[property_name] = property_group
+          property_group.instance_eval(&block)
+        else
+          if args[1] == :iterator
+            it = Iterator.new(property_name, args.first)
+            self[property_name] = it
+            it.instance_eval(&block) if block
+          elsif args[1] == :follow
+            it = Follower.new(property_name, args.first)
+            self[property_name] = it
+            it.instance_eval(&block) if block
+          else
+            self[property_name] = Property.new(property_name, *args, &block)
+          end
+        end      
+      end
+
+      def to_ary
+      end
+
+      def wombat_property_format
+        :container
+      end
+
+      def wombat_property_namespaces
+        nil
+      end
+    end
+  end
+end

data/lib/wombat/processing/node_selector.rb

@@ -0,0 +1,12 @@
+module Wombat
+	module Processing
+	  module NodeSelector
+	    def select_nodes(selector, namespaces = nil)
+	      return [selector.to_s] if selector.is_a? Symbol
+	      return @context.xpath selector[6..-1], namespaces if selector.start_with? "xpath="
+	      return @context.css selector[4..-1] if selector.start_with? "css="
+	      [selector]
+	    end
+	  end
+  end
+end

data/lib/wombat/processing/parser.rb

@@ -0,0 +1,48 @@
+#coding: utf-8
+require 'wombat/property/locators/factory'
+require 'wombat/processing/node_selector'
+require 'mechanize'
+require 'restclient'
+
+module Wombat
+  module Processing
+    module Parser
+      attr_accessor :mechanize, :context, :response_code, :page
+
+      def initialize
+        @mechanize = Mechanize.new
+      end
+
+      def parse(metadata)
+        @context = parser_for metadata
+
+        Wombat::Property::Locators::Factory.locator_for(metadata).locate(@context, @mechanize)
+      end
+
+      private
+      def parser_for(metadata)
+        url = "#{metadata[:base_url]}#{metadata[:path]}"
+        page = nil
+        parser = nil
+        begin
+          if metadata[:document_format] == :html
+            @page = @mechanize.get(url)
+            parser = @page.parser
+          else
+            @page = RestClient.get(url)
+            parser = Nokogiri::XML @page
+          end
+          @response_code = @page.code.to_i if @page.respond_to? :code
+          parser
+        rescue
+          if $!.respond_to? :http_code
+            @response_code = $!.http_code.to_i
+          elsif $!.respond_to? :response_code
+            @response_code = $!.response_code.to_i
+          end
+          raise $!
+        end
+      end
+    end
+  end
+end

data/lib/wombat/property/locators/base.rb

@@ -0,0 +1,33 @@
+#coding: utf-8
+require 'wombat/processing/node_selector'
+
+module Wombat
+  module Property
+    module Locators
+      # Abstract base class
+      class Base
+        include Wombat::Processing::NodeSelector
+        
+        def initialize(property)
+          @property = property
+        end
+
+        def locate(context, page = nil)
+          @context = context
+          
+          raw_data = yield if block_given?
+          data = @property.respond_to?(:callback) && @property.callback ? @property.callback.call(raw_data) : raw_data 
+
+          @property.wombat_property_name ? { @property.wombat_property_name => data } : data
+        end
+
+      protected
+        def locate_nodes(context)
+          @context = context
+
+          select_nodes @property.wombat_property_selector, @property.wombat_property_namespaces
+        end
+      end
+    end
+  end
+end

data/lib/wombat/property/locators/factory.rb

@@ -0,0 +1,39 @@
+#coding: utf-8
+require 'wombat/property/locators/base'
+require 'wombat/property/locators/follow'
+require 'wombat/property/locators/html'
+require 'wombat/property/locators/iterator'
+require 'wombat/property/locators/property_group'
+require 'wombat/property/locators/list'
+require 'wombat/property/locators/text'
+
+class Wombat::Property::Locators::UnknownTypeException < Exception; end;
+
+module Wombat
+	module Property
+		module Locators
+			module Factory
+				def self.locator_for(property)
+					klass = case(property.wombat_property_format)
+					when :text 
+						Text
+					when :list
+						List
+					when :html 
+						Html
+					when :iterator
+						Iterator
+					when :container
+						PropertyGroup
+					when :follow
+						Follow
+					else 
+	      		raise Wombat::Property::Locators::UnknownTypeException.new("Unknown property format #{property.format}.")
+					end
+
+					klass.new(property)
+				end
+			end
+		end
+	end
+end

data/lib/wombat/property/locators/follow.rb

@@ -0,0 +1,25 @@
+#coding: utf-8
+
+module Wombat
+	module Property
+	  module Locators
+	    class Follow < Base
+	    	def locate(context, page = nil)
+	    		super do
+            locate_nodes(context).flat_map do |node|
+              target_page = page.click node
+              context = target_page.parser
+
+              Hash.new.tap do |h|
+                @property.values
+                  .select { |v| v.is_a?(Wombat::DSL::Property) || v.is_a?(Wombat::DSL::PropertyGroup) }
+                  .map { |p| Factory.locator_for(p).locate(context, page) }
+                  .map { |p| h.merge! p }
+              end
+            end
+          end
+	    	end
+	    end
+	  end
+	end
+end

data/lib/wombat/property/locators/html.rb

@@ -0,0 +1,14 @@
+#coding: utf-8
+
+module Wombat
+	module Property
+	  module Locators
+	    class Html < Base
+	    	def locate(context, page = nil)
+	    		node = locate_nodes(context).first
+	    		super { node.inner_html.strip }
+	    	end
+	    end
+	  end
+  end
+end

data/lib/wombat/property/locators/iterator.rb

@@ -0,0 +1,23 @@
+#coding: utf-8
+require 'wombat/property/locators/property_group'
+
+module Wombat
+  module Property
+    module Locators
+      class Iterator < Base
+      	def locate(contex, page = nil)
+          super do
+            locate_nodes(contex).flat_map do |node|
+              Hash.new.tap do |h|
+                @property.values
+                  .select { |v| v.is_a?(Wombat::DSL::Property) || v.is_a?(Wombat::DSL::PropertyGroup) }
+                  .map { |p| Factory.locator_for(p).locate(node, page) }
+                  .map { |p| h.merge! p }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wombat/property/locators/list.rb

@@ -0,0 +1,17 @@
+#coding: utf-8
+
+module Wombat
+	module Property
+	  module Locators
+	    class List < Base
+	    	def locate(context, page = nil)
+	    		super do
+	    			locate_nodes(context).map do |n| 
+	    				n.is_a?(String) ? n.strip : n.inner_text.strip
+	    			end
+	    		end
+	    	end
+	    end
+	  end
+  end
+end

data/lib/wombat/property/locators/property_group.rb

@@ -0,0 +1,20 @@
+#coding: utf-8
+
+module Wombat
+  module Property
+    module Locators
+      class PropertyGroup < Base
+        def locate(context, page = nil)
+          super do
+            Hash.new.tap do |h|
+              @property.values
+                .select { |v| v.is_a?(Wombat::DSL::Property) || v.is_a?(Wombat::DSL::PropertyGroup) }
+                .map { |p| Factory.locator_for(p).locate(context, page) }
+                .map { |p| h.merge! p }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wombat/property/locators/text.rb

@@ -0,0 +1,22 @@
+#coding: utf-8
+
+module Wombat
+  module Property
+    module Locators
+      class Text < Base
+        def locate(context, page = nil)
+          node = locate_nodes(context).first
+          
+          value = 
+            unless node
+              nil
+            else 
+              node.is_a?(String) ? node.strip : node.inner_text.strip
+            end
+            
+          super { value }
+        end
+      end
+    end
+  end
+end

data/lib/wombat.rb

@@ -3,9 +3,13 @@
 require 'wombat/crawler'
 
 module Wombat
-	def self.crawl(&block)
-		klass = Class.new
-		klass.send(:include, Wombat::Crawler)
-		klass.new.crawl(&block)
+	class << self
+		def crawl(&block)
+			klass = Class.new
+			klass.send(:include, Wombat::Crawler)
+			klass.new.crawl(&block)
+		end
+		
+		alias_method :scrape, :crawl
 	end
 end

data/spec/crawler_spec.rb

@@ -1,13 +1,13 @@
 require 'spec_helper'
 
 describe Wombat::Crawler do
-  describe '#crawl' do
-    before(:each) do
-      @crawler = Class.new
-      @crawler.send(:include, Wombat::Crawler)
-      @crawler_instance = @crawler.new
-    end
+  before(:each) do
+    @crawler = Class.new
+    @crawler.send(:include, Wombat::Crawler)
+    @crawler_instance = @crawler.new
+  end
 
+  describe '#crawl' do
     it 'should call the provided block' do
       event_called = false
 
@@ -17,8 +17,8 @@ describe Wombat::Crawler do
     end
 
     it 'should provide metadata to yielded block' do
-      @crawler.event do |e|
-        e.should_not be_nil
+      @crawler.event do
+        self.class.should == Wombat::DSL::PropertyGroup
       end
     end
 
@@ -30,7 +30,10 @@ describe Wombat::Crawler do
         e.time Time.now
       end
 
-      @crawler.venue { |v| v.name "Scooba" }
+      @crawler.venue do |v| 
+        v.name "Scooba"
+      end
+
       @crawler.location { |v| v.latitude -50.2323 }
 
       @crawler_instance.should_receive(:parse) do |arg|
@@ -62,7 +65,7 @@ describe Wombat::Crawler do
 
       @crawler_instance.should_receive(:parse) do |arg|
         prop = arg['some_data']
-        prop.name.should == "some_data"
+        prop.wombat_property_name.should == "some_data"
         prop.selector.should == "/event/list"
         prop.format.should == :html
         prop.namespaces.should == "geo"
@@ -73,12 +76,12 @@ describe Wombat::Crawler do
     end
 
     it 'should be able to specify arbitrary block structure more than once' do
-      @crawler.structure do |s|
-        s.data "xpath=/xyz"
+      @crawler.structure do
+        data "xpath=/xyz"
       end
 
-      @crawler.structure do |s|
-        s.another "css=.information"
+      @crawler.structure do
+        another "css=.information"
       end
 
       @crawler_instance.should_receive(:parse) do |arg|
@@ -93,26 +96,6 @@ describe Wombat::Crawler do
       @crawler.event
     end
 
-    it 'should iterate on elements inside for_each block' do
-      @crawler.for_each "css=.element" do
-        title "css=.title"
-        body "css=.body"
-        event do |e|
-          e.all "yeah"
-        end
-      end
-
-      @crawler_instance.should_receive(:parse) do |arg|
-        it = arg.iterators.first
-        it.selector.should == "css=.element"
-        it["title"].selector.should == "css=.title"
-        it["body"].selector.should == "css=.body"
-        it["event"]["all"].selector.should == "yeah"
-      end
-
-      @crawler_instance.crawl
-    end
-
     it 'should assign metadata format' do
       @crawler_instance.should_receive(:parse) do |arg|
         arg[:document_format].should == :xml
@@ -123,22 +106,22 @@ describe Wombat::Crawler do
 
     it 'should crawl with block' do
       @crawler.base_url "danielnc.com"
-      @crawler.list_page "/itens"
+      @crawler.path "/itens"
 
       @crawler_instance.should_receive(:parse) do |arg|
         arg[:base_url].should == "danielnc.com"
-        arg[:list_page].should == "/itens/1"
+        arg[:path].should == "/itens/1"
       end
 
       @crawler_instance.crawl do
-        list_page "/itens/1"
+        path "/itens/1"
       end
 
       another_instance = @crawler.new
 
       another_instance.should_receive(:parse) do |arg|
         arg[:base_url].should == "danielnc.com"
-        arg[:list_page].should == "/itens"
+        arg[:path].should == "/itens"
       end
 
       another_instance.crawl
@@ -146,15 +129,15 @@ describe Wombat::Crawler do
 
     it 'should remove created method missing' do
       @crawler.base_url "danielnc.com"
-      @crawler.list_page "/itens"
+      @crawler.path "/itens"
 
       @crawler_instance.should_receive(:parse) do |arg|
         arg[:base_url].should == "danielnc.com"
-        arg[:list_page].should == "/itens/1"
+        arg[:path].should == "/itens/1"
       end
 
       @crawler_instance.crawl do
-        list_page "/itens/1"
+        path "/itens/1"
       end
 
       lambda { @craler_intance.undefined_method }.should raise_error(NoMethodError)
@@ -162,15 +145,15 @@ describe Wombat::Crawler do
 
     it 'should remove created instance variable' do
       @crawler.base_url "danielnc.com"
-      @crawler.list_page "/itens"
+      @crawler.path "/itens"
 
       @crawler_instance.should_receive(:parse) do |arg|
         arg[:base_url].should == "danielnc.com"
-        arg[:list_page].should == "/itens/1"
+        arg[:path].should == "/itens/1"
       end
 
       @crawler_instance.crawl do
-        list_page "/itens/1"
+        path "/itens/1"
       end
 
       @crawler_instance.instance_variables.index(:@metadata_dup).should be_nil
@@ -181,7 +164,7 @@ describe Wombat::Crawler do
         VCR.use_cassette('basic_crawler_page') do
 
           @crawler.base_url "http://www.terra.com.br"
-          @crawler.list_page '/portal'
+          @crawler.path '/portal'
 
           @crawler.search "css=.btn-search"
 
@@ -194,7 +177,7 @@ describe Wombat::Crawler do
         VCR.use_cassette('basic_crawler_page') do
 
           @crawler.base_url "http://www.terra.com.br"
-          @crawler.list_page '/portal'
+          @crawler.path '/portal'
 
           @crawler.search "css=.btn-search"
           @crawler.document_format :xml
@@ -208,7 +191,7 @@ describe Wombat::Crawler do
         VCR.use_cassette('error_page') do
 
           @crawler.base_url "http://www.terra.com.br"
-          @crawler.list_page '/portal'
+          @crawler.path '/portal'
 
           @crawler.search "css=.btn-search"
 
@@ -221,7 +204,7 @@ describe Wombat::Crawler do
         VCR.use_cassette('error_page') do
 
           @crawler.base_url "http://www.terra.com.br"
-          @crawler.list_page '/portal'
+          @crawler.path '/portal'
 
           @crawler.search "css=.btn-search"
           @crawler.document_format :xml
@@ -231,4 +214,11 @@ describe Wombat::Crawler do
       end
     end
   end
+
+  describe '#scrape' do
+    it 'should alias to crawl' do
+      @crawler_instance.should_receive :parse
+      @crawler_instance.scrape
+    end
+  end
 end