wombat 1.0.0 → 2.0.0

data/lib/wombat/metadata.rb

@@ -1,24 +0,0 @@
-#coding: utf-8
-require 'wombat/property_container'
-require 'wombat/iterator'
-
-module Wombat
-  class Metadata < PropertyContainer
-    def initialize
-      self[:document_format] = :html
-      super
-    end
-
-    def base_url(url)
-      self[:base_url] = url
-    end
-
-    def list_page(url)
-      self[:list_page] = url
-    end
-
-    def document_format(format)
-      self[:document_format] = format
-    end
-  end
-end

data/lib/wombat/node_selector.rb

@@ -1,10 +0,0 @@
-module Wombat
-  module NodeSelector
-    def select_nodes(selector, namespaces = nil)
-      return [selector.to_s] if selector.is_a? Symbol
-      return context.xpath selector[6..-1], namespaces if selector.start_with? "xpath="
-      return context.css selector[4..-1] if selector.start_with? "css="
-      [selector]
-    end
-  end
-end

data/lib/wombat/parser.rb

@@ -1,59 +0,0 @@
-#coding: utf-8
-require 'wombat/property_locator'
-require 'mechanize'
-require 'restclient'
-
-module Wombat
-  module Parser
-    include PropertyLocator
-    attr_accessor :mechanize, :context, :response_code
-
-    def initialize
-      @mechanize = Mechanize.new
-    end
-
-    def parse(metadata)
-      self.context = parser_for metadata
-      original_context = self.context
-
-      metadata.iterators.each do |it|
-        it.reset # Clean up iterator results before starting
-        select_nodes(it.selector).each do |node|
-          self.context = node
-          it.parse { |p| locate p }
-        end
-      end
-
-      self.context = original_context
-
-      metadata.parse { |p| locate p }
-
-      metadata.flatten
-    end
-
-    private
-    def parser_for(metadata)
-      url = "#{metadata[:base_url]}#{metadata[:list_page]}"
-      page = nil
-      parser = nil
-      begin
-        if metadata[:document_format] == :html
-          page = @mechanize.get(url)
-          parser = page.parser
-        else
-          page = RestClient.get(url)
-          parser = Nokogiri::XML page
-        end
-        self.response_code = page.code.to_i if page.respond_to? :code
-        parser
-      rescue
-        if $!.respond_to? :http_code
-          self.response_code = $!.http_code.to_i
-        elsif $!.respond_to? :response_code
-          self.response_code = $!.response_code.to_i
-        end
-        raise $!
-      end
-    end
-  end
-end

data/lib/wombat/property.rb

@@ -1,21 +0,0 @@
-module Wombat
-  class Property
-    attr_accessor :name, :selector, :format, :namespaces, :callback, :result
-
-    def initialize(options)
-      @name = options[:name]
-      @selector = options[:selector]
-      @format = options[:format]
-      @namespaces = options[:namespaces]
-      @callback = options[:callback]
-    end
-
-    def flatten(depth = nil)
-      depth ? result[depth] : result
-    end
-
-    def reset
-      self.result = nil
-    end
-  end
-end

data/lib/wombat/property_container.rb

@@ -1,70 +0,0 @@
-#coding: utf-8
-
-module Wombat
-  class PropertyContainer < Hash
-    attr_accessor :iterators
-
-    def initialize
-      @iterators = []
-    end
-
-    def method_missing(method, *args, &block)
-      if args.empty? && block
-        self["#{method.to_s}"] = PropertyContainer.new unless self["#{method.to_s}"]
-        block.call(self["#{method.to_s}"])
-      else
-        self[method.to_s] = Property.new(
-          name: method.to_s,
-          selector: args.first,
-          format: args[1],
-          namespaces: args[2],
-          callback: block)
-      end      
-    end
-
-    def to_ary
-    end
-
-    def all_properties
-      values.flat_map { |v|
-        if v.kind_of? PropertyContainer
-          v.all_properties
-        elsif v.kind_of? Property
-          v
-        else
-          nil
-        end
-      }.compact
-    end
-
-    def parse
-      all_properties.each do |p|
-        result = yield p if block_given?
-        p.result = p.callback ? p.callback.call(result) : result 
-      end
-    end
-
-    def flatten(depth = nil)
-      properties = Hash.new.tap do |h|
-        keys.map do |k|
-          val = self[k]
-          if val.is_a?(PropertyContainer) || val.is_a?(Property)
-            h[k] = val.flatten depth
-          end
-        end
-      end
-
-      iters = iterators.reduce({}) do |memo, i| 
-        memo.merge("iterator#{iterators.index(i)}" => i.flatten)
-      end
-
-      properties.merge iters
-    end
-
-    def for_each(selector)
-      Iterator.new(selector).tap do |i|
-        iterators << i
-      end
-    end
-  end
-end

data/lib/wombat/property_locator.rb

@@ -1,20 +0,0 @@
-#coding: utf-8
-require 'wombat/node_selector'
-
-module Wombat
-  module PropertyLocator
-    include NodeSelector
-
-    def locate(property)
-      props = _locate property
-      property.format != :list ? props.first : props
-    end
-
-    private
-    def _locate(property)
-      result = select_nodes(property.selector, property.namespaces).to_a
-      result.map! {|r| r.inner_html.strip } if property.format == :html
-      result.map {|r| r.kind_of?(String) ? r : r.inner_text }.map(&:strip)
-    end
-  end
-end

data/spec/iterator_spec.rb

@@ -1,52 +0,0 @@
-require 'spec_helper'
-
-describe Wombat::Iterator do
-	let(:it) { Wombat::Iterator.new "it_selector" }
-
-   context 'parse' do
-	   it 'should iterate in for_each properties' do
-	    it.prop_1 "some_selector"
-	    it.prop_2 "another_selector"
-	    
-	    it['prop_1'].should_receive(:result).twice.and_return([])
-	    it['prop_2'].should_receive(:result).twice.and_return([])
-	    
-	    parser = double :parser
-	    parser.should_receive(:locate).with(it['prop_1']).twice
-	    parser.should_receive(:locate).with(it['prop_2']).twice
-
-	    it.parse { |p| parser.locate p }
-	    it.parse { |p| parser.locate p }
-	  end
-
-	  it 'should raise if no block given' do
-	  	expect{
-	  		it.parse
-  		}.to raise_error(ArgumentError)
-	  end
-	end
-
-	context 'reset' do
-		it 'should clean up properties results' do
-			it.prop_1 'some_selector'
-			it['prop_1'].result = [1, 2]
-			it.reset
-			it['prop_1'].result.should be_nil
-		end
-	end
-
-  it 'should flatten properties to plain hash format' do
-  	it.prop_1 "some_selector"
-    it.prop_2 "another_selector"
-
-    it.parse {|p| }
-    it.parse {|p| }
-    it['prop_1'].result = ['result 1', 'result 2']
-    it['prop_2'].result = ['result 3', 'result 4']
-
-    it.flatten.should == [
-    	{ "prop_1" => "result 1", "prop_2" => "result 3" },
-    	{ "prop_1" => "result 2", "prop_2" => "result 4" }
-    ]
-  end
-end

data/spec/metadata_spec.rb

@@ -1,20 +0,0 @@
-require 'spec_helper'
-
-describe Wombat::Metadata do
-  before(:each) do
-    @metadata = Wombat::Metadata.new
-  end
-
-  it 'should not include non-properties in all properties list' do
-    @metadata.another_property "/some/selector", :text
-    @metadata.base_url "felipecsl.com"
-    @metadata.list_page "/yeah"
-    @metadata.all_properties.should == [@metadata['another_property']]
-  end
-
-  it 'should store iterators' do
-    @metadata.for_each("some_selector").kind_of?(Wombat::Iterator).should be_true
-    @metadata.iterators.size.should == 1
-    @metadata.iterators.first.selector.should == "some_selector"
-  end
-end

data/spec/parser_spec.rb

@@ -1,125 +0,0 @@
-require 'spec_helper'
-
-describe Wombat::Parser do
-  before(:each) do
-    crawler = Class.new
-    crawler.send(:include, Wombat::Parser)
-    @parser = crawler.new
-    @metadata = Wombat::Metadata.new
-  end
-
-  it 'should request page document with correct url' do
-    @metadata.base_url "http://www.google.com"
-    @metadata.list_page "/search"
-    fake_document = double :document
-    fake_parser = double :parser
-    fake_document.should_receive(:parser).and_return(fake_parser)
-    @parser.mechanize.should_receive(:get).with("http://www.google.com/search").and_return fake_document
-    
-    @parser.parse @metadata
-  end
-
-  it 'should send correct data to locate method' do
-    fake_document = double :document
-    fake_parser = double :parser
-    fake_document.should_receive(:parser).and_return(fake_parser)
-    @parser.mechanize.stub(:get).and_return fake_document
-    @parser.should_not_receive :locate
-    @parser.parse @metadata
-  end
-
-  it 'should invoke metadata callbacks' do
-    fake_document = double :document
-    fake_parser = double :parser
-    property = double :property
-    block_called = false
-    block = lambda { |p| block_called = true }
-    
-    property.stub(:result)
-    fake_document.should_receive(:parser).and_return(fake_parser)
-    property.should_receive(:callback).twice.and_return(block)
-    property.should_receive(:result=).with(true)
-    
-    @parser.mechanize.stub(:get).and_return fake_document
-    @metadata.stub(:all_properties).and_return [property]
-    @parser.should_receive(:locate).with(property)
-
-    @parser.parse @metadata
-
-    block_called.should be_true
-  end
-
-  it 'should invoke callback with parsed data' do
-    fake_document = double :document
-    fake_parser = double :parser
-    property = double :property
-    block_called = false
-    block = lambda { |p|
-      block_called = true 
-      p.should == "blah"
-    }
-    
-    fake_document.should_receive(:parser).and_return(fake_parser)
-    property.should_receive(:callback).twice.and_return(block)
-    property.should_receive(:result=).with(true)
-    
-    @parser.mechanize.stub(:get).and_return fake_document
-    @metadata.stub(:all_properties).and_return [property]
-    @parser.should_receive(:locate).with(property).and_return("blah")
-
-    @parser.parse @metadata
-
-    block_called.should be_true
-  end
-
-  it 'should return hash with requested properties' do
-    hash = double :results
-    fake_parser = double :parser
-    fake_document = double :document
-
-    fake_document.should_receive(:parser).and_return fake_parser
-    @parser.mechanize.stub(:get).and_return fake_document
-    @metadata.should_receive(:flatten).and_return hash
-
-    @parser.parse(@metadata).should == hash
-  end
-
-  it 'should not include null results in iterated block' do
-    fake_parser = double :parser
-    fake_document = double :document
-    c1 = double :context
-    c2 = double :context
-    it = Wombat::Iterator.new "it_selector"
-    it.prop_1 "some_selector"
-    
-    @parser.should_receive(:context=).ordered
-    @metadata.should_receive(:iterators).and_return [it]
-    @metadata.should_receive(:flatten)
-    fake_document.should_receive(:parser).and_return(fake_parser)
-    @parser.mechanize.stub(:get).and_return fake_document
-    @parser.should_receive(:select_nodes).with("it_selector").and_return [c1, c2]
-    @parser.should_receive(:context=).with(c1).ordered
-    @parser.should_receive(:context=).with(c2).ordered
-    @parser.should_receive(:context=).ordered
-    @parser.should_receive(:locate).with(it['prop_1']).and_return(12)
-    @parser.should_receive(:locate).with(it['prop_1']).and_return(nil)
-    @parser.stub(:locate)
-
-    @parser.parse(@metadata)
-
-    it["prop_1"].result.should == [12]
-  end
-
-  it 'should correctly parse xml documents' do
-    fake_document = double :xml
-    fake_parser = double :parser
-    @metadata.document_format :xml
-    @parser.mechanize.should_not_receive(:get)
-    RestClient.should_receive(:get).and_return fake_document
-    Nokogiri.should_receive(:XML).with(fake_document).and_return fake_parser
-    @parser.should_receive(:context=).with(fake_parser)
-    @parser.should_receive(:context=)
-
-    @parser.parse @metadata
-  end
-end

data/spec/property_container_spec.rb

@@ -1,62 +0,0 @@
-require 'spec_helper'
-
-describe Wombat::PropertyContainer do
-  before(:each) do
-    @metadata = Wombat::PropertyContainer.new
-  end
-
-  it 'should return an array with all the metadata properties excluding iterators' do
-    @metadata["event"] = Wombat::PropertyContainer.new
-    @metadata["venue"] = Wombat::PropertyContainer.new
-    @metadata.another_property "/some/selector", :text
-    @metadata["event"]["something"] = Wombat::PropertyContainer.new
-    @metadata["event"]["something"].else "Wohooo"
-    @metadata["venue"].awesome "whooea"
-    it = Wombat::Iterator.new "it_selector"
-    it.felipe "lima"
-    @metadata.iterators << it
-    
-    all_propes = @metadata.all_properties
-    
-    all_propes.should =~ [
-      @metadata["another_property"], 
-      @metadata["event"]["something"]["else"], 
-      @metadata["venue"]["awesome"]
-    ]
-  end
-
-  it 'should be able to change properties via all_properties' do
-    @metadata.another_property "/some/selector", :text
-    @metadata.all_properties.first.selector = "abc"
-    @metadata["another_property"].selector.should == "abc"
-  end
-
-  it 'should return metadata in plain hash format including iterators' do
-    @metadata.title "/some/selector"
-    @metadata["title"].result = "Gogobot Inc."
-    @metadata["holder"] = Wombat::PropertyContainer.new
-    @metadata["holder"].heading "css=.heading"
-    @metadata["holder"]["heading"].result = 123456
-    @metadata["holder"]["subheader"] = Wombat::PropertyContainer.new
-    @metadata["holder"]["subheader"].section "/blah"
-    @metadata["holder"]["subheader"]["section"].result = "Lorem Ipsum"
-    it = Wombat::Iterator.new "it_selector"
-    it.felipe "lima"
-    it["felipe"].result = ["correa", "de souza", "lima"]
-    @metadata.iterators = [it]
-    @metadata.footer("another thing", :html) { |a| true }
-    @metadata["footer"].result = "bla bla bla"
-    
-    @metadata.flatten.should == { 
-      "title" => "Gogobot Inc.",
-      "holder" => { 
-        "heading" => 123456,
-        "subheader" => {
-          "section" => "Lorem Ipsum"
-        }
-      },
-      "iterator0"=>[{"felipe"=>"correa"}, {"felipe"=>"de souza"}, {"felipe"=>"lima"}],
-      "footer" => "bla bla bla"
-    }
-  end
-end

data/spec/property_locator_spec.rb

@@ -1,75 +0,0 @@
-require 'spec_helper'
-
-describe Wombat::PropertyLocator do
-  before(:each) do
-    @locator = Class.new
-    @locator.send(:include, Wombat::PropertyLocator)
-    @locator_instance = @locator.new
-    @metadata = Wombat::Metadata.new
-    @metadata["event"] = Wombat::PropertyContainer.new
-    @metadata["venue"] = Wombat::PropertyContainer.new
-    @metadata["location"] = Wombat::PropertyContainer.new
-  end
-
-  it 'should locate metadata properties' do
-    context = double :context
-    abc = double :abc
-    
-    abc.stub(:inner_text).and_return("Something cool")
-
-    context.stub(:xpath).with("/abc", nil).and_return([abc])
-    context.stub(:xpath).with("/bah", nil).and_return(["abc"])
-    context.stub(:css).with("/ghi").and_return(["Another stuff"])
-
-    @metadata["event"].data1 "xpath=/abc"
-    @metadata["venue"].data2 :farms
-    @metadata["location"].data3 "css=/ghi"
-    @metadata.blah "xpath=/bah"
-
-    @locator_instance.stub(:context).and_return context
-    
-    @metadata.all_properties.each { |p| p.result = @locator_instance.locate p }
-
-    @metadata["blah"].result.should == "abc"
-    @metadata["event"]["data1"].result.should == "Something cool"
-    @metadata["venue"]["data2"].result.should == "farms"
-    @metadata["location"]["data3"].result.should == "Another stuff"
-  end
-
-  it 'should support properties with html format' do
-    context = double :context
-    html_info = double :html_info
-
-    html_info.should_receive(:inner_html).and_return("some another info ")
-    context.should_receive(:xpath).with("/anotherData", nil).and_return([html_info])
-
-    @locator_instance.stub(:context).and_return context
-
-    @metadata["event"].another_info "xpath=/anotherData", :html
-
-    @metadata.all_properties.each { |p| p.result = @locator_instance.locate p }
-
-    @metadata["event"]["another_info"].result.should == "some another info"
-  end
-
-  it 'should trim property contents and use namespaces if present' do
-    context = double :context
-    context.should_receive(:xpath).with("/event/some/description", "blah").and_return(["  awesome event    "])
-
-    @locator_instance.stub(:context).and_return context
-    @metadata["event"].description "xpath=/event/some/description", :text, "blah"
-
-    @metadata.all_properties.each { |p| p.result = @locator_instance.locate p }
-
-    @metadata["event"]["description"].result.should == "awesome event"
-  end
-
-  it 'should return array of matching nodes for list properties' do
-    context = double :context
-    @metadata.list_prop "css=.selector", :list
-    @locator_instance.stub(:context).and_return context
-    @locator_instance.should_receive(:select_nodes).with("css=.selector", nil).and_return %w(1 2 3 4 5)
-    
-    @locator_instance.locate(@metadata["list_prop"]).should == %w(1 2 3 4 5)
-  end
-end

data/spec/property_spec.rb

@@ -1,16 +0,0 @@
-require 'spec_helper'
-
-describe Wombat::Property do
-  it 'should store property data' do
-    property = Wombat::Property.new(
-      name: "title",
-      selector: "/some/selector", 
-      format: :html,
-      callback: lambda {})
-    
-    property.name.should == "title"
-    property.selector.should == "/some/selector"
-    property.format.should == :html
-    property.callback.should == lambda {}
-  end
-end