wombat 0.2.4 → 0.2.5

data/Gemfile.lock

@@ -18,11 +18,14 @@ GEM
       nokogiri (~> 1.4)
       ntlm-http (~> 0.1, >= 0.1.1)
       webrobots (~> 0.0, >= 0.0.9)
+    mime-types (1.17.2)
     net-http-digest_auth (1.2)
     net-http-persistent (2.3.3)
     nokogiri (1.5.0)
     ntlm-http (0.1.1)
     rake (0.9.2.2)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
     rspec (2.7.0)
       rspec-core (~> 2.7.0)
       rspec-expectations (~> 2.7.0)
@@ -49,6 +52,7 @@ DEPENDENCIES
   jeweler
   mechanize
   rake
+  rest-client
   rspec
   vcr (= 2.0.0.rc1)
   yard

data/README.md

@@ -63,7 +63,7 @@ my_crawler.crawl
 }
 ```
 
-### For additional documentation, please check the project [Wiki](http://github.com/felipecsl/wombat/wiki).
+### For the documentation, please check the project [Wiki](http://github.com/felipecsl/wombat/wiki).
 
 
 ## Contributing to Wombat

data/Rakefile

@@ -16,6 +16,7 @@ Jeweler::Tasks.new do |gem|
   gem.description = %Q{Generic Web crawler with a DSL that parses structured data from web pages}
   gem.email = "felipe.lima@gmail.com"
   gem.authors = ["Felipe Lima"]
+  gem.required_ruby_version = ">= 1.9"
   # dependencies defined in Gemfile
 end
 
@@ -26,4 +27,4 @@ RSpec::Core::RakeTask.new(:spec)
 task :test => :spec
 task :default => :spec
 
-YARD::Rake::YardocTask.new
+YARD::Rake::YardocTask.new

data/VERSION

@@ -1 +1 @@
-0.2.4
+0.2.5

data/lib/wombat/crawler.rb

@@ -10,13 +10,8 @@ module Wombat
     include Parser
     extend ActiveSupport::Concern
 
-    module InstanceMethods
-      def crawl
-        parse self.class.send(:metadata)
-      end
-
-      def supports_city?
-      end
+    def crawl
+      parse self.class.send(:metadata)
     end
 
     module ClassMethods
@@ -28,17 +23,10 @@ module Wombat
         metadata.for_each(selector).instance_eval(&block) if block
       end
 
-      def format type
-        metadata.document_format = type
-      end
-
       def follow_links selector
 
       end
 
-      def supported_cities
-      end
-
       def to_ary
       end
 

data/lib/wombat/metadata.rb

@@ -4,10 +4,8 @@ require 'wombat/iterator'
 
 module Wombat
   class Metadata < PropertyContainer
-    attr_accessor :document_format
-
     def initialize
-      @document_format = :html
+      self[:format] = :html
       super
     end
 
@@ -18,5 +16,9 @@ module Wombat
     def list_page url
       self[:list_page] = url
     end
+
+    def format format
+      self[:format] = format
+    end
   end
 end

data/lib/wombat/parser.rb

@@ -22,7 +22,10 @@ module Wombat
           it.all_properties.each do |p|
             p.result ||= []
             result = locate(p)
-            p.result << result if result
+            if result
+              result = p.callback ? p.callback.call(result) : result
+              p.result << result
+            end
           end
         end
       end
@@ -41,7 +44,7 @@ module Wombat
     def get_parser metadata
       url = "#{metadata[:base_url]}#{metadata[:list_page]}"
 
-      if metadata.document_format == :html
+      if metadata[:format] == :html
         @mechanize.get(url).parser
       else
         Nokogiri::XML RestClient.get(url)

data/spec/crawler_spec.rb

@@ -114,7 +114,7 @@ describe Wombat::Crawler do
 
   it 'should assign metadata forma' do
     @crawler_instance.should_receive(:parse) do |arg|
-      arg.document_format.should == :xml
+      arg[:format].should == :xml
     end
     @crawler.format :xml
     @crawler_instance.crawl

data/spec/integration/integration_spec.rb

@@ -44,7 +44,7 @@ describe 'basic crawler setup' do
 
       crawler.for_each "css=ol.ranked-repositories li" do
         repo 'css=h3'
-        description 'css=p.description'
+        description('css=p.description') { |d| d.gsub(/for/, '') }
       end
 
       crawler_instance = crawler.new
@@ -52,11 +52,11 @@ describe 'basic crawler setup' do
 
       results["repo"].should =~ ["jairajs89 / Touchy.js", "mcavage / node-restify", "notlion / streetview-stereographic", "twitter / bootstrap", "stolksdorf / Parallaxjs"]
       results["description"].should =~ [
-        "node.js REST framework specifically meant for web service APIs",
-        "A simple light-weight JavaScript library for dealing with touch events",
+        "node.js REST framework specifically meant  web service APIs",
+        "A simple light-weight JavaScript library  dealing with touch events",
         "Shader Toy + Google Map + Panoramic Explorer",
         "HTML, CSS, and JS toolkit from Twitter",
-        "a Library for Javascript that allows easy page parallaxing"
+        "a Library  Javascript that allows easy page parallaxing"
       ]
     end
   end

data/spec/parser_spec.rb

@@ -72,6 +72,10 @@ describe Wombat::Parser do
     block_called.should be_true
   end
 
+  it 'should invoke callback inside for_each block' do
+    
+  end
+
   it 'should return hash with requested properties' do
     hash = double :results
     fake_parser = double :parser
@@ -140,7 +144,7 @@ describe Wombat::Parser do
   it 'should correctly parse xml documents' do
     fake_document = double :xml
     fake_parser = double :parser
-    @metadata.document_format = :xml
+    @metadata.format :xml
     @parser.mechanize.should_not_receive(:get)
     RestClient.should_receive(:get).and_return fake_document
     Nokogiri.should_receive(:XML).with(fake_document).and_return fake_parser

data/wombat.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "wombat"
-  s.version = "0.2.4"
+  s.version = "0.2.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Felipe Lima"]
-  s.date = "2012-02-15"
+  s.date = "2012-03-21"
   s.description = "Generic Web crawler with a DSL that parses structured data from web pages"
   s.email = "felipe.lima@gmail.com"
   s.extra_rdoc_files = [
@@ -54,7 +54,8 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/felipecsl/wombat"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.11"
+  s.required_ruby_version = Gem::Requirement.new(">= 1.9")
+  s.rubygems_version = "1.8.18"
   s.summary = "Ruby DSL to crawl web pages"
 
   if s.respond_to? :specification_version then

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wombat
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-15 00:00:00.000000000 Z
+date: 2012-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &70159522946820 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70159522946820
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70159522962680 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +37,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70159522962680
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: &70159522962060 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +53,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70159522962060
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70159522961340 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +69,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70159522961340
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70159522960620 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +85,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70159522960620
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &70159522960000 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +101,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70159522960000
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70159522959520 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +117,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70159522959520
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70159522959040 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,21 +133,31 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70159522959040
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: vcr
-  requirement: &70159522958540 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - =
+    - - '='
       - !ruby/object:Gem::Version
         version: 2.0.0.rc1
   type: :development
   prerelease: false
-  version_requirements: *70159522958540
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: fakeweb
-  requirement: &70159522958060 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +165,12 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70159522958060
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Generic Web crawler with a DSL that parses structured data from web pages
 email: felipe.lima@gmail.com
 executables: []
@@ -174,7 +224,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -183,7 +233,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.18
 signing_key: 
 specification_version: 3
 summary: Ruby DSL to crawl web pages