wombat 0.1.2 → 0.1.3

data/spec/property_container_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+describe Wombat::PropertyContainer do
+  before(:each) do
+    @metadata = Wombat::PropertyContainer.new
+  end
+
+  it 'should return an array with all the metadata properties' do
+    @metadata["event"] = Wombat::PropertyContainer.new
+    @metadata["venue"] = Wombat::PropertyContainer.new
+    @metadata.another_property "/some/selector", :text
+    @metadata["event"].something "else"
+    @metadata["venue"].awesome "whooea"
+    
+    all_propes = @metadata.all_properties
+    
+    all_propes.should =~ [@metadata["another_property"], @metadata["event"]["something"], @metadata["venue"]["awesome"]]
+  end
+
+  it 'should be able to change properties via all_properties' do
+    @metadata.another_property "/some/selector", :text
+    @metadata.all_properties.first.selector = "abc"
+    @metadata["another_property"].selector.should == "abc"
+  end
+
+  it 'should return metadata in plain hash format' do
+    @metadata.title "/some/selector"
+    @metadata["title"].result = "Gogobot Inc."
+    @metadata["holder"] = Wombat::PropertyContainer.new
+    @metadata["holder"].heading "css=.heading"
+    @metadata["holder"]["heading"].result = 123456
+    @metadata["holder"]["subheader"] = Wombat::PropertyContainer.new
+    @metadata["holder"]["subheader"].section "/blah"
+    @metadata["holder"]["subheader"]["section"].result = "Lorem Ipsum"
+    @metadata.footer("another thing", :html) { |a| true }
+    @metadata["footer"].result = "bla bla bla"
+    
+    @metadata.flatten.should == { 
+      "title" => "Gogobot Inc.",
+      "holder" => { 
+        "heading" => 123456,
+        "subheader" => {
+          "section" => "Lorem Ipsum"
+        }
+      },
+      "footer" => "bla bla bla"
+    }
+  end
+end

data/spec/property_locator_spec.rb

@@ -6,24 +6,34 @@ describe Wombat::PropertyLocator do
     @locator.send(:include, Wombat::PropertyLocator)
     @locator_instance = @locator.new
     @metadata = Wombat::Metadata.new
+    @metadata["event"] = Wombat::PropertyContainer.new
+    @metadata["venue"] = Wombat::PropertyContainer.new
+    @metadata["location"] = Wombat::PropertyContainer.new
   end
 
   it 'should locate metadata properties' do
     context = double :context
-    context.stub(:xpath).with("/abc", nil).and_return(["Something cool"])
+    abc = double :abc
+    
+    abc.stub(:inner_text).and_return("Something cool")
+
+    context.stub(:xpath).with("/abc", nil).and_return([abc])
+    context.stub(:xpath).with("/bah", nil).and_return(["abc"])
     context.stub(:css).with("/ghi").and_return(["Another stuff"])
 
-    @metadata.event_props.data1 "xpath=/abc"
-    @metadata.venue_props.data2 :farms
-    @metadata.location_props.data3 "css=/ghi"
+    @metadata["event"].data1 "xpath=/abc"
+    @metadata["venue"].data2 :farms
+    @metadata["location"].data3 "css=/ghi"
+    @metadata.blah "xpath=/bah"
 
     @locator_instance.stub(:context).and_return context
     
     @locator_instance.locate @metadata
 
-    @metadata.event_props.get_property("data1").result.should == "Something cool"
-    @metadata.venue_props.get_property("data2").result.should == "farms"
-    @metadata.location_props.get_property("data3").result.should == "Another stuff"
+    @metadata.get_property("blah").result.should == "abc"
+    @metadata["event"].get_property("data1").result.should == "Something cool"
+    @metadata["venue"].get_property("data2").result.should == "farms"
+    @metadata["location"].get_property("data3").result.should == "Another stuff"
   end
 
   it 'should support properties with html format' do
@@ -35,11 +45,11 @@ describe Wombat::PropertyLocator do
 
     @locator_instance.stub(:context).and_return context
 
-    @metadata.event_props.another_info "xpath=/anotherData", :html
+    @metadata["event"].another_info "xpath=/anotherData", :html
 
     @locator_instance.locate @metadata
 
-    @metadata.event_props.get_property("another_info").result.should == "some another info"
+    @metadata["event"].get_property("another_info").result.should == "some another info"
   end
 
   it 'should trim property contents and use namespaces if present' do
@@ -47,10 +57,10 @@ describe Wombat::PropertyLocator do
     context.should_receive(:xpath).with("/event/some/description", "blah").and_return(["  awesome event    "])
 
     @locator_instance.stub(:context).and_return context
-    @metadata.event_props.description "xpath=/event/some/description", :text, "blah"
+    @metadata["event"].description "xpath=/event/some/description", :text, "blah"
 
     @locator_instance.locate @metadata
 
-    @metadata.event_props.get_property("description").result.should == "awesome event"
+    @metadata["event"].get_property("description").result.should == "awesome event"
   end
 end

data/spec/sample_crawler_spec.rb

@@ -4,17 +4,19 @@ require 'helpers/sample_crawler'
 describe SampleCrawler do
   before(:each) do
     @sample_crawler = SampleCrawler.new
-    @sample_crawler.parser = Wombat::Parser.new
   end
 
-  it 'should correctly assign event metadata' do
-    @sample_crawler.parser.should_receive(:parse) do |args|
-      args.event_props.get_property("title").selector.should == "Sample Event"
-      args.event_props.get_property("description").selector.should == "This event's description"
-      args.event_props.get_property("date").selector.should == DateTime.now.to_date
+  xit 'should correctly assign event metadata' do
+    @sample_crawler.should_receive(:parse) do |args|
+      args.event["title"].selector.should == "xpath=."
+      args.event["description"].selector.should == "css=#main-node-content"
+      args.event["date"].selector.should == DateTime.now.to_date
 
-      args.venue_props.get_property("name").selector.should == "Cafe de La Musique"
-      args.venue_props.get_property("address").selector.should == "324 Dom Pedro II Street"
+      args.venue["name"].selector.should == "Cafe de La Musique"
+      args.venue["address"].selector.should == "324 Dom Pedro II Street"
+
+      args[:base_url].should == 'http://www.google.com/'
+      args[:list_page].should == 'shows.php'
     end
 
     @sample_crawler.crawl

data/spec/spec_helper.rb

@@ -1,3 +1,9 @@
 require 'wombat'
 require 'rspec'
-require 'rspec-expectations'
+require 'rspec-expectations'
+require 'vcr'
+
+VCR.configure do |c|
+  c.cassette_library_dir = 'fixtures/vcr_cassettes'
+  c.hook_into :fakeweb
+end

data/wombat.gemspec

@@ -5,16 +5,16 @@
 
 Gem::Specification.new do |s|
   s.name = "wombat"
-  s.version = "0.1.2"
+  s.version = "0.1.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Felipe Lima"]
-  s.date = "2011-12-27"
+  s.date = "2012-02-06"
   s.description = "Generic Web crawler with a DSL that parses event-related data from web pages"
   s.email = "felipe.lima@gmail.com"
   s.extra_rdoc_files = [
     "LICENSE.txt",
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".document",
@@ -24,21 +24,23 @@ Gem::Specification.new do |s|
     "Gemfile.lock",
     "Guardfile",
     "LICENSE.txt",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "VERSION",
+    "fixtures/vcr_cassettes/basic_crawler_page.yml",
     "lib/wombat.rb",
     "lib/wombat/crawler.rb",
     "lib/wombat/metadata.rb",
     "lib/wombat/parser.rb",
-    "lib/wombat/properties.rb",
     "lib/wombat/property.rb",
+    "lib/wombat/property_container.rb",
     "lib/wombat/property_locator.rb",
     "spec/crawler_spec.rb",
     "spec/helpers/sample_crawler.rb",
+    "spec/integration/integration_spec.rb",
     "spec/metadata_spec.rb",
     "spec/parser_spec.rb",
-    "spec/properties_spec.rb",
+    "spec/property_container_spec.rb",
     "spec/property_locator_spec.rb",
     "spec/property_spec.rb",
     "spec/sample_crawler_spec.rb",
@@ -63,6 +65,10 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<jeweler>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
       s.add_development_dependency(%q<guard>, [">= 0"])
+      s.add_development_dependency(%q<guard-rspec>, [">= 0"])
+      s.add_development_dependency(%q<guard-bundler>, [">= 0"])
+      s.add_development_dependency(%q<vcr>, ["= 2.0.0.rc1"])
+      s.add_development_dependency(%q<fakeweb>, [">= 0"])
     else
       s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<rake>, [">= 0"])
@@ -72,6 +78,10 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<jeweler>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 0"])
       s.add_dependency(%q<guard>, [">= 0"])
+      s.add_dependency(%q<guard-rspec>, [">= 0"])
+      s.add_dependency(%q<guard-bundler>, [">= 0"])
+      s.add_dependency(%q<vcr>, ["= 2.0.0.rc1"])
+      s.add_dependency(%q<fakeweb>, [">= 0"])
     end
   else
     s.add_dependency(%q<bundler>, [">= 0"])
@@ -82,6 +92,10 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<jeweler>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 0"])
     s.add_dependency(%q<guard>, [">= 0"])
+    s.add_dependency(%q<guard-rspec>, [">= 0"])
+    s.add_dependency(%q<guard-bundler>, [">= 0"])
+    s.add_dependency(%q<vcr>, ["= 2.0.0.rc1"])
+    s.add_dependency(%q<fakeweb>, [">= 0"])
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wombat
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-27 00:00:00.000000000 Z
+date: 2012-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70232457949980 !ruby/object:Gem::Requirement
+  requirement: &70251197006880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232457949980
+  version_requirements: *70251197006880
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70232457965860 !ruby/object:Gem::Requirement
+  requirement: &70251197006400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232457965860
+  version_requirements: *70251197006400
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &70232457965380 !ruby/object:Gem::Requirement
+  requirement: &70251197022260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232457965380
+  version_requirements: *70251197022260
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70232457964900 !ruby/object:Gem::Requirement
+  requirement: &70251197021660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70232457964900
+  version_requirements: *70251197021660
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &70232457964420 !ruby/object:Gem::Requirement
+  requirement: &70251197021180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70232457964420
+  version_requirements: *70251197021180
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70232457963940 !ruby/object:Gem::Requirement
+  requirement: &70251197020700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70232457963940
+  version_requirements: *70251197020700
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70232457963460 !ruby/object:Gem::Requirement
+  requirement: &70251197020200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70232457963460
+  version_requirements: *70251197020200
 - !ruby/object:Gem::Dependency
   name: guard
-  requirement: &70232457962980 !ruby/object:Gem::Requirement
+  requirement: &70251197019700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,7 +98,51 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70232457962980
+  version_requirements: *70251197019700
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: &70251197019220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70251197019220
+- !ruby/object:Gem::Dependency
+  name: guard-bundler
+  requirement: &70251197018740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70251197018740
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: &70251197018260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
+        version: 2.0.0.rc1
+  type: :development
+  prerelease: false
+  version_requirements: *70251197018260
+- !ruby/object:Gem::Dependency
+  name: fakeweb
+  requirement: &70251197017720 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70251197017720
 description: Generic Web crawler with a DSL that parses event-related data from web
   pages
 email: felipe.lima@gmail.com
@@ -106,7 +150,7 @@ executables: []
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
-- README.rdoc
+- README.md
 files:
 - .document
 - .rspec
@@ -115,21 +159,23 @@ files:
 - Gemfile.lock
 - Guardfile
 - LICENSE.txt
-- README.rdoc
+- README.md
 - Rakefile
 - VERSION
+- fixtures/vcr_cassettes/basic_crawler_page.yml
 - lib/wombat.rb
 - lib/wombat/crawler.rb
 - lib/wombat/metadata.rb
 - lib/wombat/parser.rb
-- lib/wombat/properties.rb
 - lib/wombat/property.rb
+- lib/wombat/property_container.rb
 - lib/wombat/property_locator.rb
 - spec/crawler_spec.rb
 - spec/helpers/sample_crawler.rb
+- spec/integration/integration_spec.rb
 - spec/metadata_spec.rb
 - spec/parser_spec.rb
-- spec/properties_spec.rb
+- spec/property_container_spec.rb
 - spec/property_locator_spec.rb
 - spec/property_spec.rb
 - spec/sample_crawler_spec.rb

data/README.rdoc

@@ -1,20 +0,0 @@
-= Wombat
-
-Generic Web crawler with a DSL that parses event-related data from web pages.
-Still under development, it is being rewritten from scratch as a gem from an already existing project.
-
-== Contributing to Wombat
- 
-* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
-* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
-* Fork the project
-* Start a feature/bugfix branch
-* Commit and push until you are happy with your contribution
-* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
-* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
-
-== Copyright
-
-Copyright (c) 2011 Felipe Lima. See LICENSE.txt for
-further details.
-