extraloop-redis-storage 0.0.2 → 0.0.3

data/examples/amazon_reviews.rb

@@ -4,7 +4,6 @@ require "date"
 require "./lib/amazon_review_scraper.rb"
 require "../lib/extraloop/redis-storage.rb"
 
-
 class AmazonReview < ExtraLoop::Storage::Record
   attribute :title
   attribute :rank

data/examples/google_news_scraper.rb

@@ -0,0 +1,17 @@
+require "pry"
+require "rubygems"
+require "../lib/extraloop/redis-storage.rb"
+
+
+google_news_scraper = ExtraLoop::IterativeScraper.new("https://www.google.com/search?tbm=nws&q=Egypt").
+  set_iteration(:start, (1..101).step(10)).
+  loop_on("h3") { |nodes| nodes.map(&:parent) }.
+    extract(:title, "h3.r a").
+    extract(:url, "h3.r a", :href).
+    extract(:source, "br") { |node| node.next.text.split("-").first }.
+  set_storage(:GoogleNewsStory).
+  run
+
+puts "#{GoogleNewsStory.all.size} news stories fetched..."
+
+

data/lib/extraloop/redis-storage/dataset_factory.rb

@@ -3,8 +3,10 @@
 
 class ExtraLoop::Storage::DatasetFactory
   def initialize(classname, attributes=[])
-    @classname = classname.to_s.capitalize
 
+    @classname = (classname.to_s.split "").each_with_index.map { |char, index| index == 0 && char.upcase or char }.join
+
+    return if Object.const_defined? @classname
     Object.const_set(@classname, Class.new(ExtraLoop::Storage::Record) {
       attributes.each { |attr| attribute attr }
     })

data/lib/extraloop/scraper_base.rb

@@ -1,21 +1,14 @@
 class ExtraLoop::ScraperBase
   attr_reader :session
 
-  def set_storage(*args)
-    model = args.detect { |arg| arg.is_a?(Symbol) or arg.respond_to?(:new) }
-    title = args.detect { |arg| arg.is_a?(String) }
-
-    collection_name = self.class.to_s.gsub(/(:)+/,'_').downcase + "_data"
-
+  def set_storage(model, title=nil)
+    collection_name = "#{Time.now.to_i} #{model.to_s} Dataset"
     title ||= collection_name
-    model ||= collection_name.to_sym
-
     log_session! title
 
     @model = model_klass = model.respond_to?(:new) && model || ExtraLoop::Storage::DatasetFactory.new(model.to_sym, @extractor_args.map(&:first)).get_class
 
     on :data do |results|
-      # TODO: avoid calling send in the scraper object
       results = results.map { |result| @scraper.send(:instanciate_model, result) }
       block_given? && yield(results) || results.each { |result| result.save if result.respond_to?(:save) }
     end

data/spec/scraper_base_spec.rb

@@ -20,25 +20,79 @@ describe ExtraLoop::ScraperBase do
   end
 
   describe "#set_storage" do
-    context "with no arguments but a block" do
+    context "with a symbol as the 'model' parameter value" do
       before do 
         received_records = nil
 
         @scraper.
-          set_storage { |records| received_records = records }.
+          set_storage(:MyRecord) { |records| received_records = records }.
           run()
 
         @received_records = received_records
       end
-      it "all records should be openstruct instances" do
-        @received_records.all? { |record| record.is_a?(Extraloop_scraperbase_data) }.should be_true
+
+      it "Should dynamically create 'MyRecord' class" do
+        @received_records.all? { |record| record.is_a?(MyRecord) }.should be_true
+      end
+
+      it "All records should be associated to the same ScrapingSession object" do
+        @received_records.all? { |record| record.session.should be_eql @scraper.session }
+      end
+
+      it "Should auto assign a session title" do
+        @scraper.session.title.should match /^(\d)*\s(MyRecord)\sDataset$/
+      end
+    end
+
+    context "with a constant as the 'model' parameter value" do
+      before do 
+        received_records = nil
+
+        class MyModel < ExtraLoop::Storage::Record
+          attribute :foo
+        end
+
+        @scraper.
+          set_storage(MyModel) { |records| received_records = records }.
+          run()
+
+        @received_records = received_records
+      end
+
+      it "All records should be instances of MyModel" do
+        @received_records.all? { |record| record.is_a?(MyModel) }.should be_true
+      end
+
+      it "All records should be associated to the same ScrapingSession object" do
+        @received_records.all? { |record| record.session.should be_eql @scraper.session }
+      end
+
+      it "Should auto assign a session title" do
+        @scraper.session.title.should match /^(\d)*\s(MyModel)\sDataset$/
       end
     end
+  end
 
-    context "with title argument and no block" do
-      before do
-        @scraper.set_storage "my dummy dataset"
+  context "with a constant name and no block" do
+    before do 
+      received_records = nil
+
+      if !Object.const_defined? :MyModel
+        class MyModel < ExtraLoop::Storage::Record
+          attribute :foo
+        end
       end
+
+      @scraper.
+        set_storage(MyModel).
+        run()
+
+      @received_records = received_records
+    end
+
+    it "should persist 10 records" do
+      (@scraper.session.records MyModel).should have(10).records
+      (@scraper.session.records MyModel).map(&:id).reject(&:nil?).should_not be_empty
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: extraloop-redis-storage
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-20 00:00:00.000000000Z
+date: 2012-02-21 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: extraloop
-  requirement: &16470780 !ruby/object:Gem::Requirement
+  requirement: &19118640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *16470780
+  version_requirements: *19118640
 - !ruby/object:Gem::Dependency
   name: ohm
-  requirement: &16470320 !ruby/object:Gem::Requirement
+  requirement: &19118180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.1.3
   type: :runtime
   prerelease: false
-  version_requirements: *16470320
+  version_requirements: *19118180
 - !ruby/object:Gem::Dependency
   name: ohm-contrib
-  requirement: &16469860 !ruby/object:Gem::Requirement
+  requirement: &19117720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *16469860
+  version_requirements: *19117720
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &16469400 !ruby/object:Gem::Requirement
+  requirement: &19117260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 2.7.0
   type: :development
   prerelease: false
-  version_requirements: *16469400
+  version_requirements: *19117260
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &16468940 !ruby/object:Gem::Requirement
+  requirement: &19116800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.4
   type: :development
   prerelease: false
-  version_requirements: *16468940
+  version_requirements: *19116800
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &16468480 !ruby/object:Gem::Requirement
+  requirement: &19116340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +76,7 @@ dependencies:
         version: 0.9.7.4
   type: :development
   prerelease: false
-  version_requirements: *16468480
+  version_requirements: *19116340
 description: Redis+Ohm based storage for data sets extracted using the ExtraLoop toolkit.
 email: andrea.giulio.fiore@googlemail.com
 executables: []
@@ -86,6 +86,7 @@ files:
 - History.txt
 - README.rdoc
 - examples/amazon_reviews.rb
+- examples/google_news_scraper.rb
 - examples/lib/amazon_review_scraper.rb
 - lib/extraloop/redis-storage.rb
 - lib/extraloop/redis-storage/dataset_factory.rb