extraloop-redis-storage 0.0.3 → 0.0.4

data/README.rdoc

@@ -12,18 +12,18 @@ makes storing and managing Ruby objects in Redis easy and convenient.
 
 == Usage
 
-Extraloop's Redis storage module decorates +ExtraLoop::ScraperBase+ and +ExtraLoop::IterativeScraper+ instances 
+Extraloop's Redis storage module decorates <code>ExtraLoop::ScraperBase</code> and <code>ExtraLoop::IterativeScraper</code> instances 
 with the +set_storage+ method: a helper method that allows to specify how the scraped data should be stored.
 
  require "extraloop/redis-storage"
 
  class AmazonReview < ExtraLoop::Storage::Record
    attribute :title
-   attribute :author
-   attribute :star
+   attribute :rank
+   attribute :date
 
    def validate
-     (0..5).include star.to_i or  && false
+     assert (0..5).include?(rank.to_i), "Rank not in range"
    end
  end
 
@@ -32,7 +32,7 @@ with the +set_storage+ method: a helper method that allows to specify how the sc
    .run()
 
 At each scraper run, the ExtraLoop storage module internally instantiates a 
-session (see +ExtraLoop::Storage::ScrapingSession+) and link the extracted records to it.
+session (see <code>ExtraLoop::Storage::ScrapingSession</code>) and link the extracted records to it.
 The +AmazonReview+ instances extracted and stored in the example above, can in fact be fetched by calling
 Ohm's +find+ with the session id as argument.
 
@@ -48,10 +48,6 @@ session instance:
 
 The +set_storage+ method can be called with the following arguments:
 
-* _model_ A Ruby constant specifying the model to be used for storing the extracted data (optional; when this argument is not provided, a simple-non validating model will be generated on the fly).
-* _session_title_ A human readable name for the extracted dataset.
-
-== Running the test suite
-
-The test suite can be run by executing the +rspec\ \*+ command from within the  +spec/+ directory.
+* _model_ A Ruby constant specifying the model to be used for storing the extracted data .
+* _session_title_ A human readable title for the extracted dataset (optional).
 

data/examples/amazon_reviews.rb

@@ -1,18 +1,9 @@
 require "rubygems"
-require "extraloop"
 require "date"
-require "./lib/amazon_review_scraper.rb"
+require "extraloop"
 require "../lib/extraloop/redis-storage.rb"
-
-class AmazonReview < ExtraLoop::Storage::Record
-  attribute :title
-  attribute :rank
-  attribute :date
-
-  def validate
-    assert (0..5).include?(rank.to_i), "Rank not in range"
-  end
-end
+require "./lib/models/amazon_review.rb"
+require "./lib/scrapers/amazon_review_scraper.rb"
 
 scraper = AmazonReviewScraper.new("0262560992").
   set_storage(AmazonReview).

data/examples/google_news_scraper.rb

@@ -12,6 +12,6 @@ google_news_scraper = ExtraLoop::IterativeScraper.new("https://www.google.com/se
   set_storage(:GoogleNewsStory).
   run
 
-puts "#{GoogleNewsStory.all.size} news stories fetched..."
+puts "#{GoogleNewsStory.all.to_a.size} news stories fetched..."
 
 

data/examples/lib/models/amazon_review.rb

@@ -0,0 +1,9 @@
+class AmazonReview < ExtraLoop::Storage::Record
+  attribute :title
+  attribute :rank
+  attribute :date
+
+  def validate
+    assert (0..5).include?(rank.to_i), "Rank not in range"
+  end
+end

data/lib/extraloop/redis-storage.rb

@@ -1,30 +1,33 @@
 require "json"
 require "rubygems"
 require "redis"
-require 'pry'
 require "ohm"
 require "ohm/contrib"
 require "extraloop"
 
 base_path = File.realpath(File.dirname(__FILE__))
 $: << "#{base_path}"
-require "scraper_base"
-
 
+require "scraper_base"
 
 module ExtraLoop
   module Storage
     VERSION ||= "0.0.1"
 
-    class << self
-      def connect(*args)
-        Ohm.connect(*args)
-      end
+    def self.connect(*args)
+      Ohm.connect(*args)
+    end
+
+    # Tries to automatically locate the models directory and load all ruby files within in
+    def self.autoload_models(dirname='models')
+      Dir["**/**#{dirname}/*.rb"].each { |path| require "./#{path}" }
     end
   end
 end
 
 ExtraLoop::Storage.autoload :Record, "#{base_path}/redis-storage/record.rb"
 ExtraLoop::Storage.autoload :ScrapingSession, "#{base_path}/redis-storage/scraping_session.rb"
+ExtraLoop::Storage.autoload :Model,  "#{base_path}/redis-storage/model.rb"
 ExtraLoop::Storage.autoload :DatasetFactory,  "#{base_path}/redis-storage/dataset_factory.rb"
 
+

data/lib/extraloop/redis-storage/dataset_factory.rb

@@ -6,7 +6,8 @@ class ExtraLoop::Storage::DatasetFactory
 
     @classname = (classname.to_s.split "").each_with_index.map { |char, index| index == 0 && char.upcase or char }.join
 
-    return if Object.const_defined? @classname
+    return Object.const_get @classname if Object.const_defined? @classname
+
     Object.const_set(@classname, Class.new(ExtraLoop::Storage::Record) {
       attributes.each { |attr| attribute attr }
     })

data/lib/extraloop/redis-storage/model.rb

@@ -0,0 +1,13 @@
+# Meta model designed to keep track of what types of records
+# are associated to a Scraping session object.
+#
+class ExtraLoop::Storage::Model < Ohm::Model
+  attribute :name
+  index :name
+
+  def to_hash
+    super.merge(attributes.reduce({}) { |memo, attribute| 
+      memo.merge(attribute => send(attribute)) 
+    })
+  end
+end

data/lib/extraloop/redis-storage/scraping_session.rb

@@ -2,11 +2,50 @@ class ExtraLoop::Storage::ScrapingSession < Ohm::Model
 
   include Ohm::Boundaries
   include Ohm::Timestamping
+  include Ohm::Callbacks
 
   attribute :title
+  reference :model, ExtraLoop::Storage::Model
 
-  def records(collection)
-    Kernel.const_get(collection.to_s).
-      find(:session_id => self.id)
+  def records(params={})
+    klass = if Object.const_defined?(model.name)
+        Object.const_get(model.name)
+      else
+        dynamic_class = Class.new(ExtraLoop::Storage::Record) do
+          # override default to_hash so that it will return the Redis hash
+          # internally stored by Ohm
+          def to_hash
+            Ohm.redis.hgetall self.key
+          end
+        end
+
+        Object.const_set(model.name, dynamic_class)
+        dynamic_class
+    end
+
+    # set a session index, so that Ohm finder will work
+    klass.indices << :session_id unless klass.indices.include? :session_id
+
+    klass.find({
+      :session_id => self.id
+    }.merge(params))
+  end
+
+  def validate
+    assert_present :model
+  end
+
+  def to_hash
+    attrs = attributes.reduce({}) { |memo, attribute| 
+      memo.merge(attribute => send(attribute))
+    }.merge({
+      :records => records.map(&:to_hash),
+      :model => model.to_hash
+    })
+
+    super.merge attrs
+  end
+
+  def to_csv
   end
 end

data/lib/extraloop/scraper_base.rb

@@ -4,9 +4,9 @@ class ExtraLoop::ScraperBase
   def set_storage(model, title=nil)
     collection_name = "#{Time.now.to_i} #{model.to_s} Dataset"
     title ||= collection_name
-    log_session! title
 
     @model = model_klass = model.respond_to?(:new) && model || ExtraLoop::Storage::DatasetFactory.new(model.to_sym, @extractor_args.map(&:first)).get_class
+    log_session! title
 
     on :data do |results|
       results = results.map { |result| @scraper.send(:instanciate_model, result) }
@@ -17,7 +17,12 @@ class ExtraLoop::ScraperBase
   protected
   # Creates a scraping session
   def log_session!(title="")
-    @session ||= ExtraLoop::Storage::ScrapingSession.create :title => title
+    if !@session
+      ns = ExtraLoop::Storage
+      results = ns::Model.find :name => @model
+      model = results.any? && results.first || ns::Model.create(:name => @model)
+      @session = ns::ScrapingSession.create :title => title, :model =>  model
+    end
   end
 
   # Converts extracted records into instances of the dataset model specified as the first argument

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: extraloop-redis-storage
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-21 00:00:00.000000000Z
+date: 2012-02-26 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: extraloop
-  requirement: &19118640 !ruby/object:Gem::Requirement
+  requirement: &18201600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *19118640
+  version_requirements: *18201600
 - !ruby/object:Gem::Dependency
   name: ohm
-  requirement: &19118180 !ruby/object:Gem::Requirement
+  requirement: &18201140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.1.3
   type: :runtime
   prerelease: false
-  version_requirements: *19118180
+  version_requirements: *18201140
 - !ruby/object:Gem::Dependency
   name: ohm-contrib
-  requirement: &19117720 !ruby/object:Gem::Requirement
+  requirement: &18200680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,21 @@ dependencies:
         version: 0.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *19117720
+  version_requirements: *18200680
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: &18200080 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
+        version: 0.14.6
+  type: :runtime
+  prerelease: false
+  version_requirements: *18200080
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &19117260 !ruby/object:Gem::Requirement
+  requirement: &18199420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +65,10 @@ dependencies:
         version: 2.7.0
   type: :development
   prerelease: false
-  version_requirements: *19117260
+  version_requirements: *18199420
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &19116800 !ruby/object:Gem::Requirement
+  requirement: &18198820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +76,10 @@ dependencies:
         version: 1.0.4
   type: :development
   prerelease: false
-  version_requirements: *19116800
+  version_requirements: *18198820
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &19116340 !ruby/object:Gem::Requirement
+  requirement: &18187500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,7 +87,7 @@ dependencies:
         version: 0.9.7.4
   type: :development
   prerelease: false
-  version_requirements: *19116340
+  version_requirements: *18187500
 description: Redis+Ohm based storage for data sets extracted using the ExtraLoop toolkit.
 email: andrea.giulio.fiore@googlemail.com
 executables: []
@@ -87,9 +98,11 @@ files:
 - README.rdoc
 - examples/amazon_reviews.rb
 - examples/google_news_scraper.rb
-- examples/lib/amazon_review_scraper.rb
+- examples/lib/models/amazon_review.rb
+- examples/lib/scrapers/amazon_review_scraper.rb
 - lib/extraloop/redis-storage.rb
 - lib/extraloop/redis-storage/dataset_factory.rb
+- lib/extraloop/redis-storage/model.rb
 - lib/extraloop/redis-storage/record.rb
 - lib/extraloop/redis-storage/scraping_session.rb
 - lib/extraloop/scraper_base.rb