extraloop-redis-storage 0.0.2

data/History.txt

@@ -0,0 +1,5 @@
+== 0.0.2  / 2012-20-02
+  * Fixed scope bug
+
+== 0.0.1  / 2012-20-02
+  * Project Birthday!

data/README.rdoc

@@ -0,0 +1,57 @@
+= Extraloop Redis Storage
+
+== Description
+
+Persistence layer for the {ExtraLoop}[https://github.com/afiore/extraloop] data extraction toolkit.
+The module is implemented as a small wrapper built on top of {Ohm}[http://ohm.keyvalue.org], an object-hash mapping library which 
+makes storing and managing Ruby objects in Redis easy and convenient.
+
+== Installation 
+
+ gem install extraloop-redis-storage
+
+== Usage
+
+Extraloop's Redis storage module decorates +ExtraLoop::ScraperBase+ and +ExtraLoop::IterativeScraper+ instances 
+with the +set_storage+ method: a helper method that allows to specify how the scraped data should be stored.
+
+ require "extraloop/redis-storage"
+
+ class AmazonReview < ExtraLoop::Storage::Record
+   attribute :title
+   attribute :author
+   attribute :star
+
+   def validate
+     (0..5).include star.to_i or  && false
+   end
+ end
+
+ scraper = AmazonReviewScraper.new("0262560992").
+   .set_storage(AmazonReview, "Amazon reviews of 'The Little Schemer'")
+   .run()
+
+At each scraper run, the ExtraLoop storage module internally instantiates a 
+session (see +ExtraLoop::Storage::ScrapingSession+) and link the extracted records to it.
+The +AmazonReview+ instances extracted and stored in the example above, can in fact be fetched by calling
+Ohm's +find+ with the session id as argument.
+
+ reviews = AmazonReview.find :session_id => scraper.session
+
+The same set of reviews can alternatively be retrieved by calling the +record+ method on the scraping 
+session instance:
+
+ reviews = scraper.session.records AmazonReview
+
+
+== The #set_storage method
+
+The +set_storage+ method can be called with the following arguments:
+
+* _model_ A Ruby constant specifying the model to be used for storing the extracted data (optional; when this argument is not provided, a simple-non validating model will be generated on the fly).
+* _session_title_ A human readable name for the extracted dataset.
+
+== Running the test suite
+
+The test suite can be run by executing the +rspec\ \*+ command from within the  +spec/+ directory.
+

data/examples/amazon_reviews.rb

@@ -0,0 +1,24 @@
+require "rubygems"
+require "extraloop"
+require "date"
+require "./lib/amazon_review_scraper.rb"
+require "../lib/extraloop/redis-storage.rb"
+
+
+class AmazonReview < ExtraLoop::Storage::Record
+  attribute :title
+  attribute :rank
+  attribute :date
+
+  def validate
+    assert (0..5).include?(rank.to_i), "Rank not in range"
+  end
+end
+
+scraper = AmazonReviewScraper.new("0262560992").
+  set_storage(AmazonReview).
+  run
+
+records = AmazonReview.find :session_id => scraper.session.id
+puts "#{records.size} reviews have been created"
+

data/examples/lib/amazon_review_scraper.rb

@@ -0,0 +1,14 @@
+class AmazonReviewScraper < ExtraLoop::ScraperBase
+  def initialize(review_id)
+    url = "http://www.amazon.co.uk/product-reviews/#{review_id}/ref=dp_top_cm_cr_acr_txt?ie=UTF8&showViewpoints=1"
+    super(url)
+
+    loop_on("#productReviews span[class^='swSprite s_star']") do |nodes|
+      nodes.map { |node| node.parent.parent }
+    end
+
+    extract(:rank, "span[class^=swSprite]", :title)  { |title| title && title.match(/^(\d)/) && $1.to_i }
+    extract(:title, "b")
+    extract(:date, "nobr") { |date| Date.parse(date.text) if date }
+  end
+end

data/lib/extraloop/redis-storage/dataset_factory.rb

@@ -0,0 +1,16 @@
+# Creates a simple class to store an ExtraLoop 
+# generated dataset using Ohm
+
+class ExtraLoop::Storage::DatasetFactory
+  def initialize(classname, attributes=[])
+    @classname = classname.to_s.capitalize
+
+    Object.const_set(@classname, Class.new(ExtraLoop::Storage::Record) {
+      attributes.each { |attr| attribute attr }
+    })
+  end
+
+  def get_class
+    Object.const_get(@classname)
+  end
+end

data/lib/extraloop/redis-storage/record.rb

@@ -0,0 +1,44 @@
+class ExtraLoop::Storage::Record < Ohm::Model
+  include Ohm::Boundaries
+  include Ohm::Timestamping
+
+  reference :session, ExtraLoop::Storage::ScrapingSession
+  attribute :extracted_at
+  index :session_id
+
+  def initialize attrs={}
+    self.class.send :_inherit!
+    super attrs
+  end
+
+  def self.create attrs={}
+    _inherit!
+    super attrs
+  end
+
+  def to_hash
+    super.merge(attributes.reduce({}) { |memo, attribute| 
+      memo.merge(attribute => send(attribute)) 
+    })
+  end
+
+  def validate
+    assert_present :session
+  end
+
+  # 
+  # walks up the class hierarchy and incorporate
+  # Ohm attributes and indices from the superclasses
+  #
+  def self._inherit!
+    klass = self
+
+    while klass != ExtraLoop::Storage::Record
+      attributes.concat(klass.superclass.attributes).uniq!
+      indices.concat(klass.superclass.indices).uniq!
+      klass = klass.superclass
+    end
+  end
+
+  private_class_method :_inherit!
+end

data/lib/extraloop/redis-storage/scraping_session.rb

@@ -0,0 +1,12 @@
+class ExtraLoop::Storage::ScrapingSession < Ohm::Model
+
+  include Ohm::Boundaries
+  include Ohm::Timestamping
+
+  attribute :title
+
+  def records(collection)
+    Kernel.const_get(collection.to_s).
+      find(:session_id => self.id)
+  end
+end

data/lib/extraloop/redis-storage.rb

@@ -0,0 +1,30 @@
+require "json"
+require "rubygems"
+require "redis"
+require 'pry'
+require "ohm"
+require "ohm/contrib"
+require "extraloop"
+
+base_path = File.realpath(File.dirname(__FILE__))
+$: << "#{base_path}"
+require "scraper_base"
+
+
+
+module ExtraLoop
+  module Storage
+    VERSION ||= "0.0.1"
+
+    class << self
+      def connect(*args)
+        Ohm.connect(*args)
+      end
+    end
+  end
+end
+
+ExtraLoop::Storage.autoload :Record, "#{base_path}/redis-storage/record.rb"
+ExtraLoop::Storage.autoload :ScrapingSession, "#{base_path}/redis-storage/scraping_session.rb"
+ExtraLoop::Storage.autoload :DatasetFactory,  "#{base_path}/redis-storage/dataset_factory.rb"
+

data/lib/extraloop/scraper_base.rb

@@ -0,0 +1,38 @@
+class ExtraLoop::ScraperBase
+  attr_reader :session
+
+  def set_storage(*args)
+    model = args.detect { |arg| arg.is_a?(Symbol) or arg.respond_to?(:new) }
+    title = args.detect { |arg| arg.is_a?(String) }
+
+    collection_name = self.class.to_s.gsub(/(:)+/,'_').downcase + "_data"
+
+    title ||= collection_name
+    model ||= collection_name.to_sym
+
+    log_session! title
+
+    @model = model_klass = model.respond_to?(:new) && model || ExtraLoop::Storage::DatasetFactory.new(model.to_sym, @extractor_args.map(&:first)).get_class
+
+    on :data do |results|
+      # TODO: avoid calling send in the scraper object
+      results = results.map { |result| @scraper.send(:instanciate_model, result) }
+      block_given? && yield(results) || results.each { |result| result.save if result.respond_to?(:save) }
+    end
+  end
+
+  protected
+  # Creates a scraping session
+  def log_session!(title="")
+    @session ||= ExtraLoop::Storage::ScrapingSession.create :title => title
+  end
+
+  # Converts extracted records into instances of the dataset model specified as the first argument
+  # of #set_storage
+
+  def instanciate_model(record)
+    record_hash = record.respond_to?(:marshal_dump) ? record.marshal_dump : record
+    attrs = {:session => @session }.merge(record_hash)
+    @model.new(attrs)
+  end
+end

data/spec/dataset_factory_spec.rb

@@ -0,0 +1,43 @@
+load "../lib/extraloop/redis-storage.rb"
+
+describe ExtraLoop::Storage::DatasetFactory do
+  Ohm.connect :url => "redis://127.0.0.1:6379/7"
+
+  describe "#get_class" do
+    context "with invalid input" do
+      before do 
+        @factory = ExtraLoop::Storage::DatasetFactory.new(:blurb, [:a, :b, :c])
+      end
+
+      subject { @factory.get_class.new :a => 22, :b => 33, :c => 44 }
+
+      it { should respond_to :a }
+      it { should respond_to :b }
+      it { should respond_to :c }
+      it { should respond_to :save }
+      it { subject.valid?.should be_false }
+
+      after do
+        Object.send(:remove_const, :Blurb)
+      end
+    end
+
+    context "with valid input" do
+      before do 
+        @factory = ExtraLoop::Storage::DatasetFactory.new(:blurb, [:a, :b, :c])
+        @session = ExtraLoop::Storage::ScrapingSession.create
+      end
+
+      subject { @factory.get_class.new :session => @session }
+      it { subject.valid?.should be_true }
+
+    end
+
+
+    after do
+      patterns = ["Mycollection", "ExtraLoop"]
+      patterns.each { |pattern| Ohm.redis.keys("*#{pattern}*").each { |key| Ohm.redis.del(key)} }
+    end
+
+ end
+end

data/spec/record_spec.rb

@@ -0,0 +1,82 @@
+load "../lib/extraloop/redis-storage.rb"
+
+class MyRecord < ExtraLoop::Storage::Record
+  attribute :foo
+  attribute :bar
+end
+
+
+describe ExtraLoop::Storage::Record do
+
+  before do
+    @session = ExtraLoop::Storage::ScrapingSession.create
+  end
+
+  context "record subclasses" do
+
+    describe "#save" do
+
+      subject { MyRecord.new(:session => @session, :extracted_at => Time.now).save }
+
+      it { subject.extracted_at.should be_a_kind_of(Time) }
+      it { subject.session.should eql(@session) }
+
+      context "without a session attribute" do
+        subject { MyRecord.new  }
+        it { subject.valid?.should_not be_true }
+      end
+    end
+
+    describe "#create" do
+      subject { MyRecord.create(:session => @session, :extracted_at => Time.now) }
+      it { subject.extracted_at.should be_a_kind_of(Time) }
+      it { subject.session.should eql(@session) }
+    end
+
+    describe "Record::last" do
+      before do
+        2.times { MyRecord.create(:session => @session) }
+      end
+
+      subject { MyRecord }
+      it { should respond_to :last }
+    end
+
+    describe "#to_hash" do
+      before do
+        @record = MyRecord.create :foo => 'blurbzz', :bar => 'zzzzzz', :session => @session
+      end
+
+      subject { @record.to_hash }
+
+      it "should have converted its attributes as hash keys" do
+        [:session_id, :foo, :bar, :created_at, :updated_at].each { |key|  subject.should have_key(key) and subject[key].should_not be_nil }
+      end
+
+      context "excluding attributes" do
+
+        before do
+          @user_class = Class.new(ExtraLoop::Storage::Record) do
+            attribute :password
+            attribute :username
+
+            def to_hash
+              super.reject { |attr| attr === :password }
+            end
+          end
+        end
+
+        subject { @user_class.create(:session => @session, :username => 'bob', :password => 'secret' ).to_hash }
+
+        it { should_not have_key(:password) }
+        it { should have_key(:username) }
+        it { should have_key(:session_id) }
+      end
+    end
+  end
+
+  after do
+    redis = Ohm.redis
+    redis.keys("[^art]*").each { |key| redis.del(key) }
+  end
+end

data/spec/scraper_base_spec.rb

@@ -0,0 +1,45 @@
+load "../lib/extraloop/redis-storage.rb"
+
+
+describe ExtraLoop::ScraperBase do
+  Ohm.connect :url => "redis://127.0.0.1:6379/7"
+
+  before(:each) do
+    @records = records =  (1..10).to_a.map { |n| OpenStruct.new :foo => "foo#{n}" }
+    @scraper = ExtraLoop::ScraperBase.new("http://someurl.net").
+      loop_on("*").
+        extract(:foo).
+        extract(:bar)
+
+    env = ExtraLoop::ExtractionEnvironment.new(@scraper, nil, @records)
+
+    @scraper.define_singleton_method :run do
+      @environment = env
+      self.run_hook :data, [records]
+    end
+  end
+
+  describe "#set_storage" do
+    context "with no arguments but a block" do
+      before do 
+        received_records = nil
+
+        @scraper.
+          set_storage { |records| received_records = records }.
+          run()
+
+        @received_records = received_records
+      end
+      it "all records should be openstruct instances" do
+        @received_records.all? { |record| record.is_a?(Extraloop_scraperbase_data) }.should be_true
+      end
+    end
+
+    context "with title argument and no block" do
+      before do
+        @scraper.set_storage "my dummy dataset"
+      end
+    end
+  end
+end
+

data/spec/scraping_session_spec.rb

@@ -0,0 +1,25 @@
+$VERBOSE=nil
+load "../lib/extraloop/redis-storage.rb"
+
+describe ExtraLoop::Storage::ScrapingSession do
+  Ohm.connect :url => "redis://127.0.0.1:6379/7"
+
+  describe "#records" do
+    before(:each) do
+      my_collection = ExtraLoop::Storage::DatasetFactory.new(:MyCollection).get_class
+      @session = ExtraLoop::Storage::ScrapingSession.create
+      5.times do
+        item = my_collection.create(:session => @session)
+      end
+    end
+
+    context "dataset class exists" do
+      context "passing a constant" do
+        subject { @session.records(Mycollection) }
+        it { should have(5).items  }
+        it { subject.all? { |record| record.valid? }.should be_true }
+      end
+    end
+
+  end
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification
+name: extraloop-redis-storage
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Andrea Fiore
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-20 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: extraloop
+  requirement: &16470780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: *16470780
+- !ruby/object:Gem::Dependency
+  name: ohm
+  requirement: &16470320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: *16470320
+- !ruby/object:Gem::Dependency
+  name: ohm-contrib
+  requirement: &16469860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *16469860
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &16469400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  type: :development
+  prerelease: false
+  version_requirements: *16469400
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: &16468940 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+  type: :development
+  prerelease: false
+  version_requirements: *16468940
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: &16468480 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.7.4
+  type: :development
+  prerelease: false
+  version_requirements: *16468480
+description: Redis+Ohm based storage for data sets extracted using the ExtraLoop toolkit.
+email: andrea.giulio.fiore@googlemail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- History.txt
+- README.rdoc
+- examples/amazon_reviews.rb
+- examples/lib/amazon_review_scraper.rb
+- lib/extraloop/redis-storage.rb
+- lib/extraloop/redis-storage/dataset_factory.rb
+- lib/extraloop/redis-storage/record.rb
+- lib/extraloop/redis-storage/scraping_session.rb
+- lib/extraloop/scraper_base.rb
+- spec/dataset_factory_spec.rb
+- spec/record_spec.rb
+- spec/scraper_base_spec.rb
+- spec/scraping_session_spec.rb
+homepage: http://github.com/afiore/extraloop-redis-storage
+licenses: []
+post_install_message: 
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: extraloop-redis-storage
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 2
+summary: Redis storage for Extraloop.
+test_files: []
+has_rdoc: