extraloop-redis-storage 0.0.7 → 0.0.8

data/History.txt

@@ -1,3 +1,7 @@
+== 0.0.8  / 2012-03-04
+  * Allowing to type-cast attributes in Ohm models
+  * few other bug fixes here and there
+
 == 0.0.7  / 2012-11-03
   * Datasets can now be pushed to Google Fusion tables 
   * Added support for YAML export 

data/README.rdoc

@@ -76,6 +76,4 @@ Similarly, stored datasets can be uploaded to a remote datastore:
 
   extraloop datastore push 51..48 fusion_tables -c google_username:password
 
-While Google's Fusion Tables is currently the only one implemented, support for other remote datastores (e.g.
-[couchDB](http://couchdb.apache.org/), , [cartoDB](http://cartodb.com) ), and [CKAN Webstore](http://wiki.ckan.org/Webstore) will be added soon.
-
+While Google's Fusion Tables is currently the only one implemented, support for pushing dataset to other remote datastores (e.g. {couchDB}[http://couchdb.apache.org/],  {cartoDB}[http://cartodb.com], and {CKAN Webstore}[http://wiki.ckan.org/Webstore]) will be added soon.

data/bin/extraloop

@@ -10,15 +10,6 @@ class DataStoreCommand < Thor
 
   ExtraLoop::Storage::autoload_models
 
-  class << self
-    def parse_config
-      config_file = File.join(Etc.getpwuid.dir, '.extraloop.yml')
-      File.exist?(config_file) && YAML::load_file(config_file) or {}
-    end
-  end
-
-  @@config = parse_config
-
   @@sessions = ExtraLoop::Storage::ScrapingSession.all
   @@redis = Ohm.redis
 
@@ -63,18 +54,14 @@ class DataStoreCommand < Thor
 
   desc "push [sessions] [remote_store]", "Uploads one or several datasets to a remote data store"
   method_option :schema, :type => 'hash', :aliases => "-s"
-  method_option :credentials, :type => 'string', :aliases => "-a"
+  method_option :credentials, :type => 'hash', :aliases => "-a"
 
   def push(sessions, store_type=:fusion_tables)
+    schema = {:schema =>  options.fetch('schema', {})}
 
     filter(sessions).each do |session|
       store_type = store_type.to_sym
-      begin
-        credentials = options.fetch('credentials', @@config[:datastore] && @@config[:datastore][:credentials] && @@config[:datastore][store_type]).split(':')
-      rescue NoMethodError
-        abort "Cannot find credentials for remote datastore.\nPlease specify them using the --credential switch (e.g. 'andrea:mypassword')"
-      end
-      datastore = ExtraLoop::Storage::RemoteStore::get_transport(store_type, credentials)
+      datastore = ExtraLoop::Storage::RemoteStore::get_transport(store_type, options[:credentials], schema)
       datastore.push session
     end
   end

data/lib/extraloop/redis-storage/model.rb

@@ -2,8 +2,11 @@
 # are associated to a Scraping session object.
 #
 class ExtraLoop::Storage::Model < Ohm::Model
-  attribute :name
-  index :name
+
+  def self.[](id)
+    raise ArgumentError.new "model Id should be capitalized" unless id.to_s[0] =~ /[A-Z]/
+    super(id) || create(:id => id)
+  end
 
   def to_hash
     super.merge(attributes.reduce({}) { |memo, attribute| 

data/lib/extraloop/redis-storage/record.rb

@@ -1,13 +1,15 @@
 class ExtraLoop::Storage::Record < Ohm::Model
+  include Ohm::Callbacks
   include Ohm::Boundaries
+  include Ohm::Typecast
   include Ohm::Timestamping
 
   reference :session, ExtraLoop::Storage::ScrapingSession
-  attribute :extracted_at
+  attribute :extracted_at, Time
   index :session_id
 
   def initialize attrs={}
-    self.class.send :_inherit!
+    self.class.send(:_inherit!)
     super attrs
   end
 
@@ -38,8 +40,9 @@ class ExtraLoop::Storage::Record < Ohm::Model
     klass = self
 
     while klass != ExtraLoop::Storage::Record
-      attributes.concat(klass.superclass.attributes).uniq!
-      indices.concat(klass.superclass.indices).uniq!
+      %w[attributes indices counters].each do |method|
+        send(method).concat(klass.superclass.send(method)).uniq!
+      end
       klass = klass.superclass
     end
   end

data/lib/extraloop/redis-storage/remote_store/fusion_tables.rb

@@ -1,13 +1,10 @@
-class ExtraLoop::Storage::FusionTables
-  @@connection = nil
 
-  def initialize(credentials, options={})
-    @options = options
-    @credentials = credentials
-    @api = connect
-  end
+
+class ExtraLoop::Storage::FusionTables < ExtraLoop::Storage::RemoteStore
+  @@connection = nil
 
   def push(session)
+    @api = connect!
     dataset = session.to_hash
     records = dataset[:records]
     title   = dataset[:title].gsub(/\sDataset/,'')
@@ -20,22 +17,23 @@ class ExtraLoop::Storage::FusionTables
   private
   def make_schema(record)
     defaults = {
+      'id' => 'number',
       'session_id' => 'number'
     }
 
-    schema = defaults.merge(@options.fetch :schema, {})
-
+    option_schema = @options.fetch(:schema, {}).stringify_keys
+    schema = defaults.merge option_schema
     record.keys.
-      reject { |key| schema.keys.include?(key) }.
+      reject { |key| schema.keys.include?(key.to_s) }.
       map    { |key| {:name => key.to_s, :type => 'string'} }.
       concat(schema.map { |field, type| {:name => field.to_s, :type => type }})
   end
 
-  def connect
+  def connect!
     return @@connection if @@connection
-
     @@connection = GData::Client::FusionTables.new
-    @@connection.clientlogin(*@credentials)
+    @credentials = @credentials.symbolize_keys
+    @@connection.clientlogin(@credentials[:username], @credentials[:password])
     @@connection
   end
 end

data/lib/extraloop/redis-storage/remote_store.rb

@@ -1,13 +1,36 @@
-# Base class for pushing Extraloop datasets from the local Redis
-# store to remote ones (e.g. Google Fusion tables, Buzzdata, Cartodb)
+class ExtraLoop::Storage::RemoteStore
+  @@config = {}
 
-$: << path = File.dirname(__FILE__) + '/remote_store'
-Dir["#{path}/*.rb"].each { |store_adapter| require store_adapter }
+  # 
+  # Instanciates the relevant transport class for the selected datastore
+  # 
+  def self.get_transport(datastore, credentials=nil, options={})
+    classname = datastore.to_s.camel_case
+    ExtraLoop::Storage.const_get(classname).new(credentials, options) if ExtraLoop::Storage.const_defined?(classname)
+  end
 
+  def initialize(credentials, options={})
+    datastore = self.class.to_s.snake_case.split('/').last.to_sym
+    load_config
+    @options = options
+    @credentials = credentials || config_for([:datastore, datastore])
+    raise ExtraLoop::Storage::Exceptions::MissingCredentialsError.new "Missing credentials for '#{datastore}' remote store"  unless @credentials
+    @api = nil
+  end
 
-class ExtraLoop::Storage::RemoteStore
-  def self.get_transport(datastore, credentials, options={})
-    classname = datastore.to_s.gsub(/^.|_./) { |chars| chars.split("").last.upcase }
-    ExtraLoop::Storage.const_get(classname).new(credentials, options) if ExtraLoop::Storage.const_defined?(classname)
+  protected
+  def config_for(keys, context=@@config)
+    key = keys.shift.to_s
+    value = context.stringify_keys[key]
+    value && value.respond_to?(:fetch) && keys.any? ? config_for(keys, value) : value
+  end
+
+  def load_config
+    config_file = File.join(Etc.getpwuid.dir, '.extraloop.yml')
+    @@config = File.exist?(config_file) && YAML::load_file(config_file) || {}
   end
 end
+
+module ExtraLoop::Storage::Exceptions
+  class MissingCredentialsError < StandardError; end
+end

data/lib/extraloop/redis-storage/scraping_session.rb

@@ -6,20 +6,21 @@ class ExtraLoop::Storage::ScrapingSession < Ohm::Model
 
   attribute :title
   reference :model, ExtraLoop::Storage::Model
+  index :model_id
   
   def records(params={})
-    klass = if Object.const_defined?(model.name)
-      Object.const_get(model.name)
+    klass = if Object.const_defined?(model.id)
+      Object.const_get(model.id)
     else
       dynamic_class = Class.new(ExtraLoop::Storage::Record) do
-        # override default to_hash so that it will return the Redis hash
+        # override the default to_hash so that it will return the Redis hash
         # internally stored by Ohm
         def to_hash
           Ohm.redis.hgetall self.key
         end
       end
 
-      Object.const_set(model.name, dynamic_class)
+      Object.const_set(model.id, dynamic_class)
       dynamic_class
     end
 

data/lib/extraloop/redis-storage.rb

@@ -1,4 +1,5 @@
 require "rubygems"
+require "bundler/setup"
 require "json"
 require "yaml"
 require "rubygems"
@@ -6,6 +7,8 @@ require "redis"
 require "ohm"
 require "ohm/contrib"
 require "extraloop"
+require "fileutils"
+require "pry"
 
 begin
   gem "fusion_tables", "~> 0.3.1"
@@ -14,14 +17,23 @@ rescue Gem::LoadError
 end
 
 
+begin
+  gem "cartodb-rb-client"
+  require "cartodb-rb-client"
+rescue Gem::LoadError
+end
+
+
+
 base_path = File.realpath(File.dirname(__FILE__))
 $: << "#{base_path}"
 
 require "scraper_base"
+require "support"
 
 module ExtraLoop
   module Storage
-    VERSION ||= "0.0.7"
+    VERSION ||= "0.0.8"
 
     def self.connect(*args)
       Ohm.connect(*args)
@@ -29,7 +41,7 @@ module ExtraLoop
 
     # Tries to automatically locate the models directory and load all ruby files within in
     def self.autoload_models(dirname='models')
-      Dir["**/**#{dirname}/*.rb"].each { |path| require "./#{path}" }
+    #  Dir["**/**#{dirname}/*.rb"].each { |path| require "./#{path}" }
     end
   end
 end
@@ -44,4 +56,6 @@ ExtraLoop::Storage.autoload :ScrapingSession, "#{base_path}/scraping_session.rb"
 ExtraLoop::Storage.autoload :Model,  "#{base_path}/model.rb"
 ExtraLoop::Storage.autoload :DatasetFactory,  "#{base_path}/dataset_factory.rb"
 ExtraLoop::Storage.autoload :RemoteStore, "#{base_path}/remote_store.rb"
+ExtraLoop::Storage.autoload :FusionTables, "#{base_path}/remote_store/fusion_tables.rb"
+ExtraLoop::Storage.autoload :Cartodb, "#{base_path}/remote_store/cartodb.rb"
 

data/lib/extraloop/scraper_base.rb

@@ -7,6 +7,7 @@ class ExtraLoop::ScraperBase
     title ||= collection_name
 
     @model = model_klass = model.respond_to?(:new) && model || ExtraLoop::Storage::DatasetFactory.new(model.to_sym, @extractor_args.map(&:first)).get_class
+    log_session! title
 
     log_session! title
 
@@ -21,9 +22,7 @@ class ExtraLoop::ScraperBase
   def log_session!(title="")
     if !@session
       ns = ExtraLoop::Storage
-      results = ns::Model.find :name => @model
-      model = results.any? && results.first || ns::Model.create(:name => @model)
-      @session = ns::ScrapingSession.create :title => title, :model =>  model
+      @session = ns::ScrapingSession.create :title => title, :model =>  ExtraLoop::Storage::Model[@model.to_s]
     end
   end
 

data/lib/extraloop/support.rb

@@ -0,0 +1,37 @@
+class Hash
+  def stringify_keys
+    _reduce_keys &:to_s
+  end
+
+  def symbolize_keys
+    _reduce_keys &:to_sym
+  end
+
+  private
+  def _reduce_keys(&block)
+    self.reduce({}){|memo,(k,v)| memo[yield(k)] = v; memo}
+  end
+end
+
+class Object
+  def try(method, *args)
+    send method, *args if respond_to? method
+  end
+end
+
+class String
+  def capitalize_first
+    self[0].upcase + self[1, size]
+  end
+  def camel_case
+    self.gsub(/^.|_./) { |chars| chars.split("").last.upcase }
+  end
+
+  def snake_case
+    self.gsub(/::/, '/').
+      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z\d])([A-Z])/,'\1_\2').
+      tr("-", "_").
+      downcase
+  end
+end

data/spec/dataset_factory_spec.rb

@@ -1,8 +1,6 @@
-load "../lib/extraloop/redis-storage.rb"
+require "spec_helper"
 
 describe ExtraLoop::Storage::DatasetFactory do
-  Ohm.connect :url => "redis://127.0.0.1:6379/7"
-
   describe "#get_class" do
     context "with invalid input" do
       before do 
@@ -25,7 +23,7 @@ describe ExtraLoop::Storage::DatasetFactory do
     context "with valid input" do
       before do 
         @factory = ExtraLoop::Storage::DatasetFactory.new(:blurb, [:a, :b, :c])
-        @session = ExtraLoop::Storage::ScrapingSession.create
+        @session = ExtraLoop::Storage::ScrapingSession.create :model => ExtraLoop::Storage::Model[:Blurb]
       end
 
       subject { @factory.get_class.new :session => @session }

data/spec/fusion_tables_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+require 'gdata'
+
+class MyRecord < ExtraLoop::Storage::Record
+  attribute :index
+end
+
+describe ExtraLoop::Storage::FusionTables do
+
+  describe "#push" do
+
+    before do
+      @dataset = ExtraLoop::Storage::ScrapingSession.create :title => 'test dataset', :model => ExtraLoop::Storage::Model[:MyRecord]
+
+      5.times do |n|
+        MyRecord.create :index => n, :session => @dataset 
+      end
+
+      table=Object.new
+      mock(table).insert(@dataset.to_hash[:records])
+
+      any_instance_of(GData::Client::FusionTables) do |ft|
+        mock(ft).clientlogin(is_a(String), is_a(String)).times(any_times)
+        stub(ft).create_table { |title, fields|
+          @title = title
+          @fields = fields
+
+          table
+        }
+      end
+    end
+
+    context "without a schema definition" do
+
+      before do
+        @fusion_table = ExtraLoop::Storage::FusionTables.new :username => 'username', 'password' => "password"
+        @fusion_table.push(@dataset)
+      end
+
+      it "should name the table with a string starting with 'Dataset'" do
+        @title.should match /^Dataset/
+      end
+
+      subject { @fields }
+
+      it { should include :name => 'session_id', :type => 'number' }
+      it { should_not include :name => 'session_id', :type => 'string' }
+      it { should include :name => 'index', :type => 'string' }
+      it { should include :name => 'id', :type => 'number' }
+
+    end
+
+    context "with a schema definition" do
+
+      before do
+        @fusion_table = ExtraLoop::Storage::FusionTables.new(["username","password"], {:schema => {:session_id => 'string', :index => 'number'}})
+        @fusion_table.push(@dataset)
+      end
+
+      subject { @fields }
+
+      it { should_not include :name => 'session_id', :type => 'number' }
+      it { should include :name  => 'session_id', :type => 'string' }
+      it { should include :name  => 'index', :type => 'number' }
+    end
+  end
+end

data/spec/model_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe ExtraLoop::Storage::Model do
+  describe "::[]()" do
+    before do
+      @model = ExtraLoop::Storage::Model[:My_model]
+    end
+
+    it "should create a record if a model with id 'my_model' does not exist" do
+      @model.should eql(ExtraLoop::Storage::Model[:My_model])
+    end
+
+    it "should throw an argument error if the model id is not capitalized" do
+      lambda { ExtraLoop::Storage::Model[:my_model] }.should raise_error(ArgumentError)
+    end
+
+    after do
+      Ohm.flush
+    end
+  end
+end

data/spec/record_spec.rb

@@ -1,15 +1,16 @@
-load "../lib/extraloop/redis-storage.rb"
+require "spec_helper"
 
 class MyRecord < ExtraLoop::Storage::Record
   attribute :foo
   attribute :bar
+  counter :n
+  index :bar
 end
 
-
 describe ExtraLoop::Storage::Record do
 
   before do
-    @session = ExtraLoop::Storage::ScrapingSession.create
+    @session = ExtraLoop::Storage::ScrapingSession.create :model => ExtraLoop::Storage::Model[:MyRecord]
   end
 
   context "record subclasses" do
@@ -21,6 +22,11 @@ describe ExtraLoop::Storage::Record do
       it { subject.extracted_at.should be_a_kind_of(Time) }
       it { subject.session.should eql(@session) }
 
+      it "should correctly increment counters" do
+        3.times { subject.incr :n }
+        subject.n.should eql(3)
+      end
+
       context "without a session attribute" do
         subject { MyRecord.new  }
         it { subject.valid?.should_not be_true }
@@ -33,6 +39,18 @@ describe ExtraLoop::Storage::Record do
       it { subject.session.should eql(@session) }
     end
 
+    describe "#find" do
+      before do
+        MyRecord.create(:foo => 'foo', :bar => 'bar', :session => @session)
+        MyRecord.create(:foo => 'foo', :bar => 'bar', :session => @session)
+        @target = MyRecord.create(:foo => 'foo', :bar => 'foo', :session => @session)
+        MyRecord.create(:foo => 'foo', :bar => 'bar', :session => @session)
+      end
+
+      subject { MyRecord.find(:bar => 'foo').first }
+      it { should eql(@target) }
+    end
+
     describe "Record::last" do
       before do
         2.times { MyRecord.create(:session => @session) }

data/spec/remote_store_spec.rb

@@ -0,0 +1,47 @@
+require "spec_helper"
+
+
+describe ExtraLoop::Storage::RemoteStore do
+
+  describe "#get_transport" do
+    context "with credentials" do
+      subject { ExtraLoop::Storage::RemoteStore::get_transport(:fusion_tables, ['username', 'password'] ) }
+
+      it { should be_a_kind_of ExtraLoop::Storage::FusionTables }
+    end
+
+    context "without credentials or config file" do 
+      it "should raise a MissingCredentials error" do
+        expect { ExtraLoop::Storage::RemoteStore::get_transport(:fusion_tables, nil, {:schema => {:session_id => 'string', :index => 'number'}}) }.to raise_exception(ExtraLoop::Storage::Exceptions::MissingCredentialsError)
+      end
+    end
+
+    context "with credentials in config file" do
+      before do
+        config={
+          :datastore => {
+            :fusion_tables => {
+              :username => 'test_user',
+              :password => 'password'
+            }
+          }
+        }
+
+        config_file = File.join(Etc.getpwuid.dir, '.extraloop.yml')
+        FileUtils.mv(config_file, config_file + ".diabled") if File.exist? config_file
+        File.open(config_file, 'w') { |f| f.write config.to_yaml }
+      end
+
+      it "should not raise a MissingCredentials error" do
+        expect { ExtraLoop::Storage::get_transport(:fusion_tables, nil, {:schema => {:session_id => 'string', :index => 'number'}}) }.to_not raise_exception(ExtraLoop::Storage::Exceptions::MissingCredentialsError)
+      end
+
+      after do
+        config_file = File.join(Etc.getpwuid.dir, '.extraloop.yml')
+        FileUtils.rm config_file
+        FileUtils.mv(config_file + ".disabled", config_file) if File.exist? config_file + ".disabled"
+      end
+
+     end
+  end
+end

data/spec/scraper_base_spec.rb

@@ -1,9 +1,7 @@
-load "../lib/extraloop/redis-storage.rb"
+require 'spec_helper'
 
 
 describe ExtraLoop::ScraperBase do
-  Ohm.connect :url => "redis://127.0.0.1:6379/7"
-
   before(:each) do
     @records = records =  (1..10).to_a.map { |n| OpenStruct.new :foo => "foo#{n}" }
     @scraper = ExtraLoop::ScraperBase.new("http://someurl.net").
@@ -91,8 +89,8 @@ describe ExtraLoop::ScraperBase do
     end
 
     it "should persist 10 records" do
-      (@scraper.session.records MyModel).should have(10).records
-      (@scraper.session.records MyModel).map(&:id).reject(&:nil?).should_not be_empty
+      @scraper.session.records.should have(10).records
+      @scraper.session.records.map(&:id).reject(&:nil?).should_not be_empty
     end
   end
 end

data/spec/scraping_session_spec.rb

@@ -1,13 +1,13 @@
 $VERBOSE=nil
-load "../lib/extraloop/redis-storage.rb"
+require "spec_helper"
 
 describe ExtraLoop::Storage::ScrapingSession do
-  Ohm.connect :url => "redis://127.0.0.1:6379/7"
 
   describe "#records" do
     before(:each) do
       my_collection = ExtraLoop::Storage::DatasetFactory.new(:MyCollection).get_class
-      @session = ExtraLoop::Storage::ScrapingSession.create
+      @model = ExtraLoop::Storage::Model[:MyCollection]
+      @session = ExtraLoop::Storage::ScrapingSession.create :model => @model
       5.times do
         item = my_collection.create(:session => @session)
       end
@@ -15,7 +15,7 @@ describe ExtraLoop::Storage::ScrapingSession do
 
     context "dataset class exists" do
       context "passing a constant" do
-        subject { @session.records(Mycollection) }
+        subject { @session.records }
         it { should have(5).items  }
         it { subject.all? { |record| record.valid? }.should be_true }
       end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'pry'
+path = File.dirname(File.dirname(__FILE__))
+$: << path + "/lib/extraloop"
+
+require "redis-storage"
+
+RSpec.configure do |config|
+  config.mock_with :rr
+end
+
+ENV['REDIS_URL']= "redis://127.0.0.1:6379/7"
+Ohm.connect

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: extraloop-redis-storage
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-11 00:00:00.000000000Z
+date: 2012-04-03 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: extraloop
-  requirement: &14542580 !ruby/object:Gem::Requirement
+  requirement: &17153680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *14542580
+  version_requirements: *17153680
 - !ruby/object:Gem::Dependency
   name: ohm
-  requirement: &14542000 !ruby/object:Gem::Requirement
+  requirement: &17152940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.1.3
   type: :runtime
   prerelease: false
-  version_requirements: *14542000
+  version_requirements: *17152940
 - !ruby/object:Gem::Dependency
   name: ohm-contrib
-  requirement: &14507840 !ruby/object:Gem::Requirement
+  requirement: &17151620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *14507840
+  version_requirements: *17151620
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &14507360 !ruby/object:Gem::Requirement
+  requirement: &17151040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -54,10 +54,10 @@ dependencies:
         version: 0.14.6
   type: :runtime
   prerelease: false
-  version_requirements: *14507360
+  version_requirements: *17151040
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &14506900 !ruby/object:Gem::Requirement
+  requirement: &17148720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *14506900
+  version_requirements: *17148720
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &14506220 !ruby/object:Gem::Requirement
+  requirement: &17147980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,21 @@ dependencies:
         version: 2.7.0
   type: :development
   prerelease: false
-  version_requirements: *14506220
+  version_requirements: *17147980
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: &17147320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :development
+  prerelease: false
+  version_requirements: *17147320
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &14505600 !ruby/object:Gem::Requirement
+  requirement: &17146760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +98,10 @@ dependencies:
         version: 1.0.4
   type: :development
   prerelease: false
-  version_requirements: *14505600
+  version_requirements: *17146760
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &14505020 !ruby/object:Gem::Requirement
+  requirement: &17146020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,7 +109,29 @@ dependencies:
         version: 0.9.7.4
   type: :development
   prerelease: false
-  version_requirements: *14505020
+  version_requirements: *17146020
+- !ruby/object:Gem::Dependency
+  name: fusion_tables
+  requirement: &17145200 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.1
+  type: :development
+  prerelease: false
+  version_requirements: *17145200
+- !ruby/object:Gem::Dependency
+  name: geocoder
+  requirement: &17144660 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+  type: :development
+  prerelease: false
+  version_requirements: *17144660
 description: Redis-based Persistence layer for the ExtraLoop data extraction toolkit.
   Includes a convinent command line tool allowing to list, filter, delete, and export
   harvested datasets
@@ -122,10 +155,15 @@ files:
 - lib/extraloop/redis-storage/remote_store/fusion_tables.rb
 - lib/extraloop/redis-storage/scraping_session.rb
 - lib/extraloop/scraper_base.rb
+- lib/extraloop/support.rb
 - spec/dataset_factory_spec.rb
+- spec/fusion_tables_spec.rb
+- spec/model_spec.rb
 - spec/record_spec.rb
+- spec/remote_store_spec.rb
 - spec/scraper_base_spec.rb
 - spec/scraping_session_spec.rb
+- spec/spec_helper.rb
 - bin/extraloop
 homepage: http://github.com/afiore/extraloop-redis-storage
 licenses: []
@@ -142,7 +180,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1448249409185434738
+      hash: 3286007203333335840
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: