extraloop-redis-storage 0.0.5 → 0.0.6

data/History.txt

@@ -1,3 +1,10 @@
+== 0.0.6  / 2012-26-02
+
+  * Added CSV export
+
+== 0.0.5  / 2012-26-02
+  * added command line interface for listing, deleting and exporing stored records.
+
 == 0.0.2  / 2012-20-02
   * Fixed scope bug
 

data/README.rdoc

@@ -3,8 +3,9 @@
 == Description
 
 Persistence layer for the {ExtraLoop}[https://github.com/afiore/extraloop] data extraction toolkit.
-The module is implemented as a small wrapper built on top of {Ohm}[http://ohm.keyvalue.org], an object-hash mapping library which 
-makes storing and managing Ruby objects in Redis easy and convenient.
+This module is implemented as a wrapper around {Ohm}[http://ohm.keyvalue.org], an object-hash mapping library which 
+makes easy storing structured data into Redis. It comes with a convinent command line tool, which allows to 
+list, filter, delete, and export harvested datasets.
 
 == Installation 
 
@@ -44,11 +45,34 @@ session instance:
  reviews = scraper.session.records AmazonReview
 
 
-== The #set_storage method
+=== The #set_storage method
 
 The +set_storage+ method can be called with the following arguments:
 
 * _model_ A Ruby constant specifying the model to be used for storing the extracted data .
 * _session_title_ A human readable title for the extracted dataset (optional).
 
+== Command line interface 
 
+Once installed, the gem will also add to your system path the +extraloop+ executable, a command line interface to the datasets harvested through extraloop.
+A list of datasets can be obtained by running:
+
+ extraloop datastore list:
+
+This will generate a table like the following one:
+<code>
+  id | title                              | model           | records
+ --------------------------------------------------------------------
+  48 | 1330106699 GoogleNewsStory Dataset | GoogleNewsStory | 110    
+  49 | 1330106948 AmazonReview Dataset    | AmazonReview    | 0      
+  51 | 1330107087 GoogleNewsStory Dataset | GoogleNewsStory | 110    
+  52 | 1330111630 AmazonReview Dataset    | AmazonReview    | 10     
+</code>
+
+Datasets can by removed using the +delete+ subcommand:
+
+ extraloop datastore delete [id]
+
+Where +id+ is either a single scraping session id, or a session id range (e.g. 48..52).
+Finally, the +export+ subcommand allows to export one or several datasets into a JSON or CSV documents.
+Please refer to the executable inline help (<code>extraloop datastore help [command]</code>) for more usage information.

data/bin/extraloop

@@ -34,7 +34,7 @@ class DataStoreCommand < Thor
 
   desc "export [sessions]", "Export datasets by session id or session id range"
 
-  method_option :format, :default => 'json', :type => 'string'
+  method_option :format, :default => 'json', :type => 'string', :aliases => "-f"
   method_option :directory, :default => "#{Time.now.to_i}-export", :aliases => "-o"
   method_option :require, :type => 'array', :aliases => "-r"
 
@@ -43,8 +43,8 @@ class DataStoreCommand < Thor
     require_files paths if paths.any?
     format = options[:format]
     dir = options[:directory]
-
-    raise FormatNotImplemented.new unless %w[json].include? format
+    exception = DataStoreCommand::Exceptions::FormatNotImplemented.new "Format not supported #{format}"
+    raise exception unless %w[json csv].include? format
     FileUtils.mkdir(dir) unless File.exists? dir
 
     (filter sessions).each do |session|

data/lib/extraloop/redis-storage.rb

@@ -25,6 +25,8 @@ module ExtraLoop
   end
 end
 
+autoload :CSV, 'csv'
+autoload :Iconv, 'iconv'
 ExtraLoop::Storage.autoload :Record, "#{base_path}/redis-storage/record.rb"
 ExtraLoop::Storage.autoload :ScrapingSession, "#{base_path}/redis-storage/scraping_session.rb"
 ExtraLoop::Storage.autoload :Model,  "#{base_path}/redis-storage/model.rb"

data/lib/extraloop/redis-storage/scraping_session.rb

@@ -1,11 +1,15 @@
 class ExtraLoop::Storage::ScrapingSession < Ohm::Model
 
+  BOM = "\377\376" #Byte Order Mark
+
   include Ohm::Boundaries
   include Ohm::Timestamping
   include Ohm::Callbacks
 
   attribute :title
   reference :model, ExtraLoop::Storage::Model
+  
+
 
   def records(params={})
     klass = if Object.const_defined?(model.name)
@@ -47,5 +51,9 @@ class ExtraLoop::Storage::ScrapingSession < Ohm::Model
   end
 
   def to_csv
+    _records = Array records.all.map &:to_hash
+    header = _records.first && _records.first.keys.map(&:to_s)
+    data = [header].concat _records.map(&:values)
+    output = data.map { |cells| CSV.generate_line cells }.join
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: extraloop-redis-storage
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-02-26 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: extraloop
-  requirement: &16543420 !ruby/object:Gem::Requirement
+  requirement: &10773840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *16543420
+  version_requirements: *10773840
 - !ruby/object:Gem::Dependency
   name: ohm
-  requirement: &16542960 !ruby/object:Gem::Requirement
+  requirement: &10773380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.1.3
   type: :runtime
   prerelease: false
-  version_requirements: *16542960
+  version_requirements: *10773380
 - !ruby/object:Gem::Dependency
   name: ohm-contrib
-  requirement: &16542460 !ruby/object:Gem::Requirement
+  requirement: &10772860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *16542460
+  version_requirements: *10772860
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &16541860 !ruby/object:Gem::Requirement
+  requirement: &10772260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -54,10 +54,10 @@ dependencies:
         version: 0.14.6
   type: :runtime
   prerelease: false
-  version_requirements: *16541860
+  version_requirements: *10772260
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &16541180 !ruby/object:Gem::Requirement
+  requirement: &10771600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 2.7.0
   type: :development
   prerelease: false
-  version_requirements: *16541180
+  version_requirements: *10771600
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &16540580 !ruby/object:Gem::Requirement
+  requirement: &10771000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: 1.0.4
   type: :development
   prerelease: false
-  version_requirements: *16540580
+  version_requirements: *10771000
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &16540100 !ruby/object:Gem::Requirement
+  requirement: &10770520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,7 +87,7 @@ dependencies:
         version: 0.9.7.4
   type: :development
   prerelease: false
-  version_requirements: *16540100
+  version_requirements: *10770520
 description: Redis+Ohm based storage for data sets extracted using the ExtraLoop toolkit.
 email: andrea.giulio.fiore@googlemail.com
 executables: