extraloop-redis-storage 0.0.4 → 0.0.5

data/README.rdoc

@@ -51,3 +51,4 @@ The +set_storage+ method can be called with the following arguments:
 * _model_ A Ruby constant specifying the model to be used for storing the extracted data .
 * _session_title_ A human readable title for the extracted dataset (optional).
 
+

data/bin/extraloop

@@ -0,0 +1,110 @@
+#! /usr/bin/env ruby
+$: << File.realpath(File.dirname(File.dirname(__FILE__))) + "/lib/extraloop"
+
+require 'thor'
+require 'pry'
+require 'fileutils'
+require 'thor/group'
+require 'redis-storage'
+
+class DataStoreCommand < Thor
+
+  ExtraLoop::Storage::autoload_models
+
+  @@sessions = ExtraLoop::Storage::ScrapingSession.all
+  @@redis = Ohm.redis
+
+  map "l" => :list,
+      "d" => :delete,
+      "e" => :export
+
+  desc "list [sessions]", "List harvested datasets filtering by session id range (e.g '25..50')"
+  def list(sessions=nil)
+    data = (filter sessions).map { |session| [ session.id, session.title, session.model && session.model.name, session.model && session.records.size ]}
+    $stdout.puts tabularize(%w[id title model records], data)
+  end
+
+  desc "delete [sessions]", "Remove datasets by session id or session id range"
+  def delete(sessions)
+    deleted = 0
+    (filter sessions).each { |session| (session.delete && session.records.each(&:delete) ) && deleted += 1 }
+    $stderr.puts "\n => #{deleted > 0 && deleted or 'No' } record#{'s' if deleted > 1} deleted \n\n" 
+    list
+  end
+
+  desc "export [sessions]", "Export datasets by session id or session id range"
+
+  method_option :format, :default => 'json', :type => 'string'
+  method_option :directory, :default => "#{Time.now.to_i}-export", :aliases => "-o"
+  method_option :require, :type => 'array', :aliases => "-r"
+
+  def export(sessions)
+    paths = Array options[:require]
+    require_files paths if paths.any?
+    format = options[:format]
+    dir = options[:directory]
+
+    raise FormatNotImplemented.new unless %w[json].include? format
+    FileUtils.mkdir(dir) unless File.exists? dir
+
+    (filter sessions).each do |session|
+      filename, data = *[ "#{session.id}_#{session.title.gsub(/\s/,"_")}", session.send("to_#{format}")]
+      File.open("#{dir}/#{filename}.#{format}", "w") { |f| f.write data }
+    end
+  end
+
+  # override default banner
+  def self.banner(task, namespace = true, subcommand = false)
+    "datastore#{task.formatted_usage(self, true, subcommand).gsub(/data_store_command/,'')}"
+  end
+
+  private
+  def require_files(paths)
+    files = paths.map do |path|
+      path = path.gsub /^(\.?\/)|(\/$)$/m, ''
+      exception = DataStoreCommand::Exceptions::FileNotFound.new "cannot find #{path}"
+      raise exception unless File.exists?(path)
+      (File.directory? path) && Dir["#{path}/*.rb"] or path
+
+    end.flatten
+
+    files.each { |file| require "./#{file}" }
+  end
+
+
+  def filter(arg=nil)
+    out = if arg
+      start_end = /(\d+)\.\.(-?\d+)/.match(arg) ? [$1, $2] : [arg, arg]
+      Array(Range.new *start_end).map { |sid| @@sessions[sid] if @@sessions[sid] }.reject(&:nil?)
+    else
+      @@sessions
+    end
+
+  end
+
+  def tabularize(headers, data)
+    rows = [headers].concat data.map { |row| row.take(headers.size) }
+
+    max_cell_sizes = Array(Range.new 0, headers.size).map do |coll_index|
+      rows.map { |row| row[coll_index].to_s.length }.max
+    end
+
+    rows = rows.map do |row|
+      " " << row.each_with_index.map { |cell, index| cell.to_s.ljust(max_cell_sizes[index]) }.join(" | ")
+    end
+
+    separator = "-" * (rows.map &:size).max
+    rows.insert(1, separator).join "\n"
+  end
+end
+
+class DataStoreCommand::Exceptions
+  class FormatNotImplemented < StandardError; end
+  class FileNotFound < StandardError; end
+end
+
+class ExtraLoopCommand < Thor
+  register(DataStoreCommand, "datastore", "datastore <command>", "CLI interface to the ExtraLoop Redis Datastore")
+end
+
+ExtraLoopCommand.start

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: extraloop-redis-storage
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-02-26 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: extraloop
-  requirement: &18201600 !ruby/object:Gem::Requirement
+  requirement: &16543420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *18201600
+  version_requirements: *16543420
 - !ruby/object:Gem::Dependency
   name: ohm
-  requirement: &18201140 !ruby/object:Gem::Requirement
+  requirement: &16542960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.1.3
   type: :runtime
   prerelease: false
-  version_requirements: *18201140
+  version_requirements: *16542960
 - !ruby/object:Gem::Dependency
   name: ohm-contrib
-  requirement: &18200680 !ruby/object:Gem::Requirement
+  requirement: &16542460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *18200680
+  version_requirements: *16542460
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &18200080 !ruby/object:Gem::Requirement
+  requirement: &16541860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -54,10 +54,10 @@ dependencies:
         version: 0.14.6
   type: :runtime
   prerelease: false
-  version_requirements: *18200080
+  version_requirements: *16541860
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &18199420 !ruby/object:Gem::Requirement
+  requirement: &16541180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 2.7.0
   type: :development
   prerelease: false
-  version_requirements: *18199420
+  version_requirements: *16541180
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &18198820 !ruby/object:Gem::Requirement
+  requirement: &16540580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: 1.0.4
   type: :development
   prerelease: false
-  version_requirements: *18198820
+  version_requirements: *16540580
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &18187500 !ruby/object:Gem::Requirement
+  requirement: &16540100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +87,11 @@ dependencies:
         version: 0.9.7.4
   type: :development
   prerelease: false
-  version_requirements: *18187500
+  version_requirements: *16540100
 description: Redis+Ohm based storage for data sets extracted using the ExtraLoop toolkit.
 email: andrea.giulio.fiore@googlemail.com
-executables: []
+executables:
+- extraloop
 extensions: []
 extra_rdoc_files: []
 files:
@@ -110,6 +111,7 @@ files:
 - spec/record_spec.rb
 - spec/scraper_base_spec.rb
 - spec/scraping_session_spec.rb
+- bin/extraloop
 homepage: http://github.com/afiore/extraloop-redis-storage
 licenses: []
 post_install_message: