sdbcli 1.0.1 → 1.1.0

data/README

@@ -19,6 +19,11 @@ https://bitbucket.org/winebarrel/sdbcli
       -e, --eval=COMMAND
       -f, --format=YAML_OR_JSON
       -c, --consistent
+      -t, --timeout=SECOND
+          --import=DOMAIN,FILE
+          --import-replace=DOMAIN,FILE
+          --export=DOMAIN,FILE
+          --export-retry=NUM
   shell> export AWS_ACCESS_KEY_ID='...'
   shell> export AWS_SECRET_ACCESS_KEY='...'
   shell> export SDB_ENDPOINT='sdb.ap-northeast-1.amazonaws.com' # or SDB_REGION=ap-northeast-1
@@ -81,6 +86,7 @@ https://bitbucket.org/winebarrel/sdbcli
   .quit | .exit              exits sdbcli
   .format     (yaml|json)?   displays a format or changes it
   .consistent (true|false)?  displays ConsistentRead parameter or changes it
+  .timeout    SECOND         displays a timeout second or changes it
   .version                   displays a version
   
   ap-northeast-1> select * from test;
@@ -113,3 +119,17 @@ see http://docs.amazonwebservices.com/AmazonSimpleDB/latest/DeveloperGuide/Quoti
   ---
   - [itemname1, {attr1: val1, attr2: val2}]
   - [itemname2, {attr1: val1, attr2: val2}]
+
+== Import/Export
+
+  shell> sdbcli -f json  --export=employees,employees.json
+  // 2500 rows was outputted...
+  // 5000 rows was outputted...
+  // 7500 rows was outputted...
+  ...
+  shell> sdbcli -f json --import=employees,employees.json
+  // 2500 rows was inputted...
+  // 5000 rows was inputted...
+  // 7500 rows was inputted...
+
+If '-' is specified as a file name, the input/output of data will become a standard input/output.

data/bin/sdbcli

@@ -1,9 +1,10 @@
 #!/usr/bin/env ruby
 $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
 
-Version = '1.0.1'
+Version = '1.1.0'
 HISTORY_FILE = File.join((ENV['HOME'] || ENV['USERPROFILE'] || '.'), '.sdbcli_history')
 HISTSIZE = 500
+SELECT_LIMIT = 2500
 
 require 'rubygems'
 require 'sdbcli'
@@ -21,6 +22,10 @@ sdb_endpoint      = ENV['SDB_ENDPOINT'] || ENV['SDB_REGION'] || 'sdb.amazonaws.c
 command = nil
 $format = :yaml
 $consistent = false
+import = nil
+export = nil
+export_retry = 3
+timeout = 60
 
 ARGV.options do |opt|
   opt.on('-k', '--access-key=ACCESS_KEY') {|v| access_key_id = v }
@@ -29,6 +34,11 @@ ARGV.options do |opt|
   opt.on('-e', '--eval=COMMAND') {|v| command = v }
   opt.on('-f', '--format=YAML_OR_JSON', [:yaml, :json]) {|v| $format = v }
   opt.on('-c', '--consistent') { $consistent = true }
+  opt.on('-t', '--timeout=SECOND', Integer) {|v| timeout = v.to_i }
+  opt.on(''  , '--import=DOMAIN,FILE') {|v| import = v.split(/\s*,\s*/, 2) + [false] }
+  opt.on(''  , '--import-replace=DOMAIN,FILE') {|v| import = v.split(/\s*,\s*/, 2) + [true] }
+  opt.on(''  , '--export=DOMAIN,FILE') {|v| export = v.split(/\s*,\s*/, 2) }
+  opt.on(''  , '--export-retry=NUM', Integer) {|v| export_retry = v.to_i }
   opt.parse!
 
   unless access_key_id and secret_access_key and sdb_endpoint
@@ -38,17 +48,19 @@ ARGV.options do |opt|
 end
 
 $runner = SimpleDB::Runner.new(access_key_id, secret_access_key, sdb_endpoint)
+$runner.timeout = timeout
 
-def output_error(msg)
+def output_error(msg, strip = false)
   case $format
   when :yaml
-    msg = "# #{msg}\n\n"
+    msg = "# #{msg}"
   when :json
-    msg = "// #{msg}\n\n"
+    msg = "// #{msg}"
   else
-    msg = "# #{msg}\n\n"
+    msg = "# #{msg}"
   end
 
+  msg << "\n\n" unless strip
   $stderr.puts msg
 end
 
@@ -105,7 +117,17 @@ def execute(src, show_rows = false)
         when :yaml
           str = YAML.dump(out)
         when :json
-          str = JSON.pretty_generate(out)
+          if out.kind_of?(Array) and inline
+            str = "[\n"
+            out.each_with_index do |item, i|
+              str << "  #{item.to_json}"
+              str << ',' if i < (out.length - 1)
+              str << "\n"
+            end
+            str << "]"
+          else
+            str = JSON.pretty_generate(out)
+          end
         else
           output_error('Unknown format')
         end
@@ -139,6 +161,145 @@ def execute(src, show_rows = false)
   buf
 end
 
+# export mode
+if export
+  export_domain, export_file = export
+
+  begin
+    export_file = (export_file == '-') ? $stdout : open(export_file, 'wb')
+
+    case $format
+    when :yaml
+      export_file.puts '---'
+    when :json
+      export_file.puts '['
+    else
+      raise 'must not happen'
+    end
+
+    query = "SELECT * FROM #{export_domain} "
+    query_expr = "WHERE itemName() IS NOT NULL ORDER BY itemName() LIMIT #{SELECT_LIMIT}"
+    rownum = 0
+
+    loop do
+      items = nil
+
+      export_retry.times do |i|
+        begin
+          items = $runner.execute(query + query_expr, true, $consistent)
+          break
+        rescue Timeout::Error => e
+          raise e if i >= (export_retry - 1)
+        end
+
+        output_error("Retry...", true)
+      end
+
+      break if items.empty?
+      rownum += items.length
+
+      case $format
+      when :yaml
+        export_file.puts YAML.dump(items).sub(/\A---\s*/, '').strip
+      when :json
+        export_file.print "," if rownum > SELECT_LIMIT
+        items.each_with_index do |item, i|
+          row = item.to_json
+          row << ',' if i < (items.length - 1)
+          export_file.puts row
+        end
+      else
+        raise 'must not happen'
+      end
+
+      tail_item_name = "'" + items.last[0].gsub(/'/, "''") + "'"
+      query_expr = "WHERE itemName() > #{tail_item_name} ORDER BY itemName() LIMIT #{SELECT_LIMIT}"
+
+      output_error("#{rownum} #{rownum > 1 ? 'rows' : 'row'} was outputted...", true)
+    end
+
+    output_error("#{rownum} #{rownum > 1 ? 'rows' : 'row'} was processed", true)
+  rescue => e
+    output_error e.message.strip
+    exit 1
+  end
+
+  case $format
+  when :yaml
+  when :json
+    export_file.puts ']'
+  else
+    raise 'must not happen'
+  end
+
+  begin
+    if export_file.kind_of?(IO) and export_file != $stdout and not export_file.closed?
+      export_file.close
+    end
+  rescue => e
+    output_error e.message.strip
+    exit 1
+  end
+
+  exit 0
+end
+
+# import mode
+if import
+  import_domain, import_file, import_as_replace = import
+
+  unless import_file == '-' or File.exist?(import_file)
+    output_error("No such file: #{import_file}")
+    exit 1
+  end
+
+  begin
+    items = case $format
+            when :yaml
+              if import_file == '-'
+                YAML.load($stdin)
+              else
+                YAML.load_file(import_file)
+              end
+            when :json
+              if import_file == '-'
+                JSON.load($stdin)
+              else
+                open(import_file) {|f| JSON.load(f) }
+              end
+            else
+              raise 'must not happen'
+            end
+
+    unless items.kind_of?(Array)
+      output_error('Route object is not Array')
+      exit 1
+    end
+
+    rownum = 0
+
+    until (chunk = items.slice!(0, SELECT_LIMIT)).empty?
+      rownum += chunk.length
+
+      if import_as_replace
+        $runner.driver.update(import_domain, chunk)
+      else
+        $runner.driver.insert(import_domain, chunk)
+      end
+
+      output_error("#{rownum} #{rownum > 1 ? 'rows' : 'row'} was inputted...", true)
+    end
+
+    output_error("#{rownum} #{rownum > 1 ? 'rows' : 'row'} was processed", true)
+
+    exit 0
+  rescue => e
+    output_error e.message.strip
+    exit 1
+  end
+end
+
+# eval mode
 if not $stdin.tty? or command
   src = command || $stdin.read.strip
 
@@ -202,6 +363,7 @@ USE region_or_endpoint
 .quit | .exit              exits sdbcli
 .format     (yaml|json)?   displays a format or changes it
 .consistent (true|false)?  displays ConsistentRead parameter or changes it
+.timeout    SECOND         displays a timeout second or changes it
 .version                   displays a version
 
     EOS
@@ -268,6 +430,15 @@ while buf = Readline.readline(prompt, true)
       else
         output_error('Invalid argument')
       end
+    elsif r =~ 'timeout'
+      case (arg || '').strip
+      when ''
+        puts $runner.timeout
+      when /\d+/
+        $runner.timeout = arg.to_i
+      else
+        output_error('Invalid argument')
+      end
     elsif r =~ 'version'
       puts "sdbcli #{Version}"
     else

data/lib/sdbcli/sdb-client.rb

@@ -13,11 +13,13 @@ module SimpleDB
     SIGNATURE_ALGORITHM = :SHA256
 
     attr_accessor :endpoint
+    attr_accessor :timeout
 
     def initialize(accessKeyId, secretAccessKey, endpoint = 'sdb.amazonaws.com')
       @accessKeyId = accessKeyId
       @secretAccessKey = secretAccessKey
       @endpoint = endpoint
+      @timeout = 60
     end
 
     # domain action
@@ -92,6 +94,8 @@ module SimpleDB
       https = Net::HTTP.new(@endpoint, 443)
       https.use_ssl = true
       https.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      https.open_timeout = @timeout
+      https.read_timeout = @timeout
 
       doc = https.start do |w|
         req = Net::HTTP::Post.new('/',

data/lib/sdbcli/sdb-driver.rb

@@ -20,6 +20,14 @@ module SimpleDB
       @client.endpoint = v
     end
 
+    def timeout
+      @client.timeout
+    end
+
+    def timeout=(v)
+      @client.timeout = v
+    end
+
     # domain action
 
     def create_domain(domain_name)

data/lib/sdbcli/sdb-runner.rb

@@ -16,6 +16,8 @@ module SimpleDB
     }
 
   class Runner
+    attr_reader :driver
+
     def initialize(accessKeyId, secretAccessKey, endpoint = 'sdb.amazonaws.com')
       endpoint = region_to_endpoint(endpoint)
       @driver = Driver.new(accessKeyId, secretAccessKey, endpoint)
@@ -30,6 +32,14 @@ module SimpleDB
       @driver.endpoint = v
     end
 
+    def timeout
+      @driver.timeout
+    end
+
+    def timeout=(v)
+      @driver.timeout = v
+    end
+
     def region
       REGIONS[endpoint]
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sdbcli
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-02 00:00:00.000000000 Z
+date: 2013-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri