awful 0.0.61 → 0.0.62

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9de77c555253db18fb3501a860fd378a2285ae7e
-  data.tar.gz: 9874f168edca11bab394839f8a85426bc92aa9eb
+  metadata.gz: f95833d42a45ebbe11943600ad47c0154f6d971d
+  data.tar.gz: b155483d0719e88268210fbfed8d18770e3cc612
 SHA512:
-  metadata.gz: 2d63c654d168b348bd4e90a3c54fbc8b6b9f9fc053c21260ee15dea4fa6b8766aa086f6a7477d12a432ca42dbaf374f1fcb23ab2bfdc072589ea9f9a1924a128
-  data.tar.gz: 80d40f3d3dd3efb35d613ba326eff0fcaa4e4d19a1253e338481c5952865c1d90cfc3d54413ef89079677fe3e0d21f55b25329937e5d68d3a6afb0b7b36636f6
+  metadata.gz: 6f4a1db5f374f27bac8baf813c0748c4db2a913f3c6600047c943e8c20241164cbed0efdad9ba9460515a09108a49314fd6a337fdc4eb7ddefcd2d69d6bab358
+  data.tar.gz: a594f7e829d45a30c788d2ba7c822183a983b02fa44a85b4a7d491bd7cb662cb43aa0a96095c1aa9441fe15cacca3ea7d5c94470464f1b31801af13a680d2948

data/lib/awful/dynamodb.rb

@@ -75,6 +75,21 @@ module Awful
       dynamodb.create_table(params)
     end
 
+    desc 'throughput NAME', 'get or update provisioned throughput for table NAME'
+    method_option :read_capacity_units,  aliases: '-r', type: :numeric, default: nil, desc: 'Read capacity units'
+    method_option :write_capacity_units, aliases: '-w', type: :numeric, default: nil, desc: 'Write capacity units'
+    def throughput(name)
+      provisioned = dynamodb.describe_table(table_name: name).table.provisioned_throughput.to_h
+      provisioned.merge!(symbolize_keys(options))
+      if options.empty?         # just print current throughput
+        provisioned.tap do |p|
+          puts YAML.dump(stringify_keys(p))
+        end
+      else                      # update from options
+        dynamodb.update_table(table_name: name, provisioned_throughput: only_keys_matching(provisioned, %i[read_capacity_units write_capacity_units]))
+      end
+    end
+
     desc 'delete NAME', 'delete table with NAME'
     def delete_table(name)
       confirmation = ask("to delete #{name} and all its data, type the name of table to delete:", :yellow)
@@ -87,6 +102,7 @@ module Awful
     end
 
     no_commands do
+
       ## uses simple_json to get Aws::Plugins::Protocols::JsonRpc output from scan;
       ## this also means request params need to be raw strings and not symbols, etc
       def scan_to_file(name, exclusive_start_key, fd)
@@ -100,6 +116,55 @@ module Awful
           scan_to_file(name, r['LastEvaluatedKey'], fd)
         end
       end
+
+    end
+
+    desc 'copy [region/]SRC [region/]DEST', 'copy data from table region/SRC to table region/DEST'
+    method_option :dots,       aliases: '-d', type: :boolean, default: false, desc: 'Show dots for put_item progress'
+    method_option :no_clobber, aliases: '-n', type: :boolean, default: false, desc: 'Do not overwrite existing items'
+    def copy(src, dst)
+      src_table, src_region = src.split('/').reverse # parse region/table into [table, region]
+      dst_table, dst_region = dst.split('/').reverse
+
+      ## clients are potentially for different regions
+      src_client = Aws::DynamoDB::Client.new({region: src_region}.reject{|_,v| v.nil?})
+      dst_client = Aws::DynamoDB::Client.new({region: dst_region}.reject{|_,v| v.nil?})
+
+      ## params for put_item call
+      params = {table_name: dst_table}
+
+      ## add condition not to overwrite existing primary keys (hash or composite hash AND range)
+      if options[:no_clobber]
+        keys = dst_client.describe_table(table_name: dst_table).table.key_schema.map(&:attribute_name)
+        params.merge!(condition_expression: keys.map{|key| "attribute_not_exists(#{key})"}.join(' AND '))
+      end
+
+      ## lame progress indicator, pass true for put, false for skip
+      dots = options[:dots] ? ->(x){print x ? '.' : 'x'} : ->{}
+
+      ## recursive closure to scan some items from src and put to dest;
+      ## would be more studly as an anonymous y-combinator, but we should write readable code instead
+      scan_and_put = ->(myself, key) {
+        r = src_client.scan(table_name: src_table, exclusive_start_key: key, return_consumed_capacity: 'INDEXES')
+        print "[#{Time.now}] Scanned #{r.count} items; last evaluated key: #{r.last_evaluated_key}"
+        r.items.each do |item|
+          begin
+            dst_client.put_item(params.merge(item: item))
+            dots.call(true)
+          rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException #item key exists
+            dots.call(false)
+          end
+        end
+        print "\n"
+
+        ## recurse if there are more keys to scan
+        if r.last_evaluated_key
+          myself.call(myself, r.last_evaluated_key)
+        end
+      }
+
+      ## start scanning data
+      scan_and_put.call(scan_and_put, nil)
     end
 
     desc 'scan NAME', 'scan table with NAME'
@@ -111,14 +176,52 @@ module Awful
     end
 
     desc 'put_items NAME', 'puts json items into the table with NAME'
+    method_option :no_clobber, aliases: '-n', type: :boolean, default: false, desc: 'Do not overwrite existing items'
     def put_items(name, file = nil)
+      params = {'TableName' => name}
+
+      ## set a condition not to overwrite items with existing primary key(s)
+      if options[:no_clobber]
+        keys = dynamodb.describe_table(table_name: name).table.key_schema.map(&:attribute_name)
+        params.merge!('ConditionExpression' => keys.map{|key| "attribute_not_exists(#{key})"}.join(' AND '))
+      end
+
+      ## input data
       io = (file and File.open(file)) || ((not $stdin.tty?) and $stdin)
-      count = 0
+
+      put_count = 0
+      skip_count = 0
       io.each_line do |line|
-        dynamodb_simple.put_item('TableName' => name, 'Item' => JSON.parse(line))
-        count += 1
+        begin
+          dynamodb_simple.put_item(params.merge('Item' => JSON.parse(line)))
+          put_count += 1
+        rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException #item key exists
+          skip_count += 1
+        end
+      end
+
+      ## return counts
+      [put_count, skip_count].tap do |put, skip|
+        puts "put #{put} items, skipped #{skip} items"
+      end
+    end
+
+    desc 'batch_write NAME', 'batch write items to table NAME'
+    def batch_write(name)
+      items = (1..25).map do |n|
+        {
+          put_request: {
+            item: {
+              "store_id"     => "store#{n}",
+              "object_id"    => "object#{n}",
+              "object_value" => "value#{n}"
+            }
+          }
+        }
       end
-      count.tap { |c| puts "put #{c} items" }
+      p items
+      r = dynamodb.batch_write_item(request_items: {name => items})
+      p r
     end
 
   end

data/lib/awful/version.rb

@@ -1,3 +1,3 @@
 module Awful
-  VERSION = "0.0.61"
+  VERSION = "0.0.62"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: awful
 version: !ruby/object:Gem::Version
-  version: 0.0.61
+  version: 0.0.62
 platform: ruby
 authors:
 - Ric Lister
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-10 00:00:00.000000000 Z
+date: 2015-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler