dynamini 2.9.1 → 2.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d5e2c24bc6e120e7bb25eebcc819ec7081aef462
-  data.tar.gz: e484085550e7692f05431a5b79b38a6f4f002cda
+  metadata.gz: 4da5852711d8ecb8a226a9ae7c987f2ea0324927
+  data.tar.gz: eec0aaae89719060294b9ae055f8effabf79dd9b
 SHA512:
-  metadata.gz: 5e82d4a69eb80db5534cd3a70e4c1f795ba13651202ca6b228945a7fb58fd5c7289c173f7cbf12a8ca9b3a255f986b14c7b6410e63d8e8e44168c82f417dbf18
-  data.tar.gz: 387de4884bedfed45b9a475a617d2666382271221288eb45ae2d9d32fb3bcdf51f8f05e5fe5d7e511533526e7c4b7166a11977984a923abf61e9453eb71aae03
+  metadata.gz: f6eaf764f422c931ae8e5d669f2b26349cfaffedfcabb6c13654f06ac40b758ea371b0328149d1f3c0914de67972fa7971d7a01d89bef94b6ab825b3bd1527c3
+  data.tar.gz: d97dab7690b1c371052bff3c1c1a7b8d39107281b2ed71c90e6eb05b3dc18998d9f51c8ad81307ebb249de1d4924f56245ef8f4e85100d42d4fd71b870f5081c

data/Gemfile.lock

@@ -1,30 +1,29 @@
 PATH
   remote: .
   specs:
-    dynamini (2.9.1)
+    dynamini (2.9.2)
       activemodel (>= 3, < 5.0)
       aws-sdk (~> 2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (4.2.9)
-      activesupport (= 4.2.9)
+    activemodel (4.2.5.1)
+      activesupport (= 4.2.5.1)
       builder (~> 3.1)
-    activesupport (4.2.9)
+    activesupport (4.2.5.1)
       i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    aws-sdk (2.10.44)
-      aws-sdk-resources (= 2.10.44)
-    aws-sdk-core (2.10.44)
-      aws-sigv4 (~> 1.0)
+    aws-sdk (2.2.22)
+      aws-sdk-resources (= 2.2.22)
+    aws-sdk-core (2.2.22)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.10.44)
-      aws-sdk-core (= 2.10.44)
-    aws-sigv4 (1.0.2)
-    builder (3.2.3)
+    aws-sdk-resources (2.2.22)
+      aws-sdk-core (= 2.2.22)
+    builder (3.2.2)
     coderay (1.1.0)
     diff-lcs (1.2.5)
     ffi (1.9.10)
@@ -49,14 +48,15 @@ GEM
     guard-shell (0.7.1)
       guard (>= 2.0.0)
       guard-compat (~> 1.0)
-    i18n (0.8.6)
-    jmespath (1.3.1)
+    i18n (0.7.0)
+    jmespath (1.1.3)
+    json (1.8.3)
     listen (3.0.3)
       rb-fsevent (>= 0.9.3)
       rb-inotify (>= 0.9)
     lumberjack (1.0.9)
     method_source (0.8.2)
-    minitest (5.10.3)
+    minitest (5.8.4)
     nenv (0.2.0)
     notiffany (0.0.8)
       nenv (~> 0.1)
@@ -85,8 +85,8 @@ GEM
     shellany (0.0.1)
     slop (3.6.0)
     thor (0.19.1)
-    thread_safe (0.3.6)
-    tzinfo (1.2.3)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
       thread_safe (~> 0.1)
 
 PLATFORMS
@@ -99,6 +99,3 @@ DEPENDENCIES
   guard-shell
   pry (~> 0)
   rspec (~> 3)
-
-BUNDLED WITH
-   1.16.0.pre.2

data/README.md

@@ -204,18 +204,27 @@ Table scanning is a very expensive operation, and should not be undertaken witho
 
 The following options are supported:
 
-* consistent_read (default: false)
-* start_key (hash key of first desired item, default will scan from beginning)
-* index_name (if scanning a secondary index - see below)
-* limit
+* :consistent_read (default: false)
+* :start_key (key of first desired item, if unset will scan from beginning)
+* :index_name (if scanning a secondary index - see below)
+* :limit (default: no limit except for AWS chunk size)
+* :segment (for multiprocess scanning)
+* :total_segments (for multiprocess scanning)
+
+Note that start_key can be either a hash { "AttributeName" => "Value" } or a value literal. If start_key is a literal then the attribute name will be inferred automatically, either being the main hash_key of your model or the key of the secondary index matching the provided index_name. 
+
+For more information about using segment and total_segments for parallelization, see: http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Scan.html
 
-These two options are to support paralellization of table scanning, see: http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Scan.html
-* segment
-* total_segments
 
 ```ruby
-products_page_one = Product.scan(limit: 100)
-products_page_one.found # [product, product...]
+products_page_one = Product.scan(limit: 100, start_key: 'abcd')
+
+products_page_one.found 
+> [product, product...]
+
+products_page_one.last_evaluated_key
+> {'id' => 'wxyz'}
+
 page_two = Product.scan(start_key: products_page_one.last_evaluated_key)
 ```
 

data/dynamini.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'dynamini'
-  s.version = '2.9.1'
+  s.version = '2.9.2'
   s.summary = 'DynamoDB interface'
   s.description = 'Lightweight DynamoDB interface gem designed as
                    a drop-in replacement for ActiveRecord.

data/lib/dynamini/batch_operations.rb

@@ -36,14 +36,9 @@ module Dynamini
     def scan(options = {})
       validate_scan_options(options)
       response = dynamo_scan(options)
-      if options[:index_name]
-        last_evaluated_key = response.last_evaluated_key[secondary_index[options[:index_name]][:hash_key_name].to_s]
-      else
-        last_evaluated_key = response.last_evaluated_key[hash_key.to_s]
-      end
       OpenStruct.new(
-        last_evaluated_key: last_evaluated_key,
-        items: response.items.map { |i| new(i.symbolize_keys, false) }
+        items: response.items.map { |i| new(i.symbolize_keys, false) },
+        last_evaluated_key: response.last_evaluated_key
       )
     end
 
@@ -72,9 +67,18 @@ module Dynamini
     end
 
     def dynamo_scan(options)
+      if options[:start_key] && !options[:start_key].is_a?(Hash)
+        if options[:index_name]
+          start_key = { options[:index_name].to_s => options[:start_key] }
+        else
+          start_key = { hash_key.to_s => options[:start_key] }
+        end
+      else
+        start_key = options[:start_key]
+      end
       client.scan({
         consistent_read:      options[:consistent_read],
-        exclusive_start_key:  options[:exclusive_start_key],
+        exclusive_start_key:  start_key,
         secondary_index_name: options[:index_name],
         limit:                options[:limit],
         segment:              options[:segment],

data/lib/dynamini/test_client.rb

@@ -90,8 +90,8 @@ module Dynamini
       sort_scanned_records!(records, args[:secondary_index_name]) if args[:secondary_index_name]
       start_index = index_of_start_key(args, records)
       items = limit_scanned_records(args[:limit], records, start_index)
-      last_evaluated_key = get_last_evaluated_key(args[:secondary_index_name], items)
-      OpenStruct.new(items: items, last_evaluated_key: last_evaluated_key)
+      last_evaluated_key = get_last_evaluated_key(args[:secondary_index_name], items, records)
+      OpenStruct.new({items: items, last_evaluated_key: last_evaluated_key})
     end
 
     def sort_scanned_records!(records, secondary_index_name)
@@ -106,9 +106,9 @@ module Dynamini
         sec_index = secondary_index[args[:secondary_index_name]]
         start_index = records.index do |r|
           if sec_index
-            r[get_secondary_hash_key(sec_index)] == args[:exclusive_start_key]
+            r[get_secondary_hash_key(sec_index)] == args[:exclusive_start_key].values[0]
           else
-            r[hash_key_attr] == args[:exclusive_start_key]
+            r[hash_key_attr] == args[:exclusive_start_key].values[0]
           end
         end
         start_index || -1
@@ -122,16 +122,15 @@ module Dynamini
       records[start_index..end_index]
     end
 
-    def get_last_evaluated_key(secondary_index_name, items)
-      # TODO should last evaluated key be present if the args[:limit] was reached?
-      # if items.length > records.length
-      index = secondary_index[secondary_index_name]
-      if index
-        { get_secondary_hash_key(index).to_s => items.last[get_secondary_hash_key(index)] }
-      else
-        { hash_key_attr.to_s => items.last[hash_key_attr] }
+    def get_last_evaluated_key(secondary_index_name, items, records)
+      if items.last != records.last
+        index = secondary_index[secondary_index_name]
+        if index
+          { get_secondary_hash_key(index).to_s => items.last[get_secondary_hash_key(index)] }
+        else
+          { hash_key_attr.to_s => items.last[hash_key_attr] }
+        end
       end
-      # end
     end
 
     # TODO add range key support for delete, not currently implemented batch_operations.batch_delete

data/spec/dynamini/batch_operations_spec.rb

@@ -146,16 +146,24 @@ describe Dynamini::BatchOperations do
       context 'with an exclusive_start_key' do
         context 'with a limit' do
           it 'retrieves the correct items' do
-            response = SecBase.scan(exclusive_start_key: '124', limit: 1)
+            response = SecBase.scan(start_key: '124', limit: 1)
             expect(response.items.map { |i| i.id }).to eq(['124'])
-            expect(response.last_evaluated_key).to eq('124')
+            expect(response.last_evaluated_key).to eq('id' => '124')
           end
         end
         context 'without a limit' do
           it 'retrieves the correct items' do
-            response = SecBase.scan(exclusive_start_key: '124')
+            response = SecBase.scan(start_key: '124')
             expect(response.items.map { |i| i.id }).to eq(%w(124 125 126))
-            expect(response.last_evaluated_key).to eq('126')
+            expect(response.last_evaluated_key).to be_nil
+          end
+        end
+
+        context 'the start key is in aws sdk style' do
+          it 'retrieves the correct items' do
+            response = SecBase.scan(start_key: {'id' => '124'}, limit: 2)
+            expect(response.items.map { |i| i.id }).to eq(['124', '125'])
+            expect(response.last_evaluated_key).to eq('id' => '125')
           end
         end
       end
@@ -164,14 +172,14 @@ describe Dynamini::BatchOperations do
           it 'retrieves the correct items' do
             response = SecBase.scan(limit: 2)
             expect(response.items.map { |i| i.id }).to eq(%w(123 124))
-            expect(response.last_evaluated_key).to eq('124')
+            expect(response.last_evaluated_key).to eq('id' => '124')
           end
         end
         context 'without a limit' do
           it 'retrieves the correct items' do
             response = SecBase.scan
             expect(response.items.map { |i| i.id }).to eq(%w(123 124 125 126))
-            expect(response.last_evaluated_key).to eq('126')
+            expect(response.last_evaluated_key).to be_nil
           end
         end
       end
@@ -180,16 +188,16 @@ describe Dynamini::BatchOperations do
       context 'with an exclusive_start_key' do
         context 'with a limit' do
           it 'retrieves the correct items' do
-            response = SecBase.scan(index_name: 'sec', exclusive_start_key: 'B', limit: 2)
+            response = SecBase.scan(index_name: 'sec', start_key: 'B', limit: 2)
             expect(response.items.map { |i| i.sec }).to eq(%w(B C))
-            expect(response.last_evaluated_key).to eq('C')
+            expect(response.last_evaluated_key).to eq('sec' => 'C')
           end
         end
         context 'without a limit' do
           it 'retrieves the correct items' do
-            response = SecBase.scan(index_name: 'sec', exclusive_start_key: 'B')
+            response = SecBase.scan(index_name: 'sec', start_key: 'B')
             expect(response.items.map { |i| i.sec }).to eq(%w(B C D))
-            expect(response.last_evaluated_key).to eq('D')
+            expect(response.last_evaluated_key).to be_nil
           end
         end
       end
@@ -198,14 +206,14 @@ describe Dynamini::BatchOperations do
           it 'retrieves the correct items' do
             response = SecBase.scan(index_name: 'sec', limit: 3)
             expect(response.items.map { |i| i.sec }).to eq(%w(A B C))
-            expect(response.last_evaluated_key).to eq('C')
+            expect(response.last_evaluated_key).to eq('sec' => 'C')
           end
         end
         context 'without a limit' do
           it 'retrieves the correct items' do
             response = SecBase.scan(index_name: 'sec')
             expect(response.items.map { |i| i.sec }).to eq(%w(A B C D))
-            expect(response.last_evaluated_key).to eq('D')
+            expect(response.last_evaluated_key).to be_nil
           end
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynamini
 version: !ruby/object:Gem::Version
-  version: 2.9.1
+  version: 2.9.2
 platform: ruby
 authors:
 - Greg Ward