dynamini 2.8.1 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8dc1f7436267794e86bd3ba925c347fe0dbf8536
-  data.tar.gz: 202c5d8166d5dbcd95b57a3474533e38f869e9c6
+  metadata.gz: e6bc03a0efcc9da7a30ed72044a0e7733cae9dd6
+  data.tar.gz: 3f0d131c9dec2a3f1e8af932a6daf6e513eb9b62
 SHA512:
-  metadata.gz: 028d80338df534f3560290e18108c1cafecbd97cc64f7b99e5920c5c05754334e2008efe8be13488d44b48e9eca5f63777136a5c9092a28ca5828a90d0647eda
-  data.tar.gz: 8b0fe74c4f36d33adf2ae8940407e357443b7d5f81c17f3c3a45f89ae10afd19a9aa696ce6037c621819d9e383a9cec428145b2bf84d44faa64aece87ada7702
+  metadata.gz: bc65042442a651365977bd108350e7ba3d516009060696f33834ffad426903fa86e01b99e5ae9627c3b62b0d3dddbda7ca5ec2a77ac3e35bd2aa5ba96dc6d966
+  data.tar.gz: de6f8de47956820198714c9887238fd4eef803eae0dcd42907b310c57a5a9dcdc59408625011472b2fbfc4128d1efcadb8e7883fa2b183d5e2ef44da1a281f10

data/Gemfile.lock

@@ -1,30 +1,29 @@
 PATH
   remote: .
   specs:
-    dynamini (2.8.1)
+    dynamini (2.9.0)
       activemodel (>= 3, < 5.0)
       aws-sdk (~> 2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (4.2.9)
-      activesupport (= 4.2.9)
+    activemodel (4.2.5.1)
+      activesupport (= 4.2.5.1)
       builder (~> 3.1)
-    activesupport (4.2.9)
+    activesupport (4.2.5.1)
       i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    aws-sdk (2.10.42)
-      aws-sdk-resources (= 2.10.42)
-    aws-sdk-core (2.10.42)
-      aws-sigv4 (~> 1.0)
+    aws-sdk (2.2.22)
+      aws-sdk-resources (= 2.2.22)
+    aws-sdk-core (2.2.22)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.10.42)
-      aws-sdk-core (= 2.10.42)
-    aws-sigv4 (1.0.2)
-    builder (3.2.3)
+    aws-sdk-resources (2.2.22)
+      aws-sdk-core (= 2.2.22)
+    builder (3.2.2)
     coderay (1.1.0)
     diff-lcs (1.2.5)
     ffi (1.9.10)
@@ -49,14 +48,15 @@ GEM
     guard-shell (0.7.1)
       guard (>= 2.0.0)
       guard-compat (~> 1.0)
-    i18n (0.8.6)
-    jmespath (1.3.1)
+    i18n (0.7.0)
+    jmespath (1.1.3)
+    json (1.8.3)
     listen (3.0.3)
       rb-fsevent (>= 0.9.3)
       rb-inotify (>= 0.9)
     lumberjack (1.0.9)
     method_source (0.8.2)
-    minitest (5.10.3)
+    minitest (5.8.4)
     nenv (0.2.0)
     notiffany (0.0.8)
       nenv (~> 0.1)
@@ -85,8 +85,8 @@ GEM
     shellany (0.0.1)
     slop (3.6.0)
     thor (0.19.1)
-    thread_safe (0.3.6)
-    tzinfo (1.2.3)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
       thread_safe (~> 0.1)
 
 PLATFORMS
@@ -99,6 +99,3 @@ DEPENDENCIES
   guard-shell
   pry (~> 0)
   rspec (~> 3)
-
-BUNDLED WITH
-   1.16.0.pre.2

data/README.md

@@ -156,6 +156,7 @@ Query takes the following arguments:
 * :end (optional)
 * :limit (optional)
 * :scan_index_forward (optional - set to false to sort by range key in desc order)
+* :index_name (to query a secondary index - see below)
 
 Here's how you'd use it to find daily temperature data for a given city, selecting for specific date ranges:
 
@@ -197,9 +198,64 @@ DailyWeather.query(hash_key: "Toronto", limit: 2)
 DailyWeather.query(hash_key: "Toronto", scan_index_forward: false)
 > [C, B, A]
 ```
+  
+## Table Scans
+Table scanning is a very expensive operation, and should not be undertaken without a good understanding of the read/write costs. As such, Dynamini doesn't implement the traditional ActiveRecord collection methods like .all or .where. Instead, you can .scan, which has an interface much closer to DynamoDB's native SDK method.
+
+The following options are supported:
+
+* consistent_read (default: false)
+* start_key (hash key of first desired item, default will scan from beginning)
+* index_name (if scanning a secondary index - see below)
+* limit
+
+These two options are to support paralellization of table scanning, see: http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Scan.html
+* segment
+* total_segments
+
+```ruby
+products_page_one = Product.scan(limit: 100)
+products_page_one.found # [product, product...]
+page_two = Product.scan(start_key: products_page_one.last_evaluated_key)
+```
+
+## Secondary Indices
+To define a secondary index (so that you can .scan it or .query it), you can list them at the top of your Dynamini subclass. The index names have to match the names you've set up through the DynamoDB console. If your secondary index uses a range key, specify it here as well.
+
+```ruby
+class Comment < Dynamini::Base
+    set_hash_key :id
+    set_range_key :comment_date
+    set_secondary_index :score_index
+    set_secondary_index :popularity_index, range_key: :popularity
+end
+```
+For more information on how and why to use secondary indices, see http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SecondaryIndexes.html
+## Testing
+We've included an optional in-memory test client, so you don't necessarily have to connect to a real Dynamo instance when running tests. You could also use this in your development environment if you don't have a real Dynamo instance yet, but the data saved to it won't persist through a server restart.
+
+To activate this feature, just require the testing module:
+```ruby
+require 'dynamini/testing'
+```
+This module replaces all API calls Dynamini makes to AWS DynamoDB with calls to Dynamini::TestClient.
+
+The test client will not reset its database unless you tell it to, like so:
+```ruby
+Vehicle.client.reset
+```
+
+So, for instance, to get Rspec working with your test suite the way your ActiveRecord model behaved, add these lines to your spec_helper.rb:
+```ruby
+require 'dynamini/testing'
+
+config.after(:each) {
+  Vehicle.client.reset # Large test suites will be very slow and unpredictable otherwise!
+}
+```
 
 ## Batch Saving
-Dynamo allows batch saving, see: http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
+Dynamini implements DynamoDB's batch write operation, mapped to the .import method you might be used to from ActiveRecord. http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
 
 ```ruby
 class Product < Dynamini::Base
@@ -229,30 +285,6 @@ Product.find('qwerty').updated_at
 
 ````
 
-
-## Testing
-We've included an optional in-memory test client, so you don't necessarily have to connect to a real Dynamo instance when running tests. You could also use this in your development environment if you don't have a real Dynamo instance yet, but the data saved to it won't persist through a server restart.
-
-To activate this feature, just require the testing module:
-```ruby
-require 'dynamini/testing'
-```
-This module replaces all API calls Dynamini makes to AWS DynamoDB with calls to Dynamini::TestClient.
-
-The test client will not reset its database unless you tell it to, like so:
-```ruby
-Vehicle.client.reset
-```
-
-So, for instance, to get Rspec working with your test suite the way your ActiveRecord model behaved, add these lines to your spec_helper.rb:
-```ruby
-require 'dynamini/testing'
-
-config.after(:each) {
-  Vehicle.client.reset # Large test suites will be very slow and unpredictable otherwise!
-}
-```
-
 ## Things to remember
 * Since DynamoDB is schemaless, your model will respond to any method that looks like a reader, meaning model.foo will return nil.
 * You can also write any arbitrary attribute to your model.

data/dynamini.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'dynamini'
-  s.version = '2.8.1'
+  s.version = '2.9.0'
   s.summary = 'DynamoDB interface'
   s.description = 'Lightweight DynamoDB interface gem designed as
                    a drop-in replacement for ActiveRecord.

data/lib/dynamini/batch_operations.rb

@@ -33,28 +33,64 @@ module Dynamini
       client.batch_write_item(options)
     end
 
+    def scan(options = {})
+      validate_scan_options(options)
+      response = dynamo_scan(options)
+      if options[:index_name]
+        last_evaluated_key = response.last_evaluated_key[secondary_index[options[:index_name]][:hash_key_name].to_s]
+      else
+        last_evaluated_key = response.last_evaluated_key[hash_key.to_s]
+      end
+      OpenStruct.new(
+        last_evaluated_key: last_evaluated_key,
+        items: response.items.map { |i| new(i.symbolize_keys, false) }
+      )
+    end
+
     private
 
     def dynamo_batch_get(key_struct)
       client.batch_get_item(
-          request_items: {
-              table_name => {keys: key_struct}
-          }
+        request_items: {
+          table_name => {keys: key_struct}
+        }
       )
     end
 
     def dynamo_batch_save(model_array)
       put_requests = model_array.map do |model|
         {
-            put_request: {
-                item: model.attributes.reject { |_k, v| v.blank? }.stringify_keys
-            }
+          put_request: {
+            item: model.attributes.reject { |_k, v| v.blank? }.stringify_keys
+          }
         }
       end
       request_options = {
-          request_items: {table_name => put_requests}
+        request_items: {table_name => put_requests}
       }
       client.batch_write_item(request_options)
     end
+
+    def dynamo_scan(options)
+      response = client.scan(
+        consistent_read:      options[:consistent_read],
+        exclusive_start_key:  options[:exclusive_start_key],
+        secondary_index_name: options[:index_name],
+        limit:                options[:limit],
+        segment:              options[:segment],
+        total_segments:       options[:total_segments],
+        table_name:           table_name
+      )
+    end
+
+    def validate_scan_options(options)
+      if options[:total_segments] && !options[:segment]
+        raise ArgumentError, 'Must specify segment if specifying total_segments'
+      elsif options[:segment] && !options[:total_segments]
+        raise ArgumentError, 'Must specify total_segments if specifying segment'
+      elsif options[:index_name] && !self.secondary_index[options[:index_name]]
+        raise ArgumentError, "Secondary index of #{options[:index_name]} does not exist"
+      end
+    end
   end
 end

data/lib/dynamini/test_client.rb

@@ -84,6 +84,56 @@ module Dynamini
       OpenStruct.new(responses: responses)
     end
 
+    def scan(args = {})
+      records = get_table(args[:table_name]).values
+
+      sort_scanned_records!(records, args[:secondary_index_name]) if args[:secondary_index_name]
+      start_index = index_of_start_key(args, records)
+      items = limit_scanned_records(args[:limit], records, start_index)
+      last_evaluated_key = get_last_evaluated_key(args[:secondary_index_name], items)
+      OpenStruct.new(items: items, last_evaluated_key: last_evaluated_key)
+    end
+
+    def sort_scanned_records!(records, secondary_index_name)
+      index = secondary_index[secondary_index_name]
+      records.sort! do |a, b|
+        a[get_secondary_hash_key(index)] <=> b[get_secondary_hash_key(index)]
+      end
+    end
+
+    def index_of_start_key(args, records)
+      if args[:exclusive_start_key]
+        sec_index = secondary_index[args[:secondary_index_name]]
+        start_index = records.index do |r|
+          if sec_index
+            r[get_secondary_hash_key(sec_index)] == args[:exclusive_start_key]
+          else
+            r[hash_key_attr] == args[:exclusive_start_key]
+          end
+        end
+        start_index || -1
+      else
+        0
+      end
+    end
+
+    def limit_scanned_records(limit, records, start_index)
+      end_index = limit ? start_index + limit - 1 : -1
+      records[start_index..end_index]
+    end
+
+    def get_last_evaluated_key(secondary_index_name, items)
+      # TODO should last evaluated key be present if the args[:limit] was reached?
+      # if items.length > records.length
+      index = secondary_index[secondary_index_name]
+      if index
+        { get_secondary_hash_key(index).to_s => items.last[get_secondary_hash_key(index)] }
+      else
+        { hash_key_attr.to_s => items.last[hash_key_attr] }
+      end
+      # end
+    end
+
     # TODO add range key support for delete, not currently implemented batch_operations.batch_delete
     def batch_write_item(request_options)
       request_options[:request_items].each do |table_name, requests|

data/spec/dynamini/batch_operations_spec.rb

@@ -126,5 +126,90 @@ describe Dynamini::BatchOperations do
       expect{ Dynamini::Base.find('7890') }.to raise_error(Dynamini::RecordNotFound)
     end
   end
+
+  class SecBase < Dynamini::Base
+    set_hash_key :id
+    set_secondary_index :sec, hash_key: :sec
+    set_secondary_index :rev, hash_key: :sec
+  end
+
+  describe '.scan' do
+
+    before do
+      SecBase.create(id: '123', sec: 'D')
+      SecBase.create(id: '124', sec: 'C')
+      SecBase.create(id: '125', sec: 'B')
+      SecBase.create(id: '126', sec: 'A')
+    end
+
+    context 'scanning the primary key' do
+      context 'with an exclusive_start_key' do
+        context 'with a limit' do
+          it 'retrieves the correct items' do
+            response = SecBase.scan(exclusive_start_key: '124', limit: 1)
+            expect(response.items.map { |i| i.id }).to eq(['124'])
+            expect(response.last_evaluated_key).to eq('124')
+          end
+        end
+        context 'without a limit' do
+          it 'retrieves the correct items' do
+            response = SecBase.scan(exclusive_start_key: '124')
+            expect(response.items.map { |i| i.id }).to eq(%w(124 125 126))
+            expect(response.last_evaluated_key).to eq('126')
+          end
+        end
+      end
+      context 'without an exclusive_start_key' do
+        context 'with a limit' do
+          it 'retrieves the correct items' do
+            response = SecBase.scan(limit: 2)
+            expect(response.items.map { |i| i.id }).to eq(%w(123 124))
+            expect(response.last_evaluated_key).to eq('124')
+          end
+        end
+        context 'without a limit' do
+          it 'retrieves the correct items' do
+            response = SecBase.scan
+            expect(response.items.map { |i| i.id }).to eq(%w(123 124 125 126))
+            expect(response.last_evaluated_key).to eq('126')
+          end
+        end
+      end
+    end
+    context 'scanning a secondary index' do
+      context 'with an exclusive_start_key' do
+        context 'with a limit' do
+          it 'retrieves the correct items' do
+            response = SecBase.scan(index_name: 'sec', exclusive_start_key: 'B', limit: 2)
+            expect(response.items.map { |i| i.sec }).to eq(%w(B C))
+            expect(response.last_evaluated_key).to eq('C')
+          end
+        end
+        context 'without a limit' do
+          it 'retrieves the correct items' do
+            response = SecBase.scan(index_name: 'sec', exclusive_start_key: 'B')
+            expect(response.items.map { |i| i.sec }).to eq(%w(B C D))
+            expect(response.last_evaluated_key).to eq('D')
+          end
+        end
+      end
+      context 'without an exclusive_start_key' do
+        context 'with a limit' do
+          it 'retrieves the correct items' do
+            response = SecBase.scan(index_name: 'sec', limit: 3)
+            expect(response.items.map { |i| i.sec }).to eq(%w(A B C))
+            expect(response.last_evaluated_key).to eq('C')
+          end
+        end
+        context 'without a limit' do
+          it 'retrieves the correct items' do
+            response = SecBase.scan(index_name: 'sec')
+            expect(response.items.map { |i| i.sec }).to eq(%w(A B C D))
+            expect(response.last_evaluated_key).to eq('D')
+          end
+        end
+      end
+    end
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynamini
 version: !ruby/object:Gem::Version
-  version: 2.8.1
+  version: 2.9.0
 platform: ruby
 authors:
 - Greg Ward
@@ -15,7 +15,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-11 00:00:00.000000000 Z
+date: 2017-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel