dynamini 2.12.1 → 2.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 295d5d3e9ac24824561c2d3ea60328036a2a1ef08cb442c75dc07f07110f05dc
-  data.tar.gz: dc6a149f087e727efed3d20952468639a3ea0b82f3e3acc9d954d9a0a8307153
+  metadata.gz: 7c5532b785f0dddd894eb863f17905aab853a124fb3aeb6b046e67b3f0574b2a
+  data.tar.gz: 6b897e7501ec92aab731f94dbf93ee830ed4e0b615103e4b268c38f2796fbb05
 SHA512:
-  metadata.gz: 70172147217e1531d2be9a15973babb8cbd77361eff4025e39d816e0591e3c4e25278a6b73431b89a4bd5acdc7415f28ed40a9c423b1f0b36ba980151baca358
-  data.tar.gz: 3e8383facfa971c03a77e22ad8399cf5f5d024a88cc65bd67be213eaf0a967a315f5a4a34908c46afd44e1d4835de5d5a130e8d0950ec12282c6e67da206af8b
+  metadata.gz: 11dd2f65ba75d324367f9493fa8a0f2bdfca46526d2647cc738e3c7661303ada53e1aac4409770b393c837a90a892988296e20645374fff4d9d87ad513ac701c
+  data.tar.gz: fdb2715ba459b388db428af4b46a2cb11281fd6efd8a3471f3e9ce13c7c0893ed5424a6b422c20797eec8c57ba01fbf06284edd634abd311b587db4e78b05ab8

data/README.md

@@ -259,14 +259,14 @@ page_two = Product.scan(start_key: products_page_one.last_evaluated_key)
 ```
 
 ## Secondary Indices
-To define a secondary index (so that you can .scan or .query it), you can set them at the top of your Dynamini subclass. The index names have to match the names you've set up through the DynamoDB console. If your secondary index uses a range key, specify it here as well.
+To define a secondary index (so that you can .scan or .query it), you can set them at the top of your Dynamini subclass. The index names have to match the names you've set up through the DynamoDB console. You'll need to specify which attribute your index is keyed to, and if your secondary index uses a range key, specify it here as well.
 
 ```ruby
 class Comment < Dynamini::Base
     set_hash_key :id
-    set_range_key :comment_date
-    set_secondary_index :score_index
-    set_secondary_index :popularity_index, range_key: :popularity
+    set_range_key :comment_date # filter comments by date
+    set_secondary_index :score_index, hash_key: :score  # lookup comments by score
+    set_secondary_index :user_index, hash_key: :user_id, range_key: :comment_date # lookup comments by user, filtering by date
 end
 ```
 For more information on how and why to use secondary indices, see http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SecondaryIndexes.html

data/dynamini.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'dynamini'
-  s.version = '2.12.1'
+  s.version = '2.12.2'
   s.summary = 'DynamoDB interface'
   s.description = 'Lightweight DynamoDB interface gem designed as
                    a drop-in replacement for ActiveRecord.

data/lib/dynamini.rb

@@ -4,6 +4,7 @@ module Dynamini
   require 'dynamini/base'
   require 'dynamini/configuration'
   require 'dynamini/test_client'
+  require 'dynamini/item_splitter'
 
   class << self
     attr_writer :configuration

data/lib/dynamini/base.rb

@@ -130,7 +130,10 @@ module Dynamini
 
     def trigger_save(options = {})
       generate_timestamps! unless options[:skip_timestamps]
-      save_to_dynamo
+      updates = ItemSplitter.split(attribute_updates)
+      updates.each do |u|
+        save_to_dynamo(u)
+      end
       clear_changes
       @new_record = false
       true

data/lib/dynamini/client_interface.rb

@@ -10,7 +10,7 @@ module Dynamini
       end
     end
 
-    def save_to_dynamo
+    def save_to_dynamo(attribute_updates)
       self.class.client.update_item(
           table_name: self.class.table_name,
           key: key,

data/lib/dynamini/item_splitter.rb

@@ -0,0 +1,63 @@
+module Dynamini
+  class ItemSplitter
+
+    MAX_SIZE = 380_000
+
+    class << self
+
+      def split(attribute_updates)
+        unprocessed_au = attribute_updates.map { |k, v| {k => v} }
+        updates = []
+        current_update_size = 0
+        current_update = {}
+
+        while unprocessed_au.length > 0 do
+          size = au_size(unprocessed_au[0])
+          if size > MAX_SIZE
+            part_one, part_two = split_au(unprocessed_au[0])
+            unprocessed_au.shift
+            unprocessed_au.unshift(part_two)
+            unprocessed_au.unshift(part_one)
+          else
+            current_update_size += size
+            if current_update_size > MAX_SIZE
+              updates.push(current_update)
+              current_update_size = 0
+              current_update = {}
+            else
+              current_update_size += size
+              key, value = key_and_value(unprocessed_au[0])
+              current_update[key] = value
+              unprocessed_au.shift
+            end
+          end
+        end
+
+        updates.push(current_update) unless current_update.empty?
+        updates
+      end
+
+      private
+
+      def au_size(au)
+        au.to_s.bytesize
+      end
+
+      def split_au(au)
+        attribute_name = au.keys[0]
+        attribute_action = au.values[0][:action]
+        attribute_value = au.values[0][:value]
+
+        raise "#{attribute_name} is too large to save and is not splittable (not enumerable)." unless attribute_value.is_a?(Enumerable)
+
+        part_one = {attribute_name => {action: attribute_action, value: attribute_value[0..(attribute_value.length / 2) - 1]}}
+        part_two = {attribute_name => {action: "ADD", value: attribute_value[attribute_value.length / 2..-1]}}
+        [part_one, part_two]
+      end
+
+      def key_and_value(au)
+        [au.keys[0], au.values[0]]
+      end
+    end
+  end
+end

data/spec/dynamini/item_splitter_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe Dynamini::ItemSplitter do
+  before do
+    stub_const("Dynamini::ItemSplitter::MAX_SIZE", 200)
+  end
+
+  context 'regular update' do
+    it 'returns one update' do
+      input = {foo: {action: 'PUT', value: '123'}, bar: {action: 'ADD', value: [4,5,6]}}
+      expect(Dynamini::ItemSplitter.split(input)).to eq([input])
+    end
+  end
+
+  context 'large update' do
+    it 'returns multiple updates' do
+      input = {foo: {action: 'PUT', value: '123'}, bar: {action: 'ADD', value: [4,5,6]}, baz: {action: 'PUT', value: 'hello'}}
+      output = Dynamini::ItemSplitter.split(input)
+      expect(output[0]).to eq({foo: {action: 'PUT', value: '123'}, bar: {action: 'ADD', value: [4,5,6]}})
+      expect(output[1]).to eq(baz: {action: 'PUT', value: 'hello'})
+    end
+  end
+
+  context 'large enumerable attribute' do
+    it 'splits the attribute' do
+      input = {enum: {action: 'PUT', value: Array.new(20, 'hello')}}
+      output = Dynamini::ItemSplitter.split(input)
+      expect(output.length).to eq(2)
+      expect(output[0][:enum][:action]).to eq('PUT')
+      expect(output[1][:enum][:action]).to eq('ADD')
+    end
+  end
+
+  context 'large non-enumerable attribute' do
+    it 'raises an error' do
+      input = {not_enum: {action: 'PUT', value: Array.new(20, 'hello').to_s}}
+      expect{ Dynamini::ItemSplitter.split(input) }.to raise_error("not_enum is too large to save and is not splittable (not enumerable).")
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynamini
 version: !ruby/object:Gem::Version
-  version: 2.12.1
+  version: 2.12.2
 platform: ruby
 authors:
 - Greg Ward
@@ -15,7 +15,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-26 00:00:00.000000000 Z
+date: 2018-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -150,6 +150,7 @@ files:
 - lib/dynamini/errors.rb
 - lib/dynamini/global_id.rb
 - lib/dynamini/increment.rb
+- lib/dynamini/item_splitter.rb
 - lib/dynamini/querying.rb
 - lib/dynamini/test_client.rb
 - lib/dynamini/testing.rb
@@ -160,6 +161,7 @@ files:
 - spec/dynamini/dirty_spec.rb
 - spec/dynamini/global_id_spec.rb
 - spec/dynamini/increment_spec.rb
+- spec/dynamini/item_splitter_spec.rb
 - spec/dynamini/querying_spec.rb
 - spec/dynamini/test_client_spec.rb
 - spec/dynamini/type_handler_spec.rb
@@ -195,6 +197,7 @@ test_files:
 - spec/dynamini/dirty_spec.rb
 - spec/dynamini/global_id_spec.rb
 - spec/dynamini/increment_spec.rb
+- spec/dynamini/item_splitter_spec.rb
 - spec/dynamini/querying_spec.rb
 - spec/dynamini/test_client_spec.rb
 - spec/dynamini/type_handler_spec.rb