dyna_model 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca4e3bd3b871756a2b5fad4bf362c334bbf804af
-  data.tar.gz: ac13badc8bca36d88787d6cbdb6134af19c69a93
+  metadata.gz: 254b9c0ebdd7a60601af697b4702fa53d9dc3f0e
+  data.tar.gz: 9129d092e187f9284e1853d63e744ffa56e9d84a
 SHA512:
-  metadata.gz: 23243b46398ab0517a2f6a7c57e1fdbbe3f4f63c8d55b8c1a8a73abe8de75a297395ad8554d06101c02cb472de16a696e738a509b46002427812b4efa6279e49
-  data.tar.gz: 32f20b9e28c107825230eb53dfd18b092f6626e91e66a4b1fd859b0523c4f5c38c1566d2e77e9417ec80b8612c4c91ba24269f47a74379eecc1611ee37b0b38a
+  metadata.gz: 3dd8cca0fe4249f173fceb88d4a29b706409de9a017049844493a6066ef67f2238548bddd18b9278d5ba390eeeb0910d82c103ad4db3ffe4e7171903433d164f
+  data.tar.gz: 9c27a58d8bb188a5a29bf49c4ab6163ae9d3f3525d85e4d6a16f6466a586c31aed0e744a36ba7de1ff31b37f4d0aefd0a50cf95ca1bbb07cce6122d5defd1ab7

data/README.md

@@ -76,6 +76,16 @@ rake ddb:destroy CLASS=Dude
 rake ddb:destroy CLASS=all
 ```
 
+## Elasticsearch::Model compatible adapter
+```
+require 'dyna_model/adapters/elasticsearch/dyna_model_adapter'
+class Item
+  include DynaModel::Document
+  include Elasticsearch::Model
+  include Elasticsearch::Model::Callbacks
+end
+```
+
 # AWS::Record
 * http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/Record.html
 * https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/record/abstract_base.rb

data/lib/dyna_model/adapters/elasticsearch/dyna_model_adapter.rb

@@ -0,0 +1,86 @@
+module Elasticsearch
+  module Model
+    module Adapter
+      module DynaModelAdapter
+
+        Adapter.register self, lambda { |klass| !!defined?(::DynaModel::Document) && klass.ancestors.include?(::DynaModel::Document) }
+
+        module Records
+
+          def records
+            records_arr = klass.read_multiple(ids, format: :array)
+            records_arr.sort_by { |e| response.response['hits']['hits'].index { |hit| hit['_id'].to_s == e.id.to_s } }
+          end
+
+          # Intercept call to sorting methods, so we can ignore the order from Elasticsearch
+          %w| asc desc order_by |.each do |name|
+            define_method name do |*args|
+              raise "TODO - not supported yet"
+            end
+          end
+        end
+
+        module Callbacks
+          def self.included(base)
+            base.after_create  { |document| document.__elasticsearch__.index_document  }
+            base.after_update  { |document| document.__elasticsearch__.update_document }
+            base.after_destroy { |document| document.__elasticsearch__.delete_document }
+          end
+        end
+
+        module Importing
+
+          def __find_in_batches(options={}, &block)
+            items = []
+
+            # Use 1/4 or read provision
+            read_provision = self.dynamo_db_table.table_schema[:provisioned_throughput][:read_capacity_units]
+            raise "read_provision not set for class!" unless read_provision
+            default_batch_size = (read_provision / 2.0).floor
+            batch_size = options[:batch_size] || default_batch_size
+            puts "Indexing via scan with batch size of #{batch_size}..."
+
+            # :consumed_capacity
+            scan_idx = 0
+            results_hash = {}
+            while scan_idx == 0 || (results_hash && results_hash[:last_evaluated_key])
+              puts "Batch iteration #{scan_idx+1}..."
+              scan_options = {
+                batch: batch_size,
+                manual_batching: true,
+                return_consumed_capacity: :total
+              }
+              scan_options.merge!(exclusive_start_key: results_hash[:last_evaluated_key]) if results_hash[:last_evaluated_key]
+              scan_options.merge!(scan_filter: options[:scan_filter]) if options[:scan_filter]
+              results_hash = self.scan(scan_options)
+
+              unless results_hash[:results].blank?
+                puts "Indexing #{results_hash[:results].size} results..."
+                batch_for_bulk = results_hash[:results].map { |a| { index: {
+                  _id: a.id,
+                  data: a.__elasticsearch__.as_indexed_json
+                } } }
+                yield batch_for_bulk
+              end
+
+              # If more results to scan, sleep to throttle...
+              #   Local Dynamo is not returning consumed_capacity 2014-01-12
+              if results_hash[:last_evaluated_key] && results_hash[:consumed_capacity]
+                # try to keep read usage under 50% of read_provision
+                sleep_time = results_hash[:consumed_capacity][:capacity_units].to_f / (read_provision / 2.0)
+                puts "Sleeping for #{sleep_time}..."
+                sleep(sleep_time)
+              end
+              
+              scan_idx += 1
+            end
+
+          end
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/dyna_model/document.rb

@@ -34,6 +34,17 @@ module DynaModel
         end
         super
       end
+
+      # OVERRIDE
+      # https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/record/abstract_base.rb#L76
+      # AWS::Record::AbstractBase for :select attributes
+      public
+      def attributes
+        attributes = AWS::Core::IndifferentHash.new
+        (self.instance_variable_get("@_selected_attributes") || self.class.attributes.keys).inject(attributes) do |hash,attr_name|
+          hash.merge(attr_name => __send__(attr_name))
+        end
+      end
     end
 
     include ActiveModel::Conversion
@@ -47,6 +58,10 @@ module DynaModel
     include DynaModel::Schema
     include DynaModel::Query
 
+    def id
+      self.dynamo_db_guid
+    end
+
     def to_param
       self.dynamo_db_guid
     end
@@ -64,7 +79,7 @@ module DynaModel
     end
 
     def all_attributes_loaded?
-      self.instance_variable_get("@_select") == :all
+      self.instance_variable_get("@_select").nil? && self.instance_variable_get("@_select") == :all
     end
 
     # When only partial attributes were selected (via GSI or projected attributes on an index)

data/lib/dyna_model/query.rb

@@ -30,18 +30,26 @@ module DynaModel
       end
 
       def read_multiple(keys, options={})
+        options[:format] = (options[:format] && options[:format] == :array) ? :array : :hash
         results_map = {}
-        results = self.dynamo_db_table.batch_get_item(keys, options)
-        results[:responses][self.dynamo_db_table_name(options[:shard_name])].each do |result|
-          attrs = Response.strip_attr_types(result)
-          obj = self.obj_from_attrs(attrs, options)
-          if self.dynamo_db_table.range_keys.present? && primary_range_key = self.dynamo_db_table.range_keys.find{|rk| rk[:primary_range_key] }
-            (results_map[attrs[self.dynamo_db_table.hash_key[:attribute_name]]] ||= {})[attrs[primary_range_key[:attribute_name]]] = obj
-          else
-            results_map[attrs[self.dynamo_db_table.hash_key[:attribute_name]]] = obj
+        results_arr = []
+        if keys.present?
+          results = self.dynamo_db_table.batch_get_item(keys, options)
+          results[:responses][self.dynamo_db_table_name(options[:shard_name])].each do |result|
+            attrs = Response.strip_attr_types(result)
+            obj = self.obj_from_attrs(attrs, options)
+            if options[:format] == :array
+              results_arr << obj
+            else
+              if self.dynamo_db_table.range_keys.present? && primary_range_key = self.dynamo_db_table.range_keys.find{|rk| rk[:primary_range_key] }
+                (results_map[attrs[self.dynamo_db_table.hash_key[:attribute_name]]] ||= {})[attrs[primary_range_key[:attribute_name]]] = obj
+              else
+                results_map[attrs[self.dynamo_db_table.hash_key[:attribute_name]]] = obj
+              end
+            end
           end
         end
-        results_map
+        options[:format] == :array ? results_arr : results_map
       end
 
       # Read results up to the limit

data/lib/dyna_model/table.rb

@@ -281,16 +281,13 @@ module DynaModel
       keys.each do |k|
         key_request = {}
         if @primary_range_key
-          hash_value = k[:hash_value]
+          hash_value, range_value = k.split(@model.guid_delimiter)
         else
-          raise ArgumentError, "expected keys to be in the form of ['hash key here'] for table with no range keys" if hash_value.is_a?(Hash)
           hash_value = k
         end
-        raise ArgumentError, "every key must include a :hash_value" if hash_value.blank?
         key_request[@hash_key[:attribute_name]] = { @hash_key[:attribute_type] => hash_value.to_s }
         if @primary_range_key
-          range_value = k[:range_value]
-          raise ArgumentError, "every key must include a :range_value" if range_value.blank?
+          raise ArgumentError, "every key must include a range_value" if range_value.blank?
           key_request[@primary_range_key[:attribute_name]] = { @primary_range_key[:attribute_type] => range_value.to_s }
         end
         keys_request << key_request

data/lib/dyna_model/version.rb

@@ -1,3 +1,3 @@
 module DynaModel
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/dyna_model/query_spec.rb

@@ -99,7 +99,7 @@ describe "DynaModel::Query" do
     multi[@cacher2.key].should_not be_nil
     @user = User.create(@user_attrs)
     @user2 = User.create(@user2_attrs)
-    multi = User.read_multiple([{hash_value: @user.hashy, range_value: @user.ranger}, {hash_value: @user2.hashy, range_value: @user2.ranger}])
+    multi = User.read_multiple([@user.dynamo_db_guid, @user2.dynamo_db_guid])
     multi[@user.hashy].should_not be_nil
     multi[@user.hashy][@user.ranger.to_s].should_not be_nil
     multi[@user2.hashy].should_not be_nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dyna_model
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Cary Dunn
@@ -136,6 +136,7 @@ files:
 - Rakefile
 - dyna_model.gemspec
 - lib/dyna_model.rb
+- lib/dyna_model/adapters/elasticsearch/dyna_model_adapter.rb
 - lib/dyna_model/attributes.rb
 - lib/dyna_model/aws/record/attributes/serialized_attr.rb
 - lib/dyna_model/config.rb