identity_cache 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3aaaad2b3f7ef8cd9c4aaeb338756bfcccfcbd4c
-  data.tar.gz: 06ee79bbcc777f7903139d9eab772ddcd50ad1af
+  metadata.gz: 953d802de951edde24a15f5d1c073bebdaebe477
+  data.tar.gz: 8a329ec724a1b9a9dba8435a47de9df84c7e8715
 SHA512:
-  metadata.gz: 0678a544ee6f53d428517c5a44409a1f93a9e690002f6c8d3efe1ee731f50aed2eef3c68b7bba5b66f4500b0fd5bdef9fc09e990a1e9a5ece1d0fac8e496ada6
-  data.tar.gz: c1313137f940b3e7c22523b9bf331416f4824288d4e45617931042641e73aabef0bc1d659384faf6be904d503164f80748c783fb5c3d216c72b1af0a6d40b2ba
+  metadata.gz: 89fe603c343ad1fcd8c92d1bd12a7375a52b82a12c4c016256e1940ddbd0a1d66d1248b033640b661b8d30f21020cc29b5b5600a9a36591b5e03cac9238d0ca9
+  data.tar.gz: 252d8f5066cc625fb799d490c3a4e5736283e82adf8817179c35697dbbb76ac07d28feeda32ad6e1d84db2440f820277578f3b9ea253997d64f3b1735e5fa74c

data/.travis.yml

@@ -5,9 +5,15 @@ rvm:
   - 2.2.3
 
 gemfile:
-  - Gemfile.rails40
-  - Gemfile.rails41
   - Gemfile.rails42
+  - Gemfile.rails50
+
+matrix:
+  exclude:
+    - rvm: 2.1
+      gemfile: Gemfile.rails50
+    - gemfile: Gemfile.rails50
+      env: DB=postgresql
 
 env:
   - DB=mysql2

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # IdentityCache changelog
 
+#### 0.3.2
+
+- Deprecate returning non read-only records when cache is used. Set IdentityCache.fetch_readonly_records to true to avoid this. (#282)
+- Use loaded association first when fetching a cache_has_many id embedded association (#280)
+- Deprecate setting the inverse active record association on cache hits. Set IdentityCache.never_set_inverse_association to true to avoid this. (#279)
+- Fetch association returns relation or array depending on the configuration. It was only returning a relation for cache_has_many fetch association methods. (#276)
+- Stop sharing the same attributes hash between the fetched record and the memoized cache, which could interfere with dirty tracking (#267)
+
 #### 0.3.1
 
 - Fix cache_index for non-id primary key

data/Gemfile.rails50

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+gemspec
+
+gem 'activerecord', '~> 5.0.0'
+gem 'activesupport', '~> 5.0.0'
+gem 'mysql2', '>= 0.3.18', '< 0.5'

data/README.md

@@ -101,6 +101,16 @@ end
 @product.fetch_images
 ```
 
+To read multiple records in batch use `fetch_multi`.
+
+``` ruby
+class Product < ActiveRecord::Base
+  include IdentityCache
+end
+
+@product.fetch_multi([1, 2])
+```
+
 ### Embedding Associations
 
 IdentityCache can easily embed objects into the parents' cache entry. This means loading the parent object will also load the association and add it to the cache along with the parent. Subsequent cache requests will load the parent along with the association in one fetch. This can again mean some duplication in the cache if you want to be able to cache objects on their own as well, so it should be done with care. This works with both `cache_has_many` and `cache_has_one` methods.

data/dev.yml

@@ -0,0 +1,54 @@
+name: identity-cache
+
+up:
+  - homebrew:
+    - postgresql
+  - ruby:
+      version: 2.2.3p172-shopify
+      package: shopify/shopify/shopify-ruby
+  - railgun
+  - bundler
+
+env:
+  RAILGUN_HOST: identity-cache.railgun
+  MYSQL_HOST: identity-cache.railgun
+  POSTGRES_HOST: identity-cache.railgun
+  MEMCACHED_HOST: identity-cache.railgun
+
+commands:
+  test:
+    syntax:
+      optional:
+        argument: file
+        optional: args...
+    desc: 'Run tests'
+    run: |
+      if [[ $# -eq 0 ]]; then
+        bundle exec rake test
+      else
+        bundle exec ruby -I test "$@"
+      fi
+
+  benchmark-cpu:
+    desc: 'Run the identity cache CPU benchmark'
+    run: bundle exec rake benchmark:cpu
+
+  profile:
+    desc: 'Profile IDC code'
+    run: bundle exec rake profile:run
+
+  update-serialization-format:
+    desc: 'Update serialization format test fixture'
+    run: bundle exec rake update_serialization_format
+
+railgun:
+  image: dev:railgun-common-services-0.2.x
+  ip_address: 192.168.64.98
+  memory: 1G
+  cores: 1
+  disk: 1G
+  services:
+    mysql: 3306
+    postgresql: 5432
+    memcached: 11211
+

data/identity_cache.gemspec

@@ -2,8 +2,8 @@
 require File.expand_path('../lib/identity_cache/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Camilo Lopez", "Tom Burns", "Harry Brundage", "Dylan Smith", "Tobias Lutke", "Arthur Neves", "Francis Bogsanyi"]
-  gem.email         = ["harry.brundage@shopify.com"]
+  gem.authors       = ["Camilo Lopez", "Tom Burns", "Harry Brundage", "Dylan Thacker-Smith", "Tobias Lutke", "Arthur Neves", "Francis Bogsanyi"]
+  gem.email         = ["gems@shopify.com"]
   gem.description   = %q{Opt in read through ActiveRecord caching.}
   gem.summary       = %q{IdentityCache lets you specify how you want to cache your model objects, at the model level, and adds a number of convenience methods for accessing those objects through the cache. Memcached is used as the backend cache store, and the database is only hit when a copy of the object cannot be found in Memcached.}
   gem.homepage      = "https://github.com/Shopify/identity_cache"
@@ -16,9 +16,9 @@ Gem::Specification.new do |gem|
   gem.version       = IdentityCache::VERSION
 
   gem.add_dependency('ar_transaction_changes', '~> 1.0')
-  gem.add_dependency('activerecord', '>= 4.0.4')
-  gem.add_development_dependency('memcached', '~> 1.8.0')
+  gem.add_dependency('activerecord', '>= 4.2.0')
 
+  gem.add_development_dependency('memcached', '~> 1.8.0')
   gem.add_development_dependency('memcached_store', '~> 0.12.6')
   gem.add_development_dependency('rake')
   gem.add_development_dependency('mocha', '0.14.0')

data/lib/identity_cache/belongs_to_caching.rb

@@ -41,7 +41,11 @@ module IdentityCache
                 @#{options[:records_variable_name]} = association(:#{association}).klass.fetch_by_id(#{foreign_key})
               end
             else
-              #{association}
+              if IdentityCache.fetch_read_only_records && IdentityCache.should_use_cache?
+                load_and_readonlyify(:#{association})
+              else
+                #{association}
+              end
             end
           end
 

data/lib/identity_cache/cache_key_generation.rb

@@ -11,6 +11,7 @@ module IdentityCache
       schema_string = schema_to_string(klass.columns)
       if klass.include?(IdentityCache)
         klass.send(:all_cached_associations).sort.each do |name, options|
+          klass.send(:check_association_scope, name)
           case options[:embed]
           when true
             schema_string << ",#{name}:(#{denormalized_schema_hash(options[:association_reflection].klass)})"

data/lib/identity_cache/configuration_dsl.rb

@@ -205,17 +205,14 @@ module IdentityCache
         options[:cached_accessor_name]   = "fetch_#{association}"
         options[:records_variable_name]  = "cached_#{association}"
 
+        self.class_eval(<<-CODE, __FILE__, __LINE__ + 1)
+          def #{options[:cached_accessor_name]}
+            fetch_recursively_cached_association('#{options[:records_variable_name]}', :#{association})
+          end
+        CODE
 
-        unless instance_methods.include?(options[:cached_accessor_name].to_sym)
-          self.class_eval(<<-CODE, __FILE__, __LINE__ + 1)
-            def #{options[:cached_accessor_name]}
-              fetch_recursively_cached_association('#{options[:records_variable_name]}', :#{association})
-            end
-          CODE
-
-          options[:only_on_foreign_key_change] = false
-          add_parent_expiry_hook(options)
-        end
+        options[:only_on_foreign_key_change] = false
+        add_parent_expiry_hook(options)
       end
 
       def build_id_embedded_has_many_cache(association, options) #:nodoc:
@@ -236,10 +233,14 @@ module IdentityCache
           end
 
           def #{options[:cached_accessor_name]}
-            if IdentityCache.should_use_cache? || #{association}.loaded?
+            if IdentityCache.should_use_cache? && !#{association}.loaded?
               @#{options[:records_variable_name]} ||= #{options[:association_reflection].klass}.fetch_multi(#{options[:cached_ids_name]})
             else
-              #{association}
+              if IdentityCache.fetch_returns_relation
+                #{association}
+              else
+                #{association}.to_a
+              end
             end
           end
 

data/lib/identity_cache/parent_model_expiration.rb

@@ -46,7 +46,7 @@ module IdentityCache
             klass = parent_association.klass
           end
           old_parent = klass.find(transaction_changed_attributes[foreign_key])
-        rescue ActiveRecord::RecordNotFound => e
+        rescue ActiveRecord::RecordNotFound
           # suppress errors finding the old parent if its been destroyed since it will have expired itself in that case
         end
       end

data/lib/identity_cache/query_api.rb

@@ -26,7 +26,9 @@ module IdentityCache
             object = nil
             coder = IdentityCache.fetch(rails_cache_key(id)){ coder_from_record(object = resolve_cache_miss(id)) }
             object ||= record_from_coder(coder)
-            IdentityCache.logger.error "[IDC id mismatch] fetch_by_id_requested=#{id} fetch_by_id_got=#{object.id} for #{object.inspect[(0..100)]} " if object && object.id != id.to_i
+            if object && object.id.to_s != id.to_s
+              IdentityCache.logger.error "[IDC id mismatch] fetch_by_id_requested=#{id} fetch_by_id_got=#{object.id} for #{object.inspect[(0..100)]}"
+            end
             object
           end
         else
@@ -74,6 +76,38 @@ module IdentityCache
         records
       end
 
+      def prefetch_associations(associations, records)
+        records = records.to_a
+        return if records.empty?
+        unless IdentityCache.should_use_cache?
+          ActiveRecord::Associations::Preloader.new.preload(records, associations)
+          return
+        end
+
+        case associations
+        when nil
+          # do nothing
+        when Symbol
+          prefetch_one_association(associations, records)
+        when Array
+          associations.each do |association|
+            prefetch_associations(association, records)
+          end
+        when Hash
+          associations.each do |association, sub_associations|
+            next_level_records = prefetch_one_association(association, records)
+
+            associated_class = reflect_on_association(association).klass
+            if associated_class.respond_to?(:prefetch_associations)
+              associated_class.prefetch_associations(sub_associations, next_level_records)
+            end
+          end
+        else
+          raise TypeError, "Invalid associations class #{associations.class}"
+        end
+        nil
+      end
+
       private
 
       def raise_if_scoped
@@ -82,6 +116,14 @@ module IdentityCache
         end
       end
 
+      def check_association_scope(association_name)
+        association_reflection = reflect_on_association(association_name)
+        scope = association_reflection.scope
+        if scope && !association_reflection.klass.all.instance_exec(&scope).joins_values.empty?
+          raise UnsupportedAssociationError, "caching association #{self}.#{association_name} scoped with a join isn't supported"
+        end
+      end
+
       def record_from_coder(coder) #:nodoc:
         if coder
           klass = coder[:class]
@@ -89,28 +131,47 @@ module IdentityCache
 
           coder[:associations].each {|name, value| set_embedded_association(record, name, value) } if coder.has_key?(:associations)
           coder[:association_ids].each {|name, ids| record.instance_variable_set(:"@#{record.class.cached_has_manys[name][:ids_variable_name]}", ids) } if coder.has_key?(:association_ids)
+          record.readonly! if IdentityCache.fetch_read_only_records
           record
         end
       end
 
+      def set_inverse_of_cached_has_many(record, association_reflection, child_records)
+        associated_class = association_reflection.klass
+        return unless associated_class < IdentityCache
+
+        inverse_name = record.class.cached_has_manys.fetch(association_reflection.name).fetch(:inverse_name)
+        inverse_cached_association = associated_class.cached_belongs_tos[inverse_name]
+        return unless inverse_cached_association
+
+        prepopulate_method_name = inverse_cached_association.fetch(:prepopulate_method_name)
+        child_records.each { |child_record| child_record.send(prepopulate_method_name, record) }
+      end
+
       def set_embedded_association(record, association_name, coder_or_array) #:nodoc:
         value = if IdentityCache.unmap_cached_nil_for(coder_or_array).nil?
           nil
         elsif (reflection = record.class.reflect_on_association(association_name)).collection?
           association = reflection.association_class.new(record, reflection)
           association.target = coder_or_array.map {|e| record_from_coder(e) }
-          association.target.each {|e| association.set_inverse_instance(e) }
+
+          set_inverse_of_cached_has_many(record, reflection, association.target)
+
+          unless IdentityCache.never_set_inverse_association
+            association.target.each {|e| association.set_inverse_instance(e) }
+          end
+
           association
         else
           record_from_coder(coder_or_array)
         end
         variable_name = record.class.send(:recursively_embedded_associations)[association_name][:records_variable_name]
-        record.instance_variable_set(:"@#{variable_name}", IdentityCache.map_cached_nil_for(value))
+        record.instance_variable_set(:"@#{variable_name}", value)
       end
 
       def get_embedded_association(record, association, options) #:nodoc:
         embedded_variable = record.public_send(options.fetch(:cached_accessor_name))
-        if record.class.reflect_on_association(association).collection?
+        if embedded_variable.respond_to?(:to_ary)
           embedded_variable.map {|e| coder_from_record(e) }
         else
           coder_from_record(embedded_variable)
@@ -120,7 +181,7 @@ module IdentityCache
       def coder_from_record(record) #:nodoc:
         unless record.nil?
           coder = {
-            attributes: record.attributes_before_type_cast,
+            attributes: record.attributes_before_type_cast.dup,
             class: record.class,
           }
           add_cached_associations_to_coder(record, coder)
@@ -164,7 +225,10 @@ module IdentityCache
 
       def resolve_cache_miss(id)
         record = self.includes(cache_fetch_includes).reorder(nil).where(primary_key => id).first
-        preload_id_embedded_associations([record]) if record
+        if record
+          preload_id_embedded_associations([record])
+          record.readonly! if IdentityCache.fetch_read_only_records && IdentityCache.should_use_cache?
+        end
         record
       end
 
@@ -209,7 +273,7 @@ module IdentityCache
       end
 
       def all_cached_associations
-        (cached_has_manys || {}).merge(cached_has_ones || {}).merge(cached_belongs_tos || {})
+        cached_has_manys.merge(cached_has_ones).merge(cached_belongs_tos)
       end
 
       def embedded_associations
@@ -241,120 +305,138 @@ module IdentityCache
         @id_column ||= columns.detect {|c| c.name == primary_key}
         ids = ids.map{ |id| connection.type_cast(id, @id_column) }
         records = where(primary_key => ids).includes(cache_fetch_includes).to_a
+        records.each(&:readonly!) if IdentityCache.fetch_read_only_records && IdentityCache.should_use_cache?
         preload_id_embedded_associations(records)
         records_by_id = records.index_by(&:id)
         ids.map{ |id| records_by_id[id] }
       end
 
-      def prefetch_associations(associations, records)
-        associations = hashify_includes_structure(associations)
+      def fetch_embedded_associations(records)
+        associations = embedded_associations
+        return if associations.empty?
 
-        associations.each do |association, sub_associations|
-          case
-          when details = cached_has_manys[association]
+        return unless primary_cache_index_enabled
 
-            if details[:embed] == true
-              child_records = records.map(&details[:cached_accessor_name].to_sym).flatten
+        cached_records_by_id = fetch_multi(records.map(&:id)).index_by(&:id)
+
+        associations.each_value do |options|
+          records.each do |record|
+            next unless cached_record = cached_records_by_id[record.id]
+            if options[:embed] == :ids
+              cached_association = cached_record.public_send(options.fetch(:cached_ids_name))
+              record.instance_variable_set(:"@#{options.fetch(:ids_variable_name)}", cached_association)
             else
-              ids_to_parent_record = records.each_with_object({}) do |record, hash|
-                child_ids = record.send(details[:cached_ids_name])
-                child_ids.each do |child_id|
-                  hash[child_id] = record
-                end
-              end
+              cached_association = cached_record.public_send(options.fetch(:cached_accessor_name))
+              record.instance_variable_set(:"@#{options.fetch(:records_variable_name)}", cached_association)
+            end
+          end
+        end
+      end
 
-              parent_record_to_child_records = Hash.new { |h, k| h[k] = [] }
-              child_records = details[:association_reflection].klass.fetch_multi(*ids_to_parent_record.keys)
-              child_records.each do |child_record|
-                parent_record = ids_to_parent_record[child_record.id]
-                parent_record_to_child_records[parent_record] << child_record
-              end
+      def prefetch_embedded_association(records, association, details)
+        # Make the same assumption as ActiveRecord::Associations::Preloader, which is
+        # that all the records have the same associations loaded, so we can just check
+        # the first record to see if an association is loaded.
+        first_record = records.first
+        return if first_record.association(association).loaded?
+        iv_name_key = details[:embed] == true ? :records_variable_name : :ids_variable_name
+        return if first_record.instance_variable_defined?(:"@#{details[iv_name_key]}")
+        fetch_embedded_associations(records)
+      end
 
-              parent_record_to_child_records.each do |parent_record, child_records|
-                parent_record.send(details[:prepopulate_method_name], child_records)
+      def prefetch_one_association(association, records)
+        case
+        when details = cached_has_manys[association]
+          prefetch_embedded_association(records, association, details)
+          if details[:embed] == true
+            child_records = records.flat_map(&details[:cached_accessor_name].to_sym)
+          else
+            ids_to_parent_record = records.each_with_object({}) do |record, hash|
+              child_ids = record.send(details[:cached_ids_name])
+              child_ids.each do |child_id|
+                hash[child_id] = record
               end
             end
 
-            next_level_records = child_records
-
-          when details = cached_belongs_tos[association]
-            if details[:embed] == true
-              raise ArgumentError.new("Embedded belongs_to associations do not support prefetching yet.")
-            else
-              reflection = details[:association_reflection]
-              if reflection.polymorphic?
-                raise ArgumentError.new("Polymorphic belongs_to associations do not support prefetching yet.")
-              end
+            parent_record_to_child_records = Hash.new { |h, k| h[k] = [] }
+            child_records = details[:association_reflection].klass.fetch_multi(*ids_to_parent_record.keys)
+            child_records.each do |child_record|
+              parent_record = ids_to_parent_record[child_record.id]
+              parent_record_to_child_records[parent_record] << child_record
+            end
 
-              ids_to_child_record = records.each_with_object({}) do |child_record, hash|
-                parent_id = child_record.send(reflection.foreign_key)
-                hash[parent_id] = child_record if parent_id.present?
-              end
-              parent_records = reflection.klass.fetch_multi(ids_to_child_record.keys)
-              parent_records.each do |parent_record|
-                child_record = ids_to_child_record[parent_record.id]
-                child_record.send(details[:prepopulate_method_name], parent_record)
-              end
+            parent_record_to_child_records.each do |parent, children|
+              parent.send(details[:prepopulate_method_name], children)
             end
+          end
 
-            next_level_records = parent_records
+          next_level_records = child_records
 
-          when details = cached_has_ones[association]
-            if details[:embed] == true
-              parent_records = records.map(&details[:cached_accessor_name].to_sym)
-            else
-              raise ArgumentError.new("Non-embedded has_one associations do not support prefetching yet.")
+        when details = cached_belongs_tos[association]
+          if details[:embed] == true
+            raise ArgumentError.new("Embedded belongs_to associations do not support prefetching yet.")
+          else
+            reflection = details[:association_reflection]
+            if reflection.polymorphic?
+              raise ArgumentError.new("Polymorphic belongs_to associations do not support prefetching yet.")
             end
 
-            next_level_records = parent_records
+            cached_iv_name = :"@#{details.fetch(:records_variable_name)}"
+            ids_to_child_record = records.each_with_object({}) do |child_record, hash|
+              parent_id = child_record.send(reflection.foreign_key)
+              if parent_id && !child_record.instance_variable_defined?(cached_iv_name)
+                hash[parent_id] = child_record
+              end
+            end
+            parent_records = reflection.klass.fetch_multi(ids_to_child_record.keys)
+            parent_records.each do |parent_record|
+              child_record = ids_to_child_record[parent_record.id]
+              child_record.send(details[:prepopulate_method_name], parent_record)
+            end
+          end
 
+          next_level_records = parent_records
+
+        when details = cached_has_ones[association]
+          if details[:embed] == true
+            prefetch_embedded_association(records, association, details)
+            parent_records = records.map(&details[:cached_accessor_name].to_sym)
           else
-            raise ArgumentError.new("Unknown cached association #{association} listed for prefetching")
+            raise ArgumentError.new("Non-embedded has_one associations do not support prefetching yet.")
           end
 
-          if details && details[:association_reflection].klass.respond_to?(:prefetch_associations, true)
-            details[:association_reflection].klass.send(:prefetch_associations, sub_associations, next_level_records)
-          end
-        end
-      end
+          next_level_records = parent_records
 
-      def hashify_includes_structure(structure)
-        case structure
-        when nil
-          {}
-        when Symbol
-          {structure => []}
-        when Hash
-          structure.clone
-        when Array
-          structure.each_with_object({}) do |member, hash|
-            case member
-            when Hash
-              hash.merge!(member)
-            when Symbol
-              hash[member] = []
-            end
-          end
+        else
+          raise ArgumentError.new("Unknown cached association #{association} listed for prefetching")
         end
+        next_level_records
       end
     end
 
     private
 
     def fetch_recursively_cached_association(ivar_name, association_name) # :nodoc:
-      if IdentityCache.should_use_cache?
+      assoc = if IdentityCache.should_use_cache?
         ivar_full_name = :"@#{ivar_name}"
 
-        unless ivar_value = instance_variable_get(ivar_full_name)
-          ivar_value = IdentityCache.map_cached_nil_for(send(association_name))
-          instance_variable_set(ivar_full_name, ivar_value)
+        assoc = if instance_variable_defined?(ivar_full_name)
+          instance_variable_get(ivar_full_name)
+        else
+          cached_assoc = if IdentityCache.fetch_read_only_records
+            load_and_readonlyify(association_name)
+          else
+            send(association_name)
+          end
+          instance_variable_set(ivar_full_name, cached_assoc)
         end
 
-        assoc = IdentityCache.unmap_cached_nil_for(ivar_value)
         assoc.is_a?(ActiveRecord::Associations::CollectionAssociation) ? assoc.reader : assoc
       else
         send(association_name.to_sym)
       end
+      assoc = assoc.to_ary if assoc.respond_to?(:to_ary) && !IdentityCache.fetch_returns_relation
+      assoc
     end
 
     def expire_primary_index # :nodoc:
@@ -400,5 +482,15 @@ module IdentityCache
       pk = self.class.primary_key
       !destroyed? && transaction_changed_attributes.has_key?(pk) && transaction_changed_attributes[pk].nil?
     end
+
+    def load_and_readonlyify(association_name)
+      record_or_records = send(association_name)
+
+      if self.class.reflect_on_association(association_name).collection?
+        record_or_records.map { |p| p.dup.tap(&:readonly!) }
+      else
+        record_or_records.dup.tap(&:readonly!) if record_or_records
+      end
+    end
   end
 end

data/lib/identity_cache/version.rb

@@ -1,4 +1,4 @@
 module IdentityCache
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
   CACHE_VERSION = 6
 end