identity_cache 0.0.1 → 0.0.2

data/CHANGELOG

@@ -0,0 +1,3 @@
+0.0.2
+* Fix: Existent embedded entries will no longer raise when ActiveModel::MissingAttributeError when accessing a newly created attribute.
+* Fix: Do not marshal raw AcriveRecord associations

data/README.md

@@ -1,5 +1,5 @@
 # IdentityCache
-[![Build Status](https://api.travis-ci.org/Shopify/identity_cache.png)](http://travis-ci.org/Shopify/identity_cache)
+[![Build Status](https://api.travis-ci.org/Shopify/identity_cache.png?branch=master)](http://travis-ci.org/Shopify/identity_cache)
 
 Opt in read through ActiveRecord caching used in production and extracted from Shopify. IdentityCache lets you specify how you want to cache your model objects, at the model level, and adds a number of convenience methods for accessing those objects through the cache. Memcached is used as the backend cache store, and the database is only hit when a copy of the object cannot be found in Memcached.
 
@@ -10,7 +10,7 @@ IdentityCache keeps track of the objects that have cached indexes and uses an `a
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'identity_cache', :github => 'Shopify/identity_cache'
+gem 'identity_cache'
 ```
 
 And then execute:
@@ -60,7 +60,7 @@ end
 # If the object isn't in the cache it is pulled from the db and stored in the cache.
 product = Product.fetch_by_handle(handle)
 
-products = Product.fetch_by_vendor_and_product_type(handle)
+products = Product.fetch_by_vendor_and_product_type(vendor, product_type)
 ```
 
 This gives you a lot of freedom to use your objects the way you want to, and doesn't get in your way. This does keep an independent cache copy in Memcached so you might want to watch the number of different caches that are being added.

data/identity_cache.gemspec

@@ -17,8 +17,12 @@ Gem::Specification.new do |gem|
 
 
   gem.add_dependency('ar_transaction_changes', '0.0.1')
+  gem.add_dependency('activerecord', '3.2.13')
+  gem.add_dependency('activesupport', '3.2.13')
   gem.add_dependency('cityhash', '0.6.0')
+  gem.add_development_dependency('memcache-client')
   gem.add_development_dependency('rake')
   gem.add_development_dependency('mocha')
   gem.add_development_dependency('mysql2')
+  gem.add_development_dependency('debugger')
 end

data/lib/identity_cache.rb

@@ -12,8 +12,14 @@ module IdentityCache
     attr_accessor :logger, :readonly
     attr_reader :cache
 
-    def cache_backend=(memcache)
-      cache.memcache = memcache
+    # Sets the cache adaptor IdentityCache will be using
+    #
+    # == Parameters
+    #
+    # +cache_adaptor+ - A ActiveSupport::Cache::Store
+    #
+    def cache_backend=(cache_adaptor)
+      cache.memcache = cache_adaptor
     end
 
     def cache
@@ -24,10 +30,17 @@ module IdentityCache
       @logger || Rails.logger
     end
 
-    def should_cache?
+    def should_cache? # :nodoc:
       !readonly && ActiveRecord::Base.connection.open_transactions == 0
     end
 
+    # Cache retrieval and miss resolver primitive; given a key it will try to
+    # retrieve the associated value from the cache otherwise it will return the
+    # value of the execution of the block.
+    #
+    # == Parameters
+    # +key+ A cache key string
+    #
     def fetch(key, &block)
       result = cache.read(key) if should_cache?
 
@@ -37,6 +50,7 @@ module IdentityCache
             result = yield
           end
           result = map_cached_nil_for(result)
+
           if should_cache?
             cache.write(key, result)
           end
@@ -58,6 +72,11 @@ module IdentityCache
       value == IdentityCache::CACHED_NIL ? nil : value
     end
 
+    # Same as +fetch+, except that it will try a collection of keys, using the
+    # multiget operation of the cache adaptor
+    #
+    # == Parameters
+    # +keys+ A collection of key strings
     def fetch_multi(*keys, &block)
       return {} if keys.size == 0
       result = {}
@@ -93,7 +112,11 @@ module IdentityCache
       result
     end
 
-    def included(base)
+    def schema_to_string(columns)
+      columns.sort_by(&:name).map {|c| "#{c.name}:#{c.type}"} * ","
+    end
+
+    def included(base) #:nodoc:
       raise AlreadyIncludedError if base.respond_to? :cache_indexes
 
       unless ActiveRecord::Base.connection.respond_to?(:with_master)
@@ -112,26 +135,55 @@ module IdentityCache
       base.class_attribute :cache_attributes
       base.class_attribute :cached_has_manys
       base.class_attribute :cached_has_ones
+      base.class_attribute :embedded_schema_hashes
       base.send(:extend, ClassMethods)
 
+      base.cached_has_manys = {}
+      base.cached_has_ones = {}
+      base.embedded_schema_hashes = {}
+      base.cache_attributes = []
+      base.cache_indexes = []
+
       base.private_class_method :require_if_necessary, :build_normalized_has_many_cache, :build_denormalized_association_cache, :add_parent_expiry_hook,
         :identity_cache_multiple_value_dynamic_fetcher, :identity_cache_single_value_dynamic_fetcher
 
+
       base.instance_eval(ruby = <<-CODE, __FILE__, __LINE__)
         private :expire_cache, :was_new_record?, :fetch_denormalized_cached_association, :populate_denormalized_cached_association
       CODE
     end
 
-    def memcache_hash(key)
+    def memcache_hash(key) #:nodoc:
       CityHash.hash64(key)
     end
   end
 
   module ClassMethods
 
+    # Declares a new index in the cache for the class where IdentityCache was
+    # included.
+    #
+    # IdentityCache will add a fetch_by_field1_and_field2_and_...field for every
+    # index.
+    #
+    # == Example:
+    #
+    #  class Product
+    #    include IdentityCache
+    #    cache_index :name, :vendor
+    #  end
+    #
+    # Will add Product.fetch_by_name_and_vendor
+    #
+    # == Parameters
+    #
+    # +fields+ Array of symbols or strings representing the fields in the index
+    #
+    # == Options
+    # * unique: if the index would only have unique values
+    #
     def cache_index(*fields)
       options = fields.extract_options!
-      self.cache_indexes ||= []
       self.cache_indexes.push fields
 
       field_list = fields.join("_and_")
@@ -160,7 +212,7 @@ module IdentityCache
       end
     end
 
-    def identity_cache_single_value_dynamic_fetcher(fields, values, sql_on_miss)
+    def identity_cache_single_value_dynamic_fetcher(fields, values, sql_on_miss) # :nodoc:
       cache_key = rails_cache_index_key_for_fields_and_values(fields, values)
       id = IdentityCache.fetch(cache_key) { connection.select_value(sql_on_miss) }
       unless id.nil?
@@ -171,18 +223,42 @@ module IdentityCache
       record
     end
 
-    def identity_cache_multiple_value_dynamic_fetcher(fields, values, sql_on_miss)
+    def identity_cache_multiple_value_dynamic_fetcher(fields, values, sql_on_miss) # :nodoc:
       cache_key = rails_cache_index_key_for_fields_and_values(fields, values)
       ids = IdentityCache.fetch(cache_key) { connection.select_values(sql_on_miss) }
 
       ids.empty? ? [] : fetch_multi(*ids)
     end
 
+
+    # Will cache an association to the class including IdentityCache.
+    # The embed option, if set, will make IdentityCache keep the association
+    # values in the same cache entry as the parent.
+    #
+    # Embedded associations are more effective in offloading database work,
+    # however they will increase the size of the cache entries and make the
+    # whole entry expire when any of the embedded members change.
+    #
+    # == Example:
+    #   class Product
+    #    cached_has_many :options, :embed => false
+    #    cached_has_many :orders
+    #    cached_has_many :buyers, :inverse_name => 'line_item'
+    #   end
+    #
+    # == Parameters
+    # +association+ Name of the association being cached as a symbol
+    #
+    # == Options
+    #
+    # * embed: If set will cause IdentityCache to keep the values for this
+    #   association in the same cache entry as the parent, instead of its own.
+    # * inverse_name: The name of the parent in the association if the name is
+    #   not the lowercase pluralization of the parent object's class
     def cache_has_many(association, options = {})
       options[:embed] ||= false
       options[:inverse_name] ||= self.name.underscore.to_sym
       raise InverseAssociationError unless self.reflect_on_association(association)
-      self.cached_has_manys ||= {}
       self.cached_has_manys[association] = options
 
       if options[:embed]
@@ -192,21 +268,45 @@ module IdentityCache
       end
     end
 
+    # Will cache an association to the class including IdentityCache.
+    # The embed option if set will make IdentityCache keep the association
+    # values in the same cache entry as the parent.
+    #
+    # Embedded associations are more effective in offloading database work,
+    # however they will increase the size of the cache entries and make the
+    # whole entry expire with the change of any of the embedded members
+    #
+    # == Example:
+    #   class Product
+    #    cached_has_one :store, :embed => false
+    #    cached_has_one :vendor
+    #   end
+    #
+    # == Parameters
+    # +association+ Symbol with the name of the association being cached
+    #
+    # == Options
+    #
+    # * embed: If set will cause IdentityCache to keep the values for this
+    #   association in the same cache entry as the parent, instead of its own.
+    # * inverse_name: The name of the parent in the association ( only
+    #   necessary if the name is not the lowercase pluralization of the 
+    #   parent object's class)
     def cache_has_one(association, options = {})
       options[:embed] ||= true
       options[:inverse_name] ||= self.name.underscore.to_sym
       raise InverseAssociationError unless self.reflect_on_association(association)
-      self.cached_has_ones ||= {}
       self.cached_has_ones[association] = options
 
       build_denormalized_association_cache(association, options)
     end
 
-    def build_denormalized_association_cache(association, options)
+    def build_denormalized_association_cache(association, options) #:nodoc:
       options[:cached_accessor_name] ||= "fetch_#{association}"
       options[:cache_variable_name]  ||= "cached_#{association}"
       options[:population_method_name]  ||= "populate_#{association}_cache"
 
+
       unless instance_methods.include?(options[:cached_accessor_name].to_sym)
         self.class_eval(ruby = <<-CODE, __FILE__, __LINE__)
           def #{options[:cached_accessor_name]}
@@ -223,7 +323,7 @@ module IdentityCache
       end
     end
 
-    def build_normalized_has_many_cache(association, options)
+    def build_normalized_has_many_cache(association, options) #:nodoc:
       singular_association = association.to_s.singularize
       association_class    = reflect_on_association(association).klass
       options[:cached_accessor_name]    ||= "fetch_#{association}"
@@ -251,11 +351,26 @@ module IdentityCache
       add_parent_expiry_hook(association_class, options.merge(:only_on_foreign_key_change => true))
     end
 
+
+    # Will cache a single attribute on its own blob, it will add a
+    # fetch_attribute_by_id (or the value of the by option).
+    #
+    # == Example:
+    #   class Product
+    #    cache_attribute :quantity, :by => :name
+    #    cache_attribute :quantity  :by => [:name, :vendor]
+    #   end
+    #
+    # == Parameters
+    # +attribute+ Symbol with the name of the attribute being cached
+    #
+    # == Options
+    #
+    # * by: Other attribute or attributes in the model to keep values indexed. Default is :id
     def cache_attribute(attribute, options = {})
       options[:by] ||= :id
       fields = Array(options[:by])
 
-      self.cache_attributes ||= []
       self.cache_attributes.push [attribute, fields]
 
       field_list = fields.join("_and_")
@@ -270,21 +385,24 @@ module IdentityCache
       CODE
     end
 
-    def attribute_dynamic_fetcher(attribute, fields, values, sql_on_miss)
+    def attribute_dynamic_fetcher(attribute, fields, values, sql_on_miss) #:nodoc:
       cache_key = rails_cache_key_for_attribute_and_fields_and_values(attribute, fields, values)
       IdentityCache.fetch(cache_key) { connection.select_value(sql_on_miss) }
     end
 
+    # Similar to ActiveRecord::Base#exists? will return true if the id can be
+    # found in the cache.
     def exists_with_identity_cache?(id)
       !!fetch_by_id(id)
     end
 
+    # Default fetcher added to the model on inclusion, it behaves like
+    # ActiveRecord::Base.find_by_id
     def fetch_by_id(id)
       if IdentityCache.should_cache?
 
         require_if_necessary do
           object = IdentityCache.fetch(rails_cache_key(id)){ resolve_cache_miss(id) }
-          object.clear_association_cache if object.respond_to?(:clear_association_cache)
           IdentityCache.logger.error "[IDC id mismatch] fetch_by_id_requested=#{id} fetch_by_id_got=#{object.id} for #{object.inspect[(0..100)]} " if object && object.id != id.to_i
           object
         end
@@ -294,11 +412,18 @@ module IdentityCache
       end
     end
 
+    # Default fetcher added to the model on inclusion, it behaves like
+    # ActiveRecord::Base.find, will raise ActiveRecord::RecordNotFound exception
+    # if id is not in the cache or the db.
     def fetch(id)
       fetch_by_id(id) or raise(ActiveRecord::RecordNotFound, "Couldn't find #{self.class.name} with ID=#{id}")
     end
 
+
+    # Default fetcher added to the model on inclusion, if behaves like
+    # ActiveRecord::Base.find_all_by_id
     def fetch_multi(*ids)
+      options = ids.extract_options!
       if IdentityCache.should_cache?
 
         require_if_necessary do
@@ -307,25 +432,20 @@ module IdentityCache
 
           objects_by_key = IdentityCache.fetch_multi(*key_to_id_map.keys) do |unresolved_keys|
             ids = unresolved_keys.map {|key| key_to_id_map[key] }
-            records = find_batch(ids)
+            records = find_batch(ids, options)
             records.compact.each(&:populate_association_caches)
             records
           end
 
-          objects_in_order = cache_keys.map {|key| objects_by_key[key] }
-          objects_in_order.each do |object|
-            object.clear_association_cache if object.respond_to?(:clear_association_cache)
-          end
-
-          objects_in_order.compact
+          cache_keys.map {|key| objects_by_key[key] }.compact
         end
 
       else
-        find_batch(ids)
+        find_batch(ids, options)
       end
     end
 
-    def require_if_necessary
+    def require_if_necessary #:nodoc:
       # mem_cache_store returns raw value if unmarshal fails
       rval = yield
       case rval
@@ -343,7 +463,7 @@ module IdentityCache
       raise
     end
 
-    module ParentModelExpiration
+    module ParentModelExpiration # :nodoc:
       def expire_parent_cache_on_changes(parent_name, foreign_key, parent_class, options = {})
         new_parent = send(parent_name)
 
@@ -403,25 +523,43 @@ module IdentityCache
     end
 
     def all_cached_associations
-      (cached_has_manys || {}).merge(cached_has_ones || {})
+      (cached_has_manys || {}).merge(cached_has_ones || {}).merge(cached_belongs_tos || {})
+    end
+
+    def all_cached_associations_needing_population
+      all_cached_associations.select do |cached_association, options|
+        options[:population_method_name].present? # non-embedded belongs_to associations don't need population
+      end
     end
 
-    def cache_fetch_includes
-      all_cached_associations.select{|k, v| v[:embed]}.map do |child_association, options|
+    def cache_fetch_includes(additions = {})
+      additions = hashify_includes_structure(additions)
+      embedded_associations = all_cached_associations.select { |name, options| options[:embed] }
+
+      associations_for_identity_cache = embedded_associations.map do |child_association, options|
         child_class = reflect_on_association(child_association).try(:klass)
-        child_includes = child_class.respond_to?(:cache_fetch_includes) ? child_class.cache_fetch_includes : []
-        if child_includes.empty?
+
+        child_includes = additions.delete(child_association)
+
+        if child_class.respond_to?(:cache_fetch_includes)
+          child_includes = child_class.cache_fetch_includes(child_includes)
+        end
+
+        if child_includes.blank?
           child_association
         else
-          { child_association => child_class.cache_fetch_includes }
+          { child_association => child_includes }
         end
       end
+
+      associations_for_identity_cache.push(additions) if additions.keys.size > 0
+      associations_for_identity_cache.compact
     end
 
-    def find_batch(ids)
+    def find_batch(ids, options = {})
       @id_column ||= columns.detect {|c| c.name == "id"}
       ids = ids.map{ |id| @id_column.type_cast(id) }
-      records = where('id IN (?)', ids).includes(cache_fetch_includes).all
+      records = where('id IN (?)', ids).includes(cache_fetch_includes(options[:includes])).all
       records_by_id = records.index_by(&:id)
       records = ids.map{ |id| records_by_id[id] }
       mismatching_ids = records.compact.map(&:id) - ids
@@ -433,10 +571,10 @@ module IdentityCache
       rails_cache_key_prefix + id.to_s
     end
 
+
     def rails_cache_key_prefix
       @rails_cache_key_prefix ||= begin
-        column_list = columns.sort_by(&:name).map {|c| "#{c.name}:#{c.type}"} * ","
-        "IDC:blob:#{base_class.name}:#{IdentityCache.memcache_hash(column_list)}:"
+        "IDC:blob:#{base_class.name}:#{IdentityCache.memcache_hash(IdentityCache.schema_to_string(columns))}:"
       end
     end
 
@@ -451,10 +589,32 @@ module IdentityCache
     def rails_cache_string_for_fields_and_values(fields, values)
       "#{fields.join('/')}:#{IdentityCache.memcache_hash(values.join('/'))}"
     end
+
+    private
+
+    def hashify_includes_structure(structure)
+      case structure
+      when nil
+        {}
+      when Symbol
+        {structure => []}
+      when Hash
+        structure.clone
+      when Array
+        structure.each_with_object({}) do |member, hash|
+          case member
+          when Hash
+            hash.merge(hash)
+          when Symbol
+            hash[member] = []
+          end
+        end
+      end
+    end
   end
 
-  def populate_association_caches
-    self.class.all_cached_associations.each do |cached_association, options|
+  def populate_association_caches # :nodoc:
+    self.class.all_cached_associations_needing_population.each do |cached_association, options|
       send(options[:population_method_name])
       reflection = options[:embed] && self.class.reflect_on_association(cached_association)
       if reflection && reflection.klass.respond_to?(:cached_has_manys)
@@ -462,9 +622,10 @@ module IdentityCache
         child_objects.each(&:populate_association_caches)
       end
     end
+    self.clear_association_cache if self.respond_to?(:clear_association_cache)
   end
 
-  def fetch_denormalized_cached_association(ivar_name, association_name)
+  def fetch_denormalized_cached_association(ivar_name, association_name) # :nodoc:
     ivar_full_name = :"@#{ivar_name}"
     if IdentityCache.should_cache?
       populate_denormalized_cached_association(ivar_name, association_name)
@@ -474,36 +635,47 @@ module IdentityCache
     end
   end
 
-  def populate_denormalized_cached_association(ivar_name, association_name)
+  def populate_denormalized_cached_association(ivar_name, association_name) # :nodoc:
     ivar_full_name = :"@#{ivar_name}"
+    schema_hash_ivar = :"@#{ivar_name}_schema_hash"
+    reflection = association(association_name)
 
-    value = instance_variable_get(ivar_full_name)
-    return value unless value.nil?
+    current_schema_hash = self.class.embedded_schema_hashes[association_name] ||= begin
+      IdentityCache.memcache_hash(IdentityCache.schema_to_string(reflection.klass.columns))
+    end
+
+    saved_schema_hash = instance_variable_get(schema_hash_ivar)
+
+    if saved_schema_hash == current_schema_hash
+      value = instance_variable_get(ivar_full_name)
+      return value unless value.nil?
+    end
 
-    reflection = association(association_name)
     reflection.load_target unless reflection.loaded?
 
     loaded_association = send(association_name)
+
+    instance_variable_set(schema_hash_ivar, current_schema_hash)
     instance_variable_set(ivar_full_name, IdentityCache.map_cached_nil_for(loaded_association))
   end
 
-  def primary_cache_index_key
+  def primary_cache_index_key # :nodoc:
     self.class.rails_cache_key(id)
   end
 
-  def secondary_cache_index_key_for_current_values(fields)
+  def secondary_cache_index_key_for_current_values(fields) # :nodoc:
     self.class.rails_cache_index_key_for_fields_and_values(fields, fields.collect {|field| self.send(field)})
   end
 
-  def secondary_cache_index_key_for_previous_values(fields)
+  def secondary_cache_index_key_for_previous_values(fields) # :nodoc:
     self.class.rails_cache_index_key_for_fields_and_values(fields, old_values_for_fields(fields))
   end
 
-  def attribute_cache_key_for_attribute_and_previous_values(attribute, fields)
+  def attribute_cache_key_for_attribute_and_previous_values(attribute, fields) # :nodoc:
     self.class.rails_cache_key_for_attribute_and_fields_and_values(attribute, fields, old_values_for_fields(fields))
   end
 
-  def old_values_for_fields(fields)
+  def old_values_for_fields(fields) # :nodoc:
     fields.map do |field|
       field_string = field.to_s
       if destroyed? && transaction_changed_attributes.has_key?(field_string)
@@ -516,7 +688,7 @@ module IdentityCache
     end
   end
 
-  def expire_primary_index
+  def expire_primary_index # :nodoc:
     extra_keys = if respond_to? :updated_at
       old_updated_at = old_values_for_fields([:updated_at]).first
       "expiring_last_updated_at=#{old_updated_at}"
@@ -528,7 +700,7 @@ module IdentityCache
     IdentityCache.cache.delete(primary_cache_index_key)
   end
 
-  def expire_secondary_indexes
+  def expire_secondary_indexes # :nodoc:
     cache_indexes.try(:each) do |fields|
       if self.destroyed?
         IdentityCache.cache.delete(secondary_cache_index_key_for_previous_values(fields))
@@ -544,20 +716,20 @@ module IdentityCache
     end
   end
 
-  def expire_attribute_indexes
+  def expire_attribute_indexes # :nodoc:
     cache_attributes.try(:each) do |(attribute, fields)|
       IdentityCache.cache.delete(attribute_cache_key_for_attribute_and_previous_values(attribute, fields)) unless was_new_record?
     end
   end
 
-  def expire_cache
+  def expire_cache # :nodoc:
     expire_primary_index
     expire_secondary_indexes
     expire_attribute_indexes
     true
   end
 
-  def was_new_record?
+  def was_new_record? # :nodoc:
     !destroyed? && transaction_changed_attributes.has_key?('id') && transaction_changed_attributes['id'].nil?
   end
 

data/lib/identity_cache/version.rb

@@ -1,3 +1,3 @@
 module IdentityCache
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/test/cache_fetch_includes_test.rb

@@ -0,0 +1,88 @@
+require "test_helper"
+
+class CacheFetchIncludesTest < IdentityCache::TestCase
+  def setup
+    super
+  end
+
+  def test_cached_embedded_has_manys_are_included_in_includes
+    Record.send(:cache_has_many, :associated_records, :embed => true)
+    assert_equal [:associated_records], Record.cache_fetch_includes
+  end
+
+  def test_cached_nonembedded_has_manys_are_included_in_includes
+    Record.send(:cache_has_many, :associated_records, :embed => false)
+    assert_equal [], Record.cache_fetch_includes
+  end
+
+  def test_cached_has_ones_are_included_in_includes
+    Record.send(:cache_has_one, :associated)
+    assert_equal [:associated], Record.cache_fetch_includes
+  end
+
+  def test_cached_nonembedded_belongs_tos_are_not_included_in_includes
+    Record.send(:cache_belongs_to, :record)
+    assert_equal [], Record.cache_fetch_includes
+  end
+
+  def test_cached_child_associations_are_included_in_includes
+    Record.send(:cache_has_many, :associated_records, :embed => true)
+    AssociatedRecord.send(:cache_has_many, :deeply_associated_records, :embed => true)
+    assert_equal [{:associated_records => [:deeply_associated_records]}], Record.cache_fetch_includes
+  end
+
+  def test_multiple_cached_associations_and_child_associations_are_included_in_includes
+    Record.send(:cache_has_many, :associated_records, :embed => true)
+    Record.send(:cache_has_many, :polymorphic_records, {:inverse_name => :owner, :embed => true})
+    Record.send(:cache_has_one, :associated, :embed => true)
+    AssociatedRecord.send(:cache_has_many, :deeply_associated_records, :embed => true)
+    assert_equal [
+      {:associated_records => [:deeply_associated_records]},
+      :polymorphic_records,
+      {:associated => [:deeply_associated_records]}
+    ],  Record.cache_fetch_includes
+  end
+
+  def test_empty_additions_for_top_level_associations_makes_no_difference
+    Record.send(:cache_has_many, :associated_records, :embed => true)
+    assert_equal [:associated_records], Record.cache_fetch_includes({})
+  end
+
+  def test_top_level_additions_are_included_in_includes
+    assert_equal [{:associated_records => []}], Record.cache_fetch_includes({:associated_records => []})
+  end
+
+  def test_top_level_additions_alongside_top_level_cached_associations_are_included_in_includes
+    Record.send(:cache_has_many, :associated_records, :embed => true)
+    assert_equal [
+      :associated_records,
+      {:polymorphic_records => []}
+    ], Record.cache_fetch_includes({:polymorphic_records => []})
+  end
+
+  def test_child_level_additions_for_top_level_cached_associations_are_included_in_includes
+    Record.send(:cache_has_many, :associated_records, :embed => true)
+    assert_equal [
+      {:associated_records => [{:deeply_associated_records => []}]}
+    ], Record.cache_fetch_includes({:associated_records => :deeply_associated_records})
+  end
+
+  def test_array_child_level_additions_for_top_level_cached_associations_are_included_in_includes
+    Record.send(:cache_has_many, :associated_records, :embed => true)
+    assert_equal [
+      {:associated_records => [{:deeply_associated_records => []}]}
+    ], Record.cache_fetch_includes({:associated_records => [:deeply_associated_records]})
+  end
+
+  def test_array_child_level_additions_for_child_level_cached_associations_are_included_in_includes
+    Record.send(:cache_has_many, :associated_records, :embed => true)
+    AssociatedRecord.send(:cache_has_many, :deeply_associated_records, :embed => true)
+    assert_equal [
+      {:associated_records => [
+        :deeply_associated_records,
+        {:record => []}
+      ]}
+    ], Record.cache_fetch_includes({:associated_records => [:record]})
+  end
+
+end

data/test/denormalized_has_one_test.rb

@@ -24,7 +24,10 @@ class DenormalizedHasOneTest < IdentityCache::TestCase
   end
 
   def test_on_cache_miss_record_should_embed_nil_object
-    @record.expects(:associated => nil)
+
+    @record.associated = nil
+    @record.save!
+    @record.reload
     Record.expects(:find_by_id).with(@record.id, :include => Record.cache_fetch_includes).returns(@record)
     IdentityCache.cache.expects(:read).with(@record.secondary_cache_index_key_for_current_values([:title]))
     IdentityCache.cache.expects(:read).with(@record.primary_cache_index_key)
@@ -57,7 +60,10 @@ class DenormalizedHasOneTest < IdentityCache::TestCase
   end
 
   def test_on_cache_hit_record_should_come_back_with_cached_nil_association
-    @record.expects(:associated => nil)
+    @record.associated = nil
+    @record.save!
+    @record.reload
+
     Record.expects(:find_by_id).with(1, :include => Record.cache_fetch_includes).once.returns(@record)
     Record.fetch_by_title('foo')
 

data/test/fetch_multi_test.rb

@@ -110,6 +110,7 @@ class FetchMultiTest < IdentityCache::TestCase
   def test_fetch_multi_includes_cached_associations
     Record.send(:cache_has_many, :associated_records, :embed => true)
     Record.send(:cache_has_one, :associated)
+    Record.send(:cache_belongs_to, :record)
 
     cache_response = {}
     cache_response[@bob_blob_key] = nil
@@ -124,6 +125,24 @@ class FetchMultiTest < IdentityCache::TestCase
     assert_equal [@bob, @joe, @fred], Record.fetch_multi(@bob.id, @joe.id, @fred.id)
   end
 
+  def test_fetch_multi_includes_cached_associations_and_other_asked_for_associations
+    Record.send(:cache_has_many, :associated_records, :embed => true)
+    Record.send(:cache_has_one, :associated)
+    Record.send(:cache_belongs_to, :record)
+
+    cache_response = {}
+    cache_response[@bob_blob_key] = nil
+    cache_response[@joe_blob_key] = nil
+    cache_response[@fred_blob_key] = nil
+
+    IdentityCache.cache.expects(:read_multi).with(@bob_blob_key, @joe_blob_key, @fred_blob_key).returns(cache_response)
+
+    mock_relation = mock("ActiveRecord::Relation")
+    Record.expects(:where).returns(mock_relation)
+    mock_relation.expects(:includes).with([:associated_records, :associated, {:record => []}]).returns(stub(:all => [@bob, @joe, @fred]))
+    assert_equal [@bob, @joe, @fred], Record.fetch_multi(@bob.id, @joe.id, @fred.id, {:includes => :record})
+  end
+
   def test_find_batch_coerces_ids_to_primary_key_type
     mock_relation = mock("ActiveRecord::Relation")
     Record.expects(:where).returns(mock_relation)

data/test/helpers/cache.rb

@@ -1,46 +1,10 @@
 module Rails
-  class Cache
 
-    def initialize
-      @cache = {}
-    end
-
-    def fetch(e)
-      if @cache.key?(e)
-        return read(e)
-      else
-        a = yield
-        write(e,a)
-        return a
-      end
-    end
-
-    def clear
-      @cache.clear
-    end
-
-    def write(a,b)
-      @cache[a] = b
-    end
-
-    def delete(a)
-      @cache.delete(a)
-    end
-
-    def read(a)
-      @cache[a]
-    end
-
-    def read_multi(*keys)
-      keys.reduce({}) do |hash, key|
-        hash[key] = @cache[key]
-        hash
-      end
-    end
+  class Cache < ActiveSupport::Cache::MemCacheStore
   end
 
   def self.cache
-    @@cache ||= Cache.new
+    @@cache ||= Cache.new("localhost:#{$memcached_port}")
   end
 
   class Logger

data/test/helpers/database_connection.rb

@@ -1,5 +1,6 @@
 module DatabaseConnection
   def self.setup
+    DATABASE_CONFIG['port'] ||= $mysql_port
     ActiveRecord::Base.establish_connection(DATABASE_CONFIG)
     ActiveRecord::Base.connection
   rescue

data/test/recursive_denormalized_has_many_test.rb

@@ -55,13 +55,16 @@ class RecursiveDenormalizedHasManyTest < IdentityCache::TestCase
   end
 
   def test_on_cache_miss_child_record_fetch_should_include_nested_associations_to_avoid_n_plus_ones
-    record_from_cache_miss = Record.fetch(@record.id)
-    expected = @associated_record.deeply_associated_records
-    
-    assert record_from_cache_miss.fetch_associated_records[0].deeply_associated_records.loaded?
-    assert record_from_cache_miss.fetch_associated_records[1].deeply_associated_records.loaded?
+    assert_queries(5) do
+      # one for the top level record
+      # one for the mid level has_many association
+      # one for the mid level has_one association
+      # one for the deep level level has_many on the mid level has_many association
+      # one for the deep level level has_many on the mid level has_one association
+      record_from_cache_miss = Record.fetch(@record.id)
+    end
   end
-  
+
   def test_saving_child_record_should_expire_parent_record
     IdentityCache.cache.expects(:delete).with(@record.primary_cache_index_key)
     IdentityCache.cache.expects(:delete).with(@associated_record.primary_cache_index_key)

data/test/schema_change_test.rb

@@ -0,0 +1,88 @@
+require "test_helper"
+
+class SchemaChangeTest < IdentityCache::TestCase
+  class AddColumnToChild < ActiveRecord::Migration
+    def up
+      add_column :associated_records, :shiny, :string
+    end
+  end
+
+  class AddColumnToDeepChild < ActiveRecord::Migration
+    def up
+      add_column :deeply_associated_records, :new_column, :string
+    end
+  end
+
+  def setup
+    super
+    ActiveRecord::Migration.verbose = false
+
+    read_new_schema
+    Record.cache_has_one :associated, :embed => true
+    AssociatedRecord.cache_has_many :deeply_associated_records, :embed => true
+
+    @associated_record = AssociatedRecord.new(:name => 'bar')
+    @deeply_associated_record = DeeplyAssociatedRecord.new(:name => "corge")
+    @associated_record.deeply_associated_records << @deeply_associated_record
+    @associated_record.deeply_associated_records << DeeplyAssociatedRecord.new(:name => "qux")
+    @record = Record.new(:title => 'foo')
+    @record.associated = @associated_record
+
+    @associated_record.save!
+    @record.save!
+
+    @record.reload
+  end
+
+  # This helper simulates the models being reloaded
+  def read_new_schema
+    AssociatedRecord.reset_column_information
+    DeeplyAssociatedRecord.reset_column_information
+
+    AssociatedRecord.embedded_schema_hashes = {}
+    Record.embedded_schema_hashes = {}
+  end
+
+  def test_schema_changes_on_embedded_association_when_the_cached_object_is_already_in_the_cache_should_request_from_the_db
+    record = Record.fetch(@record.id)
+    record.fetch_associated
+
+    AddColumnToChild.new.up
+    read_new_schema
+
+    # Reloading the association queries
+    # SHOW FULL FIELDS FROM `associated_records`
+    # SHOW TABLES LIKE 'associated_records'
+    # SELECT  `associated_records`.* FROM `associated_records`  WHERE `associated_records`.`record_id` = 1 ORDER BY id ASC LIMIT 1.
+    assert_queries(3) do
+      assert_nothing_raised { record.fetch_associated.shiny }
+    end
+
+    assert_no_queries { record.fetch_associated.shiny }
+  end
+
+  def test_schema_changes_on_deeply_embedded_association_when_the_cached_object_is_already_in_the_cache_should_request_from_the_db
+    record = Record.fetch(@record.id)
+    associated_record_from_cache = record.fetch_associated
+    associated_record_from_cache.fetch_deeply_associated_records
+
+    AddColumnToDeepChild.new.up
+    read_new_schema
+
+    # Loading association queries
+    # SHOW FULL FIELDS FROM `deeply_associated_records`
+    # SHOW FULL FIELDS FROM `associated_records`
+    # SHOW TABLES LIKE 'deeply_associated_records'
+    # SELECT `deeply_associated_records`.* FROM `deeply_associated_records` WHERE `deeply_associated_records`.`associated_record_id` = 1 ORDER BY name DESC.
+    assert_queries(4) do
+      assert_nothing_raised do
+        associated_record_from_cache.fetch_deeply_associated_records.map(&:new_column)
+      end
+    end
+
+    assert_no_queries do
+      associated_record_from_cache.fetch_deeply_associated_records.each{ |obj| assert_nil obj.new_column }
+      record.fetch_associated.fetch_deeply_associated_records.each{ |obj| assert_nil obj.new_column }
+    end
+  end
+end

data/test/test_helper.rb

@@ -6,6 +6,14 @@ require 'helpers/database_connection'
 
 require File.dirname(__FILE__) + '/../lib/identity_cache'
 
+if ENV['BOXEN_HOME'].present?
+  $memcached_port = 21211
+  $mysql_port = 13306
+else
+  $memcached_port = 11211
+  $mysql_port = 3306
+end
+
 DatabaseConnection.setup
 
 class IdentityCache::TestCase < MiniTest::Unit::TestCase
@@ -35,6 +43,21 @@ class IdentityCache::TestCase < MiniTest::Unit::TestCase
     assert *args
   end
 
+  def assert_queries(num = 1)
+    counter = SQLCounter.new
+    subscriber = ActiveSupport::Notifications.subscribe('sql.active_record', counter)
+      yield
+    ensure
+      ActiveSupport::Notifications.unsubscribe(subscriber)
+      assert_equal num, counter.log.size, "#{counter.log.size} instead of #{num} queries were executed.#{counter.log.size == 0 ? '' : "\nQueries:\n#{counter.log.join("\n")}"}"
+  end
+
+  def assert_no_queries
+    assert_queries(0) do
+      yield
+    end
+  end
+
   def cache_hash(key)
     CityHash.hash64(key)
   end
@@ -71,3 +94,29 @@ class IdentityCache::TestCase < MiniTest::Unit::TestCase
     }
   end
 end
+
+class SQLCounter
+  cattr_accessor :ignored_sql
+  self.ignored_sql = [/^PRAGMA (?!(table_info))/, /^SELECT currval/, /^SELECT CAST/, /^SELECT @@IDENTITY/, /^SELECT @@ROWCOUNT/, /^SAVEPOINT/, /^ROLLBACK TO SAVEPOINT/, /^RELEASE SAVEPOINT/, /^SHOW max_identifier_length/, /^BEGIN/, /^COMMIT/]
+
+  # FIXME: this needs to be refactored so specific database can add their own
+  # ignored SQL.  This ignored SQL is for Oracle.
+  ignored_sql.concat [/^select .*nextval/i, /^SAVEPOINT/, /^ROLLBACK TO/, /^\s*select .* from all_triggers/im]
+
+  attr_reader :ignore
+  attr_accessor :log
+
+  def initialize(ignore = self.class.ignored_sql)
+    @ignore   = ignore
+    @log = []
+  end
+
+  def call(name, start, finish, message_id, values)
+    sql = values[:sql]
+
+    # FIXME: this seems bad. we should probably have a better way to indicate
+    # the query was cached
+    return if 'CACHE' == values[:name] || ignore.any? { |x| x =~ sql }
+    self.log << sql
+  end
+end

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: identity_cache
 version: !ruby/object:Gem::Version
-  version: 0.0.1
   prerelease: 
+  version: 0.0.2
 platform: ruby
 authors:
 - Camilo Lopez
@@ -13,63 +13,152 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-17 00:00:00.000000000 Z
+date: 2013-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  prerelease: false
   name: ar_transaction_changes
-  requirement: &70221938344660 !ruby/object:Gem::Requirement
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - =
+    - - '='
       - !ruby/object:Gem::Version
         version: 0.0.1
+    none: false
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  name: activerecord
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.2.13
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.2.13
+    none: false
+- !ruby/object:Gem::Dependency
   prerelease: false
-  version_requirements: *70221938344660
+  name: activesupport
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.2.13
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.2.13
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
   name: cityhash
-  requirement: &70221938344160 !ruby/object:Gem::Requirement
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.6.0
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - =
+    - - '='
       - !ruby/object:Gem::Version
         version: 0.6.0
-  type: :runtime
+    none: false
+- !ruby/object:Gem::Dependency
   prerelease: false
-  version_requirements: *70221938344160
+  name: memcache-client
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
   name: rake
-  requirement: &70221938343780 !ruby/object:Gem::Requirement
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *70221938343780
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
   name: mocha
-  requirement: &70221938343320 !ruby/object:Gem::Requirement
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *70221938343320
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
   name: mysql2
-  requirement: &70221938342900 !ruby/object:Gem::Requirement
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
     none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+    none: false
+- !ruby/object:Gem::Dependency
   prerelease: false
-  version_requirements: *70221938342900
+  name: debugger
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
 description: Opt in read through ActiveRecord caching.
 email:
 - harry.brundage@shopify.com
@@ -79,6 +168,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .travis.yml
+- CHANGELOG
 - Gemfile
 - LICENSE
 - README.md
@@ -89,6 +179,7 @@ files:
 - lib/identity_cache/version.rb
 - lib/memoized_cache_proxy.rb
 - test/attribute_cache_test.rb
+- test/cache_fetch_includes_test.rb
 - test/denormalized_has_many_test.rb
 - test/denormalized_has_one_test.rb
 - test/fetch_multi_test.rb
@@ -102,6 +193,7 @@ files:
 - test/normalized_has_many_test.rb
 - test/recursive_denormalized_has_many_test.rb
 - test/save_test.rb
+- test/schema_change_test.rb
 - test/test_helper.rb
 homepage: https://github.com/Shopify/identity_cache
 licenses: []
@@ -110,26 +202,26 @@ rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
       segments:
       - 0
-      hash: 460007319304896635
-required_rubygems_version: !ruby/object:Gem::Requirement
+      hash: -901854368761295247
+      version: '0'
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
       segments:
       - 0
-      hash: 460007319304896635
+      hash: -901854368761295247
+      version: '0'
+  none: false
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: IdentityCache lets you specify how you want to cache your model objects,
@@ -138,6 +230,7 @@ summary: IdentityCache lets you specify how you want to cache your model objects
   database is only hit when a copy of the object cannot be found in Memcached.
 test_files:
 - test/attribute_cache_test.rb
+- test/cache_fetch_includes_test.rb
 - test/denormalized_has_many_test.rb
 - test/denormalized_has_one_test.rb
 - test/fetch_multi_test.rb
@@ -151,4 +244,5 @@ test_files:
 - test/normalized_has_many_test.rb
 - test/recursive_denormalized_has_many_test.rb
 - test/save_test.rb
+- test/schema_change_test.rb
 - test/test_helper.rb