identity_cache 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: ba4de54fb34aecd2a44702ea1bb950c718cdc715
-  data.tar.gz: 752b53a7d7399f5273fd7083152c4b28ee5cbdeb
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YmQyZjllMTVlOTNiMmUxMDkxMzJlNzYzYjY0ODZlMmQ4Y2E0YTNkNA==
+  data.tar.gz: !binary |-
+    OGQ2MDc5Zjk2NmY3MTcwMmIxNDkzYjczNTZjMmJhNjMwMWI1ZjJjYg==
 SHA512:
-  metadata.gz: ffa0486c327789a3d975ea768ffe63ed73878f4916aa54b24fa06c1771ece940b4e8bd67a2b8838e224d996a9088b2bd6696237209040608b30c10ed28bb0438
-  data.tar.gz: 903fc321ed4092d797d230302cfb8a09bfb3a4f47e58f0e67b605391ca0ea6dbc5916dd801ed06ee58e65ffae8793e6f25d8f7e94a875d2dd35d1fa427e62825
+  metadata.gz: !binary |-
+    MmM4NmJmMTY2MDRkNDdkYzRjMDg3NTQ4OTRlMzgyOWJiYzY4Y2RjMDQxZDQ0
+    NDA2NzYwZGIyN2UzMjVkNGU1MTI1ZTViNDU4YThkNTAxNGYyY2UxMWU5Y2U4
+    MzhjN2JjMDkyMWVmYjhkMWRmMmIyMmVmZjI4NzJkYTExZGZiYmM=
+  data.tar.gz: !binary |-
+    Yjg4MTdkOGI0OTJmY2ZkMWEwMDY0NjViMDE2MjdiMDg4NTkzZTc0MmU1YjEy
+    NzI1Nzc0ZjQ0ODk1YWY0M2ExMDdjNWNjMjQ5OGY3Y2YyMTcyNzQ3MjA3OTY3
+    YWMxMTlkZTgxNjQ0OTgyYmY3YzY5OWYzY2QyY2NlZjZkNzUwYTQ=

data/.travis.yml

@@ -2,10 +2,12 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-
+  - jruby-19mode
 services:
   - memcache
   - mysql
-
+matrix:
+  allow_failures:
+    - rvm: jruby-19mode
 before_script:
   - mysql -e 'create database identity_cache_test'

data/CHANGELOG

@@ -1,3 +1,8 @@
+0.0.4
+* Fix: only marshal attributes, embedded associations and normalized association IDs
+* Add cache version number to cache keys
+* Add test case to ensure version number is updated when the marshalled format changes
+
 0.0.3
 * Fix: memoization for multi hits actually work
 * Fix: quotes SELECT projection elements on cache misses

data/README.md

@@ -11,6 +11,7 @@ Add this line to your application's Gemfile:
 
 ```ruby
 gem 'identity_cache'
+gem 'cityhash'        # optional, for faster hashing (C-Ruby only)
 ```
 
 And then execute:
@@ -205,12 +206,18 @@ class ApplicationController < ActionController::Base
 end
 ```
 
+## Versioning
+
+Cache keys include a version number by default, specified in `IdentityCache::CACHE_VERSION`. This version number is updated whenever the storage format for cache values is modified. If you modify the cache value format, you must run `rake update_serialization_format` in order to pass the unit tests, and include the modified `test/fixtures/serialized_record` file in your pull request.
+
 ## Caveats
 
 A word of warning. Some versions of rails will silently rescue all exceptions in `after_commit` hooks. If an `after_commit` fails before the cache expiry `after_commit` the cache will not be expired and you will be left with stale data.
 
 Since everything is being marshalled and unmarshalled from Memcached changing Ruby or Rails versions could mean your objects cannot be unmarshalled from Memcached. There are a number of ways to get around this such as namespacing keys when you upgrade or rescuing marshal load errors and treating it as a cache miss. Just something to be aware of if you are using IdentityCache and upgrade Ruby or Rails.
 
+IdentityCache is also very much _opt-in_ by deliberate design. This means IdentityCache does not mess with the way normal Rails associations work, and including it in a model won't change any clients of that model until you switch them to use `fetch` instead of `find`. This is because there is no way IdentityCache is ever going to be 100% consistent. Processes die, execeptions happen, and network blips occur, which means there is a chance that some database transaction might commit but the corresponding memcached DEL operation does not make it. This means that you need to think carefully about when you use `fetch` and when you use `find`. For example, at Shopify, we never use any `fetch`ers on the path which moves money around, because IdentityCache could simply be wrong, and we want to charge people the right amount of money. We do however use the fetchers on performance critical paths where absolute correctness isn't the most important thing, and this is what IdentityCache is intended for.
+
 ## Contributing
 
 Caching is hard. Chances are that if some feature was left out, it was left out on purpose because it didn't make sense to cache in that way. This is used in production at Shopify so we are very opinionated about the types of features we're going to add. Please start the discussion early, before even adding code, so that we can talk about the feature you are proposing and decide if it makes sense in IdentityCache.
@@ -236,4 +243,5 @@ Tom Burns (@boourns)
 Harry Brundage (@hornairs)  
 Dylan Smith (@dylanahsmith)  
 Tobias Lütke (@tobi)  
-John Duff (@jduff)
+John Duff (@jduff)  
+Francis Bogsany (@fbogsany)

data/Rakefile

@@ -15,6 +15,11 @@ Rake::TestTask.new(:test) do |t|
   t.verbose = true
 end
 
+desc 'Update serialization format test fixture.'
+task :update_serialization_format do
+  ruby './test/helpers/update_serialization_format.rb'
+end
+
 namespace :benchmark do
   desc "Run the identity cache CPU benchmark"
   task :cpu do
@@ -32,4 +37,3 @@ namespace :profile do
     ruby "./performance/profile.rb"
   end
 end
-

data/identity_cache.gemspec

@@ -17,13 +17,21 @@ Gem::Specification.new do |gem|
 
 
   gem.add_dependency('ar_transaction_changes', '0.0.1')
-  gem.add_dependency('activerecord', '3.2.13')
-  gem.add_dependency('activesupport', '3.2.13')
-  gem.add_dependency('cityhash', '0.6.0')
+  gem.add_dependency('activerecord', '~> 3.2.12')
+  gem.add_dependency('activesupport', '~> 3.2.12')
+
   gem.add_development_dependency('memcache-client')
   gem.add_development_dependency('rake')
   gem.add_development_dependency('mocha', '0.14.0')
-  gem.add_development_dependency('mysql2')
-  gem.add_development_dependency('debugger')
-  gem.add_development_dependency('ruby-prof')
+
+  if RUBY_PLATFORM == 'java'
+    gem.add_development_dependency 'jruby-openssl'
+    gem.add_development_dependency 'activerecord-jdbcmysql-adapter'
+    gem.add_development_dependency 'jdbc-mysql'
+  else
+    gem.add_development_dependency('debugger')
+    gem.add_development_dependency('ruby-prof')
+    gem.add_development_dependency('cityhash', '0.6.0')
+    gem.add_development_dependency('mysql2')
+  end
 end

data/lib/identity_cache.rb

@@ -1,5 +1,6 @@
-require 'cityhash'
+require 'active_record'
 require 'ar_transaction_changes'
+
 require "identity_cache/version"
 require 'identity_cache/memoized_cache_proxy'
 require 'identity_cache/belongs_to_caching'
@@ -7,6 +8,7 @@ require 'identity_cache/cache_key_generation'
 require 'identity_cache/configuration_dsl'
 require 'identity_cache/parent_model_expiration'
 require 'identity_cache/query_api'
+require "identity_cache/cache_hash"
 
 module IdentityCache
   CACHED_NIL = :idc_cached_nil
@@ -19,9 +21,20 @@ module IdentityCache
   end
 
   class << self
+    include IdentityCache::CacheHash
+
+    attr_accessor :readonly
+    attr_writer :logger
 
-    attr_accessor :logger, :readonly
-    attr_reader :cache
+    def included(base) #:nodoc:
+      raise AlreadyIncludedError if base.respond_to? :cache_indexes
+
+      base.send(:include, ArTransactionChanges) unless base.include?(ArTransactionChanges)
+      base.send(:include, IdentityCache::BelongsToCaching)
+      base.send(:include, IdentityCache::CacheKeyGeneration)
+      base.send(:include, IdentityCache::ConfigurationDSL)
+      base.send(:include, IdentityCache::QueryAPI)
+    end
 
     # Sets the cache adaptor IdentityCache will be using
     #
@@ -30,7 +43,7 @@ module IdentityCache
     # +cache_adaptor+ - A ActiveSupport::Cache::Store
     #
     def cache_backend=(cache_adaptor)
-      cache.memcache = cache_adaptor
+      cache.cache_backend = cache_adaptor
     end
 
     def cache
@@ -73,7 +86,6 @@ module IdentityCache
       value.nil? ? IdentityCache::CACHED_NIL : value
     end
 
-
     def unmap_cached_nil_for(value)
       value == IdentityCache::CACHED_NIL ? nil : value
     end
@@ -113,34 +125,5 @@ module IdentityCache
 
       result
     end
-
-    def schema_to_string(columns)
-      columns.sort_by(&:name).map{|c| "#{c.name}:#{c.type}"}.join(',')
-    end
-
-    def denormalized_schema_hash(klass)
-      schema_string = schema_to_string(klass.columns)
-      if klass.respond_to?(:all_cached_associations_needing_population) && !(embeded_associations = klass.all_cached_associations_needing_population).empty?
-        embedded_schema = embeded_associations.map do |name, options|
-          "#{name}:(#{denormalized_schema_hash(options[:association_class])})"
-        end.sort.join(',')
-        schema_string << "," << embedded_schema
-      end
-      IdentityCache.memcache_hash(schema_string)
-    end
-
-    def included(base) #:nodoc:
-      raise AlreadyIncludedError if base.respond_to? :cache_indexes
-
-      base.send(:include, ArTransactionChanges) unless base.include?(ArTransactionChanges)
-      base.send(:include, IdentityCache::BelongsToCaching)
-      base.send(:include, IdentityCache::CacheKeyGeneration)
-      base.send(:include, IdentityCache::ConfigurationDSL)
-      base.send(:include, IdentityCache::QueryAPI)
-    end
-
-    def memcache_hash(key) #:nodoc:
-      CityHash.hash64(key)
-    end
   end
 end

data/lib/identity_cache/cache_hash.rb

@@ -0,0 +1,36 @@
+# Use CityHash for fast hashing if it is available; use Digest::MD5 otherwise
+begin
+  require 'cityhash'
+rescue LoadError
+  unless RUBY_PLATFORM == 'java'
+    warn <<-NOTICE
+      ** Notice: CityHash was not loaded. **
+
+      For optimal performance, use of the cityhash gem is recommended.
+
+      Run the following command, or add it to your Gemfile:
+
+        gem install cityhash
+    NOTICE
+  end
+
+  require 'digest/md5'
+end
+
+module IdentityCache
+  module CacheHash
+
+    if defined?(CityHash)
+      
+      def memcache_hash(key) #:nodoc:
+        CityHash.hash64(key)
+      end
+    else
+      
+      def memcache_hash(key) #:nodoc:
+        a = Digest::MD5.digest(key).unpack('LL')
+        (a[0] << 32) | a[1]
+      end
+    end
+  end
+end

data/lib/identity_cache/cache_key_generation.rb

@@ -1,26 +1,54 @@
 module IdentityCache
   module CacheKeyGeneration
     extend ActiveSupport::Concern
+    DEFAULT_NAMESPACE = "IDC:#{CACHE_VERSION}:".freeze
+
+    def self.schema_to_string(columns)
+      columns.sort_by(&:name).map{|c| "#{c.name}:#{c.type}"}.join(',')
+    end
+
+    def self.denormalized_schema_hash(klass)
+      schema_string = schema_to_string(klass.columns)
+      if !(associations = embedded_associations(klass)).empty?
+        embedded_schema = associations.map do |name, options|
+          "#{name}:(#{denormalized_schema_hash(options[:association_class])})"
+        end.sort.join(',')
+        schema_string << "," << embedded_schema
+      end
+      IdentityCache.memcache_hash(schema_string)
+    end
+
+    def self.embedded_associations(klass)
+      if klass.respond_to?(:all_cached_associations_needing_population)
+        klass.all_cached_associations_needing_population
+      else
+        {}
+      end
+    end
 
     module ClassMethods
       def rails_cache_key(id)
-        rails_cache_key_prefix + id.to_s
+        "#{rails_cache_key_prefix}#{id}"
       end
 
       def rails_cache_key_prefix
-        @rails_cache_key_prefix ||= begin
-          "IDC:blob:#{base_class.name}:#{IdentityCache.denormalized_schema_hash(self)}:"
-        end
+        @rails_cache_key_prefix ||= IdentityCache::CacheKeyGeneration.denormalized_schema_hash(self)
+        "#{rails_cache_key_namespace}blob:#{base_class.name}:#{@rails_cache_key_prefix}:"
       end
 
       def rails_cache_index_key_for_fields_and_values(fields, values)
-        "IDC:index:#{base_class.name}:#{rails_cache_string_for_fields_and_values(fields, values)}"
+        "#{rails_cache_key_namespace}index:#{base_class.name}:#{rails_cache_string_for_fields_and_values(fields, values)}"
       end
 
       def rails_cache_key_for_attribute_and_fields_and_values(attribute, fields, values)
-        "IDC:attribute:#{base_class.name}:#{attribute}:#{rails_cache_string_for_fields_and_values(fields, values)}"
+        "#{rails_cache_key_namespace}attribute:#{base_class.name}:#{attribute}:#{rails_cache_string_for_fields_and_values(fields, values)}"
+      end
+
+      def rails_cache_key_namespace
+        DEFAULT_NAMESPACE
       end
 
+      private
       def rails_cache_string_for_fields_and_values(fields, values)
         "#{fields.join('/')}:#{IdentityCache.memcache_hash(values.join('/'))}"
       end

data/lib/identity_cache/configuration_dsl.rb

@@ -183,7 +183,7 @@ module IdentityCache
       end
 
       def identity_cache_single_value_dynamic_fetcher(fields, values) # :nodoc:
-        sql_on_miss = "SELECT `id` FROM `#{table_name}` WHERE #{identity_cache_sql_conditions(fields, values)} LIMIT 1"
+        sql_on_miss = "SELECT #{quoted_primary_key} FROM #{quoted_table_name} WHERE #{identity_cache_sql_conditions(fields, values)} LIMIT 1"
         cache_key = rails_cache_index_key_for_fields_and_values(fields, values)
         id = IdentityCache.fetch(cache_key) { connection.select_value(sql_on_miss) }
         unless id.nil?
@@ -195,7 +195,7 @@ module IdentityCache
       end
 
       def identity_cache_multiple_value_dynamic_fetcher(fields, values) # :nodoc
-        sql_on_miss = "SELECT `id` FROM `#{table_name}` WHERE #{identity_cache_sql_conditions(fields, values)}"
+        sql_on_miss = "SELECT #{quoted_primary_key} FROM #{quoted_table_name} WHERE #{identity_cache_sql_conditions(fields, values)}"
         cache_key = rails_cache_index_key_for_fields_and_values(fields, values)
         ids = IdentityCache.fetch(cache_key) { connection.select_values(sql_on_miss) }
 
@@ -267,8 +267,7 @@ module IdentityCache
 
       def attribute_dynamic_fetcher(attribute, fields, values) #:nodoc:
         cache_key = rails_cache_key_for_attribute_and_fields_and_values(attribute, fields, values)
-        sql_on_miss = "SELECT `#{attribute}` FROM `#{table_name}` WHERE #{identity_cache_sql_conditions(fields, values)} LIMIT 1"
-
+        sql_on_miss = "SELECT #{connection.quote_column_name(attribute)} FROM #{quoted_table_name} WHERE #{identity_cache_sql_conditions(fields, values)} LIMIT 1"
         IdentityCache.fetch(cache_key) { connection.select_value(sql_on_miss) }
       end
 
@@ -278,7 +277,6 @@ module IdentityCache
         raise InverseAssociationError unless child_association
         foreign_key = child_association.association_foreign_key
         parent_class ||= self.name
-        new_parent = options[:inverse_name]
 
         child_class.send(:include, ArTransactionChanges) unless child_class.include?(ArTransactionChanges)
         child_class.send(:include, ParentModelExpiration) unless child_class.include?(ParentModelExpiration)
@@ -294,7 +292,7 @@ module IdentityCache
       end
 
       def identity_cache_sql_conditions(fields, values)
-        fields.each_with_index.collect { |f, i| "`#{f}` = #{quote_value(values[i])}" }.join(" AND ")
+        fields.each_with_index.collect { |f, i| "#{connection.quote_column_name(f)} = #{quote_value(values[i])}" }.join(" AND ")
       end
     end
   end

data/lib/identity_cache/memoized_cache_proxy.rb

@@ -2,10 +2,10 @@ require 'monitor'
 
 module IdentityCache
   class MemoizedCacheProxy
-    attr_writer :memcache
+    attr_accessor :cache_backend
 
-    def initialize(memcache = nil)
-      @memcache = memcache || Rails.cache
+    def initialize(cache_backend = nil)
+      @cache_backend = cache_backend || Rails.cache
       @key_value_maps = Hash.new {|h, k| h[k] = {} }
     end
 
@@ -23,27 +23,27 @@ module IdentityCache
 
     def write(key, value)
       memoized_key_values[key] = value if memoizing?
-      @memcache.write(key, value)
+      @cache_backend.write(key, value)
     end
 
     def read(key)
-      used_memcached = true
+      used_cache_backend = true
 
       result = if memoizing?
-        used_memcached = false
+        used_cache_backend = false
         mkv = memoized_key_values
 
         mkv.fetch(key) do
-          used_memcached = true
-          mkv[key] = @memcache.read(key)
+          used_cache_backend = true
+          mkv[key] = @cache_backend.read(key)
         end
 
       else
-        @memcache.read(key)
+        @cache_backend.read(key)
       end
 
       if result
-        IdentityCache.logger.debug { "[IdentityCache] #{ used_memcached ? '(memcache)'  : '(memoized)'  } cache hit for #{key}" }
+        IdentityCache.logger.debug { "[IdentityCache] #{ used_cache_backend ? '(cache_backend)'  : '(memoized)'  } cache hit for #{key}" }
       else
         IdentityCache.logger.debug { "[IdentityCache] cache miss for #{key}" }
       end
@@ -53,13 +53,15 @@ module IdentityCache
 
     def delete(key)
       memoized_key_values.delete(key) if memoizing?
-      @memcache.delete(key)
+      result = @cache_backend.delete(key)
+      IdentityCache.logger.debug { "[IdentityCache] delete #{ result ? 'hit'  : 'miss'  } for #{key}" }
+      result
     end
 
     def read_multi(*keys)
 
       if IdentityCache.logger.debug?
-        memoized_keys , memcache_keys = [], []
+        memoized_keys , cache_backend_keys = [], []
       end
 
       result = if memoizing?
@@ -75,7 +77,7 @@ module IdentityCache
           end
         end
 
-        hits =   missing_keys.empty? ? {} : @memcache.read_multi(*missing_keys)
+        hits =   missing_keys.empty? ? {} : @cache_backend.read_multi(*missing_keys)
 
         missing_keys.each do |key|
           hit = hits[key]
@@ -84,17 +86,17 @@ module IdentityCache
         end
         hash
       else
-        @memcache.read_multi(*keys)
+        @cache_backend.read_multi(*keys)
       end
 
       if IdentityCache.logger.debug?
 
         result.each do |k, v|
-          memcache_keys << k if !v.nil? && !memoized_keys.include?(k)
+          cache_backend_keys << k if !v.nil? && !memoized_keys.include?(k)
         end
 
         memoized_keys.each{ |k| IdentityCache.logger.debug "[IdentityCache] (memoized) cache hit for #{k} (multi)" }
-        memcache_keys.each{ |k| IdentityCache.logger.debug "[IdentityCache] (memcache) cache hit for #{k} (multi)" }
+        cache_backend_keys.each{ |k| IdentityCache.logger.debug "[IdentityCache] (cache_backend) cache hit for #{k} (multi)" }
       end
 
       result
@@ -102,7 +104,7 @@ module IdentityCache
 
     def clear
       clear_memoization
-      @memcache.clear
+      @cache_backend.clear
     end
 
     private