air18n 0.1.32 → 0.1.33

data/.gitignore

@@ -17,3 +17,4 @@ test/tmp
 test/version_tmp
 tmp
 .rvmrc
+*.sw[op]

data/air18n.gemspec

@@ -10,6 +10,7 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency 'i18n', '>= 0.5.0'
   gem.add_runtime_dependency 'activerecord', '~> 3.0'
+  gem.add_runtime_dependency 'activesupport', '~> 3.0'
   gem.add_development_dependency "rspec"
   gem.add_development_dependency 'sqlite3'
   gem.add_development_dependency 'guard'

data/lib/air18n.rb

@@ -54,6 +54,9 @@ module Air18n
         @distance_unit = 'KM'
         attr_reader :distance_unit
 
+        # Cache used to store translation data from the database.
+        attr_accessor :cache
+
         # Whether to wrap phrases in HTML that allows them to be translated by
         # the user from directly on the page.
         @contextual_translation = false

data/lib/air18n/backend.rb

@@ -4,6 +4,7 @@ require 'air18n/logging_helper'
 require 'air18n/prim_and_proper'
 require 'air18n/pseudo_locales'
 require 'air18n/smart_count'
+require 'air18n/chunk_cache'
 
 module Air18n
   class Backend
@@ -17,10 +18,19 @@ module Air18n
     #   be able to guard against flipflops.
     attr_accessor :translation_data, :phrase_screenshots, :default_text_change_observer
 
+    # Define constants used as cache keys
+    T_LAST_LOADED_AT = 'Air18n::translations_last_loaded_at_%s'
+    T_LAST_UPDATED_AT = 'Air18n::translations_last_updated_at_%s'
+    T_DATA = 'Air18n::translation_data_%s'
+    # This value allows one app instance to make a DB call while other still
+    # pull slightly stale data from the cache. The unit is seconds.
+    RACE_CONDITION_TTL = 5
+
     # Stores translations for a given locale.
     def store_translations locale, data, options = {}
       @translation_data ||= {}
       @translation_data[locale.to_sym] = data
+      ChunkCache::set(I18n.cache, T_DATA % locale, data, 1.day) if I18n.cache
     end
 
     def available_locales
@@ -66,7 +76,7 @@ module Air18n
     def init_translations(locale)
       reset_phrase_screenshots unless @phrase_screenshots
       if @translation_data.nil? || !@translation_data.include?(locale)
-        reload_translations([locale])
+        get_from_cache_or_reload(locale)
       end
     end
 
@@ -74,6 +84,39 @@ module Air18n
       @phrase_screenshots = PhraseScreenshot.all_phrase_urls
     end
 
+    def check_last_timestamps(locale)
+      # Potential race condition here but the order of operations will at worst
+      # cause an extra DB call.
+      if I18n.cache
+        translation_last_loaded_at = I18n.cache.read(T_LAST_LOADED_AT % locale)
+        last_updated_at = I18n.cache.read(T_LAST_UPDATED_AT % locale)
+      else
+        translation_last_loaded_at = @translations_last_loaded_at[locale]
+        last_updated_at ||= Time.now
+      end
+      [translation_last_loaded_at, last_updated_at]
+    end
+
+    def get_from_cache_or_reload(locale)
+      # This function will check the cache if available to see if there is data.
+      # If it is available, read it and store it in a instance variable.
+      # If it is not available, increment the last_loaded_at cache key by
+      # a few seconds in the future. (This will prevent the majority of application
+      # processes from doing a database lookup by serving them slightly stale data
+      # from the cache. If multiple instances start up when the cache is empty, then multiple 
+      # call to the database are unavoidable unless a locking and wait style is used.
+      # Refer to the specs 'I18n cache' for more information.
+      cache_results = nil
+      cache_results = ChunkCache::get(I18n.cache, T_DATA % locale) if I18n.cache
+      if !cache_results.nil?
+        @translation_data ||= {}
+        @translation_data[locale.to_sym] = cache_results
+      else
+        I18n.cache.write(T_LAST_LOADED_AT % locale, Time.now) if I18n.cache
+        reload_translations([locale])
+      end
+    end
+
     def check_for_new_translations(locale)
       # When TranslateController makes a new translation, it sets
       # translations_last_updated_at to the current time in the cache.
@@ -81,10 +124,12 @@ module Air18n
 
       # No-op if locale is the default locale; its translations never change.
       if locale != I18n.default_locale
-        # TODO(jkb, sri) Add a layer of abstraction between air18n and Rails.cache.
-        last_updated_at = Rails.cache.read("#{locale}_translations_last_updated_at")
-        if last_updated_at && last_updated_at.to_i > @translations_last_loaded_at[locale].to_i
+        translation_last_loaded_at, last_updated_at = check_last_timestamps(locale)
+        if (translation_last_loaded_at.nil?) || (last_updated_at && last_updated_at.to_i >= translation_last_loaded_at.to_i)
+          I18n.cache.write(T_LAST_LOADED_AT % locale, Time.now + RACE_CONDITION_TTL) if I18n.cache
           reload_translations([locale])
+        else
+          get_from_cache_or_reload(locale)
         end
       end
     end

data/lib/air18n/chunk_cache.rb

@@ -0,0 +1,61 @@
+require 'json'
+require 'zlib'
+
+module Air18n
+  module ChunkCache
+    # This class augments the caching layer by splitting up (into chunks)
+    # values > 1MB.
+
+    # Define constants and cache keys.
+    #
+    # We divide things into chunks of 900000 bytes to accomodate memcaches with
+    # 1mb max value size.
+    MAX_CHUNK_SIZE = 900000
+    CHECKSUM_CACHE_KEY = "%s_0_nchunks+checksum"
+    CHUNK_CACHE_KEY = "%s_%d"
+
+    module_function
+
+    def set(cache, key, value, expires_in)
+      # We put the value in an array because plain strings are not valid JSON.
+      value_json = [value].to_json
+
+      # We use divmod to avoid the edge condition where the floating point math
+      # result is 5.000001 and the ceil would result in 6.
+      quotient, modulus = value_json.size.divmod(MAX_CHUNK_SIZE)
+      num_chunks = modulus > 0 ? quotient + 1 : quotient
+      cache.write(CHECKSUM_CACHE_KEY % key, [num_chunks, Zlib::crc32(value_json)].to_json)
+
+      1.upto(num_chunks) do |i|
+        chunk = value_json[(i-1)*MAX_CHUNK_SIZE, MAX_CHUNK_SIZE]
+        result = cache.write(CHUNK_CACHE_KEY % [key, i], chunk, :expires_in => expires_in)
+        raise "ChunkCache set: Failed to write chunk #{i} of #{chunk.length} bytes to cache" unless result
+      end
+    end
+
+    # This returns nil if the chunks are corrupt or missing.
+    def get(cache, key)
+      buffer = ""
+      chunk = cache.read(CHECKSUM_CACHE_KEY % key)
+      if chunk
+        num_chunks, checksum = JSON.parse(chunk)
+      else
+        LoggingHelper.error("ChunkCache get: No checksum stored for #{key}")
+        return nil
+      end
+
+      for i in 1..num_chunks
+        chunk = cache.read(CHUNK_CACHE_KEY % [key, i])
+        buffer << chunk
+      end
+
+      if Zlib::crc32(buffer) != checksum
+        LoggingHelper.error("ChunkCache get: Invalid checksum on #{key}")
+        return nil
+      else
+        return JSON.parse(buffer)[0]
+      end
+    end
+  end
+end
+

data/lib/air18n/class_methods.rb

@@ -192,6 +192,22 @@ module Air18n
       fallbacks_for(the_locale, :exclude_default => true).include?(other_locale)
     end
 
+    # Call this when translations have been updated. Next time
+    # 'check_for_new_translations' is called, it will do a database read 
+    # and update the cache.
+    def mark_translations_as_changed(locale)
+      I18n.cache.write(Air18n::Backend::T_LAST_UPDATED_AT % locale, Time.now) if I18n.cache
+    end
+
+    # This will cause cached translations to be updated and updates the
+    # 'last_loaded_at' timestamp in the cache. WARNING: You may still get stale
+    # data if another app is already in the process of updating a warm but stale
+    # cache.
+    def force_cache_refresh(locale)
+      I18n.mark_translations_as_changed(locale)
+      I18n.check_for_new_translations(locale)
+    end
+
     def check_for_new_translations
       @air18n_backend.check_for_new_translations(I18n.full_locale)
     end
@@ -201,6 +217,7 @@ module Air18n
       reload_translations_in(I18n.full_locale)
     end
 
+    # This is unaware of the cache and ignores it.
     def reload_translations_in(locale)
       @air18n_backend.reload_translations([locale])
     end

data/lib/air18n/phrase_translation.rb

@@ -494,6 +494,7 @@ module Air18n
         phrase_to_phrase_translations = pt_scope.all.group_by { |pt| [pt.locale, pt.phrase_id] }
         phrase_to_phrase_translations.each do |(locale, phrase_id), phrase_translations|
           phrase_translations.sort_by! { |pt| pt.created_at }
+
           previous_translation = PhraseTranslation.where("created_at < ?", phrase_translations.first.created_at - 1.second).
                                                    where(:locale => locale).
                                                    where(:phrase_id => phrase_id).

data/lib/air18n/version.rb

@@ -1,3 +1,3 @@
 module Air18n
-  VERSION = "0.1.32"
+  VERSION = "0.1.33"
 end

data/spec/lib/air18n/backend_spec.rb

@@ -100,6 +100,47 @@ describe Air18n::Backend do
     end
   end
 
+  context 'I18n cache' do
+    before :each do
+      @backend = Air18n::Backend.new
+      I18n.cache = ActiveSupport::Cache::MemoryStore.new
+          end
+
+    after :all do
+      I18n.cache = nil
+    end
+
+    it 'should return stale results from the cache' do
+      @phraseA = FactoryGirl.create(:phrase, :key => 'init, key A', :value => 'value A')
+      @tAs = FactoryGirl.create(:phrase_translation, :phrase => @phraseA, :value => 'Spanish value A', :locale => :es)
+      @tAf = FactoryGirl.create(:phrase_translation, :phrase => @phraseA, :value => 'French value A', :locale => :fr)
+
+      @backend.lookup(:es, @phraseA.key).should == 'Spanish value A'
+      @tAs.update_attributes(:value => 'Updated Spanish value A')
+      @backend.lookup(:es, @phraseA.key, [], :default => @phraseA.value).should == 'Spanish value A'
+      @backend.lookup(:fr, @phraseA.key).should == 'French value A'
+      @backend.lookup(:it, @phraseA.key, [], :default => @phraseA.value).should == 'value A'
+    end
+
+    it 'should check for new translations' do
+      @phraseC = FactoryGirl.create(:phrase, :key => 'init, key C', :value => 'value C')
+      @tCs = FactoryGirl.create(:phrase_translation, :phrase => @phraseC, :value => 'Spanish value C', :locale => :es)
+      @tCf = FactoryGirl.create(:phrase_translation, :phrase => @phraseC, :value => 'French value C', :locale => :fr)
+
+      @backend.lookup(:es, @phraseC.key, [], :default => @phraseC.value).should == 'Spanish value C'
+      @backend.lookup(:fr, @phraseC.key).should == 'French value C'
+      @tCs.update_attributes(:value => 'Newer Spanish value C')
+      @tCf.update_attributes(:value => 'Newer French value C')
+      I18n.mark_translations_as_changed(:es)
+      I18n.mark_translations_as_changed(:fr)
+      @backend.check_for_new_translations(:es)
+      @backend.check_for_new_translations(:fr)
+      @backend.lookup(:es, @phraseC.key).should == 'Newer Spanish value C'
+      @backend.lookup(:fr, @phraseC.key).should == 'Newer French value C'
+      @backend.lookup(:it, @phraseC.key, [], :default => @phraseC.value).should == 'value C'
+    end
+  end
+
   context 'Pseudolocales' do
     it 'should use pseudolocales for xx locale' do
       @backend = Air18n::Backend.new

data/spec/lib/air18n/chunk_cache_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+require 'air18n/chunk_cache'
+
+describe Air18n::ChunkCache do
+  context 'Chunk cache get and set' do
+    before :all do
+      @cache = ActiveSupport::Cache::MemoryStore.new
+    end
+
+    it 'should return nil for non-existant keys' do
+      Air18n::ChunkCache::get(@cache, "ANY_KEY").should == nil
+    end
+
+    it 'should work for short values < 1M' do
+      tiny_string = SecureRandom.urlsafe_base64(5)
+      Air18n::ChunkCache::set(@cache, "TINY", tiny_string, 60)
+      num_chunks, checksum = JSON.parse(@cache.read(Air18n::ChunkCache::CHECKSUM_CACHE_KEY % "TINY"))
+      num_chunks.should == 1
+      Air18n::ChunkCache::get(@cache, "TINY").should == tiny_string
+    end
+
+    it 'should work for values < 1M' do
+      small_string = SecureRandom.urlsafe_base64(5000)
+      Air18n::ChunkCache::set(@cache, "SMALL", small_string, 60)
+      num_chunks, checksum = JSON.parse(@cache.read(Air18n::ChunkCache::CHECKSUM_CACHE_KEY % "SMALL"))
+      num_chunks.should == 1
+      Air18n::ChunkCache::get(@cache, "SMALL").should == small_string
+    end
+
+    it 'should work for values > 1M' do
+      large_string = SecureRandom.urlsafe_base64(3000000)
+      Air18n::ChunkCache::set(@cache, "LARGE", large_string, 60)
+      num_chunks, checksum = JSON.parse(@cache.read(Air18n::ChunkCache::CHECKSUM_CACHE_KEY % "LARGE"))
+      num_chunks.should == 5
+      Air18n::ChunkCache::get(@cache, "LARGE").should == large_string
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: air18n
 version: !ruby/object:Gem::Version
-  version: 0.1.32
+  version: 0.1.33
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-15 00:00:00.000000000 Z
+date: 2012-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: i18n
@@ -47,6 +47,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -147,6 +163,7 @@ files:
 - config/colonial_spelling_variants.yml
 - lib/air18n.rb
 - lib/air18n/backend.rb
+- lib/air18n/chunk_cache.rb
 - lib/air18n/class_methods.rb
 - lib/air18n/default_text_change_observer.rb
 - lib/air18n/less_silly_chain.rb
@@ -174,6 +191,7 @@ files:
 - spec/factories.rb
 - spec/lib/air18n/air18n_spec.rb
 - spec/lib/air18n/backend_spec.rb
+- spec/lib/air18n/chunk_cache_spec.rb
 - spec/lib/air18n/phrase_spec.rb
 - spec/lib/air18n/phrase_translation_spec.rb
 - spec/lib/air18n/prim_and_proper_spec.rb
@@ -210,6 +228,7 @@ test_files:
 - spec/factories.rb
 - spec/lib/air18n/air18n_spec.rb
 - spec/lib/air18n/backend_spec.rb
+- spec/lib/air18n/chunk_cache_spec.rb
 - spec/lib/air18n/phrase_spec.rb
 - spec/lib/air18n/phrase_translation_spec.rb
 - spec/lib/air18n/prim_and_proper_spec.rb