air18n 0.1.32 → 0.1.33

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore CHANGED
@@ -17,3 +17,4 @@ test/tmp
17
17
  test/version_tmp
18
18
  tmp
19
19
  .rvmrc
20
+ *.sw[op]
@@ -10,6 +10,7 @@ Gem::Specification.new do |gem|
10
10
 
11
11
  gem.add_runtime_dependency 'i18n', '>= 0.5.0'
12
12
  gem.add_runtime_dependency 'activerecord', '~> 3.0'
13
+ gem.add_runtime_dependency 'activesupport', '~> 3.0'
13
14
  gem.add_development_dependency "rspec"
14
15
  gem.add_development_dependency 'sqlite3'
15
16
  gem.add_development_dependency 'guard'
@@ -54,6 +54,9 @@ module Air18n
54
54
  @distance_unit = 'KM'
55
55
  attr_reader :distance_unit
56
56
 
57
+ # Cache used to store translation data from the database.
58
+ attr_accessor :cache
59
+
57
60
  # Whether to wrap phrases in HTML that allows them to be translated by
58
61
  # the user from directly on the page.
59
62
  @contextual_translation = false
@@ -4,6 +4,7 @@ require 'air18n/logging_helper'
4
4
  require 'air18n/prim_and_proper'
5
5
  require 'air18n/pseudo_locales'
6
6
  require 'air18n/smart_count'
7
+ require 'air18n/chunk_cache'
7
8
 
8
9
  module Air18n
9
10
  class Backend
@@ -17,10 +18,19 @@ module Air18n
17
18
  # be able to guard against flipflops.
18
19
  attr_accessor :translation_data, :phrase_screenshots, :default_text_change_observer
19
20
 
21
+ # Define constants used as cache keys
22
+ T_LAST_LOADED_AT = 'Air18n::translations_last_loaded_at_%s'
23
+ T_LAST_UPDATED_AT = 'Air18n::translations_last_updated_at_%s'
24
+ T_DATA = 'Air18n::translation_data_%s'
25
+ # This value allows one app instance to make a DB call while other still
26
+ # pull slightly stale data from the cache. The unit is seconds.
27
+ RACE_CONDITION_TTL = 5
28
+
20
29
  # Stores translations for a given locale.
21
30
  def store_translations locale, data, options = {}
22
31
  @translation_data ||= {}
23
32
  @translation_data[locale.to_sym] = data
33
+ ChunkCache::set(I18n.cache, T_DATA % locale, data, 1.day) if I18n.cache
24
34
  end
25
35
 
26
36
  def available_locales
@@ -66,7 +76,7 @@ module Air18n
66
76
  def init_translations(locale)
67
77
  reset_phrase_screenshots unless @phrase_screenshots
68
78
  if @translation_data.nil? || !@translation_data.include?(locale)
69
- reload_translations([locale])
79
+ get_from_cache_or_reload(locale)
70
80
  end
71
81
  end
72
82
 
@@ -74,6 +84,39 @@ module Air18n
74
84
  @phrase_screenshots = PhraseScreenshot.all_phrase_urls
75
85
  end
76
86
 
87
+ def check_last_timestamps(locale)
88
+ # Potential race condition here but the order of operations will at worst
89
+ # cause an extra DB call.
90
+ if I18n.cache
91
+ translation_last_loaded_at = I18n.cache.read(T_LAST_LOADED_AT % locale)
92
+ last_updated_at = I18n.cache.read(T_LAST_UPDATED_AT % locale)
93
+ else
94
+ translation_last_loaded_at = @translations_last_loaded_at[locale]
95
+ last_updated_at ||= Time.now
96
+ end
97
+ [translation_last_loaded_at, last_updated_at]
98
+ end
99
+
100
+ def get_from_cache_or_reload(locale)
101
+ # This function will check the cache if available to see if there is data.
102
+ # If it is available, read it and store it in a instance variable.
103
+ # If it is not available, increment the last_loaded_at cache key by
104
+ # a few seconds in the future. (This will prevent the majority of application
105
+ # processes from doing a database lookup by serving them slightly stale data
106
+ # from the cache. If multiple instances start up when the cache is empty, then multiple
107
+ # call to the database are unavoidable unless a locking and wait style is used.
108
+ # Refer to the specs 'I18n cache' for more information.
109
+ cache_results = nil
110
+ cache_results = ChunkCache::get(I18n.cache, T_DATA % locale) if I18n.cache
111
+ if !cache_results.nil?
112
+ @translation_data ||= {}
113
+ @translation_data[locale.to_sym] = cache_results
114
+ else
115
+ I18n.cache.write(T_LAST_LOADED_AT % locale, Time.now) if I18n.cache
116
+ reload_translations([locale])
117
+ end
118
+ end
119
+
77
120
  def check_for_new_translations(locale)
78
121
  # When TranslateController makes a new translation, it sets
79
122
  # translations_last_updated_at to the current time in the cache.
@@ -81,10 +124,12 @@ module Air18n
81
124
 
82
125
  # No-op if locale is the default locale; its translations never change.
83
126
  if locale != I18n.default_locale
84
- # TODO(jkb, sri) Add a layer of abstraction between air18n and Rails.cache.
85
- last_updated_at = Rails.cache.read("#{locale}_translations_last_updated_at")
86
- if last_updated_at && last_updated_at.to_i > @translations_last_loaded_at[locale].to_i
127
+ translation_last_loaded_at, last_updated_at = check_last_timestamps(locale)
128
+ if (translation_last_loaded_at.nil?) || (last_updated_at && last_updated_at.to_i >= translation_last_loaded_at.to_i)
129
+ I18n.cache.write(T_LAST_LOADED_AT % locale, Time.now + RACE_CONDITION_TTL) if I18n.cache
87
130
  reload_translations([locale])
131
+ else
132
+ get_from_cache_or_reload(locale)
88
133
  end
89
134
  end
90
135
  end
@@ -0,0 +1,61 @@
1
+ require 'json'
2
+ require 'zlib'
3
+
4
+ module Air18n
5
+ module ChunkCache
6
+ # This class augments the caching layer by splitting up (into chunks)
7
+ # values > 1MB.
8
+
9
+ # Define constants and cache keys.
10
+ #
11
+ # We divide things into chunks of 900000 bytes to accomodate memcaches with
12
+ # 1mb max value size.
13
+ MAX_CHUNK_SIZE = 900000
14
+ CHECKSUM_CACHE_KEY = "%s_0_nchunks+checksum"
15
+ CHUNK_CACHE_KEY = "%s_%d"
16
+
17
+ module_function
18
+
19
+ def set(cache, key, value, expires_in)
20
+ # We put the value in an array because plain strings are not valid JSON.
21
+ value_json = [value].to_json
22
+
23
+ # We use divmod to avoid the edge condition where the floating point math
24
+ # result is 5.000001 and the ceil would result in 6.
25
+ quotient, modulus = value_json.size.divmod(MAX_CHUNK_SIZE)
26
+ num_chunks = modulus > 0 ? quotient + 1 : quotient
27
+ cache.write(CHECKSUM_CACHE_KEY % key, [num_chunks, Zlib::crc32(value_json)].to_json)
28
+
29
+ 1.upto(num_chunks) do |i|
30
+ chunk = value_json[(i-1)*MAX_CHUNK_SIZE, MAX_CHUNK_SIZE]
31
+ result = cache.write(CHUNK_CACHE_KEY % [key, i], chunk, :expires_in => expires_in)
32
+ raise "ChunkCache set: Failed to write chunk #{i} of #{chunk.length} bytes to cache" unless result
33
+ end
34
+ end
35
+
36
+ # This returns nil if the chunks are corrupt or missing.
37
+ def get(cache, key)
38
+ buffer = ""
39
+ chunk = cache.read(CHECKSUM_CACHE_KEY % key)
40
+ if chunk
41
+ num_chunks, checksum = JSON.parse(chunk)
42
+ else
43
+ LoggingHelper.error("ChunkCache get: No checksum stored for #{key}")
44
+ return nil
45
+ end
46
+
47
+ for i in 1..num_chunks
48
+ chunk = cache.read(CHUNK_CACHE_KEY % [key, i])
49
+ buffer << chunk
50
+ end
51
+
52
+ if Zlib::crc32(buffer) != checksum
53
+ LoggingHelper.error("ChunkCache get: Invalid checksum on #{key}")
54
+ return nil
55
+ else
56
+ return JSON.parse(buffer)[0]
57
+ end
58
+ end
59
+ end
60
+ end
61
+
@@ -192,6 +192,22 @@ module Air18n
192
192
  fallbacks_for(the_locale, :exclude_default => true).include?(other_locale)
193
193
  end
194
194
 
195
+ # Call this when translations have been updated. Next time
196
+ # 'check_for_new_translations' is called, it will do a database read
197
+ # and update the cache.
198
+ def mark_translations_as_changed(locale)
199
+ I18n.cache.write(Air18n::Backend::T_LAST_UPDATED_AT % locale, Time.now) if I18n.cache
200
+ end
201
+
202
+ # This will cause cached translations to be updated and updates the
203
+ # 'last_loaded_at' timestamp in the cache. WARNING: You may still get stale
204
+ # data if another app is already in the process of updating a warm but stale
205
+ # cache.
206
+ def force_cache_refresh(locale)
207
+ I18n.mark_translations_as_changed(locale)
208
+ I18n.check_for_new_translations(locale)
209
+ end
210
+
195
211
  def check_for_new_translations
196
212
  @air18n_backend.check_for_new_translations(I18n.full_locale)
197
213
  end
@@ -201,6 +217,7 @@ module Air18n
201
217
  reload_translations_in(I18n.full_locale)
202
218
  end
203
219
 
220
+ # This is unaware of the cache and ignores it.
204
221
  def reload_translations_in(locale)
205
222
  @air18n_backend.reload_translations([locale])
206
223
  end
@@ -494,6 +494,7 @@ module Air18n
494
494
  phrase_to_phrase_translations = pt_scope.all.group_by { |pt| [pt.locale, pt.phrase_id] }
495
495
  phrase_to_phrase_translations.each do |(locale, phrase_id), phrase_translations|
496
496
  phrase_translations.sort_by! { |pt| pt.created_at }
497
+
497
498
  previous_translation = PhraseTranslation.where("created_at < ?", phrase_translations.first.created_at - 1.second).
498
499
  where(:locale => locale).
499
500
  where(:phrase_id => phrase_id).
@@ -1,3 +1,3 @@
1
1
  module Air18n
2
- VERSION = "0.1.32"
2
+ VERSION = "0.1.33"
3
3
  end
@@ -100,6 +100,47 @@ describe Air18n::Backend do
100
100
  end
101
101
  end
102
102
 
103
+ context 'I18n cache' do
104
+ before :each do
105
+ @backend = Air18n::Backend.new
106
+ I18n.cache = ActiveSupport::Cache::MemoryStore.new
107
+ end
108
+
109
+ after :all do
110
+ I18n.cache = nil
111
+ end
112
+
113
+ it 'should return stale results from the cache' do
114
+ @phraseA = FactoryGirl.create(:phrase, :key => 'init, key A', :value => 'value A')
115
+ @tAs = FactoryGirl.create(:phrase_translation, :phrase => @phraseA, :value => 'Spanish value A', :locale => :es)
116
+ @tAf = FactoryGirl.create(:phrase_translation, :phrase => @phraseA, :value => 'French value A', :locale => :fr)
117
+
118
+ @backend.lookup(:es, @phraseA.key).should == 'Spanish value A'
119
+ @tAs.update_attributes(:value => 'Updated Spanish value A')
120
+ @backend.lookup(:es, @phraseA.key, [], :default => @phraseA.value).should == 'Spanish value A'
121
+ @backend.lookup(:fr, @phraseA.key).should == 'French value A'
122
+ @backend.lookup(:it, @phraseA.key, [], :default => @phraseA.value).should == 'value A'
123
+ end
124
+
125
+ it 'should check for new translations' do
126
+ @phraseC = FactoryGirl.create(:phrase, :key => 'init, key C', :value => 'value C')
127
+ @tCs = FactoryGirl.create(:phrase_translation, :phrase => @phraseC, :value => 'Spanish value C', :locale => :es)
128
+ @tCf = FactoryGirl.create(:phrase_translation, :phrase => @phraseC, :value => 'French value C', :locale => :fr)
129
+
130
+ @backend.lookup(:es, @phraseC.key, [], :default => @phraseC.value).should == 'Spanish value C'
131
+ @backend.lookup(:fr, @phraseC.key).should == 'French value C'
132
+ @tCs.update_attributes(:value => 'Newer Spanish value C')
133
+ @tCf.update_attributes(:value => 'Newer French value C')
134
+ I18n.mark_translations_as_changed(:es)
135
+ I18n.mark_translations_as_changed(:fr)
136
+ @backend.check_for_new_translations(:es)
137
+ @backend.check_for_new_translations(:fr)
138
+ @backend.lookup(:es, @phraseC.key).should == 'Newer Spanish value C'
139
+ @backend.lookup(:fr, @phraseC.key).should == 'Newer French value C'
140
+ @backend.lookup(:it, @phraseC.key, [], :default => @phraseC.value).should == 'value C'
141
+ end
142
+ end
143
+
103
144
  context 'Pseudolocales' do
104
145
  it 'should use pseudolocales for xx locale' do
105
146
  @backend = Air18n::Backend.new
@@ -0,0 +1,39 @@
1
+ require 'spec_helper'
2
+
3
+ require 'air18n/chunk_cache'
4
+
5
+ describe Air18n::ChunkCache do
6
+ context 'Chunk cache get and set' do
7
+ before :all do
8
+ @cache = ActiveSupport::Cache::MemoryStore.new
9
+ end
10
+
11
+ it 'should return nil for non-existant keys' do
12
+ Air18n::ChunkCache::get(@cache, "ANY_KEY").should == nil
13
+ end
14
+
15
+ it 'should work for short values < 1M' do
16
+ tiny_string = SecureRandom.urlsafe_base64(5)
17
+ Air18n::ChunkCache::set(@cache, "TINY", tiny_string, 60)
18
+ num_chunks, checksum = JSON.parse(@cache.read(Air18n::ChunkCache::CHECKSUM_CACHE_KEY % "TINY"))
19
+ num_chunks.should == 1
20
+ Air18n::ChunkCache::get(@cache, "TINY").should == tiny_string
21
+ end
22
+
23
+ it 'should work for values < 1M' do
24
+ small_string = SecureRandom.urlsafe_base64(5000)
25
+ Air18n::ChunkCache::set(@cache, "SMALL", small_string, 60)
26
+ num_chunks, checksum = JSON.parse(@cache.read(Air18n::ChunkCache::CHECKSUM_CACHE_KEY % "SMALL"))
27
+ num_chunks.should == 1
28
+ Air18n::ChunkCache::get(@cache, "SMALL").should == small_string
29
+ end
30
+
31
+ it 'should work for values > 1M' do
32
+ large_string = SecureRandom.urlsafe_base64(3000000)
33
+ Air18n::ChunkCache::set(@cache, "LARGE", large_string, 60)
34
+ num_chunks, checksum = JSON.parse(@cache.read(Air18n::ChunkCache::CHECKSUM_CACHE_KEY % "LARGE"))
35
+ num_chunks.should == 5
36
+ Air18n::ChunkCache::get(@cache, "LARGE").should == large_string
37
+ end
38
+ end
39
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: air18n
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.32
4
+ version: 0.1.33
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -13,7 +13,7 @@ authors:
13
13
  autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
- date: 2012-11-15 00:00:00.000000000 Z
16
+ date: 2012-11-20 00:00:00.000000000 Z
17
17
  dependencies:
18
18
  - !ruby/object:Gem::Dependency
19
19
  name: i18n
@@ -47,6 +47,22 @@ dependencies:
47
47
  - - ~>
48
48
  - !ruby/object:Gem::Version
49
49
  version: '3.0'
50
+ - !ruby/object:Gem::Dependency
51
+ name: activesupport
52
+ requirement: !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - ~>
56
+ - !ruby/object:Gem::Version
57
+ version: '3.0'
58
+ type: :runtime
59
+ prerelease: false
60
+ version_requirements: !ruby/object:Gem::Requirement
61
+ none: false
62
+ requirements:
63
+ - - ~>
64
+ - !ruby/object:Gem::Version
65
+ version: '3.0'
50
66
  - !ruby/object:Gem::Dependency
51
67
  name: rspec
52
68
  requirement: !ruby/object:Gem::Requirement
@@ -147,6 +163,7 @@ files:
147
163
  - config/colonial_spelling_variants.yml
148
164
  - lib/air18n.rb
149
165
  - lib/air18n/backend.rb
166
+ - lib/air18n/chunk_cache.rb
150
167
  - lib/air18n/class_methods.rb
151
168
  - lib/air18n/default_text_change_observer.rb
152
169
  - lib/air18n/less_silly_chain.rb
@@ -174,6 +191,7 @@ files:
174
191
  - spec/factories.rb
175
192
  - spec/lib/air18n/air18n_spec.rb
176
193
  - spec/lib/air18n/backend_spec.rb
194
+ - spec/lib/air18n/chunk_cache_spec.rb
177
195
  - spec/lib/air18n/phrase_spec.rb
178
196
  - spec/lib/air18n/phrase_translation_spec.rb
179
197
  - spec/lib/air18n/prim_and_proper_spec.rb
@@ -210,6 +228,7 @@ test_files:
210
228
  - spec/factories.rb
211
229
  - spec/lib/air18n/air18n_spec.rb
212
230
  - spec/lib/air18n/backend_spec.rb
231
+ - spec/lib/air18n/chunk_cache_spec.rb
213
232
  - spec/lib/air18n/phrase_spec.rb
214
233
  - spec/lib/air18n/phrase_translation_spec.rb
215
234
  - spec/lib/air18n/prim_and_proper_spec.rb