air18n 0.1.32 → 0.1.33
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/air18n.gemspec +1 -0
- data/lib/air18n.rb +3 -0
- data/lib/air18n/backend.rb +49 -4
- data/lib/air18n/chunk_cache.rb +61 -0
- data/lib/air18n/class_methods.rb +17 -0
- data/lib/air18n/phrase_translation.rb +1 -0
- data/lib/air18n/version.rb +1 -1
- data/spec/lib/air18n/backend_spec.rb +41 -0
- data/spec/lib/air18n/chunk_cache_spec.rb +39 -0
- metadata +21 -2
data/.gitignore
CHANGED
data/air18n.gemspec
CHANGED
@@ -10,6 +10,7 @@ Gem::Specification.new do |gem|
|
|
10
10
|
|
11
11
|
gem.add_runtime_dependency 'i18n', '>= 0.5.0'
|
12
12
|
gem.add_runtime_dependency 'activerecord', '~> 3.0'
|
13
|
+
gem.add_runtime_dependency 'activesupport', '~> 3.0'
|
13
14
|
gem.add_development_dependency "rspec"
|
14
15
|
gem.add_development_dependency 'sqlite3'
|
15
16
|
gem.add_development_dependency 'guard'
|
data/lib/air18n.rb
CHANGED
@@ -54,6 +54,9 @@ module Air18n
|
|
54
54
|
@distance_unit = 'KM'
|
55
55
|
attr_reader :distance_unit
|
56
56
|
|
57
|
+
# Cache used to store translation data from the database.
|
58
|
+
attr_accessor :cache
|
59
|
+
|
57
60
|
# Whether to wrap phrases in HTML that allows them to be translated by
|
58
61
|
# the user from directly on the page.
|
59
62
|
@contextual_translation = false
|
data/lib/air18n/backend.rb
CHANGED
@@ -4,6 +4,7 @@ require 'air18n/logging_helper'
|
|
4
4
|
require 'air18n/prim_and_proper'
|
5
5
|
require 'air18n/pseudo_locales'
|
6
6
|
require 'air18n/smart_count'
|
7
|
+
require 'air18n/chunk_cache'
|
7
8
|
|
8
9
|
module Air18n
|
9
10
|
class Backend
|
@@ -17,10 +18,19 @@ module Air18n
|
|
17
18
|
# be able to guard against flipflops.
|
18
19
|
attr_accessor :translation_data, :phrase_screenshots, :default_text_change_observer
|
19
20
|
|
21
|
+
# Define constants used as cache keys
|
22
|
+
T_LAST_LOADED_AT = 'Air18n::translations_last_loaded_at_%s'
|
23
|
+
T_LAST_UPDATED_AT = 'Air18n::translations_last_updated_at_%s'
|
24
|
+
T_DATA = 'Air18n::translation_data_%s'
|
25
|
+
# This value allows one app instance to make a DB call while other still
|
26
|
+
# pull slightly stale data from the cache. The unit is seconds.
|
27
|
+
RACE_CONDITION_TTL = 5
|
28
|
+
|
20
29
|
# Stores translations for a given locale.
|
21
30
|
def store_translations locale, data, options = {}
|
22
31
|
@translation_data ||= {}
|
23
32
|
@translation_data[locale.to_sym] = data
|
33
|
+
ChunkCache::set(I18n.cache, T_DATA % locale, data, 1.day) if I18n.cache
|
24
34
|
end
|
25
35
|
|
26
36
|
def available_locales
|
@@ -66,7 +76,7 @@ module Air18n
|
|
66
76
|
def init_translations(locale)
|
67
77
|
reset_phrase_screenshots unless @phrase_screenshots
|
68
78
|
if @translation_data.nil? || !@translation_data.include?(locale)
|
69
|
-
|
79
|
+
get_from_cache_or_reload(locale)
|
70
80
|
end
|
71
81
|
end
|
72
82
|
|
@@ -74,6 +84,39 @@ module Air18n
|
|
74
84
|
@phrase_screenshots = PhraseScreenshot.all_phrase_urls
|
75
85
|
end
|
76
86
|
|
87
|
+
def check_last_timestamps(locale)
|
88
|
+
# Potential race condition here but the order of operations will at worst
|
89
|
+
# cause an extra DB call.
|
90
|
+
if I18n.cache
|
91
|
+
translation_last_loaded_at = I18n.cache.read(T_LAST_LOADED_AT % locale)
|
92
|
+
last_updated_at = I18n.cache.read(T_LAST_UPDATED_AT % locale)
|
93
|
+
else
|
94
|
+
translation_last_loaded_at = @translations_last_loaded_at[locale]
|
95
|
+
last_updated_at ||= Time.now
|
96
|
+
end
|
97
|
+
[translation_last_loaded_at, last_updated_at]
|
98
|
+
end
|
99
|
+
|
100
|
+
def get_from_cache_or_reload(locale)
|
101
|
+
# This function will check the cache if available to see if there is data.
|
102
|
+
# If it is available, read it and store it in a instance variable.
|
103
|
+
# If it is not available, increment the last_loaded_at cache key by
|
104
|
+
# a few seconds in the future. (This will prevent the majority of application
|
105
|
+
# processes from doing a database lookup by serving them slightly stale data
|
106
|
+
# from the cache. If multiple instances start up when the cache is empty, then multiple
|
107
|
+
# call to the database are unavoidable unless a locking and wait style is used.
|
108
|
+
# Refer to the specs 'I18n cache' for more information.
|
109
|
+
cache_results = nil
|
110
|
+
cache_results = ChunkCache::get(I18n.cache, T_DATA % locale) if I18n.cache
|
111
|
+
if !cache_results.nil?
|
112
|
+
@translation_data ||= {}
|
113
|
+
@translation_data[locale.to_sym] = cache_results
|
114
|
+
else
|
115
|
+
I18n.cache.write(T_LAST_LOADED_AT % locale, Time.now) if I18n.cache
|
116
|
+
reload_translations([locale])
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
77
120
|
def check_for_new_translations(locale)
|
78
121
|
# When TranslateController makes a new translation, it sets
|
79
122
|
# translations_last_updated_at to the current time in the cache.
|
@@ -81,10 +124,12 @@ module Air18n
|
|
81
124
|
|
82
125
|
# No-op if locale is the default locale; its translations never change.
|
83
126
|
if locale != I18n.default_locale
|
84
|
-
|
85
|
-
last_updated_at
|
86
|
-
|
127
|
+
translation_last_loaded_at, last_updated_at = check_last_timestamps(locale)
|
128
|
+
if (translation_last_loaded_at.nil?) || (last_updated_at && last_updated_at.to_i >= translation_last_loaded_at.to_i)
|
129
|
+
I18n.cache.write(T_LAST_LOADED_AT % locale, Time.now + RACE_CONDITION_TTL) if I18n.cache
|
87
130
|
reload_translations([locale])
|
131
|
+
else
|
132
|
+
get_from_cache_or_reload(locale)
|
88
133
|
end
|
89
134
|
end
|
90
135
|
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'zlib'
|
3
|
+
|
4
|
+
module Air18n
|
5
|
+
module ChunkCache
|
6
|
+
# This class augments the caching layer by splitting up (into chunks)
|
7
|
+
# values > 1MB.
|
8
|
+
|
9
|
+
# Define constants and cache keys.
|
10
|
+
#
|
11
|
+
# We divide things into chunks of 900000 bytes to accomodate memcaches with
|
12
|
+
# 1mb max value size.
|
13
|
+
MAX_CHUNK_SIZE = 900000
|
14
|
+
CHECKSUM_CACHE_KEY = "%s_0_nchunks+checksum"
|
15
|
+
CHUNK_CACHE_KEY = "%s_%d"
|
16
|
+
|
17
|
+
module_function
|
18
|
+
|
19
|
+
def set(cache, key, value, expires_in)
|
20
|
+
# We put the value in an array because plain strings are not valid JSON.
|
21
|
+
value_json = [value].to_json
|
22
|
+
|
23
|
+
# We use divmod to avoid the edge condition where the floating point math
|
24
|
+
# result is 5.000001 and the ceil would result in 6.
|
25
|
+
quotient, modulus = value_json.size.divmod(MAX_CHUNK_SIZE)
|
26
|
+
num_chunks = modulus > 0 ? quotient + 1 : quotient
|
27
|
+
cache.write(CHECKSUM_CACHE_KEY % key, [num_chunks, Zlib::crc32(value_json)].to_json)
|
28
|
+
|
29
|
+
1.upto(num_chunks) do |i|
|
30
|
+
chunk = value_json[(i-1)*MAX_CHUNK_SIZE, MAX_CHUNK_SIZE]
|
31
|
+
result = cache.write(CHUNK_CACHE_KEY % [key, i], chunk, :expires_in => expires_in)
|
32
|
+
raise "ChunkCache set: Failed to write chunk #{i} of #{chunk.length} bytes to cache" unless result
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# This returns nil if the chunks are corrupt or missing.
|
37
|
+
def get(cache, key)
|
38
|
+
buffer = ""
|
39
|
+
chunk = cache.read(CHECKSUM_CACHE_KEY % key)
|
40
|
+
if chunk
|
41
|
+
num_chunks, checksum = JSON.parse(chunk)
|
42
|
+
else
|
43
|
+
LoggingHelper.error("ChunkCache get: No checksum stored for #{key}")
|
44
|
+
return nil
|
45
|
+
end
|
46
|
+
|
47
|
+
for i in 1..num_chunks
|
48
|
+
chunk = cache.read(CHUNK_CACHE_KEY % [key, i])
|
49
|
+
buffer << chunk
|
50
|
+
end
|
51
|
+
|
52
|
+
if Zlib::crc32(buffer) != checksum
|
53
|
+
LoggingHelper.error("ChunkCache get: Invalid checksum on #{key}")
|
54
|
+
return nil
|
55
|
+
else
|
56
|
+
return JSON.parse(buffer)[0]
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
data/lib/air18n/class_methods.rb
CHANGED
@@ -192,6 +192,22 @@ module Air18n
|
|
192
192
|
fallbacks_for(the_locale, :exclude_default => true).include?(other_locale)
|
193
193
|
end
|
194
194
|
|
195
|
+
# Call this when translations have been updated. Next time
|
196
|
+
# 'check_for_new_translations' is called, it will do a database read
|
197
|
+
# and update the cache.
|
198
|
+
def mark_translations_as_changed(locale)
|
199
|
+
I18n.cache.write(Air18n::Backend::T_LAST_UPDATED_AT % locale, Time.now) if I18n.cache
|
200
|
+
end
|
201
|
+
|
202
|
+
# This will cause cached translations to be updated and updates the
|
203
|
+
# 'last_loaded_at' timestamp in the cache. WARNING: You may still get stale
|
204
|
+
# data if another app is already in the process of updating a warm but stale
|
205
|
+
# cache.
|
206
|
+
def force_cache_refresh(locale)
|
207
|
+
I18n.mark_translations_as_changed(locale)
|
208
|
+
I18n.check_for_new_translations(locale)
|
209
|
+
end
|
210
|
+
|
195
211
|
def check_for_new_translations
|
196
212
|
@air18n_backend.check_for_new_translations(I18n.full_locale)
|
197
213
|
end
|
@@ -201,6 +217,7 @@ module Air18n
|
|
201
217
|
reload_translations_in(I18n.full_locale)
|
202
218
|
end
|
203
219
|
|
220
|
+
# This is unaware of the cache and ignores it.
|
204
221
|
def reload_translations_in(locale)
|
205
222
|
@air18n_backend.reload_translations([locale])
|
206
223
|
end
|
@@ -494,6 +494,7 @@ module Air18n
|
|
494
494
|
phrase_to_phrase_translations = pt_scope.all.group_by { |pt| [pt.locale, pt.phrase_id] }
|
495
495
|
phrase_to_phrase_translations.each do |(locale, phrase_id), phrase_translations|
|
496
496
|
phrase_translations.sort_by! { |pt| pt.created_at }
|
497
|
+
|
497
498
|
previous_translation = PhraseTranslation.where("created_at < ?", phrase_translations.first.created_at - 1.second).
|
498
499
|
where(:locale => locale).
|
499
500
|
where(:phrase_id => phrase_id).
|
data/lib/air18n/version.rb
CHANGED
@@ -100,6 +100,47 @@ describe Air18n::Backend do
|
|
100
100
|
end
|
101
101
|
end
|
102
102
|
|
103
|
+
context 'I18n cache' do
|
104
|
+
before :each do
|
105
|
+
@backend = Air18n::Backend.new
|
106
|
+
I18n.cache = ActiveSupport::Cache::MemoryStore.new
|
107
|
+
end
|
108
|
+
|
109
|
+
after :all do
|
110
|
+
I18n.cache = nil
|
111
|
+
end
|
112
|
+
|
113
|
+
it 'should return stale results from the cache' do
|
114
|
+
@phraseA = FactoryGirl.create(:phrase, :key => 'init, key A', :value => 'value A')
|
115
|
+
@tAs = FactoryGirl.create(:phrase_translation, :phrase => @phraseA, :value => 'Spanish value A', :locale => :es)
|
116
|
+
@tAf = FactoryGirl.create(:phrase_translation, :phrase => @phraseA, :value => 'French value A', :locale => :fr)
|
117
|
+
|
118
|
+
@backend.lookup(:es, @phraseA.key).should == 'Spanish value A'
|
119
|
+
@tAs.update_attributes(:value => 'Updated Spanish value A')
|
120
|
+
@backend.lookup(:es, @phraseA.key, [], :default => @phraseA.value).should == 'Spanish value A'
|
121
|
+
@backend.lookup(:fr, @phraseA.key).should == 'French value A'
|
122
|
+
@backend.lookup(:it, @phraseA.key, [], :default => @phraseA.value).should == 'value A'
|
123
|
+
end
|
124
|
+
|
125
|
+
it 'should check for new translations' do
|
126
|
+
@phraseC = FactoryGirl.create(:phrase, :key => 'init, key C', :value => 'value C')
|
127
|
+
@tCs = FactoryGirl.create(:phrase_translation, :phrase => @phraseC, :value => 'Spanish value C', :locale => :es)
|
128
|
+
@tCf = FactoryGirl.create(:phrase_translation, :phrase => @phraseC, :value => 'French value C', :locale => :fr)
|
129
|
+
|
130
|
+
@backend.lookup(:es, @phraseC.key, [], :default => @phraseC.value).should == 'Spanish value C'
|
131
|
+
@backend.lookup(:fr, @phraseC.key).should == 'French value C'
|
132
|
+
@tCs.update_attributes(:value => 'Newer Spanish value C')
|
133
|
+
@tCf.update_attributes(:value => 'Newer French value C')
|
134
|
+
I18n.mark_translations_as_changed(:es)
|
135
|
+
I18n.mark_translations_as_changed(:fr)
|
136
|
+
@backend.check_for_new_translations(:es)
|
137
|
+
@backend.check_for_new_translations(:fr)
|
138
|
+
@backend.lookup(:es, @phraseC.key).should == 'Newer Spanish value C'
|
139
|
+
@backend.lookup(:fr, @phraseC.key).should == 'Newer French value C'
|
140
|
+
@backend.lookup(:it, @phraseC.key, [], :default => @phraseC.value).should == 'value C'
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
103
144
|
context 'Pseudolocales' do
|
104
145
|
it 'should use pseudolocales for xx locale' do
|
105
146
|
@backend = Air18n::Backend.new
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
require 'air18n/chunk_cache'
|
4
|
+
|
5
|
+
describe Air18n::ChunkCache do
|
6
|
+
context 'Chunk cache get and set' do
|
7
|
+
before :all do
|
8
|
+
@cache = ActiveSupport::Cache::MemoryStore.new
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should return nil for non-existant keys' do
|
12
|
+
Air18n::ChunkCache::get(@cache, "ANY_KEY").should == nil
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'should work for short values < 1M' do
|
16
|
+
tiny_string = SecureRandom.urlsafe_base64(5)
|
17
|
+
Air18n::ChunkCache::set(@cache, "TINY", tiny_string, 60)
|
18
|
+
num_chunks, checksum = JSON.parse(@cache.read(Air18n::ChunkCache::CHECKSUM_CACHE_KEY % "TINY"))
|
19
|
+
num_chunks.should == 1
|
20
|
+
Air18n::ChunkCache::get(@cache, "TINY").should == tiny_string
|
21
|
+
end
|
22
|
+
|
23
|
+
it 'should work for values < 1M' do
|
24
|
+
small_string = SecureRandom.urlsafe_base64(5000)
|
25
|
+
Air18n::ChunkCache::set(@cache, "SMALL", small_string, 60)
|
26
|
+
num_chunks, checksum = JSON.parse(@cache.read(Air18n::ChunkCache::CHECKSUM_CACHE_KEY % "SMALL"))
|
27
|
+
num_chunks.should == 1
|
28
|
+
Air18n::ChunkCache::get(@cache, "SMALL").should == small_string
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'should work for values > 1M' do
|
32
|
+
large_string = SecureRandom.urlsafe_base64(3000000)
|
33
|
+
Air18n::ChunkCache::set(@cache, "LARGE", large_string, 60)
|
34
|
+
num_chunks, checksum = JSON.parse(@cache.read(Air18n::ChunkCache::CHECKSUM_CACHE_KEY % "LARGE"))
|
35
|
+
num_chunks.should == 5
|
36
|
+
Air18n::ChunkCache::get(@cache, "LARGE").should == large_string
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: air18n
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.33
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -13,7 +13,7 @@ authors:
|
|
13
13
|
autorequire:
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
|
-
date: 2012-11-
|
16
|
+
date: 2012-11-20 00:00:00.000000000 Z
|
17
17
|
dependencies:
|
18
18
|
- !ruby/object:Gem::Dependency
|
19
19
|
name: i18n
|
@@ -47,6 +47,22 @@ dependencies:
|
|
47
47
|
- - ~>
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: '3.0'
|
50
|
+
- !ruby/object:Gem::Dependency
|
51
|
+
name: activesupport
|
52
|
+
requirement: !ruby/object:Gem::Requirement
|
53
|
+
none: false
|
54
|
+
requirements:
|
55
|
+
- - ~>
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: '3.0'
|
58
|
+
type: :runtime
|
59
|
+
prerelease: false
|
60
|
+
version_requirements: !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - ~>
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: '3.0'
|
50
66
|
- !ruby/object:Gem::Dependency
|
51
67
|
name: rspec
|
52
68
|
requirement: !ruby/object:Gem::Requirement
|
@@ -147,6 +163,7 @@ files:
|
|
147
163
|
- config/colonial_spelling_variants.yml
|
148
164
|
- lib/air18n.rb
|
149
165
|
- lib/air18n/backend.rb
|
166
|
+
- lib/air18n/chunk_cache.rb
|
150
167
|
- lib/air18n/class_methods.rb
|
151
168
|
- lib/air18n/default_text_change_observer.rb
|
152
169
|
- lib/air18n/less_silly_chain.rb
|
@@ -174,6 +191,7 @@ files:
|
|
174
191
|
- spec/factories.rb
|
175
192
|
- spec/lib/air18n/air18n_spec.rb
|
176
193
|
- spec/lib/air18n/backend_spec.rb
|
194
|
+
- spec/lib/air18n/chunk_cache_spec.rb
|
177
195
|
- spec/lib/air18n/phrase_spec.rb
|
178
196
|
- spec/lib/air18n/phrase_translation_spec.rb
|
179
197
|
- spec/lib/air18n/prim_and_proper_spec.rb
|
@@ -210,6 +228,7 @@ test_files:
|
|
210
228
|
- spec/factories.rb
|
211
229
|
- spec/lib/air18n/air18n_spec.rb
|
212
230
|
- spec/lib/air18n/backend_spec.rb
|
231
|
+
- spec/lib/air18n/chunk_cache_spec.rb
|
213
232
|
- spec/lib/air18n/phrase_spec.rb
|
214
233
|
- spec/lib/air18n/phrase_translation_spec.rb
|
215
234
|
- spec/lib/air18n/prim_and_proper_spec.rb
|