viximo-cache-money 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. data/LICENSE +201 -0
  2. data/README +204 -0
  3. data/README.markdown +204 -0
  4. data/TODO +17 -0
  5. data/UNSUPPORTED_FEATURES +13 -0
  6. data/config/environment.rb +8 -0
  7. data/config/memcached.yml +4 -0
  8. data/db/schema.rb +18 -0
  9. data/init.rb +1 -0
  10. data/lib/cache_money.rb +105 -0
  11. data/lib/cash/accessor.rb +83 -0
  12. data/lib/cash/adapter/memcache_client.rb +36 -0
  13. data/lib/cash/adapter/memcached.rb +127 -0
  14. data/lib/cash/adapter/redis.rb +144 -0
  15. data/lib/cash/buffered.rb +137 -0
  16. data/lib/cash/config.rb +78 -0
  17. data/lib/cash/fake.rb +83 -0
  18. data/lib/cash/finders.rb +50 -0
  19. data/lib/cash/index.rb +211 -0
  20. data/lib/cash/local.rb +105 -0
  21. data/lib/cash/lock.rb +63 -0
  22. data/lib/cash/mock.rb +158 -0
  23. data/lib/cash/query/abstract.rb +219 -0
  24. data/lib/cash/query/calculation.rb +45 -0
  25. data/lib/cash/query/primary_key.rb +50 -0
  26. data/lib/cash/query/select.rb +16 -0
  27. data/lib/cash/request.rb +3 -0
  28. data/lib/cash/transactional.rb +43 -0
  29. data/lib/cash/util/array.rb +9 -0
  30. data/lib/cash/util/marshal.rb +19 -0
  31. data/lib/cash/version.rb +3 -0
  32. data/lib/cash/write_through.rb +71 -0
  33. data/lib/mem_cached_session_store.rb +49 -0
  34. data/lib/mem_cached_support_store.rb +143 -0
  35. data/rails/init.rb +1 -0
  36. data/spec/cash/accessor_spec.rb +186 -0
  37. data/spec/cash/active_record_spec.rb +224 -0
  38. data/spec/cash/buffered_spec.rb +9 -0
  39. data/spec/cash/calculations_spec.rb +78 -0
  40. data/spec/cash/finders_spec.rb +455 -0
  41. data/spec/cash/local_buffer_spec.rb +9 -0
  42. data/spec/cash/local_spec.rb +9 -0
  43. data/spec/cash/lock_spec.rb +110 -0
  44. data/spec/cash/marshal_spec.rb +60 -0
  45. data/spec/cash/order_spec.rb +172 -0
  46. data/spec/cash/transactional_spec.rb +602 -0
  47. data/spec/cash/window_spec.rb +195 -0
  48. data/spec/cash/without_caching_spec.rb +32 -0
  49. data/spec/cash/write_through_spec.rb +252 -0
  50. data/spec/spec_helper.rb +87 -0
  51. metadata +300 -0
@@ -0,0 +1,105 @@
1
+ module Cash
2
+ class Local
3
+ def initialize(remote_cache)
4
+ @remote_cache = remote_cache
5
+ end
6
+
7
+ def cache_locally
8
+ @remote_cache = LocalBuffer.new(original_cache = @remote_cache)
9
+ yield if block_given?
10
+ ensure
11
+ @remote_cache = original_cache
12
+ end
13
+
14
+ def autoload_missing_constants
15
+ yield if block_given?
16
+ rescue ArgumentError, *@remote_cache.exception_classes => error
17
+ lazy_load ||= Hash.new { |hash, hash_key| hash[hash_key] = true; false }
18
+ if error.to_s[/undefined class|referred/] && !lazy_load[error.to_s.split.last.constantize]
19
+ retry
20
+ else
21
+ raise error
22
+ end
23
+ end
24
+
25
+ def respond_to?(method)
26
+ super || @remote_cache.respond_to?(method)
27
+ end
28
+
29
+ private
30
+
31
+ def method_missing(method, *args, &block)
32
+ autoload_missing_constants do
33
+ @remote_cache.send(method, *args, &block)
34
+ end
35
+ end
36
+ end
37
+
38
+ class LocalBuffer
39
+ delegate :respond_to?, :to => :@remote_cache
40
+ MULTI_GET_RESULTS_THRESHOLD = 1
41
+
42
+ def initialize(remote_cache)
43
+ @local_cache = {}
44
+ @remote_cache = remote_cache
45
+ end
46
+
47
+ def get(key, *options)
48
+ if @local_cache.has_key?(key)
49
+ @local_cache[key]
50
+ else
51
+ @local_cache[key] = @remote_cache.get(key, *options)
52
+ end
53
+ end
54
+
55
+ # Model.find_by_index() uses get_multi to get the object after getting the id through the index.
56
+ # Simplest way of utilizing the local cache is to override get_multi and use
57
+ # a threshold of 1. Using a higher threshold could be an optimization for get_multi calls
58
+ # returning more than one value.
59
+ def get_multi(keys, *options)
60
+ results = {}
61
+ remaining_keys = []
62
+ keys.each do |key|
63
+ if @local_cache.has_key?(key)
64
+ results[key] = @local_cache[key]
65
+ else
66
+ remaining_keys << key
67
+ end
68
+ end
69
+ if !remaining_keys.empty?
70
+ multi = @remote_cache.get_multi(remaining_keys, *options)
71
+ if multi
72
+ multi.each do |k,v|
73
+ @local_cache[k] = v if multi.size() <= MULTI_GET_RESULTS_THRESHOLD
74
+ results[k] = v
75
+ end
76
+ end
77
+ end
78
+ results
79
+ end
80
+
81
+ def set(key, value, *options)
82
+ @remote_cache.set(key, value, *options)
83
+ @local_cache[key] = value
84
+ end
85
+
86
+ def add(key, value, *options)
87
+ result = @remote_cache.add(key, value, *options)
88
+ if result == "STORED\r\n"
89
+ @local_cache[key] = value
90
+ end
91
+ result
92
+ end
93
+
94
+ def delete(key, *options)
95
+ @remote_cache.delete(key, *options)
96
+ @local_cache.delete(key)
97
+ end
98
+
99
+ private
100
+
101
+ def method_missing(method, *args, &block)
102
+ @remote_cache.send(method, *args, &block)
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,63 @@
1
+ require 'socket'
2
+
3
+ module Cash
4
+ class Lock
5
+ class Error < RuntimeError; end
6
+
7
+ INITIAL_WAIT = 2
8
+ DEFAULT_RETRY = 8
9
+ DEFAULT_EXPIRY = 30
10
+
11
+ def initialize(cache)
12
+ @cache = cache
13
+ end
14
+
15
+ def synchronize(key, lock_expiry = DEFAULT_EXPIRY, retries = DEFAULT_RETRY, initial_wait = INITIAL_WAIT)
16
+ if recursive_lock?(key)
17
+ yield
18
+ else
19
+ acquire_lock(key, lock_expiry, retries, initial_wait)
20
+ begin
21
+ yield
22
+ ensure
23
+ release_lock(key)
24
+ end
25
+ end
26
+ end
27
+
28
+ def acquire_lock(key, lock_expiry = DEFAULT_EXPIRY, retries = DEFAULT_RETRY, initial_wait = INITIAL_WAIT)
29
+ retries.times do |count|
30
+ response = @cache.add("lock/#{key}", host_pid, lock_expiry)
31
+ return if response == "STORED\r\n"
32
+ return if recursive_lock?(key)
33
+ exponential_sleep(count, initial_wait) unless count == retries - 1
34
+ end
35
+ debug_lock(key)
36
+ raise Error, "Couldn't acquire memcache lock on #{@cache.get_server_for_key("lock/#{key}")}"
37
+ end
38
+
39
+ def release_lock(key)
40
+ @cache.delete("lock/#{key}")
41
+ end
42
+
43
+ def exponential_sleep(count, initial_wait)
44
+ sleep((2**count) / initial_wait)
45
+ end
46
+
47
+ private
48
+
49
+ def recursive_lock?(key)
50
+ @cache.get("lock/#{key}") == host_pid
51
+ end
52
+
53
+ def debug_lock(key)
54
+ @cache.logger.warn("Cash::Lock[#{key}]: #{@cache.get("lock/#{key}")}") if @cache.respond_to?(:logger) && @cache.logger.respond_to?(:warn)
55
+ rescue
56
+ @cache.logger.warn("#{$!}") if @cache.respond_to?(:logger) && @cache.logger.respond_to?(:warn)
57
+ end
58
+
59
+ def host_pid
60
+ "#{Socket.gethostname} #{Process.pid}"
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,158 @@
1
+ module Cash
2
+ class Mock < HashWithIndifferentAccess
3
+ attr_accessor :servers
4
+
5
+ class CacheEntry
6
+ attr_reader :value
7
+
8
+ def self.default_ttl
9
+ 1_000_000
10
+ end
11
+
12
+ def self.now
13
+ Time.now
14
+ end
15
+
16
+ def initialize(value, raw, ttl)
17
+ if raw
18
+ @value = value.to_s
19
+ else
20
+ @value = Marshal.dump(value)
21
+ end
22
+
23
+ if ttl.nil? || ttl.zero?
24
+ @ttl = self.class.default_ttl
25
+ else
26
+ @ttl = ttl
27
+ end
28
+
29
+ @expires_at = self.class.now + @ttl
30
+ end
31
+
32
+
33
+ def expired?
34
+ self.class.now > @expires_at
35
+ end
36
+
37
+ def increment(amount = 1)
38
+ @value = (@value.to_i + amount).to_s
39
+ end
40
+
41
+ def decrement(amount = 1)
42
+ @value = (@value.to_i - amount).to_s
43
+ end
44
+
45
+ def unmarshal
46
+ Marshal.load(@value)
47
+ end
48
+
49
+ def to_i
50
+ @value.to_i
51
+ end
52
+ end
53
+
54
+ attr_accessor :logging
55
+
56
+ def initialize
57
+ @logging = false
58
+ end
59
+
60
+ def get_multi(*keys)
61
+ slice(*keys).collect { |k,v| [k, v.unmarshal] }.to_hash_without_nils
62
+ end
63
+
64
+ def set(key, value, ttl = CacheEntry.default_ttl, raw = false)
65
+ log "< set #{key} #{ttl}"
66
+ self[key] = CacheEntry.new(value, raw, ttl)
67
+ log('> STORED')
68
+ end
69
+
70
+ def get(key, raw = false)
71
+ if key.is_a?(Array)
72
+ get_multi(*key)
73
+ else
74
+ log "< get #{key}"
75
+ unless self.has_unexpired_key?(key)
76
+ log('> END')
77
+ return nil
78
+ end
79
+
80
+ log("> sending key #{key}")
81
+ log('> END')
82
+ if raw
83
+ self[key].value
84
+ else
85
+ self[key].unmarshal
86
+ end
87
+ end
88
+ end
89
+
90
+ def delete(key, options = {})
91
+ log "< delete #{key}"
92
+ if self.has_unexpired_key?(key)
93
+ log "> DELETED"
94
+ super(key)
95
+ else
96
+ log "> NOT FOUND"
97
+ end
98
+ end
99
+
100
+ def incr(key, amount = 1)
101
+ if self.has_unexpired_key?(key)
102
+ self[key].increment(amount)
103
+ self[key].to_i
104
+ end
105
+ end
106
+
107
+ def decr(key, amount = 1)
108
+ if self.has_unexpired_key?(key)
109
+ self[key].decrement(amount)
110
+ self[key].to_i
111
+ end
112
+ end
113
+
114
+ def add(key, value, ttl = CacheEntry.default_ttl, raw = false)
115
+ if self.has_unexpired_key?(key)
116
+ "NOT_STORED\r\n"
117
+ else
118
+ set(key, value, ttl, raw)
119
+ "STORED\r\n"
120
+ end
121
+ end
122
+
123
+ def append(key, value)
124
+ set(key, get(key, true).to_s + value.to_s, nil, true)
125
+ end
126
+
127
+ def namespace
128
+ nil
129
+ end
130
+
131
+ def flush_all
132
+ log('< flush_all')
133
+ clear
134
+ end
135
+
136
+ def stats
137
+ {}
138
+ end
139
+
140
+ def reset_runtime
141
+ [0, Hash.new(0)]
142
+ end
143
+
144
+ def has_unexpired_key?(key)
145
+ self.has_key?(key) && !self[key].expired?
146
+ end
147
+
148
+ def log(message)
149
+ return unless logging
150
+ logger.debug(message)
151
+ end
152
+
153
+ def logger
154
+ @logger ||= ActiveSupport::BufferedLogger.new(Rails.root.join('log/cash_mock.log'))
155
+ end
156
+
157
+ end
158
+ end
@@ -0,0 +1,219 @@
1
+ module Cash
2
+ module Query
3
+ class Abstract
4
+ delegate :with_exclusive_scope, :get, :table_name, :indices, :find_from_ids_without_cache, :cache_key, :columns_hash, :logger, :to => :@active_record
5
+
6
+ def self.perform(*args)
7
+ new(*args).perform
8
+ end
9
+
10
+ def initialize(active_record, options1, options2)
11
+ @active_record, @options1, @options2 = active_record, options1, options2 || {}
12
+
13
+ # if @options2.empty? and active_record.base_class != active_record
14
+ # @options2 = { :conditions => { active_record.inheritance_column => active_record.to_s }}
15
+ # end
16
+ # if active_record.base_class != active_record
17
+ # @options2[:conditions] = active_record.merge_conditions(
18
+ # @options2[:conditions], { active_record.inheritance_column => active_record.to_s }
19
+ # )
20
+ # end
21
+ end
22
+
23
+ def perform(find_options = {}, get_options = {})
24
+ if cache_config = cacheable?(@options1, @options2, find_options)
25
+ cache_keys, index = cache_keys(cache_config[0]), cache_config[1]
26
+
27
+ misses, missed_keys, objects = hit_or_miss(cache_keys, index, get_options)
28
+ format_results(cache_keys, choose_deserialized_objects_if_possible(missed_keys, cache_keys, misses, objects))
29
+ else
30
+ logger.debug(" \e[1;4;31mUNCACHEABLE\e[0m #{table_name} - #{find_options.inspect} - #{get_options.inspect} - #{@options1.inspect} - #{@options2.inspect}") if logger
31
+ uncacheable
32
+ end
33
+ end
34
+
35
+ DESC = /DESC/i
36
+
37
+ def order
38
+ @order ||= begin
39
+ if order_sql = @options1[:order] || @options2[:order]
40
+ matched, table_name, column_name, direction = *(ORDER.match(order_sql.to_s))
41
+ [column_name, direction =~ DESC ? :desc : :asc]
42
+ else
43
+ ['id', :asc]
44
+ end
45
+ end
46
+ rescue TypeError
47
+ ['id', :asc]
48
+ end
49
+
50
+ def limit
51
+ @limit ||= @options1[:limit] || @options2[:limit]
52
+ end
53
+
54
+ def offset
55
+ @offset ||= @options1[:offset] || @options2[:offset] || 0
56
+ end
57
+
58
+ def calculation?
59
+ false
60
+ end
61
+
62
+ private
63
+ def cacheable?(*optionss)
64
+ if @active_record.respond_to?(:cacheable?) && ! @active_record.cacheable?(*optionss)
65
+ if logger
66
+ if @active_record.respond_to?(:cacheable?)
67
+ logger.debug(" \e[1;4;31mUNCACHEABLE CLASS\e[0m #{table_name}")
68
+ else
69
+ logger.debug(" \e[1;4;31mUNCACHEABLE INSTANCE\e[0m #{table_name} - #{optionss.inspect}")
70
+ end
71
+ end
72
+ return false
73
+ end
74
+ optionss.each do |options|
75
+ unless safe_options_for_cache?(options)
76
+ logger.debug(" \e[1;4;31mUNCACHEABLE UNSAFE\e[0m #{table_name} - #{options.inspect}") if logger
77
+ return false
78
+ end
79
+ end
80
+ partial_indices = optionss.collect { |options| attribute_value_pairs_for_conditions(options[:conditions]) }
81
+ return if partial_indices.include?(nil)
82
+ attribute_value_pairs = partial_indices.sum.sort { |x, y| x[0] <=> y[0] }
83
+
84
+ # attribute_value_pairs.each do |attribute_value_pair|
85
+ # return false if attribute_value_pair.last.is_a?(Array)
86
+ # end
87
+
88
+ if index = indexed_on?(attribute_value_pairs.collect { |pair| pair[0] })
89
+ if index.matches?(self)
90
+ [attribute_value_pairs, index]
91
+ else
92
+ logger.debug(" \e[1;4;31mUNCACHEABLE NO MATCHING INDEX\e[0m #{table_name} - #{index.order_column.inspect} #{index.order.inspect} #{index.limit.inspect}") if logger
93
+ false
94
+ end
95
+ else
96
+ logger.debug(" \e[1;4;31mUNCACHEABLE NOT INDEXED\e[0m #{table_name} - #{attribute_value_pairs.collect { |pair| pair[0] }.inspect}") if logger
97
+ false
98
+ end
99
+ end
100
+
101
+ def hit_or_miss(cache_keys, index, options)
102
+ misses, missed_keys = nil, nil
103
+ objects = @active_record.get(cache_keys, options.merge(:ttl => index.ttl)) do |missed_keys|
104
+ misses = miss(missed_keys, @options1.merge(:limit => index.window))
105
+ serialize_objects(index, misses)
106
+ end
107
+ [misses, missed_keys, objects]
108
+ end
109
+
110
+ def cache_keys(attribute_value_pairs)
111
+ attribute_value_pairs.flatten.join('/')
112
+ end
113
+
114
+ def safe_options_for_cache?(options)
115
+ return false unless options.kind_of?(Hash)
116
+ options.except(:conditions, :readonly, :limit, :offset, :order).values.compact.empty? && !options[:readonly]
117
+ end
118
+
119
+ def attribute_value_pairs_for_conditions(conditions)
120
+ case conditions
121
+ when Hash
122
+ conditions.to_a.collect { |key, value| [key.to_s, value.is_a?(ActiveRecord::Base) ? value.id : value] }
123
+ when String
124
+ parse_indices_from_condition(conditions.gsub('1 = 1 AND ', '')) #ignore unnecessary conditions
125
+ when Array
126
+ parse_indices_from_condition(*conditions)
127
+ when NilClass
128
+ []
129
+ end
130
+ end
131
+
132
+ AND = /\s+AND\s+/i
133
+ TABLE_AND_COLUMN = /(?:(?:`|")?(\w+)(?:`|")?\.)?(?:`|")?(\w+)(?:`|")?/ # Matches: `users`.id, `users`.`id`, users.id, id
134
+ VALUE = /'?(\d+|\?|(?:(?:[^']|'')*))'?/ # Matches: 123, ?, '123', '12''3'
135
+ KEY_EQ_VALUE = /^\(?#{TABLE_AND_COLUMN}\s+=\s+#{VALUE}\)?$/ # Matches: KEY = VALUE, (KEY = VALUE)
136
+ ORDER = /^#{TABLE_AND_COLUMN}\s*(ASC|DESC)?$/i # Matches: COLUMN ASC, COLUMN DESC, COLUMN
137
+
138
+ def parse_indices_from_condition(conditions = '', *values)
139
+ values = values.dup
140
+ conditions.split(AND).inject([]) do |indices, condition|
141
+ matched, table_name, column_name, sql_value = *(KEY_EQ_VALUE.match(condition))
142
+ if matched
143
+ # value = sql_value == '?' ? values.shift : columns_hash[column_name].type_cast(sql_value)
144
+ if sql_value == '?'
145
+ value = values.shift
146
+ else
147
+ column = columns_hash[column_name]
148
+ raise "could not find column #{column_name} in columns #{columns_hash.keys.join(',')}" if column.nil?
149
+ if sql_value[0..0] == ':' && values && values.count > 0 && values[0].is_a?(Hash)
150
+ symb = sql_value[1..-1].to_sym
151
+ value = column.type_cast(values[0][symb])
152
+ else
153
+ value = column.type_cast(sql_value)
154
+ end
155
+ end
156
+ indices << [column_name, value.is_a?(ActiveRecord::Base) ? value.id : value]
157
+ else
158
+ return nil
159
+ end
160
+ end
161
+ end
162
+
163
+ def indexed_on?(attributes)
164
+ indices.detect { |index| index == attributes }
165
+ rescue NoMethodError
166
+ nil
167
+ end
168
+ alias_method :index_for, :indexed_on?
169
+
170
+ def format_results(cache_keys, objects)
171
+ return objects if objects.blank?
172
+
173
+ objects = convert_to_array(cache_keys, objects)
174
+ objects = apply_limits_and_offsets(objects, @options1)
175
+ deserialize_objects(objects)
176
+ end
177
+
178
+ def choose_deserialized_objects_if_possible(missed_keys, cache_keys, misses, objects)
179
+ missed_keys == cache_keys ? misses : objects
180
+ end
181
+
182
+ def serialize_objects(index, objects)
183
+ Array(objects).collect { |missed| index.serialize_object(missed) }
184
+ end
185
+
186
+ def convert_to_array(cache_keys, object)
187
+ if object.kind_of?(Hash)
188
+ cache_keys.collect { |key| object[cache_key(key)] }.flatten.compact
189
+ else
190
+ Array(object)
191
+ end
192
+ end
193
+
194
+ def apply_limits_and_offsets(results, options)
195
+ results.slice((options[:offset] || 0), (options[:limit] || results.length))
196
+ end
197
+
198
+ def deserialize_objects(objects)
199
+ if objects.first.kind_of?(ActiveRecord::Base)
200
+ objects
201
+ else
202
+ cache_keys = objects.collect { |id| "id/#{id}" }
203
+ with_exclusive_scope(:find => {}) { objects = get(cache_keys, &method(:find_from_keys)) }
204
+ convert_to_array(cache_keys, objects)
205
+ end
206
+ end
207
+
208
+ def find_from_keys(*missing_keys)
209
+ missing_ids = Array(missing_keys).flatten.collect { |key| key.split('/')[2].to_i }
210
+ options = {}
211
+ order_sql = @options1[:order] || @options2[:order]
212
+ options[:order] = order_sql if order_sql
213
+ results = find_from_ids_without_cache(missing_ids, options)
214
+ results.each {|o| @active_record.add_to_caches(o) } if results && results.is_a?(Array)
215
+ results
216
+ end
217
+ end
218
+ end
219
+ end