queris 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +4 -0
  3. data/Gemfile.lock +34 -0
  4. data/README.md +53 -0
  5. data/Rakefile +1 -0
  6. data/data/redis_scripts/add_low_ttl.lua +10 -0
  7. data/data/redis_scripts/copy_key_if_absent.lua +13 -0
  8. data/data/redis_scripts/copy_ttl.lua +13 -0
  9. data/data/redis_scripts/create_page_if_absent.lua +24 -0
  10. data/data/redis_scripts/debuq.lua +20 -0
  11. data/data/redis_scripts/delete_if_string.lua +8 -0
  12. data/data/redis_scripts/delete_matching_keys.lua +7 -0
  13. data/data/redis_scripts/expire_temp_query_keys.lua +7 -0
  14. data/data/redis_scripts/make_rangehack_if_needed.lua +30 -0
  15. data/data/redis_scripts/master_expire.lua +15 -0
  16. data/data/redis_scripts/match_key_type.lua +9 -0
  17. data/data/redis_scripts/move_key.lua +11 -0
  18. data/data/redis_scripts/multisize.lua +19 -0
  19. data/data/redis_scripts/paged_query_ready.lua +35 -0
  20. data/data/redis_scripts/periodic_zremrangebyscore.lua +9 -0
  21. data/data/redis_scripts/persist_reusable_temp_query_keys.lua +14 -0
  22. data/data/redis_scripts/query_ensure_existence.lua +23 -0
  23. data/data/redis_scripts/query_intersect_optimization.lua +31 -0
  24. data/data/redis_scripts/remove_from_keyspace.lua +27 -0
  25. data/data/redis_scripts/remove_from_sets.lua +13 -0
  26. data/data/redis_scripts/results_from_hash.lua +54 -0
  27. data/data/redis_scripts/results_with_ttl.lua +20 -0
  28. data/data/redis_scripts/subquery_intersect_optimization.lua +25 -0
  29. data/data/redis_scripts/subquery_intersect_optimization_cleanup.lua +5 -0
  30. data/data/redis_scripts/undo_add_low_ttl.lua +8 -0
  31. data/data/redis_scripts/unpaged_query_ready.lua +17 -0
  32. data/data/redis_scripts/unpersist_reusable_temp_query_keys.lua +11 -0
  33. data/data/redis_scripts/update_live_expiring_presence_index.lua +20 -0
  34. data/data/redis_scripts/update_query.lua +126 -0
  35. data/data/redis_scripts/update_rangehacks.lua +94 -0
  36. data/data/redis_scripts/zrangestore.lua +12 -0
  37. data/lib/queris.rb +400 -0
  38. data/lib/queris/errors.rb +8 -0
  39. data/lib/queris/indices.rb +735 -0
  40. data/lib/queris/mixin/active_record.rb +74 -0
  41. data/lib/queris/mixin/object.rb +398 -0
  42. data/lib/queris/mixin/ohm.rb +81 -0
  43. data/lib/queris/mixin/queris_model.rb +59 -0
  44. data/lib/queris/model.rb +455 -0
  45. data/lib/queris/profiler.rb +275 -0
  46. data/lib/queris/query.rb +1215 -0
  47. data/lib/queris/query/operations.rb +398 -0
  48. data/lib/queris/query/page.rb +101 -0
  49. data/lib/queris/query/timer.rb +42 -0
  50. data/lib/queris/query/trace.rb +108 -0
  51. data/lib/queris/query_store.rb +137 -0
  52. data/lib/queris/version.rb +3 -0
  53. data/lib/rails/log_subscriber.rb +22 -0
  54. data/lib/rails/request_timing.rb +29 -0
  55. data/lib/tasks/queris.rake +138 -0
  56. data/queris.gemspec +41 -0
  57. data/test.rb +39 -0
  58. data/test/current.rb +74 -0
  59. data/test/dsl.rb +35 -0
  60. data/test/ohm.rb +37 -0
  61. metadata +161 -0
@@ -0,0 +1,74 @@
1
+ module Queris
2
+
3
+ module ActiveRecordMixin
4
+ def self.included base
5
+ base.after_create :create_redis_indices
6
+ base.before_save :update_redis_indices
7
+ base.before_destroy :delete_redis_indices
8
+
9
+ def changed_cacheable_attributes
10
+ changed
11
+ end
12
+
13
+ def all_cacheable_attributes
14
+ attribute_names
15
+ end
16
+
17
+ base.extend ActiveRecordClassMixin
18
+ end
19
+
20
+ module ActiveRecordClassMixin
21
+ def redis_query(arg={}, &block)
22
+ @hashcache ||= stored_in_redis?
23
+ @hashkey ||= @hashcache.key '%s' if @hashcache
24
+ ActiveRecordQuery.new self, arg.merge(from_hash: @hashkey), &block
25
+ end
26
+ def find_all
27
+ find :all
28
+ end
29
+ def stored_in_redis?
30
+ @hashcache = redis_index(:all_attribute_hashcache, Queris::HashCache, false) || false if @hashcache.nil?
31
+ end
32
+ def find_cached(id, opt={})
33
+ @hashcache ||= stored_in_redis?
34
+ if (!opt[:assume_missing] && (obj = @hashcache.fetch(id, opt)))
35
+ return obj
36
+ elsif !opt[:nofallback]
37
+ begin
38
+ obj = find(id)
39
+ rescue
40
+ obj = nil
41
+ end
42
+ @hashcache.create obj if obj
43
+ obj
44
+ end
45
+ end
46
+ def restore(hash, id=nil)
47
+ unless (@hashcache ||= stored_in_redis?)
48
+ raise SchemaError, "Can't restore ActiveRecord model from hash -- there isn't a HashCache index present. (Don't forget to use cache_all_attributes on the model)"
49
+ end
50
+ unless (restored = @hashcache.load_cached hash)
51
+ restored = find_cached id if id
52
+ end
53
+ restored
54
+ end
55
+
56
+ end
57
+ end
58
+
59
+ class ActiveRecordQuery < Query
60
+ attr_accessor :params
61
+ def initialize(model, arg=nil)
62
+ if model.kind_of?(Hash) and arg.nil?
63
+ arg, model = model, model[:model]
64
+ elsif arg.nil?
65
+ arg= {}
66
+ end
67
+ @params = {}
68
+ unless model.kind_of?(Class) && model < ActiveRecord::Base
69
+ raise ArgumentError, ":model arg must be an ActiveRecord model, got #{model.respond_to?(:superclass) ? model.superclass.name : model} instead."
70
+ end
71
+ super model, arg
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,398 @@
1
+ module Queris
2
+
3
+ #black hole, does nothing, is nil.
4
+ class DummyProfiler
5
+ def method_missing(*args)
6
+ self
7
+ end
8
+ def initialize(*args); end
9
+ def nil?; true; end
10
+ end
11
+
12
+ module ObjectMixin
13
+ def self.included base
14
+ base.extend ClassMixin
15
+ end
16
+
17
+ module ClassMixin
18
+ def redis_index(index_match=nil, index_class = Index, strict=true)
19
+ raise ArgumentError, "#{index_class} must be a subclass of Queris::Index" unless index_class <= Index
20
+ case index_match
21
+ when index_class, NilClass, Query
22
+ return index_match
23
+ when String
24
+ index_match = index_match.to_sym
25
+ end
26
+ index = redis_index_hash[index_match]
27
+ if strict
28
+ raise SchemaError, "Index #{index_match} not found in #{name}" unless index
29
+ raise SchemaError, "Found wrong index class: expected #{index_class.name}, found #{index.class.name}" unless index.kind_of? index_class
30
+ end
31
+ index
32
+ end
33
+
34
+ #get all redis indices
35
+ #options:
36
+ # :foreign => true - look in foreign indices (other models' indices indexing from this model)
37
+ # :live => true - look in live indices
38
+ # ONLY ONE of the following will be respected
39
+ # :except => [index_name, ...] - exclude these
40
+ # :attributes => [...] - indices matching any of the given attribute names
41
+ # :names => [...] - indices with any of the given names
42
+ # :class => Queris::IndexClass - indices that are descendants of given class
43
+ def redis_indices(opt={})
44
+ unless Hash === opt
45
+ tmp, opt = opt, {}
46
+ opt[tmp]=true
47
+ end
48
+ if opt[:foreign]
49
+ indices = @foreign_redis_indices || []
50
+ else
51
+ indices = @redis_indices || (superclass.respond_to?(:redis_indices) ? superclass.redis_indices.clone : [])
52
+ end
53
+ if !opt[:attributes].nil?
54
+ attrs = opt[:attributes].map{|v| v.to_sym}.to_set
55
+ indices.select { |index| attrs.member? index.attribute }
56
+ elsif opt[:except]
57
+ except = Array === opt[:except] ? opt[:except] : [ opt[:except] ]
58
+ indices.select { |index| !except.member? index.name }
59
+ elsif opt[:live]
60
+ indices.select { |index| index.live? }
61
+ elsif !opt[:names].nil?
62
+ names = opt[:names].map{|v| v.to_sym}.to_set
63
+ indices.select { |index| names.member? index.name }
64
+ elsif !opt[:class].nil?
65
+ indices.select { |index| opt[:class] === index }
66
+ else
67
+ indices
68
+ end
69
+ #BUG: redis_indices is very static. superclass modifications after class declaration will not count.
70
+ end
71
+ def live_index?(index)
72
+ redis_indices(:live).member? redis_index(index)
73
+ end
74
+ def foreign_index?(index)
75
+ redis_indices(:foreign).member? redis_index(index)
76
+ end
77
+
78
+ def query_profiler; @profiler || DummyProfiler; end
79
+ def profile_queries?; query_profiler.nil?; end
80
+
81
+
82
+ def query(arg={}, &block)
83
+ redis_query arg, &block
84
+ end
85
+
86
+ def page_size(n=nil)
87
+ if n.nil?
88
+ @page_size || 1000
89
+ else
90
+ @page_size=n
91
+ end
92
+ end
93
+
94
+ def info(arg={})
95
+ puts "#{self.name} model info:"
96
+ redis_indices.each do |i|
97
+ next if arg[:live] && !i.live?
98
+ next if arg[:class] && !(arg[:class] === i)
99
+ puts " #{i.info}"
100
+ end
101
+ querykeys = arg[:query_keys] || ((redis || Queris.redis).keys(query.results_key(nil, "*")))
102
+ puts " #{querykeys.count} query-related keys"
103
+ end
104
+
105
+ def clear_queries!
106
+ q = query
107
+ querykeys = redis.keys "#{q.redis_prefix}*"
108
+ #old keys, too
109
+ querykeys.concat redis.keys("#{self.prefix}#{q.class.name}*")
110
+ querykeys.uniq!
111
+ print "Deleting #{querykeys.count} query keys for #{name}..."
112
+ redis.multi do |r|
113
+ querykeys.each {|k| r.del k}
114
+ end
115
+ puts "ok"
116
+ querykeys.count
117
+ end
118
+ def clear_cache!
119
+ indices = redis_indices(:class => HashCache)
120
+ total = 0
121
+ indices.each do |i|
122
+ keymatch = i.key("*", nil, true)
123
+ allkeys = redis.keys(keymatch)
124
+ print "Clearing #{allkeys.count} cache keys for #{name} ..."
125
+ redis.multi do |r|
126
+ allkeys.each { |key| r.del key }
127
+ end
128
+ total += allkeys.count
129
+ puts "ok"
130
+ end
131
+ total
132
+ end
133
+ def redis_query(arg={})
134
+ Queris::Query.new self, arg
135
+ end
136
+
137
+ def redis
138
+ Queris.redis
139
+ end
140
+
141
+ def build_missing_redis_indices
142
+ missing_indices = redis_indices.select do |index|
143
+
144
+ unless index.skip_create?
145
+ found = false
146
+ cursor = "0"
147
+ loop do
148
+ cursor, res = redis.scan [cursor, "match", index.keypattern]
149
+ if res.count > 0
150
+ found = true
151
+ break
152
+ end
153
+ break if cursor == "0"
154
+ end
155
+
156
+ if not found then
157
+ puts "index #{name}:#{index.name} missing (#{index.keypattern})"
158
+ true
159
+ else
160
+ false
161
+ end
162
+ end
163
+ end
164
+ if missing_indices .count > 0
165
+ build_redis_indices missing_indices
166
+ else
167
+ puts "No missing indices for model #{name}."
168
+ end
169
+ end
170
+
171
+ def build_redis_indices(indices=nil, build_foreign = true, incremental_delete=false)
172
+ indices ||= redis_indices
173
+ foreign_indices = []
174
+ indices.select! do |index|
175
+ if index.skip_create?
176
+ false
177
+ elsif Queris::ForeignIndex === index
178
+ foreign_indices << index
179
+ false
180
+ else
181
+ true
182
+ end
183
+ end
184
+ start_time = Time.now
185
+ if indices.count > 0
186
+ print "Loading #{name} data..."
187
+ all = self.find_all
188
+ fetch_time = Time.now - start_time
189
+ puts "\rLoaded #{all.count} entries for #{name} (#{fetch_time.to_f.round} sec)"
190
+ redis_start_time, printy, total =Time.now, 0, all.count - 1
191
+ index_keys = []
192
+
193
+ print "\rDeleting existing indices..." unless incremental_delete
194
+ indices.each {|index| index_keys.concat(redis.keys index.keypattern)} #BUG - race condition may prevent all index values from being deleted
195
+ no_indexing_pipeline do #prevent create_redis_index from running in its own personal multi
196
+ unless incremental_delete
197
+ redis.multi
198
+ index_keys.each { |k| redis.del k }
199
+ end
200
+ next_print, last_i, gap=Time.now, 0, 2
201
+ all.each_with_index do |row, i|
202
+ now = Time.now
203
+ if next_print <= now
204
+ rate = (i - last_i)/(now.to_f - (next_print-gap).to_f)
205
+ last_i=i
206
+ print "\rBuilding redis indices... #{((i.to_f/total) * 100).round.to_i}% (#{i}/#{all.count}) (#{rate.round}/sec)" unless total == 0
207
+ next_print = now + gap
208
+ end
209
+ if incremental_delete
210
+ redis.multi
211
+ row.eliminate_redis_indices indices
212
+ end
213
+ row.create_redis_indices indices
214
+ redis.exec if incremental_delete
215
+ end
216
+ print "\rBuilding redis indices... ok. Committing to redis..."
217
+ redis.exec unless incremental_delete
218
+ print "\rBuilt #{name} redis indices for #{total} rows in #{(Time.now - redis_start_time).round 3} sec. (#{fetch_time.round 3} sec. to fetch all data).\r\n"
219
+ end
220
+ end
221
+ #update all foreign indices
222
+ foreign_by_model={}
223
+ if build_foreign
224
+ foreign_indices.each do |index|
225
+ m = index.real_index.model
226
+ foreign_by_model[m] ||= []
227
+ foreign_by_model[m] << index.real_index
228
+ end
229
+ foreign_by_model.each { |model, indices| model.build_redis_indices indices, true, incremental_delete }
230
+ end
231
+ puts "Built #{indices.count} ind#{indices.count == 1 ? "ex" : "ices"} (#{build_foreign ? foreign_indices.count : 'skipped'} foreign) for #{self.name} in #{(Time.now - start_time).round(3)} seconds."
232
+ self
233
+ end
234
+ def build_redis_index(index_name, incremental_delete=false)
235
+ build_redis_indices [ redis_index(index_name) ], true, incremental_delete
236
+ end
237
+
238
+ def before_query(&block)
239
+ @before_query_callbacks ||= []
240
+ @before_query_callbacks << block
241
+ end
242
+ def before_query_flush(&block)
243
+ @before_query_flush_callbacks ||= []
244
+ @before_query_flush_callbacks << block
245
+ end
246
+
247
+ def run_query_callbacks(which_ones, query)
248
+ callbacks = case which_ones
249
+ when :before, :before_run, :run
250
+ @before_query_callbacks
251
+ when :before_flush, :flush
252
+ @before_query_flush_callbacks
253
+ end
254
+ if callbacks
255
+ callbacks.each {|block| block.call(query)}
256
+ callbacks.count
257
+ else
258
+ 0
259
+ end
260
+ end
261
+
262
+ def add_redis_index(index, opt={})
263
+ raise SchemaError, "Not an index" unless index.kind_of? Index
264
+ #if (found = redis_index_hash[index.name])
265
+ #todo: same-name indices are allowed, but with some caveats.
266
+ #define and check them here.
267
+ #raise "Index #{index.name} (#{found.class.name}) already exists. Trying to add #{index.class.name}"
268
+ #end
269
+ @redis_indices ||= []
270
+ redis_indices.push index
271
+ redis_index_hash[index.name.to_sym]=index
272
+ redis_index_hash[index.class]=index
273
+ index
274
+ end
275
+ def live_queries?
276
+ @live_redis_indices && @live_redis_indices.count
277
+ end
278
+
279
+ def no_indexing_pipeline
280
+ if block_given?
281
+ @no_indexing_pipeline = true
282
+ res = yield
283
+ @no_indexing_pipeline = nil
284
+ return res
285
+ else
286
+ @no_indexing_pipeline
287
+ end
288
+ end
289
+
290
+ private
291
+ def redis_index_hash
292
+ @redis_index_hash ||= superclass.respond_to?(:redis_index_hash) ? superclass.redis_index_hash.clone : {}
293
+ end
294
+
295
+ def profile_queries(another_profiler=nil)
296
+ require "queris/profiler"
297
+ @profiler = case another_profiler
298
+ when :lite
299
+ Queris::QueryProfilerLite
300
+ else
301
+ Queris::QueryProfiler
302
+ end
303
+ end
304
+ attr_accessor :live_redis_indices
305
+
306
+ def index_attribute(arg={}, &block)
307
+ if arg.kind_of? Symbol
308
+ arg = {:attribute => arg }
309
+ end
310
+ index_class = arg[:index] || Queris::SearchIndex
311
+ raise ArgumentError, "index argument must be in Queris::Index if given" unless index_class <= Queris::Index
312
+ Queris.register_model self
313
+ index_class.new(arg.merge(:model => self), &block)
314
+ end
315
+ def index_attribute_for(arg)
316
+ raise ArgumentError ,"index_attribute_for requires :model argument" unless arg[:model]
317
+ index = index_attribute(arg) do |i|
318
+ i.name = "foreign_index_#{i.name}".to_sym
319
+ end
320
+
321
+ foreigner = arg[:model].send :index_attribute, arg.merge(:index=> Queris::ForeignIndex, :real_index => index)
322
+ @foreign_redis_indices ||= []
323
+ @foreign_redis_indices.push foreigner
324
+ foreigner
325
+ end
326
+ def index_attribute_from(arg)
327
+ model = arg[:model]
328
+ model.send(:include, Queris) unless model.include? Queris
329
+ model.send(:index_attribute_for, arg.merge(:model => self))
330
+ end
331
+ def index_range_attribute(arg)
332
+ if arg.kind_of? Symbol
333
+ arg = {:attribute => arg }
334
+ end
335
+ index_attribute arg.merge :index => Queris::RangeIndex
336
+ end
337
+ def index_range_attributes(*arg)
338
+ base_param = arg.last.kind_of?(Hash) ? arg.pop : {}
339
+ arg.each do |attr|
340
+ index_range_attribute base_param.merge :attribute => attr
341
+ end
342
+ end
343
+ alias index_sort_attribute index_range_attribute
344
+ def index_attributes(*arg)
345
+ base_param = arg.last.kind_of?(Hash) ? arg.pop : {}
346
+ arg.each do |attr|
347
+ index_attribute base_param.merge :attribute => attr
348
+ end
349
+ self
350
+ end
351
+ def cache_attribute(attribute_name)
352
+ Queris.register_model self
353
+ Queris::HashCache.new :model => self, :attribute => attribute_name
354
+ end
355
+ def cache_all_attributes
356
+ Queris.register_model self
357
+ Queris::HashCache.new :model => self
358
+ end
359
+ def stored_in_redis?
360
+ nil
361
+ end
362
+ alias :cache_object :cache_all_attributes
363
+ def cache_attribute_from(arg)
364
+ arg[:index]=Queris::HashCache
365
+ index_attribute_from arg
366
+ end
367
+
368
+ end
369
+
370
+ def get_cached_attribute(attr_name)
371
+ redis_index(attr_name, Queris::HashCache).fetch id
372
+ end
373
+
374
+ #in lieu of code cleanup, instance method pollution
375
+ %w(redis_index redis_indices query_profiler profile_queries redis).each do |method_name|
376
+ define_method method_name do |*arg|
377
+ self.class.send method_name, *arg
378
+ end
379
+ end
380
+
381
+ %w(create update delete eliminate).each do |op|
382
+ define_method "#{op}_redis_indices" do |indices=nil, redis=nil|
383
+ if self.class.no_indexing_pipeline || Redis::Pipeline === Queris.redis.client
384
+ (indices || self.class.redis_indices).each do |index|
385
+ index.send op, self unless index.respond_to?("skip_#{op}?") and index.send("skip_#{op}?")
386
+ end
387
+ else
388
+ Queris.redis.multi do #hacky, might bug out on weird configs
389
+ (indices || self.class.redis_indices).each do |index|
390
+ index.send op, self unless index.respond_to?("skip_#{op}?") and index.send("skip_#{op}?")
391
+ end
392
+ end
393
+ end
394
+ end
395
+ end
396
+
397
+ end
398
+ end