zermelo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +16 -0
- data/.rspec +10 -0
- data/.travis.yml +27 -0
- data/Gemfile +20 -0
- data/LICENSE.txt +22 -0
- data/README.md +512 -0
- data/Rakefile +1 -0
- data/lib/zermelo/associations/association_data.rb +24 -0
- data/lib/zermelo/associations/belongs_to.rb +115 -0
- data/lib/zermelo/associations/class_methods.rb +244 -0
- data/lib/zermelo/associations/has_and_belongs_to_many.rb +128 -0
- data/lib/zermelo/associations/has_many.rb +120 -0
- data/lib/zermelo/associations/has_one.rb +109 -0
- data/lib/zermelo/associations/has_sorted_set.rb +124 -0
- data/lib/zermelo/associations/index.rb +50 -0
- data/lib/zermelo/associations/index_data.rb +18 -0
- data/lib/zermelo/associations/unique_index.rb +44 -0
- data/lib/zermelo/backends/base.rb +115 -0
- data/lib/zermelo/backends/influxdb_backend.rb +178 -0
- data/lib/zermelo/backends/redis_backend.rb +281 -0
- data/lib/zermelo/filters/base.rb +235 -0
- data/lib/zermelo/filters/influxdb_filter.rb +162 -0
- data/lib/zermelo/filters/redis_filter.rb +558 -0
- data/lib/zermelo/filters/steps/base_step.rb +22 -0
- data/lib/zermelo/filters/steps/diff_range_step.rb +17 -0
- data/lib/zermelo/filters/steps/diff_step.rb +17 -0
- data/lib/zermelo/filters/steps/intersect_range_step.rb +17 -0
- data/lib/zermelo/filters/steps/intersect_step.rb +17 -0
- data/lib/zermelo/filters/steps/limit_step.rb +17 -0
- data/lib/zermelo/filters/steps/offset_step.rb +17 -0
- data/lib/zermelo/filters/steps/sort_step.rb +17 -0
- data/lib/zermelo/filters/steps/union_range_step.rb +17 -0
- data/lib/zermelo/filters/steps/union_step.rb +17 -0
- data/lib/zermelo/locks/no_lock.rb +16 -0
- data/lib/zermelo/locks/redis_lock.rb +221 -0
- data/lib/zermelo/records/base.rb +62 -0
- data/lib/zermelo/records/class_methods.rb +127 -0
- data/lib/zermelo/records/collection.rb +14 -0
- data/lib/zermelo/records/errors.rb +24 -0
- data/lib/zermelo/records/influxdb_record.rb +35 -0
- data/lib/zermelo/records/instance_methods.rb +224 -0
- data/lib/zermelo/records/key.rb +19 -0
- data/lib/zermelo/records/redis_record.rb +27 -0
- data/lib/zermelo/records/type_validator.rb +20 -0
- data/lib/zermelo/version.rb +3 -0
- data/lib/zermelo.rb +102 -0
- data/spec/lib/zermelo/associations/belongs_to_spec.rb +6 -0
- data/spec/lib/zermelo/associations/has_many_spec.rb +6 -0
- data/spec/lib/zermelo/associations/has_one_spec.rb +6 -0
- data/spec/lib/zermelo/associations/has_sorted_set.spec.rb +6 -0
- data/spec/lib/zermelo/associations/index_spec.rb +6 -0
- data/spec/lib/zermelo/associations/unique_index_spec.rb +6 -0
- data/spec/lib/zermelo/backends/influxdb_backend_spec.rb +0 -0
- data/spec/lib/zermelo/backends/moneta_backend_spec.rb +0 -0
- data/spec/lib/zermelo/filters/influxdb_filter_spec.rb +0 -0
- data/spec/lib/zermelo/filters/redis_filter_spec.rb +0 -0
- data/spec/lib/zermelo/locks/redis_lock_spec.rb +170 -0
- data/spec/lib/zermelo/records/influxdb_record_spec.rb +258 -0
- data/spec/lib/zermelo/records/key_spec.rb +6 -0
- data/spec/lib/zermelo/records/redis_record_spec.rb +1426 -0
- data/spec/lib/zermelo/records/type_validator_spec.rb +6 -0
- data/spec/lib/zermelo/version_spec.rb +6 -0
- data/spec/lib/zermelo_spec.rb +6 -0
- data/spec/spec_helper.rb +67 -0
- data/spec/support/profile_all_formatter.rb +44 -0
- data/spec/support/uncolored_doc_formatter.rb +74 -0
- data/zermelo.gemspec +30 -0
- metadata +174 -0
@@ -0,0 +1,558 @@
|
|
1
|
+
require 'zermelo/filters/base'
|
2
|
+
|
3
|
+
# TODO check escaping of ids and index_keys -- shouldn't allow bare :
|
4
|
+
|
5
|
+
module Zermelo
|
6
|
+
|
7
|
+
module Filters
|
8
|
+
|
9
|
+
class RedisFilter
|
10
|
+
|
11
|
+
# abstraction for a set or list of record ids
|
12
|
+
class Collection
|
13
|
+
attr_reader :name, :type
|
14
|
+
def initialize(opts = {})
|
15
|
+
@name = opts[:name]
|
16
|
+
@type = opts[:type]
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
include Zermelo::Filters::Base
|
21
|
+
|
22
|
+
# more step users
|
23
|
+
def first
|
24
|
+
unless [:list, :sorted_set].include?(@initial_set.type) ||
|
25
|
+
@steps.any? {|s| s.is_a?(Zermelo::Filters::Steps::SortStep) }
|
26
|
+
|
27
|
+
raise "Can't get first member of a non-sorted set"
|
28
|
+
end
|
29
|
+
|
30
|
+
lock {
|
31
|
+
first_id = resolve_steps do |collection|
|
32
|
+
op = {:list => :lrange, :sorted_set => :zrange}[collection.type]
|
33
|
+
Zermelo.redis.send(op, collection.name, 0, 0).first
|
34
|
+
end
|
35
|
+
first_id.nil? ? nil : _load(first_id)
|
36
|
+
}
|
37
|
+
end
|
38
|
+
|
39
|
+
def last
|
40
|
+
unless [:list, :sorted_set].include?(@initial_set.type) ||
|
41
|
+
@steps.any? {|s| s.is_a?(Zermelo::Filters::Steps::SortStep) }
|
42
|
+
|
43
|
+
raise "Can't get last member of a non-sorted set"
|
44
|
+
end
|
45
|
+
|
46
|
+
lock {
|
47
|
+
last_id = resolve_steps do |collection|
|
48
|
+
op = {:list => :lrevrange, :sorted_set => :zrevrange}[collection.type]
|
49
|
+
Zermelo.redis.send(op, collection.name, 0, 0).first
|
50
|
+
end
|
51
|
+
last_id.nil? ? nil : _load(last_id)
|
52
|
+
}
|
53
|
+
end
|
54
|
+
# end step users
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def _count
|
59
|
+
resolve_steps(:list => :llen,
|
60
|
+
:set => :scard,
|
61
|
+
:sorted_set => :zcard)
|
62
|
+
end
|
63
|
+
|
64
|
+
def _ids
|
65
|
+
resolve_steps(:list => :lrange,
|
66
|
+
:set => :smembers,
|
67
|
+
:sorted_set => :zrange)
|
68
|
+
end
|
69
|
+
|
70
|
+
def _exists?(id)
|
71
|
+
return if id.nil?
|
72
|
+
resolve_steps do |collection|
|
73
|
+
case collection.type
|
74
|
+
when :list
|
75
|
+
Zermelo.redis.lrange(collection.name, 0, -1).include?(id)
|
76
|
+
when :set
|
77
|
+
Zermelo.redis.sismember(collection.name, id)
|
78
|
+
when :sorted_set
|
79
|
+
!Zermelo.redis.zscore(collection.name, id).nil?
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def temp_set_name
|
85
|
+
"#{class_key}::tmp:#{SecureRandom.hex(16)}"
|
86
|
+
end
|
87
|
+
|
88
|
+
def class_key
|
89
|
+
@class_key ||= @associated_class.send(:class_key)
|
90
|
+
end
|
91
|
+
|
92
|
+
def indexed_step_to_set(att, idx_class, value, attr_type)
|
93
|
+
# TODO (maybe) if a filter from different backend, resolve to ids and
|
94
|
+
# put that in a Redis temp set
|
95
|
+
|
96
|
+
case value
|
97
|
+
when Zermelo::Filters::RedisFilter
|
98
|
+
|
99
|
+
collection, should_be_deleted = value.resolve_steps
|
100
|
+
|
101
|
+
if should_be_deleted
|
102
|
+
temp_sets << collection.name
|
103
|
+
end
|
104
|
+
|
105
|
+
unless :set.eql?(collection.type)
|
106
|
+
raise "Unsure as yet if non-sets are safe as Filter step values"
|
107
|
+
end
|
108
|
+
|
109
|
+
when Regexp
|
110
|
+
raise "Can't query non-string values via regexp" unless :string.eql?(attr_type)
|
111
|
+
idx_result = temp_set_name
|
112
|
+
starts_with_string_re = /^string:/
|
113
|
+
case idx_class.name
|
114
|
+
when 'Zermelo::Associations::UniqueIndex'
|
115
|
+
index_key = backend.key_to_redis_key(Zermelo::Records::Key.new(
|
116
|
+
:klass => class_key,
|
117
|
+
:name => "by_#{att}",
|
118
|
+
:type => :hash,
|
119
|
+
:object => :index
|
120
|
+
))
|
121
|
+
candidates = Zermelo.redis.hgetall(index_key)
|
122
|
+
matching_ids = candidates.values_at(*candidates.keys.select {|k|
|
123
|
+
(starts_with_string_re === k) &&
|
124
|
+
(value === backend.unescape_key_name(k.sub(starts_with_string_re, '')))
|
125
|
+
})
|
126
|
+
Zermelo.redis.sadd(idx_result, matching_ids) unless matching_ids.empty?
|
127
|
+
when 'Zermelo::Associations::Index'
|
128
|
+
key_root = backend.key_to_redis_key(Zermelo::Records::Key.new(
|
129
|
+
:klass => class_key,
|
130
|
+
:name => "by_#{att}:string",
|
131
|
+
:type => :set,
|
132
|
+
:object => :index
|
133
|
+
))
|
134
|
+
|
135
|
+
matching_sets = Zermelo.redis.keys(key_root + ":*").inject([]) do |memo, k|
|
136
|
+
k =~ /^#{key_root}:(.+)$/
|
137
|
+
memo << k if value === $1
|
138
|
+
memo
|
139
|
+
end
|
140
|
+
|
141
|
+
Zermelo.redis.sunionstore(idx_result, matching_sets) unless matching_sets.empty?
|
142
|
+
end
|
143
|
+
[idx_result, true]
|
144
|
+
else
|
145
|
+
index = @associated_class.send("#{att}_index")
|
146
|
+
|
147
|
+
case index
|
148
|
+
when Zermelo::Associations::UniqueIndex
|
149
|
+
idx_result = temp_set_name
|
150
|
+
Zermelo.redis.sadd(idx_result,
|
151
|
+
Zermelo.redis.hget(backend.key_to_redis_key(index.key),
|
152
|
+
backend.index_keys(attr_type, value).join(':')))
|
153
|
+
[idx_result, true]
|
154
|
+
when Zermelo::Associations::Index
|
155
|
+
[backend.key_to_redis_key(index.key(value)), false]
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def resolve_step(step, source, idx_attrs, attr_types, &block)
|
161
|
+
temp_sets = []
|
162
|
+
source_keys = []
|
163
|
+
|
164
|
+
case step
|
165
|
+
when Zermelo::Filters::Steps::IntersectStep,
|
166
|
+
Zermelo::Filters::Steps::UnionStep,
|
167
|
+
Zermelo::Filters::Steps::DiffStep
|
168
|
+
|
169
|
+
source_keys += step.attributes.inject([]) do |memo, (att, value)|
|
170
|
+
|
171
|
+
val = value.is_a?(Set) ? value.to_a : value
|
172
|
+
|
173
|
+
if :id.eql?(att)
|
174
|
+
ts = temp_set_name
|
175
|
+
temp_sets << ts
|
176
|
+
Zermelo.redis.sadd(ts, val)
|
177
|
+
memo << ts
|
178
|
+
else
|
179
|
+
idx_class = idx_attrs[att.to_s]
|
180
|
+
raise "'#{att}' property is not indexed" if idx_class.nil?
|
181
|
+
|
182
|
+
if val.is_a?(Enumerable)
|
183
|
+
conditions_set = temp_set_name
|
184
|
+
temp_idx_sets = []
|
185
|
+
Zermelo.redis.sunionstore(conditions_set, *val.collect {|v|
|
186
|
+
idx_set, clear = indexed_step_to_set(att, idx_class, v, attr_types[att])
|
187
|
+
temp_idx_sets << idx_set if clear
|
188
|
+
idx_set
|
189
|
+
})
|
190
|
+
Zermelo.redis.del(temp_idx_sets) unless temp_idx_sets.empty?
|
191
|
+
temp_sets << conditions_set
|
192
|
+
memo << conditions_set
|
193
|
+
else
|
194
|
+
idx_set, clear = indexed_step_to_set(att, idx_class, val, attr_types[att])
|
195
|
+
temp_sets << idx_set if clear
|
196
|
+
memo << idx_set
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
memo
|
201
|
+
end
|
202
|
+
|
203
|
+
when Zermelo::Filters::Steps::IntersectRangeStep,
|
204
|
+
Zermelo::Filters::Steps::UnionRangeStep,
|
205
|
+
Zermelo::Filters::Steps::DiffRangeStep
|
206
|
+
|
207
|
+
range_ids_set = temp_set_name
|
208
|
+
|
209
|
+
options = step.options || {}
|
210
|
+
|
211
|
+
start = options[:start]
|
212
|
+
finish = options[:finish]
|
213
|
+
|
214
|
+
order_desc = options[:desc].is_a?(TrueClass)
|
215
|
+
|
216
|
+
if options[:by_score]
|
217
|
+
start = '-inf' if start.nil? || (start <= 0)
|
218
|
+
finish = '+inf' if finish.nil? || (finish <= 0)
|
219
|
+
else
|
220
|
+
start = 0 if start.nil?
|
221
|
+
finish = -1 if finish.nil?
|
222
|
+
end
|
223
|
+
|
224
|
+
args = [start, finish]
|
225
|
+
|
226
|
+
if order_desc
|
227
|
+
if options[:by_score]
|
228
|
+
query = :zrevrangebyscore
|
229
|
+
args = args.map(&:to_s).reverse
|
230
|
+
else
|
231
|
+
query = :zrevrange
|
232
|
+
end
|
233
|
+
elsif options[:by_score]
|
234
|
+
query = :zrangebyscore
|
235
|
+
args = args.map(&:to_s)
|
236
|
+
else
|
237
|
+
query = :zrange
|
238
|
+
end
|
239
|
+
|
240
|
+
args << {:with_scores => :true}
|
241
|
+
|
242
|
+
if options[:limit]
|
243
|
+
args.last.update(:limit => [0, options[:limit].to_i])
|
244
|
+
end
|
245
|
+
|
246
|
+
args.unshift(source)
|
247
|
+
|
248
|
+
range_ids_scores = Zermelo.redis.send(query, *args)
|
249
|
+
|
250
|
+
unless range_ids_scores.empty?
|
251
|
+
Zermelo.redis.zadd(range_ids_set, range_ids_scores.map(&:reverse))
|
252
|
+
end
|
253
|
+
source_keys << range_ids_set
|
254
|
+
temp_sets << range_ids_set
|
255
|
+
end
|
256
|
+
|
257
|
+
yield(source_keys)
|
258
|
+
|
259
|
+
unless temp_sets.empty?
|
260
|
+
Zermelo.redis.del(*temp_sets)
|
261
|
+
temp_sets.clear
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
# TODO could parts of this move to a stored Lua script in the redis server?
|
266
|
+
|
267
|
+
# If called with a block -- takes a block and passes the name of a set to
|
268
|
+
# it; deletes all temporary sets once done
|
269
|
+
|
270
|
+
# If called with any arguments -- treats them as a hash of shortcuts
|
271
|
+
|
272
|
+
# If not called with any arguments -- returns two values, the first is
|
273
|
+
# the name of a set containing the filtered ids, the second is a boolean
|
274
|
+
# for whether or not to clear up that set once it's been used
|
275
|
+
|
276
|
+
def resolve_steps(shortcuts = {}, &block)
|
277
|
+
source = backend.key_to_redis_key(@initial_set)
|
278
|
+
source_type = @initial_set.type
|
279
|
+
|
280
|
+
if @steps.empty?
|
281
|
+
ret = if shortcuts.empty?
|
282
|
+
data = Zermelo::Filters::RedisFilter::Collection.new(
|
283
|
+
:name => source, :type => source_type)
|
284
|
+
if block.nil?
|
285
|
+
[data, false]
|
286
|
+
else
|
287
|
+
block.call(data)
|
288
|
+
end
|
289
|
+
else
|
290
|
+
if :sorted_set.eql?(source_type) && :zrange.eql?(shortcuts[source_type])
|
291
|
+
Zermelo.redis.zrange(source, 0, -1)
|
292
|
+
elsif :list.eql?(source_type) && :lrange.eql?(shortcuts[source_type])
|
293
|
+
Zermelo.redis.lrange(source, 0, -1)
|
294
|
+
else
|
295
|
+
Zermelo.redis.send(shortcuts[source_type], source)
|
296
|
+
end
|
297
|
+
end
|
298
|
+
return ret
|
299
|
+
end
|
300
|
+
|
301
|
+
temp_sets = []
|
302
|
+
dest_set = nil
|
303
|
+
ret = nil
|
304
|
+
|
305
|
+
idx_attrs = @associated_class.send(:with_index_data) do |d|
|
306
|
+
d.each_with_object({}) do |(name, data), memo|
|
307
|
+
memo[name.to_s] = data.index_klass
|
308
|
+
end
|
309
|
+
end
|
310
|
+
|
311
|
+
attr_types = @associated_class.send(:attribute_types)
|
312
|
+
|
313
|
+
offset = nil
|
314
|
+
limit = nil
|
315
|
+
order_desc = nil
|
316
|
+
|
317
|
+
members = nil
|
318
|
+
|
319
|
+
begin
|
320
|
+
|
321
|
+
@steps.each_with_index do |step, idx|
|
322
|
+
|
323
|
+
resolve_step(step, source, idx_attrs, attr_types) do |source_keys|
|
324
|
+
options = step.options || {}
|
325
|
+
|
326
|
+
order_desc = order_desc ^ options[:desc].is_a?(TrueClass)
|
327
|
+
|
328
|
+
unless step.class.accepted_types.include?(source_type)
|
329
|
+
raise "'#{step.class.name}' does not accept input type #{source_type}"
|
330
|
+
end
|
331
|
+
|
332
|
+
case source_type
|
333
|
+
when :set
|
334
|
+
sort_set = if step.is_a?(Zermelo::Filters::Steps::SortStep)
|
335
|
+
|
336
|
+
proc {
|
337
|
+
|
338
|
+
# TODO raise error in step construction if keys not
|
339
|
+
# passed as expected below
|
340
|
+
sort_attrs_and_orders = case options[:keys]
|
341
|
+
when String, Symbol
|
342
|
+
{options[:keys].to_s => options[:desc].is_a?(TrueClass) ? :desc : :asc}
|
343
|
+
when Array
|
344
|
+
options[:keys].each_with_object({}) do |k, memo|
|
345
|
+
memo[k.to_sym] = (options[:desc].is_a?(TrueClass) ? :desc : :asc)
|
346
|
+
end
|
347
|
+
when Hash
|
348
|
+
options[:keys]
|
349
|
+
end
|
350
|
+
|
351
|
+
# TODO check if complex attribute types or associations
|
352
|
+
# can be used for sorting
|
353
|
+
|
354
|
+
Zermelo.redis.sunionstore(dest_set, source, *source_keys)
|
355
|
+
|
356
|
+
sort_attrs_and_orders.keys.reverse.each_with_index do |sort_attr, idx|
|
357
|
+
|
358
|
+
order = sort_attrs_and_orders[sort_attr]
|
359
|
+
|
360
|
+
opts = {}
|
361
|
+
|
362
|
+
unless 'id'.eql?(sort_attr.to_s)
|
363
|
+
opts.update(:by => "#{class_key}:*:attrs->#{sort_attr}")
|
364
|
+
end
|
365
|
+
|
366
|
+
if (idx + 1) == sort_attrs_and_orders.size
|
367
|
+
# only apply offset & limit on the last sort
|
368
|
+
o = options[:offset]
|
369
|
+
l = options[:limit]
|
370
|
+
|
371
|
+
if !(l.nil? && o.nil?)
|
372
|
+
o = o.nil? ? 0 : o.to_i
|
373
|
+
l = (l.nil? || (l.to_i < 1)) ? (Zermelo.redis.llen(dest_set) - o) : l
|
374
|
+
opts.update(:limit => [o, l])
|
375
|
+
end
|
376
|
+
end
|
377
|
+
|
378
|
+
order_parts = []
|
379
|
+
sort_attr_type = attr_types[sort_attr.to_sym]
|
380
|
+
unless [:integer, :float, :timestamp].include?(sort_attr_type)
|
381
|
+
order_parts << 'alpha'
|
382
|
+
end
|
383
|
+
order_parts << 'desc' if 'desc'.eql?(order.to_s)
|
384
|
+
|
385
|
+
unless order_parts.empty?
|
386
|
+
opts.update(:order => order_parts.join(' '))
|
387
|
+
end
|
388
|
+
|
389
|
+
opts.update(:store => dest_set)
|
390
|
+
Zermelo.redis.sort(dest_set, opts)
|
391
|
+
end
|
392
|
+
|
393
|
+
source_type = :list
|
394
|
+
}
|
395
|
+
else
|
396
|
+
nil
|
397
|
+
end
|
398
|
+
|
399
|
+
last_step_and_smembers = (idx == (@steps.size - 1)) && :smembers.eql?(shortcuts[:set])
|
400
|
+
|
401
|
+
unless last_step_and_smembers && step.is_a?(Zermelo::Filters::Steps::IntersectStep)
|
402
|
+
dest_set = temp_set_name
|
403
|
+
temp_sets << dest_set
|
404
|
+
end
|
405
|
+
|
406
|
+
if last_step_and_smembers
|
407
|
+
members = case step
|
408
|
+
when Zermelo::Filters::Steps::UnionStep
|
409
|
+
Zermelo.redis.sinterstore(dest_set, *source_keys)
|
410
|
+
Zermelo.redis.sunion(dest_set, source)
|
411
|
+
when Zermelo::Filters::Steps::IntersectStep
|
412
|
+
Zermelo.redis.sinter(source, *source_keys)
|
413
|
+
when Zermelo::Filters::Steps::DiffStep
|
414
|
+
Zermelo.redis.sinterstore(dest_set, *source_keys)
|
415
|
+
Zermelo.redis.sdiff(source, dest_set)
|
416
|
+
when Zermelo::Filters::Steps::SortStep
|
417
|
+
sort_set.call
|
418
|
+
Zermelo.redis.send((order_desc ? :lrevrange : :lrange),
|
419
|
+
dest_set, 0, -1)
|
420
|
+
end
|
421
|
+
else
|
422
|
+
|
423
|
+
case step
|
424
|
+
when Zermelo::Filters::Steps::UnionStep
|
425
|
+
Zermelo.redis.sinterstore(dest_set, *source_keys)
|
426
|
+
Zermelo.redis.sunionstore(dest_set, source, dest_set)
|
427
|
+
when Zermelo::Filters::Steps::IntersectStep
|
428
|
+
Zermelo.redis.sinterstore(dest_set, *source_keys)
|
429
|
+
when Zermelo::Filters::Steps::DiffStep
|
430
|
+
Zermelo.redis.sinterstore(dest_set, *source_keys)
|
431
|
+
Zermelo.redis.sdiffstore(dest_set, source, dest_set)
|
432
|
+
when Zermelo::Filters::Steps::SortStep
|
433
|
+
sort_set.call
|
434
|
+
end
|
435
|
+
|
436
|
+
source = dest_set
|
437
|
+
end
|
438
|
+
|
439
|
+
when :list
|
440
|
+
# TODO could allow reversion into set by :union, :intersect, :diff,
|
441
|
+
# or application of :sort again to re-order. For now, YAGNI, and
|
442
|
+
# document the limitations.
|
443
|
+
|
444
|
+
case step
|
445
|
+
when Zermelo::Filters::Steps::OffsetStep
|
446
|
+
offset = options[:amount]
|
447
|
+
when Zermelo::Filters::Steps::LimitStep
|
448
|
+
limit = options[:amount]
|
449
|
+
end
|
450
|
+
|
451
|
+
when :sorted_set
|
452
|
+
weights = case step
|
453
|
+
when Zermelo::Filters::Steps::UnionStep, Zermelo::Filters::Steps::UnionRangeStep
|
454
|
+
[0.0] * source_keys.length
|
455
|
+
when Zermelo::Filters::Steps::DiffStep, Zermelo::Filters::Steps::DiffRangeStep
|
456
|
+
[-1.0] * source_keys.length
|
457
|
+
end
|
458
|
+
|
459
|
+
dest_set = temp_set_name
|
460
|
+
temp_sets << dest_set
|
461
|
+
|
462
|
+
case step
|
463
|
+
when Zermelo::Filters::Steps::UnionStep, Zermelo::Filters::Steps::UnionRangeStep
|
464
|
+
Zermelo.redis.zinterstore(dest_set, source_keys, :weights => weights, :aggregate => 'max')
|
465
|
+
Zermelo.redis.zunionstore(dest_set, [source, dest_set])
|
466
|
+
when Zermelo::Filters::Steps::IntersectStep, Zermelo::Filters::Steps::IntersectRangeStep
|
467
|
+
Zermelo.redis.zinterstore(dest_set, [source] + source_keys, :weights => weights, :aggregate => 'max')
|
468
|
+
when Zermelo::Filters::Steps::DiffStep, Zermelo::Filters::Steps::DiffRangeStep
|
469
|
+
# 'zdiffstore' via weights, relies on non-zero scores being used
|
470
|
+
# see https://code.google.com/p/redis/issues/detail?id=579
|
471
|
+
Zermelo.redis.zinterstore(dest_set, source_keys, :weights => weights, :aggregate => 'max')
|
472
|
+
Zermelo.redis.zunionstore(dest_set, [source, dest_set])
|
473
|
+
Zermelo.redis.zremrangebyscore(dest_set, "0", "0")
|
474
|
+
end
|
475
|
+
|
476
|
+
if (idx == (@steps.size - 1)) && :zrange.eql?(shortcuts[:sorted_set])
|
477
|
+
# supporting shortcut structures here as it helps preserve the
|
478
|
+
# win gained by the shortcut for empty steps, but this is
|
479
|
+
# no better than passing it through to a block would be; if
|
480
|
+
# Redis still supported ZINTER and ZUNION it would work better
|
481
|
+
members = Zermelo.redis.send((order_desc ? :zrevrange : :zrange),
|
482
|
+
dest_set, 0, -1)
|
483
|
+
else
|
484
|
+
source = dest_set
|
485
|
+
end
|
486
|
+
|
487
|
+
end
|
488
|
+
end
|
489
|
+
|
490
|
+
end
|
491
|
+
|
492
|
+
ret = if members.nil?
|
493
|
+
if :list.eql?(source_type) && !(offset.nil? && limit.nil?)
|
494
|
+
|
495
|
+
# TODO need a guaranteed non-existing key for non-sorting 'sort'
|
496
|
+
o = offset.to_i
|
497
|
+
l = limit.to_i
|
498
|
+
l = (Zermelo.redis.llen(dest_set) - o) if (limit < 1)
|
499
|
+
|
500
|
+
opts = {:by => 'no_sort', :limit => [o, l]}
|
501
|
+
|
502
|
+
# https://github.com/antirez/redis/issues/2079, fixed in redis 2.8.19
|
503
|
+
if (Zermelo.redis_version <=> ['2', '8', '18']) == 1
|
504
|
+
opts.update(:store => dest_set)
|
505
|
+
Zermelo.redis.sort(dest_set, opts)
|
506
|
+
else
|
507
|
+
data = Zermelo.redis.sort(dest_set, opts)
|
508
|
+
|
509
|
+
if data.empty?
|
510
|
+
Zermelo.redis.del(dest_set)
|
511
|
+
else
|
512
|
+
limited = temp_set_name
|
513
|
+
temp_sets << limited
|
514
|
+
|
515
|
+
Zermelo.redis.rpush(limited, data)
|
516
|
+
|
517
|
+
dest_set = limited
|
518
|
+
end
|
519
|
+
end
|
520
|
+
end
|
521
|
+
|
522
|
+
if shortcuts.empty?
|
523
|
+
data = Zermelo::Filters::RedisFilter::Collection.new(
|
524
|
+
:name => dest_set, :type => source_type)
|
525
|
+
if block.nil?
|
526
|
+
should_be_deleted = !temp_sets.delete(dest_set).nil?
|
527
|
+
[data, should_be_deleted]
|
528
|
+
else
|
529
|
+
block.call(data)
|
530
|
+
end
|
531
|
+
elsif :sorted_set.eql?(source_type) && :zrange.eql?(shortcuts[source_type])
|
532
|
+
Zermelo.redis.zrange(dest_set, 0, -1)
|
533
|
+
elsif :list.eql?(source_type) && :lrange.eql?(shortcuts[source_type])
|
534
|
+
Zermelo.redis.lrange(dest_set, 0, -1)
|
535
|
+
else
|
536
|
+
Zermelo.redis.send(shortcuts[source_type], dest_set)
|
537
|
+
end
|
538
|
+
|
539
|
+
else
|
540
|
+
members
|
541
|
+
end
|
542
|
+
|
543
|
+
rescue
|
544
|
+
raise
|
545
|
+
ensure
|
546
|
+
unless temp_sets.empty?
|
547
|
+
Zermelo.redis.del(*temp_sets)
|
548
|
+
temp_sets.clear
|
549
|
+
end
|
550
|
+
end
|
551
|
+
|
552
|
+
ret
|
553
|
+
end
|
554
|
+
end
|
555
|
+
|
556
|
+
end
|
557
|
+
|
558
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Zermelo
|
2
|
+
module Filters
|
3
|
+
class Steps
|
4
|
+
class BaseStep
|
5
|
+
def self.accepted_types
|
6
|
+
raise "Must be implemented in subclass"
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.returns_type
|
10
|
+
raise "Must be implemented in subclass"
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_reader :options, :attributes
|
14
|
+
|
15
|
+
def initialize(opts, attrs)
|
16
|
+
@options = opts
|
17
|
+
@attributes = attrs
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'zermelo/filters/steps/base_step'
|
2
|
+
|
3
|
+
module Zermelo
|
4
|
+
module Filters
|
5
|
+
class Steps
|
6
|
+
class DiffRangeStep < Zermelo::Filters::Steps::BaseStep
|
7
|
+
def self.accepted_types
|
8
|
+
[:sorted_set]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.returns_type
|
12
|
+
:sorted_set
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'zermelo/filters/steps/base_step'
|
2
|
+
|
3
|
+
module Zermelo
|
4
|
+
module Filters
|
5
|
+
class Steps
|
6
|
+
class DiffStep < Zermelo::Filters::Steps::BaseStep
|
7
|
+
def self.accepted_types
|
8
|
+
[:set, :sorted_set]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.returns_type
|
12
|
+
:set
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'zermelo/filters/steps/base_step'
|
2
|
+
|
3
|
+
module Zermelo
|
4
|
+
module Filters
|
5
|
+
class Steps
|
6
|
+
class IntersectRangeStep < Zermelo::Filters::Steps::BaseStep
|
7
|
+
def self.accepted_types
|
8
|
+
[:sorted_set]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.returns_type
|
12
|
+
:sorted_set
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'zermelo/filters/steps/base_step'
|
2
|
+
|
3
|
+
module Zermelo
|
4
|
+
module Filters
|
5
|
+
class Steps
|
6
|
+
class IntersectStep < Zermelo::Filters::Steps::BaseStep
|
7
|
+
def self.accepted_types
|
8
|
+
[:set, :sorted_set]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.returns_type
|
12
|
+
:set
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'zermelo/filters/steps/base_step'
|
2
|
+
|
3
|
+
module Zermelo
|
4
|
+
module Filters
|
5
|
+
class Steps
|
6
|
+
class LimitStep < Zermelo::Filters::Steps::BaseStep
|
7
|
+
def self.accepted_types
|
8
|
+
[:list]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.returns_type
|
12
|
+
:list
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'zermelo/filters/steps/base_step'
|
2
|
+
|
3
|
+
module Zermelo
|
4
|
+
module Filters
|
5
|
+
class Steps
|
6
|
+
class OffsetStep < Zermelo::Filters::Steps::BaseStep
|
7
|
+
def self.accepted_types
|
8
|
+
[:list]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.returns_type
|
12
|
+
:list
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|