zermelo 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec +1 -1
- data/.travis.yml +3 -5
- data/CHANGELOG.md +8 -2
- data/lib/zermelo/associations/belongs_to.rb +3 -3
- data/lib/zermelo/associations/has_and_belongs_to_many.rb +3 -3
- data/lib/zermelo/associations/has_many.rb +3 -3
- data/lib/zermelo/associations/has_one.rb +3 -3
- data/lib/zermelo/associations/has_sorted_set.rb +4 -4
- data/lib/zermelo/associations/index.rb +1 -2
- data/lib/zermelo/associations/unique_index.rb +1 -2
- data/lib/zermelo/backends/base.rb +1 -1
- data/lib/zermelo/backends/influxdb_backend.rb +29 -18
- data/lib/zermelo/backends/redis_backend.rb +106 -6
- data/lib/zermelo/filters/base.rb +34 -57
- data/lib/zermelo/filters/influxdb_filter.rb +22 -70
- data/lib/zermelo/filters/redis_filter.rb +35 -482
- data/lib/zermelo/filters/steps/list_step.rb +79 -0
- data/lib/zermelo/filters/steps/set_step.rb +176 -0
- data/lib/zermelo/filters/steps/sort_step.rb +85 -0
- data/lib/zermelo/filters/steps/sorted_set_step.rb +156 -0
- data/lib/zermelo/records/class_methods.rb +16 -4
- data/lib/zermelo/records/influxdb_record.rb +2 -0
- data/lib/zermelo/records/instance_methods.rb +4 -4
- data/lib/zermelo/records/key.rb +2 -0
- data/lib/zermelo/version.rb +1 -1
- data/lib/zermelo.rb +9 -1
- data/spec/lib/zermelo/records/influxdb_record_spec.rb +186 -10
- data/spec/lib/zermelo/records/redis_record_spec.rb +11 -4
- data/spec/spec_helper.rb +12 -10
- metadata +5 -11
- data/lib/zermelo/filters/steps/diff_range_step.rb +0 -17
- data/lib/zermelo/filters/steps/diff_step.rb +0 -17
- data/lib/zermelo/filters/steps/intersect_range_step.rb +0 -17
- data/lib/zermelo/filters/steps/intersect_step.rb +0 -17
- data/lib/zermelo/filters/steps/limit_step.rb +0 -17
- data/lib/zermelo/filters/steps/offset_step.rb +0 -17
- data/lib/zermelo/filters/steps/union_range_step.rb +0 -17
- data/lib/zermelo/filters/steps/union_step.rb +0 -17
- data/lib/zermelo/records/collection.rb +0 -14
@@ -8,46 +8,21 @@ module Zermelo
|
|
8
8
|
|
9
9
|
class RedisFilter
|
10
10
|
|
11
|
-
# abstraction for a set or list of record ids
|
12
|
-
class Collection
|
13
|
-
attr_reader :name, :type
|
14
|
-
def initialize(opts = {})
|
15
|
-
@name = opts[:name]
|
16
|
-
@type = opts[:type]
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
11
|
include Zermelo::Filters::Base
|
21
12
|
|
13
|
+
# TODO polite error when first/last applied to set
|
14
|
+
|
22
15
|
# more step users
|
23
16
|
def first
|
24
|
-
unless [:list, :sorted_set].include?(@initial_set.type) ||
|
25
|
-
@steps.any? {|s| s.is_a?(Zermelo::Filters::Steps::SortStep) }
|
26
|
-
|
27
|
-
raise "Can't get first member of a non-sorted set"
|
28
|
-
end
|
29
|
-
|
30
17
|
lock {
|
31
|
-
first_id = resolve_steps
|
32
|
-
op = {:list => :lrange, :sorted_set => :zrange}[collection.type]
|
33
|
-
Zermelo.redis.send(op, collection.name, 0, 0).first
|
34
|
-
end
|
18
|
+
first_id = resolve_steps(:first)
|
35
19
|
first_id.nil? ? nil : _load(first_id)
|
36
20
|
}
|
37
21
|
end
|
38
22
|
|
39
23
|
def last
|
40
|
-
unless [:list, :sorted_set].include?(@initial_set.type) ||
|
41
|
-
@steps.any? {|s| s.is_a?(Zermelo::Filters::Steps::SortStep) }
|
42
|
-
|
43
|
-
raise "Can't get last member of a non-sorted set"
|
44
|
-
end
|
45
|
-
|
46
24
|
lock {
|
47
|
-
last_id = resolve_steps
|
48
|
-
op = {:list => :lrevrange, :sorted_set => :zrevrange}[collection.type]
|
49
|
-
Zermelo.redis.send(op, collection.name, 0, 0).first
|
50
|
-
end
|
25
|
+
last_id = resolve_steps(:last)
|
51
26
|
last_id.nil? ? nil : _load(last_id)
|
52
27
|
}
|
53
28
|
end
|
@@ -56,214 +31,18 @@ module Zermelo
|
|
56
31
|
private
|
57
32
|
|
58
33
|
def _count
|
59
|
-
resolve_steps(:
|
60
|
-
:set => :scard,
|
61
|
-
:sorted_set => :zcard)
|
34
|
+
resolve_steps(:count)
|
62
35
|
end
|
63
36
|
|
64
37
|
def _ids
|
65
|
-
resolve_steps(:
|
66
|
-
:set => :smembers,
|
67
|
-
:sorted_set => :zrange)
|
38
|
+
resolve_steps(:ids)
|
68
39
|
end
|
69
40
|
|
70
41
|
def _exists?(id)
|
71
42
|
return if id.nil?
|
72
|
-
resolve_steps
|
73
|
-
case collection.type
|
74
|
-
when :list
|
75
|
-
Zermelo.redis.lrange(collection.name, 0, -1).include?(id)
|
76
|
-
when :set
|
77
|
-
Zermelo.redis.sismember(collection.name, id)
|
78
|
-
when :sorted_set
|
79
|
-
!Zermelo.redis.zscore(collection.name, id).nil?
|
80
|
-
end
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
def temp_set_name
|
85
|
-
"#{class_key}::tmp:#{SecureRandom.hex(16)}"
|
86
|
-
end
|
87
|
-
|
88
|
-
def class_key
|
89
|
-
@class_key ||= @associated_class.send(:class_key)
|
90
|
-
end
|
91
|
-
|
92
|
-
def indexed_step_to_set(att, idx_class, value, attr_type)
|
93
|
-
# TODO (maybe) if a filter from different backend, resolve to ids and
|
94
|
-
# put that in a Redis temp set
|
95
|
-
|
96
|
-
case value
|
97
|
-
when Zermelo::Filters::RedisFilter
|
98
|
-
|
99
|
-
collection, should_be_deleted = value.resolve_steps
|
100
|
-
|
101
|
-
if should_be_deleted
|
102
|
-
temp_sets << collection.name
|
103
|
-
end
|
104
|
-
|
105
|
-
unless :set.eql?(collection.type)
|
106
|
-
raise "Unsure as yet if non-sets are safe as Filter step values"
|
107
|
-
end
|
108
|
-
|
109
|
-
when Regexp
|
110
|
-
raise "Can't query non-string values via regexp" unless :string.eql?(attr_type)
|
111
|
-
idx_result = temp_set_name
|
112
|
-
starts_with_string_re = /^string:/
|
113
|
-
case idx_class.name
|
114
|
-
when 'Zermelo::Associations::UniqueIndex'
|
115
|
-
index_key = backend.key_to_redis_key(Zermelo::Records::Key.new(
|
116
|
-
:klass => class_key,
|
117
|
-
:name => "by_#{att}",
|
118
|
-
:type => :hash,
|
119
|
-
:object => :index
|
120
|
-
))
|
121
|
-
candidates = Zermelo.redis.hgetall(index_key)
|
122
|
-
matching_ids = candidates.values_at(*candidates.keys.select {|k|
|
123
|
-
(starts_with_string_re === k) &&
|
124
|
-
(value === backend.unescape_key_name(k.sub(starts_with_string_re, '')))
|
125
|
-
})
|
126
|
-
Zermelo.redis.sadd(idx_result, matching_ids) unless matching_ids.empty?
|
127
|
-
when 'Zermelo::Associations::Index'
|
128
|
-
key_root = backend.key_to_redis_key(Zermelo::Records::Key.new(
|
129
|
-
:klass => class_key,
|
130
|
-
:name => "by_#{att}:string",
|
131
|
-
:type => :set,
|
132
|
-
:object => :index
|
133
|
-
))
|
134
|
-
|
135
|
-
matching_sets = Zermelo.redis.keys(key_root + ":*").inject([]) do |memo, k|
|
136
|
-
k =~ /^#{key_root}:(.+)$/
|
137
|
-
memo << k if value === $1
|
138
|
-
memo
|
139
|
-
end
|
140
|
-
|
141
|
-
Zermelo.redis.sunionstore(idx_result, matching_sets) unless matching_sets.empty?
|
142
|
-
end
|
143
|
-
[idx_result, true]
|
144
|
-
else
|
145
|
-
index = @associated_class.send("#{att}_index")
|
146
|
-
|
147
|
-
case index
|
148
|
-
when Zermelo::Associations::UniqueIndex
|
149
|
-
idx_result = temp_set_name
|
150
|
-
Zermelo.redis.sadd(idx_result,
|
151
|
-
Zermelo.redis.hget(backend.key_to_redis_key(index.key),
|
152
|
-
backend.index_keys(attr_type, value).join(':')))
|
153
|
-
[idx_result, true]
|
154
|
-
when Zermelo::Associations::Index
|
155
|
-
[backend.key_to_redis_key(index.key(value)), false]
|
156
|
-
end
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
|
-
def resolve_step(step, source, idx_attrs, attr_types, &block)
|
161
|
-
temp_sets = []
|
162
|
-
source_keys = []
|
163
|
-
|
164
|
-
case step
|
165
|
-
when Zermelo::Filters::Steps::IntersectStep,
|
166
|
-
Zermelo::Filters::Steps::UnionStep,
|
167
|
-
Zermelo::Filters::Steps::DiffStep
|
168
|
-
|
169
|
-
source_keys += step.attributes.inject([]) do |memo, (att, value)|
|
170
|
-
|
171
|
-
val = value.is_a?(Set) ? value.to_a : value
|
172
|
-
|
173
|
-
if :id.eql?(att)
|
174
|
-
ts = temp_set_name
|
175
|
-
temp_sets << ts
|
176
|
-
Zermelo.redis.sadd(ts, val)
|
177
|
-
memo << ts
|
178
|
-
else
|
179
|
-
idx_class = idx_attrs[att.to_s]
|
180
|
-
raise "'#{att}' property is not indexed" if idx_class.nil?
|
181
|
-
|
182
|
-
if val.is_a?(Enumerable)
|
183
|
-
conditions_set = temp_set_name
|
184
|
-
temp_idx_sets = []
|
185
|
-
Zermelo.redis.sunionstore(conditions_set, *val.collect {|v|
|
186
|
-
idx_set, clear = indexed_step_to_set(att, idx_class, v, attr_types[att])
|
187
|
-
temp_idx_sets << idx_set if clear
|
188
|
-
idx_set
|
189
|
-
})
|
190
|
-
Zermelo.redis.del(temp_idx_sets) unless temp_idx_sets.empty?
|
191
|
-
temp_sets << conditions_set
|
192
|
-
memo << conditions_set
|
193
|
-
else
|
194
|
-
idx_set, clear = indexed_step_to_set(att, idx_class, val, attr_types[att])
|
195
|
-
temp_sets << idx_set if clear
|
196
|
-
memo << idx_set
|
197
|
-
end
|
198
|
-
end
|
199
|
-
|
200
|
-
memo
|
201
|
-
end
|
202
|
-
|
203
|
-
when Zermelo::Filters::Steps::IntersectRangeStep,
|
204
|
-
Zermelo::Filters::Steps::UnionRangeStep,
|
205
|
-
Zermelo::Filters::Steps::DiffRangeStep
|
206
|
-
|
207
|
-
range_ids_set = temp_set_name
|
208
|
-
|
209
|
-
options = step.options || {}
|
210
|
-
|
211
|
-
start = options[:start]
|
212
|
-
finish = options[:finish]
|
213
|
-
|
214
|
-
order_desc = options[:desc].is_a?(TrueClass)
|
215
|
-
|
216
|
-
if options[:by_score]
|
217
|
-
start = '-inf' if start.nil? || (start <= 0)
|
218
|
-
finish = '+inf' if finish.nil? || (finish <= 0)
|
219
|
-
else
|
220
|
-
start = 0 if start.nil?
|
221
|
-
finish = -1 if finish.nil?
|
222
|
-
end
|
223
|
-
|
224
|
-
args = [start, finish]
|
225
|
-
|
226
|
-
if order_desc
|
227
|
-
if options[:by_score]
|
228
|
-
query = :zrevrangebyscore
|
229
|
-
args = args.map(&:to_s).reverse
|
230
|
-
else
|
231
|
-
query = :zrevrange
|
232
|
-
end
|
233
|
-
elsif options[:by_score]
|
234
|
-
query = :zrangebyscore
|
235
|
-
args = args.map(&:to_s)
|
236
|
-
else
|
237
|
-
query = :zrange
|
238
|
-
end
|
239
|
-
|
240
|
-
args << {:with_scores => :true}
|
241
|
-
|
242
|
-
if options[:limit]
|
243
|
-
args.last.update(:limit => [0, options[:limit].to_i])
|
244
|
-
end
|
245
|
-
|
246
|
-
args.unshift(source)
|
247
|
-
|
248
|
-
range_ids_scores = Zermelo.redis.send(query, *args)
|
249
|
-
|
250
|
-
unless range_ids_scores.empty?
|
251
|
-
Zermelo.redis.zadd(range_ids_set, range_ids_scores.map(&:reverse))
|
252
|
-
end
|
253
|
-
source_keys << range_ids_set
|
254
|
-
temp_sets << range_ids_set
|
255
|
-
end
|
256
|
-
|
257
|
-
yield(source_keys)
|
258
|
-
|
259
|
-
unless temp_sets.empty?
|
260
|
-
Zermelo.redis.del(*temp_sets)
|
261
|
-
temp_sets.clear
|
262
|
-
end
|
43
|
+
resolve_steps(:exists?, id)
|
263
44
|
end
|
264
45
|
|
265
|
-
# TODO could parts of this move to a stored Lua script in the redis server?
|
266
|
-
|
267
46
|
# If called with a block -- takes a block and passes the name of a set to
|
268
47
|
# it; deletes all temporary sets once done
|
269
48
|
|
@@ -273,35 +52,22 @@ module Zermelo
|
|
273
52
|
# the name of a set containing the filtered ids, the second is a boolean
|
274
53
|
# for whether or not to clear up that set once it's been used
|
275
54
|
|
276
|
-
def resolve_steps(
|
277
|
-
source = backend.key_to_redis_key(@initial_set)
|
278
|
-
source_type = @initial_set.type
|
279
|
-
|
55
|
+
def resolve_steps(shortcut, *args)
|
280
56
|
if @steps.empty?
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
57
|
+
sc_klass = {
|
58
|
+
:list => Zermelo::Filters::Steps::ListStep,
|
59
|
+
:set => Zermelo::Filters::Steps::SetStep,
|
60
|
+
:sorted_set => Zermelo::Filters::Steps::SortedSetStep
|
61
|
+
}[@initial_key.type]
|
62
|
+
sc = sc_klass.const_get(:REDIS_SHORTCUTS)[shortcut]
|
63
|
+
ret = if sc.nil?
|
64
|
+
yield(@initial_key)
|
289
65
|
else
|
290
|
-
|
291
|
-
Zermelo.redis.zrange(source, 0, -1)
|
292
|
-
elsif :list.eql?(source_type) && :lrange.eql?(shortcuts[source_type])
|
293
|
-
Zermelo.redis.lrange(source, 0, -1)
|
294
|
-
else
|
295
|
-
Zermelo.redis.send(shortcuts[source_type], source)
|
296
|
-
end
|
66
|
+
sc.call(*([backend.key_to_redis_key(@initial_key)] + args))
|
297
67
|
end
|
298
|
-
return
|
68
|
+
return(ret)
|
299
69
|
end
|
300
70
|
|
301
|
-
temp_sets = []
|
302
|
-
dest_set = nil
|
303
|
-
ret = nil
|
304
|
-
|
305
71
|
idx_attrs = @associated_class.send(:with_index_data) do |d|
|
306
72
|
d.each_with_object({}) do |(name, data), memo|
|
307
73
|
memo[name.to_s] = data.index_klass
|
@@ -310,246 +76,33 @@ module Zermelo
|
|
310
76
|
|
311
77
|
attr_types = @associated_class.send(:attribute_types)
|
312
78
|
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
members = nil
|
318
|
-
|
319
|
-
begin
|
320
|
-
|
321
|
-
@steps.each_with_index do |step, idx|
|
322
|
-
|
323
|
-
resolve_step(step, source, idx_attrs, attr_types) do |source_keys|
|
324
|
-
options = step.options || {}
|
325
|
-
|
326
|
-
order_desc = order_desc ^ options[:desc].is_a?(TrueClass)
|
327
|
-
|
328
|
-
unless step.class.accepted_types.include?(source_type)
|
329
|
-
raise "'#{step.class.name}' does not accept input type #{source_type}"
|
330
|
-
end
|
331
|
-
|
332
|
-
case source_type
|
333
|
-
when :set
|
334
|
-
sort_set = if step.is_a?(Zermelo::Filters::Steps::SortStep)
|
335
|
-
|
336
|
-
proc {
|
337
|
-
|
338
|
-
# TODO raise error in step construction if keys not
|
339
|
-
# passed as expected below
|
340
|
-
sort_attrs_and_orders = case options[:keys]
|
341
|
-
when String, Symbol
|
342
|
-
{options[:keys].to_s => options[:desc].is_a?(TrueClass) ? :desc : :asc}
|
343
|
-
when Array
|
344
|
-
options[:keys].each_with_object({}) do |k, memo|
|
345
|
-
memo[k.to_sym] = (options[:desc].is_a?(TrueClass) ? :desc : :asc)
|
346
|
-
end
|
347
|
-
when Hash
|
348
|
-
options[:keys]
|
349
|
-
end
|
350
|
-
|
351
|
-
# TODO check if complex attribute types or associations
|
352
|
-
# can be used for sorting
|
353
|
-
|
354
|
-
Zermelo.redis.sunionstore(dest_set, source, *source_keys)
|
355
|
-
|
356
|
-
sort_attrs_and_orders.keys.reverse.each_with_index do |sort_attr, idx|
|
357
|
-
|
358
|
-
order = sort_attrs_and_orders[sort_attr]
|
359
|
-
|
360
|
-
opts = {}
|
79
|
+
backend.temp_key_wrap do |temp_keys|
|
80
|
+
result = nil
|
81
|
+
last_step = @steps.last
|
361
82
|
|
362
|
-
|
363
|
-
|
364
|
-
|
83
|
+
step_opts = {
|
84
|
+
:index_attrs => idx_attrs,
|
85
|
+
:attr_types => attr_types,
|
86
|
+
:temp_keys => temp_keys,
|
87
|
+
:source => @initial_key
|
88
|
+
}
|
365
89
|
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
l = options[:limit]
|
370
|
-
|
371
|
-
if !(l.nil? && o.nil?)
|
372
|
-
o = o.nil? ? 0 : o.to_i
|
373
|
-
l = (l.nil? || (l.to_i < 1)) ? (Zermelo.redis.llen(dest_set) - o) : l
|
374
|
-
opts.update(:limit => [o, l])
|
375
|
-
end
|
376
|
-
end
|
377
|
-
|
378
|
-
order_parts = []
|
379
|
-
sort_attr_type = attr_types[sort_attr.to_sym]
|
380
|
-
unless [:integer, :float, :timestamp].include?(sort_attr_type)
|
381
|
-
order_parts << 'alpha'
|
382
|
-
end
|
383
|
-
order_parts << 'desc' if 'desc'.eql?(order.to_s)
|
384
|
-
|
385
|
-
unless order_parts.empty?
|
386
|
-
opts.update(:order => order_parts.join(' '))
|
387
|
-
end
|
388
|
-
|
389
|
-
opts.update(:store => dest_set)
|
390
|
-
Zermelo.redis.sort(dest_set, opts)
|
391
|
-
end
|
392
|
-
|
393
|
-
source_type = :list
|
394
|
-
}
|
395
|
-
else
|
396
|
-
nil
|
397
|
-
end
|
398
|
-
|
399
|
-
last_step_and_smembers = (idx == (@steps.size - 1)) && :smembers.eql?(shortcuts[:set])
|
400
|
-
|
401
|
-
unless last_step_and_smembers && step.is_a?(Zermelo::Filters::Steps::IntersectStep)
|
402
|
-
dest_set = temp_set_name
|
403
|
-
temp_sets << dest_set
|
404
|
-
end
|
405
|
-
|
406
|
-
if last_step_and_smembers
|
407
|
-
members = case step
|
408
|
-
when Zermelo::Filters::Steps::UnionStep
|
409
|
-
Zermelo.redis.sinterstore(dest_set, *source_keys)
|
410
|
-
Zermelo.redis.sunion(dest_set, source)
|
411
|
-
when Zermelo::Filters::Steps::IntersectStep
|
412
|
-
Zermelo.redis.sinter(source, *source_keys)
|
413
|
-
when Zermelo::Filters::Steps::DiffStep
|
414
|
-
Zermelo.redis.sinterstore(dest_set, *source_keys)
|
415
|
-
Zermelo.redis.sdiff(source, dest_set)
|
416
|
-
when Zermelo::Filters::Steps::SortStep
|
417
|
-
sort_set.call
|
418
|
-
Zermelo.redis.send((order_desc ? :lrevrange : :lrange),
|
419
|
-
dest_set, 0, -1)
|
420
|
-
end
|
421
|
-
else
|
422
|
-
|
423
|
-
case step
|
424
|
-
when Zermelo::Filters::Steps::UnionStep
|
425
|
-
Zermelo.redis.sinterstore(dest_set, *source_keys)
|
426
|
-
Zermelo.redis.sunionstore(dest_set, source, dest_set)
|
427
|
-
when Zermelo::Filters::Steps::IntersectStep
|
428
|
-
Zermelo.redis.sinterstore(dest_set, *source_keys)
|
429
|
-
when Zermelo::Filters::Steps::DiffStep
|
430
|
-
Zermelo.redis.sinterstore(dest_set, *source_keys)
|
431
|
-
Zermelo.redis.sdiffstore(dest_set, source, dest_set)
|
432
|
-
when Zermelo::Filters::Steps::SortStep
|
433
|
-
sort_set.call
|
434
|
-
end
|
435
|
-
|
436
|
-
source = dest_set
|
437
|
-
end
|
438
|
-
|
439
|
-
when :list
|
440
|
-
# TODO could allow reversion into set by :union, :intersect, :diff,
|
441
|
-
# or application of :sort again to re-order. For now, YAGNI, and
|
442
|
-
# document the limitations.
|
443
|
-
|
444
|
-
case step
|
445
|
-
when Zermelo::Filters::Steps::OffsetStep
|
446
|
-
offset = options[:amount]
|
447
|
-
when Zermelo::Filters::Steps::LimitStep
|
448
|
-
limit = options[:amount]
|
449
|
-
end
|
450
|
-
|
451
|
-
when :sorted_set
|
452
|
-
weights = case step
|
453
|
-
when Zermelo::Filters::Steps::UnionStep, Zermelo::Filters::Steps::UnionRangeStep
|
454
|
-
[0.0] * source_keys.length
|
455
|
-
when Zermelo::Filters::Steps::DiffStep, Zermelo::Filters::Steps::DiffRangeStep
|
456
|
-
[-1.0] * source_keys.length
|
457
|
-
end
|
458
|
-
|
459
|
-
dest_set = temp_set_name
|
460
|
-
temp_sets << dest_set
|
461
|
-
|
462
|
-
case step
|
463
|
-
when Zermelo::Filters::Steps::UnionStep, Zermelo::Filters::Steps::UnionRangeStep
|
464
|
-
Zermelo.redis.zinterstore(dest_set, source_keys, :weights => weights, :aggregate => 'max')
|
465
|
-
Zermelo.redis.zunionstore(dest_set, [source, dest_set])
|
466
|
-
when Zermelo::Filters::Steps::IntersectStep, Zermelo::Filters::Steps::IntersectRangeStep
|
467
|
-
Zermelo.redis.zinterstore(dest_set, [source] + source_keys, :weights => weights, :aggregate => 'max')
|
468
|
-
when Zermelo::Filters::Steps::DiffStep, Zermelo::Filters::Steps::DiffRangeStep
|
469
|
-
# 'zdiffstore' via weights, relies on non-zero scores being used
|
470
|
-
# see https://code.google.com/p/redis/issues/detail?id=579
|
471
|
-
Zermelo.redis.zinterstore(dest_set, source_keys, :weights => weights, :aggregate => 'max')
|
472
|
-
Zermelo.redis.zunionstore(dest_set, [source, dest_set])
|
473
|
-
Zermelo.redis.zremrangebyscore(dest_set, "0", "0")
|
474
|
-
end
|
475
|
-
|
476
|
-
if (idx == (@steps.size - 1)) && :zrange.eql?(shortcuts[:sorted_set])
|
477
|
-
# supporting shortcut structures here as it helps preserve the
|
478
|
-
# win gained by the shortcut for empty steps, but this is
|
479
|
-
# no better than passing it through to a block would be; if
|
480
|
-
# Redis still supported ZINTER and ZUNION it would work better
|
481
|
-
members = Zermelo.redis.send((order_desc ? :zrevrange : :zrange),
|
482
|
-
dest_set, 0, -1)
|
483
|
-
else
|
484
|
-
source = dest_set
|
485
|
-
end
|
486
|
-
|
487
|
-
end
|
90
|
+
@steps.each do |step|
|
91
|
+
unless step.class.accepted_types.include?(step_opts[:source].type)
|
92
|
+
raise "'#{step.class.name}' does not accept input type #{step_opts[:source].type}"
|
488
93
|
end
|
489
94
|
|
490
|
-
|
491
|
-
|
492
|
-
ret = if members.nil?
|
493
|
-
if :list.eql?(source_type) && !(offset.nil? && limit.nil?)
|
494
|
-
|
495
|
-
# TODO need a guaranteed non-existing key for non-sorting 'sort'
|
496
|
-
o = offset.to_i
|
497
|
-
l = limit.to_i
|
498
|
-
l = (Zermelo.redis.llen(dest_set) - o) if (limit < 1)
|
499
|
-
|
500
|
-
opts = {:by => 'no_sort', :limit => [o, l]}
|
501
|
-
|
502
|
-
# https://github.com/antirez/redis/issues/2079, fixed in redis 2.8.19
|
503
|
-
if (Zermelo.redis_version <=> ['2', '8', '18']) == 1
|
504
|
-
opts.update(:store => dest_set)
|
505
|
-
Zermelo.redis.sort(dest_set, opts)
|
506
|
-
else
|
507
|
-
data = Zermelo.redis.sort(dest_set, opts)
|
508
|
-
|
509
|
-
if data.empty?
|
510
|
-
Zermelo.redis.del(dest_set)
|
511
|
-
else
|
512
|
-
limited = temp_set_name
|
513
|
-
temp_sets << limited
|
514
|
-
|
515
|
-
Zermelo.redis.rpush(limited, data)
|
516
|
-
|
517
|
-
dest_set = limited
|
518
|
-
end
|
519
|
-
end
|
95
|
+
if step == last_step
|
96
|
+
step_opts.update(:shortcut => shortcut, :shortcut_args => args)
|
520
97
|
end
|
521
98
|
|
522
|
-
|
523
|
-
data = Zermelo::Filters::RedisFilter::Collection.new(
|
524
|
-
:name => dest_set, :type => source_type)
|
525
|
-
if block.nil?
|
526
|
-
should_be_deleted = !temp_sets.delete(dest_set).nil?
|
527
|
-
[data, should_be_deleted]
|
528
|
-
else
|
529
|
-
block.call(data)
|
530
|
-
end
|
531
|
-
elsif :sorted_set.eql?(source_type) && :zrange.eql?(shortcuts[source_type])
|
532
|
-
Zermelo.redis.zrange(dest_set, 0, -1)
|
533
|
-
elsif :list.eql?(source_type) && :lrange.eql?(shortcuts[source_type])
|
534
|
-
Zermelo.redis.lrange(dest_set, 0, -1)
|
535
|
-
else
|
536
|
-
Zermelo.redis.send(shortcuts[source_type], dest_set)
|
537
|
-
end
|
99
|
+
result = step.resolve(backend, @associated_class, step_opts)
|
538
100
|
|
539
|
-
|
540
|
-
members
|
101
|
+
step_opts[:source] = result unless step == last_step
|
541
102
|
end
|
542
103
|
|
543
|
-
|
544
|
-
raise
|
545
|
-
ensure
|
546
|
-
unless temp_sets.empty?
|
547
|
-
Zermelo.redis.del(*temp_sets)
|
548
|
-
temp_sets.clear
|
549
|
-
end
|
104
|
+
result
|
550
105
|
end
|
551
|
-
|
552
|
-
ret
|
553
106
|
end
|
554
107
|
end
|
555
108
|
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'zermelo/filters/steps/base_step'
|
2
|
+
|
3
|
+
module Zermelo
|
4
|
+
module Filters
|
5
|
+
class Steps
|
6
|
+
class ListStep < Zermelo::Filters::Steps::BaseStep
|
7
|
+
def self.accepted_types
|
8
|
+
[:list]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.returns_type
|
12
|
+
:list
|
13
|
+
end
|
14
|
+
|
15
|
+
REDIS_SHORTCUTS = {
|
16
|
+
:ids => proc {|key| Zermelo.redis.lrange(key, 0, -1) },
|
17
|
+
:count => proc {|key| Zermelo.redis.llen(key) },
|
18
|
+
:exists? => proc {|key, id| Zermelo.redis.lrange(key, 0, -1).include?(id) },
|
19
|
+
:first => proc {|key| Zermelo.redis.lrange(key, 0, 0).first },
|
20
|
+
:last => proc {|key| Zermelo.redis.lrevrange(key, 0, 0).first }
|
21
|
+
}
|
22
|
+
|
23
|
+
def resolve(backend, associated_class, opts = {})
|
24
|
+
shortcut = opts[:shortcut]
|
25
|
+
|
26
|
+
offset = @options[:offset]
|
27
|
+
limit = @options[:limit]
|
28
|
+
|
29
|
+
o = offset.to_i
|
30
|
+
l = limit.to_i
|
31
|
+
|
32
|
+
case backend
|
33
|
+
when Zermelo::Backends::RedisBackend
|
34
|
+
|
35
|
+
source = opts[:source]
|
36
|
+
idx_attrs = opts[:index_attrs]
|
37
|
+
attr_types = opts[:attr_types]
|
38
|
+
temp_keys = opts[:temp_keys]
|
39
|
+
|
40
|
+
# TODO apply these transformations via a subset?
|
41
|
+
# TODO need a guaranteed non-existing key for non-sorting 'sort'
|
42
|
+
|
43
|
+
# TODO check if source is in temp_keys, use a generated temp_key instead if not
|
44
|
+
r_source = backend.key_to_redis_key(source)
|
45
|
+
|
46
|
+
l = (Zermelo.redis.llen(r_source) - o) if (l < 1)
|
47
|
+
|
48
|
+
sort_opts = {:by => 'no_sort', :limit => [o, l]}
|
49
|
+
|
50
|
+
# https://github.com/antirez/redis/issues/2079, fixed in redis 2.8.19
|
51
|
+
result, r_result = if (Zermelo.redis_version.split('.') <=> ['2', '8', '18']) == 1
|
52
|
+
sort_opts.update(:store => r_source)
|
53
|
+
Zermelo.redis.sort(r_source, sort_opts)
|
54
|
+
[source, r_source]
|
55
|
+
else
|
56
|
+
data = Zermelo.redis.sort(r_source, sort_opts)
|
57
|
+
|
58
|
+
if data.empty?
|
59
|
+
# TODO fix
|
60
|
+
else
|
61
|
+
limited = associated_class.send(:temp_key, :list)
|
62
|
+
temp_keys << limited
|
63
|
+
r_limited = backend.key_to_redis_key(limited)
|
64
|
+
|
65
|
+
Zermelo.redis.rpush(r_limited, data)
|
66
|
+
|
67
|
+
[limited, r_limited]
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
return result if shortcut.nil?
|
72
|
+
REDIS_SHORTCUTS[shortcut].call(*([r_result] + opts[:shortcut_args]))
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|