rhcf-timeseries 0.0.6 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rhcf/extensions/fixnum.rb +34 -0
- data/lib/rhcf/timeseries/constants.rb +25 -0
- data/lib/rhcf/timeseries/manager.rb +119 -0
- data/lib/rhcf/timeseries/query.rb +69 -0
- data/lib/rhcf/timeseries/redis_strategies.rb +175 -0
- data/lib/rhcf/timeseries/version.rb +1 -1
- data/rhcf-timeseries.gemspec +2 -2
- data/spec/lib/rhcf/timeseries/manager_spec.rb +193 -0
- data/spec/lib/rhcf/timeseries/{result_spec.rb → query_spec.rb} +6 -6
- data/spec/lib/rhcf/timeseries/redis_get_strategy_spec.rb +7 -0
- data/spec/lib/rhcf/timeseries/redis_hgetall_strategy_spec.rb +7 -0
- data/spec/lib/rhcf/timeseries/redis_mget_lua_strategy_spec.rb +7 -0
- data/spec/lib/rhcf/timeseries/redis_mget_strategy_spec.rb +7 -0
- data/spec/spec_helper.rb +1 -2
- data/spec/support/database_cleaner.rb +14 -0
- data/spec/support/simplecov.rb +2 -0
- data/spec/support/valid_strategy_spec.rb +119 -0
- metadata +53 -7
- data/lib/rhcf/timeseries/redis.rb +0 -269
- data/spec/lib/rhcf/timeseries/redis_spec.rb +0 -182
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6ac0f81efa0f044a261c6919b0107b48e68efe24
|
4
|
+
data.tar.gz: 9e4248aec431ffb9c0f8dc07817c4af1189ba458
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 15e810d8f9f5b309f4fe92fcd0cc38a18d9ec3522ac85c00eb8f546113fd17cd23d5074fcaec349f49ae0699adb3e5cdede6c7ceff7b583844a25015a46dabaa
|
7
|
+
data.tar.gz: d3781b3ab20ef7f918d5f74353b37023323a7b4253b15b52d695efe2f2df50615aeaae8672c07cb3f9eab374ccb5408f56e2f57836695073d642733c4e3199d3
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class Fixnum
|
2
|
+
def minutes
|
3
|
+
self * 60
|
4
|
+
end
|
5
|
+
|
6
|
+
def hours
|
7
|
+
self.minutes * 60
|
8
|
+
end
|
9
|
+
|
10
|
+
def days
|
11
|
+
self.hours * 24
|
12
|
+
end
|
13
|
+
|
14
|
+
def seconds
|
15
|
+
self
|
16
|
+
end
|
17
|
+
|
18
|
+
def weeks
|
19
|
+
self.days * 7
|
20
|
+
end
|
21
|
+
|
22
|
+
def years
|
23
|
+
self.days * 365
|
24
|
+
end
|
25
|
+
|
26
|
+
alias_method :day, :days
|
27
|
+
alias_method :week, :weeks
|
28
|
+
alias_method :hour, :hours
|
29
|
+
alias_method :second, :seconds
|
30
|
+
alias_method :minute, :minutes
|
31
|
+
alias_method :year, :years
|
32
|
+
end
|
33
|
+
|
34
|
+
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Rhcf
|
2
|
+
module Timeseries
|
3
|
+
EVENT_SET_TOKEN = 'ES'
|
4
|
+
EVENT_POINT_TOKEN = 'P'
|
5
|
+
DEFAULT_PREFIX = 'TS'
|
6
|
+
|
7
|
+
DEFAULT_RESOLUTIONS_MAP={
|
8
|
+
:ever => {span:Float::INFINITY, formatter: "ever", ttl: (2 * 366).days},
|
9
|
+
:year => {span: 365.days,formatter: "%Y", ttl: (2 * 366).days},
|
10
|
+
:week => {span: 1.week, formatter: "%Y-CW%w", ttl: 90.days},
|
11
|
+
:month => {span: 30.days, formatter: "%Y-%m", ttl: 366.days},
|
12
|
+
:day => {span: 1.day, formatter: "%Y-%m-%d", ttl: 30.days},
|
13
|
+
:hour => {span: 1.hour, formatter: "%Y-%m-%dT%H", ttl: 24.hours},
|
14
|
+
:minute => {span: 1.minute, formatter: "%Y-%m-%dT%H:%M", ttl: 120.minutes},
|
15
|
+
:second => {span: 1, formatter: "%Y-%m-%dT%H:%M:%S", ttl: 1.hour},
|
16
|
+
:"5seconds" => {span: 5.seconds, formatter: ->(time){ [time.strftime("%Y-%m-%dT%H:%M:") , time.to_i % 60/5, '*',5].join('') }, ttl: 1.hour},
|
17
|
+
:"5minutes" => {span: 5.minutes, formatter: ->(time){ [time.strftime("%Y-%m-%dT%H:") , (time.to_i/60) % 60/5, '*',5].join('') }, ttl: 3.hour},
|
18
|
+
:"15minutes" => {span: 15.minutes, formatter: ->(time){ [time.strftime("%Y-%m-%dT%H:") , (time.to_i/60) % 60/15, '*',15].join('') }, ttl: 24.hours}
|
19
|
+
|
20
|
+
}
|
21
|
+
|
22
|
+
DEFAULT_RESOLUTIONS = DEFAULT_RESOLUTIONS_MAP.keys
|
23
|
+
NAMESPACE_SEPARATOR = '|'
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
unless 1.respond_to?(:minute)
|
2
|
+
require_relative '../extensions/fixnum'
|
3
|
+
end
|
4
|
+
require 'rhcf/timeseries/constants'
|
5
|
+
require 'rhcf/timeseries/query'
|
6
|
+
require 'rhcf/timeseries/redis_strategies'
|
7
|
+
|
8
|
+
module Rhcf
|
9
|
+
module Timeseries
|
10
|
+
|
11
|
+
class Filter
|
12
|
+
attr_reader :regex
|
13
|
+
def initialize(keys, values)
|
14
|
+
@keys = keys
|
15
|
+
@values = values
|
16
|
+
end
|
17
|
+
|
18
|
+
def regex
|
19
|
+
@regex ||= Regexp.new('\A' + @keys.map{|key| @values[key]|| '.*'}.join('\/') + '\z')
|
20
|
+
end
|
21
|
+
|
22
|
+
def match?(value)
|
23
|
+
value =~ regex
|
24
|
+
end
|
25
|
+
|
26
|
+
def to_lua_pattern
|
27
|
+
@lua_pattern ||= @keys.map{|key| @values[key]|| '.*'}.join('/')
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class Manager
|
32
|
+
DEFAULT_STRATEGY = RedisHgetallStrategy
|
33
|
+
attr_reader :prefix
|
34
|
+
def initialize(options = {})
|
35
|
+
@strategy = ( options[:strategy] || DEFAULT_STRATEGY).new
|
36
|
+
@resolution_ids = options[:resolutions] || DEFAULT_RESOLUTIONS
|
37
|
+
@prefix = [(options[:prefix] || DEFAULT_PREFIX) , @strategy.id].join(NAMESPACE_SEPARATOR)
|
38
|
+
@connection_to_use = options[:connection]
|
39
|
+
end
|
40
|
+
|
41
|
+
def on_connection(conn)
|
42
|
+
old_connection = @connection_to_use
|
43
|
+
@connection_to_use = conn
|
44
|
+
yield self
|
45
|
+
@connection_to_use = old_connection
|
46
|
+
end
|
47
|
+
|
48
|
+
def connection_to_use
|
49
|
+
@connection_to_use || fail("No connection given")
|
50
|
+
end
|
51
|
+
|
52
|
+
def store(subject, event_point_hash, moment = Time.now, descend_subject = true, descend_event = true)
|
53
|
+
resolutions = resolutions_of(moment)
|
54
|
+
|
55
|
+
descend(subject, descend_subject) do |subject_path|
|
56
|
+
event_point_hash.each do |event, point_value|
|
57
|
+
descend(event, descend_event) do |event_path|
|
58
|
+
resolutions.each do |res|
|
59
|
+
resolution_name, resolution_value = *res
|
60
|
+
store_point_value(subject_path, resolution_name, resolution_value, point_value, event_path)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def resolutions_of(moment)
|
68
|
+
@resolution_ids.collect do |res_id|
|
69
|
+
[res_id, resolution_value_at(moment, res_id)]
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def resolution_value_at(moment, res_id)
|
74
|
+
res_config = DEFAULT_RESOLUTIONS_MAP[res_id] # TODO configurable
|
75
|
+
if res_config.nil?
|
76
|
+
fail "No resolution config for id: #{res_id.class}:#{res_id}"
|
77
|
+
end
|
78
|
+
|
79
|
+
time_resolution_formater = res_config[:formatter]
|
80
|
+
case time_resolution_formater
|
81
|
+
when String
|
82
|
+
moment.strftime(time_resolution_formater)
|
83
|
+
when Proc
|
84
|
+
time_resolution_formater.call(moment)
|
85
|
+
else
|
86
|
+
fail ArgumentError, "Unexpected moment formater type #{time_resolution_formater.class}"
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def descend(path, do_descend = true , &block)
|
91
|
+
return if path.empty? or path == "."
|
92
|
+
block.call(path)
|
93
|
+
descend(File.dirname(path), do_descend, &block) if do_descend
|
94
|
+
end
|
95
|
+
|
96
|
+
def store_point_value( subject_path, resolution_name, resolution_value, point_value, event_path)
|
97
|
+
@strategy.store_point_value(self, subject_path, resolution_name, resolution_value, point_value, event_path)
|
98
|
+
end
|
99
|
+
|
100
|
+
def find(subject, from, to = Time.now, filter = nil)
|
101
|
+
Rhcf::Timeseries::Query.new(subject, from, to, self, filter)
|
102
|
+
end
|
103
|
+
|
104
|
+
def resolution(id)
|
105
|
+
res = DEFAULT_RESOLUTIONS_MAP[id] # TODO configurable
|
106
|
+
fail ArgumentError, "Invalid resolution name #{id} for this time series" if res.nil?
|
107
|
+
res.merge(:id => id)
|
108
|
+
end
|
109
|
+
|
110
|
+
def resolutions
|
111
|
+
@_resolutions ||= @resolution_ids.map { |id| resolution(id) }
|
112
|
+
end
|
113
|
+
|
114
|
+
def crunch_values(subject, resolution_id, point, filter)
|
115
|
+
@strategy.crunch_values(self, subject, resolution_id, point, filter)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
module Rhcf
|
2
|
+
module Timeseries
|
3
|
+
class Query
|
4
|
+
def initialize(subject, from, to, series, filter = nil)
|
5
|
+
fail ArgumentError, "Argument 'from' can not be bigger then 'to'" if from > to
|
6
|
+
@series = series
|
7
|
+
@subject = subject
|
8
|
+
@from = from
|
9
|
+
@to = to
|
10
|
+
|
11
|
+
@filter = filter
|
12
|
+
end
|
13
|
+
|
14
|
+
def total(resolution_id=nil)
|
15
|
+
accumulator={}
|
16
|
+
points(resolution_id || better_resolution[:id]) do |data|
|
17
|
+
|
18
|
+
data[:values].each do |key, value|
|
19
|
+
accumulator[key]||=0
|
20
|
+
accumulator[key]+=value
|
21
|
+
end
|
22
|
+
end
|
23
|
+
accumulator
|
24
|
+
end
|
25
|
+
|
26
|
+
def points(resolution_id)
|
27
|
+
list =[]
|
28
|
+
|
29
|
+
point_range(resolution_id) do |point|
|
30
|
+
|
31
|
+
values = @series.crunch_values(@subject, resolution_id, point, @filter)
|
32
|
+
|
33
|
+
next if values.empty?
|
34
|
+
data = {moment: point, values: values }
|
35
|
+
if block_given?
|
36
|
+
yield data
|
37
|
+
else
|
38
|
+
list << data
|
39
|
+
end
|
40
|
+
end
|
41
|
+
list unless block_given?
|
42
|
+
end
|
43
|
+
|
44
|
+
def point_range(resolution_id)
|
45
|
+
resolution = @series.resolution(resolution_id)
|
46
|
+
span = resolution[:span]
|
47
|
+
ptr = @from.dup
|
48
|
+
while ptr < @to
|
49
|
+
point = @series.resolution_value_at(ptr, resolution_id)
|
50
|
+
yield point
|
51
|
+
ptr += span.to_i
|
52
|
+
end
|
53
|
+
rescue FloatDomainError
|
54
|
+
# OK
|
55
|
+
end
|
56
|
+
|
57
|
+
def better_resolution
|
58
|
+
span = @to.to_time - @from.to_time
|
59
|
+
|
60
|
+
resolutions = @series.resolutions.sort_by{|h| h[:span]}.reverse
|
61
|
+
5.downto(1) do |div|
|
62
|
+
res = resolutions.find{|r| r[:span] < span / div }
|
63
|
+
return res if res
|
64
|
+
end
|
65
|
+
return nil
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,175 @@
|
|
1
|
+
module Rhcf
|
2
|
+
module Timeseries
|
3
|
+
class RedisHgetallStrategy
|
4
|
+
def id
|
5
|
+
'H'
|
6
|
+
end
|
7
|
+
|
8
|
+
def crunch_values(manager, subject, resolution_id, point, filter)
|
9
|
+
values = hgetall(manager, EVENT_POINT_TOKEN, subject, resolution_id, point)
|
10
|
+
values.reject!{|event, value| !filter.match?(event) } if filter
|
11
|
+
values
|
12
|
+
end
|
13
|
+
|
14
|
+
def store_point_value(manager, subject_path, resolution_name, resolution_value, point_value, event_path)
|
15
|
+
key = [manager.prefix, EVENT_POINT_TOKEN ,subject_path, resolution_name, resolution_value].join(NAMESPACE_SEPARATOR)
|
16
|
+
manager.connection_to_use.hincrby(key, event_path, point_value)
|
17
|
+
manager.connection_to_use.expire(key, DEFAULT_RESOLUTIONS_MAP[resolution_name][:ttl])
|
18
|
+
end
|
19
|
+
|
20
|
+
def hgetall(manager, k,s,r,p)
|
21
|
+
key = [ manager.prefix, k,s,r,p].join(NAMESPACE_SEPARATOR)
|
22
|
+
manager.connection_to_use.hgetall(key).each_with_object({}) do |(_k, value), hash|
|
23
|
+
hash[_k] = value.to_i
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
class RedisStringBasedStrategy
|
29
|
+
def id
|
30
|
+
fail 'AbstractStrategy'
|
31
|
+
end
|
32
|
+
|
33
|
+
def store_point_value(manager, subject_path, resolution_name, resolution_value, point_value, event_path)
|
34
|
+
store_point_event(manager, resolution_name, resolution_value, subject_path, event_path)
|
35
|
+
key = [manager.prefix, EVENT_POINT_TOKEN ,subject_path, resolution_name, resolution_value, event_path].join(NAMESPACE_SEPARATOR)
|
36
|
+
manager.connection_to_use.incrby(key, point_value)
|
37
|
+
manager.connection_to_use.expire(key, DEFAULT_RESOLUTIONS_MAP[resolution_name][:ttl])
|
38
|
+
end
|
39
|
+
|
40
|
+
def store_point_event(manager, resolution_name, resolution_value, subject_path, event_path)
|
41
|
+
key = [manager.prefix, EVENT_SET_TOKEN, resolution_name, resolution_value, subject_path].join(NAMESPACE_SEPARATOR)
|
42
|
+
manager.connection_to_use.sadd(key, event_path)
|
43
|
+
manager.connection_to_use.expire(key, DEFAULT_RESOLUTIONS_MAP[resolution_name][:ttl])
|
44
|
+
end
|
45
|
+
|
46
|
+
def events_for_subject_on(manager, subject, point, resolution_id, filter)
|
47
|
+
key = [manager.prefix, EVENT_SET_TOKEN, resolution_id, point, subject].join(NAMESPACE_SEPARATOR)
|
48
|
+
events = manager.connection_to_use.smembers(key)
|
49
|
+
events = events.select{|event| filter.match?(event) } if filter
|
50
|
+
events
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
class RedisMgetStrategy < RedisStringBasedStrategy
|
55
|
+
def id
|
56
|
+
'M'
|
57
|
+
end
|
58
|
+
|
59
|
+
def crunch_values(manager, subject, resolution_id, point, filter)
|
60
|
+
events = events_for_subject_on(manager, subject, point, resolution_id, filter)
|
61
|
+
mget(manager, EVENT_POINT_TOKEN, subject, resolution_id, point, events)
|
62
|
+
end
|
63
|
+
|
64
|
+
def mget(manager, k, s, r, p, es)
|
65
|
+
return {} if es.empty?
|
66
|
+
keys = es.map{|e| [manager.prefix, k, s, r, p, e].flatten.join(NAMESPACE_SEPARATOR)}
|
67
|
+
values = manager.connection_to_use.mget(*keys)
|
68
|
+
data = {}
|
69
|
+
keys.each_with_index do |key, index|
|
70
|
+
data[es[index]] = values[index].to_i
|
71
|
+
end
|
72
|
+
data
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
class RedisMgetLuaStrategy < RedisMgetStrategy
|
77
|
+
def id; 'ME'; end
|
78
|
+
|
79
|
+
def events_for_subject_on(manager, subject, point, resolution_id, filter)
|
80
|
+
key = [manager.prefix, EVENT_SET_TOKEN, resolution_id, point, subject].join(NAMESPACE_SEPARATOR)
|
81
|
+
events = if filter
|
82
|
+
manager.connection_to_use.evalsha(evalsha_for(:smembers_matching),
|
83
|
+
keys: [key], argv: [filter.to_lua_pattern])
|
84
|
+
else
|
85
|
+
manager.connection_to_use.smembers(key)
|
86
|
+
end
|
87
|
+
events
|
88
|
+
end
|
89
|
+
|
90
|
+
def crunch_values(manager, subject, resolution_id, point, filter)
|
91
|
+
register_lua_scripts!(manager.connection_to_use)
|
92
|
+
point_prefix = [manager.prefix, EVENT_POINT_TOKEN, subject, resolution_id, point].join(NAMESPACE_SEPARATOR)
|
93
|
+
set_key = [manager.prefix, EVENT_SET_TOKEN, resolution_id, point, subject].join(NAMESPACE_SEPARATOR)
|
94
|
+
|
95
|
+
data = manager.connection_to_use.evalsha(evalsha_for(:mget_matching_smembers),
|
96
|
+
keys: [set_key], argv: [point_prefix, filter && filter.to_lua_pattern])
|
97
|
+
|
98
|
+
return {} if data.nil?
|
99
|
+
result = {}
|
100
|
+
data.first.each_with_index do |evt, idx|
|
101
|
+
value = data.last[idx].to_i
|
102
|
+
result[evt] = value
|
103
|
+
end
|
104
|
+
|
105
|
+
result
|
106
|
+
end
|
107
|
+
|
108
|
+
def evalsha_for(sym_os_lua_script)
|
109
|
+
@lua_script_register[sym_os_lua_script] || fail("Script for '#{sym_os_lua_script}' not registered")
|
110
|
+
end
|
111
|
+
|
112
|
+
def register_lua_scripts!(connection)
|
113
|
+
@lua_script_register ||=
|
114
|
+
begin
|
115
|
+
smembers_matching = <<-EOF
|
116
|
+
local matches = {}
|
117
|
+
for _, val in ipairs(redis.call('smembers', KEYS[1])) do
|
118
|
+
if string.match(val, ARGV[1]) then
|
119
|
+
table.insert(matches, val)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
return matches
|
123
|
+
EOF
|
124
|
+
|
125
|
+
mget_matching_smembers = <<-EOF
|
126
|
+
local matches = {}
|
127
|
+
local set_key = KEYS[1]
|
128
|
+
local key_prefix = ARGV[1]
|
129
|
+
local filter_pattern = ARGV[2]
|
130
|
+
local keys = {}
|
131
|
+
local keys_to_mget = {}
|
132
|
+
|
133
|
+
for _, val in ipairs(redis.call('smembers', set_key)) do
|
134
|
+
if (filter_pattern and string.match(val, filter_pattern)) or not filter_pattern then
|
135
|
+
table.insert(keys, val)
|
136
|
+
table.insert(keys_to_mget, key_prefix .. '#{NAMESPACE_SEPARATOR}' .. val)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
if table.getn(keys) > 0 then
|
141
|
+
return {keys, redis.call('MGET', unpack(keys_to_mget)) }
|
142
|
+
end
|
143
|
+
return {{},{}}
|
144
|
+
EOF
|
145
|
+
|
146
|
+
{
|
147
|
+
mget_matching_smembers: connection.script(:load, mget_matching_smembers),
|
148
|
+
smembers_matching: connection.script(:load, smembers_matching)
|
149
|
+
}
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
class RedisGetStrategy < RedisStringBasedStrategy
|
155
|
+
def id
|
156
|
+
'G'
|
157
|
+
end
|
158
|
+
|
159
|
+
def crunch_values(manager, subject, resolution_id, point, filter)
|
160
|
+
events = events_for_subject_on(manager, subject, point, resolution_id, filter)
|
161
|
+
values = {}
|
162
|
+
events.each do |event|
|
163
|
+
value = get(manager, EVENT_POINT_TOKEN, subject, resolution_id, point, event)
|
164
|
+
values[event] = value.to_i
|
165
|
+
end
|
166
|
+
values
|
167
|
+
end
|
168
|
+
|
169
|
+
def get(manager, *a_key)
|
170
|
+
a_key = [manager.prefix, a_key].flatten.join(NAMESPACE_SEPARATOR)
|
171
|
+
manager.connection_to_use.get(a_key)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
data/rhcf-timeseries.gemspec
CHANGED
@@ -25,8 +25,8 @@ Gem::Specification.new do |spec|
|
|
25
25
|
#spec.add_development_dependency "guard"
|
26
26
|
#spec.add_development_dependency "guard-rspec"
|
27
27
|
#spec.add_development_dependency "guard-bundler"
|
28
|
-
|
28
|
+
spec.add_development_dependency "simplecov"
|
29
29
|
spec.add_development_dependency "timecop"
|
30
30
|
spec.add_development_dependency "stackprof"
|
31
|
-
|
31
|
+
spec.add_development_dependency "database_cleaner"
|
32
32
|
end
|
@@ -0,0 +1,193 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'timecop'
|
3
|
+
require 'redis'
|
4
|
+
require 'rhcf/timeseries/manager'
|
5
|
+
require 'benchmark'
|
6
|
+
require 'stackprof'
|
7
|
+
|
8
|
+
describe Rhcf::Timeseries::Manager do
|
9
|
+
let(:redis){Redis.new}
|
10
|
+
subject{Rhcf::Timeseries::Manager.new(connection: redis)}
|
11
|
+
|
12
|
+
before(:each) do
|
13
|
+
Timecop.return
|
14
|
+
end
|
15
|
+
|
16
|
+
describe 'descending' do
|
17
|
+
it "is fast to store and read" do
|
18
|
+
total = 0
|
19
|
+
start_time = Time.now
|
20
|
+
|
21
|
+
bench = Benchmark.measure {
|
22
|
+
StackProf.run(mode: :cpu, out: p('/tmp/stackprof-cpu-store-descend.dump')) do
|
23
|
+
1000.times do
|
24
|
+
total +=1
|
25
|
+
subject.store("a/b", {"e/f" => 1} ) #, time)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
}
|
29
|
+
|
30
|
+
|
31
|
+
Benchmark.measure {
|
32
|
+
expect(subject.find("a", start_time - 11100, Time.now + 11100).total['e'].to_i).to eq(total)
|
33
|
+
}
|
34
|
+
|
35
|
+
Benchmark.measure {
|
36
|
+
expect(subject.find("a", start_time - 100000, Time.now + 100000).total(:year)['e/f'].to_i).to eq(total)
|
37
|
+
}
|
38
|
+
|
39
|
+
puts "Descend write speed %d points/seg | points:%d, duration:%0.3fs" % [speed = (1.0 * total / (bench.total + 0.00000001)), total, bench.total]
|
40
|
+
expect(speed).to be > 100
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe 'not descending' do
|
45
|
+
it "is be fast to store and read" do
|
46
|
+
total = 0
|
47
|
+
bench = Benchmark.measure {
|
48
|
+
StackProf.run(mode: :cpu, out: p('/tmp/stackprof-cpu-store-nodescend.dump')) do
|
49
|
+
1000.times do
|
50
|
+
total +=1
|
51
|
+
subject.store("a/b/c/d", {"e/f/g/h" => 1} , Time.now, false, false)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
}
|
55
|
+
|
56
|
+
puts "No descend write speed %d points/seg | points:%d, duration:%0.3fs" % [new_speed = (1.0 * total / (bench.total + 0.00000001)), total, bench.total]
|
57
|
+
expect(new_speed).to be > 300
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
describe "find and total" do
|
62
|
+
let(:start_time){ Time.parse("2000-01-01 00:00:00") }
|
63
|
+
before do
|
64
|
+
Timecop.travel(start_time)
|
65
|
+
subject.store("views/product/15", {"web/firefox/3" => 1})
|
66
|
+
|
67
|
+
Timecop.travel(15.minutes) #00:00:15
|
68
|
+
subject.store("views/product/13", {"web/firefox/3" => 1}, Time.now)
|
69
|
+
subject.store("views/product/13", {"web/firefox/3" => 1}, Time.now)
|
70
|
+
subject.store("views/product/13", {"web/firefox/3" => 0}, Time.now)
|
71
|
+
|
72
|
+
Timecop.travel(15.minutes) #00:00:30
|
73
|
+
subject.store("views/product/15", {"web/ie/6" => 3})
|
74
|
+
|
75
|
+
Timecop.travel(15.minutes) #00:00:45
|
76
|
+
subject.store("views/product/15", {"web/ie/6" => 2})
|
77
|
+
|
78
|
+
Timecop.travel(15.minutes) #00:00:00
|
79
|
+
subject.store("views/product/11", {"web/ie/5" => 2})
|
80
|
+
|
81
|
+
Timecop.travel(15.minutes) #00:00:15
|
82
|
+
subject.store("views/product/11", {"web/chrome/11"=> 4})
|
83
|
+
|
84
|
+
Timecop.travel(15.minutes) #00:00:30
|
85
|
+
subject.store("views/product/11", {"web/chrome/11"=> 2})
|
86
|
+
end
|
87
|
+
|
88
|
+
it "is similar to redistat" do
|
89
|
+
|
90
|
+
expect(subject.find("views/product", start_time, start_time + 55.minutes).total(:ever)).to eq({
|
91
|
+
"web" => 16.0,
|
92
|
+
"web/chrome" => 6.0,
|
93
|
+
"web/chrome/11" => 6.0,
|
94
|
+
"web/firefox" => 3.0,
|
95
|
+
"web/firefox/3" => 3.0,
|
96
|
+
"web/ie" => 7.0,
|
97
|
+
"web/ie/5" => 2.0,
|
98
|
+
"web/ie/6" => 5.0
|
99
|
+
})
|
100
|
+
|
101
|
+
expect(subject.find("views/product", start_time, start_time + 55.minutes).total(:year)).to eq({
|
102
|
+
"web" => 16.0,
|
103
|
+
"web/chrome" => 6.0,
|
104
|
+
"web/chrome/11" => 6.0,
|
105
|
+
"web/firefox" => 3.0,
|
106
|
+
"web/firefox/3" => 3.0,
|
107
|
+
"web/ie" => 7.0,
|
108
|
+
"web/ie/5" => 2.0,
|
109
|
+
"web/ie/6" => 5.0
|
110
|
+
})
|
111
|
+
|
112
|
+
expect( subject.find("views/product", start_time, start_time + 55.minutes).total ).to eq({
|
113
|
+
'web' => 8,
|
114
|
+
'web/firefox' => 3,
|
115
|
+
'web/firefox/3' => 3,
|
116
|
+
'web/ie' => 5,
|
117
|
+
'web/ie/6' => 5,
|
118
|
+
})
|
119
|
+
|
120
|
+
expect(subject.find("views/product/15", start_time, start_time + 55.minutes).points(:minute)).to eq([
|
121
|
+
{:moment=>"2000-01-01T00:00", :values=>{"web/firefox"=>1, "web/firefox/3"=>1, "web"=>1}},
|
122
|
+
{:moment=>"2000-01-01T00:30", :values=>{"web"=>3, "web/ie/6"=>3, "web/ie"=>3}},
|
123
|
+
{:moment=>"2000-01-01T00:45", :values=>{"web"=>2, "web/ie/6"=>2, "web/ie"=>2}}
|
124
|
+
])
|
125
|
+
|
126
|
+
expect(subject.find("views/product/13", start_time, start_time + 55.minutes).points(:minute)).to eq([
|
127
|
+
{:moment=>"2000-01-01T00:15", :values=>{"web/firefox"=>2, "web/firefox/3"=>2, "web"=>2}},
|
128
|
+
])
|
129
|
+
|
130
|
+
expect(subject.find("views/product", start_time, start_time + 55.minutes).points(:minute)).to eq([
|
131
|
+
{:moment=>"2000-01-01T00:00", :values=>{"web/firefox"=>1, "web/firefox/3"=>1, "web"=>1}},
|
132
|
+
{:moment=>"2000-01-01T00:15", :values=>{"web/firefox"=>2, "web/firefox/3"=>2, "web"=>2}},
|
133
|
+
{:moment=>"2000-01-01T00:30", :values=>{"web"=>3, "web/ie/6"=>3, "web/ie"=>3}},
|
134
|
+
{:moment=>"2000-01-01T00:45", :values=>{"web"=>2, "web/ie/6"=>2, "web/ie"=>2}}
|
135
|
+
])
|
136
|
+
|
137
|
+
expect(subject.find("views", start_time, start_time + 55.minutes).points(:minute)).to eq([
|
138
|
+
{:moment=>"2000-01-01T00:00", :values=>{"web/firefox"=>1, "web/firefox/3"=>1, "web"=>1}},
|
139
|
+
{:moment=>"2000-01-01T00:15", :values=>{"web/firefox"=>2, "web/firefox/3"=>2, "web"=>2}},
|
140
|
+
{:moment=>"2000-01-01T00:30", :values=>{"web"=>3, "web/ie/6"=>3, "web/ie"=>3}},
|
141
|
+
{:moment=>"2000-01-01T00:45", :values=>{"web"=>2, "web/ie/6"=>2, "web/ie"=>2}}
|
142
|
+
])
|
143
|
+
|
144
|
+
expect(subject.find("views", start_time).points(:hour)).to eq([
|
145
|
+
{
|
146
|
+
:moment=>"2000-01-01T00",
|
147
|
+
:values=> {
|
148
|
+
"web/ie"=>5.0,
|
149
|
+
"web"=>8.0,
|
150
|
+
"web/firefox"=>3.0,
|
151
|
+
"web/ie/6"=>5.0,
|
152
|
+
"web/firefox/3"=>3.0}
|
153
|
+
},{
|
154
|
+
:moment=>"2000-01-01T01",
|
155
|
+
:values=>{
|
156
|
+
"web/ie"=>2.0,
|
157
|
+
"web/chrome"=>6.0,
|
158
|
+
"web/chrome/11"=>6.0,
|
159
|
+
"web"=>8.0,
|
160
|
+
"web/ie/5"=>2.0
|
161
|
+
}
|
162
|
+
}
|
163
|
+
])
|
164
|
+
end
|
165
|
+
let(:filter) { Rhcf::Timeseries::Filter.new([:source, :browser], browser: 'firefox.*' )}
|
166
|
+
it "can find with filter" do
|
167
|
+
expect(subject.find("views", start_time, start_time + 55.minutes, filter).points(:minute)).to eq([
|
168
|
+
{:moment=>"2000-01-01T00:00", :values=>{"web/firefox"=>1, "web/firefox/3"=>1}},
|
169
|
+
{:moment=>"2000-01-01T00:15", :values=>{"web/firefox"=>2, "web/firefox/3"=>2}},
|
170
|
+
])
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
it "causes no stack overflow" do
|
175
|
+
params_hash = {
|
176
|
+
sender_domain: 'example.com',
|
177
|
+
realm: 'realm',
|
178
|
+
destination_domain: 'lvh.me',
|
179
|
+
mail_server: 'aserver',
|
180
|
+
bind_interface: '11.1.1.11'
|
181
|
+
}
|
182
|
+
|
183
|
+
{
|
184
|
+
'sender_domain' => '%{sender_domain}',
|
185
|
+
'realm_and_sender_domain' => '%{realm}/%{sender_domain}',
|
186
|
+
'mail_server_and_interface' => '%{mail_server}/%{bind_interface}',
|
187
|
+
'realm_and_destination_domain' => '%{realm}/%{destination_domain}',
|
188
|
+
'destination_domain' => '%{destination_domain}'
|
189
|
+
}.each do |known, unknown|
|
190
|
+
subject.store(known % params_hash, {[(unknown % params_hash),'sent'].join('/') => 1})
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
@@ -1,28 +1,28 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
-
require 'rhcf/timeseries/
|
2
|
+
require 'rhcf/timeseries/manager'
|
3
3
|
|
4
4
|
|
5
5
|
describe "Query" do
|
6
6
|
describe "#better_resolution" do
|
7
7
|
|
8
|
-
let(:
|
8
|
+
let(:redis){nil}
|
9
9
|
describe "When having a smaller then 1/5 " do
|
10
|
-
let(:series) { Rhcf::Timeseries::
|
10
|
+
let(:series) { Rhcf::Timeseries::Manager.new(connection: redis, resolutions: [:hour, :"15minutes", :minute]) }
|
11
11
|
it { expect(series.find('bla', Time.now - 3600, Time.now).better_resolution[:id]).to eq :minute }
|
12
12
|
end
|
13
13
|
|
14
14
|
describe "When having a smaller but greather then 1/5" do
|
15
|
-
let(:series) { Rhcf::Timeseries::
|
15
|
+
let(:series) { Rhcf::Timeseries::Manager.new(connection: redis, resolutions: [:hour, :"15minutes"]) }
|
16
16
|
it { expect(series.find('bla', Time.now - 3600, Time.now).better_resolution[:id]).to eq :"15minutes" }
|
17
17
|
end
|
18
18
|
|
19
19
|
describe "When having no smaller, only its size" do
|
20
|
-
let(:series) { Rhcf::Timeseries::
|
20
|
+
let(:series) { Rhcf::Timeseries::Manager.new(connection: redis, resolutions: [:hour]) }
|
21
21
|
it { expect(series.find('bla', DateTime.parse('2015-01-01 01:50:00'), DateTime.parse('2015-01-01 03:10:00')).better_resolution[:id]).to eq :hour }
|
22
22
|
end
|
23
23
|
|
24
24
|
describe "When having only bigger" do
|
25
|
-
let(:series) { Rhcf::Timeseries::
|
25
|
+
let(:series) { Rhcf::Timeseries::Manager.new(connection: redis, resolutions: [:month]) }
|
26
26
|
it { expect(series.find('bla', DateTime.parse('2015-01-01 01:50:00'), DateTime.parse('2015-01-01 03:10:00')).better_resolution).to be_nil}
|
27
27
|
end
|
28
28
|
|