simple_metrics 0.3.2 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/bin/populate CHANGED
@@ -21,7 +21,7 @@ def create_dps(name)
21
21
  previous_value = 20
22
22
  (1..1).inject([]) do |result, index|
23
23
  value = previous_value+rand(20)
24
- result << SimpleMetrics::DataPoint.create_counter(:name => name, :value => value)
24
+ result << SimpleMetrics::DataPoint::Counter.new(:name => name, :value => value)
25
25
  previous_value = value
26
26
  result
27
27
  end
@@ -32,7 +32,7 @@ current = now - 1 * hour
32
32
  while (current < now)
33
33
  dps = create_dps(name)
34
34
  puts "flush data for #{Time.at(current)}, #{counter}"
35
- SimpleMetrics::Bucket.flush_data_points(dps, current)
35
+ SimpleMetrics::Importer.flush_data_points(dps, current)
36
36
  current += 10
37
37
  counter += 1
38
38
  end
@@ -13,18 +13,6 @@ module SimpleMetrics
13
13
  set :public_folder, File.expand_path('../public', __FILE__)
14
14
 
15
15
  helpers do
16
- def graph_title(time)
17
- case time
18
- when "minute"
19
- "Minute"
20
- when "hour"
21
- "Hour"
22
- when "day"
23
- "Day"
24
- when "week"
25
- "Week"
26
- end
27
- end
28
16
  end
29
17
 
30
18
  get "/api/metrics" do
@@ -33,9 +21,9 @@ module SimpleMetrics
33
21
  metrics.inject([]) { |result, m| result << m.attributes }.to_json
34
22
  end
35
23
 
36
- get "/api/metrics/:id" do
24
+ get "/api/metrics/:name" do
37
25
  content_type :json
38
- metric = SimpleMetrics::MetricRepository.find_one(to_bson_id(params[:id]))
26
+ metric = SimpleMetrics::MetricRepository.find_one_by_name(params[:name])
39
27
  metric.attributes.to_json
40
28
  end
41
29
 
@@ -52,85 +40,13 @@ module SimpleMetrics
52
40
  erb :index
53
41
  end
54
42
 
55
- # get "/metrics" do
56
- # erb :index
57
- # end
58
-
59
- # get "/metrics/:id" do
60
- # erb :index
61
- # end
62
-
63
- # get "/metric" do
64
- # @from = (params[:from] || Time.now).to_i
65
- # erb :show
66
- # end
67
-
68
- # get "/graph" do
69
- # @from = (params[:from] || Time.now).to_i
70
- # @time = params[:time] || 'minute'
71
- # @targets = params[:target]
72
- # @data_points = prepare_data_points(@from, @time, *@targets)
73
- # @series = @data_points
74
- # erb :graph, :layout => false
75
- # end
76
-
77
43
  private
78
44
 
79
- # params[:id]
80
- def to_bson_id(id)
81
- BSON::ObjectId.from_string(id)
82
- end
83
-
84
45
  def prepare_data_points(from, time, *targets)
85
- to = from - time_range(time)
86
- result = SimpleMetrics::Graph.query_all(bucket(time), to, from, *targets)
87
- result.map do |data_point|
88
- { :name => data_point.first, :data => data_point.last.map { |p| { :x => p[:ts], :y => p[:value] || 0 } } }
89
- end
90
- end
91
-
92
- def one_minute
93
- 60
94
- end
95
-
96
- def one_hour
97
- one_minute * 60
98
- end
99
-
100
- def one_day
101
- one_hour * 24
102
- end
103
-
104
- def one_week
105
- one_day * 7
106
- end
107
-
108
- def time_range(time)
109
- case time
110
- when 'minute'
111
- 5*one_minute
112
- when 'hour'
113
- one_hour
114
- when 'day'
115
- one_day
116
- when 'week'
117
- one_week
118
- else
119
- raise "Unknown time param: #{time}"
120
- end
46
+ bucket = SimpleMetrics::Bucket.for_time(time)
47
+ to = from - SimpleMetrics::Graph.time_range(time)
48
+ SimpleMetrics::Graph.query_all(bucket, to, from, *targets)
121
49
  end
122
50
 
123
- def bucket(time)
124
- case time
125
- when 'minute'
126
- SimpleMetrics::Bucket[0]
127
- when 'hour'
128
- SimpleMetrics::Bucket[1]
129
- when 'day'
130
- SimpleMetrics::Bucket[2]
131
- when 'week'
132
- SimpleMetrics::Bucket[3]
133
- end
134
- end
135
51
  end
136
52
  end
@@ -11,80 +11,33 @@ module SimpleMetrics
11
11
  def first
12
12
  all.first
13
13
  end
14
- alias :finest :first
15
14
 
16
15
  def [](index)
17
16
  all[index]
18
17
  end
19
18
 
20
- def flush_raw_data(data)
21
- data_points = []
22
- data.each do |str|
23
- begin
24
- data_points << DataPoint.parse(str)
25
- rescue DataPoint::ParserError => e
26
- SimpleMetrics.logger.debug "Invalid Data skipped: #{str}, #{e}"
27
- end
19
+ def for_time(time)
20
+ case time
21
+ when 'minute'
22
+ self[0]
23
+ when 'hour'
24
+ self[1]
25
+ when 'day'
26
+ self[2]
27
+ when 'week'
28
+ self[3]
28
29
  end
29
- flush_data_points(data_points, Time.now.utc.to_i)
30
30
  end
31
31
 
32
- def flush_data_points(data_points, ts = nil)
33
- return if data_points.empty?
34
- SimpleMetrics.logger.info "#{Time.now} Flushing #{data_points.count} counters to MongoDB"
35
-
36
- ts ||= Time.now.utc.to_i
37
- bucket = Bucket.first
38
-
39
- data_points.group_by { |dp| dp.name }.each_pair do |name,dps|
40
- dp = ValueAggregation.aggregate(dps)
41
- bucket.save(dp, ts)
42
- update_metric(dp)
43
- aggregate(dp)
44
- end
45
- end
46
-
47
- def aggregate(dp)
48
- coarse_buckets.each do |bucket|
49
- existing_dp = bucket.find_data_point_at_ts(dp.ts, dp.name)
50
- if existing_dp
51
- UpdateAggregation.aggregate(existing_dp, dp)
52
- bucket.update(existing_dp, existing_dp.ts)
53
- else
54
- dp.sum = dp.value
55
- dp.total = 1
56
- bucket.save(dp, dp.ts)
57
- end
58
- end
59
- end
60
-
61
- private
62
-
63
- def update_metric(dp)
64
- metric = MetricRepository.find_one_by_name(dp.name)
65
- if metric
66
- MetricRepository.update(Metric.new(:name => dp.name, :total => metric.total + 1))
67
- else
68
- MetricRepository.save(Metric.new(:name => dp.name, :total => 1))
69
- end
70
- end
71
-
72
- def coarse_buckets
73
- Bucket.all.sort_by! { |r| r.seconds }[1..-1]
74
- end
75
-
76
- def humanized_timestamp(ts)
77
- Time.at(ts).utc
78
- end
79
32
  end
80
33
 
81
34
  attr_reader :name, :capped
82
35
 
83
36
  def initialize(attributes)
84
- @name = attributes['name']
85
- @seconds = attributes['seconds']
86
- @capped = attributes['capped']
87
- @size = attributes['size']
37
+ @name = attributes.fetch(:name)
38
+ @seconds = attributes.fetch(:seconds)
39
+ @capped = attributes.fetch(:capped)
40
+ @size = attributes.fetch(:size)
88
41
  end
89
42
 
90
43
  def seconds
@@ -116,11 +69,6 @@ module SimpleMetrics
116
69
  repository.find_data_point_at_ts(ts_bucket(ts), name)
117
70
  end
118
71
 
119
- # TODO: only used in tests, do we need it?
120
- def find_all_in_ts_range(from, to)
121
- repository.find_all_in_ts_range(from, to)
122
- end
123
-
124
72
  def find_all_in_ts_range_by_name(from, to, name)
125
73
  repository.find_all_in_ts_range_by_name(from, to, name)
126
74
  end
@@ -129,54 +77,32 @@ module SimpleMetrics
129
77
  repository.find_all_in_ts_range_by_wildcard(from, to, target)
130
78
  end
131
79
 
132
- # TODO: only used in tests, do we need it?
133
- def data_points_exist_at_ts?(ts, name)
134
- repository.count_for_name_at(ts, name) > 0
135
- end
136
-
137
- # TODO: only used in tests, do we need it?
138
- def stats_exist_in_previous_ts?(ts)
139
- repository.count_at(ts) > 0
140
- end
141
-
142
- def find_all_distinct_names
143
- repository.find_all_distinct_names
144
- end
145
-
146
80
  def save(dp, ts)
147
- dp.ts = ts_bucket(ts)
81
+ dp.ts = ts_bucket(ts)
82
+ dp.sum = dp.value
83
+ dp.total = 1
148
84
  repository.save(dp)
149
- SimpleMetrics.logger.debug "SERVER: MongoDB - insert in #{name}: #{dp.inspect}"
150
85
  end
151
86
 
152
87
  def update(dp, ts)
153
88
  dp.ts = ts_bucket(ts)
154
89
  repository.update(dp, ts)
155
- SimpleMetrics.logger.debug "SERVER: MongoDB - update in #{name}: #{dp.inspect}"
156
90
  end
157
91
 
158
92
  def capped?
159
93
  @capped == true
160
94
  end
161
95
 
162
- # TODO refactor, move to graph.rb
163
96
  def fill_gaps(from, to, query_result)
164
97
  return query_result if query_result.nil? || query_result.size == 0
165
98
 
166
- tmp_hash = DataPoint.ts_hash(query_result)
99
+ existing_ts_entries = query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
167
100
  dp_template = query_result.first
168
101
 
169
102
  result = []
170
- each_ts(from, to) do |current_bucket_ts|
171
- result <<
172
- if tmp_hash.key?(current_bucket_ts)
173
- tmp_hash[current_bucket_ts]
174
- else
175
- dp = dp_template.dup
176
- dp.value = nil
177
- dp.ts = current_bucket_ts
178
- dp
179
- end
103
+ each_ts(from, to) do |ts_bucket|
104
+ dp = existing_ts_entries[ts_bucket] || DataPoint::Base.new(:name => dp_template.name, :ts => ts_bucket)
105
+ result << dp
180
106
  end
181
107
  result
182
108
  end
@@ -188,10 +114,10 @@ module SimpleMetrics
188
114
  end
189
115
 
190
116
  def each_ts(from, to)
191
- current_bucket_ts = ts_bucket(from)
192
- while (current_bucket_ts <= ts_bucket(to))
193
- yield(current_bucket_ts)
194
- current_bucket_ts = current_bucket_ts + seconds
117
+ ts_bucket = ts_bucket(from)
118
+ while (ts_bucket <= ts_bucket(to))
119
+ yield(ts_bucket)
120
+ ts_bucket = ts_bucket + seconds
195
121
  end
196
122
  end
197
123
 
@@ -6,7 +6,7 @@ module SimpleMetrics
6
6
  attr_reader :config
7
7
 
8
8
  def initialize(hash = {}, &block)
9
- @config = load_defaults.merge(hash)
9
+ @config = load_defaults.merge(symbolize_keys(hash))
10
10
  end
11
11
 
12
12
  def configure(hash = {}, &block)
@@ -15,7 +15,7 @@ module SimpleMetrics
15
15
  end
16
16
 
17
17
  def db
18
- @db ||= config['db']
18
+ @db ||= config.fetch(:db)
19
19
  end
20
20
 
21
21
  def db=(db)
@@ -23,7 +23,10 @@ module SimpleMetrics
23
23
  end
24
24
 
25
25
  def buckets
26
- @buckets ||= config['buckets']
26
+ @buckets ||= begin
27
+ tmp = config.fetch(:buckets)
28
+ tmp.map { |b| symbolize_keys(b)}
29
+ end
27
30
  end
28
31
 
29
32
  def buckets=(buckets)
@@ -31,7 +34,7 @@ module SimpleMetrics
31
34
  end
32
35
 
33
36
  def server
34
- @server ||= config['server']
37
+ @server ||= config.fetch(:server)
35
38
  end
36
39
 
37
40
  def server=(server)
@@ -39,7 +42,7 @@ module SimpleMetrics
39
42
  end
40
43
 
41
44
  def web
42
- @web ||= config['web']
45
+ @web ||= config.fetch(:web)
43
46
  end
44
47
 
45
48
  def web=(web)
@@ -49,7 +52,7 @@ module SimpleMetrics
49
52
  private
50
53
 
51
54
  def load_defaults
52
- @config = load_config
55
+ @config = symbolize_keys(load_config)
53
56
  rescue Errno::ENOENT # not found error
54
57
  logger.info "Creating initial config file: #{config_file}"
55
58
  FileUtils.cp(default_config_file, config_file)
@@ -72,6 +75,21 @@ module SimpleMetrics
72
75
  logger.error "Error reading config file: #{e}"
73
76
  end
74
77
 
78
+ def symbolize_keys(hash)
79
+ hash.inject({}){|result, (key, value)|
80
+ new_key = case key
81
+ when String then key.to_sym
82
+ else key
83
+ end
84
+ new_value = case value
85
+ when Hash then symbolize_keys(value)
86
+ else value
87
+ end
88
+ result[new_key] = new_value
89
+ result
90
+ }
91
+ end
92
+
75
93
  def logger
76
94
  SimpleMetrics.logger
77
95
  end
@@ -8,6 +8,13 @@ module SimpleMetrics
8
8
  @value = (@value.to_i || 1) * (1.0 / (@sample_rate || 1).to_f)
9
9
  end
10
10
 
11
+ def combine(dp)
12
+ @total += 1
13
+ @value += dp.value
14
+ @sum += dp.value
15
+ self
16
+ end
17
+
11
18
  end
12
19
  end
13
20
  end
@@ -7,6 +7,10 @@ module SimpleMetrics
7
7
  @type = 'ev'
8
8
  end
9
9
 
10
+ def combine(dp)
11
+ raise "Implement me!"
12
+ end
13
+
10
14
  end
11
15
  end
12
16
  end
@@ -8,6 +8,12 @@ module SimpleMetrics
8
8
  @value = (@value.to_i || 1) * (1.0 / (@sample_rate || 1).to_f)
9
9
  end
10
10
 
11
+ def combine(dp)
12
+ @total += 1
13
+ @sum += dp.value
14
+ @value = @sum / @total
15
+ self
16
+ end
11
17
  end
12
18
  end
13
19
  end
@@ -7,6 +7,9 @@ module SimpleMetrics
7
7
  @type = 'ms'
8
8
  end
9
9
 
10
+ def combine(dp)
11
+ raise "Implement me!"
12
+ end
10
13
  end
11
14
  end
12
15
  end
@@ -26,7 +26,7 @@ module SimpleMetrics
26
26
  def build(attributes)
27
27
  case attributes[:type]
28
28
  when 'c'
29
- DataPoint.create_counter(attributes)
29
+ Counter.new(attributes)
30
30
  when 'g'
31
31
  Gauge.new(attributes)
32
32
  when 'ms'
@@ -37,22 +37,6 @@ module SimpleMetrics
37
37
  raise UnknownTypeError, "Unknown Type Error: #{attributes[:type]}"
38
38
  end
39
39
  end
40
-
41
- def create_counter(attributes)
42
- Counter.new(attributes)
43
- end
44
-
45
- def create_gauge(attributes)
46
- Gauge.new(attributes)
47
- end
48
-
49
- def create_timing(attributes)
50
- Timing.new(attributes)
51
- end
52
-
53
- def ts_hash(query_result)
54
- query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
55
- end
56
40
 
57
41
  end
58
42
  end
@@ -19,25 +19,25 @@ module SimpleMetrics
19
19
  def ensure_collections_exist
20
20
  SimpleMetrics.logger.debug "SERVER: MongoDB - found following collections: #{db.collection_names.inspect}"
21
21
  buckets.each do |retention|
22
- unless db.collection_names.include?(retention['name'])
23
- db.create_collection(retention['name'], :capped => retention['capped'], :size => retention['size'])
24
- SimpleMetrics.logger.debug "SERVER: MongoDB - created collection #{retention['name']}, capped: #{retention['capped']}, size: #{retention['size']}"
22
+ unless db.collection_names.include?(retention.fetch(:name))
23
+ db.create_collection(retention.fetch(:name), :capped => retention.fetch(:capped), :size => retention.fetch(:size))
24
+ SimpleMetrics.logger.debug "SERVER: MongoDB - created collection #{retention.fetch(:name)}, capped: #{retention.fetch(:capped)}, size: #{retention.fetch(:size)}"
25
25
  end
26
26
 
27
- db.collection(retention['name']).ensure_index([['ts', ::Mongo::ASCENDING]])
28
- SimpleMetrics.logger.debug "SERVER: MongoDB - ensure index on column ts for collection #{retention['name']}"
27
+ db.collection(retention.fetch(:name)).ensure_index([['ts', ::Mongo::ASCENDING]])
28
+ SimpleMetrics.logger.debug "SERVER: MongoDB - ensure index on column ts for collection #{retention.fetch(:name)}"
29
29
  end
30
30
  end
31
31
 
32
32
  def truncate_collections
33
33
  buckets.each do |retention|
34
- if db.collection_names.include?(retention['name'])
35
- if retention['capped']
36
- collection(retention['name']).drop # capped collections can't remove elements, drop it instead
34
+ if db.collection_names.include?(retention.fetch(:name))
35
+ if retention.fetch(:capped)
36
+ collection(retention.fetch(:name)).drop # capped collections can't remove elements, drop it instead
37
37
  else
38
- collection(retention['name']).remove
38
+ collection(retention.fetch(:name)).remove
39
39
  end
40
- SimpleMetrics.logger.debug "SERVER: MongoDB - truncated collection #{retention['name']}"
40
+ SimpleMetrics.logger.debug "SERVER: MongoDB - truncated collection #{retention.fetch(:name)}"
41
41
  end
42
42
  end
43
43
  end
@@ -49,7 +49,7 @@ module SimpleMetrics
49
49
  end
50
50
 
51
51
  def retention_names
52
- buckets.map { |r| r['name'] }
52
+ buckets.map { |r| r.fetch(:name) }
53
53
  end
54
54
 
55
55
  def buckets
@@ -79,11 +79,6 @@ module SimpleMetrics
79
79
  data_point(result) if result
80
80
  end
81
81
 
82
- def find_all_in_ts_range(from, to)
83
- results = @collection.find(range_query(from, to)).to_a
84
- data_points(results)
85
- end
86
-
87
82
  def find_all_in_ts_range_by_name(from, to, name)
88
83
  results = @collection.find({ :name => name }.merge(range_query(from, to))).to_a
89
84
  data_points(results)
@@ -94,18 +89,6 @@ module SimpleMetrics
94
89
  data_points(results)
95
90
  end
96
91
 
97
- def count_at(ts)
98
- @collection.find({ :ts => ts }).count
99
- end
100
-
101
- def count_for_name_at(ts, name)
102
- @collection.find({ :ts => ts, :name => name }).count
103
- end
104
-
105
- def find_all_distinct_names
106
- @collection.distinct(:name).to_a
107
- end
108
-
109
92
  private
110
93
 
111
94
  def regexp(target)
@@ -3,28 +3,32 @@ module SimpleMetrics
3
3
  module Graph
4
4
  extend self
5
5
 
6
- def minutes
7
- Bucket[0]
8
- end
9
-
10
- def hours
11
- Bucket[1]
12
- end
13
-
14
- def day
15
- Bucket[2]
16
- end
6
+ def minutes; Bucket[0]; end
7
+ def hours; Bucket[1]; end
8
+ def day; Bucket[2]; end
9
+ def week; Bucket[3]; end
17
10
 
18
- def week
19
- Bucket[3]
11
+ def time_range(time)
12
+ case time
13
+ when 'minute'
14
+ 5 * one_minute
15
+ when 'hour'
16
+ one_hour
17
+ when 'day'
18
+ one_day
19
+ when 'week'
20
+ one_week
21
+ else
22
+ raise "Unknown time param: #{time}"
23
+ end
20
24
  end
21
-
25
+
22
26
  def query_all(bucket, from, to, *targets)
23
- result = {}
27
+ results = []
24
28
  Array(targets).each do |target|
25
- result[target.inspect] = values_only(query(bucket, from, to, target))
29
+ results << { :name => target, :data => query(bucket, from, to, target).map { |data| { :x => data.ts, :y => data.value || 0 } } }
26
30
  end
27
- result
31
+ results
28
32
  end
29
33
 
30
34
  def query(bucket, from, to, target)
@@ -42,12 +46,24 @@ module SimpleMetrics
42
46
 
43
47
  private
44
48
 
45
- def wild_card_query?(target)
46
- target.is_a?(String) && target.include?('*')
49
+ def one_minute
50
+ 60
47
51
  end
48
52
 
49
- def values_only(data_point_array)
50
- data_point_array.map { |data| { :ts => data.ts, :value => data.value } }
53
+ def one_hour
54
+ one_minute * 60
55
+ end
56
+
57
+ def one_day
58
+ one_hour * 24
59
+ end
60
+
61
+ def one_week
62
+ one_day * 7
63
+ end
64
+
65
+ def wild_card_query?(target)
66
+ target.is_a?(String) && target.include?('*')
51
67
  end
52
68
 
53
69
  end