simple_metrics 0.4.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. data/.travis.yml +3 -0
  2. data/README.markdown +7 -59
  3. data/Rakefile +17 -0
  4. data/bin/populate +25 -12
  5. data/bin/populate_for_demo +65 -0
  6. data/bin/simple_metrics_server +4 -3
  7. data/config.ru +13 -0
  8. data/default_config.yml +34 -0
  9. data/lib/simple_metrics/bucket.rb +43 -106
  10. data/lib/simple_metrics/configuration.rb +82 -0
  11. data/lib/simple_metrics/dashboard.rb +29 -0
  12. data/lib/simple_metrics/dashboard_repository.rb +59 -0
  13. data/lib/simple_metrics/data_point/base.rb +59 -0
  14. data/lib/simple_metrics/data_point/counter.rb +20 -0
  15. data/lib/simple_metrics/data_point/event.rb +16 -0
  16. data/lib/simple_metrics/data_point/gauge.rb +19 -0
  17. data/lib/simple_metrics/data_point/timing.rb +15 -0
  18. data/lib/simple_metrics/data_point.rb +45 -137
  19. data/lib/simple_metrics/data_point_repository.rb +115 -0
  20. data/lib/simple_metrics/functions.rb +5 -5
  21. data/lib/simple_metrics/graph.rb +69 -32
  22. data/lib/simple_metrics/importer.rb +64 -0
  23. data/lib/simple_metrics/instrument.rb +28 -0
  24. data/lib/simple_metrics/instrument_repository.rb +64 -0
  25. data/lib/simple_metrics/metric.rb +29 -0
  26. data/lib/simple_metrics/metric_repository.rb +54 -0
  27. data/lib/simple_metrics/repository.rb +34 -0
  28. data/lib/simple_metrics/udp_server.rb +81 -0
  29. data/lib/simple_metrics/version.rb +1 -1
  30. data/lib/simple_metrics.rb +21 -76
  31. data/simple_metrics.gemspec +4 -0
  32. data/spec/bucket_spec.rb +17 -152
  33. data/spec/dashboard_repository_spec.rb +52 -0
  34. data/spec/data_point_repository_spec.rb +77 -0
  35. data/spec/data_point_spec.rb +14 -64
  36. data/spec/graph_spec.rb +30 -19
  37. data/spec/importer_spec.rb +126 -0
  38. data/spec/instrument_repository_spec.rb +52 -0
  39. data/spec/metric_repository_spec.rb +53 -0
  40. data/spec/spec_helper.rb +4 -3
  41. metadata +93 -23
  42. data/bin/simple_metrics_client +0 -64
  43. data/lib/simple_metrics/client.rb +0 -83
  44. data/lib/simple_metrics/mongo.rb +0 -48
  45. data/lib/simple_metrics/server.rb +0 -66
@@ -0,0 +1,59 @@
1
+ module SimpleMetrics
2
+ class DashboardRepository
3
+
4
+ class << self
5
+
6
+ def find_one(id)
7
+ dashboard(collection.find_one(BSON::ObjectId.from_string(id)))
8
+ end
9
+
10
+ def find_one_by_name(name)
11
+ result = collection.find({ :name => name }).to_a.first
12
+ dashboard(result) if result
13
+ end
14
+
15
+ def find_all
16
+ results = collection.find.sort([['name', ::Mongo::ASCENDING]]).to_a
17
+ dashboards(results) if results
18
+ end
19
+
20
+ def save(dashboard)
21
+ collection.insert(dashboard.attributes.merge(:created_at => Time.now.utc, :updated_at => Time.now.utc))
22
+ end
23
+
24
+ def update(dashboard)
25
+ collection.update({ "_id" => dashboard.id }, { "$set" => dashboard.attributes.merge(:updated_at => Time.now.utc).reject { |k, v| k == 'id' } })
26
+ end
27
+
28
+ def remove(id)
29
+ collection.remove("_id" => BSON::ObjectId.from_string(id))
30
+ end
31
+
32
+ def truncate_collections
33
+ collection.remove
34
+ end
35
+
36
+ def ensure_index
37
+ collection.ensure_index([['created_at', ::Mongo::ASCENDING]])
38
+ collection.ensure_index([['updated_at', ::Mongo::ASCENDING]])
39
+ collection.ensure_index([['name', ::Mongo::ASCENDING]])
40
+ end
41
+
42
+ private
43
+
44
+ def collection
45
+ Repository.db.collection('dashboards')
46
+ end
47
+
48
+ def dashboard(result)
49
+ Dashboard.new(:id => result["_id"], :name => result["name"], :instruments => result["instruments"], :created_at => result["created_at"], :updated_at => result["updated_at"])
50
+ end
51
+
52
+ def dashboards(results)
53
+ results.inject([]) { |result, a| result << dashboard(a); }
54
+ end
55
+
56
+ end # class << self
57
+
58
+ end
59
+ end
@@ -0,0 +1,59 @@
1
+ # encoding: utf-8
2
+ module SimpleMetrics
3
+ module DataPoint
4
+ class Base
5
+
6
+ attr_accessor :name, :ts, :type, :value, :total, :sum
7
+ attr_reader :id
8
+
9
+ def initialize(attributes)
10
+ @id = attributes[:id]
11
+ @name = attributes[:name]
12
+ @value = attributes[:value]
13
+ @ts = attributes[:ts]
14
+ @sample_rate = attributes[:sample_rate]
15
+ @sum = attributes[:sum]
16
+ @total = attributes[:total]
17
+ end
18
+
19
+ def counter?
20
+ @type == 'c'
21
+ end
22
+
23
+ def gauge?
24
+ @type == 'g'
25
+ end
26
+
27
+ def timing?
28
+ @type == 'ms'
29
+ end
30
+
31
+ def event?
32
+ @type == 'ev'
33
+ end
34
+
35
+ def timestamp
36
+ ts
37
+ end
38
+
39
+ def value
40
+ @value.to_i if @value
41
+ end
42
+
43
+ def attributes
44
+ {
45
+ :name => @name,
46
+ :value => @value,
47
+ :ts => @ts,
48
+ :type => @type,
49
+ :total => @total,
50
+ :sum => @sum
51
+ }
52
+ end
53
+
54
+ def to_s
55
+ attributes.to_s
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,20 @@
1
+ module SimpleMetrics
2
+ module DataPoint
3
+ class Counter < Base
4
+
5
+ def initialize(attributes)
6
+ super(attributes)
7
+ @type = 'c'
8
+ @value = (@value.to_i || 1) * (1.0 / (@sample_rate || 1).to_f)
9
+ end
10
+
11
+ def combine(dp)
12
+ @total += 1
13
+ @value += dp.value
14
+ @sum += dp.value
15
+ self
16
+ end
17
+
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,16 @@
1
+ module SimpleMetrics
2
+ module DataPoint
3
+ class Event < Base
4
+
5
+ def initialize(attributes)
6
+ super(attributes)
7
+ @type = 'ev'
8
+ end
9
+
10
+ def combine(dp)
11
+ raise "Implement me!"
12
+ end
13
+
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,19 @@
1
+ module SimpleMetrics
2
+ module DataPoint
3
+ class Gauge < Base
4
+
5
+ def initialize(attributes)
6
+ super(attributes)
7
+ @type = 'g'
8
+ @value = (@value.to_i || 1) * (1.0 / (@sample_rate || 1).to_f)
9
+ end
10
+
11
+ def combine(dp)
12
+ @total += 1
13
+ @sum += dp.value
14
+ @value = @sum / @total
15
+ self
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,15 @@
1
+ module SimpleMetrics
2
+ module DataPoint
3
+ class Timing < Base
4
+
5
+ def initialize(attributes)
6
+ super(attributes)
7
+ @type = 'ms'
8
+ end
9
+
10
+ def combine(dp)
11
+ raise "Implement me!"
12
+ end
13
+ end
14
+ end
15
+ end
@@ -1,10 +1,10 @@
1
- # encoding: utf-8
2
1
  module SimpleMetrics
3
-
4
- class DataPoint
2
+ module DataPoint
3
+ extend self
5
4
 
6
5
  class NonMatchingTypesError < Exception; end
7
6
  class ParserError < Exception; end
7
+ class UnknownTypeError < Exception; end
8
8
 
9
9
  # examples:
10
10
  # com.example.test1:1|c
@@ -14,149 +14,57 @@ module SimpleMetrics
14
14
  # com.example.test4:44|ms
15
15
  REGEXP = /^([\d\w_.]*):(-?[\d]*)\|(c|g|ms){1}(\|@([.\d]+))?$/i
16
16
 
17
- class << self
18
-
19
- def parse(str)
20
- if str =~ REGEXP
21
- name, value, type, sample_rate = $1, $2, $3, $5
22
- if type == "ms"
23
- # TODO: implement sample_rate handling
24
- create_timing(:name => name, :value => value)
25
- elsif type == "g"
26
- create_gauge(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
27
- elsif type == "c"
28
- create_counter(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
29
- end
30
- else
31
- raise ParserError, "Parser Error - Invalid Stat: #{str}"
32
- end
33
- end
34
-
35
- def create_counter(attributes)
36
- self.new(attributes.merge(:type => 'c'))
37
- end
38
-
39
- def create_gauge(attributes)
40
- self.new(attributes.merge(:type => 'g'))
41
- end
42
-
43
- def create_timing(attributes)
44
- self.new(attributes.merge(:type => 'ms'))
45
- end
46
-
47
- def aggregate(stats_array, name = nil)
48
- raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
49
-
50
- result_stat = stats_array.first.dup
51
- result_stat.name = name if name
52
- if stats_array.first.counter?
53
- result_stat.value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
54
- result_stat
55
- elsif stats_array.first.gauge?
56
- total_value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
57
- result_stat.value = total_value / stats_array.size
58
- result_stat
59
- elsif stats_array.first.timing?
60
- # TODO implement timing aggregation
61
- elsif stats_array.first.event?
62
- # TODO implement event aggregation
63
- else
64
- raise ArgumentError, "Unknown data point type"
65
- end
66
- end
67
-
68
- def aggregate_array(stats_array, name = nil)
69
- raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
70
-
71
- if stats_array.first.counter?
72
- tmp_hash = ts_hash_aggregated(stats_array) do |value1, value2|
73
- value1 + value2
74
- end
75
-
76
- result = []
77
- tmp_hash.each_pair do |key, value|
78
- result << self.new(:name => name, :ts => key, :value => value, :type => 'c')
79
- end
80
- result
81
- elsif stats_array.first.gauge?
82
- tmp_hash = ts_hash_aggregated(stats_array) do |value1, value2|
83
- (value1 + value2)/2
84
- end
85
-
86
- result = []
87
- tmp_hash.each_pair do |key, value|
88
- result << self.new(:name => name, :ts => key, :value => value, :type => 'g')
89
- end
90
- result
91
- elsif stats_array.first.timing?
92
- # TODO implement timing aggregation
93
- elsif stats_array.first.event?
94
- # TODO implement event aggregation
95
- else
96
- raise ArgumentError, "Unknown data point type"
97
- end
98
- end
99
-
100
- def create_from_db(attributes)
101
- self.new(:name => attributes["name"], :value => attributes["value"], :ts => attributes["ts"], :type => attributes["type"])
102
- end
103
-
104
- def ts_hash(query_result)
105
- query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
106
- end
107
-
108
- private
109
-
110
- def ts_hash_aggregated(data_points, &block)
111
- tmp = {}
112
- data_points.each do |dp|
113
- if tmp.key?(dp.ts)
114
- tmp[dp.ts] = block.call(tmp[dp.ts], dp.value)
115
- else
116
- tmp[dp.ts] = dp.value
117
- end
118
- end
119
- tmp
17
+ def parse(str)
18
+ if str =~ REGEXP
19
+ name, value, type, sample_rate = $1, $2, $3, $5
20
+ build(:name => name, :value => value, :type => type, :sample_rate => sample_rate)
21
+ else
22
+ raise ParserError, "Parser Error - Invalid data point: #{str}"
120
23
  end
121
24
  end
122
25
 
123
- attr_accessor :name, :ts, :type, :value
124
-
125
- def initialize(attributes)
126
- @name = attributes[:name]
127
- @value = attributes[:value]
128
- @ts = attributes[:ts]
129
- @type = attributes[:type]
130
- end
131
-
132
- def counter?
133
- type == 'c'
134
- end
135
-
136
- def gauge?
137
- type == 'g'
26
+ def build(attributes)
27
+ case attributes[:type]
28
+ when 'c'
29
+ Counter.new(attributes)
30
+ when 'g'
31
+ Gauge.new(attributes)
32
+ when 'ms'
33
+ Timing.new(attributes)
34
+ when 'ev'
35
+ Event.new(attributes)
36
+ else
37
+ raise UnknownTypeError, "Unknown Type Error: #{attributes[:type]}"
38
+ end
138
39
  end
139
-
140
- def timing?
141
- type == 'ms'
40
+
41
+ def aggregate_values(dps)
42
+ raise SimpleMetrics::DataPoint::NonMatchingTypesError if has_non_matching_types?(dps)
43
+
44
+ dp = dps.first.dup
45
+ dp.value = if dp.counter?
46
+ sum(dps)
47
+ elsif dp.gauge?
48
+ sum(dps) / dps.size
49
+ elsif dp.event?
50
+ raise "Implement me!"
51
+ elsif dp.timing?
52
+ raise "Implement me!"
53
+ else
54
+ raise ArgumentError("Unknown data point type: #{dp}")
55
+ end
56
+ dp
142
57
  end
143
58
 
144
- def timestamp
145
- ts
146
- end
59
+ private
147
60
 
148
- def value
149
- @value.to_i if @value
61
+ def sum(dps)
62
+ dps.map { |dp| dp.value }.inject(0) { |result, value| result += value }
150
63
  end
151
64
 
152
- def attributes
153
- {
154
- :name => name,
155
- :value => value,
156
- :ts => ts,
157
- :type => type
158
- }
65
+ def has_non_matching_types?(dps)
66
+ dps.group_by { |dp| dp.type }.size != 1
159
67
  end
160
68
 
161
69
  end
162
- end
70
+ end
@@ -0,0 +1,115 @@
1
+ require "mongo"
2
+
3
+ module SimpleMetrics
4
+ class DataPointRepository
5
+
6
+ class << self
7
+
8
+ @@collection = {}
9
+
10
+ def for_retention(name)
11
+ self.new(collection(name))
12
+ end
13
+
14
+ def collection(name)
15
+ raise ArgumentError, "Unknown retention: #{name}" unless retention_names.include?(name)
16
+ @@collection[name] ||= db.collection(name)
17
+ end
18
+
19
+ def ensure_collections_exist
20
+ SimpleMetrics.logger.debug "SERVER: MongoDB - found following collections: #{db.collection_names.inspect}"
21
+ buckets.each do |retention|
22
+ unless db.collection_names.include?(retention.fetch(:name))
23
+ db.create_collection(retention.fetch(:name), :capped => retention.fetch(:capped), :size => retention.fetch(:size))
24
+ SimpleMetrics.logger.debug "SERVER: MongoDB - created collection #{retention.fetch(:name)}, capped: #{retention.fetch(:capped)}, size: #{retention.fetch(:size)}"
25
+ end
26
+
27
+ db.collection(retention.fetch(:name)).ensure_index([['ts', ::Mongo::ASCENDING]])
28
+ db.collection(retention.fetch(:name)).ensure_index([['_id', ::Mongo::ASCENDING]])
29
+ SimpleMetrics.logger.debug "SERVER: MongoDB - ensure index on column ts for collection #{retention.fetch(:name)}"
30
+ end
31
+ end
32
+
33
+ def truncate_collections
34
+ buckets.each do |retention|
35
+ if db.collection_names.include?(retention.fetch(:name))
36
+ if retention.fetch(:capped)
37
+ collection(retention.fetch(:name)).drop # capped collections can't remove elements, drop it instead
38
+ else
39
+ collection(retention.fetch(:name)).remove
40
+ end
41
+ SimpleMetrics.logger.debug "SERVER: MongoDB - truncated collection #{retention.fetch(:name)}"
42
+ end
43
+ end
44
+ end
45
+
46
+ private
47
+
48
+ def db
49
+ Repository.db
50
+ end
51
+
52
+ def retention_names
53
+ buckets.map { |r| r.fetch(:name) }
54
+ end
55
+
56
+ def buckets
57
+ SimpleMetrics.config.buckets
58
+ end
59
+ end
60
+
61
+ def initialize(collection)
62
+ @collection = collection
63
+ end
64
+
65
+ def save(result)
66
+ @collection.insert(result.attributes.reject { |k, v| k == 'id' })
67
+ end
68
+
69
+ def update(dp, ts)
70
+ @collection.update({ "_id" => dp.id }, { "$set" => { :value => dp.value, :sum => dp.sum, :total => dp.total }})
71
+ end
72
+
73
+ def find_all_at_ts(ts)
74
+ results = @collection.find({ :ts => ts }).to_a
75
+ data_points(results)
76
+ end
77
+
78
+ def find_data_point_at_ts(ts, name)
79
+ result = @collection.find_one({ :ts => ts, :name => name })
80
+ data_point(result) if result
81
+ end
82
+
83
+ def find_all_in_ts_range_by_name(from, to, name)
84
+ results = @collection.find({ :name => name }.merge(range_query(from, to))).to_a
85
+ data_points(results)
86
+ end
87
+
88
+ def find_all_in_ts_range_by_wildcard(from, to, target)
89
+ results = @collection.find({ :name => regexp(target) }.merge(range_query(from, to))).to_a
90
+ data_points(results)
91
+ end
92
+
93
+ private
94
+
95
+ def regexp(target)
96
+ /#{wildcard_replace(target)}/
97
+ end
98
+
99
+ def wildcard_replace(target)
100
+ target.gsub('.', '\.').gsub('*', '.*')
101
+ end
102
+
103
+ def range_query(from, to)
104
+ { :ts => { "$gte" => from, "$lte" => to } }
105
+ end
106
+
107
+ def data_point(result)
108
+ DataPoint.build(:id => result["_id"], :name => result["name"], :value => result["value"], :ts => result["ts"], :type => result["type"], :sum => result["sum"], :total => result["total"])
109
+ end
110
+
111
+ def data_points(results)
112
+ results.inject([]) { |result, a| result << data_point(a); }
113
+ end
114
+ end
115
+ end
@@ -2,14 +2,14 @@ module SimpleMetrics
2
2
 
3
3
  module Functions
4
4
 
5
- # calculate the maximum value for multiple targets
6
- #
7
- # params:
5
+ # calculate the maximum value for multiple targets
6
+ #
7
+ # params:
8
8
  # data_points [1, 3, 5], [2, 1, 6]
9
9
  #
10
10
  # return:
11
11
  # array [2, 3, 6]
12
- def max(*data_points)
12
+ def max(*data_points)
13
13
  end
14
14
 
15
15
  # calculate the minimum value for multiple targets
@@ -17,7 +17,7 @@ module SimpleMetrics
17
17
  end
18
18
 
19
19
  # add offset to each value
20
- def offset(*data_points)
20
+ def offset(*data_points)
21
21
  end
22
22
 
23
23
  # multiple each value
@@ -1,58 +1,95 @@
1
1
  module SimpleMetrics
2
2
 
3
- #
4
- # url format examples:
5
- # * target=com.post.clicks (1 line in graph)
6
- # * target=com.post.clicks.text&target=com.post.clicks.logo (2 lines in graph)
7
- # * target=com.post.clicks.* (1 aggregated line in graph)
8
- #
9
3
  module Graph
10
4
  extend self
11
5
 
12
- def minutes
13
- Bucket[0]
14
- end
15
-
16
- def hours
17
- Bucket[1]
18
- end
6
+ def minutes; Bucket[0]; end
7
+ def hours; Bucket[1]; end
8
+ def day; Bucket[2]; end
9
+ def week; Bucket[3]; end
19
10
 
20
- def day
21
- Bucket[2]
22
- end
23
-
24
- def week
25
- Bucket[3]
11
+ def time_range(time)
12
+ case time
13
+ when 'minute'
14
+ 5 * one_minute
15
+ when 'hour'
16
+ one_hour
17
+ when 'day'
18
+ one_day
19
+ when 'week'
20
+ one_week
21
+ else
22
+ raise "Unknown time param: #{time}"
23
+ end
26
24
  end
27
-
25
+
28
26
  def query_all(bucket, from, to, *targets)
29
- result = {}
27
+ results = []
30
28
  Array(targets).each do |target|
31
- result[target.inspect] = values_only(query(bucket, from, to, target))
29
+ results << { :name => target, :data => query(bucket, from, to, target).map { |data| { :x => data.ts, :y => data.value || 0 } } }
32
30
  end
33
- result
31
+ results
34
32
  end
35
33
 
36
34
  def query(bucket, from, to, target)
37
- if target.is_a?(Regexp)
38
- result = bucket.find_all_in_ts_range_by_regexp(from, to, target)
39
- result = DataPoint.aggregate_array(result, target.inspect)
40
- bucket.fill_gaps(from, to, result)
41
- elsif target.is_a?(String) && target.include?('*')
35
+ if wild_card_query?(target)
42
36
  result = bucket.find_all_in_ts_range_by_wildcard(from, to, target)
43
- result = DataPoint.aggregate_array(result, target)
37
+ result = aggregate(result, target)
44
38
  bucket.fill_gaps(from, to, result)
45
39
  elsif target.is_a?(String)
46
40
  result = bucket.find_all_in_ts_range_by_name(from, to, target)
47
41
  bucket.fill_gaps(from, to, result)
48
42
  else
49
- raise ArgumentError, "Unknown target: #{target.inspect}"
43
+ raise ArgumentError, "Unknown target format: #{target.inspect}"
44
+ end
45
+ end
46
+
47
+ private
48
+
49
+ def aggregate(dps, target)
50
+ raise SimpleMetrics::DataPoint::NonMatchingTypesError if has_non_matching_types?(dps)
51
+
52
+ tmp = {}
53
+ dps.each do |dp|
54
+ dp.name = target
55
+ if tmp.key?(dp.ts)
56
+ tmp[dp.ts] << dp
57
+ else
58
+ tmp[dp.ts] = [dp]
59
+ end
60
+ end
61
+ tmp
62
+
63
+ result = []
64
+ tmp.each_pair do |ts, dps|
65
+ result << DataPoint.aggregate_values(dps)
50
66
  end
67
+ result
51
68
  end
52
69
 
53
- def values_only(data_point_array)
54
- data_point_array.map { |data| { :ts => data.ts, :value => data.value } }
70
+ def has_non_matching_types?(dps)
71
+ dps.group_by { |dp| dp.type }.size != 1
55
72
  end
56
73
 
74
+ def one_minute
75
+ 60
76
+ end
77
+
78
+ def one_hour
79
+ one_minute * 60
80
+ end
81
+
82
+ def one_day
83
+ one_hour * 24
84
+ end
85
+
86
+ def one_week
87
+ one_day * 7
88
+ end
89
+
90
+ def wild_card_query?(target)
91
+ target.is_a?(String) && target.include?('*')
92
+ end
93
+
57
94
  end
58
95
  end