simple_metrics 0.4.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. data/.travis.yml +3 -0
  2. data/README.markdown +7 -59
  3. data/Rakefile +17 -0
  4. data/bin/populate +25 -12
  5. data/bin/populate_for_demo +65 -0
  6. data/bin/simple_metrics_server +4 -3
  7. data/config.ru +13 -0
  8. data/default_config.yml +34 -0
  9. data/lib/simple_metrics/bucket.rb +43 -106
  10. data/lib/simple_metrics/configuration.rb +82 -0
  11. data/lib/simple_metrics/dashboard.rb +29 -0
  12. data/lib/simple_metrics/dashboard_repository.rb +59 -0
  13. data/lib/simple_metrics/data_point/base.rb +59 -0
  14. data/lib/simple_metrics/data_point/counter.rb +20 -0
  15. data/lib/simple_metrics/data_point/event.rb +16 -0
  16. data/lib/simple_metrics/data_point/gauge.rb +19 -0
  17. data/lib/simple_metrics/data_point/timing.rb +15 -0
  18. data/lib/simple_metrics/data_point.rb +45 -137
  19. data/lib/simple_metrics/data_point_repository.rb +115 -0
  20. data/lib/simple_metrics/functions.rb +5 -5
  21. data/lib/simple_metrics/graph.rb +69 -32
  22. data/lib/simple_metrics/importer.rb +64 -0
  23. data/lib/simple_metrics/instrument.rb +28 -0
  24. data/lib/simple_metrics/instrument_repository.rb +64 -0
  25. data/lib/simple_metrics/metric.rb +29 -0
  26. data/lib/simple_metrics/metric_repository.rb +54 -0
  27. data/lib/simple_metrics/repository.rb +34 -0
  28. data/lib/simple_metrics/udp_server.rb +81 -0
  29. data/lib/simple_metrics/version.rb +1 -1
  30. data/lib/simple_metrics.rb +21 -76
  31. data/simple_metrics.gemspec +4 -0
  32. data/spec/bucket_spec.rb +17 -152
  33. data/spec/dashboard_repository_spec.rb +52 -0
  34. data/spec/data_point_repository_spec.rb +77 -0
  35. data/spec/data_point_spec.rb +14 -64
  36. data/spec/graph_spec.rb +30 -19
  37. data/spec/importer_spec.rb +126 -0
  38. data/spec/instrument_repository_spec.rb +52 -0
  39. data/spec/metric_repository_spec.rb +53 -0
  40. data/spec/spec_helper.rb +4 -3
  41. metadata +93 -23
  42. data/bin/simple_metrics_client +0 -64
  43. data/lib/simple_metrics/client.rb +0 -83
  44. data/lib/simple_metrics/mongo.rb +0 -48
  45. data/lib/simple_metrics/server.rb +0 -66
@@ -0,0 +1,64 @@
1
+ module SimpleMetrics
2
+ module Importer
3
+ extend self
4
+
5
+ def flush_raw(data)
6
+ data_points = []
7
+ data.each do |str|
8
+ begin
9
+ data_points << DataPoint.parse(str)
10
+ rescue DataPoint::ParserError => e
11
+ SimpleMetrics.logger.debug "Invalid Data skipped: #{str}, #{e}"
12
+ end
13
+ end
14
+ flush_data_points(data_points, Time.now.utc.to_i)
15
+ end
16
+
17
+ def flush_data_points(data_points, ts = nil)
18
+ return if data_points.empty?
19
+ SimpleMetrics.logger.info "#{Time.now} Flushing #{data_points.count} to MongoDB"
20
+
21
+ ts ||= Time.now.utc.to_i
22
+ bucket = Bucket.first
23
+
24
+ group_by_name(data_points) do |name, dps|
25
+ dp = DataPoint.aggregate_values(dps)
26
+ bucket.save(dp, ts)
27
+ update_metric(dp, dps.size)
28
+ aggregate_coarse_buckets(dp)
29
+ end
30
+ end
31
+
32
+ def aggregate_coarse_buckets(dp)
33
+ coarse_buckets.each do |bucket|
34
+ if existing_dp = bucket.find_data_point_at_ts(dp.ts, dp.name)
35
+ bucket.update(existing_dp.combine(dp), existing_dp.ts)
36
+ else
37
+ bucket.save(dp, dp.ts)
38
+ end
39
+ end
40
+ end
41
+
42
+ private
43
+
44
+ def group_by_name(dps, &block)
45
+ dps.group_by { |dp| dp.name }.each_pair do |name,dps|
46
+ block.call(name, dps)
47
+ end
48
+ end
49
+
50
+ def update_metric(dp, total)
51
+ if metric = MetricRepository.find_one_by_name(dp.name)
52
+ metric.total += total
53
+ MetricRepository.update(metric)
54
+ else
55
+ MetricRepository.save(Metric.new(:name => dp.name, :total => total))
56
+ end
57
+ end
58
+
59
+ def coarse_buckets
60
+ Bucket.all.sort_by! { |r| r.seconds }[1..-1]
61
+ end
62
+
63
+ end
64
+ end
@@ -0,0 +1,28 @@
1
+ module SimpleMetrics
2
+ class Instrument
3
+
4
+ attr_accessor :id, :name, :metrics, :created_at, :updated_at
5
+
6
+ def initialize(attributes)
7
+ @id = attributes[:id]
8
+ @name = attributes[:name]
9
+ @created_at = attributes[:created_at]
10
+ @updated_at = attributes[:updated_at]
11
+ @metrics = attributes[:metrics] || []
12
+ end
13
+
14
+ def attributes
15
+ {
16
+ :id => @id.to_s, # convert bson id to str
17
+ :name => @name,
18
+ :metrics => @metrics,
19
+ :created_at => @created_at,
20
+ :updated_at => @updated_at
21
+ }
22
+ end
23
+
24
+ def to_s
25
+ attributes.to_s
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,64 @@
1
+ module SimpleMetrics
2
+ class InstrumentRepository
3
+
4
+ class << self
5
+
6
+ def find_one(id)
7
+ instrument(collection.find_one(BSON::ObjectId.from_string(id)))
8
+ end
9
+
10
+ def find_one_by_name(name)
11
+ result = collection.find({ :name => name }).to_a.first
12
+ instrument(result) if result
13
+ end
14
+
15
+ def find_all
16
+ results = collection.find.sort([['name', ::Mongo::ASCENDING]]).to_a
17
+ instruments(results) if results
18
+ end
19
+
20
+ def save(instrument)
21
+ instrument.created_at = Time.now.utc
22
+ instrument.updated_at = Time.now.utc
23
+ attributes = instrument.attributes.reject { |key, value| key.to_s == "id" }
24
+ id = collection.insert(attributes)
25
+ instrument.id = id
26
+ id
27
+ end
28
+
29
+ def update(instrument)
30
+ collection.update({ "_id" => instrument.id }, "$set" => instrument.attributes.merge(:updated_at => Time.now.utc).reject { |k, v| k == 'id' })
31
+ end
32
+
33
+ def remove(id)
34
+ collection.remove("_id" => BSON::ObjectId.from_string(id))
35
+ end
36
+
37
+ def truncate_collections
38
+ collection.remove
39
+ end
40
+
41
+ def ensure_index
42
+ collection.ensure_index([['created_at', ::Mongo::ASCENDING]])
43
+ collection.ensure_index([['updated_at', ::Mongo::ASCENDING]])
44
+ collection.ensure_index([['name', ::Mongo::ASCENDING]])
45
+ end
46
+
47
+ private
48
+
49
+ def collection
50
+ Repository.db.collection('instruments')
51
+ end
52
+
53
+ def instrument(result)
54
+ Instrument.new(:id => result["_id"], :name => result["name"], :metrics => result["metrics"], :created_at => result["created_at"], :updated_at => result["updated_at"])
55
+ end
56
+
57
+ def instruments(results)
58
+ results.inject([]) { |result, a| result << instrument(a); }
59
+ end
60
+
61
+ end # class << self
62
+
63
+ end
64
+ end
@@ -0,0 +1,29 @@
1
+ module SimpleMetrics
2
+ class Metric
3
+
4
+ attr_reader :id, :name, :created_at, :updated_at
5
+ attr_accessor :total
6
+
7
+ def initialize(attributes)
8
+ @id = attributes[:id]
9
+ @name = attributes[:name]
10
+ @total = attributes[:total]
11
+ @created_at = attributes[:created_at]
12
+ @updated_at = attributes[:updated_at]
13
+ end
14
+
15
+ def attributes
16
+ {
17
+ :id => @id.to_s, # convert bson id to str
18
+ :name => @name,
19
+ :total => @total,
20
+ :created_at => @created_at,
21
+ :updated_at => @updated_at
22
+ }
23
+ end
24
+
25
+ def to_s
26
+ attributes.to_s
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,54 @@
1
+ module SimpleMetrics
2
+ class MetricRepository
3
+
4
+ class << self
5
+
6
+ def find_one(id)
7
+ metric(collection.find_one(id))
8
+ end
9
+
10
+ def find_one_by_name(name)
11
+ result = collection.find({ :name => name }).to_a.first
12
+ metric(result) if result
13
+ end
14
+
15
+ def find_all
16
+ results = collection.find.sort([['name', ::Mongo::ASCENDING]]).to_a
17
+ metrics(results) if results
18
+ end
19
+
20
+ def save(metric)
21
+ collection.insert(metric.attributes.merge(:created_at => Time.now.utc, :updated_at => Time.now.utc))
22
+ end
23
+
24
+ def update(metric)
25
+ collection.update({ "_id" => metric.id }, { "$set" => { :total => metric.total, :updated_at => Time.now.utc }})
26
+ end
27
+
28
+ def truncate_collections
29
+ collection.remove
30
+ end
31
+
32
+ def ensure_index
33
+ collection.ensure_index([['created_at', ::Mongo::ASCENDING]])
34
+ collection.ensure_index([['updated_at', ::Mongo::ASCENDING]])
35
+ collection.ensure_index([['name', ::Mongo::ASCENDING]])
36
+ end
37
+
38
+ private
39
+
40
+ def collection
41
+ Repository.db.collection('metrics')
42
+ end
43
+
44
+ def metric(result)
45
+ Metric.new(:id => result["_id"], :name => result["name"], :total => result["total"], :created_at => result["created_at"], :updated_at => result["updated_at"])
46
+ end
47
+
48
+ def metrics(results)
49
+ results.inject([]) { |result, a| result << metric(a); }
50
+ end
51
+ end
52
+
53
+ end
54
+ end
@@ -0,0 +1,34 @@
1
+ module SimpleMetrics
2
+ module Repository
3
+ extend self
4
+
5
+ def connection
6
+ @@connection ||= ::Mongo::Connection.new(host, port)
7
+ end
8
+
9
+ def db
10
+ @@db ||= connection.db(db_name, config.fetch(:options))
11
+ end
12
+
13
+ def db_name
14
+ "simple_metrics_#{prefix}"
15
+ end
16
+
17
+ def prefix
18
+ config[:prefix] || 'development'
19
+ end
20
+
21
+ def host
22
+ config[:host] || 'localhost'
23
+ end
24
+
25
+ def port
26
+ config[:port] || 27017
27
+ end
28
+
29
+ def config
30
+ SimpleMetrics.config.db
31
+ end
32
+
33
+ end
34
+ end
@@ -0,0 +1,81 @@
1
+ # encoding: utf-8
2
+ require "eventmachine"
3
+
4
+ if defined? Encoding
5
+ Encoding.default_external = Encoding::UTF_8
6
+ end
7
+
8
+ module SimpleMetrics
9
+
10
+ module ClientHandler
11
+
12
+ @@data = []
13
+
14
+ class << self
15
+ def get_and_clear_data
16
+ data = @@data.dup
17
+ @@data = []
18
+ data
19
+ end
20
+ end
21
+
22
+ def post_init
23
+ SimpleMetrics.logger.info "ClientHandler entering post_init"
24
+ end
25
+
26
+ def receive_data(data)
27
+ SimpleMetrics.logger.debug "received_data: #{data.inspect}"
28
+ @@data << data
29
+ end
30
+ end
31
+
32
+ class UDPServer
33
+
34
+ def start
35
+ SimpleMetrics.logger.info "SERVER: starting up on #{host}:#{port}..."
36
+
37
+ trap('TERM') { stop }
38
+ trap('INT') { stop }
39
+
40
+ DataPointRepository.ensure_collections_exist
41
+
42
+ EM.run do
43
+ EM.open_datagram_socket(host, port, SimpleMetrics::ClientHandler) do |con|
44
+ EventMachine::add_periodic_timer(flush_interval) do
45
+ SimpleMetrics.logger.debug "SERVER: period timer triggered after #{flush_interval} seconds"
46
+
47
+ EM.defer { Bucket.flush_raw(ClientHandler.get_and_clear_data) }
48
+ end
49
+ end
50
+ end
51
+ end
52
+
53
+ def stop
54
+ SimpleMetrics.logger.info "EventMachine stop"
55
+ EM.stop
56
+ end
57
+
58
+ def to_s
59
+ "#{host}:#{port}"
60
+ end
61
+
62
+ private
63
+
64
+ def host
65
+ config['host'] || 'localhost'
66
+ end
67
+
68
+ def port
69
+ config['port'] || 8125
70
+ end
71
+
72
+ def flush_interval
73
+ config['flush_interval'] || 10
74
+ end
75
+
76
+ def config
77
+ SimpleMetrics.config.server
78
+ end
79
+
80
+ end
81
+ end
@@ -1,4 +1,4 @@
1
1
  # encoding: utf-8
2
2
  module SimpleMetrics
3
- VERSION = "0.4.0"
3
+ VERSION = "0.4.2"
4
4
  end
@@ -1,14 +1,28 @@
1
1
  # encoding: utf-8
2
2
  require "logger"
3
+ require "active_support/all"
3
4
 
4
5
  require "simple_metrics/version"
5
- require "simple_metrics/client"
6
- require "simple_metrics/server"
6
+ require "simple_metrics/configuration"
7
+ require "simple_metrics/udp_server"
8
+ require "simple_metrics/repository"
9
+ require "simple_metrics/data_point_repository"
7
10
  require "simple_metrics/data_point"
11
+ require "simple_metrics/data_point/base"
12
+ require "simple_metrics/data_point/counter"
13
+ require "simple_metrics/data_point/event"
14
+ require "simple_metrics/data_point/gauge"
15
+ require "simple_metrics/data_point/timing"
16
+ require "simple_metrics/importer"
8
17
  require "simple_metrics/bucket"
9
18
  require "simple_metrics/graph"
10
19
  require "simple_metrics/functions"
11
- require "simple_metrics/mongo"
20
+ require "simple_metrics/metric"
21
+ require "simple_metrics/metric_repository"
22
+ require "simple_metrics/instrument"
23
+ require "simple_metrics/instrument_repository"
24
+ require "simple_metrics/dashboard"
25
+ require "simple_metrics/dashboard_repository"
12
26
 
13
27
  module SimpleMetrics
14
28
  extend self
@@ -21,81 +35,12 @@ module SimpleMetrics
21
35
  @@logger = logger
22
36
  end
23
37
 
24
- CONFIG_DEFAULTS = {
25
- :host => 'localhost',
26
- :port => 8125,
27
- :flush_interval => 10
28
- }.freeze
29
-
30
38
  def config
31
- @@config ||= CONFIG_DEFAULTS
32
- end
33
-
34
- def config=(options)
35
- @@config = CONFIG_DEFAULTS.merge(options)
36
- end
37
-
38
- BUCKETS_DEFAULTS = [
39
- {
40
- :name => 'stats_per_10s',
41
- :seconds => 10,
42
- :capped => true,
43
- :size => 100_100_100
44
- },
45
- {
46
- :name => 'stats_per_1min',
47
- :seconds => 60,
48
- :capped => true,
49
- :size => 1_100_100_100
50
- },
51
- {
52
- :name => 'stats_per_10min',
53
- :seconds => 600,
54
- :size => 0 ,
55
- :capped => false
56
- },
57
- {
58
- :name => 'stats_per_day',
59
- :seconds => 86400,
60
- :size => 0,
61
- :capped => false
62
- }
63
- ].freeze
64
-
65
- def buckets_config
66
- @@buckets ||= BUCKETS_DEFAULTS
67
- end
68
-
69
- def buckets_config=(buckets)
70
- @@buckets = buckets
71
- end
72
-
73
- MONGODB_DEFAULTS = {
74
- :pool_size => 5,
75
- :timeout => 5,
76
- :strict => true
77
- }.freeze
78
-
79
- DB_CONFIG_DEFAULTS = {
80
- :host => 'localhost',
81
- :port => 27017,
82
- :prefix => 'development'
83
- }.freeze
84
-
85
- def db_config=(options)
86
- @@db_config = {
87
- :host => options.delete(:host) || 'localhost',
88
- :port => options.delete(:port) || 27017,
89
- :db_name => "simple_metrics_#{options.delete(:prefix)}",
90
- :options => MONGODB_DEFAULTS.merge(options)
91
- }
39
+ @@config ||= Configuration.new
92
40
  end
93
41
 
94
- def db_config
95
- @@db_config ||= DB_CONFIG_DEFAULTS.merge(
96
- :db_name => "simple_metrics_#{DB_CONFIG_DEFAULTS[:prefix]}",
97
- :options => MONGODB_DEFAULTS
98
- )
42
+ def configure(hash = {}, &block)
43
+ config.configure(hash, &block)
99
44
  end
100
45
 
101
- end
46
+ end
@@ -19,9 +19,13 @@ Gem::Specification.new do |s|
19
19
  s.add_development_dependency "rake"
20
20
  s.add_development_dependency "rspec"
21
21
  s.add_development_dependency "rr"
22
+ s.add_development_dependency "shotgun"
23
+ s.add_development_dependency "rack-test"
22
24
 
23
25
  s.add_dependency "eventmachine"
24
26
  s.add_dependency "daemons"
25
27
  s.add_dependency "mongo", '~> 1.6'
26
28
  s.add_dependency "bson", '~> 1.6'
29
+ s.add_dependency "bson_ext", '~> 1.6'
30
+ s.add_dependency "activesupport"
27
31
  end