simple_metrics 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.travis.yml +3 -0
- data/README.markdown +7 -59
- data/Rakefile +17 -0
- data/bin/populate +25 -12
- data/bin/populate_for_demo +65 -0
- data/bin/simple_metrics_server +4 -3
- data/config.ru +13 -0
- data/default_config.yml +34 -0
- data/lib/simple_metrics/bucket.rb +43 -106
- data/lib/simple_metrics/configuration.rb +82 -0
- data/lib/simple_metrics/dashboard.rb +29 -0
- data/lib/simple_metrics/dashboard_repository.rb +59 -0
- data/lib/simple_metrics/data_point/base.rb +59 -0
- data/lib/simple_metrics/data_point/counter.rb +20 -0
- data/lib/simple_metrics/data_point/event.rb +16 -0
- data/lib/simple_metrics/data_point/gauge.rb +19 -0
- data/lib/simple_metrics/data_point/timing.rb +15 -0
- data/lib/simple_metrics/data_point.rb +45 -137
- data/lib/simple_metrics/data_point_repository.rb +115 -0
- data/lib/simple_metrics/functions.rb +5 -5
- data/lib/simple_metrics/graph.rb +69 -32
- data/lib/simple_metrics/importer.rb +64 -0
- data/lib/simple_metrics/instrument.rb +28 -0
- data/lib/simple_metrics/instrument_repository.rb +64 -0
- data/lib/simple_metrics/metric.rb +29 -0
- data/lib/simple_metrics/metric_repository.rb +54 -0
- data/lib/simple_metrics/repository.rb +34 -0
- data/lib/simple_metrics/udp_server.rb +81 -0
- data/lib/simple_metrics/version.rb +1 -1
- data/lib/simple_metrics.rb +21 -76
- data/simple_metrics.gemspec +4 -0
- data/spec/bucket_spec.rb +17 -152
- data/spec/dashboard_repository_spec.rb +52 -0
- data/spec/data_point_repository_spec.rb +77 -0
- data/spec/data_point_spec.rb +14 -64
- data/spec/graph_spec.rb +30 -19
- data/spec/importer_spec.rb +126 -0
- data/spec/instrument_repository_spec.rb +52 -0
- data/spec/metric_repository_spec.rb +53 -0
- data/spec/spec_helper.rb +4 -3
- metadata +93 -23
- data/bin/simple_metrics_client +0 -64
- data/lib/simple_metrics/client.rb +0 -83
- data/lib/simple_metrics/mongo.rb +0 -48
- data/lib/simple_metrics/server.rb +0 -66
@@ -0,0 +1,64 @@
|
|
1
|
+
module SimpleMetrics
|
2
|
+
module Importer
|
3
|
+
extend self
|
4
|
+
|
5
|
+
def flush_raw(data)
|
6
|
+
data_points = []
|
7
|
+
data.each do |str|
|
8
|
+
begin
|
9
|
+
data_points << DataPoint.parse(str)
|
10
|
+
rescue DataPoint::ParserError => e
|
11
|
+
SimpleMetrics.logger.debug "Invalid Data skipped: #{str}, #{e}"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
flush_data_points(data_points, Time.now.utc.to_i)
|
15
|
+
end
|
16
|
+
|
17
|
+
def flush_data_points(data_points, ts = nil)
|
18
|
+
return if data_points.empty?
|
19
|
+
SimpleMetrics.logger.info "#{Time.now} Flushing #{data_points.count} to MongoDB"
|
20
|
+
|
21
|
+
ts ||= Time.now.utc.to_i
|
22
|
+
bucket = Bucket.first
|
23
|
+
|
24
|
+
group_by_name(data_points) do |name, dps|
|
25
|
+
dp = DataPoint.aggregate_values(dps)
|
26
|
+
bucket.save(dp, ts)
|
27
|
+
update_metric(dp, dps.size)
|
28
|
+
aggregate_coarse_buckets(dp)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def aggregate_coarse_buckets(dp)
|
33
|
+
coarse_buckets.each do |bucket|
|
34
|
+
if existing_dp = bucket.find_data_point_at_ts(dp.ts, dp.name)
|
35
|
+
bucket.update(existing_dp.combine(dp), existing_dp.ts)
|
36
|
+
else
|
37
|
+
bucket.save(dp, dp.ts)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def group_by_name(dps, &block)
|
45
|
+
dps.group_by { |dp| dp.name }.each_pair do |name,dps|
|
46
|
+
block.call(name, dps)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def update_metric(dp, total)
|
51
|
+
if metric = MetricRepository.find_one_by_name(dp.name)
|
52
|
+
metric.total += total
|
53
|
+
MetricRepository.update(metric)
|
54
|
+
else
|
55
|
+
MetricRepository.save(Metric.new(:name => dp.name, :total => total))
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def coarse_buckets
|
60
|
+
Bucket.all.sort_by! { |r| r.seconds }[1..-1]
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module SimpleMetrics
|
2
|
+
class Instrument
|
3
|
+
|
4
|
+
attr_accessor :id, :name, :metrics, :created_at, :updated_at
|
5
|
+
|
6
|
+
def initialize(attributes)
|
7
|
+
@id = attributes[:id]
|
8
|
+
@name = attributes[:name]
|
9
|
+
@created_at = attributes[:created_at]
|
10
|
+
@updated_at = attributes[:updated_at]
|
11
|
+
@metrics = attributes[:metrics] || []
|
12
|
+
end
|
13
|
+
|
14
|
+
def attributes
|
15
|
+
{
|
16
|
+
:id => @id.to_s, # convert bson id to str
|
17
|
+
:name => @name,
|
18
|
+
:metrics => @metrics,
|
19
|
+
:created_at => @created_at,
|
20
|
+
:updated_at => @updated_at
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
def to_s
|
25
|
+
attributes.to_s
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
module SimpleMetrics
|
2
|
+
class InstrumentRepository
|
3
|
+
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def find_one(id)
|
7
|
+
instrument(collection.find_one(BSON::ObjectId.from_string(id)))
|
8
|
+
end
|
9
|
+
|
10
|
+
def find_one_by_name(name)
|
11
|
+
result = collection.find({ :name => name }).to_a.first
|
12
|
+
instrument(result) if result
|
13
|
+
end
|
14
|
+
|
15
|
+
def find_all
|
16
|
+
results = collection.find.sort([['name', ::Mongo::ASCENDING]]).to_a
|
17
|
+
instruments(results) if results
|
18
|
+
end
|
19
|
+
|
20
|
+
def save(instrument)
|
21
|
+
instrument.created_at = Time.now.utc
|
22
|
+
instrument.updated_at = Time.now.utc
|
23
|
+
attributes = instrument.attributes.reject { |key, value| key.to_s == "id" }
|
24
|
+
id = collection.insert(attributes)
|
25
|
+
instrument.id = id
|
26
|
+
id
|
27
|
+
end
|
28
|
+
|
29
|
+
def update(instrument)
|
30
|
+
collection.update({ "_id" => instrument.id }, "$set" => instrument.attributes.merge(:updated_at => Time.now.utc).reject { |k, v| k == 'id' })
|
31
|
+
end
|
32
|
+
|
33
|
+
def remove(id)
|
34
|
+
collection.remove("_id" => BSON::ObjectId.from_string(id))
|
35
|
+
end
|
36
|
+
|
37
|
+
def truncate_collections
|
38
|
+
collection.remove
|
39
|
+
end
|
40
|
+
|
41
|
+
def ensure_index
|
42
|
+
collection.ensure_index([['created_at', ::Mongo::ASCENDING]])
|
43
|
+
collection.ensure_index([['updated_at', ::Mongo::ASCENDING]])
|
44
|
+
collection.ensure_index([['name', ::Mongo::ASCENDING]])
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def collection
|
50
|
+
Repository.db.collection('instruments')
|
51
|
+
end
|
52
|
+
|
53
|
+
def instrument(result)
|
54
|
+
Instrument.new(:id => result["_id"], :name => result["name"], :metrics => result["metrics"], :created_at => result["created_at"], :updated_at => result["updated_at"])
|
55
|
+
end
|
56
|
+
|
57
|
+
def instruments(results)
|
58
|
+
results.inject([]) { |result, a| result << instrument(a); }
|
59
|
+
end
|
60
|
+
|
61
|
+
end # class << self
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module SimpleMetrics
|
2
|
+
class Metric
|
3
|
+
|
4
|
+
attr_reader :id, :name, :created_at, :updated_at
|
5
|
+
attr_accessor :total
|
6
|
+
|
7
|
+
def initialize(attributes)
|
8
|
+
@id = attributes[:id]
|
9
|
+
@name = attributes[:name]
|
10
|
+
@total = attributes[:total]
|
11
|
+
@created_at = attributes[:created_at]
|
12
|
+
@updated_at = attributes[:updated_at]
|
13
|
+
end
|
14
|
+
|
15
|
+
def attributes
|
16
|
+
{
|
17
|
+
:id => @id.to_s, # convert bson id to str
|
18
|
+
:name => @name,
|
19
|
+
:total => @total,
|
20
|
+
:created_at => @created_at,
|
21
|
+
:updated_at => @updated_at
|
22
|
+
}
|
23
|
+
end
|
24
|
+
|
25
|
+
def to_s
|
26
|
+
attributes.to_s
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module SimpleMetrics
|
2
|
+
class MetricRepository
|
3
|
+
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def find_one(id)
|
7
|
+
metric(collection.find_one(id))
|
8
|
+
end
|
9
|
+
|
10
|
+
def find_one_by_name(name)
|
11
|
+
result = collection.find({ :name => name }).to_a.first
|
12
|
+
metric(result) if result
|
13
|
+
end
|
14
|
+
|
15
|
+
def find_all
|
16
|
+
results = collection.find.sort([['name', ::Mongo::ASCENDING]]).to_a
|
17
|
+
metrics(results) if results
|
18
|
+
end
|
19
|
+
|
20
|
+
def save(metric)
|
21
|
+
collection.insert(metric.attributes.merge(:created_at => Time.now.utc, :updated_at => Time.now.utc))
|
22
|
+
end
|
23
|
+
|
24
|
+
def update(metric)
|
25
|
+
collection.update({ "_id" => metric.id }, { "$set" => { :total => metric.total, :updated_at => Time.now.utc }})
|
26
|
+
end
|
27
|
+
|
28
|
+
def truncate_collections
|
29
|
+
collection.remove
|
30
|
+
end
|
31
|
+
|
32
|
+
def ensure_index
|
33
|
+
collection.ensure_index([['created_at', ::Mongo::ASCENDING]])
|
34
|
+
collection.ensure_index([['updated_at', ::Mongo::ASCENDING]])
|
35
|
+
collection.ensure_index([['name', ::Mongo::ASCENDING]])
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def collection
|
41
|
+
Repository.db.collection('metrics')
|
42
|
+
end
|
43
|
+
|
44
|
+
def metric(result)
|
45
|
+
Metric.new(:id => result["_id"], :name => result["name"], :total => result["total"], :created_at => result["created_at"], :updated_at => result["updated_at"])
|
46
|
+
end
|
47
|
+
|
48
|
+
def metrics(results)
|
49
|
+
results.inject([]) { |result, a| result << metric(a); }
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module SimpleMetrics
|
2
|
+
module Repository
|
3
|
+
extend self
|
4
|
+
|
5
|
+
def connection
|
6
|
+
@@connection ||= ::Mongo::Connection.new(host, port)
|
7
|
+
end
|
8
|
+
|
9
|
+
def db
|
10
|
+
@@db ||= connection.db(db_name, config.fetch(:options))
|
11
|
+
end
|
12
|
+
|
13
|
+
def db_name
|
14
|
+
"simple_metrics_#{prefix}"
|
15
|
+
end
|
16
|
+
|
17
|
+
def prefix
|
18
|
+
config[:prefix] || 'development'
|
19
|
+
end
|
20
|
+
|
21
|
+
def host
|
22
|
+
config[:host] || 'localhost'
|
23
|
+
end
|
24
|
+
|
25
|
+
def port
|
26
|
+
config[:port] || 27017
|
27
|
+
end
|
28
|
+
|
29
|
+
def config
|
30
|
+
SimpleMetrics.config.db
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "eventmachine"
|
3
|
+
|
4
|
+
if defined? Encoding
|
5
|
+
Encoding.default_external = Encoding::UTF_8
|
6
|
+
end
|
7
|
+
|
8
|
+
module SimpleMetrics
|
9
|
+
|
10
|
+
module ClientHandler
|
11
|
+
|
12
|
+
@@data = []
|
13
|
+
|
14
|
+
class << self
|
15
|
+
def get_and_clear_data
|
16
|
+
data = @@data.dup
|
17
|
+
@@data = []
|
18
|
+
data
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def post_init
|
23
|
+
SimpleMetrics.logger.info "ClientHandler entering post_init"
|
24
|
+
end
|
25
|
+
|
26
|
+
def receive_data(data)
|
27
|
+
SimpleMetrics.logger.debug "received_data: #{data.inspect}"
|
28
|
+
@@data << data
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class UDPServer
|
33
|
+
|
34
|
+
def start
|
35
|
+
SimpleMetrics.logger.info "SERVER: starting up on #{host}:#{port}..."
|
36
|
+
|
37
|
+
trap('TERM') { stop }
|
38
|
+
trap('INT') { stop }
|
39
|
+
|
40
|
+
DataPointRepository.ensure_collections_exist
|
41
|
+
|
42
|
+
EM.run do
|
43
|
+
EM.open_datagram_socket(host, port, SimpleMetrics::ClientHandler) do |con|
|
44
|
+
EventMachine::add_periodic_timer(flush_interval) do
|
45
|
+
SimpleMetrics.logger.debug "SERVER: period timer triggered after #{flush_interval} seconds"
|
46
|
+
|
47
|
+
EM.defer { Bucket.flush_raw(ClientHandler.get_and_clear_data) }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def stop
|
54
|
+
SimpleMetrics.logger.info "EventMachine stop"
|
55
|
+
EM.stop
|
56
|
+
end
|
57
|
+
|
58
|
+
def to_s
|
59
|
+
"#{host}:#{port}"
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def host
|
65
|
+
config['host'] || 'localhost'
|
66
|
+
end
|
67
|
+
|
68
|
+
def port
|
69
|
+
config['port'] || 8125
|
70
|
+
end
|
71
|
+
|
72
|
+
def flush_interval
|
73
|
+
config['flush_interval'] || 10
|
74
|
+
end
|
75
|
+
|
76
|
+
def config
|
77
|
+
SimpleMetrics.config.server
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
end
|
data/lib/simple_metrics.rb
CHANGED
@@ -1,14 +1,28 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logger"
|
3
|
+
require "active_support/all"
|
3
4
|
|
4
5
|
require "simple_metrics/version"
|
5
|
-
require "simple_metrics/
|
6
|
-
require "simple_metrics/
|
6
|
+
require "simple_metrics/configuration"
|
7
|
+
require "simple_metrics/udp_server"
|
8
|
+
require "simple_metrics/repository"
|
9
|
+
require "simple_metrics/data_point_repository"
|
7
10
|
require "simple_metrics/data_point"
|
11
|
+
require "simple_metrics/data_point/base"
|
12
|
+
require "simple_metrics/data_point/counter"
|
13
|
+
require "simple_metrics/data_point/event"
|
14
|
+
require "simple_metrics/data_point/gauge"
|
15
|
+
require "simple_metrics/data_point/timing"
|
16
|
+
require "simple_metrics/importer"
|
8
17
|
require "simple_metrics/bucket"
|
9
18
|
require "simple_metrics/graph"
|
10
19
|
require "simple_metrics/functions"
|
11
|
-
require "simple_metrics/
|
20
|
+
require "simple_metrics/metric"
|
21
|
+
require "simple_metrics/metric_repository"
|
22
|
+
require "simple_metrics/instrument"
|
23
|
+
require "simple_metrics/instrument_repository"
|
24
|
+
require "simple_metrics/dashboard"
|
25
|
+
require "simple_metrics/dashboard_repository"
|
12
26
|
|
13
27
|
module SimpleMetrics
|
14
28
|
extend self
|
@@ -21,81 +35,12 @@ module SimpleMetrics
|
|
21
35
|
@@logger = logger
|
22
36
|
end
|
23
37
|
|
24
|
-
CONFIG_DEFAULTS = {
|
25
|
-
:host => 'localhost',
|
26
|
-
:port => 8125,
|
27
|
-
:flush_interval => 10
|
28
|
-
}.freeze
|
29
|
-
|
30
38
|
def config
|
31
|
-
@@config ||=
|
32
|
-
end
|
33
|
-
|
34
|
-
def config=(options)
|
35
|
-
@@config = CONFIG_DEFAULTS.merge(options)
|
36
|
-
end
|
37
|
-
|
38
|
-
BUCKETS_DEFAULTS = [
|
39
|
-
{
|
40
|
-
:name => 'stats_per_10s',
|
41
|
-
:seconds => 10,
|
42
|
-
:capped => true,
|
43
|
-
:size => 100_100_100
|
44
|
-
},
|
45
|
-
{
|
46
|
-
:name => 'stats_per_1min',
|
47
|
-
:seconds => 60,
|
48
|
-
:capped => true,
|
49
|
-
:size => 1_100_100_100
|
50
|
-
},
|
51
|
-
{
|
52
|
-
:name => 'stats_per_10min',
|
53
|
-
:seconds => 600,
|
54
|
-
:size => 0 ,
|
55
|
-
:capped => false
|
56
|
-
},
|
57
|
-
{
|
58
|
-
:name => 'stats_per_day',
|
59
|
-
:seconds => 86400,
|
60
|
-
:size => 0,
|
61
|
-
:capped => false
|
62
|
-
}
|
63
|
-
].freeze
|
64
|
-
|
65
|
-
def buckets_config
|
66
|
-
@@buckets ||= BUCKETS_DEFAULTS
|
67
|
-
end
|
68
|
-
|
69
|
-
def buckets_config=(buckets)
|
70
|
-
@@buckets = buckets
|
71
|
-
end
|
72
|
-
|
73
|
-
MONGODB_DEFAULTS = {
|
74
|
-
:pool_size => 5,
|
75
|
-
:timeout => 5,
|
76
|
-
:strict => true
|
77
|
-
}.freeze
|
78
|
-
|
79
|
-
DB_CONFIG_DEFAULTS = {
|
80
|
-
:host => 'localhost',
|
81
|
-
:port => 27017,
|
82
|
-
:prefix => 'development'
|
83
|
-
}.freeze
|
84
|
-
|
85
|
-
def db_config=(options)
|
86
|
-
@@db_config = {
|
87
|
-
:host => options.delete(:host) || 'localhost',
|
88
|
-
:port => options.delete(:port) || 27017,
|
89
|
-
:db_name => "simple_metrics_#{options.delete(:prefix)}",
|
90
|
-
:options => MONGODB_DEFAULTS.merge(options)
|
91
|
-
}
|
39
|
+
@@config ||= Configuration.new
|
92
40
|
end
|
93
41
|
|
94
|
-
def
|
95
|
-
|
96
|
-
:db_name => "simple_metrics_#{DB_CONFIG_DEFAULTS[:prefix]}",
|
97
|
-
:options => MONGODB_DEFAULTS
|
98
|
-
)
|
42
|
+
def configure(hash = {}, &block)
|
43
|
+
config.configure(hash, &block)
|
99
44
|
end
|
100
45
|
|
101
|
-
end
|
46
|
+
end
|
data/simple_metrics.gemspec
CHANGED
@@ -19,9 +19,13 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.add_development_dependency "rake"
|
20
20
|
s.add_development_dependency "rspec"
|
21
21
|
s.add_development_dependency "rr"
|
22
|
+
s.add_development_dependency "shotgun"
|
23
|
+
s.add_development_dependency "rack-test"
|
22
24
|
|
23
25
|
s.add_dependency "eventmachine"
|
24
26
|
s.add_dependency "daemons"
|
25
27
|
s.add_dependency "mongo", '~> 1.6'
|
26
28
|
s.add_dependency "bson", '~> 1.6'
|
29
|
+
s.add_dependency "bson_ext", '~> 1.6'
|
30
|
+
s.add_dependency "activesupport"
|
27
31
|
end
|