ganymed 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. data/Gemfile +1 -2
  2. data/README.md +6 -14
  3. data/ganymed.gemspec +4 -17
  4. data/lib/ganymed/collector.rb +6 -30
  5. data/lib/ganymed/collectors/cpu.rb +8 -8
  6. data/lib/ganymed/collectors/disk.rb +2 -2
  7. data/lib/ganymed/collectors/iostat.rb +2 -2
  8. data/lib/ganymed/collectors/load.rb +1 -1
  9. data/lib/ganymed/collectors/memory.rb +8 -8
  10. data/lib/ganymed/collectors/network.rb +6 -6
  11. data/lib/ganymed/collectors/process.rb +4 -4
  12. data/lib/ganymed/config.yml +0 -32
  13. data/lib/ganymed/master.rb +9 -8
  14. data/lib/ganymed/version.rb +1 -1
  15. metadata +8 -152
  16. data/contrib/cpuhog +0 -0
  17. data/contrib/cpuhog.c +0 -21
  18. data/lib/ganymed/collectors/uptime.rb +0 -10
  19. data/lib/ganymed/console.rb +0 -147
  20. data/lib/ganymed/event.rb +0 -97
  21. data/lib/ganymed/ext/array.rb +0 -38
  22. data/lib/ganymed/mongodb.rb +0 -44
  23. data/lib/ganymed/processor.rb +0 -137
  24. data/lib/ganymed/sampler/counter.rb +0 -19
  25. data/lib/ganymed/sampler/datasource.rb +0 -85
  26. data/lib/ganymed/sampler/derive.rb +0 -27
  27. data/lib/ganymed/sampler/gauge.rb +0 -20
  28. data/lib/ganymed/sampler.rb +0 -94
  29. data/lib/ganymed/websocket/authentication.rb +0 -37
  30. data/lib/ganymed/websocket/connection.rb +0 -71
  31. data/lib/ganymed/websocket/filter.rb +0 -23
  32. data/lib/ganymed/websocket/metadata.rb +0 -21
  33. data/lib/ganymed/websocket/query.rb +0 -30
  34. data/lib/ganymed/websocket/subscribe.rb +0 -45
  35. data/lib/ganymed/websocket.rb +0 -45
  36. data/spec/sampler/counter_spec.rb +0 -11
  37. data/spec/sampler/datasource_examples.rb +0 -49
  38. data/spec/sampler/datasource_spec.rb +0 -23
  39. data/spec/sampler/derive_spec.rb +0 -34
  40. data/spec/sampler/gauge_spec.rb +0 -35
  41. data/spec/sampler_spec.rb +0 -5
@@ -1,147 +0,0 @@
1
- require 'ascii_charts'
2
- require 'digest'
3
- require 'madvertise/ext/config'
4
- require 'ripl'
5
- require 'terminal-table'
6
-
7
- require 'ganymed'
8
- require 'ganymed/mongodb'
9
-
10
- module Ganymed
11
- class Console
12
- include Configuration::Helpers
13
-
14
- attr_accessor :db
15
-
16
- def initialize
17
- @cache = {}
18
-
19
- # load config file
20
- @default_config_file = File.join(LIB_DIR, 'ganymed/config.yml')
21
-
22
- # store fqdn for later use
23
- config.fqdn = ::Socket.gethostbyname(::Socket.gethostname).first
24
-
25
- # initialize db before loading ripl
26
- log.silence do
27
- @db = Ganymed::MongoDB.new(config.processor.mongodb)
28
- end
29
-
30
- Ripl.start(binding: binding)
31
- end
32
-
33
- def adduser(username, password)
34
- hashed = Digest::SHA256.hexdigest(password)
35
- db['users'].find_and_modify(
36
- query: {_id: username},
37
- update: {password: hashed},
38
- new: true
39
- )
40
- end
41
-
42
- def origins
43
- headings = ["Origin"]
44
- rows = db['metadata'].find.to_a.map do |row|
45
- [row['_id']]
46
- end
47
- puts Terminal::Table.new(rows: rows, headings: headings)
48
- end
49
-
50
- alias_method :o, :origins
51
-
52
- def list
53
- db.collection_names.reject do |col|
54
- %w(
55
- metadata
56
- system.indexes
57
- users
58
- ).include?(col)
59
- end.sort.each do |col|
60
- puts col
61
- end
62
- nil
63
- end
64
-
65
- alias_method :ls, :list
66
- alias_method :l, :list
67
-
68
- def drop
69
- db.connection.drop_database(config.processor.mongodb.database)
70
- end
71
-
72
- def method_missing(method, *args)
73
- pattern = /^#{method}\./
74
-
75
- namespaces = db.collection_names.select do |col|
76
- col =~ pattern
77
- end
78
-
79
- if namespaces.empty?
80
- super
81
- else
82
- @cache[method] ||= CollectionProxy.new(method, db, config)
83
- end
84
- end
85
-
86
- class CollectionProxy
87
- attr_reader :ns, :db, :config
88
-
89
- def initialize(ns, db, config)
90
- @ns, @db, @config = ns, db, config
91
- @cache = {}
92
- @consolidation = "avg"
93
-
94
- metaclass = class << self; self; end
95
- config.sampler.consolidations.each do |key, _|
96
- metaclass.send(:define_method, key.to_sym) do
97
- @consolidation = key
98
- self
99
- end
100
- end
101
- end
102
-
103
- def top(limit=10, selector={})
104
- selector.merge!({c: { :$in => [nil, @consolidation]}})
105
- opts = {limit: limit, sort: [:t, -1]}
106
- db[ns].find(selector, opts).to_a.map do |row|
107
- row.delete('_id')
108
- row.delete('c')
109
- row['t'] = Time.at(row['t'])
110
- row
111
- end
112
- end
113
-
114
- def to_s
115
- headings = ["Time", "Origin", "Value"]
116
- rows = top.map do |row|
117
- [row['t'], row['o'], row['v']]
118
- end
119
- table = Terminal::Table.new(rows: rows, headings: headings)
120
- "top-10 most recent events:\n" + table.to_s
121
- end
122
-
123
- def chart(limit=10, origin=nil)
124
- origin ||= config.fqdn
125
- values = top(limit, {o: origin}).reverse.map do |event|
126
- [event['t'].strftime("%H:%M"), event['v']]
127
- end
128
- puts AsciiCharts::Cartesian.new(values, bar: true).draw
129
- end
130
-
131
- def method_missing(method, *args)
132
- subns = "#{@ns}.#{method}"
133
- pattern = /^#{subns}(\.|$)/
134
-
135
- namespaces = db.collection_names.select do |col|
136
- col =~ pattern
137
- end
138
-
139
- if namespaces.empty?
140
- db[ns].send(method, *args)
141
- else
142
- @cache[method] ||= CollectionProxy.new(subns, db, config)
143
- end
144
- end
145
- end
146
- end
147
- end
data/lib/ganymed/event.rb DELETED
@@ -1,97 +0,0 @@
1
- module Ganymed
2
-
3
- ##
4
- # An incoming event is a hash-like object and contains the following keys:
5
- #
6
- # n:: namespace (e.g. +os.cpu.idle+, +nginx.requests+, etc)
7
- # c:: a consolidation function used to build this event from multiple
8
- # samples (_optional_)
9
- # o:: origin (e.g. +web1.example.com+, +myapplication+)
10
- # t:: timestamp as seconds since epoch
11
- # r:: resolution of this event (i.e. how often is this event sent to the processor, _optional_)
12
- # v:: the event value (any object that can be serialized with BSON and JSON)
13
- #
14
- class Event
15
- # The namespace of this event.
16
- # @return [String]
17
- attr_accessor :ns
18
-
19
- # The consolidation function used to build this event from multiple
20
- # samples if any.
21
- # @return [String]
22
- attr_accessor :cf
23
-
24
- # The origin of this event.
25
- # @return [String]
26
- attr_accessor :origin
27
-
28
- # The timestamp of this event.
29
- # @return [Fixnum]
30
- attr_accessor :timestamp
31
-
32
- # The event resolution
33
- # @return [Fixnum]
34
- #
35
- # @note +nil+ or +0+ will bypass range checks for MongoDB, refer to the
36
- # {file:doc/configuration.md configuration} for details.
37
- attr_accessor :resolution
38
-
39
- # The actual value of this sample.
40
- # @return [Object]
41
- attr_accessor :value
42
-
43
- # Create a new {Event} instance from +obj+.
44
- #
45
- # @return [Event] The parsed event instance.
46
- def self.parse(obj)
47
- event = new
48
- event.ns = obj['n']
49
- event.cf = obj['c']
50
- event.origin = obj['o']
51
- event.timestamp = obj['t'].to_i
52
- event.resolution = obj['r'].to_i
53
-
54
- value = obj['v']
55
- if value.is_a?(Float) and (value.nan? or value.infinite?)
56
- event.value = nil
57
- else
58
- event.value = value
59
- end
60
-
61
- event.validate
62
- event
63
- end
64
-
65
- # Convert this event into a hash with long keys. This hash cannot be parsed
66
- # by #parse which accepts short keys for effiency.
67
- #
68
- # @return [Hash]
69
- def to_h
70
- {
71
- :ns => ns,
72
- :cf => cf,
73
- :origin => origin,
74
- :timestamp => timestamp,
75
- :resolution => resolution,
76
- :value => value
77
- }
78
- end
79
-
80
- # Validate this event.
81
- #
82
- # @return [void]
83
- def validate
84
- unless @ns =~ /^[a-z][a-z0-9_\.-]+[a-z0-9]$/i
85
- raise "invalid namespace: #{@ns.inspect}"
86
- end
87
-
88
- if @ns =~ /^(system\.|metadata)/
89
- raise "namespace is reserved: #{@ns.inspect}"
90
- end
91
-
92
- if @timestamp < 0
93
- raise "invalid timestamp: #{@timestamp.inspect}"
94
- end
95
- end
96
- end
97
- end
@@ -1,38 +0,0 @@
1
- require 'inline'
2
-
3
- class Array
4
- # this is incredibly faster than doing ary.inject(0, :+)
5
- # should only be called on numbers-only arrays, though ;-)
6
- inline :C do |builder|
7
- builder.c_raw <<-EOC
8
- static VALUE sum(int argc, VALUE *argv, VALUE self) {
9
- double result = 0;
10
- VALUE *arr = RARRAY_PTR(self);
11
- long i, len = RARRAY_LEN(self);
12
-
13
- for (i = 0; i < len; i++) {
14
- result += NUM2DBL(arr[i]);
15
- }
16
-
17
- return rb_float_new(result);
18
- }
19
- EOC
20
- end
21
-
22
- def mean
23
- sum.to_f / length
24
- end
25
-
26
- def variance
27
- m = mean
28
- reduce(0) {|accum, item| accum + (item - m) ** 2}.to_f / (length - 1)
29
- end
30
-
31
- def stdev
32
- Math.sqrt(variance)
33
- end
34
-
35
- def percentile(pc)
36
- sort[(pc * length).ceil - 1]
37
- end
38
- end
@@ -1,44 +0,0 @@
1
- require 'mongo'
2
-
3
- module Ganymed
4
- # @private
5
- class MongoDB
6
- attr_reader :config
7
-
8
- def initialize(config)
9
- @config = config
10
- @collections = {}
11
- log.info("using MongoDB at #{config.host}:#{config.port}/#{config.database}")
12
- end
13
-
14
- def connection
15
- @connection ||= ::Mongo::Connection.new(config.host, config.port,
16
- :pool_size => config.pool_size,
17
- :pool_timeout => config.pool_timeout)
18
- end
19
-
20
- def db
21
- @db ||= connection.db(config.database)
22
- end
23
-
24
- def collection(ns)
25
- return @collections[ns] if @collections.has_key?(ns)
26
- col = @collections[ns] = db.collection(ns)
27
- col.ensure_index([['c', ::Mongo::ASCENDING]])
28
- col.ensure_index([['o', ::Mongo::ASCENDING]])
29
- col.ensure_index([['t', ::Mongo::ASCENDING]])
30
- col.ensure_index([['c', ::Mongo::ASCENDING], ['o', ::Mongo::ASCENDING]])
31
- col.ensure_index([['o', ::Mongo::ASCENDING], ['t', ::Mongo::ASCENDING]])
32
- col.ensure_index([['c', ::Mongo::ASCENDING], ['o', ::Mongo::ASCENDING], ['t', ::Mongo::ASCENDING]], :unique => true)
33
- col
34
- end
35
-
36
- def [](*args)
37
- db.collection(*args)
38
- end
39
-
40
- def method_missing(method, *args)
41
- db.send(method, *args)
42
- end
43
- end
44
- end
@@ -1,137 +0,0 @@
1
- require 'eventmachine'
2
- require 'msgpack'
3
-
4
- require 'ganymed/client'
5
- require 'ganymed/event'
6
- require 'ganymed/mongodb'
7
- require 'ganymed/websocket'
8
-
9
- module Ganymed
10
-
11
- ##
12
- # The Processor accepts {Event events} from a UDP socket and stores these
13
- # metrics into MongoDB. All metrics are also published to interested parties
14
- # via WebSocket.
15
- #
16
- class Processor
17
- def initialize(config)
18
- @config = config
19
- @db = MongoDB.new(config.processor.mongodb)
20
- @websocket = Websocket.new(config.processor.websocket, @db)
21
-
22
- @config.client.sampler.tap do |sampler|
23
- # schedule connect on next tick so that our sampler socket is
24
- # available. Processor#initialize is called from EM.run and therefore
25
- # no sockets have been opened yet.
26
- EM.next_tick { @sampler = Ganymed::Client::Sampler.connect(sampler.host, sampler.port) }
27
- end
28
-
29
- cache!
30
- listen!
31
- end
32
-
33
- def cache!
34
- @metadata = Hash.new do |hsh, key|
35
- hsh[key] = []
36
- end
37
-
38
- @db['metadata'].find({}, fields: [:_id, :namespaces]).each do |result|
39
- @metadata[result['_id']] = result['namespaces']
40
- end
41
- end
42
-
43
- def listen!
44
- @config.processor.listen.tap do |listen|
45
- log.info("processing metrics on udp##{listen.host}:#{listen.port}")
46
- EM.open_datagram_socket(listen.host, listen.port, Connection) do |connection|
47
- connection.processor = self
48
- end
49
-
50
- log.info("processing metrics on tcp##{listen.host}:#{listen.port}")
51
- EM.start_server(listen.host, listen.port, Connection) do |connection|
52
- connection.processor = self
53
- end
54
- end
55
- end
56
-
57
- def process(data)
58
- case data['_type'].to_sym
59
- when :event
60
- process_event(Event.parse(data))
61
- when :metadata
62
- process_metadata(data)
63
- end
64
- rescue Exception => exc
65
- log.exception(exc)
66
- end
67
-
68
- def process_event(event)
69
- @sampler.emit(:counter, 'ganymed.processor.events', 1) rescue nil
70
-
71
- if not @metadata[event.origin].include?(event.ns)
72
- # insert origin + namespace into metadata
73
- metadata = @db['metadata'].find_and_modify({
74
- query: {:_id => event.origin},
75
- update: {:$addToSet => {:namespaces => event.ns}},
76
- upsert: true,
77
- })
78
-
79
- # set defaults in case find_and_modify returned an empty document
80
- metadata['_id'] ||= event.origin
81
- metadata['namespaces'] ||= [event.ns]
82
-
83
- # notify websocket clients
84
- @websocket.send(:metadata, [metadata])
85
- end
86
-
87
- # normalize timestamp for events with a resolution
88
- if event.resolution > 1
89
- event.timestamp -= event.timestamp % event.resolution
90
- end
91
-
92
- # insert event into ns collection
93
- @db.collection(event.ns).update({
94
- :c => event.cf,
95
- :o => event.origin,
96
- :t => event.timestamp
97
- }, {
98
- :$set => {:v => event.value}
99
- }, :upsert => true)
100
-
101
- # send the event to websocket clients
102
- @websocket.each do |connection|
103
- connection.publish(event)
104
- end
105
- end
106
-
107
- def process_metadata(data)
108
- metadata = @db['metadata'].find_and_modify({
109
- query: {:_id => data['origin']},
110
- update: {:$set => {:data => data['data']}},
111
- upsert: true,
112
- new: true
113
- })
114
-
115
- @websocket.send(:metadata, [metadata])
116
- end
117
-
118
- # @private
119
- class Connection < EM::Connection
120
- attr_accessor :processor
121
-
122
- def initialize
123
- @pac = MessagePack::Unpacker.new
124
- end
125
-
126
- def receive_data(data)
127
- @pac.feed(data)
128
- @pac.each do |obj|
129
- EM.defer { processor.process(obj) }
130
- end
131
- rescue Exception => exc
132
- log.exception(exc)
133
- close_connection
134
- end
135
- end
136
- end
137
- end
@@ -1,19 +0,0 @@
1
- require 'ganymed/ext/array'
2
- require 'ganymed/sampler/datasource'
3
-
4
- module Ganymed
5
- class Sampler
6
-
7
- ##
8
- # A Counter {DataSource} processes samples counting a specific event (such
9
- # as a webserver request, user login, etc) and produces a rate/s gauge.
10
- #
11
- class Counter < DataSource
12
- def flush
13
- each do |ns, origin, ts|
14
- yield ns, origin, ts.values.map(&:sum)
15
- end
16
- end
17
- end
18
- end
19
- end
@@ -1,85 +0,0 @@
1
- require 'inline'
2
-
3
- module Ganymed
4
- class Sampler
5
-
6
- ##
7
- # A DataSource processes samples from various sources. Samples are stored
8
- # to an in-memory buffer using a sliding-window algorithm. Values are kept
9
- # for a configurable time and consolidated values can be flushed to the
10
- # processor at any time.
11
- #
12
- # All DataSources interpolate their samples to gauge style values before
13
- # flushing to the processor, e.g. a counter (like requests, or bytes sent
14
- # over the network) is interpolated to a rate/s gauge (e.g. requests/s or
15
- # bytes/s).
16
- #
17
- class DataSource
18
-
19
- attr_reader :buffer, :window
20
-
21
- # Create a new DataSource.
22
- #
23
- # @param [Fixnum] window Sliding window size.
24
- def initialize(window)
25
- @window = window
26
- @buffer = Hash.new do |buffer, origin|
27
- buffer[origin] = Hash.new do |origin, ns|
28
- origin[ns] = Hash.new do |ns, timestamp|
29
- ns[timestamp] = []
30
- end
31
- end
32
- end
33
- end
34
-
35
- # Feed data from the source.
36
- #
37
- # @param [String] ns The namespace for this value.
38
- # @param [String] origin The origin of this value.
39
- # @param [Fixnum,Float] value The actual value.
40
- # @return [void]
41
- def feed(ns, origin, value)
42
- @buffer[origin][ns][now] << value
43
- end
44
-
45
- # Flush and consolidate the buffer.
46
- #
47
- # @yield [ns, origin, values] Run the block once for every
48
- # namespace/origin, passing in all the
49
- # consolidated values.
50
- # @return [void]
51
- def flush
52
- raise NotImplementedError
53
- end
54
-
55
- # Yield values for every origin/ns combination and remove old elements
56
- # from the buffer (sliding window). This is used internally to iterate
57
- # through the values in the current window.
58
- #
59
- # @yield [ns, origin, ts] Run the block once for every namespace
60
- # passing in all the collected samples grouped
61
- # by timestamp.
62
- def each(&block)
63
- oldest = now - window
64
- @buffer.each do |origin, ns|
65
- ns.each do |ns, ts|
66
- ts.delete_if { |ts, _| ts < oldest }
67
- yield ns, origin, ts
68
- end
69
- end
70
- end
71
-
72
- # Time.now.utc.to_i is way too expensive
73
- inline :C do |builder|
74
- builder.include "<sys/time.h>"
75
- builder.c <<-EOC
76
- long now(void) {
77
- struct timeval tv;
78
- gettimeofday(&tv, NULL);
79
- return tv.tv_sec;
80
- }
81
- EOC
82
- end
83
- end
84
- end
85
- end
@@ -1,27 +0,0 @@
1
- require 'ganymed/sampler/datasource'
2
-
3
- module Ganymed
4
- class Sampler
5
-
6
- ##
7
- # A Derive {DataSource data source} will store the derivative of the line
8
- # going from the last to the current value of the data source. Derive is
9
- # similar to a {Counter} {DataSource data source} in that it produces a
10
- # rate/s gauge, but only accepts absolute values and produces the
11
- # derivative instead of summation of all samples.
12
- #
13
- class Derive < DataSource
14
- def flush
15
- each do |ns, origin, ts|
16
- values = ts.each_cons(2).map do |a, b|
17
- (b[1].last - a[1].last)/(b[0] - a[0]) # ∆value / ∆ts
18
- end.reject do |value|
19
- value < 0 # ignore overflow/counter wrap
20
- end
21
-
22
- yield ns, origin, values
23
- end
24
- end
25
- end
26
- end
27
- end
@@ -1,20 +0,0 @@
1
- require 'ganymed/sampler/datasource'
2
-
3
- module Ganymed
4
- class Sampler
5
-
6
- ##
7
- # A Gauge is the simplest DataSource type. It simply records the given
8
- # value in the buffer and emits all values in the buffer upon flush
9
- # assuming the given values are in gauge-style (e.g. free memory, users
10
- # currently logged in, etc).
11
- #
12
- class Gauge < DataSource
13
- def flush
14
- each do |ns, origin, ts|
15
- yield ns, origin, ts.values.flatten
16
- end
17
- end
18
- end
19
- end
20
- end