fnordmetric 0.3.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +6 -0
- data/Gemfile.lock +21 -0
- data/Procfile +1 -2
- data/VERSION +1 -1
- data/_spec/app_spec.rb +178 -0
- data/{spec → _spec}/cache_spec.rb +0 -0
- data/{spec → _spec}/combine_metric_spec.rb +0 -0
- data/{spec → _spec}/core_spec.rb +0 -0
- data/{spec → _spec}/count_metric_spec.rb +0 -0
- data/_spec/dashboard_spec.rb +67 -0
- data/_spec/event_spec.rb +46 -0
- data/{spec → _spec}/metric_spec.rb +0 -0
- data/{spec → _spec}/report_spec.rb +0 -0
- data/{spec → _spec}/sum_metric_spec.rb +0 -0
- data/_spec/widget_spec.rb +107 -0
- data/doc/import_dump.rb +26 -0
- data/em_runner.rb +33 -0
- data/fnordmetric.gemspec +59 -20
- data/haml/app.haml +26 -12
- data/lib/fnordmetric.rb +150 -15
- data/lib/fnordmetric/app.rb +70 -11
- data/lib/fnordmetric/cache.rb +4 -4
- data/lib/fnordmetric/context.rb +65 -0
- data/lib/fnordmetric/dashboard.rb +16 -12
- data/lib/fnordmetric/event.rb +65 -15
- data/lib/fnordmetric/gauge.rb +46 -0
- data/lib/fnordmetric/gauge_calculations.rb +43 -0
- data/lib/fnordmetric/gauge_modifiers.rb +43 -0
- data/lib/fnordmetric/inbound_stream.rb +66 -0
- data/lib/fnordmetric/logger.rb +38 -0
- data/lib/fnordmetric/namespace.rb +120 -0
- data/lib/fnordmetric/numbers_widget.rb +29 -11
- data/lib/fnordmetric/session.rb +131 -0
- data/lib/fnordmetric/standalone.rb +31 -0
- data/lib/fnordmetric/timeline_widget.rb +29 -9
- data/lib/fnordmetric/widget.rb +50 -45
- data/lib/fnordmetric/worker.rb +80 -0
- data/pub/fnordmetric/fnordmetric.css +76 -9
- data/pub/fnordmetric/fnordmetric.js +541 -42
- data/pub/raphael-min.js +8 -0
- data/pub/raphael-utils.js +221 -0
- data/readme.rdoc +172 -27
- data/server.rb +22 -0
- data/spec/app_spec.rb +359 -117
- data/spec/context_spec.rb +42 -0
- data/spec/dashboard_spec.rb +7 -47
- data/spec/event_spec.rb +114 -33
- data/spec/gauge_modifiers_spec.rb +276 -0
- data/spec/gauge_spec.rb +128 -0
- data/spec/namespace_spec.rb +104 -0
- data/spec/session_spec.rb +231 -0
- data/spec/spec_helper.rb +27 -4
- data/spec/widget_spec.rb +81 -75
- data/spec/worker_spec.rb +37 -0
- data/test_stream.sh +187 -0
- data/ulm_stats.rb +198 -0
- metadata +114 -35
- data/lib/fnordmetric/core.rb +0 -66
- data/lib/fnordmetric/engine.rb +0 -3
@@ -0,0 +1,66 @@
|
|
1
|
+
class FnordMetric::InboundStream < EventMachine::Connection
|
2
|
+
|
3
|
+
@@opts = nil
|
4
|
+
|
5
|
+
def self.start(opts)
|
6
|
+
@@opts = opts
|
7
|
+
EM.start_server(*opts[:inbound_stream], self)
|
8
|
+
end
|
9
|
+
|
10
|
+
def receive_data(chunk)
|
11
|
+
@buffer << chunk
|
12
|
+
EM.defer{ next_event }
|
13
|
+
end
|
14
|
+
|
15
|
+
def push_event(event_id, event_data)
|
16
|
+
prefix = @@opts[:redis_prefix]
|
17
|
+
|
18
|
+
@redis.hincrby "#{prefix}-stats", "events_received", 1
|
19
|
+
@redis.set "#{prefix}-event-#{event_id}", event_data
|
20
|
+
@redis.lpush "#{prefix}-queue", event_id
|
21
|
+
@redis.expire "#{prefix}-event-#{event_id}", @@opts[:event_queue_ttl]
|
22
|
+
|
23
|
+
@events_buffered -= 1
|
24
|
+
close_connection?
|
25
|
+
end
|
26
|
+
|
27
|
+
def next_event
|
28
|
+
read_next_event
|
29
|
+
push_next_event
|
30
|
+
end
|
31
|
+
|
32
|
+
def read_next_event
|
33
|
+
while (event = @buffer.slice!(/^(.*)\n/))
|
34
|
+
@events_buffered += 1
|
35
|
+
@events << event
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def push_next_event
|
40
|
+
return true if @events.empty?
|
41
|
+
push_event(get_next_uuid, @events.pop)
|
42
|
+
EM.next_tick(&method(:push_next_event))
|
43
|
+
end
|
44
|
+
|
45
|
+
def get_next_uuid
|
46
|
+
rand(9999999999999999999).to_s # FIXME
|
47
|
+
end
|
48
|
+
|
49
|
+
def close_connection?
|
50
|
+
@redis.quit unless @streaming || (@events_buffered!=0)
|
51
|
+
end
|
52
|
+
|
53
|
+
def post_init
|
54
|
+
@redis = Redis.new
|
55
|
+
@events_buffered = 0
|
56
|
+
@streaming = true
|
57
|
+
@buffer = ""
|
58
|
+
@events = []
|
59
|
+
end
|
60
|
+
|
61
|
+
def unbind
|
62
|
+
@streaming = false
|
63
|
+
close_connection?
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
class FnordMetric::Logger
|
2
|
+
|
3
|
+
def self.start(logfile_path)
|
4
|
+
require 'json'
|
5
|
+
event_ids = Queue.new
|
6
|
+
dump_file = File.open(logfile_path, 'a+')
|
7
|
+
|
8
|
+
fetcher = Thread.new do
|
9
|
+
redis = Redis.new
|
10
|
+
loop do
|
11
|
+
event_id = event_ids.pop
|
12
|
+
event_data = redis.get("fnordmetric-event-#{event_id}")
|
13
|
+
event_hash = JSON.parse(event_data) rescue next
|
14
|
+
|
15
|
+
event_hash.merge!(:_time => Time.now.to_i)
|
16
|
+
|
17
|
+
dump_file.write(event_hash.to_json+"\n")
|
18
|
+
dump_file.flush
|
19
|
+
|
20
|
+
print "\033[1;34m"
|
21
|
+
print event_hash.inspect
|
22
|
+
print "\033[0m\n"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
listener = Thread.new do
|
27
|
+
redis = Redis.new
|
28
|
+
redis.subscribe("fnordmetric-announce") do |on|
|
29
|
+
on.message do |channel, event_id|
|
30
|
+
event_ids << event_id
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
fetcher.join
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
class FnordMetric::Namespace
|
2
|
+
|
3
|
+
attr_reader :handlers, :gauges, :opts, :key, :dashboards
|
4
|
+
|
5
|
+
@@opts = [:event, :gauge, :widget]
|
6
|
+
|
7
|
+
def initialize(key, opts)
|
8
|
+
@gauges = Hash.new
|
9
|
+
@dashboards = Hash.new
|
10
|
+
@handlers = Hash.new
|
11
|
+
@opts = opts
|
12
|
+
@key = key
|
13
|
+
end
|
14
|
+
|
15
|
+
def ready!(redis)
|
16
|
+
@redis = redis
|
17
|
+
@gauges.map{ |k,g| g.add_redis(@redis) }
|
18
|
+
self
|
19
|
+
end
|
20
|
+
|
21
|
+
def announce(event)
|
22
|
+
announce_to_timeline(event)
|
23
|
+
announce_to_typelist(event)
|
24
|
+
|
25
|
+
if event[:_session]
|
26
|
+
event[:_session_key] = announce_to_session(event).session_key
|
27
|
+
end
|
28
|
+
|
29
|
+
[
|
30
|
+
@handlers[event[:_type].to_s],
|
31
|
+
@handlers["*"]
|
32
|
+
].flatten.compact.each do |context|
|
33
|
+
context.call(event, @redis)
|
34
|
+
end
|
35
|
+
|
36
|
+
self
|
37
|
+
end
|
38
|
+
|
39
|
+
def announce_to_session(event)
|
40
|
+
FnordMetric::Session.create(@opts.clone.merge(
|
41
|
+
:namespace_key => @key,
|
42
|
+
:namespace_prefix => key_prefix,
|
43
|
+
:redis => @redis,
|
44
|
+
:event => event
|
45
|
+
))
|
46
|
+
end
|
47
|
+
|
48
|
+
def announce_to_timeline(event)
|
49
|
+
timeline_key = key_prefix(:timeline)
|
50
|
+
@redis.zadd(timeline_key, event[:_time], event[:_eid])
|
51
|
+
end
|
52
|
+
|
53
|
+
def announce_to_typelist(event)
|
54
|
+
typelist_key = key_prefix("type-#{event[:_type]}")
|
55
|
+
@redis.lpush(typelist_key, event[:_eid])
|
56
|
+
end
|
57
|
+
|
58
|
+
|
59
|
+
def key_prefix(append=nil)
|
60
|
+
[@opts[:redis_prefix], @key, append].compact.join("-")
|
61
|
+
end
|
62
|
+
|
63
|
+
def token
|
64
|
+
@key
|
65
|
+
end
|
66
|
+
|
67
|
+
def dashboards(name=nil)
|
68
|
+
return @dashboards unless name
|
69
|
+
@dashboards[name] ||= FnordMetric::Dashboard.new(
|
70
|
+
:title => name
|
71
|
+
)
|
72
|
+
end
|
73
|
+
|
74
|
+
def sessions(_ids, opts={})
|
75
|
+
return FnordMetric::Session.all(extend_opts(opts)) if _ids == :all
|
76
|
+
end
|
77
|
+
|
78
|
+
def events(_ids, opts={})
|
79
|
+
return FnordMetric::Event.all(extend_opts(opts)) if _ids == :all
|
80
|
+
return FnordMetric::Event.by_type(opts.delete(:type), extend_opts(opts)) if _ids == :by_type
|
81
|
+
end
|
82
|
+
|
83
|
+
def method_missing(m, *args, &block)
|
84
|
+
raise "unknown option: #{m}" unless @@opts.include?(m)
|
85
|
+
send(:"opt_#{m}", *args, &block)
|
86
|
+
end
|
87
|
+
|
88
|
+
def opt_event(event_type, opts={}, &block)
|
89
|
+
opts.merge!(:redis => @redis, :gauges => @gauges)
|
90
|
+
FnordMetric::Context.new(opts, block).tap do |context|
|
91
|
+
@handlers[event_type.to_s] ||= []
|
92
|
+
@handlers[event_type.to_s] << context
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def opt_gauge(gauge_key, opts={})
|
97
|
+
opts.merge!(:key => gauge_key, :key_prefix => key_prefix)
|
98
|
+
@gauges[gauge_key] ||= FnordMetric::Gauge.new(opts)
|
99
|
+
end
|
100
|
+
|
101
|
+
def opt_widget(dashboard, widget)
|
102
|
+
widget = build_widget(widget) if widget.is_a?(Hash)
|
103
|
+
dashboards(dashboard).add_widget(widget)
|
104
|
+
end
|
105
|
+
|
106
|
+
def build_widget(opts)
|
107
|
+
_gauges = [opts[:gauges]].flatten.map{ |g| @gauges.fetch(g) }
|
108
|
+
widget_klass = "FnordMetric::#{opts.fetch(:type).to_s.capitalize}Widget"
|
109
|
+
widget_klass.constantize.new(opts.merge(:gauges => _gauges))
|
110
|
+
end
|
111
|
+
|
112
|
+
def extend_opts(opts)
|
113
|
+
opts.merge(
|
114
|
+
:namespace_prefix => key_prefix,
|
115
|
+
:redis_prefix => @opts[:redis_prefix],
|
116
|
+
:redis => @redis
|
117
|
+
)
|
118
|
+
end
|
119
|
+
|
120
|
+
end
|
@@ -1,18 +1,36 @@
|
|
1
|
-
class FnordMetric::NumbersWidget
|
1
|
+
class FnordMetric::NumbersWidget < FnordMetric::Widget
|
2
2
|
|
3
3
|
def data
|
4
4
|
super.merge(
|
5
|
-
:
|
6
|
-
:
|
7
|
-
:metrics => metrics.map{ |m| m.token },
|
8
|
-
:intervals => {
|
9
|
-
"Current" => "",
|
10
|
-
"Today" => "at=#{1.day.ago.to_i}-#{Time.now.to_i}",
|
11
|
-
"Month" => "at=#{30.days.ago.to_i}-#{Time.now.to_i}"
|
12
|
-
},
|
13
|
-
:widget_url => "/fnordmetric/widget_numbers.js?#{(rand*999).to_i}",
|
14
|
-
:widget_height => 100
|
5
|
+
:offsets => offsets,
|
6
|
+
:gauges => data_gauges
|
15
7
|
)
|
16
8
|
end
|
17
9
|
|
10
|
+
def data_gauges
|
11
|
+
Hash.new.tap do |hash|
|
12
|
+
gauges.each do |g|
|
13
|
+
hash[g.name] = {
|
14
|
+
:values => data_gauge(g),
|
15
|
+
:title => g.name
|
16
|
+
}
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def data_gauge(gauge)
|
22
|
+
offsets.map do |offset|
|
23
|
+
offset_time = Time.now.to_i - offset*gauge.tick
|
24
|
+
[gauge.tick_at(offset_time), gauge.value_at(offset_time)]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def has_tick?
|
29
|
+
false
|
30
|
+
end
|
31
|
+
|
32
|
+
def offsets
|
33
|
+
[0, 1, 30]
|
34
|
+
end
|
35
|
+
|
18
36
|
end
|
@@ -0,0 +1,131 @@
|
|
1
|
+
class FnordMetric::Session
|
2
|
+
|
3
|
+
attr_accessor :updated_at, :name, :picture
|
4
|
+
|
5
|
+
@@meta_attributes = %w(name picture)
|
6
|
+
|
7
|
+
def self.create(opts)
|
8
|
+
redis = opts.fetch(:redis)
|
9
|
+
event = opts[:event]
|
10
|
+
|
11
|
+
hash = Digest::MD5.hexdigest(event[:_session])
|
12
|
+
set_key = "#{opts[:namespace_prefix]}-session"
|
13
|
+
|
14
|
+
self.new(hash).tap do |session|
|
15
|
+
session.add_redis(redis, set_key)
|
16
|
+
session.add_event(event)
|
17
|
+
session.expire(opts[:session_data_ttl])
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.find(session_key, opts)
|
22
|
+
set_key = "#{opts[:namespace_prefix]}-session"
|
23
|
+
self.new(session_key, [opts[:redis], set_key])
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.all(opts)
|
27
|
+
set_key = "#{opts[:namespace_prefix]}-session"
|
28
|
+
limit = (opts[:limit].try(:to_i)||0)-1
|
29
|
+
session_ids = opts[:redis].zrevrange(set_key, 0, limit, :withscores => true)
|
30
|
+
session_ids.in_groups_of(2).map do |session_key, ts|
|
31
|
+
find(session_key, opts).tap{ |s| s.updated_at = ts }
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize(session_key, redis_opts=nil)
|
36
|
+
@session_key = session_key
|
37
|
+
add_redis(*redis_opts) if redis_opts
|
38
|
+
end
|
39
|
+
|
40
|
+
def session_key
|
41
|
+
@session_key
|
42
|
+
end
|
43
|
+
|
44
|
+
def picture
|
45
|
+
@picture
|
46
|
+
end
|
47
|
+
|
48
|
+
def name
|
49
|
+
@name
|
50
|
+
end
|
51
|
+
|
52
|
+
def data(key=nil)
|
53
|
+
key ? @data[key] : @data
|
54
|
+
end
|
55
|
+
|
56
|
+
def event_ids
|
57
|
+
@event_ids || []
|
58
|
+
end
|
59
|
+
|
60
|
+
def events
|
61
|
+
[]
|
62
|
+
end
|
63
|
+
|
64
|
+
def to_json
|
65
|
+
{ :session_key => session_key }.tap do |hash|
|
66
|
+
hash.merge!(:_picture => @picture) if @picture
|
67
|
+
hash.merge!(:_name => @name) if @name
|
68
|
+
hash.merge!(:_updated_at => @updated_at) if @updated_at
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def redis_key(append=nil)
|
73
|
+
[@redis_prefix, @session_key, append].compact.join("-")
|
74
|
+
end
|
75
|
+
|
76
|
+
def add_redis(redis, prefix)
|
77
|
+
@redis_prefix = prefix
|
78
|
+
@redis = redis
|
79
|
+
end
|
80
|
+
|
81
|
+
def touch(time=Time.now.to_i)
|
82
|
+
@redis.zadd(@redis_prefix, time, @session_key)
|
83
|
+
end
|
84
|
+
|
85
|
+
def expire(time)
|
86
|
+
@redis.expire(redis_key(:events), time)
|
87
|
+
@redis.expire(redis_key(:data), time)
|
88
|
+
end
|
89
|
+
|
90
|
+
def add_event(event)
|
91
|
+
@redis.zadd(redis_key(:events), event[:_time], event[:_eid])
|
92
|
+
add_data(:_picture, event[:url]) if event[:_type] == "_set_picture"
|
93
|
+
add_data(:_name, event[:name]) if event[:_type] == "_set_name"
|
94
|
+
add_event_data(event) if event[:_type] == "_set_data"
|
95
|
+
touch(event[:_time])
|
96
|
+
end
|
97
|
+
|
98
|
+
def add_event_data(event)
|
99
|
+
event.each do |key,value|
|
100
|
+
add_data(key, value) unless key[0]=="_"
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def add_data(key, value)
|
105
|
+
@redis.hset(redis_key(:data), key, value)
|
106
|
+
end
|
107
|
+
|
108
|
+
def fetch_data!
|
109
|
+
@data = Hash.new
|
110
|
+
@redis.hgetall(redis_key(:data)).each do |key, value|
|
111
|
+
if key[0]=="_"
|
112
|
+
fetch_meta_key(key, value)
|
113
|
+
else
|
114
|
+
@data[key.intern] = value
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def fetch_meta_key(key, value)
|
120
|
+
meta_key = key[1..-1]
|
121
|
+
if @@meta_attributes.include?(meta_key)
|
122
|
+
instance_variable_set(:"@#{meta_key}", value)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def fetch_event_ids!(since=-1)
|
127
|
+
# FIXME: use WITHSCORE to get the timestamps and return event objects
|
128
|
+
@event_ids = @redis.zrange(redis_key(:events), 0, since)
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'rake'
|
2
|
+
require 'redis'
|
3
|
+
|
4
|
+
task :run do
|
5
|
+
FnordMetric.run
|
6
|
+
end
|
7
|
+
|
8
|
+
task :log do
|
9
|
+
FnordMetric::Logger.start(dump_file_path)
|
10
|
+
end
|
11
|
+
|
12
|
+
task :import do
|
13
|
+
puts 'not yet implemented :('
|
14
|
+
end
|
15
|
+
|
16
|
+
task :help do
|
17
|
+
puts "usage: #{$0} {run|log|import} [DUMP_FILE=fm_dump.json]"
|
18
|
+
end
|
19
|
+
|
20
|
+
task :default => :help
|
21
|
+
|
22
|
+
def dump_file_path
|
23
|
+
if ENV["DUMP_FILE"].blank?
|
24
|
+
Rake::Task[:help].execute; exit!
|
25
|
+
else
|
26
|
+
::File.expand_path(ENV["DUMP_FILE"], ::File.dirname($0))
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
Rake.application.init('fnordmetric')
|
31
|
+
Rake.application.top_level
|
@@ -1,17 +1,37 @@
|
|
1
1
|
class FnordMetric::TimelineWidget < FnordMetric::Widget
|
2
2
|
|
3
|
+
def data_labels
|
4
|
+
ticks.map do |t|
|
5
|
+
Time.at(t).strftime('%d.%m.%y %H:%M')
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
def data_series
|
10
|
+
gauges.map do |gauge|
|
11
|
+
{
|
12
|
+
:color => next_series_colour,
|
13
|
+
:data => ticks.map{ |t| gauge.value_at(t)||0 }
|
14
|
+
}
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def next_series_colour
|
19
|
+
@series_colors.pop.tap do |color|
|
20
|
+
@series_colors.unshift(color)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
3
24
|
def data
|
25
|
+
@series_colors = ["#FACE4F", "#42436B", "#CD645A", "#2F635E"]
|
26
|
+
|
4
27
|
super.merge(
|
5
|
-
:
|
6
|
-
:
|
7
|
-
:tick => tick.to_i,
|
8
|
-
:delta => !!@options[:delta],
|
9
|
-
:autoupdate => !!@options[:autoupdate],
|
10
|
-
:metrics => metrics.map{ |m| m.token },
|
11
|
-
:widget_url => "/fnordmetric/widget_timeline.js?#{(rand*999).to_i}",
|
12
|
-
:chart_type => (@options[:chart] || "line"),
|
13
|
-
:widget_height => 320
|
28
|
+
:labels => data_labels,
|
29
|
+
:series => data_series
|
14
30
|
)
|
15
31
|
end
|
16
32
|
|
33
|
+
def has_tick?
|
34
|
+
true
|
35
|
+
end
|
36
|
+
|
17
37
|
end
|