fnordmetric 0.3.2 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. data/Gemfile +6 -0
  2. data/Gemfile.lock +21 -0
  3. data/Procfile +1 -2
  4. data/VERSION +1 -1
  5. data/_spec/app_spec.rb +178 -0
  6. data/{spec → _spec}/cache_spec.rb +0 -0
  7. data/{spec → _spec}/combine_metric_spec.rb +0 -0
  8. data/{spec → _spec}/core_spec.rb +0 -0
  9. data/{spec → _spec}/count_metric_spec.rb +0 -0
  10. data/_spec/dashboard_spec.rb +67 -0
  11. data/_spec/event_spec.rb +46 -0
  12. data/{spec → _spec}/metric_spec.rb +0 -0
  13. data/{spec → _spec}/report_spec.rb +0 -0
  14. data/{spec → _spec}/sum_metric_spec.rb +0 -0
  15. data/_spec/widget_spec.rb +107 -0
  16. data/doc/import_dump.rb +26 -0
  17. data/em_runner.rb +33 -0
  18. data/fnordmetric.gemspec +59 -20
  19. data/haml/app.haml +26 -12
  20. data/lib/fnordmetric.rb +150 -15
  21. data/lib/fnordmetric/app.rb +70 -11
  22. data/lib/fnordmetric/cache.rb +4 -4
  23. data/lib/fnordmetric/context.rb +65 -0
  24. data/lib/fnordmetric/dashboard.rb +16 -12
  25. data/lib/fnordmetric/event.rb +65 -15
  26. data/lib/fnordmetric/gauge.rb +46 -0
  27. data/lib/fnordmetric/gauge_calculations.rb +43 -0
  28. data/lib/fnordmetric/gauge_modifiers.rb +43 -0
  29. data/lib/fnordmetric/inbound_stream.rb +66 -0
  30. data/lib/fnordmetric/logger.rb +38 -0
  31. data/lib/fnordmetric/namespace.rb +120 -0
  32. data/lib/fnordmetric/numbers_widget.rb +29 -11
  33. data/lib/fnordmetric/session.rb +131 -0
  34. data/lib/fnordmetric/standalone.rb +31 -0
  35. data/lib/fnordmetric/timeline_widget.rb +29 -9
  36. data/lib/fnordmetric/widget.rb +50 -45
  37. data/lib/fnordmetric/worker.rb +80 -0
  38. data/pub/fnordmetric/fnordmetric.css +76 -9
  39. data/pub/fnordmetric/fnordmetric.js +541 -42
  40. data/pub/raphael-min.js +8 -0
  41. data/pub/raphael-utils.js +221 -0
  42. data/readme.rdoc +172 -27
  43. data/server.rb +22 -0
  44. data/spec/app_spec.rb +359 -117
  45. data/spec/context_spec.rb +42 -0
  46. data/spec/dashboard_spec.rb +7 -47
  47. data/spec/event_spec.rb +114 -33
  48. data/spec/gauge_modifiers_spec.rb +276 -0
  49. data/spec/gauge_spec.rb +128 -0
  50. data/spec/namespace_spec.rb +104 -0
  51. data/spec/session_spec.rb +231 -0
  52. data/spec/spec_helper.rb +27 -4
  53. data/spec/widget_spec.rb +81 -75
  54. data/spec/worker_spec.rb +37 -0
  55. data/test_stream.sh +187 -0
  56. data/ulm_stats.rb +198 -0
  57. metadata +114 -35
  58. data/lib/fnordmetric/core.rb +0 -66
  59. data/lib/fnordmetric/engine.rb +0 -3
@@ -0,0 +1,66 @@
1
+ class FnordMetric::InboundStream < EventMachine::Connection
2
+
3
+ @@opts = nil
4
+
5
+ def self.start(opts)
6
+ @@opts = opts
7
+ EM.start_server(*opts[:inbound_stream], self)
8
+ end
9
+
10
+ def receive_data(chunk)
11
+ @buffer << chunk
12
+ EM.defer{ next_event }
13
+ end
14
+
15
+ def push_event(event_id, event_data)
16
+ prefix = @@opts[:redis_prefix]
17
+
18
+ @redis.hincrby "#{prefix}-stats", "events_received", 1
19
+ @redis.set "#{prefix}-event-#{event_id}", event_data
20
+ @redis.lpush "#{prefix}-queue", event_id
21
+ @redis.expire "#{prefix}-event-#{event_id}", @@opts[:event_queue_ttl]
22
+
23
+ @events_buffered -= 1
24
+ close_connection?
25
+ end
26
+
27
+ def next_event
28
+ read_next_event
29
+ push_next_event
30
+ end
31
+
32
+ def read_next_event
33
+ while (event = @buffer.slice!(/^(.*)\n/))
34
+ @events_buffered += 1
35
+ @events << event
36
+ end
37
+ end
38
+
39
+ def push_next_event
40
+ return true if @events.empty?
41
+ push_event(get_next_uuid, @events.pop)
42
+ EM.next_tick(&method(:push_next_event))
43
+ end
44
+
45
+ def get_next_uuid
46
+ rand(9999999999999999999).to_s # FIXME
47
+ end
48
+
49
+ def close_connection?
50
+ @redis.quit unless @streaming || (@events_buffered!=0)
51
+ end
52
+
53
+ def post_init
54
+ @redis = Redis.new
55
+ @events_buffered = 0
56
+ @streaming = true
57
+ @buffer = ""
58
+ @events = []
59
+ end
60
+
61
+ def unbind
62
+ @streaming = false
63
+ close_connection?
64
+ end
65
+
66
+ end
@@ -0,0 +1,38 @@
1
+ class FnordMetric::Logger
2
+
3
+ def self.start(logfile_path)
4
+ require 'json'
5
+ event_ids = Queue.new
6
+ dump_file = File.open(logfile_path, 'a+')
7
+
8
+ fetcher = Thread.new do
9
+ redis = Redis.new
10
+ loop do
11
+ event_id = event_ids.pop
12
+ event_data = redis.get("fnordmetric-event-#{event_id}")
13
+ event_hash = JSON.parse(event_data) rescue next
14
+
15
+ event_hash.merge!(:_time => Time.now.to_i)
16
+
17
+ dump_file.write(event_hash.to_json+"\n")
18
+ dump_file.flush
19
+
20
+ print "\033[1;34m"
21
+ print event_hash.inspect
22
+ print "\033[0m\n"
23
+ end
24
+ end
25
+
26
+ listener = Thread.new do
27
+ redis = Redis.new
28
+ redis.subscribe("fnordmetric-announce") do |on|
29
+ on.message do |channel, event_id|
30
+ event_ids << event_id
31
+ end
32
+ end
33
+ end
34
+
35
+ fetcher.join
36
+ end
37
+
38
+ end
@@ -0,0 +1,120 @@
1
+ class FnordMetric::Namespace
2
+
3
+ attr_reader :handlers, :gauges, :opts, :key, :dashboards
4
+
5
+ @@opts = [:event, :gauge, :widget]
6
+
7
+ def initialize(key, opts)
8
+ @gauges = Hash.new
9
+ @dashboards = Hash.new
10
+ @handlers = Hash.new
11
+ @opts = opts
12
+ @key = key
13
+ end
14
+
15
+ def ready!(redis)
16
+ @redis = redis
17
+ @gauges.map{ |k,g| g.add_redis(@redis) }
18
+ self
19
+ end
20
+
21
+ def announce(event)
22
+ announce_to_timeline(event)
23
+ announce_to_typelist(event)
24
+
25
+ if event[:_session]
26
+ event[:_session_key] = announce_to_session(event).session_key
27
+ end
28
+
29
+ [
30
+ @handlers[event[:_type].to_s],
31
+ @handlers["*"]
32
+ ].flatten.compact.each do |context|
33
+ context.call(event, @redis)
34
+ end
35
+
36
+ self
37
+ end
38
+
39
+ def announce_to_session(event)
40
+ FnordMetric::Session.create(@opts.clone.merge(
41
+ :namespace_key => @key,
42
+ :namespace_prefix => key_prefix,
43
+ :redis => @redis,
44
+ :event => event
45
+ ))
46
+ end
47
+
48
+ def announce_to_timeline(event)
49
+ timeline_key = key_prefix(:timeline)
50
+ @redis.zadd(timeline_key, event[:_time], event[:_eid])
51
+ end
52
+
53
+ def announce_to_typelist(event)
54
+ typelist_key = key_prefix("type-#{event[:_type]}")
55
+ @redis.lpush(typelist_key, event[:_eid])
56
+ end
57
+
58
+
59
+ def key_prefix(append=nil)
60
+ [@opts[:redis_prefix], @key, append].compact.join("-")
61
+ end
62
+
63
+ def token
64
+ @key
65
+ end
66
+
67
+ def dashboards(name=nil)
68
+ return @dashboards unless name
69
+ @dashboards[name] ||= FnordMetric::Dashboard.new(
70
+ :title => name
71
+ )
72
+ end
73
+
74
+ def sessions(_ids, opts={})
75
+ return FnordMetric::Session.all(extend_opts(opts)) if _ids == :all
76
+ end
77
+
78
+ def events(_ids, opts={})
79
+ return FnordMetric::Event.all(extend_opts(opts)) if _ids == :all
80
+ return FnordMetric::Event.by_type(opts.delete(:type), extend_opts(opts)) if _ids == :by_type
81
+ end
82
+
83
+ def method_missing(m, *args, &block)
84
+ raise "unknown option: #{m}" unless @@opts.include?(m)
85
+ send(:"opt_#{m}", *args, &block)
86
+ end
87
+
88
+ def opt_event(event_type, opts={}, &block)
89
+ opts.merge!(:redis => @redis, :gauges => @gauges)
90
+ FnordMetric::Context.new(opts, block).tap do |context|
91
+ @handlers[event_type.to_s] ||= []
92
+ @handlers[event_type.to_s] << context
93
+ end
94
+ end
95
+
96
+ def opt_gauge(gauge_key, opts={})
97
+ opts.merge!(:key => gauge_key, :key_prefix => key_prefix)
98
+ @gauges[gauge_key] ||= FnordMetric::Gauge.new(opts)
99
+ end
100
+
101
+ def opt_widget(dashboard, widget)
102
+ widget = build_widget(widget) if widget.is_a?(Hash)
103
+ dashboards(dashboard).add_widget(widget)
104
+ end
105
+
106
+ def build_widget(opts)
107
+ _gauges = [opts[:gauges]].flatten.map{ |g| @gauges.fetch(g) }
108
+ widget_klass = "FnordMetric::#{opts.fetch(:type).to_s.capitalize}Widget"
109
+ widget_klass.constantize.new(opts.merge(:gauges => _gauges))
110
+ end
111
+
112
+ def extend_opts(opts)
113
+ opts.merge(
114
+ :namespace_prefix => key_prefix,
115
+ :redis_prefix => @opts[:redis_prefix],
116
+ :redis => @redis
117
+ )
118
+ end
119
+
120
+ end
@@ -1,18 +1,36 @@
1
- class FnordMetric::NumbersWidget < FnordMetric::Widget
1
+ class FnordMetric::NumbersWidget < FnordMetric::Widget
2
2
 
3
3
  def data
4
4
  super.merge(
5
- :autoupdate => !!@options[:autoupdate],
6
- :round_to => @options[:round_to]||2,
7
- :metrics => metrics.map{ |m| m.token },
8
- :intervals => {
9
- "Current" => "",
10
- "Today" => "at=#{1.day.ago.to_i}-#{Time.now.to_i}",
11
- "Month" => "at=#{30.days.ago.to_i}-#{Time.now.to_i}"
12
- },
13
- :widget_url => "/fnordmetric/widget_numbers.js?#{(rand*999).to_i}",
14
- :widget_height => 100
5
+ :offsets => offsets,
6
+ :gauges => data_gauges
15
7
  )
16
8
  end
17
9
 
10
+ def data_gauges
11
+ Hash.new.tap do |hash|
12
+ gauges.each do |g|
13
+ hash[g.name] = {
14
+ :values => data_gauge(g),
15
+ :title => g.name
16
+ }
17
+ end
18
+ end
19
+ end
20
+
21
+ def data_gauge(gauge)
22
+ offsets.map do |offset|
23
+ offset_time = Time.now.to_i - offset*gauge.tick
24
+ [gauge.tick_at(offset_time), gauge.value_at(offset_time)]
25
+ end
26
+ end
27
+
28
+ def has_tick?
29
+ false
30
+ end
31
+
32
+ def offsets
33
+ [0, 1, 30]
34
+ end
35
+
18
36
  end
@@ -0,0 +1,131 @@
1
+ class FnordMetric::Session
2
+
3
+ attr_accessor :updated_at, :name, :picture
4
+
5
+ @@meta_attributes = %w(name picture)
6
+
7
+ def self.create(opts)
8
+ redis = opts.fetch(:redis)
9
+ event = opts[:event]
10
+
11
+ hash = Digest::MD5.hexdigest(event[:_session])
12
+ set_key = "#{opts[:namespace_prefix]}-session"
13
+
14
+ self.new(hash).tap do |session|
15
+ session.add_redis(redis, set_key)
16
+ session.add_event(event)
17
+ session.expire(opts[:session_data_ttl])
18
+ end
19
+ end
20
+
21
+ def self.find(session_key, opts)
22
+ set_key = "#{opts[:namespace_prefix]}-session"
23
+ self.new(session_key, [opts[:redis], set_key])
24
+ end
25
+
26
+ def self.all(opts)
27
+ set_key = "#{opts[:namespace_prefix]}-session"
28
+ limit = (opts[:limit].try(:to_i)||0)-1
29
+ session_ids = opts[:redis].zrevrange(set_key, 0, limit, :withscores => true)
30
+ session_ids.in_groups_of(2).map do |session_key, ts|
31
+ find(session_key, opts).tap{ |s| s.updated_at = ts }
32
+ end
33
+ end
34
+
35
+ def initialize(session_key, redis_opts=nil)
36
+ @session_key = session_key
37
+ add_redis(*redis_opts) if redis_opts
38
+ end
39
+
40
+ def session_key
41
+ @session_key
42
+ end
43
+
44
+ def picture
45
+ @picture
46
+ end
47
+
48
+ def name
49
+ @name
50
+ end
51
+
52
+ def data(key=nil)
53
+ key ? @data[key] : @data
54
+ end
55
+
56
+ def event_ids
57
+ @event_ids || []
58
+ end
59
+
60
+ def events
61
+ []
62
+ end
63
+
64
+ def to_json
65
+ { :session_key => session_key }.tap do |hash|
66
+ hash.merge!(:_picture => @picture) if @picture
67
+ hash.merge!(:_name => @name) if @name
68
+ hash.merge!(:_updated_at => @updated_at) if @updated_at
69
+ end
70
+ end
71
+
72
+ def redis_key(append=nil)
73
+ [@redis_prefix, @session_key, append].compact.join("-")
74
+ end
75
+
76
+ def add_redis(redis, prefix)
77
+ @redis_prefix = prefix
78
+ @redis = redis
79
+ end
80
+
81
+ def touch(time=Time.now.to_i)
82
+ @redis.zadd(@redis_prefix, time, @session_key)
83
+ end
84
+
85
+ def expire(time)
86
+ @redis.expire(redis_key(:events), time)
87
+ @redis.expire(redis_key(:data), time)
88
+ end
89
+
90
+ def add_event(event)
91
+ @redis.zadd(redis_key(:events), event[:_time], event[:_eid])
92
+ add_data(:_picture, event[:url]) if event[:_type] == "_set_picture"
93
+ add_data(:_name, event[:name]) if event[:_type] == "_set_name"
94
+ add_event_data(event) if event[:_type] == "_set_data"
95
+ touch(event[:_time])
96
+ end
97
+
98
+ def add_event_data(event)
99
+ event.each do |key,value|
100
+ add_data(key, value) unless key[0]=="_"
101
+ end
102
+ end
103
+
104
+ def add_data(key, value)
105
+ @redis.hset(redis_key(:data), key, value)
106
+ end
107
+
108
+ def fetch_data!
109
+ @data = Hash.new
110
+ @redis.hgetall(redis_key(:data)).each do |key, value|
111
+ if key[0]=="_"
112
+ fetch_meta_key(key, value)
113
+ else
114
+ @data[key.intern] = value
115
+ end
116
+ end
117
+ end
118
+
119
+ def fetch_meta_key(key, value)
120
+ meta_key = key[1..-1]
121
+ if @@meta_attributes.include?(meta_key)
122
+ instance_variable_set(:"@#{meta_key}", value)
123
+ end
124
+ end
125
+
126
+ def fetch_event_ids!(since=-1)
127
+ # FIXME: use WITHSCORE to get the timestamps and return event objects
128
+ @event_ids = @redis.zrange(redis_key(:events), 0, since)
129
+ end
130
+
131
+ end
@@ -0,0 +1,31 @@
1
+ require 'rake'
2
+ require 'redis'
3
+
4
+ task :run do
5
+ FnordMetric.run
6
+ end
7
+
8
+ task :log do
9
+ FnordMetric::Logger.start(dump_file_path)
10
+ end
11
+
12
+ task :import do
13
+ puts 'not yet implemented :('
14
+ end
15
+
16
+ task :help do
17
+ puts "usage: #{$0} {run|log|import} [DUMP_FILE=fm_dump.json]"
18
+ end
19
+
20
+ task :default => :help
21
+
22
+ def dump_file_path
23
+ if ENV["DUMP_FILE"].blank?
24
+ Rake::Task[:help].execute; exit!
25
+ else
26
+ ::File.expand_path(ENV["DUMP_FILE"], ::File.dirname($0))
27
+ end
28
+ end
29
+
30
+ Rake.application.init('fnordmetric')
31
+ Rake.application.top_level
@@ -1,17 +1,37 @@
1
1
  class FnordMetric::TimelineWidget < FnordMetric::Widget
2
2
 
3
+ def data_labels
4
+ ticks.map do |t|
5
+ Time.at(t).strftime('%d.%m.%y %H:%M')
6
+ end
7
+ end
8
+
9
+ def data_series
10
+ gauges.map do |gauge|
11
+ {
12
+ :color => next_series_colour,
13
+ :data => ticks.map{ |t| gauge.value_at(t)||0 }
14
+ }
15
+ end
16
+ end
17
+
18
+ def next_series_colour
19
+ @series_colors.pop.tap do |color|
20
+ @series_colors.unshift(color)
21
+ end
22
+ end
23
+
3
24
  def data
25
+ @series_colors = ["#FACE4F", "#42436B", "#CD645A", "#2F635E"]
26
+
4
27
  super.merge(
5
- :start_timestamp => range_to_i.first,
6
- :end_timestamp => range_to_i.last,
7
- :tick => tick.to_i,
8
- :delta => !!@options[:delta],
9
- :autoupdate => !!@options[:autoupdate],
10
- :metrics => metrics.map{ |m| m.token },
11
- :widget_url => "/fnordmetric/widget_timeline.js?#{(rand*999).to_i}",
12
- :chart_type => (@options[:chart] || "line"),
13
- :widget_height => 320
28
+ :labels => data_labels,
29
+ :series => data_series
14
30
  )
15
31
  end
16
32
 
33
+ def has_tick?
34
+ true
35
+ end
36
+
17
37
  end