dasht 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +7 -0
  2. data/.agignore +3 -0
  3. data/Gemfile +14 -0
  4. data/Gemfile.lock +88 -0
  5. data/LICENSE.txt +20 -0
  6. data/README.md +329 -0
  7. data/Rakefile +51 -0
  8. data/VERSION +1 -0
  9. data/assets/css/style.css +123 -0
  10. data/assets/images/background.jpg +0 -0
  11. data/assets/js/Chart.min.js +11 -0
  12. data/assets/js/dasht.js +141 -0
  13. data/assets/js/jquery-1.11.3.min.js +5 -0
  14. data/assets/js/masonry.pkgd.min.js +9 -0
  15. data/assets/js/mustache.min.js +1 -0
  16. data/assets/js/underscore-min.js +6 -0
  17. data/assets/plugins/chart.css +11 -0
  18. data/assets/plugins/chart.html +2 -0
  19. data/assets/plugins/chart.js +50 -0
  20. data/assets/plugins/map.css +17 -0
  21. data/assets/plugins/map.html +4 -0
  22. data/assets/plugins/map.js +125 -0
  23. data/assets/plugins/scroll.css +2 -0
  24. data/assets/plugins/scroll.html +2 -0
  25. data/assets/plugins/scroll.js +14 -0
  26. data/assets/plugins/value.css +18 -0
  27. data/assets/plugins/value.html +4 -0
  28. data/assets/plugins/value.js +26 -0
  29. data/examples/simple_heroku_dashboard.rb +34 -0
  30. data/lib/dasht.rb +25 -0
  31. data/lib/dasht/array_monkeypatching.rb +5 -0
  32. data/lib/dasht/base.rb +117 -0
  33. data/lib/dasht/board.rb +108 -0
  34. data/lib/dasht/collector.rb +33 -0
  35. data/lib/dasht/list.rb +93 -0
  36. data/lib/dasht/log_thread.rb +94 -0
  37. data/lib/dasht/metric.rb +82 -0
  38. data/lib/dasht/rack_app.rb +74 -0
  39. data/lib/dasht/reloader.rb +28 -0
  40. data/screenshot_1.png +0 -0
  41. data/test/helper.rb +34 -0
  42. data/test/test_list.rb +65 -0
  43. data/test/test_metric.rb +58 -0
  44. data/tests.rb +6 -0
  45. data/views/dashboard.erb +27 -0
  46. metadata +189 -0
@@ -0,0 +1,33 @@
1
+ module Dasht
2
+ class Metrics
3
+ attr_accessor :parent
4
+ def initialize(parent)
5
+ @parent = parent
6
+ @metric_values = {}
7
+ @metric_operations = {}
8
+ end
9
+
10
+ def set(metric, value, op, ts)
11
+ metric = metric.to_s
12
+ @metric_operations[metric] = op
13
+ m = (@metric_values[metric] ||= Metric.new)
14
+ m.append(value, ts) do |old_value, new_value|
15
+ [old_value, new_value].compact.flatten.send(op)
16
+ end
17
+ end
18
+
19
+ def get(metric, start_ts, end_ts)
20
+ metric = metric.to_s
21
+ m = @metric_values[metric]
22
+ return [] if m.nil?
23
+ op = @metric_operations[metric]
24
+ m.enum(start_ts, end_ts).to_a.flatten.send(op)
25
+ end
26
+
27
+ def trim_to(ts)
28
+ @metric_values.each do |k, v|
29
+ v.trim_to(ts)
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,93 @@
1
+ # Dasht::List - Simple list structure following properties:
2
+ #
3
+ # 1. Fast writes. Appends to a list.
4
+ # 2. Fast reads by index. Indexed by position in the list.
5
+ # 3. Fast deletes that preserve indexes. Removes items from the front
6
+ # of the list.
7
+ # 4. Simple (but not necessarily fast) aggregation. Enumerate values
8
+ # between pointers.
9
+ #
10
+ # The Dasht::List structure is formed using a Ruby Array (values),
11
+ # plus a counter of how many items have been deleted (offset).
12
+ # Whenever data is deleted from the head of the list, the offset is
13
+ # incremented.
14
+
15
+ module Dasht
16
+ class List
17
+ attr_accessor :values
18
+ attr_accessor :offset
19
+
20
+ def initialize
21
+ @offset = 0
22
+ @values = []
23
+ end
24
+
25
+ def to_s
26
+ return @values.to_s
27
+ end
28
+
29
+ # Public: Get a pointer to the first value.
30
+ def head_pointer
31
+ return offset
32
+ end
33
+
34
+ # Public: Get a pointer to right after the last value.
35
+ def tail_pointer
36
+ return offset + @values.length
37
+ end
38
+
39
+ # Public: Get the value at a given pointer, or nil if the pointer
40
+ # is no longer valid.
41
+ def get(pointer)
42
+ index = _pointer_to_index(pointer)
43
+ return @values[index]
44
+ end
45
+
46
+ # Public: Return an enumerator that walks through the list, yielding
47
+ # data.
48
+ def enum(start_pointer = nil, end_pointer = nil)
49
+ index = _pointer_to_index(start_pointer || head_pointer)
50
+ end_index = _pointer_to_index(end_pointer || tail_pointer)
51
+ return Enumerator.new do |yielder|
52
+ while index < end_index
53
+ yielder << @values[index]
54
+ index += 1
55
+ end
56
+ end
57
+ end
58
+
59
+ # Public: Add data to the list.
60
+ # Returns a pointer to the new data.
61
+ def append(data)
62
+ pointer = self.tail_pointer
63
+ @values << data
64
+ return pointer
65
+ end
66
+
67
+ # Public: Remove data up to (but not including) the specified pointer.
68
+ def trim_to(pointer)
69
+ return if pointer.nil?
70
+ index = _pointer_to_index(pointer)
71
+ @offset += index
72
+ @values = @values.slice(index, @values.length)
73
+ return
74
+ end
75
+
76
+ # Public: Walk through the list, removing links from the list while
77
+ # the block returns true. Stop when it returns false.
78
+ def trim_while(&block)
79
+ while (@values.length > 0) && yield(@values.first)
80
+ @values.shift
81
+ @offset += 1
82
+ end
83
+ return
84
+ end
85
+
86
+ private
87
+
88
+ # Convert a pointer to an index in the list.
89
+ def _pointer_to_index(pointer)
90
+ return [pointer - offset, 0].max
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,94 @@
1
+ module Dasht
2
+ class LogThread
3
+ attr_accessor :parent
4
+
5
+ def self.update_global_stats(line)
6
+ @total_lines ||= 0
7
+ @total_bytes ||= 0
8
+ @total_lines += 1
9
+ @total_bytes += line.length
10
+ print "\rConsumed #{@total_lines} lines (#{@total_bytes} bytes)..."
11
+ end
12
+
13
+ def initialize(parent, command)
14
+ @parent = parent
15
+ @command = command
16
+ @event_definitions = []
17
+ end
18
+
19
+ def run
20
+ parent.log "Starting `#{@command}`..."
21
+ @thread = Thread.new do
22
+ begin
23
+ while true
24
+ begin
25
+ IO.popen(@command) do |process|
26
+ process.each do |line|
27
+ _consume_line(line)
28
+ end
29
+ end
30
+ rescue => e
31
+ parent.log e
32
+ end
33
+ sleep 2
34
+ end
35
+ rescue => e
36
+ parent.log e
37
+ raise e
38
+ end
39
+ end
40
+ end
41
+
42
+ def event(metric, regex, op, value = nil, &block)
43
+ @event_definitions << [metric, regex, op, value, block]
44
+ end
45
+
46
+ def count(metric, regex, &block)
47
+ event(metric, regex, :dasht_sum, 1, &block)
48
+ end
49
+
50
+ def gauge(metric, regex, &block)
51
+ event(metric, regex, :last, nil, &block)
52
+ end
53
+
54
+ def min(metric, regex, &block)
55
+ event(metric, regex, :min, nil, &block)
56
+ end
57
+
58
+ def max(metric, regex, &block)
59
+ event(metric, regex, :max, nil, &block)
60
+ end
61
+
62
+ def append(metric, regex, &block)
63
+ event(metric, regex, :to_a, nil, &block)
64
+ end
65
+
66
+ def unique(metric, regex, &block)
67
+ event(metric, regex, :uniq, nil, &block)
68
+ end
69
+
70
+ def terminate
71
+ @thread.terminate
72
+ end
73
+
74
+ private
75
+
76
+ def _consume_line(line)
77
+ self.class.update_global_stats(line)
78
+ ts = Time.now.to_i
79
+ @event_definitions.each do |metric, regex, op, value, block|
80
+ begin
81
+ regex.match(line) do |matches|
82
+ value = matches[0] if value.nil?
83
+ value = block.call(matches) if block
84
+ parent.metrics.set(metric, value, op, ts) if value
85
+ end
86
+ rescue => e
87
+ parent.log e
88
+ raise e
89
+ end
90
+ parent.metrics.trim_to(ts - (parent.history || (60 * 60)))
91
+ end
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,82 @@
1
+ # Dasht::Metric - Simple in-memory time-series data structure with the
2
+ # following properties:
3
+ #
4
+ # 1. Sparse. Only stores time stamps for intervals with known data,
5
+ # and only stores one timestamp per interval.
6
+ # 2. Flexible aggregation using Ruby blocks during both read and
7
+ # write.
8
+ # 3. Read values between two timestamps.
9
+ #
10
+ # The Dasht::Metric structure is formed using two Dasht::List
11
+ # objects. One object tracks data, the other object tracks a list of
12
+ # checkpoints and their corresponding index into the data.
13
+ module Dasht
14
+ class Metric
15
+ attr_reader :data, :checkpoints
16
+
17
+ def initialize
18
+ @checkpoints = List.new
19
+ @data = List.new
20
+ @last_item = nil
21
+ @last_ts = nil
22
+ end
23
+
24
+ def to_s
25
+ return @data.to_s + " (last: #{@last_item})"
26
+ end
27
+
28
+ def append(data, ts, &block)
29
+ # Maybe checkpoint the time.
30
+ if @last_ts == ts
31
+ @last_item = yield(@last_item, data)
32
+ else
33
+ if @last_ts
34
+ pointer = @data.append(@last_item)
35
+ @checkpoints.append([@last_ts, pointer])
36
+ end
37
+ @last_ts = ts
38
+ @last_item = nil
39
+ @last_item = yield(@last_item, data)
40
+ end
41
+ return
42
+ end
43
+
44
+ def trim_to(ts)
45
+ pointer = nil
46
+ @checkpoints.trim_while do |s, p|
47
+ pointer = p
48
+ s < ts
49
+ end
50
+ @data.trim_to(pointer)
51
+ return
52
+ end
53
+
54
+ def enum(start_ts, end_ts = nil)
55
+ # Get a pointer to our location in the data.
56
+ start_pointer = nil
57
+ end_pointer = nil
58
+ prev_p = nil
59
+ @checkpoints.enum.each do |s, p|
60
+ start_pointer ||= p if start_ts <= s
61
+ end_pointer ||= prev_p if end_ts && end_ts <= s
62
+ break if start_pointer && (end_ts.nil? || end_pointer)
63
+ prev_p = p
64
+ end
65
+ start_pointer ||= @data.tail_pointer
66
+ end_pointer ||= @data.tail_pointer
67
+
68
+ # Enumerate through the data, then tack on the last item.
69
+ return Enumerator.new do |yielder|
70
+ @data.enum(start_pointer, end_pointer).each do |data|
71
+ yielder << data
72
+ end
73
+ # Maybe include the last item.
74
+ if @last_item &&
75
+ (start_ts <= @last_ts) &&
76
+ (end_ts.nil? || (@last_ts < end_ts))
77
+ yielder << @last_item
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,74 @@
1
+ module Dasht
2
+ class RackApp
3
+ attr_accessor :parent
4
+
5
+ def initialize(parent)
6
+ @parent = parent
7
+ end
8
+
9
+ def root_path
10
+ File.join(File.dirname(__FILE__), "..", "..")
11
+ end
12
+
13
+ def run(port)
14
+ context = self
15
+ app = Rack::Builder.new do
16
+ use Rack::Static, :urls => ["/assets"], :root => context.root_path
17
+ run lambda { |env|
18
+ begin
19
+ context._call(env)
20
+ rescue => e
21
+ context.parent.log e
22
+ raise e
23
+ end
24
+ }
25
+ end
26
+ Rack::Server.start(:app => app, :Port => (port || 8080))
27
+ end
28
+
29
+ def _call(env)
30
+ if "/" == env["REQUEST_PATH"] && parent.boards["default"]
31
+ return ['200', {'Content-Type' => 'text/html'}, [parent.boards["default"].to_html]]
32
+ end
33
+
34
+ /^\/boards\/(.+)$/.match(env["REQUEST_PATH"]) do |match|
35
+ board = match[1]
36
+ if parent.boards[board]
37
+ return ['200', {'Content-Type' => 'text/html'}, [parent.boards[board].to_html]]
38
+ else
39
+ return ['404', {'Content-Type' => 'text/html'}, ["Board #{board} not found."]]
40
+ end
41
+ end
42
+
43
+ /^\/data$/.match(env["REQUEST_PATH"]) do |match|
44
+ queries = JSON.parse(env['rack.input'].gets)
45
+ now = Time.now.to_i
46
+ data = queries.map do |query|
47
+ metric = query["metric"]
48
+ resolution = query["resolution"]
49
+ periods = query["periods"]
50
+ ts = now - (resolution * periods)
51
+ (1..periods).map do |n|
52
+ parent.metrics.get(metric, ts, ts += resolution) || 0
53
+ end
54
+ end
55
+
56
+ return ['200', {'Content-Type' => 'application/json'}, [data.to_json]]
57
+ end
58
+
59
+ /^\/data\/(.+)/.match(env["REQUEST_PATH"]) do |match|
60
+ parts = match[1].split('/')
61
+ metric = parts.shift
62
+ resolution = parts.shift.to_i
63
+ periods = (parts.shift || 1).to_i
64
+ ts = Time.now.to_i - (resolution * periods)
65
+ data = (1..periods).map do |n|
66
+ parent.metrics.get(metric, ts, ts += resolution) || 0
67
+ end
68
+ return ['200', {'Content-Type' => 'application/json'}, [data.to_json]]
69
+ end
70
+
71
+ return ['404', {'Content-Type' => 'text/html'}, ["Path not found: #{env['REQUEST_PATH']}"]]
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,28 @@
1
+ module Dasht
2
+ class Reloader
3
+ attr_accessor :parent
4
+
5
+ def initialize(parent)
6
+ @parent = parent
7
+ @last_modified = File.mtime($PROGRAM_NAME)
8
+ end
9
+
10
+ def changed?
11
+ @last_modified != File.mtime($PROGRAM_NAME)
12
+ end
13
+
14
+ def run
15
+ Thread.new do
16
+ while true
17
+ unless changed?
18
+ sleep 0.3
19
+ next
20
+ end
21
+ parent.log("Reloading #{$PROGRAM_NAME}...")
22
+ eval(IO.read($PROGRAM_NAME))
23
+ @last_modified = File.mtime($PROGRAM_NAME)
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
Binary file
@@ -0,0 +1,34 @@
1
+ require 'simplecov'
2
+
3
+ module SimpleCov::Configuration
4
+ def clean_filters
5
+ @filters = []
6
+ end
7
+ end
8
+
9
+ SimpleCov.configure do
10
+ clean_filters
11
+ load_adapter 'test_frameworks'
12
+ end
13
+
14
+ ENV["COVERAGE"] && SimpleCov.start do
15
+ add_filter "/.rvm/"
16
+ end
17
+ require 'rubygems'
18
+ require 'bundler'
19
+ begin
20
+ Bundler.setup(:default, :development)
21
+ rescue Bundler::BundlerError => e
22
+ $stderr.puts e.message
23
+ $stderr.puts "Run `bundle install` to install missing gems"
24
+ exit e.status_code
25
+ end
26
+ require 'test/unit'
27
+ require 'shoulda'
28
+
29
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
30
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
31
+ require 'dasht'
32
+
33
+ class Test::Unit::TestCase
34
+ end