dasht 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.agignore +3 -0
- data/Gemfile +14 -0
- data/Gemfile.lock +88 -0
- data/LICENSE.txt +20 -0
- data/README.md +329 -0
- data/Rakefile +51 -0
- data/VERSION +1 -0
- data/assets/css/style.css +123 -0
- data/assets/images/background.jpg +0 -0
- data/assets/js/Chart.min.js +11 -0
- data/assets/js/dasht.js +141 -0
- data/assets/js/jquery-1.11.3.min.js +5 -0
- data/assets/js/masonry.pkgd.min.js +9 -0
- data/assets/js/mustache.min.js +1 -0
- data/assets/js/underscore-min.js +6 -0
- data/assets/plugins/chart.css +11 -0
- data/assets/plugins/chart.html +2 -0
- data/assets/plugins/chart.js +50 -0
- data/assets/plugins/map.css +17 -0
- data/assets/plugins/map.html +4 -0
- data/assets/plugins/map.js +125 -0
- data/assets/plugins/scroll.css +2 -0
- data/assets/plugins/scroll.html +2 -0
- data/assets/plugins/scroll.js +14 -0
- data/assets/plugins/value.css +18 -0
- data/assets/plugins/value.html +4 -0
- data/assets/plugins/value.js +26 -0
- data/examples/simple_heroku_dashboard.rb +34 -0
- data/lib/dasht.rb +25 -0
- data/lib/dasht/array_monkeypatching.rb +5 -0
- data/lib/dasht/base.rb +117 -0
- data/lib/dasht/board.rb +108 -0
- data/lib/dasht/collector.rb +33 -0
- data/lib/dasht/list.rb +93 -0
- data/lib/dasht/log_thread.rb +94 -0
- data/lib/dasht/metric.rb +82 -0
- data/lib/dasht/rack_app.rb +74 -0
- data/lib/dasht/reloader.rb +28 -0
- data/screenshot_1.png +0 -0
- data/test/helper.rb +34 -0
- data/test/test_list.rb +65 -0
- data/test/test_metric.rb +58 -0
- data/tests.rb +6 -0
- data/views/dashboard.erb +27 -0
- metadata +189 -0
@@ -0,0 +1,33 @@
|
|
1
|
+
module Dasht
|
2
|
+
class Metrics
|
3
|
+
attr_accessor :parent
|
4
|
+
def initialize(parent)
|
5
|
+
@parent = parent
|
6
|
+
@metric_values = {}
|
7
|
+
@metric_operations = {}
|
8
|
+
end
|
9
|
+
|
10
|
+
def set(metric, value, op, ts)
|
11
|
+
metric = metric.to_s
|
12
|
+
@metric_operations[metric] = op
|
13
|
+
m = (@metric_values[metric] ||= Metric.new)
|
14
|
+
m.append(value, ts) do |old_value, new_value|
|
15
|
+
[old_value, new_value].compact.flatten.send(op)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def get(metric, start_ts, end_ts)
|
20
|
+
metric = metric.to_s
|
21
|
+
m = @metric_values[metric]
|
22
|
+
return [] if m.nil?
|
23
|
+
op = @metric_operations[metric]
|
24
|
+
m.enum(start_ts, end_ts).to_a.flatten.send(op)
|
25
|
+
end
|
26
|
+
|
27
|
+
def trim_to(ts)
|
28
|
+
@metric_values.each do |k, v|
|
29
|
+
v.trim_to(ts)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/dasht/list.rb
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
# Dasht::List - Simple list structure following properties:
|
2
|
+
#
|
3
|
+
# 1. Fast writes. Appends to a list.
|
4
|
+
# 2. Fast reads by index. Indexed by position in the list.
|
5
|
+
# 3. Fast deletes that preserve indexes. Removes items from the front
|
6
|
+
# of the list.
|
7
|
+
# 4. Simple (but not necessarily fast) aggregation. Enumerate values
|
8
|
+
# between pointers.
|
9
|
+
#
|
10
|
+
# The Dasht::List structure is formed using a Ruby Array (values),
|
11
|
+
# plus a counter of how many items have been deleted (offset).
|
12
|
+
# Whenever data is deleted from the head of the list, the offset is
|
13
|
+
# incremented.
|
14
|
+
|
15
|
+
module Dasht
|
16
|
+
class List
|
17
|
+
attr_accessor :values
|
18
|
+
attr_accessor :offset
|
19
|
+
|
20
|
+
def initialize
|
21
|
+
@offset = 0
|
22
|
+
@values = []
|
23
|
+
end
|
24
|
+
|
25
|
+
def to_s
|
26
|
+
return @values.to_s
|
27
|
+
end
|
28
|
+
|
29
|
+
# Public: Get a pointer to the first value.
|
30
|
+
def head_pointer
|
31
|
+
return offset
|
32
|
+
end
|
33
|
+
|
34
|
+
# Public: Get a pointer to right after the last value.
|
35
|
+
def tail_pointer
|
36
|
+
return offset + @values.length
|
37
|
+
end
|
38
|
+
|
39
|
+
# Public: Get the value at a given pointer, or nil if the pointer
|
40
|
+
# is no longer valid.
|
41
|
+
def get(pointer)
|
42
|
+
index = _pointer_to_index(pointer)
|
43
|
+
return @values[index]
|
44
|
+
end
|
45
|
+
|
46
|
+
# Public: Return an enumerator that walks through the list, yielding
|
47
|
+
# data.
|
48
|
+
def enum(start_pointer = nil, end_pointer = nil)
|
49
|
+
index = _pointer_to_index(start_pointer || head_pointer)
|
50
|
+
end_index = _pointer_to_index(end_pointer || tail_pointer)
|
51
|
+
return Enumerator.new do |yielder|
|
52
|
+
while index < end_index
|
53
|
+
yielder << @values[index]
|
54
|
+
index += 1
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# Public: Add data to the list.
|
60
|
+
# Returns a pointer to the new data.
|
61
|
+
def append(data)
|
62
|
+
pointer = self.tail_pointer
|
63
|
+
@values << data
|
64
|
+
return pointer
|
65
|
+
end
|
66
|
+
|
67
|
+
# Public: Remove data up to (but not including) the specified pointer.
|
68
|
+
def trim_to(pointer)
|
69
|
+
return if pointer.nil?
|
70
|
+
index = _pointer_to_index(pointer)
|
71
|
+
@offset += index
|
72
|
+
@values = @values.slice(index, @values.length)
|
73
|
+
return
|
74
|
+
end
|
75
|
+
|
76
|
+
# Public: Walk through the list, removing links from the list while
|
77
|
+
# the block returns true. Stop when it returns false.
|
78
|
+
def trim_while(&block)
|
79
|
+
while (@values.length > 0) && yield(@values.first)
|
80
|
+
@values.shift
|
81
|
+
@offset += 1
|
82
|
+
end
|
83
|
+
return
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
# Convert a pointer to an index in the list.
|
89
|
+
def _pointer_to_index(pointer)
|
90
|
+
return [pointer - offset, 0].max
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
module Dasht
|
2
|
+
class LogThread
|
3
|
+
attr_accessor :parent
|
4
|
+
|
5
|
+
def self.update_global_stats(line)
|
6
|
+
@total_lines ||= 0
|
7
|
+
@total_bytes ||= 0
|
8
|
+
@total_lines += 1
|
9
|
+
@total_bytes += line.length
|
10
|
+
print "\rConsumed #{@total_lines} lines (#{@total_bytes} bytes)..."
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(parent, command)
|
14
|
+
@parent = parent
|
15
|
+
@command = command
|
16
|
+
@event_definitions = []
|
17
|
+
end
|
18
|
+
|
19
|
+
def run
|
20
|
+
parent.log "Starting `#{@command}`..."
|
21
|
+
@thread = Thread.new do
|
22
|
+
begin
|
23
|
+
while true
|
24
|
+
begin
|
25
|
+
IO.popen(@command) do |process|
|
26
|
+
process.each do |line|
|
27
|
+
_consume_line(line)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
rescue => e
|
31
|
+
parent.log e
|
32
|
+
end
|
33
|
+
sleep 2
|
34
|
+
end
|
35
|
+
rescue => e
|
36
|
+
parent.log e
|
37
|
+
raise e
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def event(metric, regex, op, value = nil, &block)
|
43
|
+
@event_definitions << [metric, regex, op, value, block]
|
44
|
+
end
|
45
|
+
|
46
|
+
def count(metric, regex, &block)
|
47
|
+
event(metric, regex, :dasht_sum, 1, &block)
|
48
|
+
end
|
49
|
+
|
50
|
+
def gauge(metric, regex, &block)
|
51
|
+
event(metric, regex, :last, nil, &block)
|
52
|
+
end
|
53
|
+
|
54
|
+
def min(metric, regex, &block)
|
55
|
+
event(metric, regex, :min, nil, &block)
|
56
|
+
end
|
57
|
+
|
58
|
+
def max(metric, regex, &block)
|
59
|
+
event(metric, regex, :max, nil, &block)
|
60
|
+
end
|
61
|
+
|
62
|
+
def append(metric, regex, &block)
|
63
|
+
event(metric, regex, :to_a, nil, &block)
|
64
|
+
end
|
65
|
+
|
66
|
+
def unique(metric, regex, &block)
|
67
|
+
event(metric, regex, :uniq, nil, &block)
|
68
|
+
end
|
69
|
+
|
70
|
+
def terminate
|
71
|
+
@thread.terminate
|
72
|
+
end
|
73
|
+
|
74
|
+
private
|
75
|
+
|
76
|
+
def _consume_line(line)
|
77
|
+
self.class.update_global_stats(line)
|
78
|
+
ts = Time.now.to_i
|
79
|
+
@event_definitions.each do |metric, regex, op, value, block|
|
80
|
+
begin
|
81
|
+
regex.match(line) do |matches|
|
82
|
+
value = matches[0] if value.nil?
|
83
|
+
value = block.call(matches) if block
|
84
|
+
parent.metrics.set(metric, value, op, ts) if value
|
85
|
+
end
|
86
|
+
rescue => e
|
87
|
+
parent.log e
|
88
|
+
raise e
|
89
|
+
end
|
90
|
+
parent.metrics.trim_to(ts - (parent.history || (60 * 60)))
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
data/lib/dasht/metric.rb
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
# Dasht::Metric - Simple in-memory time-series data structure with the
|
2
|
+
# following properties:
|
3
|
+
#
|
4
|
+
# 1. Sparse. Only stores time stamps for intervals with known data,
|
5
|
+
# and only stores one timestamp per interval.
|
6
|
+
# 2. Flexible aggregation using Ruby blocks during both read and
|
7
|
+
# write.
|
8
|
+
# 3. Read values between two timestamps.
|
9
|
+
#
|
10
|
+
# The Dasht::Metric structure is formed using two Dasht::List
|
11
|
+
# objects. One object tracks data, the other object tracks a list of
|
12
|
+
# checkpoints and their corresponding index into the data.
|
13
|
+
module Dasht
|
14
|
+
class Metric
|
15
|
+
attr_reader :data, :checkpoints
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
@checkpoints = List.new
|
19
|
+
@data = List.new
|
20
|
+
@last_item = nil
|
21
|
+
@last_ts = nil
|
22
|
+
end
|
23
|
+
|
24
|
+
def to_s
|
25
|
+
return @data.to_s + " (last: #{@last_item})"
|
26
|
+
end
|
27
|
+
|
28
|
+
def append(data, ts, &block)
|
29
|
+
# Maybe checkpoint the time.
|
30
|
+
if @last_ts == ts
|
31
|
+
@last_item = yield(@last_item, data)
|
32
|
+
else
|
33
|
+
if @last_ts
|
34
|
+
pointer = @data.append(@last_item)
|
35
|
+
@checkpoints.append([@last_ts, pointer])
|
36
|
+
end
|
37
|
+
@last_ts = ts
|
38
|
+
@last_item = nil
|
39
|
+
@last_item = yield(@last_item, data)
|
40
|
+
end
|
41
|
+
return
|
42
|
+
end
|
43
|
+
|
44
|
+
def trim_to(ts)
|
45
|
+
pointer = nil
|
46
|
+
@checkpoints.trim_while do |s, p|
|
47
|
+
pointer = p
|
48
|
+
s < ts
|
49
|
+
end
|
50
|
+
@data.trim_to(pointer)
|
51
|
+
return
|
52
|
+
end
|
53
|
+
|
54
|
+
def enum(start_ts, end_ts = nil)
|
55
|
+
# Get a pointer to our location in the data.
|
56
|
+
start_pointer = nil
|
57
|
+
end_pointer = nil
|
58
|
+
prev_p = nil
|
59
|
+
@checkpoints.enum.each do |s, p|
|
60
|
+
start_pointer ||= p if start_ts <= s
|
61
|
+
end_pointer ||= prev_p if end_ts && end_ts <= s
|
62
|
+
break if start_pointer && (end_ts.nil? || end_pointer)
|
63
|
+
prev_p = p
|
64
|
+
end
|
65
|
+
start_pointer ||= @data.tail_pointer
|
66
|
+
end_pointer ||= @data.tail_pointer
|
67
|
+
|
68
|
+
# Enumerate through the data, then tack on the last item.
|
69
|
+
return Enumerator.new do |yielder|
|
70
|
+
@data.enum(start_pointer, end_pointer).each do |data|
|
71
|
+
yielder << data
|
72
|
+
end
|
73
|
+
# Maybe include the last item.
|
74
|
+
if @last_item &&
|
75
|
+
(start_ts <= @last_ts) &&
|
76
|
+
(end_ts.nil? || (@last_ts < end_ts))
|
77
|
+
yielder << @last_item
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Dasht
|
2
|
+
class RackApp
|
3
|
+
attr_accessor :parent
|
4
|
+
|
5
|
+
def initialize(parent)
|
6
|
+
@parent = parent
|
7
|
+
end
|
8
|
+
|
9
|
+
def root_path
|
10
|
+
File.join(File.dirname(__FILE__), "..", "..")
|
11
|
+
end
|
12
|
+
|
13
|
+
def run(port)
|
14
|
+
context = self
|
15
|
+
app = Rack::Builder.new do
|
16
|
+
use Rack::Static, :urls => ["/assets"], :root => context.root_path
|
17
|
+
run lambda { |env|
|
18
|
+
begin
|
19
|
+
context._call(env)
|
20
|
+
rescue => e
|
21
|
+
context.parent.log e
|
22
|
+
raise e
|
23
|
+
end
|
24
|
+
}
|
25
|
+
end
|
26
|
+
Rack::Server.start(:app => app, :Port => (port || 8080))
|
27
|
+
end
|
28
|
+
|
29
|
+
def _call(env)
|
30
|
+
if "/" == env["REQUEST_PATH"] && parent.boards["default"]
|
31
|
+
return ['200', {'Content-Type' => 'text/html'}, [parent.boards["default"].to_html]]
|
32
|
+
end
|
33
|
+
|
34
|
+
/^\/boards\/(.+)$/.match(env["REQUEST_PATH"]) do |match|
|
35
|
+
board = match[1]
|
36
|
+
if parent.boards[board]
|
37
|
+
return ['200', {'Content-Type' => 'text/html'}, [parent.boards[board].to_html]]
|
38
|
+
else
|
39
|
+
return ['404', {'Content-Type' => 'text/html'}, ["Board #{board} not found."]]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
/^\/data$/.match(env["REQUEST_PATH"]) do |match|
|
44
|
+
queries = JSON.parse(env['rack.input'].gets)
|
45
|
+
now = Time.now.to_i
|
46
|
+
data = queries.map do |query|
|
47
|
+
metric = query["metric"]
|
48
|
+
resolution = query["resolution"]
|
49
|
+
periods = query["periods"]
|
50
|
+
ts = now - (resolution * periods)
|
51
|
+
(1..periods).map do |n|
|
52
|
+
parent.metrics.get(metric, ts, ts += resolution) || 0
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
return ['200', {'Content-Type' => 'application/json'}, [data.to_json]]
|
57
|
+
end
|
58
|
+
|
59
|
+
/^\/data\/(.+)/.match(env["REQUEST_PATH"]) do |match|
|
60
|
+
parts = match[1].split('/')
|
61
|
+
metric = parts.shift
|
62
|
+
resolution = parts.shift.to_i
|
63
|
+
periods = (parts.shift || 1).to_i
|
64
|
+
ts = Time.now.to_i - (resolution * periods)
|
65
|
+
data = (1..periods).map do |n|
|
66
|
+
parent.metrics.get(metric, ts, ts += resolution) || 0
|
67
|
+
end
|
68
|
+
return ['200', {'Content-Type' => 'application/json'}, [data.to_json]]
|
69
|
+
end
|
70
|
+
|
71
|
+
return ['404', {'Content-Type' => 'text/html'}, ["Path not found: #{env['REQUEST_PATH']}"]]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Dasht
|
2
|
+
class Reloader
|
3
|
+
attr_accessor :parent
|
4
|
+
|
5
|
+
def initialize(parent)
|
6
|
+
@parent = parent
|
7
|
+
@last_modified = File.mtime($PROGRAM_NAME)
|
8
|
+
end
|
9
|
+
|
10
|
+
def changed?
|
11
|
+
@last_modified != File.mtime($PROGRAM_NAME)
|
12
|
+
end
|
13
|
+
|
14
|
+
def run
|
15
|
+
Thread.new do
|
16
|
+
while true
|
17
|
+
unless changed?
|
18
|
+
sleep 0.3
|
19
|
+
next
|
20
|
+
end
|
21
|
+
parent.log("Reloading #{$PROGRAM_NAME}...")
|
22
|
+
eval(IO.read($PROGRAM_NAME))
|
23
|
+
@last_modified = File.mtime($PROGRAM_NAME)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/screenshot_1.png
ADDED
Binary file
|
data/test/helper.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'simplecov'
|
2
|
+
|
3
|
+
module SimpleCov::Configuration
|
4
|
+
def clean_filters
|
5
|
+
@filters = []
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
SimpleCov.configure do
|
10
|
+
clean_filters
|
11
|
+
load_adapter 'test_frameworks'
|
12
|
+
end
|
13
|
+
|
14
|
+
ENV["COVERAGE"] && SimpleCov.start do
|
15
|
+
add_filter "/.rvm/"
|
16
|
+
end
|
17
|
+
require 'rubygems'
|
18
|
+
require 'bundler'
|
19
|
+
begin
|
20
|
+
Bundler.setup(:default, :development)
|
21
|
+
rescue Bundler::BundlerError => e
|
22
|
+
$stderr.puts e.message
|
23
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
24
|
+
exit e.status_code
|
25
|
+
end
|
26
|
+
require 'test/unit'
|
27
|
+
require 'shoulda'
|
28
|
+
|
29
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
30
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
31
|
+
require 'dasht'
|
32
|
+
|
33
|
+
class Test::Unit::TestCase
|
34
|
+
end
|