dasht 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.agignore +3 -0
- data/Gemfile +14 -0
- data/Gemfile.lock +88 -0
- data/LICENSE.txt +20 -0
- data/README.md +329 -0
- data/Rakefile +51 -0
- data/VERSION +1 -0
- data/assets/css/style.css +123 -0
- data/assets/images/background.jpg +0 -0
- data/assets/js/Chart.min.js +11 -0
- data/assets/js/dasht.js +141 -0
- data/assets/js/jquery-1.11.3.min.js +5 -0
- data/assets/js/masonry.pkgd.min.js +9 -0
- data/assets/js/mustache.min.js +1 -0
- data/assets/js/underscore-min.js +6 -0
- data/assets/plugins/chart.css +11 -0
- data/assets/plugins/chart.html +2 -0
- data/assets/plugins/chart.js +50 -0
- data/assets/plugins/map.css +17 -0
- data/assets/plugins/map.html +4 -0
- data/assets/plugins/map.js +125 -0
- data/assets/plugins/scroll.css +2 -0
- data/assets/plugins/scroll.html +2 -0
- data/assets/plugins/scroll.js +14 -0
- data/assets/plugins/value.css +18 -0
- data/assets/plugins/value.html +4 -0
- data/assets/plugins/value.js +26 -0
- data/examples/simple_heroku_dashboard.rb +34 -0
- data/lib/dasht.rb +25 -0
- data/lib/dasht/array_monkeypatching.rb +5 -0
- data/lib/dasht/base.rb +117 -0
- data/lib/dasht/board.rb +108 -0
- data/lib/dasht/collector.rb +33 -0
- data/lib/dasht/list.rb +93 -0
- data/lib/dasht/log_thread.rb +94 -0
- data/lib/dasht/metric.rb +82 -0
- data/lib/dasht/rack_app.rb +74 -0
- data/lib/dasht/reloader.rb +28 -0
- data/screenshot_1.png +0 -0
- data/test/helper.rb +34 -0
- data/test/test_list.rb +65 -0
- data/test/test_metric.rb +58 -0
- data/tests.rb +6 -0
- data/views/dashboard.erb +27 -0
- metadata +189 -0
@@ -0,0 +1,33 @@
|
|
1
|
+
module Dasht
|
2
|
+
class Metrics
|
3
|
+
attr_accessor :parent
|
4
|
+
def initialize(parent)
|
5
|
+
@parent = parent
|
6
|
+
@metric_values = {}
|
7
|
+
@metric_operations = {}
|
8
|
+
end
|
9
|
+
|
10
|
+
def set(metric, value, op, ts)
|
11
|
+
metric = metric.to_s
|
12
|
+
@metric_operations[metric] = op
|
13
|
+
m = (@metric_values[metric] ||= Metric.new)
|
14
|
+
m.append(value, ts) do |old_value, new_value|
|
15
|
+
[old_value, new_value].compact.flatten.send(op)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def get(metric, start_ts, end_ts)
|
20
|
+
metric = metric.to_s
|
21
|
+
m = @metric_values[metric]
|
22
|
+
return [] if m.nil?
|
23
|
+
op = @metric_operations[metric]
|
24
|
+
m.enum(start_ts, end_ts).to_a.flatten.send(op)
|
25
|
+
end
|
26
|
+
|
27
|
+
def trim_to(ts)
|
28
|
+
@metric_values.each do |k, v|
|
29
|
+
v.trim_to(ts)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/dasht/list.rb
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
# Dasht::List - Simple list structure following properties:
|
2
|
+
#
|
3
|
+
# 1. Fast writes. Appends to a list.
|
4
|
+
# 2. Fast reads by index. Indexed by position in the list.
|
5
|
+
# 3. Fast deletes that preserve indexes. Removes items from the front
|
6
|
+
# of the list.
|
7
|
+
# 4. Simple (but not necessarily fast) aggregation. Enumerate values
|
8
|
+
# between pointers.
|
9
|
+
#
|
10
|
+
# The Dasht::List structure is formed using a Ruby Array (values),
|
11
|
+
# plus a counter of how many items have been deleted (offset).
|
12
|
+
# Whenever data is deleted from the head of the list, the offset is
|
13
|
+
# incremented.
|
14
|
+
|
15
|
+
module Dasht
|
16
|
+
class List
|
17
|
+
attr_accessor :values
|
18
|
+
attr_accessor :offset
|
19
|
+
|
20
|
+
def initialize
|
21
|
+
@offset = 0
|
22
|
+
@values = []
|
23
|
+
end
|
24
|
+
|
25
|
+
def to_s
|
26
|
+
return @values.to_s
|
27
|
+
end
|
28
|
+
|
29
|
+
# Public: Get a pointer to the first value.
|
30
|
+
def head_pointer
|
31
|
+
return offset
|
32
|
+
end
|
33
|
+
|
34
|
+
# Public: Get a pointer to right after the last value.
|
35
|
+
def tail_pointer
|
36
|
+
return offset + @values.length
|
37
|
+
end
|
38
|
+
|
39
|
+
# Public: Get the value at a given pointer, or nil if the pointer
|
40
|
+
# is no longer valid.
|
41
|
+
def get(pointer)
|
42
|
+
index = _pointer_to_index(pointer)
|
43
|
+
return @values[index]
|
44
|
+
end
|
45
|
+
|
46
|
+
# Public: Return an enumerator that walks through the list, yielding
|
47
|
+
# data.
|
48
|
+
def enum(start_pointer = nil, end_pointer = nil)
|
49
|
+
index = _pointer_to_index(start_pointer || head_pointer)
|
50
|
+
end_index = _pointer_to_index(end_pointer || tail_pointer)
|
51
|
+
return Enumerator.new do |yielder|
|
52
|
+
while index < end_index
|
53
|
+
yielder << @values[index]
|
54
|
+
index += 1
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# Public: Add data to the list.
|
60
|
+
# Returns a pointer to the new data.
|
61
|
+
def append(data)
|
62
|
+
pointer = self.tail_pointer
|
63
|
+
@values << data
|
64
|
+
return pointer
|
65
|
+
end
|
66
|
+
|
67
|
+
# Public: Remove data up to (but not including) the specified pointer.
|
68
|
+
def trim_to(pointer)
|
69
|
+
return if pointer.nil?
|
70
|
+
index = _pointer_to_index(pointer)
|
71
|
+
@offset += index
|
72
|
+
@values = @values.slice(index, @values.length)
|
73
|
+
return
|
74
|
+
end
|
75
|
+
|
76
|
+
# Public: Walk through the list, removing links from the list while
|
77
|
+
# the block returns true. Stop when it returns false.
|
78
|
+
def trim_while(&block)
|
79
|
+
while (@values.length > 0) && yield(@values.first)
|
80
|
+
@values.shift
|
81
|
+
@offset += 1
|
82
|
+
end
|
83
|
+
return
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
# Convert a pointer to an index in the list.
|
89
|
+
def _pointer_to_index(pointer)
|
90
|
+
return [pointer - offset, 0].max
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
module Dasht
|
2
|
+
class LogThread
|
3
|
+
attr_accessor :parent
|
4
|
+
|
5
|
+
def self.update_global_stats(line)
|
6
|
+
@total_lines ||= 0
|
7
|
+
@total_bytes ||= 0
|
8
|
+
@total_lines += 1
|
9
|
+
@total_bytes += line.length
|
10
|
+
print "\rConsumed #{@total_lines} lines (#{@total_bytes} bytes)..."
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(parent, command)
|
14
|
+
@parent = parent
|
15
|
+
@command = command
|
16
|
+
@event_definitions = []
|
17
|
+
end
|
18
|
+
|
19
|
+
def run
|
20
|
+
parent.log "Starting `#{@command}`..."
|
21
|
+
@thread = Thread.new do
|
22
|
+
begin
|
23
|
+
while true
|
24
|
+
begin
|
25
|
+
IO.popen(@command) do |process|
|
26
|
+
process.each do |line|
|
27
|
+
_consume_line(line)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
rescue => e
|
31
|
+
parent.log e
|
32
|
+
end
|
33
|
+
sleep 2
|
34
|
+
end
|
35
|
+
rescue => e
|
36
|
+
parent.log e
|
37
|
+
raise e
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def event(metric, regex, op, value = nil, &block)
|
43
|
+
@event_definitions << [metric, regex, op, value, block]
|
44
|
+
end
|
45
|
+
|
46
|
+
def count(metric, regex, &block)
|
47
|
+
event(metric, regex, :dasht_sum, 1, &block)
|
48
|
+
end
|
49
|
+
|
50
|
+
def gauge(metric, regex, &block)
|
51
|
+
event(metric, regex, :last, nil, &block)
|
52
|
+
end
|
53
|
+
|
54
|
+
def min(metric, regex, &block)
|
55
|
+
event(metric, regex, :min, nil, &block)
|
56
|
+
end
|
57
|
+
|
58
|
+
def max(metric, regex, &block)
|
59
|
+
event(metric, regex, :max, nil, &block)
|
60
|
+
end
|
61
|
+
|
62
|
+
def append(metric, regex, &block)
|
63
|
+
event(metric, regex, :to_a, nil, &block)
|
64
|
+
end
|
65
|
+
|
66
|
+
def unique(metric, regex, &block)
|
67
|
+
event(metric, regex, :uniq, nil, &block)
|
68
|
+
end
|
69
|
+
|
70
|
+
def terminate
|
71
|
+
@thread.terminate
|
72
|
+
end
|
73
|
+
|
74
|
+
private
|
75
|
+
|
76
|
+
def _consume_line(line)
|
77
|
+
self.class.update_global_stats(line)
|
78
|
+
ts = Time.now.to_i
|
79
|
+
@event_definitions.each do |metric, regex, op, value, block|
|
80
|
+
begin
|
81
|
+
regex.match(line) do |matches|
|
82
|
+
value = matches[0] if value.nil?
|
83
|
+
value = block.call(matches) if block
|
84
|
+
parent.metrics.set(metric, value, op, ts) if value
|
85
|
+
end
|
86
|
+
rescue => e
|
87
|
+
parent.log e
|
88
|
+
raise e
|
89
|
+
end
|
90
|
+
parent.metrics.trim_to(ts - (parent.history || (60 * 60)))
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
data/lib/dasht/metric.rb
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
# Dasht::Metric - Simple in-memory time-series data structure with the
|
2
|
+
# following properties:
|
3
|
+
#
|
4
|
+
# 1. Sparse. Only stores time stamps for intervals with known data,
|
5
|
+
# and only stores one timestamp per interval.
|
6
|
+
# 2. Flexible aggregation using Ruby blocks during both read and
|
7
|
+
# write.
|
8
|
+
# 3. Read values between two timestamps.
|
9
|
+
#
|
10
|
+
# The Dasht::Metric structure is formed using two Dasht::List
|
11
|
+
# objects. One object tracks data, the other object tracks a list of
|
12
|
+
# checkpoints and their corresponding index into the data.
|
13
|
+
module Dasht
|
14
|
+
class Metric
|
15
|
+
attr_reader :data, :checkpoints
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
@checkpoints = List.new
|
19
|
+
@data = List.new
|
20
|
+
@last_item = nil
|
21
|
+
@last_ts = nil
|
22
|
+
end
|
23
|
+
|
24
|
+
def to_s
|
25
|
+
return @data.to_s + " (last: #{@last_item})"
|
26
|
+
end
|
27
|
+
|
28
|
+
def append(data, ts, &block)
|
29
|
+
# Maybe checkpoint the time.
|
30
|
+
if @last_ts == ts
|
31
|
+
@last_item = yield(@last_item, data)
|
32
|
+
else
|
33
|
+
if @last_ts
|
34
|
+
pointer = @data.append(@last_item)
|
35
|
+
@checkpoints.append([@last_ts, pointer])
|
36
|
+
end
|
37
|
+
@last_ts = ts
|
38
|
+
@last_item = nil
|
39
|
+
@last_item = yield(@last_item, data)
|
40
|
+
end
|
41
|
+
return
|
42
|
+
end
|
43
|
+
|
44
|
+
def trim_to(ts)
|
45
|
+
pointer = nil
|
46
|
+
@checkpoints.trim_while do |s, p|
|
47
|
+
pointer = p
|
48
|
+
s < ts
|
49
|
+
end
|
50
|
+
@data.trim_to(pointer)
|
51
|
+
return
|
52
|
+
end
|
53
|
+
|
54
|
+
def enum(start_ts, end_ts = nil)
|
55
|
+
# Get a pointer to our location in the data.
|
56
|
+
start_pointer = nil
|
57
|
+
end_pointer = nil
|
58
|
+
prev_p = nil
|
59
|
+
@checkpoints.enum.each do |s, p|
|
60
|
+
start_pointer ||= p if start_ts <= s
|
61
|
+
end_pointer ||= prev_p if end_ts && end_ts <= s
|
62
|
+
break if start_pointer && (end_ts.nil? || end_pointer)
|
63
|
+
prev_p = p
|
64
|
+
end
|
65
|
+
start_pointer ||= @data.tail_pointer
|
66
|
+
end_pointer ||= @data.tail_pointer
|
67
|
+
|
68
|
+
# Enumerate through the data, then tack on the last item.
|
69
|
+
return Enumerator.new do |yielder|
|
70
|
+
@data.enum(start_pointer, end_pointer).each do |data|
|
71
|
+
yielder << data
|
72
|
+
end
|
73
|
+
# Maybe include the last item.
|
74
|
+
if @last_item &&
|
75
|
+
(start_ts <= @last_ts) &&
|
76
|
+
(end_ts.nil? || (@last_ts < end_ts))
|
77
|
+
yielder << @last_item
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Dasht
|
2
|
+
class RackApp
|
3
|
+
attr_accessor :parent
|
4
|
+
|
5
|
+
def initialize(parent)
|
6
|
+
@parent = parent
|
7
|
+
end
|
8
|
+
|
9
|
+
def root_path
|
10
|
+
File.join(File.dirname(__FILE__), "..", "..")
|
11
|
+
end
|
12
|
+
|
13
|
+
def run(port)
|
14
|
+
context = self
|
15
|
+
app = Rack::Builder.new do
|
16
|
+
use Rack::Static, :urls => ["/assets"], :root => context.root_path
|
17
|
+
run lambda { |env|
|
18
|
+
begin
|
19
|
+
context._call(env)
|
20
|
+
rescue => e
|
21
|
+
context.parent.log e
|
22
|
+
raise e
|
23
|
+
end
|
24
|
+
}
|
25
|
+
end
|
26
|
+
Rack::Server.start(:app => app, :Port => (port || 8080))
|
27
|
+
end
|
28
|
+
|
29
|
+
def _call(env)
|
30
|
+
if "/" == env["REQUEST_PATH"] && parent.boards["default"]
|
31
|
+
return ['200', {'Content-Type' => 'text/html'}, [parent.boards["default"].to_html]]
|
32
|
+
end
|
33
|
+
|
34
|
+
/^\/boards\/(.+)$/.match(env["REQUEST_PATH"]) do |match|
|
35
|
+
board = match[1]
|
36
|
+
if parent.boards[board]
|
37
|
+
return ['200', {'Content-Type' => 'text/html'}, [parent.boards[board].to_html]]
|
38
|
+
else
|
39
|
+
return ['404', {'Content-Type' => 'text/html'}, ["Board #{board} not found."]]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
/^\/data$/.match(env["REQUEST_PATH"]) do |match|
|
44
|
+
queries = JSON.parse(env['rack.input'].gets)
|
45
|
+
now = Time.now.to_i
|
46
|
+
data = queries.map do |query|
|
47
|
+
metric = query["metric"]
|
48
|
+
resolution = query["resolution"]
|
49
|
+
periods = query["periods"]
|
50
|
+
ts = now - (resolution * periods)
|
51
|
+
(1..periods).map do |n|
|
52
|
+
parent.metrics.get(metric, ts, ts += resolution) || 0
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
return ['200', {'Content-Type' => 'application/json'}, [data.to_json]]
|
57
|
+
end
|
58
|
+
|
59
|
+
/^\/data\/(.+)/.match(env["REQUEST_PATH"]) do |match|
|
60
|
+
parts = match[1].split('/')
|
61
|
+
metric = parts.shift
|
62
|
+
resolution = parts.shift.to_i
|
63
|
+
periods = (parts.shift || 1).to_i
|
64
|
+
ts = Time.now.to_i - (resolution * periods)
|
65
|
+
data = (1..periods).map do |n|
|
66
|
+
parent.metrics.get(metric, ts, ts += resolution) || 0
|
67
|
+
end
|
68
|
+
return ['200', {'Content-Type' => 'application/json'}, [data.to_json]]
|
69
|
+
end
|
70
|
+
|
71
|
+
return ['404', {'Content-Type' => 'text/html'}, ["Path not found: #{env['REQUEST_PATH']}"]]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Dasht
|
2
|
+
class Reloader
|
3
|
+
attr_accessor :parent
|
4
|
+
|
5
|
+
def initialize(parent)
|
6
|
+
@parent = parent
|
7
|
+
@last_modified = File.mtime($PROGRAM_NAME)
|
8
|
+
end
|
9
|
+
|
10
|
+
def changed?
|
11
|
+
@last_modified != File.mtime($PROGRAM_NAME)
|
12
|
+
end
|
13
|
+
|
14
|
+
def run
|
15
|
+
Thread.new do
|
16
|
+
while true
|
17
|
+
unless changed?
|
18
|
+
sleep 0.3
|
19
|
+
next
|
20
|
+
end
|
21
|
+
parent.log("Reloading #{$PROGRAM_NAME}...")
|
22
|
+
eval(IO.read($PROGRAM_NAME))
|
23
|
+
@last_modified = File.mtime($PROGRAM_NAME)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/screenshot_1.png
ADDED
Binary file
|
data/test/helper.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'simplecov'
|
2
|
+
|
3
|
+
module SimpleCov::Configuration
|
4
|
+
def clean_filters
|
5
|
+
@filters = []
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
SimpleCov.configure do
|
10
|
+
clean_filters
|
11
|
+
load_adapter 'test_frameworks'
|
12
|
+
end
|
13
|
+
|
14
|
+
ENV["COVERAGE"] && SimpleCov.start do
|
15
|
+
add_filter "/.rvm/"
|
16
|
+
end
|
17
|
+
require 'rubygems'
|
18
|
+
require 'bundler'
|
19
|
+
begin
|
20
|
+
Bundler.setup(:default, :development)
|
21
|
+
rescue Bundler::BundlerError => e
|
22
|
+
$stderr.puts e.message
|
23
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
24
|
+
exit e.status_code
|
25
|
+
end
|
26
|
+
require 'test/unit'
|
27
|
+
require 'shoulda'
|
28
|
+
|
29
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
30
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
31
|
+
require 'dasht'
|
32
|
+
|
33
|
+
class Test::Unit::TestCase
|
34
|
+
end
|