metric_system 0.1.0 → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c76a2843de05d5ccaaa70865f23e76b09b287dc3
4
- data.tar.gz: 34156de4921e7fb09874a3d47bedc4174ca4985b
3
+ metadata.gz: d1483853542b6eb79dc9c7a0e349591d2ee14d52
4
+ data.tar.gz: 9f50ee6bc48e72b83002aaf165576ce586414873
5
5
  SHA512:
6
- metadata.gz: 13f1207dee0f2cfb89850d1b637f4015abc0507231b332640848f88d6295e30ca222d59c8df2ae972139f01a184c04c1af997f4752151eb6a2cb709701e06cc9
7
- data.tar.gz: 89948cf352c4e282b5e0bb7c91dc3f6bca221312767bec1a84f72c5254fa3aa7952d0f2abef10366750a101b2e1f0891526df258d4cb7f60c4f8bf6717090eeb
6
+ metadata.gz: 422740c79d19c46fce188e3b33d6cdbf45fc34c79ab37cda1560dfeae0aed607411b6678d0a6085ba6e0569249ec50ac91967fa8bb5bce38bdb048195cfa60b4
7
+ data.tar.gz: 03deb35f0a2baa9a5842de40355cd8118120a49ec371e900491d458455bab093acbac6bfecc2dd1a34829bbb5122ce47294593ff7c40e3d120959d732bd34475
data/README.md CHANGED
@@ -1,7 +1,11 @@
1
1
  # metric_system
2
2
 
3
3
  A simple metric collector
4
-
4
+
5
+ **Note!**
6
+
7
+ This is alpha software. Use at your own risk!
8
+
5
9
  ## Installation
6
10
 
7
11
  gem install metric_system
@@ -0,0 +1,164 @@
1
+ class MetricSystem::Database
2
+ def initialize(path)
3
+ require_relative "./sqlite3_extensions"
4
+ open path
5
+ end
6
+
7
+ extend Forwardable
8
+
9
+ delegate [:exec, :select, :transaction, :rollback, :print, :run, :ask, :register] => :"@db"
10
+
11
+ PERIODS = [
12
+ [ :year, 31536000, "strftime('%Y-01-01', starts_at, 'unixepoch')" ],
13
+ [ :month, 2592000, "strftime('%Y-%m-01', starts_at, 'unixepoch')" ],
14
+ [ :week, 604800, "strftime('%Y-%m-%d', starts_at, 'unixepoch', 'weekday 1', '-7 days')" ],
15
+ [ :day, 86400, "strftime('%Y-%m-%d', starts_at, 'unixepoch')" ],
16
+ [ :hour, 3600, "strftime('%Y-%m-%d %H:00:00', starts_at, 'unixepoch')" ],
17
+ [ :minute, 60, "strftime('%Y-%m-%d %H:%M:00', starts_at, 'unixepoch')" ],
18
+ # [ :second, 1, "strftime('%Y-%m-%d %H:%M:%S', starts_at, 'unixepoch')" ],
19
+ ]
20
+
21
+ private
22
+
23
+ def open(path)
24
+ @db = SQLite3::Database.new(path)
25
+
26
+ [ :counters, :gauges ].each do |name|
27
+ exec <<-SQL
28
+ CREATE TABLE IF NOT EXISTS #{name}(
29
+ id INTEGER PRIMARY KEY,
30
+
31
+ name NOT NULL, -- the event name
32
+ value NOT NULL, -- the value
33
+ starts_at TIMESTAMP NOT NULL DEFAULT (strftime('%s','now')) -- the timestamp
34
+ );
35
+
36
+ CREATE INDEX IF NOT EXISTS #{name}_idx1 ON #{name}(name, starts_at);
37
+
38
+ CREATE TABLE IF NOT EXISTS aggregated_#{name}(
39
+ id INTEGER PRIMARY KEY,
40
+
41
+ name NOT NULL, -- the event name
42
+ starts_at TIMESTAMP NOT NULL, -- the start-at timestamp
43
+ duration NOT NULL, -- the duration (estimate, in secs.)
44
+ period, -- the name of the period (year, day, etc.)
45
+ sum, -- the sum of event values
46
+ count, -- the count of events
47
+ value -- the aggregated value
48
+ );
49
+
50
+ CREATE UNIQUE INDEX IF NOT EXISTS aggregated_#{name}_uidx1 ON aggregated_#{name}(name, starts_at, duration);
51
+ CREATE INDEX IF NOT EXISTS aggregated_#{name}_idx2 ON aggregated_#{name}(starts_at);
52
+ CREATE INDEX IF NOT EXISTS aggregated_#{name}_idx3 ON aggregated_#{name}(duration);
53
+ SQL
54
+ end
55
+
56
+ exec <<-SQL
57
+ PRAGMA synchronous = NORMAL;
58
+ PRAGMA journal_mode = WAL;
59
+
60
+ CREATE VIEW IF NOT EXISTS aggregates AS
61
+ SELECT * FROM aggregated_gauges
62
+ UNION
63
+ SELECT * FROM aggregated_counters
64
+ SQL
65
+ end
66
+
67
+ public
68
+
69
+ def add_event(table, name, value, starts_at)
70
+ # get names of all related events. An event "a.b.c" is actually
71
+ # 3 events: "a", "a.b", "a.b.c"
72
+ names = begin
73
+ parts = name.split(".")
74
+ parts.length.downto(1).map do |cnt|
75
+ parts[0,cnt].join(".")
76
+ end
77
+ end
78
+
79
+ if starts_at
80
+ starts_at = Time.parse(starts_at) if starts_at.is_a?(String)
81
+
82
+ names.each do |name|
83
+ run "INSERT INTO #{table}(name, value, starts_at) VALUES(?, ?, ?)", name, value, starts_at.to_i
84
+ end
85
+ else
86
+ names.each do |name|
87
+ run "INSERT INTO #{table}(name, value) VALUES(?, ?)", name, value
88
+ end
89
+ end
90
+ rescue
91
+ STDERR.puts "#{$!}: #{table} #{name.inspect}, #{value.inspect}"
92
+ end
93
+
94
+ PERIODS_BY_KEY = PERIODS.by(&:first)
95
+
96
+ def aggregate(*keys)
97
+ if keys.empty?
98
+ keys = PERIODS.map(&:first)
99
+ end
100
+
101
+ transaction do
102
+ keys.each do |period|
103
+ aggregate_for_period :period => period, :source => :counters, :dest => :aggregated_counters, :aggregate => "sum"
104
+ end
105
+
106
+ @db.exec "DELETE FROM counters"
107
+ end
108
+
109
+ transaction do
110
+ keys.each do |period|
111
+ aggregate_for_period :period => period, :source => :gauges, :dest => :aggregated_gauges, :aggregate => "CAST(sum AS FLOAT) / count"
112
+ end
113
+
114
+ @db.exec "DELETE FROM gauges"
115
+ end
116
+ end
117
+
118
+ private
119
+
120
+ def aggregate_for_period(options)
121
+ expect! options => {
122
+ :period => PERIODS.map(&:first)
123
+ }
124
+ period, source, dest, aggregate = options.values_at :period, :source, :dest, :aggregate
125
+
126
+ _, duration, starts_at = PERIODS_BY_KEY[period]
127
+
128
+ # sql expression to calculate value from sum and count of event values
129
+ aggregate = source == :gauges ? "CAST(sum AS FLOAT) / count" : "sum"
130
+
131
+ @db.exec <<-SQL
132
+ CREATE TEMPORARY TABLE batch AS
133
+ SELECT name, starts_at, SUM(sum) AS sum, SUM(count) AS count FROM
134
+ (
135
+ SELECT name AS name,
136
+ #{starts_at} AS starts_at,
137
+ SUM(value) AS sum,
138
+ COUNT(value) AS count
139
+ FROM #{source}
140
+ GROUP BY name, starts_at
141
+
142
+ UNION
143
+
144
+ SELECT #{dest}.name AS name,
145
+ #{dest}.starts_at AS starts_at,
146
+ sum AS sum,
147
+ count AS count
148
+ FROM #{dest}
149
+ -- INNER JOIN #{source} ON #{dest}.name=#{source}.name
150
+ WHERE duration=#{duration}
151
+ AND #{dest}.starts_at >= (SELECT MIN(starts_at) FROM #{source})
152
+ )
153
+ GROUP BY name, starts_at;
154
+
155
+ INSERT OR REPLACE INTO #{dest}(name, starts_at, period, duration, sum, count, value)
156
+ SELECT name, starts_at, '#{period}', #{duration}, sum, count, #{aggregate}
157
+ FROM batch;
158
+ SQL
159
+
160
+ @db.exec <<-SQL
161
+ DROP TABLE batch;
162
+ SQL
163
+ end
164
+ end
@@ -0,0 +1,15 @@
1
+ class MetricSystem::IO
2
+ def initialize(io)
3
+ expect! io => ::IO
4
+ @io = io
5
+ end
6
+
7
+ def add_event(table, name, value, starts_at)
8
+ starts_at = Time.now unless starts_at
9
+ @io.puts "#{table} #{name} #{value} #{starts_at}"
10
+ end
11
+
12
+ def quit_server!
13
+ @io.puts "SHUTDOWN:SERVER"
14
+ end
15
+ end
@@ -0,0 +1,126 @@
1
+ require 'eventmachine'
2
+ #require 'eventmachine/timer'
3
+
4
+ module MetricSystem::Server
5
+ include EM::P::LineProtocol
6
+ extend self
7
+
8
+ module Buffer
9
+ extend self
10
+
11
+ def buffer
12
+ @buffer ||= []
13
+ end
14
+
15
+ def take
16
+ taken, @buffer = @buffer, []
17
+ taken
18
+ end
19
+
20
+ def push(event)
21
+ buffer << event
22
+ end
23
+
24
+ def length
25
+ buffer.length
26
+ end
27
+ end
28
+
29
+ class Event < Struct.new(:table, :name, :value, :time)
30
+ def self.parse(line)
31
+ table, name, value, time, remainder = line.split(" ", 5)
32
+ value = value.to_f
33
+ time = time ? time.to_i : Time.now.to_i
34
+ new(table, name, value, time)
35
+ end
36
+ end
37
+
38
+ def receive_line(line)
39
+ if line == "SHUTDOWN:SERVER"
40
+ MetricSystem::Server.shutdown
41
+ return
42
+ end
43
+
44
+ return if MetricSystem::Server.shutting_down?
45
+
46
+ return unless event = Event.parse(line)
47
+
48
+ Buffer.push event
49
+
50
+ MetricSystem::Server.flush if Buffer.length % 1000 == 0
51
+ rescue
52
+ STDERR.puts "#{$!}, from\n\t" + $!.backtrace.join("\t")
53
+ end
54
+
55
+ def self.flush
56
+ return unless Buffer.length > 0
57
+
58
+ if @busy
59
+ STDERR.puts " Waiting for writer: backlog is #{Buffer.length} entries"
60
+ return
61
+ end
62
+
63
+ @busy = true
64
+
65
+ events = Buffer.take
66
+
67
+ operation = proc {
68
+ flush_events events
69
+ }
70
+ callback = proc {
71
+ @busy = nil
72
+
73
+ if shutting_down?
74
+ flush_events(Buffer.take)
75
+ EM.stop
76
+ end
77
+ }
78
+ EventMachine.defer operation, callback
79
+ rescue
80
+ STDERR.puts "#{$!}, from\n\t" + $!.backtrace.join("\t")
81
+ end
82
+
83
+ def self.flush_events(events)
84
+ return if events.empty?
85
+
86
+ starts_at = Time.now
87
+ MetricSystem.transaction do
88
+ events.each do |event|
89
+ MetricSystem.add_event event.table, event.name, event.value, event.time
90
+ end
91
+ end
92
+ STDERR.puts " Writing #{events.count} events: %.3f secs" % (Time.now - starts_at)
93
+
94
+ starts_at = Time.now
95
+ MetricSystem.aggregate
96
+ STDERR.puts " Merging #{events.count} events: %.3f secs" % (Time.now - starts_at)
97
+ end
98
+
99
+ # Note that this will block current thread.
100
+ def self.run(db, socket_path, options = {})
101
+ @options = options || {}
102
+ MetricSystem.target = db
103
+
104
+ STDERR.puts "Starting server at socket: #{socket_path}"
105
+
106
+ EventMachine.run {
107
+ EventMachine::PeriodicTimer.new(1) do
108
+ MetricSystem::Server.flush
109
+ end
110
+
111
+ EventMachine.start_server socket_path, MetricSystem::Server
112
+ }
113
+ end
114
+
115
+ def self.shutting_down?
116
+ @shutting_down
117
+ end
118
+
119
+ def self.shutdown
120
+ return if shutting_down?
121
+ return unless @options[:quit_server]
122
+
123
+ @shutting_down = true
124
+ MetricSystem::Server.flush
125
+ end
126
+ end
@@ -1,158 +1,4 @@
1
1
  require "sqlite3"
2
-
3
- # manage SQLite3::Records
4
- #
5
- # The SQLite3::Record module is able to generate classes that are optimized
6
- # for a specific set of columns. It is build on top of Struct, which is way
7
- # faster than Hashes, for example.
8
- module SQLite3::Record
9
- module ClassMethods
10
- attr :columns, true
11
-
12
- private
13
-
14
- def to_time(s)
15
- case s
16
- when String then Time.parse(s)
17
- when Fixnum then Time.at(s)
18
- else s
19
- end
20
- end
21
-
22
- def to_date(s)
23
- return unless time = to_time(s)
24
- time.to_date
25
- end
26
-
27
- public
28
-
29
- def build(*attrs)
30
- attrs = columns.zip(attrs).map do |key, value|
31
- case key
32
- when /_at$/ then to_time(value)
33
- when /_on$/ then to_date(value)
34
- else value
35
- end
36
- end
37
-
38
- new *attrs
39
- end
40
- end
41
-
42
- def to_a
43
- self.class.columns.map do |column| send(column) end
44
- end
45
-
46
- def self.for_columns(columns)
47
- columns = columns.map(&:to_sym)
48
-
49
- @@classes ||= {}
50
- @@classes[columns] ||= begin
51
- struct = Struct.new(*columns)
52
- struct.extend SQLite3::Record::ClassMethods
53
- struct.include SQLite3::Record
54
-
55
- struct.columns = columns
56
- struct
57
- end
58
- end
59
- end
60
-
61
- class SQLite3::Query
62
- def initialize(sql, statement)
63
- expect! statement => SQLite3::Statement
64
-
65
- @sql, @statement = sql, statement
66
- end
67
-
68
- def run(*args)
69
- # STDERR.puts "Q: #{@sql} #{args.map(&:inspect).join(", ")}"
70
- @statement.execute *args
71
- end
72
-
73
- def select(*args)
74
- @klass ||= SQLite3::Record.for_columns(@statement.columns)
75
-
76
- run(*args).map do |rec|
77
- @klass.build *rec
78
- end
79
- end
80
-
81
- def ask(*args)
82
- results = run(*args)
83
- row = results.first
84
- results.reset
85
-
86
- if !row then nil
87
- elsif row.length == 1 then row.first
88
- else row
89
- end
90
- end
91
- end
92
-
93
- class SQLite3::Database
94
- # execute multiple SQL statements at once.
95
- def exec(sql, *args)
96
- args = prepare_arguments(args)
97
-
98
- while sql =~ /\S/ do
99
- statement = prepare(sql)
100
-
101
- sql = statement.remainder
102
- if statement.active?
103
- statement.execute!(*args)
104
- end
105
- end
106
-
107
- rescue
108
- STDERR.puts "#{sql}: #{$!}"
109
- raise
110
- end
111
-
112
- # -- cached queries ---------------------------------------------------------
113
-
114
- private
115
-
116
- def query(sql)
117
- @queries ||= {}
118
- @queries[sql] ||= SQLite3::Query.new sql, prepare(sql)
119
- end
120
-
121
- def prepare_arguments(args)
122
- args.map do |arg|
123
- case arg
124
- when Time then arg.to_i
125
- when Date then arg.to_time.to_i
126
- else arg
127
- end
128
- end
129
- end
130
-
131
- public
132
-
133
- def run(sql, *args)
134
- query(sql).run *prepare_arguments(args)
135
- end
136
-
137
- def ask(sql, *args)
138
- query(sql).ask *prepare_arguments(args)
139
- end
140
-
141
- # run a select like query. Returns an array of records.
142
- def select(sql, *args)
143
- query(sql).select *prepare_arguments(args)
144
- end
145
-
146
- def print(sql, *args)
147
- results = select sql, *args
148
- log_sql = sql.gsub(/\n/, " ").gsub(/\s+/, " ")
149
- puts "=" * log_sql.length
150
- puts log_sql
151
- puts "-" * log_sql.length
152
-
153
- results.each do |result|
154
- pp result.to_a
155
- end
156
- puts "=" * log_sql.length
157
- end
158
- end
2
+ require "sqlite3/database_extension"
3
+ require "sqlite3/query"
4
+ require "sqlite3/record"
@@ -1,3 +1,3 @@
1
- class MetricSystem
2
- VERSION = "0.1.0"
1
+ module MetricSystem
2
+ VERSION = "0.1.2"
3
3
  end
@@ -0,0 +1,163 @@
1
+ <html>
2
+ <head>
3
+ <!--Load the AJAX API-->
4
+ <script type="text/javascript" src="https://www.google.com/jsapi"></script>
5
+ <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
6
+ <script type="text/javascript">
7
+
8
+ // Load the Visualization API and the piechart package.
9
+ google.load('visualization', '1', {'packages':['corechart', 'table', 'annotationchart']});
10
+
11
+ $.fn.visualize = function() {
12
+ var CHARTS = {
13
+ pie: google.visualization.PieChart,
14
+ area: google.visualization.AreaChart,
15
+ bar: google.visualization.BarChart,
16
+ column: google.visualization.ColumnChart,
17
+ annotation: google.visualization.AnnotationChart
18
+ }
19
+
20
+ this.each(function(idx, node) {
21
+ node = $(node);
22
+
23
+ var ctor = CHARTS[$(node).data("chart")];
24
+ if(!ctor) ctor = google.visualization.Table;
25
+
26
+ var chart = new ctor(node[0]); // the chart object
27
+ var data = null; // the chart's data, will be loaded from url
28
+
29
+ var url = $(node).data("src");
30
+ if(url) {
31
+ loadDataFromURL();
32
+ }
33
+ else {
34
+ var raw_data = eval("r = " + node.text());
35
+ node.html("");
36
+ data = new google.visualization.DataTable(raw_data);
37
+ redraw();
38
+
39
+ }
40
+
41
+ // automatically redraw when resizing.
42
+ (function() {
43
+ var timeout = null;
44
+
45
+ $(window).resize(function() {
46
+ if(timeout) clearTimeout(timeout);
47
+ timeout = setTimeout(redraw, 50);
48
+ });
49
+ })();
50
+
51
+ // redraw();
52
+
53
+ // -- callbacks and helpers ---------------------------------------------
54
+
55
+ // (re)draw chart
56
+ function redraw() {
57
+ chart.draw(data, {width: node.width(), height: node.height()});
58
+ }
59
+
60
+ // fetch data from \a url, never run two requests in parallel.
61
+ var xhrRequest = null;
62
+ function loadDataFromURL(callback) {
63
+ if(xhrRequest) return;
64
+
65
+ xhrRequest = $.ajax({url: url, dataType: "text",
66
+ success: function(js) {
67
+ xhrRequest = null;
68
+ var raw_data = eval("r = " + js);
69
+ data = new google.visualization.DataTable(raw_data);
70
+ redraw();
71
+ }
72
+ });
73
+ }
74
+ });
75
+ };
76
+
77
+ $(function() {
78
+ $('.google-visualization').visualize();
79
+ });
80
+ </script>
81
+ </head>
82
+
83
+ <body>
84
+ <style type="text/css" media="screen">
85
+ @import url(http://fonts.googleapis.com/css?family=Open+Sans:400,700);
86
+
87
+ h1, h2, h3, h4 {
88
+ font-weight: 700;
89
+ font-family: 'Open Sans', sans-serif;
90
+ }
91
+
92
+ body {
93
+ padding: 0;
94
+ margin: 0;
95
+ }
96
+
97
+ .google-visualization {
98
+ overflow: auto;
99
+ height: 300px;
100
+ width: 400px;
101
+ border: 1px solid red;
102
+ }
103
+ #gallery > div {
104
+ width: 400px;
105
+ display: inline-block;
106
+ position: relative;
107
+ }
108
+ #gallery > div > h3 {
109
+ position: relative;
110
+ top: 0;
111
+ }
112
+
113
+ </style>
114
+
115
+ <div id="gallery">
116
+ <div>
117
+ <h3>Table</h3>
118
+ <div class="google-visualization">
119
+ <%=
120
+ select "SELECT date(starts_at), value FROM aggregates WHERE period='day'"
121
+ %>
122
+ </div>
123
+ </div>
124
+ <div>
125
+ <h3>Pie Chart</h3>
126
+ <div class="google-visualization" data-src="/value_by_day_name.js" data-chart="pie">
127
+ </div>
128
+ </div>
129
+ <div>
130
+ <h3>Area Chart</h3>
131
+ <div class="google-visualization" data-chart="area">
132
+ <%=
133
+ select "SELECT date(starts_at), value FROM aggregates WHERE period='day'"
134
+ %>
135
+ </div>
136
+ </div>
137
+ <div>
138
+ <h3>Area Chart</h3>
139
+ <div class="google-visualization" data-chart="area">
140
+ <%= select :value_by_day %>
141
+ </div>
142
+ </div>
143
+ <div>
144
+ <h3>Bar Chart</h3>
145
+ <div class="google-visualization" data-chart="bar">
146
+ <%= select :value_by_day %>
147
+ </div>
148
+ </div>
149
+ <div>
150
+ <h3>Column Chart</h3>
151
+ <div class="google-visualization" data-chart="column">
152
+ <%= select :value_by_day %>
153
+ </div>
154
+ </div>
155
+ <div>
156
+ <h3>Annotation Chart</h3>
157
+ <div class="google-visualization" data-chart="annotation">
158
+ <%= select :value_by_day %>
159
+ </div>
160
+ </div>
161
+ </div>
162
+ </body>
163
+ </html>
@@ -0,0 +1,36 @@
1
+ require 'sinatra/base'
2
+ require 'to_js'
3
+
4
+ module GoogleCharts
5
+ extend self
6
+
7
+ def convert(results)
8
+ results.description.to_js
9
+ end
10
+
11
+ private
12
+
13
+ end
14
+
15
+ class MetricSystem::Web < Sinatra::Base
16
+ set :environment, :development
17
+ set :raise_errors, true
18
+ set :views, "#{File.dirname(__FILE__)}/web"
19
+ set :dump_errors, true
20
+
21
+ helpers do
22
+ def select(query, *args)
23
+ @result_cache ||= {}
24
+ @result_cache[query] ||= MetricSystem.database.select(query, *args).data_table.to_js
25
+ end
26
+ end
27
+
28
+ get '/:query.js' do
29
+ content_type "application/javascript"
30
+ select params[:query].to_sym
31
+ end
32
+
33
+ get '/' do
34
+ erb :dashboard
35
+ end
36
+ end