metric_system 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c76a2843de05d5ccaaa70865f23e76b09b287dc3
4
- data.tar.gz: 34156de4921e7fb09874a3d47bedc4174ca4985b
3
+ metadata.gz: d1483853542b6eb79dc9c7a0e349591d2ee14d52
4
+ data.tar.gz: 9f50ee6bc48e72b83002aaf165576ce586414873
5
5
  SHA512:
6
- metadata.gz: 13f1207dee0f2cfb89850d1b637f4015abc0507231b332640848f88d6295e30ca222d59c8df2ae972139f01a184c04c1af997f4752151eb6a2cb709701e06cc9
7
- data.tar.gz: 89948cf352c4e282b5e0bb7c91dc3f6bca221312767bec1a84f72c5254fa3aa7952d0f2abef10366750a101b2e1f0891526df258d4cb7f60c4f8bf6717090eeb
6
+ metadata.gz: 422740c79d19c46fce188e3b33d6cdbf45fc34c79ab37cda1560dfeae0aed607411b6678d0a6085ba6e0569249ec50ac91967fa8bb5bce38bdb048195cfa60b4
7
+ data.tar.gz: 03deb35f0a2baa9a5842de40355cd8118120a49ec371e900491d458455bab093acbac6bfecc2dd1a34829bbb5122ce47294593ff7c40e3d120959d732bd34475
data/README.md CHANGED
@@ -1,7 +1,11 @@
1
1
  # metric_system
2
2
 
3
3
  A simple metric collector
4
-
4
+
5
+ **Note!**
6
+
7
+ This is alpha software. Use at your own risk!
8
+
5
9
  ## Installation
6
10
 
7
11
  gem install metric_system
@@ -0,0 +1,164 @@
1
+ class MetricSystem::Database
2
+ def initialize(path)
3
+ require_relative "./sqlite3_extensions"
4
+ open path
5
+ end
6
+
7
+ extend Forwardable
8
+
9
+ delegate [:exec, :select, :transaction, :rollback, :print, :run, :ask, :register] => :"@db"
10
+
11
+ PERIODS = [
12
+ [ :year, 31536000, "strftime('%Y-01-01', starts_at, 'unixepoch')" ],
13
+ [ :month, 2592000, "strftime('%Y-%m-01', starts_at, 'unixepoch')" ],
14
+ [ :week, 604800, "strftime('%Y-%m-%d', starts_at, 'unixepoch', 'weekday 1', '-7 days')" ],
15
+ [ :day, 86400, "strftime('%Y-%m-%d', starts_at, 'unixepoch')" ],
16
+ [ :hour, 3600, "strftime('%Y-%m-%d %H:00:00', starts_at, 'unixepoch')" ],
17
+ [ :minute, 60, "strftime('%Y-%m-%d %H:%M:00', starts_at, 'unixepoch')" ],
18
+ # [ :second, 1, "strftime('%Y-%m-%d %H:%M:%S', starts_at, 'unixepoch')" ],
19
+ ]
20
+
21
+ private
22
+
23
+ def open(path)
24
+ @db = SQLite3::Database.new(path)
25
+
26
+ [ :counters, :gauges ].each do |name|
27
+ exec <<-SQL
28
+ CREATE TABLE IF NOT EXISTS #{name}(
29
+ id INTEGER PRIMARY KEY,
30
+
31
+ name NOT NULL, -- the event name
32
+ value NOT NULL, -- the value
33
+ starts_at TIMESTAMP NOT NULL DEFAULT (strftime('%s','now')) -- the timestamp
34
+ );
35
+
36
+ CREATE INDEX IF NOT EXISTS #{name}_idx1 ON #{name}(name, starts_at);
37
+
38
+ CREATE TABLE IF NOT EXISTS aggregated_#{name}(
39
+ id INTEGER PRIMARY KEY,
40
+
41
+ name NOT NULL, -- the event name
42
+ starts_at TIMESTAMP NOT NULL, -- the start-at timestamp
43
+ duration NOT NULL, -- the duration (estimate, in secs.)
44
+ period, -- the name of the period (year, day, etc.)
45
+ sum, -- the sum of event values
46
+ count, -- the count of events
47
+ value -- the aggregated value
48
+ );
49
+
50
+ CREATE UNIQUE INDEX IF NOT EXISTS aggregated_#{name}_uidx1 ON aggregated_#{name}(name, starts_at, duration);
51
+ CREATE INDEX IF NOT EXISTS aggregated_#{name}_idx2 ON aggregated_#{name}(starts_at);
52
+ CREATE INDEX IF NOT EXISTS aggregated_#{name}_idx3 ON aggregated_#{name}(duration);
53
+ SQL
54
+ end
55
+
56
+ exec <<-SQL
57
+ PRAGMA synchronous = NORMAL;
58
+ PRAGMA journal_mode = WAL;
59
+
60
+ CREATE VIEW IF NOT EXISTS aggregates AS
61
+ SELECT * FROM aggregated_gauges
62
+ UNION
63
+ SELECT * FROM aggregated_counters
64
+ SQL
65
+ end
66
+
67
+ public
68
+
69
+ def add_event(table, name, value, starts_at)
70
+ # get names of all related events. An event "a.b.c" is actually
71
+ # 3 events: "a", "a.b", "a.b.c"
72
+ names = begin
73
+ parts = name.split(".")
74
+ parts.length.downto(1).map do |cnt|
75
+ parts[0,cnt].join(".")
76
+ end
77
+ end
78
+
79
+ if starts_at
80
+ starts_at = Time.parse(starts_at) if starts_at.is_a?(String)
81
+
82
+ names.each do |name|
83
+ run "INSERT INTO #{table}(name, value, starts_at) VALUES(?, ?, ?)", name, value, starts_at.to_i
84
+ end
85
+ else
86
+ names.each do |name|
87
+ run "INSERT INTO #{table}(name, value) VALUES(?, ?)", name, value
88
+ end
89
+ end
90
+ rescue
91
+ STDERR.puts "#{$!}: #{table} #{name.inspect}, #{value.inspect}"
92
+ end
93
+
94
+ PERIODS_BY_KEY = PERIODS.by(&:first)
95
+
96
+ def aggregate(*keys)
97
+ if keys.empty?
98
+ keys = PERIODS.map(&:first)
99
+ end
100
+
101
+ transaction do
102
+ keys.each do |period|
103
+ aggregate_for_period :period => period, :source => :counters, :dest => :aggregated_counters, :aggregate => "sum"
104
+ end
105
+
106
+ @db.exec "DELETE FROM counters"
107
+ end
108
+
109
+ transaction do
110
+ keys.each do |period|
111
+ aggregate_for_period :period => period, :source => :gauges, :dest => :aggregated_gauges, :aggregate => "CAST(sum AS FLOAT) / count"
112
+ end
113
+
114
+ @db.exec "DELETE FROM gauges"
115
+ end
116
+ end
117
+
118
+ private
119
+
120
+ def aggregate_for_period(options)
121
+ expect! options => {
122
+ :period => PERIODS.map(&:first)
123
+ }
124
+ period, source, dest, aggregate = options.values_at :period, :source, :dest, :aggregate
125
+
126
+ _, duration, starts_at = PERIODS_BY_KEY[period]
127
+
128
+ # sql expression to calculate value from sum and count of event values
129
+ aggregate = source == :gauges ? "CAST(sum AS FLOAT) / count" : "sum"
130
+
131
+ @db.exec <<-SQL
132
+ CREATE TEMPORARY TABLE batch AS
133
+ SELECT name, starts_at, SUM(sum) AS sum, SUM(count) AS count FROM
134
+ (
135
+ SELECT name AS name,
136
+ #{starts_at} AS starts_at,
137
+ SUM(value) AS sum,
138
+ COUNT(value) AS count
139
+ FROM #{source}
140
+ GROUP BY name, starts_at
141
+
142
+ UNION
143
+
144
+ SELECT #{dest}.name AS name,
145
+ #{dest}.starts_at AS starts_at,
146
+ sum AS sum,
147
+ count AS count
148
+ FROM #{dest}
149
+ -- INNER JOIN #{source} ON #{dest}.name=#{source}.name
150
+ WHERE duration=#{duration}
151
+ AND #{dest}.starts_at >= (SELECT MIN(starts_at) FROM #{source})
152
+ )
153
+ GROUP BY name, starts_at;
154
+
155
+ INSERT OR REPLACE INTO #{dest}(name, starts_at, period, duration, sum, count, value)
156
+ SELECT name, starts_at, '#{period}', #{duration}, sum, count, #{aggregate}
157
+ FROM batch;
158
+ SQL
159
+
160
+ @db.exec <<-SQL
161
+ DROP TABLE batch;
162
+ SQL
163
+ end
164
+ end
@@ -0,0 +1,15 @@
1
+ class MetricSystem::IO
2
+ def initialize(io)
3
+ expect! io => ::IO
4
+ @io = io
5
+ end
6
+
7
+ def add_event(table, name, value, starts_at)
8
+ starts_at = Time.now unless starts_at
9
+ @io.puts "#{table} #{name} #{value} #{starts_at}"
10
+ end
11
+
12
+ def quit_server!
13
+ @io.puts "SHUTDOWN:SERVER"
14
+ end
15
+ end
@@ -0,0 +1,126 @@
1
+ require 'eventmachine'
2
+ #require 'eventmachine/timer'
3
+
4
+ module MetricSystem::Server
5
+ include EM::P::LineProtocol
6
+ extend self
7
+
8
+ module Buffer
9
+ extend self
10
+
11
+ def buffer
12
+ @buffer ||= []
13
+ end
14
+
15
+ def take
16
+ taken, @buffer = @buffer, []
17
+ taken
18
+ end
19
+
20
+ def push(event)
21
+ buffer << event
22
+ end
23
+
24
+ def length
25
+ buffer.length
26
+ end
27
+ end
28
+
29
+ class Event < Struct.new(:table, :name, :value, :time)
30
+ def self.parse(line)
31
+ table, name, value, time, remainder = line.split(" ", 5)
32
+ value = value.to_f
33
+ time = time ? time.to_i : Time.now.to_i
34
+ new(table, name, value, time)
35
+ end
36
+ end
37
+
38
+ def receive_line(line)
39
+ if line == "SHUTDOWN:SERVER"
40
+ MetricSystem::Server.shutdown
41
+ return
42
+ end
43
+
44
+ return if MetricSystem::Server.shutting_down?
45
+
46
+ return unless event = Event.parse(line)
47
+
48
+ Buffer.push event
49
+
50
+ MetricSystem::Server.flush if Buffer.length % 1000 == 0
51
+ rescue
52
+ STDERR.puts "#{$!}, from\n\t" + $!.backtrace.join("\t")
53
+ end
54
+
55
+ def self.flush
56
+ return unless Buffer.length > 0
57
+
58
+ if @busy
59
+ STDERR.puts " Waiting for writer: backlog is #{Buffer.length} entries"
60
+ return
61
+ end
62
+
63
+ @busy = true
64
+
65
+ events = Buffer.take
66
+
67
+ operation = proc {
68
+ flush_events events
69
+ }
70
+ callback = proc {
71
+ @busy = nil
72
+
73
+ if shutting_down?
74
+ flush_events(Buffer.take)
75
+ EM.stop
76
+ end
77
+ }
78
+ EventMachine.defer operation, callback
79
+ rescue
80
+ STDERR.puts "#{$!}, from\n\t" + $!.backtrace.join("\t")
81
+ end
82
+
83
+ def self.flush_events(events)
84
+ return if events.empty?
85
+
86
+ starts_at = Time.now
87
+ MetricSystem.transaction do
88
+ events.each do |event|
89
+ MetricSystem.add_event event.table, event.name, event.value, event.time
90
+ end
91
+ end
92
+ STDERR.puts " Writing #{events.count} events: %.3f secs" % (Time.now - starts_at)
93
+
94
+ starts_at = Time.now
95
+ MetricSystem.aggregate
96
+ STDERR.puts " Merging #{events.count} events: %.3f secs" % (Time.now - starts_at)
97
+ end
98
+
99
+ # Note that this will block current thread.
100
+ def self.run(db, socket_path, options = {})
101
+ @options = options || {}
102
+ MetricSystem.target = db
103
+
104
+ STDERR.puts "Starting server at socket: #{socket_path}"
105
+
106
+ EventMachine.run {
107
+ EventMachine::PeriodicTimer.new(1) do
108
+ MetricSystem::Server.flush
109
+ end
110
+
111
+ EventMachine.start_server socket_path, MetricSystem::Server
112
+ }
113
+ end
114
+
115
+ def self.shutting_down?
116
+ @shutting_down
117
+ end
118
+
119
+ def self.shutdown
120
+ return if shutting_down?
121
+ return unless @options[:quit_server]
122
+
123
+ @shutting_down = true
124
+ MetricSystem::Server.flush
125
+ end
126
+ end
@@ -1,158 +1,4 @@
1
1
  require "sqlite3"
2
-
3
- # manage SQLite3::Records
4
- #
5
- # The SQLite3::Record module is able to generate classes that are optimized
6
- # for a specific set of columns. It is build on top of Struct, which is way
7
- # faster than Hashes, for example.
8
- module SQLite3::Record
9
- module ClassMethods
10
- attr :columns, true
11
-
12
- private
13
-
14
- def to_time(s)
15
- case s
16
- when String then Time.parse(s)
17
- when Fixnum then Time.at(s)
18
- else s
19
- end
20
- end
21
-
22
- def to_date(s)
23
- return unless time = to_time(s)
24
- time.to_date
25
- end
26
-
27
- public
28
-
29
- def build(*attrs)
30
- attrs = columns.zip(attrs).map do |key, value|
31
- case key
32
- when /_at$/ then to_time(value)
33
- when /_on$/ then to_date(value)
34
- else value
35
- end
36
- end
37
-
38
- new *attrs
39
- end
40
- end
41
-
42
- def to_a
43
- self.class.columns.map do |column| send(column) end
44
- end
45
-
46
- def self.for_columns(columns)
47
- columns = columns.map(&:to_sym)
48
-
49
- @@classes ||= {}
50
- @@classes[columns] ||= begin
51
- struct = Struct.new(*columns)
52
- struct.extend SQLite3::Record::ClassMethods
53
- struct.include SQLite3::Record
54
-
55
- struct.columns = columns
56
- struct
57
- end
58
- end
59
- end
60
-
61
- class SQLite3::Query
62
- def initialize(sql, statement)
63
- expect! statement => SQLite3::Statement
64
-
65
- @sql, @statement = sql, statement
66
- end
67
-
68
- def run(*args)
69
- # STDERR.puts "Q: #{@sql} #{args.map(&:inspect).join(", ")}"
70
- @statement.execute *args
71
- end
72
-
73
- def select(*args)
74
- @klass ||= SQLite3::Record.for_columns(@statement.columns)
75
-
76
- run(*args).map do |rec|
77
- @klass.build *rec
78
- end
79
- end
80
-
81
- def ask(*args)
82
- results = run(*args)
83
- row = results.first
84
- results.reset
85
-
86
- if !row then nil
87
- elsif row.length == 1 then row.first
88
- else row
89
- end
90
- end
91
- end
92
-
93
- class SQLite3::Database
94
- # execute multiple SQL statements at once.
95
- def exec(sql, *args)
96
- args = prepare_arguments(args)
97
-
98
- while sql =~ /\S/ do
99
- statement = prepare(sql)
100
-
101
- sql = statement.remainder
102
- if statement.active?
103
- statement.execute!(*args)
104
- end
105
- end
106
-
107
- rescue
108
- STDERR.puts "#{sql}: #{$!}"
109
- raise
110
- end
111
-
112
- # -- cached queries ---------------------------------------------------------
113
-
114
- private
115
-
116
- def query(sql)
117
- @queries ||= {}
118
- @queries[sql] ||= SQLite3::Query.new sql, prepare(sql)
119
- end
120
-
121
- def prepare_arguments(args)
122
- args.map do |arg|
123
- case arg
124
- when Time then arg.to_i
125
- when Date then arg.to_time.to_i
126
- else arg
127
- end
128
- end
129
- end
130
-
131
- public
132
-
133
- def run(sql, *args)
134
- query(sql).run *prepare_arguments(args)
135
- end
136
-
137
- def ask(sql, *args)
138
- query(sql).ask *prepare_arguments(args)
139
- end
140
-
141
- # run a select like query. Returns an array of records.
142
- def select(sql, *args)
143
- query(sql).select *prepare_arguments(args)
144
- end
145
-
146
- def print(sql, *args)
147
- results = select sql, *args
148
- log_sql = sql.gsub(/\n/, " ").gsub(/\s+/, " ")
149
- puts "=" * log_sql.length
150
- puts log_sql
151
- puts "-" * log_sql.length
152
-
153
- results.each do |result|
154
- pp result.to_a
155
- end
156
- puts "=" * log_sql.length
157
- end
158
- end
2
+ require "sqlite3/database_extension"
3
+ require "sqlite3/query"
4
+ require "sqlite3/record"
@@ -1,3 +1,3 @@
1
- class MetricSystem
2
- VERSION = "0.1.0"
1
+ module MetricSystem
2
+ VERSION = "0.1.2"
3
3
  end
@@ -0,0 +1,163 @@
1
+ <html>
2
+ <head>
3
+ <!--Load the AJAX API-->
4
+ <script type="text/javascript" src="https://www.google.com/jsapi"></script>
5
+ <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
6
+ <script type="text/javascript">
7
+
8
+ // Load the Visualization API and the piechart package.
9
+ google.load('visualization', '1', {'packages':['corechart', 'table', 'annotationchart']});
10
+
11
+ $.fn.visualize = function() {
12
+ var CHARTS = {
13
+ pie: google.visualization.PieChart,
14
+ area: google.visualization.AreaChart,
15
+ bar: google.visualization.BarChart,
16
+ column: google.visualization.ColumnChart,
17
+ annotation: google.visualization.AnnotationChart
18
+ }
19
+
20
+ this.each(function(idx, node) {
21
+ node = $(node);
22
+
23
+ var ctor = CHARTS[$(node).data("chart")];
24
+ if(!ctor) ctor = google.visualization.Table;
25
+
26
+ var chart = new ctor(node[0]); // the chart object
27
+ var data = null; // the chart's data, will be loaded from url
28
+
29
+ var url = $(node).data("src");
30
+ if(url) {
31
+ loadDataFromURL();
32
+ }
33
+ else {
34
+ var raw_data = eval("r = " + node.text());
35
+ node.html("");
36
+ data = new google.visualization.DataTable(raw_data);
37
+ redraw();
38
+
39
+ }
40
+
41
+ // automatically redraw when resizing.
42
+ (function() {
43
+ var timeout = null;
44
+
45
+ $(window).resize(function() {
46
+ if(timeout) clearTimeout(timeout);
47
+ timeout = setTimeout(redraw, 50);
48
+ });
49
+ })();
50
+
51
+ // redraw();
52
+
53
+ // -- callbacks and helpers ---------------------------------------------
54
+
55
+ // (re)draw chart
56
+ function redraw() {
57
+ chart.draw(data, {width: node.width(), height: node.height()});
58
+ }
59
+
60
+ // fetch data from \a url, never run two requests in parallel.
61
+ var xhrRequest = null;
62
+ function loadDataFromURL(callback) {
63
+ if(xhrRequest) return;
64
+
65
+ xhrRequest = $.ajax({url: url, dataType: "text",
66
+ success: function(js) {
67
+ xhrRequest = null;
68
+ var raw_data = eval("r = " + js);
69
+ data = new google.visualization.DataTable(raw_data);
70
+ redraw();
71
+ }
72
+ });
73
+ }
74
+ });
75
+ };
76
+
77
+ $(function() {
78
+ $('.google-visualization').visualize();
79
+ });
80
+ </script>
81
+ </head>
82
+
83
+ <body>
84
+ <style type="text/css" media="screen">
85
+ @import url(http://fonts.googleapis.com/css?family=Open+Sans:400,700);
86
+
87
+ h1, h2, h3, h4 {
88
+ font-weight: 700;
89
+ font-family: 'Open Sans', sans-serif;
90
+ }
91
+
92
+ body {
93
+ padding: 0;
94
+ margin: 0;
95
+ }
96
+
97
+ .google-visualization {
98
+ overflow: auto;
99
+ height: 300px;
100
+ width: 400px;
101
+ border: 1px solid red;
102
+ }
103
+ #gallery > div {
104
+ width: 400px;
105
+ display: inline-block;
106
+ position: relative;
107
+ }
108
+ #gallery > div > h3 {
109
+ position: relative;
110
+ top: 0;
111
+ }
112
+
113
+ </style>
114
+
115
+ <div id="gallery">
116
+ <div>
117
+ <h3>Table</h3>
118
+ <div class="google-visualization">
119
+ <%=
120
+ select "SELECT date(starts_at), value FROM aggregates WHERE period='day'"
121
+ %>
122
+ </div>
123
+ </div>
124
+ <div>
125
+ <h3>Pie Chart</h3>
126
+ <div class="google-visualization" data-src="/value_by_day_name.js" data-chart="pie">
127
+ </div>
128
+ </div>
129
+ <div>
130
+ <h3>Area Chart</h3>
131
+ <div class="google-visualization" data-chart="area">
132
+ <%=
133
+ select "SELECT date(starts_at), value FROM aggregates WHERE period='day'"
134
+ %>
135
+ </div>
136
+ </div>
137
+ <div>
138
+ <h3>Area Chart</h3>
139
+ <div class="google-visualization" data-chart="area">
140
+ <%= select :value_by_day %>
141
+ </div>
142
+ </div>
143
+ <div>
144
+ <h3>Bar Chart</h3>
145
+ <div class="google-visualization" data-chart="bar">
146
+ <%= select :value_by_day %>
147
+ </div>
148
+ </div>
149
+ <div>
150
+ <h3>Column Chart</h3>
151
+ <div class="google-visualization" data-chart="column">
152
+ <%= select :value_by_day %>
153
+ </div>
154
+ </div>
155
+ <div>
156
+ <h3>Annotation Chart</h3>
157
+ <div class="google-visualization" data-chart="annotation">
158
+ <%= select :value_by_day %>
159
+ </div>
160
+ </div>
161
+ </div>
162
+ </body>
163
+ </html>
@@ -0,0 +1,36 @@
1
+ require 'sinatra/base'
2
+ require 'to_js'
3
+
4
+ module GoogleCharts
5
+ extend self
6
+
7
+ def convert(results)
8
+ results.description.to_js
9
+ end
10
+
11
+ private
12
+
13
+ end
14
+
15
+ class MetricSystem::Web < Sinatra::Base
16
+ set :environment, :development
17
+ set :raise_errors, true
18
+ set :views, "#{File.dirname(__FILE__)}/web"
19
+ set :dump_errors, true
20
+
21
+ helpers do
22
+ def select(query, *args)
23
+ @result_cache ||= {}
24
+ @result_cache[query] ||= MetricSystem.database.select(query, *args).data_table.to_js
25
+ end
26
+ end
27
+
28
+ get '/:query.js' do
29
+ content_type "application/javascript"
30
+ select params[:query].to_sym
31
+ end
32
+
33
+ get '/' do
34
+ erb :dashboard
35
+ end
36
+ end