metric_system 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c76a2843de05d5ccaaa70865f23e76b09b287dc3
-  data.tar.gz: 34156de4921e7fb09874a3d47bedc4174ca4985b
+  metadata.gz: d1483853542b6eb79dc9c7a0e349591d2ee14d52
+  data.tar.gz: 9f50ee6bc48e72b83002aaf165576ce586414873
 SHA512:
-  metadata.gz: 13f1207dee0f2cfb89850d1b637f4015abc0507231b332640848f88d6295e30ca222d59c8df2ae972139f01a184c04c1af997f4752151eb6a2cb709701e06cc9
-  data.tar.gz: 89948cf352c4e282b5e0bb7c91dc3f6bca221312767bec1a84f72c5254fa3aa7952d0f2abef10366750a101b2e1f0891526df258d4cb7f60c4f8bf6717090eeb
+  metadata.gz: 422740c79d19c46fce188e3b33d6cdbf45fc34c79ab37cda1560dfeae0aed607411b6678d0a6085ba6e0569249ec50ac91967fa8bb5bce38bdb048195cfa60b4
+  data.tar.gz: 03deb35f0a2baa9a5842de40355cd8118120a49ec371e900491d458455bab093acbac6bfecc2dd1a34829bbb5122ce47294593ff7c40e3d120959d732bd34475

data/README.md

@@ -1,7 +1,11 @@
 # metric_system
 
 A simple metric collector
- 
+
+**Note!** 
+
+This is alpha software. Use at your own risk!
+
 ## Installation
 
     gem install metric_system

data/lib/metric_system/database.rb

@@ -0,0 +1,164 @@
+class MetricSystem::Database
+  def initialize(path)
+    require_relative "./sqlite3_extensions"
+    open path
+  end
+
+  extend Forwardable
+
+  delegate [:exec, :select, :transaction, :rollback, :print, :run, :ask, :register] => :"@db"
+
+  PERIODS = [
+    [ :year,   31536000, "strftime('%Y-01-01',          starts_at, 'unixepoch')" ],
+    [ :month,   2592000, "strftime('%Y-%m-01',          starts_at, 'unixepoch')" ],
+    [ :week,     604800, "strftime('%Y-%m-%d',          starts_at, 'unixepoch',  'weekday 1', '-7 days')" ],
+    [ :day,       86400, "strftime('%Y-%m-%d',          starts_at, 'unixepoch')" ],
+    [ :hour,       3600, "strftime('%Y-%m-%d %H:00:00', starts_at, 'unixepoch')" ],
+    [ :minute,       60, "strftime('%Y-%m-%d %H:%M:00', starts_at, 'unixepoch')" ],
+    # [ :second,        1, "strftime('%Y-%m-%d %H:%M:%S', starts_at, 'unixepoch')" ],
+  ]
+
+  private
+
+  def open(path)
+    @db = SQLite3::Database.new(path)
+
+    [ :counters, :gauges ].each do |name|
+      exec <<-SQL
+      CREATE TABLE IF NOT EXISTS #{name}(
+        id INTEGER PRIMARY KEY,
+
+        name NOT NULL,                                                -- the event name
+        value NOT NULL,                                               -- the value
+        starts_at TIMESTAMP NOT NULL DEFAULT (strftime('%s','now'))   -- the timestamp
+      );
+
+      CREATE INDEX IF NOT EXISTS #{name}_idx1 ON #{name}(name, starts_at);
+
+      CREATE TABLE IF NOT EXISTS aggregated_#{name}(
+          id INTEGER PRIMARY KEY,
+
+          name NOT NULL,                                      -- the event name
+          starts_at TIMESTAMP NOT NULL,                       -- the start-at timestamp
+          duration NOT NULL,                                  -- the duration (estimate, in secs.)
+          period,                                             -- the name of the period (year, day, etc.)
+          sum,                                                -- the sum of event values
+          count,                                              -- the count of events
+          value                                               -- the aggregated value
+      );
+
+      CREATE UNIQUE INDEX IF NOT EXISTS aggregated_#{name}_uidx1 ON aggregated_#{name}(name, starts_at, duration);
+      CREATE INDEX IF NOT EXISTS aggregated_#{name}_idx2 ON aggregated_#{name}(starts_at);
+      CREATE INDEX IF NOT EXISTS aggregated_#{name}_idx3 ON aggregated_#{name}(duration);
+      SQL
+    end
+
+    exec <<-SQL
+    PRAGMA synchronous = NORMAL;
+    PRAGMA journal_mode = WAL;
+
+    CREATE VIEW IF NOT EXISTS aggregates AS
+      SELECT * FROM aggregated_gauges
+        UNION
+      SELECT * FROM aggregated_counters
+    SQL
+  end
+
+  public
+
+  def add_event(table, name, value, starts_at)
+    # get names of all related events. An event "a.b.c" is actually
+    # 3 events: "a", "a.b", "a.b.c"
+    names = begin
+      parts = name.split(".")
+      parts.length.downto(1).map do |cnt|
+        parts[0,cnt].join(".")
+      end
+    end
+
+    if starts_at
+      starts_at = Time.parse(starts_at) if starts_at.is_a?(String)
+
+      names.each do |name|
+        run "INSERT INTO #{table}(name, value, starts_at) VALUES(?, ?, ?)", name, value, starts_at.to_i
+      end
+    else
+      names.each do |name|
+        run "INSERT INTO #{table}(name, value) VALUES(?, ?)", name, value
+      end
+    end
+  rescue
+    STDERR.puts "#{$!}: #{table} #{name.inspect}, #{value.inspect}"
+  end
+
+  PERIODS_BY_KEY = PERIODS.by(&:first)
+
+  def aggregate(*keys)
+    if keys.empty?
+      keys = PERIODS.map(&:first)
+    end
+
+    transaction do
+      keys.each do |period|
+        aggregate_for_period :period => period, :source => :counters, :dest => :aggregated_counters, :aggregate => "sum"
+      end
+
+      @db.exec "DELETE FROM counters"
+    end
+
+    transaction do
+      keys.each do |period|
+        aggregate_for_period :period => period, :source => :gauges, :dest => :aggregated_gauges, :aggregate => "CAST(sum AS FLOAT) / count"
+      end
+
+      @db.exec "DELETE FROM gauges"
+    end
+  end
+
+  private
+
+  def aggregate_for_period(options)
+    expect! options => {
+      :period => PERIODS.map(&:first)
+    }
+    period, source, dest, aggregate = options.values_at :period, :source, :dest, :aggregate
+
+    _, duration, starts_at = PERIODS_BY_KEY[period]
+
+    # sql expression to calculate value from sum and count of event values
+    aggregate = source == :gauges ? "CAST(sum AS FLOAT) / count" : "sum"
+
+    @db.exec <<-SQL
+      CREATE TEMPORARY TABLE batch AS
+        SELECT name, starts_at, SUM(sum) AS sum, SUM(count) AS count FROM
+        (
+          SELECT name     AS name,
+            #{starts_at}  AS starts_at,
+            SUM(value)    AS sum,
+            COUNT(value)  AS count
+            FROM #{source}
+          GROUP BY name, starts_at
+
+          UNION
+
+          SELECT #{dest}.name     AS name,
+            #{dest}.starts_at     AS starts_at,
+            sum                   AS sum,
+            count                 AS count
+          FROM #{dest}
+          -- INNER JOIN #{source} ON #{dest}.name=#{source}.name
+          WHERE duration=#{duration}
+            AND #{dest}.starts_at >= (SELECT MIN(starts_at) FROM #{source})
+        )
+        GROUP BY name, starts_at;
+
+      INSERT OR REPLACE INTO #{dest}(name, starts_at, period, duration, sum, count, value)
+                      SELECT name, starts_at, '#{period}', #{duration}, sum, count, #{aggregate}
+                      FROM batch;
+    SQL
+
+    @db.exec <<-SQL
+      DROP TABLE batch;
+    SQL
+  end
+end

data/lib/metric_system/io.rb

@@ -0,0 +1,15 @@
+class MetricSystem::IO
+  def initialize(io)
+    expect! io => ::IO
+    @io = io
+  end
+
+  def add_event(table, name, value, starts_at)
+    starts_at = Time.now unless starts_at
+    @io.puts "#{table} #{name} #{value} #{starts_at}"
+  end
+
+  def quit_server!
+    @io.puts "SHUTDOWN:SERVER"
+  end
+end

data/lib/metric_system/server.rb

@@ -0,0 +1,126 @@
+require 'eventmachine'
+#require 'eventmachine/timer'
+
+module MetricSystem::Server
+  include EM::P::LineProtocol
+  extend self
+
+  module Buffer
+    extend self
+
+    def buffer
+      @buffer ||= []
+    end
+
+    def take
+      taken, @buffer = @buffer, []
+      taken
+    end
+
+    def push(event)
+      buffer << event
+    end
+
+    def length
+      buffer.length
+    end
+  end
+
+  class Event < Struct.new(:table, :name, :value, :time)
+    def self.parse(line)
+      table, name, value, time, remainder = line.split(" ", 5)
+      value = value.to_f
+      time = time ? time.to_i : Time.now.to_i
+      new(table, name, value, time)
+    end
+  end
+
+  def receive_line(line)
+    if line == "SHUTDOWN:SERVER"
+      MetricSystem::Server.shutdown
+      return
+    end
+
+    return if MetricSystem::Server.shutting_down?
+
+    return unless event = Event.parse(line)
+
+    Buffer.push event
+
+    MetricSystem::Server.flush if Buffer.length % 1000 == 0
+  rescue
+    STDERR.puts "#{$!}, from\n\t" + $!.backtrace.join("\t")
+  end
+
+  def self.flush
+    return unless Buffer.length > 0
+
+    if @busy
+      STDERR.puts "    Waiting for writer: backlog is #{Buffer.length} entries"
+      return
+    end
+
+    @busy = true
+
+    events = Buffer.take
+
+    operation = proc {
+      flush_events events
+    }
+    callback = proc {
+      @busy = nil
+
+      if shutting_down?
+        flush_events(Buffer.take)
+        EM.stop
+      end
+    }
+    EventMachine.defer operation, callback
+  rescue
+    STDERR.puts "#{$!}, from\n\t" + $!.backtrace.join("\t")
+  end
+
+  def self.flush_events(events)
+    return if events.empty?
+
+    starts_at = Time.now
+    MetricSystem.transaction do
+      events.each do |event|
+        MetricSystem.add_event event.table, event.name, event.value, event.time
+      end
+    end
+    STDERR.puts "    Writing #{events.count} events: %.3f secs" % (Time.now - starts_at)
+
+    starts_at = Time.now
+    MetricSystem.aggregate
+    STDERR.puts "    Merging #{events.count} events: %.3f secs" % (Time.now - starts_at)
+  end
+
+  # Note that this will block current thread.
+  def self.run(db, socket_path, options = {})
+    @options = options || {}
+    MetricSystem.target = db
+
+    STDERR.puts "Starting server at socket: #{socket_path}"
+
+    EventMachine.run {
+      EventMachine::PeriodicTimer.new(1) do
+        MetricSystem::Server.flush
+      end
+
+      EventMachine.start_server socket_path, MetricSystem::Server
+    }
+  end
+
+  def self.shutting_down?
+    @shutting_down
+  end
+
+  def self.shutdown
+    return if shutting_down?
+    return unless @options[:quit_server]
+
+    @shutting_down = true
+    MetricSystem::Server.flush
+  end
+end

data/lib/metric_system/sqlite3_extensions.rb

@@ -1,158 +1,4 @@
 require "sqlite3"
-
-# manage SQLite3::Records
-#
-# The SQLite3::Record module is able to generate classes that are optimized
-# for a specific set of columns. It is build on top of Struct, which is way
-# faster than Hashes, for example.
-module SQLite3::Record
-  module ClassMethods
-    attr :columns, true
-
-    private
-
-    def to_time(s)
-      case s
-      when String then Time.parse(s)
-      when Fixnum then Time.at(s)
-      else s
-      end
-    end
-
-    def to_date(s)
-      return unless time = to_time(s)
-      time.to_date
-    end
-
-    public
-
-    def build(*attrs)
-      attrs = columns.zip(attrs).map do |key, value|
-        case key
-        when /_at$/ then to_time(value)
-        when /_on$/ then to_date(value)
-        else value
-        end
-      end
-
-      new *attrs
-    end
-  end
-
-  def to_a
-    self.class.columns.map do |column| send(column) end
-  end
-
-  def self.for_columns(columns)
-    columns = columns.map(&:to_sym)
-
-    @@classes ||= {}
-    @@classes[columns] ||= begin
-      struct = Struct.new(*columns)
-      struct.extend SQLite3::Record::ClassMethods
-      struct.include SQLite3::Record
-
-      struct.columns = columns
-      struct
-    end
-  end
-end
-
-class SQLite3::Query
-  def initialize(sql, statement)
-    expect! statement => SQLite3::Statement
-
-    @sql, @statement = sql, statement
-  end
-
-  def run(*args)
-    # STDERR.puts "Q: #{@sql} #{args.map(&:inspect).join(", ")}"
-    @statement.execute *args
-  end
-
-  def select(*args)
-    @klass ||= SQLite3::Record.for_columns(@statement.columns)
-
-    run(*args).map do |rec|
-      @klass.build *rec
-    end
-  end
-
-  def ask(*args)
-    results = run(*args)
-    row = results.first
-    results.reset
-
-    if !row               then  nil
-    elsif row.length == 1 then  row.first
-    else                        row
-    end
-  end
-end
-
-class SQLite3::Database
-  # execute multiple SQL statements at once.
-  def exec(sql, *args)
-    args = prepare_arguments(args)
-
-    while sql =~ /\S/ do
-      statement = prepare(sql)
-
-      sql = statement.remainder
-      if statement.active?
-        statement.execute!(*args)
-      end
-    end
-
-  rescue
-    STDERR.puts "#{sql}: #{$!}"
-    raise
-  end
-
-  # -- cached queries ---------------------------------------------------------
-
-  private
-
-  def query(sql)
-    @queries ||= {}
-    @queries[sql] ||= SQLite3::Query.new sql, prepare(sql)
-  end
-
-  def prepare_arguments(args)
-    args.map do |arg|
-      case arg
-      when Time then arg.to_i
-      when Date then arg.to_time.to_i
-      else arg
-      end
-    end
-  end
-
-  public
-
-  def run(sql, *args)
-    query(sql).run *prepare_arguments(args)
-  end
-
-  def ask(sql, *args)
-    query(sql).ask *prepare_arguments(args)
-  end
-
-  # run a select like query. Returns an array of records.
-  def select(sql, *args)
-    query(sql).select *prepare_arguments(args)
-  end
-
-  def print(sql, *args)
-    results = select sql, *args
-    log_sql = sql.gsub(/\n/, " ").gsub(/\s+/, " ")
-    puts "=" * log_sql.length
-    puts log_sql
-    puts "-" * log_sql.length
-
-    results.each do |result|
-      pp result.to_a
-    end
-    puts "=" * log_sql.length
-  end
-end
+require "sqlite3/database_extension"
+require "sqlite3/query"
+require "sqlite3/record"

data/lib/metric_system/version.rb

@@ -1,3 +1,3 @@
-class MetricSystem
-  VERSION = "0.1.0"
+module MetricSystem
+  VERSION = "0.1.2"
 end

data/lib/metric_system/web/dashboard.erb

@@ -0,0 +1,163 @@
+<html>
+  <head>
+    <!--Load the AJAX API-->
+    <script type="text/javascript" src="https://www.google.com/jsapi"></script>
+    <script type="text/javascript" src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
+    <script type="text/javascript">
+    
+    // Load the Visualization API and the piechart package.
+    google.load('visualization', '1', {'packages':['corechart', 'table', 'annotationchart']});
+    
+    $.fn.visualize = function() {
+      var CHARTS = {
+        pie: google.visualization.PieChart,
+        area: google.visualization.AreaChart,
+        bar: google.visualization.BarChart,
+        column: google.visualization.ColumnChart,
+        annotation: google.visualization.AnnotationChart
+      }
+
+      this.each(function(idx, node) {
+        node = $(node);
+        
+        var ctor = CHARTS[$(node).data("chart")];
+        if(!ctor) ctor = google.visualization.Table;
+      
+        var chart = new ctor(node[0]);          // the chart object
+        var data = null;                        // the chart's data, will be loaded from url
+
+        var url = $(node).data("src");
+        if(url) {
+          loadDataFromURL();
+        }
+        else {
+          var raw_data = eval("r = " + node.text());
+          node.html("");
+          data = new google.visualization.DataTable(raw_data);
+          redraw();
+          
+        }
+
+        // automatically redraw when resizing.
+        (function() {
+          var timeout = null;
+  
+          $(window).resize(function() {
+            if(timeout) clearTimeout(timeout);
+            timeout = setTimeout(redraw, 50);
+          });
+        })();
+
+        // redraw();
+    
+        // -- callbacks and helpers ---------------------------------------------
+    
+        // (re)draw chart
+        function redraw() {
+          chart.draw(data, {width: node.width(), height: node.height()});
+        }
+  
+        // fetch data from \a url, never run two requests in parallel.
+        var xhrRequest = null;
+        function loadDataFromURL(callback) {
+          if(xhrRequest) return;
+      
+          xhrRequest = $.ajax({url: url, dataType: "text",
+            success: function(js) { 
+              xhrRequest = null;
+              var raw_data = eval("r = " + js);
+              data = new google.visualization.DataTable(raw_data);
+              redraw();
+            }
+          });
+        }        
+      });
+    };
+    
+    $(function() {
+      $('.google-visualization').visualize();
+    });
+    </script>
+  </head>
+
+  <body>
+    <style type="text/css" media="screen">
+    @import url(http://fonts.googleapis.com/css?family=Open+Sans:400,700);
+    
+    h1, h2, h3, h4 {
+      font-weight: 700;
+      font-family: 'Open Sans', sans-serif;
+    }
+
+    body {
+      padding: 0;
+      margin: 0;
+    }
+    
+    .google-visualization {
+      overflow: auto;
+      height: 300px;
+      width: 400px;
+      border: 1px solid red;
+    }
+    #gallery > div {
+      width: 400px;
+      display: inline-block;
+      position: relative;
+    }
+    #gallery > div > h3 {
+      position: relative;
+      top: 0;
+    }
+    
+    </style>
+
+    <div id="gallery">
+      <div>
+        <h3>Table</h3>
+        <div class="google-visualization">
+          <%=
+            select "SELECT date(starts_at), value FROM aggregates WHERE period='day'"
+          %>
+        </div>
+      </div>
+      <div>
+        <h3>Pie Chart</h3>
+        <div class="google-visualization" data-src="/value_by_day_name.js" data-chart="pie">
+        </div>
+      </div>
+      <div>
+        <h3>Area Chart</h3>
+        <div class="google-visualization" data-chart="area">
+          <%=
+            select "SELECT date(starts_at), value FROM aggregates WHERE period='day'"
+          %>
+        </div>
+      </div>
+      <div>
+        <h3>Area Chart</h3>
+        <div class="google-visualization" data-chart="area">
+          <%= select :value_by_day %>
+        </div>
+      </div>
+      <div>
+        <h3>Bar Chart</h3>
+        <div class="google-visualization" data-chart="bar">
+          <%= select :value_by_day %>
+        </div>
+      </div>
+      <div>
+        <h3>Column Chart</h3>
+        <div class="google-visualization" data-chart="column">
+          <%= select :value_by_day %>
+        </div>
+      </div>
+      <div>
+        <h3>Annotation Chart</h3>
+        <div class="google-visualization" data-chart="annotation">
+          <%= select :value_by_day %>
+        </div>
+      </div>
+    </div>
+  </body>
+</html>

data/lib/metric_system/web.rb

@@ -0,0 +1,36 @@
+require 'sinatra/base'
+require 'to_js'
+
+module GoogleCharts
+  extend self
+
+  def convert(results)
+    results.description.to_js
+  end
+
+  private
+
+end
+
+class MetricSystem::Web < Sinatra::Base
+  set :environment, :development
+  set :raise_errors, true
+  set :views, "#{File.dirname(__FILE__)}/web"
+  set :dump_errors, true
+
+  helpers do
+    def select(query, *args)
+      @result_cache ||= {}
+      @result_cache[query] ||= MetricSystem.database.select(query, *args).data_table.to_js
+    end
+  end
+
+  get '/:query.js' do
+    content_type "application/javascript"
+    select params[:query].to_sym
+  end
+
+  get '/' do
+    erb :dashboard
+  end
+end