metric_system 0.1.0 → 0.1.2

data/lib/metric_system.rb

@@ -1,67 +1,33 @@
 require "expectation"
-require_relative "metric_system/core_extensions"
-require_relative "metric_system/sqlite3_extensions"
-
-# -- The MetricSystem module --------------------------------------------------
 
+require_relative "metric_system/core_extensions"
 require "forwardable"
-class MetricSystem
-  extend Forwardable
-  delegate [:exec, :select, :transaction, :rollback] => :@db
-
-  PERIODS = [
-    [ :year,   31536000, "strftime('%Y-01-01',          starts_at, 'unixepoch')" ],
-    [ :month,   2592000, "strftime('%Y-%m-01',          starts_at, 'unixepoch')" ],
-    [ :week,     604800, "strftime('%Y-%m-%d',          starts_at, 'unixepoch',  'weekday 1', '-7 days')" ],
-    [ :day,       86400, "strftime('%Y-%m-%d',          starts_at, 'unixepoch')" ],
-    [ :hour,       3600, "strftime('%Y-%m-%d %H:00:00', starts_at, 'unixepoch')" ],
-    [ :minute,       60, "strftime('%Y-%m-%d %H:%M:00', starts_at, 'unixepoch')" ],
-    [ :second,        1, "strftime('%Y-%m-%d %H:%M:%S', starts_at, 'unixepoch')" ],
-  ]
 
-  def initialize(path)
-    @db = SQLite3::Database.new(path)
+module MetricSystem
+  extend self
 
-    [ :counters, :gauges ].each do |name|
-      exec <<-SQL
-      CREATE TABLE IF NOT EXISTS #{name}(
-        id INTEGER PRIMARY KEY,
+  attr :target, :database
 
-        name NOT NULL,                                                -- the event name
-        value NOT NULL,                                               -- the value
-        starts_at TIMESTAMP NOT NULL DEFAULT (strftime('%s','now'))   -- the timestamp
-      );
+  def target=(target)
+    @target = @database = nil
 
-      CREATE INDEX IF NOT EXISTS #{name}_idx1 ON #{name}(name, starts_at);
-
-      CREATE TABLE IF NOT EXISTS aggregated_#{name}(
-          id INTEGER PRIMARY KEY,
-
-          name NOT NULL,                                      -- the event name
-          starts_at TIMESTAMP NOT NULL,                       -- the start-at timestamp
-          duration NOT NULL,                                  -- the duration (estimate, in secs.)
-          period,                                             -- the name of the period (year, day, etc.)
-          sum,                                                -- the sum of event values
-          count,                                              -- the count of events
-          value                                               -- the aggregated value
-      );
-
-      CREATE UNIQUE INDEX IF NOT EXISTS aggregated_#{name}_uidx1 ON aggregated_#{name}(name, starts_at, duration);
-      CREATE INDEX IF NOT EXISTS aggregated_#{name}_idx2 ON aggregated_#{name}(starts_at);
-      CREATE INDEX IF NOT EXISTS aggregated_#{name}_idx3 ON aggregated_#{name}(duration);
-      SQL
+    case target
+    when nil
+    when String
+      require_relative "metric_system/database"
+      @target = @database = MetricSystem::Database.new(target)
+    else
+      require_relative "metric_system/io"
+      @target = MetricSystem::IO.new(target)
     end
-
-    exec <<-SQL
-    PRAGMA synchronous = NORMAL;
-
-    CREATE VIEW IF NOT EXISTS aggregates AS
-      SELECT * FROM aggregated_gauges
-        UNION
-      SELECT * FROM aggregated_counters
-    SQL
   end
 
+  extend Forwardable
+  delegate [:aggregate, :select, :print, :run, :ask, :register] => :"@target"
+  delegate [:transaction] => :"@target"
+  delegate [:add_event] => :"@target"
+  delegate [:quit_server!] => :"@target"
+
   def gauge(name, value, starts_at = nil)
     add_event :gauges, name, value, starts_at
   end
@@ -70,99 +36,10 @@ class MetricSystem
     add_event :counters, name, value, starts_at
   end
 
-  private
-
-  def add_event(table, name, value, starts_at)
-    # get names of all related events. An event "a.b.c" is actually
-    # 3 events: "a", "a.b", "a.b.c"
-    names = begin
-      parts = name.split(".")
-      parts.length.downto(1).map do |cnt|
-        parts[0,cnt].join(".")
-      end
-    end
-
-    if starts_at
-      starts_at = Time.parse(starts_at) if starts_at.is_a?(String)
-
-      names.each do |name|
-        @db.run "INSERT INTO #{table}(name, value, starts_at) VALUES(?, ?, ?)", name, value, starts_at.to_i
-      end
-    else
-      names.each do |name|
-        @db.run "INSERT INTO #{table}(name, value) VALUES(?, ?)", name, value
-      end
-    end
-  end
-
-  public
-
-  PERIODS_BY_KEY = PERIODS.by(&:first)
-
-  def aggregate(*keys)
-    if keys.empty?
-      keys = PERIODS.map(&:first)
-    end
-
-    transaction do
-      keys.each do |period|
-        aggregate_for_period :period => period, :source => :counters, :dest => :aggregated_counters, :aggregate => "sum"
-      end
-
-      @db.exec "DELETE FROM counters"
-    end
-
-    transaction do
-      keys.each do |period|
-        aggregate_for_period :period => period, :source => :gauges, :dest => :aggregated_gauges, :aggregate => "CAST(sum AS FLOAT) / count"
-      end
-
-      @db.exec "DELETE FROM gauges"
+  def measure(name, starts_at = nil, &block)
+    start = Time.now
+    yield.tap do
+      gauge name, Time.now - start, starts_at
     end
   end
-
-  def aggregate_for_period(options)
-    expect! options => {
-      :period => PERIODS.map(&:first)
-    }
-    period, source, dest, aggregate = options.values_at :period, :source, :dest, :aggregate
-
-    _, duration, starts_at = PERIODS_BY_KEY[period]
-
-    # sql expression to calculate value from sum and count of event values
-    aggregate = source == :gauges ? "CAST(sum AS FLOAT) / count" : "sum"
-
-    @db.exec <<-SQL
-      CREATE TEMPORARY TABLE batch AS
-        SELECT name, starts_at, SUM(sum) AS sum, SUM(count) AS count FROM
-        (
-          SELECT name     AS name,
-            #{starts_at}  AS starts_at,
-            SUM(value)    AS sum,
-            COUNT(value)  AS count
-            FROM #{source}
-          GROUP BY name, starts_at
-
-          UNION
-
-          SELECT #{dest}.name     AS name,
-            #{dest}.starts_at     AS starts_at,
-            sum                   AS sum,
-            count                 AS count
-          FROM #{dest}
-          INNER JOIN #{source} ON #{dest}.name=#{source}.name
-          WHERE duration=#{duration} 
-            AND #{dest}.starts_at >= (SELECT MIN(starts_at) FROM #{source})
-        )
-        GROUP BY name, starts_at;
-
-      INSERT OR REPLACE INTO #{dest}(name, starts_at, period, duration, sum, count, value)
-                      SELECT name, starts_at, '#{period}', #{duration}, sum, count, #{aggregate}
-                      FROM batch;
-    SQL
-
-    @db.exec <<-SQL
-      DROP TABLE batch;
-    SQL
-  end
 end

data/lib/sqlite3/database_extension.rb

@@ -0,0 +1,85 @@
+class SQLite3::Database
+  # execute multiple SQL statements at once.
+  def exec(sql, *args)
+    args = prepare_arguments(args)
+
+    while sql =~ /\S/ do
+      statement = prepare(sql)
+
+      sql = statement.remainder
+      if statement.active?
+        statement.execute!(*args)
+      end
+    end
+
+  rescue
+    STDERR.puts "#{sql}: #{$!}"
+    raise
+  end
+
+  # -- cached queries ---------------------------------------------------------
+
+  private
+
+  def query(sql)
+    expect! sql => [ String, Symbol ]
+
+    if sql.is_a?(Symbol)
+      expect! sql => registry.keys
+      sql = registry.fetch(sql)
+    end
+
+    @queries ||= {}
+    @queries[sql] ||= SQLite3::Query.new sql, prepare(sql)
+  end
+
+  def prepare_arguments(args)
+    args.map do |arg|
+      case arg
+      when Time then arg.to_i
+      when Date then arg.to_time.to_i
+      else arg
+      end
+    end
+  end
+
+  public
+
+  def run(sql, *args)
+    query(sql).run *prepare_arguments(args)
+  end
+
+  def ask(sql, *args)
+    query(sql).ask *prepare_arguments(args)
+  end
+
+  # run a select like query. Returns an array of records.
+  def select(sql, *args)
+    query(sql).select *prepare_arguments(args)
+  end
+
+  def print(sql, *args)
+    require "pp"
+
+    results = select sql, *args
+    log_sql = sql.gsub(/\n/, " ").gsub(/\s+/, " ")
+    puts "=" * log_sql.length
+    puts log_sql
+    puts "-" * log_sql.length
+
+    results.each do |result|
+      pp result.to_a
+    end
+    puts "=" * log_sql.length
+  end
+
+  # -- query registry
+
+  def registry
+    @registry ||= {}
+  end
+
+  def register(name, query)
+    registry[name] = query
+  end
+end

data/lib/sqlite3/query.rb

@@ -0,0 +1,77 @@
+class SQLite3::Query
+  def initialize(sql, statement)
+    expect! statement => SQLite3::Statement
+
+    @sql, @statement = sql, statement
+  end
+
+  def run(*args)
+    # STDERR.puts "Q: #{@sql} #{args.map(&:inspect).join(", ")}"
+    @statement.execute *args
+  end
+
+  def select(*args)
+    @klass ||= SQLite3::Record.for_columns(@statement.columns)
+
+    ary = run(*args).map do |rec|
+      @klass.build *rec
+    end
+
+    ary.extend Description
+    ary.columns = @statement.columns
+    ary
+  end
+
+  module Description
+    attr :columns, true
+
+    # A Google Chart compatible data table; see
+    # https://developers.google.com/chart/interactive/docs/reference#dataparam
+    def data_table
+      cols = columns.map do |column|
+        type = case column
+        when /_at$/   then :datetime
+        when /_on$/   then :date
+        when /value/  then :number
+        else               :string
+        end
+
+        { id: column, type: type, label: column }
+      end
+
+      rows = map { |record| convert_record record, cols }
+
+      { cols: cols, rows: rows }
+    end
+
+    def convert_record(record, cols)
+      values = cols.map do |col|
+        id, type = col.values_at(:id, :type)
+        v = record.send(id)
+
+        case type
+        when :date      then f = v.strftime("%a %b, %Y")
+        when :datetime  then f = v.inspect
+        when :number    then f = v
+        else            f = v
+        end
+
+
+        { v: v, f: f }
+      end
+
+      { c: values }
+    end
+  end
+
+  def ask(*args)
+    results = run(*args)
+    row = results.first
+    results.reset
+
+    if !row               then  nil
+    elsif row.length == 1 then  row.first
+    else                        row
+    end
+  end
+end

data/lib/sqlite3/record.rb

@@ -0,0 +1,65 @@
+# manage SQLite3::Records
+#
+# The SQLite3::Record module is able to generate classes that are optimized
+# for a specific set of columns. It is build on top of Struct, which is way
+# faster than Hashes, for example.
+module SQLite3::Record
+  module ClassMethods
+    attr :columns, true
+
+    private
+
+    def to_time(s)
+      case s
+      when String then Time.parse(s)
+      when Fixnum then Time.at(s)
+      else s
+      end
+    end
+
+    def to_date(s)
+      return unless time = to_time(s)
+      time.to_date
+    end
+
+    public
+
+    def build(*attrs)
+      attrs = columns.zip(attrs).map do |key, value|
+        case key
+        when /_at$/ then to_time(value)
+        when /_on$/ then to_date(value)
+        else value
+        end
+      end
+
+      new *attrs
+    end
+  end
+
+  def to_a
+    self.class.columns.map do |column| send(column) end
+  end
+
+  def to_hash
+    kvs = self.class.columns.inject([]) do |ary, column|
+      ary << column << send(column)
+    end
+
+    Hash[*kvs]
+  end
+
+  def self.for_columns(columns)
+    columns = columns.map(&:to_sym)
+
+    @@classes ||= {}
+    @@classes[columns] ||= begin
+      struct = Struct.new(*columns)
+      struct.extend SQLite3::Record::ClassMethods
+      struct.include SQLite3::Record
+
+      struct.columns = columns
+      struct
+    end
+  end
+end

data/lib/to_js.rb

@@ -0,0 +1,55 @@
+require "json"
+
+class Object
+  def to_js
+    convert_to_js
+  end
+
+  alias :convert_to_js :to_json
+end
+
+class Array
+  def convert_to_js
+    "[" + map(&:convert_to_js).join(", ") + "]"
+  end
+end
+
+class Hash
+  def convert_to_js
+    "{" + map { |k,v| "#{k.convert_to_js}: #{v.convert_to_js}" }.join(", ") + "}"
+  end
+end
+
+class Date
+  def convert_to_js
+    "new Date(#{year}, #{month-1}, #{day})"
+  end
+end
+
+class Time
+  def convert_to_js
+    "new Date(#{year}, #{month-1}, #{day}, #{hour}, #{min}, #{sec}, #{usec / 1000})"
+  end
+end
+
+class Numeric
+  def convert_to_js
+    "%f" % self
+  end
+end
+
+class OpenStruct
+  def convert_to_js
+    @table.to_js
+  end
+end
+
+if defined?(SQLite3::Record)
+
+module SQLite3::Record::ClassMethods
+  def convert_to_js
+    to_hash.convert_to_js
+  end
+end
+
+end

data/test/benchmark.rb

@@ -0,0 +1,48 @@
+$: << "#{File.dirname(__FILE__)}/../lib"
+require "metric_system"
+
+def benchmark(msg, &block)
+  starts = Time.now
+
+  yield
+
+ensure
+  runtime = Time.now - starts
+  if runtime > 0.1
+    STDERR.puts "%s: %.3f secs" % [ msg, runtime = Time.now - starts ]
+  end
+end
+
+TICKS_PER_DAY = 86400
+TICKS_PER_DAY = 10
+DBPATH = "samples.sqlite"
+
+File.unlink(DBPATH) rescue nil
+MetricSystem.target = DBPATH
+
+benchmark "Building metric_system" do
+  distance = 24 * 3600.0 / TICKS_PER_DAY
+
+  0.upto(365) do |day|
+    midnight = Time.parse("2013-01-01").to_i + day * 24 * 3600
+
+    benchmark "Day ##{day}: add metrics" do
+      MetricSystem.transaction do
+        1.upto(TICKS_PER_DAY) do |step|
+          time = midnight + distance * step
+          STDERR.print "."
+          MetricSystem.count "clicks", 1, time
+        end
+      end
+    end
+
+    benchmark "Day ##{day}: aggregate" do
+      MetricSystem.aggregate
+    end
+  end
+end
+
+MetricSystem.print "SELECT COUNT(*) FROM aggregates"
+
+__END__
+db.print "SELECT * FROM aggregates"

data/test/counters_test.rb

@@ -3,8 +3,16 @@ require "metric_system"
 require "test/unit"
 
 class MetricSystem::TestCounters < Test::Unit::TestCase
+  def setup
+    MetricSystem.target = ":memory:"
+  end
+
+  def teardown
+    MetricSystem.target = nil
+  end
+
   def db
-    @db ||= MetricSystem.new ":memory:"
+    MetricSystem
   end
 
   def test_two_events
@@ -25,6 +33,8 @@ class MetricSystem::TestCounters < Test::Unit::TestCase
   end
 
   def test_conversion
+    db = MetricSystem.target
+    expect! db => MetricSystem::Database
     db.exec "CREATE TABLE tmp(value, starts_at, starts_on)"
     now = Time.parse("2014-03-02 11:10:11 +0100")
     day = Date.parse("2014-03-02")
@@ -42,7 +52,6 @@ class MetricSystem::TestCounters < Test::Unit::TestCase
 
     r = db.select("SELECT name, value, period, starts_at FROM aggregates ORDER BY duration, name")
     assert_equal(r.map(&:to_a), [
-      ["foo", 1, "second",  Time.parse("2014-03-02 11:10:11 +0100")],
       ["foo", 1, "minute",  Time.parse("2014-03-02 11:10:00 +0100")],
       ["foo", 1, "hour",    Time.parse("2014-03-02 11:00:00 +0100")],
       ["foo", 1, "day",     Time.parse("2014-03-02 00:00:00 +0100")],
@@ -66,7 +75,7 @@ class MetricSystem::TestCounters < Test::Unit::TestCase
   def test_combined_name
     db.count "foo",     3, "2014-03-02 12:10:11"
     db.count "foo.bar", 2, "2014-03-02 12:10:11"
-    db.aggregate :second, :minute, :hour
+    db.aggregate :minute, :hour
 
     r = db.select <<-SQL
       SELECT name, value, period, starts_at
@@ -75,8 +84,6 @@ class MetricSystem::TestCounters < Test::Unit::TestCase
     SQL
 
     assert_equal(r.map(&:to_a), [
-      ["foo"    , 5, "second",  Time.parse("2014-03-02 11:10:11 +0100")],
-      ["foo.bar", 2, "second",  Time.parse("2014-03-02 11:10:11 +0100")],
       ["foo"    , 5, "minute",  Time.parse("2014-03-02 11:10:00 +0100")],
       ["foo.bar", 2, "minute",  Time.parse("2014-03-02 11:10:00 +0100")],
       ["foo"    , 5, "hour",    Time.parse("2014-03-02 11:00:00 +0100")],
@@ -87,8 +94,8 @@ class MetricSystem::TestCounters < Test::Unit::TestCase
   def test_double_aggregate
     db.count "foo",     3, "2014-03-02 12:10:11"
     db.count "foo.bar", 2, "2014-03-02 12:10:11"
-    db.aggregate :second, :minute, :hour
-    db.aggregate :second, :minute, :hour
+    db.aggregate :minute, :hour
+    db.aggregate :minute, :hour
 
     r = db.select <<-SQL
       SELECT name, value, period, starts_at
@@ -97,8 +104,6 @@ class MetricSystem::TestCounters < Test::Unit::TestCase
     SQL
 
     assert_equal(r.map(&:to_a), [
-      ["foo"    , 5, "second",  Time.parse("2014-03-02 11:10:11 +0100")],
-      ["foo.bar", 2, "second",  Time.parse("2014-03-02 11:10:11 +0100")],
       ["foo"    , 5, "minute",  Time.parse("2014-03-02 11:10:00 +0100")],
       ["foo.bar", 2, "minute",  Time.parse("2014-03-02 11:10:00 +0100")],
       ["foo"    , 5, "hour",    Time.parse("2014-03-02 11:00:00 +0100")],

data/test/gauges_test.rb

@@ -3,8 +3,16 @@ require "metric_system"
 require "test/unit"
 
 class MetricSystem::TestGauging < Test::Unit::TestCase
+  def setup
+    MetricSystem.target = ":memory:"
+  end
+
+  def teardown
+    MetricSystem.target = nil
+  end
+
   def db
-    @db ||= MetricSystem.new ":memory:"
+    MetricSystem
   end
 
   def test_two_events

data/test/mixed_test.rb

@@ -3,11 +3,19 @@ require "metric_system"
 require "test/unit"
 
 class MetricSystem::TestMixed < Test::Unit::TestCase
+  def setup
+    MetricSystem.target = ":memory:"
+  end
+
+  def teardown
+    MetricSystem.target = nil
+  end
+
   def db
-    @db ||= MetricSystem.new ":memory:"
+    MetricSystem
   end
 
-  def xtest_two_events
+  def test_two_events
     db.gauge "foo", 1, "2014-03-02 12:10:11"
     db.gauge "foo", 2, "2014-03-02 14:10:11"
     db.count "bar", 1, "2014-03-02 12:10:11"

data/test/parallel.rb

@@ -0,0 +1,68 @@
+$: << "#{File.dirname(__FILE__)}/../lib"
+require "metric_system"
+
+SOCKET = "performance.socket"
+DBPATH = "samples.sqlite"
+
+mode, _ = *ARGV
+if mode == nil then
+  require "metric_system/server"
+
+  Thread.new do
+    sleep 1
+    system "ruby #{__FILE__} sender"
+  end
+
+  MetricSystem::Server.run DBPATH, SOCKET, :quit_server => true
+
+  puts "server stopped"
+
+  MetricSystem.print "SELECT COUNT(*) FROM aggregates"
+  exit
+end
+
+# ---------------------------------------------------------------------
+
+def benchmark(msg, &block)
+  starts = Time.now
+
+  yield
+
+ensure
+  runtime = Time.now - starts
+  if runtime > 0.1
+    STDERR.puts "%s: %.3f secs" % [ msg, runtime = Time.now - starts ]
+  end
+end
+
+# TICKS_PER_DAY = 86400
+TICKS_PER_DAY = 10
+
+File.unlink(DBPATH) rescue nil
+
+require "time"
+require "socket"
+
+benchmark "Sending #{365 * TICKS_PER_DAY} events to metric_system" do
+  distance = 24 * 3600.0 / TICKS_PER_DAY
+  MetricSystem.target = UNIXSocket.new(SOCKET)
+  0.upto(364) do |day|
+    midnight = Time.parse("2013-01-01").to_i + day * 24 * 3600
+
+    benchmark "Day ##{day}: add metrics" do
+      1.upto(TICKS_PER_DAY) do |step|
+        time = midnight + distance * step
+        MetricSystem.count "clicks", 1, time
+      end
+    end
+  end
+end
+
+STDERR.puts "Quitting server"
+MetricSystem.quit_server!
+
+__END__
+
+db.print "SELECT COUNT(*) FROM aggregates"
+
+db.print "SELECT * FROM aggregates"