dasht 0.1.0

data/lib/dasht/collector.rb

@@ -0,0 +1,33 @@
+module Dasht
+  class Metrics
+    attr_accessor :parent
+    def initialize(parent)
+      @parent            = parent
+      @metric_values     = {}
+      @metric_operations = {}
+    end
+
+    def set(metric, value, op, ts)
+      metric = metric.to_s
+      @metric_operations[metric] = op
+      m = (@metric_values[metric] ||= Metric.new)
+      m.append(value, ts) do |old_value, new_value|
+        [old_value, new_value].compact.flatten.send(op)
+      end
+    end
+
+    def get(metric, start_ts, end_ts)
+      metric = metric.to_s
+      m = @metric_values[metric]
+      return [] if m.nil?
+      op = @metric_operations[metric]
+      m.enum(start_ts, end_ts).to_a.flatten.send(op)
+    end
+
+    def trim_to(ts)
+      @metric_values.each do |k, v|
+        v.trim_to(ts)
+      end
+    end
+  end
+end

data/lib/dasht/list.rb

@@ -0,0 +1,93 @@
+# Dasht::List - Simple list structure following properties:
+#
+# 1. Fast writes. Appends to a list.
+# 2. Fast reads by index. Indexed by position in the list.
+# 3. Fast deletes that preserve indexes. Removes items from the front
+#    of the list.
+# 4. Simple (but not necessarily fast) aggregation. Enumerate values
+#    between pointers.
+#
+# The Dasht::List structure is formed using a Ruby Array (values),
+# plus a counter of how many items have been deleted (offset).
+# Whenever data is deleted from the head of the list, the offset is
+# incremented.
+
+module Dasht
+  class List
+    attr_accessor :values
+    attr_accessor :offset
+
+    def initialize
+      @offset = 0
+      @values = []
+    end
+
+    def to_s
+      return @values.to_s
+    end
+
+    # Public: Get a pointer to the first value.
+    def head_pointer
+      return offset
+    end
+
+    # Public: Get a pointer to right after the last value.
+    def tail_pointer
+      return offset + @values.length
+    end
+
+    # Public: Get the value at a given pointer, or nil if the pointer
+    # is no longer valid.
+    def get(pointer)
+      index = _pointer_to_index(pointer)
+      return @values[index]
+    end
+
+    # Public: Return an enumerator that walks through the list, yielding
+    # data.
+    def enum(start_pointer = nil, end_pointer = nil)
+      index = _pointer_to_index(start_pointer || head_pointer)
+      end_index = _pointer_to_index(end_pointer || tail_pointer)
+      return Enumerator.new do |yielder|
+        while index < end_index
+          yielder << @values[index]
+          index += 1
+        end
+      end
+    end
+
+    # Public: Add data to the list.
+    # Returns a pointer to the new data.
+    def append(data)
+      pointer = self.tail_pointer
+      @values << data
+      return pointer
+    end
+
+    # Public: Remove data up to (but not including) the specified pointer.
+    def trim_to(pointer)
+      return if pointer.nil?
+      index = _pointer_to_index(pointer)
+      @offset += index
+      @values = @values.slice(index, @values.length)
+      return
+    end
+
+    # Public: Walk through the list, removing links from the list while
+    # the block returns true. Stop when it returns false.
+    def trim_while(&block)
+      while (@values.length > 0) && yield(@values.first)
+        @values.shift
+        @offset += 1
+      end
+      return
+    end
+
+    private
+
+    # Convert a pointer to an index in the list.
+    def _pointer_to_index(pointer)
+      return [pointer - offset, 0].max
+    end
+  end
+end

data/lib/dasht/log_thread.rb

@@ -0,0 +1,94 @@
+module Dasht
+  class LogThread
+    attr_accessor :parent
+
+    def self.update_global_stats(line)
+      @total_lines ||= 0
+      @total_bytes ||= 0
+      @total_lines += 1
+      @total_bytes += line.length
+      print "\rConsumed #{@total_lines} lines (#{@total_bytes} bytes)..."
+    end
+
+    def initialize(parent, command)
+      @parent            = parent
+      @command           = command
+      @event_definitions = []
+    end
+
+    def run
+      parent.log "Starting `#{@command}`..."
+      @thread = Thread.new do
+        begin
+          while true
+            begin
+              IO.popen(@command) do |process|
+                process.each do |line|
+                  _consume_line(line)
+                end
+              end
+            rescue => e
+              parent.log e
+            end
+            sleep 2
+          end
+        rescue => e
+          parent.log e
+          raise e
+        end
+      end
+    end
+
+    def event(metric, regex, op, value = nil, &block)
+      @event_definitions << [metric, regex, op, value, block]
+    end
+
+    def count(metric, regex, &block)
+      event(metric, regex, :dasht_sum, 1, &block)
+    end
+
+    def gauge(metric, regex, &block)
+      event(metric, regex, :last, nil, &block)
+    end
+
+    def min(metric, regex, &block)
+      event(metric, regex, :min, nil, &block)
+    end
+
+    def max(metric, regex, &block)
+      event(metric, regex, :max, nil, &block)
+    end
+
+    def append(metric, regex, &block)
+      event(metric, regex, :to_a, nil, &block)
+    end
+
+    def unique(metric, regex, &block)
+      event(metric, regex, :uniq, nil, &block)
+    end
+
+    def terminate
+      @thread.terminate
+    end
+
+    private
+
+    def _consume_line(line)
+      self.class.update_global_stats(line)
+      ts = Time.now.to_i
+      @event_definitions.each do |metric, regex, op, value, block|
+        begin
+          regex.match(line) do |matches|
+            value = matches[0] if value.nil?
+            value = block.call(matches) if block
+            parent.metrics.set(metric, value, op, ts) if value
+          end
+        rescue => e
+          parent.log e
+          raise e
+        end
+        parent.metrics.trim_to(ts - (parent.history || (60 * 60)))
+      end
+    end
+  end
+end

data/lib/dasht/metric.rb

@@ -0,0 +1,82 @@
+# Dasht::Metric - Simple in-memory time-series data structure with the
+# following properties:
+#
+# 1. Sparse. Only stores time stamps for intervals with known data,
+#    and only stores one timestamp per interval.
+# 2. Flexible aggregation using Ruby blocks during both read and
+#    write.
+# 3. Read values between two timestamps.
+#
+# The Dasht::Metric structure is formed using two Dasht::List
+# objects. One object tracks data, the other object tracks a list of
+# checkpoints and their corresponding index into the data.
+module Dasht
+  class Metric
+    attr_reader :data, :checkpoints
+
+    def initialize
+      @checkpoints = List.new
+      @data = List.new
+      @last_item = nil
+      @last_ts = nil
+    end
+
+    def to_s
+      return @data.to_s + " (last: #{@last_item})"
+    end
+
+    def append(data, ts, &block)
+      # Maybe checkpoint the time.
+      if @last_ts == ts
+        @last_item = yield(@last_item, data)
+      else
+        if @last_ts
+          pointer = @data.append(@last_item)
+          @checkpoints.append([@last_ts, pointer])
+        end
+        @last_ts = ts
+        @last_item = nil
+        @last_item = yield(@last_item, data)
+      end
+      return
+    end
+
+    def trim_to(ts)
+      pointer = nil
+      @checkpoints.trim_while do |s, p|
+        pointer = p
+        s < ts
+      end
+      @data.trim_to(pointer)
+      return
+    end
+
+    def enum(start_ts, end_ts = nil)
+      # Get a pointer to our location in the data.
+      start_pointer = nil
+      end_pointer = nil
+      prev_p = nil
+      @checkpoints.enum.each do |s, p|
+        start_pointer ||= p if start_ts <= s
+        end_pointer   ||= prev_p if end_ts && end_ts <= s
+        break if start_pointer && (end_ts.nil? || end_pointer)
+        prev_p = p
+      end
+      start_pointer ||= @data.tail_pointer
+      end_pointer ||= @data.tail_pointer
+
+      # Enumerate through the data, then tack on the last item.
+      return Enumerator.new do |yielder|
+        @data.enum(start_pointer, end_pointer).each do |data|
+          yielder << data
+        end
+        # Maybe include the last item.
+        if @last_item &&
+           (start_ts <= @last_ts) &&
+           (end_ts.nil? || (@last_ts < end_ts))
+          yielder << @last_item
+        end
+      end
+    end
+  end
+end

data/lib/dasht/rack_app.rb

@@ -0,0 +1,74 @@
+module Dasht
+  class RackApp
+    attr_accessor :parent
+
+    def initialize(parent)
+      @parent = parent
+    end
+
+    def root_path
+      File.join(File.dirname(__FILE__), "..", "..")
+    end
+
+    def run(port)
+      context = self
+      app = Rack::Builder.new do
+        use Rack::Static, :urls => ["/assets"], :root  => context.root_path
+        run lambda { |env|
+          begin
+            context._call(env)
+          rescue => e
+            context.parent.log e
+            raise e
+          end
+        }
+      end
+      Rack::Server.start(:app => app, :Port => (port || 8080))
+    end
+
+    def _call(env)
+      if "/" == env["REQUEST_PATH"] && parent.boards["default"]
+        return ['200', {'Content-Type' => 'text/html'}, [parent.boards["default"].to_html]]
+      end
+
+      /^\/boards\/(.+)$/.match(env["REQUEST_PATH"]) do |match|
+        board = match[1]
+        if parent.boards[board]
+          return ['200', {'Content-Type' => 'text/html'}, [parent.boards[board].to_html]]
+        else
+          return ['404', {'Content-Type' => 'text/html'}, ["Board #{board} not found."]]
+        end
+      end
+
+      /^\/data$/.match(env["REQUEST_PATH"]) do |match|
+        queries = JSON.parse(env['rack.input'].gets)
+        now = Time.now.to_i
+        data = queries.map do |query|
+          metric     = query["metric"]
+          resolution = query["resolution"]
+          periods    = query["periods"]
+          ts = now - (resolution * periods)
+          (1..periods).map do |n|
+            parent.metrics.get(metric, ts, ts += resolution) || 0
+          end
+        end
+
+        return ['200', {'Content-Type' => 'application/json'}, [data.to_json]]
+      end
+
+      /^\/data\/(.+)/.match(env["REQUEST_PATH"]) do |match|
+        parts = match[1].split('/')
+        metric     = parts.shift
+        resolution = parts.shift.to_i
+        periods    = (parts.shift || 1).to_i
+        ts = Time.now.to_i - (resolution * periods)
+        data = (1..periods).map do |n|
+          parent.metrics.get(metric, ts, ts += resolution) || 0
+        end
+        return ['200', {'Content-Type' => 'application/json'}, [data.to_json]]
+      end
+
+      return ['404', {'Content-Type' => 'text/html'}, ["Path not found: #{env['REQUEST_PATH']}"]]
+    end
+  end
+end

data/lib/dasht/reloader.rb

@@ -0,0 +1,28 @@
+module Dasht
+  class Reloader
+    attr_accessor :parent
+
+    def initialize(parent)
+      @parent        = parent
+      @last_modified = File.mtime($PROGRAM_NAME)
+    end
+
+    def changed?
+      @last_modified != File.mtime($PROGRAM_NAME)
+    end
+
+    def run
+      Thread.new do
+        while true
+          unless changed?
+            sleep 0.3
+            next
+          end
+          parent.log("Reloading #{$PROGRAM_NAME}...")
+          eval(IO.read($PROGRAM_NAME))
+          @last_modified = File.mtime($PROGRAM_NAME)
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,34 @@
+require 'simplecov'
+
+module SimpleCov::Configuration
+  def clean_filters
+    @filters = []
+  end
+end
+
+SimpleCov.configure do
+  clean_filters
+  load_adapter 'test_frameworks'
+end
+
+ENV["COVERAGE"] && SimpleCov.start do
+  add_filter "/.rvm/"
+end
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'dasht'
+
+class Test::Unit::TestCase
+end