solanum 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e9ebaaa5f2e140d9a0b2d821bc685a363f103ea1
+  data.tar.gz: 4fc6fbb8e60b919d8a9d1d8c8d36d557a80269ca
+SHA512:
+  metadata.gz: 30e99d6aa3fba2cf78f4ba69bde44e799212ca457a7de325ee40585e111c237188b3b1d0770c642a86641da5c253eab09a64310ae4a39fc2f7be6cee19a7d4f6
+  data.tar.gz: 254e37a291a4e3925e3a0e38cb7fde12f4a177508ba0354ed6ba035d2844fd086411ed040eedb63d421f51f2841f4d9277d9b8b32a26f232497f547a2a7e0160

data/README.md

@@ -0,0 +1,53 @@
+Solanum
+=======
+
+This gem provides a domain-specific language (DSL) for collecting metrics
+data in Ruby. The `solanum` script takes a number of monitoring configuration
+scripts as arguments and periodically collects the metrics defined. The results
+can be printed to the console or sent to a [Riemann](http://riemann.io/) server.
+This requires the `riemann-client` gem to work.
+
+## Structure
+
+Solanum scripts define _sources_, which provide some string input when they are
+read. This input is processed by a set of _matchers_ for each source, which can
+generate named measurements from that data. A simple example would be a file
+source, which is read and matched line-by-line against a set of regular
+expressions.
+
+The emitted measurements can undergo a bit more processing before being
+reported. For example, some metrics are monotonically-increasing counters, and
+what we actually want is the _difference_ between each reading. For others, we
+may want to apply threshold-based states to the events. These are set by
+_service prototypes_, which are also defined in the scripts.
+
+## Examples
+
+Here's an example of reading some information about the current system memory:
+
+```ruby
+# Read memory usage.
+read "/proc/meminfo" do
+  match /^MemTotal:\s+(\d+) kB$/, cast: :to_i, scale: 1024, record: 'memory total bytes'
+  match /^MemFree:\s+(\d+) kB$/,  cast: :to_i, scale: 1024, record: 'memory free bytes'
+end
+
+# Calculate percentages from total space.
+compute do |metrics|
+  total = metrics['memory total bytes']
+  free = metrics['memory free bytes']
+  if total && free
+    metrics['memory free pct'] = free.to_f/total
+  end
+end
+
+# Define a service prototype with a threshold-based state.
+service 'memory free pct', state: thresholds(0.00, :critical, 0.10, :warning, 0.25, :ok)
+```
+
+See the files in the `examples` directory for more monitor configuration samples.
+
+## License
+
+This is free and unencumbered software released into the public domain.
+See the UNLICENSE file for more information.

data/bin/solanum

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+
+$: << File.expand_path("../../lib", __FILE__)
+
+require 'optparse'
+require 'solanum'
+
+$options = {
+  riemann_host: nil,
+  riemann_port: 5555,
+  interval: 5,
+  verbose: false,
+}
+
+$defaults = {
+  host: %x{hostname}.chomp,
+  tags: [],
+  ttl: 10,
+}
+
+def fail(msg, code=1)
+  STDERR.puts(msg)
+  exit code
+end
+
+def log(msg)
+  puts "%s %s" % [Time.now.strftime("%H:%M:%S"), msg] if $options[:verbose]
+end
+
+# Parse command-line options.
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename($0)} [options] <monitor config> [monitor config] ..."
+  opts.separator ""
+  opts.separator "Event Attributes:"
+  opts.on(      '--host HOST', "Event hostname (default: #{$defaults[:host]})") {|v| $defaults[:host] = v }
+  opts.on('-a', '--attribute KEY=VAL', "Attribute to add to the event (may be given multiple times)") {|attr| k,v = attr.split(/=/); if k and v then $defaults[k.intern] = v end }
+  opts.on('-t', '--tag TAG', "Tag to add to events (may be given multiple times)") {|v| $defaults[:tags] << v }
+  opts.on(      '--ttl SECONDS', "Default TTL for events (default: #{$options[:ttl]})") {|v| $defaults[:ttl] = v.to_i }
+  opts.separator ""
+  opts.separator "General Options:"
+  opts.on(      '--riemann-host HOST', "Riemann host to report events to") {|v| $options[:riemann_host] = v }
+  opts.on(      '--riemann-port PORT', "Riemann port (default: #{$options[:riemann_port]})") {|v| $options[:riemann_port] = v.to_i }
+  opts.on('-i', '--interval SECONDS', "Seconds between updates (default: #{$options[:interval]})") {|v| $options[:interval] = v.to_i }
+  opts.on('-v', '--verbose', "Print additional information to stdout") { $options[:verbose] = true }
+  opts.on('-h', '--help', "Displays usage information") { print opts; exit }
+end
+options.parse!
+
+# Check usage.
+fail options if ARGV.empty?
+
+
+
+##### MONITORING CONFIGS #####
+
+$solanum = Solanum.new(ARGV)
+fail "No sources loaded!" if $solanum.sources.empty?
+
+if $options[:riemann_host]
+  begin
+    require 'riemann/client'
+  rescue LoadError
+    fail "ERROR: could not load Riemann client library! `gem install riemann-client` to enable reporting"
+  end
+
+  $riemann = Riemann::Client.new(host: $options[:riemann_host], port: $options[:riemann_port])
+end
+
+
+
+##### REPORT LOOP #####
+
+trap "SIGINT" do
+  exit
+end
+
+loop do
+  $solanum.collect!
+  events = $solanum.build_events($defaults)
+
+  events.each do |event|
+    if $options[:verbose] || $riemann.nil?
+      puts "%-40s %5s (%s) %s" % [
+        event[:service], event[:metric],
+        event[:state].nil? ? "--" : event[:state],
+        event.inspect
+      ]
+    end
+
+    $riemann << event if $riemann
+  end
+
+  sleep $options[:interval]
+end

data/lib/solanum/config.rb

@@ -0,0 +1,97 @@
+require 'solanum/source'
+
+class Solanum::Config
+  attr_reader :sources, :services
+
+  def initialize(path)
+    @sources = []
+    @services = []
+
+    instance_eval ::File.readlines(path).join, path, 1
+
+    raise "No sources loaded from monitor script: #{path}" if @sources.empty?
+  end
+
+
+  private
+
+  # Registers a new source object. If a block is given, it is used to configure
+  # the source with instance_exec.
+  def register_source(source, config=nil)
+    source.instance_exec &config if config
+    @sources << source
+    source
+  end
+
+
+  # Registers a source which runs a command and matches against output lines.
+  def run(command, &config)
+    register_source Solanum::Source::Command.new(command), config
+  end
+
+
+  # Registers a source which matches against the lines in a file.
+  def read(path, &config)
+    register_source Solanum::Source::File.new(path), config
+  end
+
+
+  # Registers a source which computes metrics directly.
+  def compute(&block)
+    register_source Solanum::Source::Compute.new(block)
+  end
+
+
+  # Registers a pair of [matcher, prototype] where matcher is generally a string
+  # or regex to match a service name, and prototype is a map of :ttl, :state,
+  # :tags, etc.
+  def service(service, prototype={})
+    @services << [service, prototype]
+  end
+
+
+  ##### HELPER METHODS #####
+
+  # Creates a state function based on thresholds. If the first argument is a
+  # symbol, it is taken as the default service state. Otherwise, arguments should
+  # be alternating numeric thresholds and state values to assign if the metric
+  # value exceeds the threshold.
+  #
+  # For example, for an 'availability' metric you often want to warn on low
+  # values. To assign a 'critical' state to values between 0% and 10%,
+  # 'warning' between 10% and 25%, and 'ok' above, use the following:
+  #
+  #     thresholds(0.00, :critical, 0.10, :warning, 0.25, :ok)
+  #
+  # For 'usage' metrics it's the inverse, giving low values ok states and
+  # warning about high values:
+  #
+  #     thresholds(:ok, 55, :warning, 65, :critical)
+  #
+  def thresholds(*args)
+    default_state = nil
+    default_state = args.shift unless args.first.kind_of? Numeric
+
+    # Check arguments.
+    raise "Thresholds must be paired with state values" unless args.count.even?
+    args.each_slice(2) do |threshold|
+      limit, state = *threshold
+      raise "Limits must be numeric: #{limit}" unless limit.kind_of? Numeric
+      raise "State values must be strings or symbols: #{state}" unless state.instance_of?(String) || state.instance_of?(Symbol)
+    end
+
+    # State block.
+    lambda do |v|
+      state = default_state
+      args.each_slice(2) do |threshold|
+        if threshold[0] < v
+          state = threshold[1]
+        else
+          break
+        end
+      end
+      state
+    end
+  end
+
+end

data/lib/solanum/matcher.rb

@@ -0,0 +1,70 @@
+require 'json'
+
+# A matcher takes in an input string and returns a hash of measurement names to
+# numeric values.
+#
+# Author:: Greg Look
+class Solanum::Matcher
+  attr_reader :fn
+
+  # Creates a new Matcher which will run the given function on input.
+  def initialize(fn)
+    raise "function must be provided" if fn.nil?
+    @fn = fn
+  end
+
+  # Attempts to match the given input, returning a hash of metrics.
+  def call(input)
+    {}
+  end
+
+
+  ### MATCHER TYPES ###
+
+  public
+
+  # LinePattern matchers define a regular expression which is tested against
+  # each line of input. The given function is called for **each** matched line,
+  # and the resulting measurements are merged together.
+  class LinePattern < Solanum::Matcher
+    def initialize(fn, pattern)
+      super fn
+      raise "pattern must be provided" if pattern.nil?
+      @pattern = pattern
+    end
+
+    def call(input)
+      raise "No input provided!" if input.nil?
+      lines = input.split("\n")
+      metrics = {}
+
+      lines.each do |line|
+        begin
+          if @pattern === line
+            measurements = @fn.call($~)
+            metrics.merge!(measurements) if measurements
+          end
+        rescue => e
+          STDERR.puts("Error calculating metrics from line match: #{e.inspect}")
+        end
+      end
+
+      metrics
+    end
+  end
+
+
+  # JsonReader matches a JSON-formatted input string and passes the parsed
+  # object to the block body.
+  class JSONReader < Solanum::Matcher
+    def call(input)
+      begin
+        json = JSON.parse(input)
+        @fn.call(json)
+      rescue => e
+        STDERR.puts("Error matching JSON input: #{e.inspect}")
+        {}
+      end
+    end
+  end
+end

data/lib/solanum/source.rb

@@ -0,0 +1,116 @@
+require 'solanum/matcher'
+
+# This class represents a source of data, whether read from command output,
+# a file on the system, or just calculated from other values. Each source
+# may have multiple matchers, which will be run against the input data to
+# produce metrics. Each matcher sees the _whole_ input.
+#
+# Author:: Greg Look
+class Solanum::Source
+  attr_reader :config, :matchers
+
+  # Creates a new Source
+  def initialize(config)
+    @config = config
+    @matchers = []
+  end
+
+  # Collect input and process it with the defined matchers to produce some
+  # output measurements. The current set of metrics collected in this cycle
+  # will be passed to the measure function.
+  def collect(current_metrics)
+    input = read_input!
+    metrics = {}
+
+    unless input.nil?
+      @matchers.each do |matcher|
+        measurements = matcher.call(input)
+        metrics.merge!(measurements) if measurements
+      end
+    end
+
+    metrics
+  end
+
+  private
+
+  # Collect input data from the given source.
+  def read_input!
+    nil
+  end
+
+  # Declares a matcher for a single line of input.
+  def match(pattern, options={}, &block)
+    raise "pattern must be provided" if pattern.nil?
+
+    commands = 0
+    commands += 1 if options[:record]
+    commands += 1 if block_given?
+    raise "Must specify :record or provide a block to execute" if commands == 0
+    raise "Only one of :record or a block should be provided" if commands > 1
+
+    if options[:record]
+      block = lambda do |matches|
+        value = matches[1]
+        value = value.send(options[:cast]) if options[:cast]
+        value *= options[:scale] if options[:scale]
+        {options[:record] => value}
+      end
+    end
+
+    @matchers << Solanum::Matcher::LinePattern.new(block, pattern)
+  end
+
+  # Declares a matcher for JSON input.
+  def json(&block)
+    @matchers << Solanum::Matcher::JSONReader.new(block)
+  end
+
+
+
+  ### SOURCE TYPES ###
+
+  public
+
+  class File < Solanum::Source
+    def read_input!
+      # Check that file exists and is readable.
+      raise "File does not exist: #{@config}" unless ::File.exists? @config
+      raise "File is not readable: #{@config}" unless ::File.readable? @config
+
+      # Read lines from the file.
+      ::File.read(@config)
+    end
+  end
+
+  class Command < Solanum::Source
+    def read_input!
+      # Locate absolute command path.
+      command, args = @config.split(/\s/, 2)
+      abs_command =
+        if ::File.executable? command
+          command
+        else
+          %x{which #{command} 2> /dev/null}.chomp
+        end
+
+      # Check that command exists and is executable.
+      raise "Command #{command} not found" unless ::File.exist? abs_command
+      raise "Command #{abs_command} not executable" unless ::File.executable? abs_command
+
+      # Run command for output.
+      input = %x{#{abs_command} #{args}}
+      raise "Error executing command: #{abs_command} #{args}" unless $?.success?
+
+      input
+    end
+  end
+
+  class Compute < Solanum::Source
+    def collect(current_metrics)
+      # Compute metrics directly, but don't let the block change the existing
+      # metrics in-place.
+      @config.call(current_metrics.dup.freeze)
+    end
+  end
+end

data/lib/solanum.rb

@@ -0,0 +1,81 @@
+# Class which wraps up an active Solanum monitoring system into an object.
+#
+# Author:: Greg Look
+class Solanum
+  attr_reader :sources, :services, :metrics
+
+  require 'solanum/config'
+  require 'solanum/source'
+
+
+  # Loads the given monitoring scripts and initializes the sources and service
+  # definitions.
+  def initialize(scripts)
+    @sources = []
+    @services = []
+    @metrics = {}
+
+    scripts.each do |path|
+      begin
+        config = Solanum::Config.new(path)
+        @sources.concat(config.sources)
+        @services.concat(config.services)
+      rescue => e
+        STDERR.puts "Error loading monitor script #{path}: #{e}"
+      end
+    end
+
+    @sources.freeze
+    @services.freeze
+  end
+
+
+  # Collects metrics from the given sources, in order. Updates the internal
+  # merged map of metric data.
+  def collect!
+    @old_metrics = @metrics
+    @metrics = @sources.reduce({}) do |metrics, source|
+      begin
+        new_metrics = source.collect(metrics) || {}
+        metrics.merge(new_metrics)
+      rescue => e
+        STDERR.puts "Error collecting metrics from #{source}: #{e}"
+        metrics
+      end
+    end
+  end
+
+
+  # Builds full events from a set of service prototypes, old metrics, and new
+  # metrics.
+  def build_events(defaults={})
+    @metrics.keys.sort.map do |service|
+      value = @metrics[service]
+      prototype = @services.select{|m| m[0] === service }.map{|m| m[1] }.reduce({}, &:merge)
+
+      state = prototype[:state] ? prototype[:state].call(value) : :ok
+      tags = ((prototype[:tags] || []) + (defaults[:tags] || [])).uniq
+      ttl = prototype[:ttl] || defaults[:ttl]
+
+      if prototype[:diff]
+        last = @old_metrics[service]
+        if last && last <= value
+          value = value - last
+        else
+          value = nil
+        end
+      end
+
+      if value
+        defaults.merge({
+          service: service,
+          metric: value,
+          state: state.to_s,
+          tags: tags,
+          ttl: ttl
+        })
+      end
+    end.compact
+  end
+
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: solanum
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Greg Look
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: riemann-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+description: 
+email: greg@greg-look.net
+executables:
+- solanum
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/solanum/config.rb
+- lib/solanum/matcher.rb
+- lib/solanum/source.rb
+- lib/solanum.rb
+- bin/solanum
+- README.md
+homepage: https://github.com/greglook/solanum
+licenses:
+- Public Domain
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: DSL for custom monitoring configuration
+test_files: []