salus 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0adb5863d10056a25ec392981e682cdc8a27fe88
+  data.tar.gz: 54011bd7dc54a824778e604ea85f54040ad1445c
+SHA512:
+  metadata.gz: bfc09ce8d330fea7668c58062fba29cff5d80d6cfd4394fa80f922d6a97d6fe636cb49cef1f2c344d3915578ab715200adb28a871857c9de87b1da2daf53b262
+  data.tar.gz: a23b0c69dcd2e6e5f6caf23f8513a5a8678f58857aa30b710652096ef121c0946fa1f4fbf32b85f62e88f2df2621f26f84d5a176deb24a2a08cffa0904e3b5a2

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.15.3

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in salus.gemspec
+gemspec

data/LICENSE.md

@@ -0,0 +1,29 @@
+Salus is licensed under the BSD license.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    (1) Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+    (2) Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.  
+
+    (3) The name of the author may not be used to
+    endorse or promote products derived from this software without
+    specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,254 @@
+# Salus
+
+Salus is a simple DSL for writing collector agents for different monitoring systems. I'm just tired of rewriting those primitives from scratch for every new check I'm willing to add to a monitoring system.
+
+_This is alpha quality software right now, but you might help to improve it_
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'salus'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install salus
+
+## Usage
+
+The gem can be used from your own script or by using CLI.
+
+Quick sample:
+
+```ruby
+require "json"
+default ttl: 60
+var state_file: "system.state.yml"
+
+group "cpu" do
+  data = File.open("/proc/stat").read.split(/\n/).grep(/^cpu /)
+  data.each do |l|
+    name, user, nice, csystem, idle, iowait, irq, softirq = l.split(/\s+/)
+
+    busy  = user.to_i + nice.to_i + csystem.to_i + iowait.to_i + irq.to_i + softirq.to_i
+    total = busy + idle.to_i
+
+    counter "busy", value: busy, mute: true
+    counter "total", value: total, mute: true
+    gauge "usage" do
+      value("busy") / value("total") * 100
+    end
+  end
+end
+
+group "memory" do
+  data = File.open("/proc/meminfo").read.split(/\n/).grep(/^Mem/)
+  data.each do |l|
+    name, value = l.match(/Mem(?<name>.+):\s+(?<value>\d+)\s/)[1,2]
+    gauge name.downcase, value: value.to_i, mute: true
+  end
+  gauge "usage" do
+    (value("total") - value("free")) / value("total").to_f * 100
+  end
+end
+
+render do |data|
+  iterate(data) do |name, metric|
+    puts ({:name => name, :value => metric.value, :timestamp => metric.timestamp, :ttl => metric.ttl}.to_json)
+  end
+end
+```
+
+Save it as `sample.salus` and run `salus`.
+
+```bash
+$ salus -f sample.salus
+{"name":"cpu.usage","value":6.433521607455635,"timestamp":1510699623.8592708,"ttl":60}
+{"name":"memory.usage","value":25.60121068625779,"timestamp":1510699623.863265,"ttl":60}
+```
+
+Because `cpu.usage` is made of counters, you'll have to run the command at least twice. Be aware of `salus` making `system.state.yml` for persisting it's state. You may redefine file name and location with `-s` switch.
+
+You can also run in infinite loop mode
+
+```bash
+$ salus loop -f sample.salus
+{"name":"cpu.usage","value":12.528473606797895,"timestamp":1510700076.4455612,"ttl":60}
+{"name":"memory.usage","value":25.57683405209171,"timestamp":1510700076.447059,"ttl":60}
+{"name":"cpu.usage","value":8.239946939924048,"timestamp":1510700106.3334389,"ttl":60}
+{"name":"memory.usage","value":25.535366304014968,"timestamp":1510700106.33444,"ttl":60}
+{"name":"cpu.usage","value":2.840158185017875,"timestamp":1510700136.364328,"ttl":60}
+{"name":"memory.usage","value":25.537265316671707,"timestamp":1510700136.36533,"ttl":60}
+^C
+```
+
+You may invoke several salus scripts at once, just specify all of them in a space delimited list. You might also specify a directory with `Salusfile` or many `*.salus` files. By default `salus` does search `*.salus` and `Salusfile` in the current directory.
+
+### Primitives
+
+#### Group
+
+Group is the base unit of work. You should write your metric collecting code inside groups. Groups can be nested. Top level groups are run in separate threads. Group might include metrics. Groups must be named.
+
+```ruby
+group "test" do
+  gauge "test1", value: 10
+  gauge "test2", value: 30, mute: true
+end
+```
+
+#### Metrics
+
+Could be one of the following (mimicking RRDtool data sources):
+ * Gauge: values stored as is
+ * Derive: a rate of something per second, best suites for values that rarely overflow
+ * Counter: almost same as derive, but with overflow detection for 32- and 64-bit counters
+ * Absolute: a rate of a counter, which resets on reading
+ * Text: just a text stored as is
+
+A metric should have a name and a value at the very minimum. You might also specify a custom or default TTL. A metric can be also `mute`, which means it wouldn't appear in the output, unless told to do so.
+
+An expired metric is considered invalid, so if you use counter or derive with ttl less than collecting interval, you'll always get nils.
+
+```ruby
+group "test" do
+  gauge   "test1", value: 10, ttl: 50
+  counter "test2", value: 10, mute: true
+  derive  "test3", value: 30, timestamp: Time.now.to_f + 10
+end
+```
+
+You might also use a block to calculate metrics value. In this case any exception which happened in the block would be muted, and the nil value returned instead.
+
+```ruby
+group "test" do
+  # this would always produce nil
+  gauge "division by zero" do
+    100 / 0
+  end
+end
+```
+
+#### Renderers
+
+A renderer is a class, which is used to render the actual output, would it be just STDOUT, a file or tcp/ip service.
+
+```ruby
+render(StdoutRenderer.new(separator: "/"))
+```
+
+This code would produce something like that
+
+```
+[2017-11-15 02:25:16 +0300] cpu/usage - 4.00
+[2017-11-15 02:25:16 +0300] memory/usage - 26.23
+```
+
+You may add more than one renderer at once and send your data to as many monitoring services as you want.
+
+Check sample renderers for examples.
+
+### Pipeline
+
+Salus pipeline is rather straightforward and consists of two stages:
+ * Collect data (execute groups' code)
+ * Send data (execute renderers' code)
+
+You might run it once by cron or in infinite loop mode. Each stage is executed using embed thread pool with pre-set timeouts.
+
+Thread pool and timeouts could be configured using `configure`
+
+```ruby
+Salus.configure do |config|
+  # Thread pool settings
+  config.min_threads = (CPU.count / 2 == 0) ? 1 : CPU.count / 2
+  config.max_threads = CPU.count * 2
+
+  config.interval = 30 # Interval between runs in loop mode
+  config.tick_timeout   = 15 # Data collection timeout
+  config.render_timeout = 10 # Data rendering timeout
+  config.logger   = Logger.new(STDERR) # Default logger
+end
+```
+
+### Zabbix
+
+Zabbix uses two stage collecting. First of all, it queries (discovers) the list of objects to be checked. Next, it would ask for exact values of specified metrics of an object one by one. Sometimes this means making a lot of requests to a monitored service. So many script writers use some kind of result caching to lower unnecessary work. Salus also writes a result cache file (`-c` flag). Cache TTL for a metric is a half of it's real TTL or 60 seconds if TTL is unspecified. Upon parameter request it is loaded from cache and if it's expired, whole cache is invalidated and recalculated using salus script.
+
+Sample Salus script for collecting CPU usage ratio on Linux for Zabbix Agent is something like that:
+
+```ruby
+require "salus/zabbix"
+
+default ttl: 60
+var state_file: "cpu.state.yml"
+var zabbix_cache_file: "cpu.cache.yml"
+
+discover "cpus" do |data|
+  stat = File.open("/proc/stat").read.split(/\n/).grep(/^cpu\d/)
+  stat.each do |l|
+    name, = l.split(/\s+/)
+    data << {"\#{CPUNAME}" => name}
+  end
+end
+
+group "cpu" do
+  stat = File.open("/proc/stat").read.split(/\n/).grep(/^cpu\d/)
+  stat.each do |l|
+    name, user, nice, csystem, idle, iowait, irq, softirq = l.split(/\s+/)
+
+    busy  = user.to_i + nice.to_i + csystem.to_i + iowait.to_i + irq.to_i + softirq.to_i
+    total = busy + idle.to_i
+
+    counter "busy[#{name}]", value: busy, mute: true
+    counter "total[#{name}]", value: total, mute: true
+    gauge "usage[#{name}]" do
+      value("busy[#{name}]") / value("total[#{name}]") * 100
+    end
+  end
+end
+```
+
+You can run it using command `salus`.
+
+```bash
+$ salus zabbix discover cpus -f zabbix.salus
+{"data":[{"#{CPUNAME}":"cpu0"},{"#{CPUNAME}":"cpu1"},{"#{CPUNAME}":"cpu2"},{"#{CPUNAME}":"cpu3"},{"#{CPUNAME}":"cpu4"},{"#{CPUNAME}":"cpu5"},{"#{CPUNAME}":"cpu6"},{"#{CPUNAME}":"cpu7"}]}
+```
+
+Later you can get a cpu usage ratio
+
+```bash
+$ salus zabbix parameter cpu.usage[cpu5] -f zabbix.salus && sleep 31 && salus zabbix parameter cpu.usage[cpu5] -f zabbix.salus
+8.619550926524335
+```
+
+**NOTE!** You won't get the result on the first run, because cpu usage on Linux needs to get two points in time to be calculated. `zabbix` subcommand uses caching of the results, so you have to wait for 30 seconds to get next result. But if you'll wait for more than TTL (60 seconds), you'll get empty result again.
+
+New dependant items mode of Zabbix 3.4+ is also supported. Output is in JSON.
+
+```bash
+$ salus zabbix bulk cpu -f zabbix.salus
+{"usage[cpu0]":8.362702709145317,"usage[cpu1]":2.2395325673158424,"usage[cpu2]":3.570268951237728,"usage[cpu3]":4.641349636608478,"usage[cpu4]":3.7313418117547834,"usage[cpu5]":4.023359928822469,"usage[cpu6]":3.0834133549568365,"usage[cpu7]":4.47470699450407}
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+## Special thanks
+ * meh for [ruby-thread](https://github.com/meh/ruby-thread)
+ * all folks of [concurrent](https://github.com/ruby-concurrency/concurrent-ruby) project
+ * all folks of [thor](https://github.com/erikhuda/thor) project
+
+Salus uses portions of code (meh's thread pool and future implementation) and concepts from both project and thor for CLI implementation.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/divanikus/salus.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "salus"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/salus

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+require "salus"
+require "salus/cli"
+
+begin
+  Salus::CLI.start
+rescue Exception => e
+  unless e.is_a? Interrupt
+    STDERR.puts "Error: " + e.message
+    exit(1)
+  end
+end

data/lib/salus.rb

@@ -0,0 +1,183 @@
+require "salus/version"
+require "salus/logging"
+require "salus/thread"
+require "salus/group"
+require "salus/configuration"
+require "salus/renderer"
+
+module Salus
+  extend Configuration
+
+  class << self
+    include Logging
+    @@_groups = {}
+    @@_renders= []
+    @@_opts   = {}
+    @@_vars   = {}
+    @@_lazy   = []
+
+    def on_win?
+      @@_win ||= !(RUBY_PLATFORM =~ /bccwin|cygwin|djgpp|mingw|mswin|wince/i).nil?
+    end
+
+    def group(title, &block)
+      unless @@_groups.key?(title)
+        @@_groups[title] = Group.new(@@_opts, &block)
+      end
+    end
+
+    def groups
+      @@_groups
+    end
+    alias root groups
+
+    def default(opts)
+      return unless opts.is_a?(Hash)
+      opts.each do |k, v|
+        next if [:value, :timestamp].include?(k)
+        @@_opts[k] = v
+      end
+    end
+
+    def defaults
+      @@_opts
+    end
+
+    def var(arg, default=nil, &block)
+      if arg.is_a?(Hash)
+        arg.each {|k, v| @@_vars[k] = v}
+      elsif block_given?
+        @@_vars[arg.to_sym] = block
+      else
+        value = @@_vars.fetch(arg.to_sym, default)
+        # Dumb lazy loading
+        @@_vars[arg.to_sym] = if value.is_a?(Proc)
+          begin
+            value = value.call
+          rescue Exception => e
+            log DEBUG, e
+            value = default
+          end
+        end
+        value
+      end
+    end
+    alias let var
+
+    def vars
+      @@_vars
+    end
+
+    def render(obj=nil, &block)
+      if block_given?
+        @@_renders << BlockRenderer.new(&block)
+      else
+        unless obj.is_a? Salus::BaseRenderer
+          log ERROR, "#{obj.class} must be a subclass of Salus::BaseRenderer"
+          return
+        end
+        @@_renders << obj
+      end
+    end
+
+    def renders
+      @@_renders
+    end
+
+    def reset
+      @@_groups  = {}
+      @@_renders = []
+      @@_opts    = {}
+      @@_vars    = {}
+      @@_lazy    = []
+      if defined?(@@_pool) && @@_pool.is_a?(Salus::ThreadPool)
+        @@_pool.shutdown!
+        @@_pool = nil
+      end
+    end
+
+    def lazy(&block)
+      raise ArgumentError, "Block should be given" unless block_given?
+      @@_lazy << block
+    end
+
+    def lazy_eval
+      # Lazy eval blocks once
+      return if @@_lazy.empty?
+      @@_lazy.each { |block| instance_eval(&block) }
+      @@_lazy.clear
+    end
+
+    def load(file)
+      instance_eval(File.read(file), File.basename(file), 0) if File.exists?(file)
+    end
+
+    def load_state(&block)
+      data = block.call
+      return unless data
+      return if data.empty?
+      lazy_eval
+      data.each do |k, v|
+        @@_groups[k].load(v) if @@_groups.key?(k)
+      end
+    end
+
+    def save_state(&block)
+      data = {}
+      @@_groups.each { |k, v| data[k]  = v.to_h }
+      block.call(data)
+    end
+
+    def tick
+      lazy_eval
+      return if @@_groups.empty?
+      pause = (Salus.interval - Salus.tick_timeout - Salus.render_timeout) / 2
+      pause = 1 if (pause <= 0)
+
+      latch = CountDownLatch.new(@@_groups.count)
+      @@_groups.each do |k, v|
+        pool.process do
+          begin
+            v.tick
+            latch.count_down
+          rescue Exception => e
+            log ERROR, e
+            latch.count_down
+          end
+        end.timeout_after(Salus.tick_timeout)
+      end
+      latch.wait(Salus.tick_timeout + pause)
+      log DEBUG, "Collection finished. Threads: #{pool.spawned} spawned, #{pool.waiting} waiting, #{Thread.list.count} total"
+
+      return if @@_renders.empty?
+      latch = CountDownLatch.new(@@_renders.count)
+      @@_renders.each do |v|
+        pool.process do
+          begin
+            v.render(root)
+            latch.count_down
+          rescue Exception => e
+            log ERROR, e
+            latch.count_down
+          end
+        end.timeout_after(Salus.render_timeout)
+      end
+      latch.wait(Salus.render_timeout + pause)
+      log DEBUG, "Rendering finished. Threads: #{pool.spawned} spawned, #{pool.waiting} waiting, #{Thread.list.count} total"
+    end
+
+    def run
+      loop do
+        pool.process do
+          tick
+        end
+        sleep Salus.interval
+      end
+    end
+
+    protected
+    def pool
+      @@_pool ||= ThreadPool.new(self.min_threads, self.max_threads).auto_trim!
+    end
+  end
+end