bipbip 0.5.24 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ebd236b49b6ae015737b235b91ccc37957c4e003
-  data.tar.gz: e11213f8f95ea6d62e4f074ae1a51d4926139f3f
+  metadata.gz: 3dad495dcac92d89bce9202405e212fc5395cdc2
+  data.tar.gz: 63e38ea535ac06437ab9f15bcb916f74d8b6fc1a
 SHA512:
-  metadata.gz: a299ac1d5dd426a73b07f32a68bed3035289347895b7902a41a7951017bebb0aa5bb37f237f5ca8e2f34cdd0e68b3db25225842b1e9bf4cd79383f41c0b9af92
-  data.tar.gz: a2f6f16f4e0bf25f49f195ef00ce794e9fb9f12b6a12050431a8752d4fae2f836395252577ec3070c3f8df7eab1a8181d218cad2c34ec754eaab61e9b86843cb
+  metadata.gz: 18dfce67a0feeb23f8370ff3d20c49dd574f2087a2e6f9fc6e3a9cd215144bff93bb73e6b887fd156396492542e74aed87c3b914a54b711c48b159996ef09fd0
+  data.tar.gz: 80f6389027697b47ed98bad817bea8335cee6ef8ee0f741eadf44626dc73e65e86ac3252615aa96aa615e75fc26ce7fb71ac02e45135462ba3359f685a3aa3e1

data/bin/bipbip

@@ -29,7 +29,7 @@ end
 
 begin
   optparse.parse!
-  if not options[:config]
+  unless options[:config]
     puts "Missing options: config file [-c PATH]"
     exit
   end
@@ -40,8 +40,10 @@ rescue OptionParser::InvalidOption, OptionParser::MissingArgument
 end
 
 begin
-  Bipbip::Agent.new(options[:config]).run
+  config = Bipbip::Config.factory_from_file(options[:config])
+  agent = Bipbip::Agent.new(config)
+  agent.run
 rescue => e
-  Bipbip.logger.fatal e.message + "\n" + e.backtrace.map{|s| "\t#{s}"}.join("\n")
+  Bipbip.logger.fatal e.message + "\n" + e.backtrace.map { |s| "\t#{s}" }.join("\n")
   exit 1
 end

data/lib/bipbip.rb

@@ -7,12 +7,14 @@ module Bipbip
   require 'logger'
   require 'socket'
   require 'shellwords'
+  require 'thwait'
+  require 'timeout'
 
   require 'interruptible_sleep'
-  require 'process_exists'
 
   require 'bipbip/version'
   require 'bipbip/helper'
+  require 'bipbip/config'
   require 'bipbip/agent'
   require 'bipbip/storage'
   require 'bipbip/plugin'

data/lib/bipbip/agent.rb

@@ -1,17 +1,21 @@
 module Bipbip
 
   class Agent
+    include InterruptibleSleep
 
     PLUGIN_RESPAWN_DELAY = 5
 
     attr_accessor :plugins
     attr_accessor :storages
+    attr_accessor :threads
 
-    def initialize(config_file = nil)
-      @plugins = []
-      @storages = []
+    # @param [Bipbip::Config] config
+    def initialize(config)
+      @plugins = config.plugins
+      @storages = config.storages
+      Bipbip.logger = config.logger
 
-      load_config(config_file) if config_file
+      @threads = []
     end
 
     def run
@@ -29,88 +33,53 @@ module Bipbip
         end
       end
 
-      ['INT', 'TERM'].each { |sig| trap(sig) {
-        Thread.new do
+      ['INT', 'TERM'].each do |sig|
+        trap(sig) do
+          Bipbip.logger.info "Received signal #{sig}, interrupting..."
           interrupt
-          exit
         end
-      } }
+      end
 
       @plugins.each do |plugin|
         Bipbip.logger.info "Starting plugin #{plugin.name} with config #{plugin.config}"
-        plugin.run(@storages)
+        start_plugin(plugin, @storages)
       end
 
       @interrupted = false
       until @interrupted
-        pid = Process.wait(-1)
+        thread = ThreadsWait.new(@threads).next_wait
+        @threads.delete(thread)
+        plugin = thread['plugin']
         next if @interrupted
-        plugin = plugin_by_pid(pid)
-        Bipbip.logger.error "Plugin #{plugin.name} with config #{plugin.config} died. Respawning..."
-        sleep(PLUGIN_RESPAWN_DELAY)
-        plugin.run(@storages)
-      end
-    end
 
-    def load_config(config_file)
-      config = YAML.load(File.open(config_file))
-      config = {
-          'logfile' => STDOUT,
-          'loglevel' => 'INFO',
-          'frequency' => 60,
-          'include' => nil,
-          'services' => [],
-          'tags' => [],
-      }.merge(config)
-
-      Bipbip.logger = Logger.new(config['logfile'])
-      Bipbip.logger.level = Logger::const_get(config['loglevel'])
-
-      services = config['services'].to_a
-      if config['include']
-        include_path = File.expand_path(config['include'].to_s, File.dirname(config_file))
-
-        files = Dir[include_path + '/**/*.yaml', include_path + '/**/*.yml']
-        services += files.map { |file| YAML.load(File.open(file)) }
-      end
-
-      @plugins = services.map do |service|
-        plugin_name = service['plugin']
-        metric_group = service['metric_group']
-        frequency = service['frequency'].nil? ? config['frequency'] : service['frequency']
-        tags = config['tags'].to_a + service['tags'].to_a
-        plugin_config = service.reject { |key, value| ['plugin', 'frequency', 'tags', 'metric_group'].include?(key) }
-        Bipbip::Plugin.factory(plugin_name, plugin_config, frequency, tags, metric_group)
-      end
+        Bipbip.logger.error "Plugin #{plugin.name} with config #{plugin.config} terminated. Restarting..."
+        interruptible_sleep(PLUGIN_RESPAWN_DELAY)
+        next if @interrupted
 
-      storages = config['storages'].to_a
-      @storages = storages.map do |storage|
-        storage_name = storage['name'].to_s
-        storage_config = storage.reject { |key, value| ['name'].include?(key) }
-        Bipbip::Storage.factory(storage_name, storage_config)
+        # Re-instantiate plugin to get rid of existing database-connections etc
+        plugin_new = Bipbip::Plugin.factory_from_plugin(plugin)
+        @plugins.delete(plugin)
+        @plugins.push(plugin_new)
+        start_plugin(plugin_new, @storages)
       end
     end
 
     def interrupt
       @interrupted = true
-
-      Bipbip.logger.info 'Interrupt, killing plugin processes...'
-      @plugins.each do |plugin|
-        Process.kill('TERM', plugin.pid) if Process.exists?(plugin.pid)
+      @threads.each do |thread|
+        thread.terminate
       end
-
-      Bipbip.logger.info 'Waiting for all plugin processes to exit...'
-      Process.waitall
+      interrupt_sleep
     end
 
     private
 
-    def plugin_by_pid(pid)
-      plugin = @plugins.find { |plugin| plugin.pid == pid }
-      if plugin.nil?
-        raise "Cannot find plugin with pid #{pid}"
-      end
-      plugin
+    # @param [Bipbip::Plugin] plugin
+    # @param [Array<Bipbip::Storage>] storages
+    def start_plugin(plugin, storages)
+      thread = Thread.new { plugin.run(storages) }
+      thread['plugin'] = plugin
+      @threads.push(thread)
     end
   end
 end

data/lib/bipbip/config.rb

@@ -0,0 +1,62 @@
+module Bipbip
+
+  class Config
+
+    attr_reader :plugins
+    attr_reader :storages
+    attr_reader :logger
+
+    # @param [String] file_path
+    # @return [Bipbip::Config]
+    def self.factory_from_file(file_path)
+      config = YAML.load(File.open(file_path))
+      config = {
+        'logfile' => STDOUT,
+        'loglevel' => 'INFO',
+        'frequency' => 60,
+        'include' => nil,
+        'services' => [],
+        'tags' => [],
+      }.merge(config)
+
+      logger = Logger.new(config['logfile'])
+      logger.level = Logger::const_get(config['loglevel'])
+
+      plugins_config = config['services'].to_a
+      if config['include']
+        include_path = File.expand_path(config['include'].to_s, File.dirname(file_path))
+
+        files = Dir[include_path + '/**/*.yaml', include_path + '/**/*.yml']
+        plugins_config += files.map { |file| YAML.load(File.open(file)) }
+      end
+
+      plugins = plugins_config.map do |service|
+        plugin_name = service['plugin']
+        metric_group = service['metric_group']
+        frequency = service['frequency'].nil? ? config['frequency'] : service['frequency']
+        tags = config['tags'].to_a + service['tags'].to_a
+        plugin_config = service.reject { |key, value| ['plugin', 'frequency', 'tags', 'metric_group'].include?(key) }
+        Bipbip::Plugin.factory(plugin_name, plugin_config, frequency, tags, metric_group)
+      end
+
+      storages_config = config['storages'].to_a
+      storages = storages_config.map do |storage|
+        storage_name = storage['name'].to_s
+        storage_config = storage.reject { |key, value| ['name'].include?(key) }
+        Bipbip::Storage.factory(storage_name, storage_config)
+      end
+
+      Bipbip::Config.new(plugins, storages, logger)
+    end
+
+    # @param [Array<Bipbip::Plugin>] [plugins]
+    # @param [Array<Bipbip::Storage>] [storages]
+    # @param [Logger] [logger]
+    def initialize(plugins = nil, storages = nil, logger = nil)
+      @plugins = plugins || []
+      @storages = storages || []
+      @logger = logger || Logger.new(STDOUT)
+    end
+
+  end
+end

data/lib/bipbip/plugin.rb

@@ -1,19 +1,30 @@
 module Bipbip
 
   class Plugin
+
+    class MeasurementTimeout < RuntimeError
+    end
+
     include InterruptibleSleep
 
     attr_accessor :name
     attr_accessor :config
-    attr_accessor :metric_group
+    attr_accessor :frequency
     attr_accessor :tags
-    attr_accessor :pid
+    attr_accessor :metric_group
 
     def self.factory(name, config, frequency, tags, metric_group = nil)
       require "bipbip/plugin/#{Bipbip::Helper.name_to_filename(name)}"
       Plugin::const_get(Bipbip::Helper.name_to_classname(name)).new(name, config, frequency, tags, metric_group)
     end
 
+    # @param [Bipbip::Plugin] plugin
+    # @return [Bipbip::Plugin]
+    def self.factory_from_plugin(plugin)
+      class_name = plugin.class.name.gsub(/^Bipbip::/, '')
+      Bipbip::const_get(class_name).new(plugin.name, plugin.config, plugin.frequency, plugin.tags, plugin.metric_group)
+    end
+
     def initialize(name, config, frequency, tags = nil, metric_group = nil)
       @name = name.to_s
       @config = config.to_hash
@@ -22,49 +33,30 @@ module Bipbip
       @metric_group = (metric_group || name).to_s
     end
 
+    # @param [Array] storages
     def run(storages)
-      @pid = fork do
-        ['INT', 'TERM'].each { |sig| trap(sig) {
-          Thread.new { interrupt } if !@interrupted
-        } }
-
-        retry_delay = frequency
-        begin
-          until interrupted? do
-            time = Time.now
-            data = monitor
-            if data.empty?
-              raise "#{name} #{source_identifier}: Empty data"
-            end
-            log(Logger::DEBUG, "Data: #{data}")
-            storages.each do |storage|
-              storage.store_sample(self, time, data)
-            end
-            retry_delay = frequency
-            interruptible_sleep (frequency - (Time.now - time))
+      begin
+        timeout = frequency * 2
+        while true
+          time = Time.now
+          Timeout::timeout(timeout, MeasurementTimeout) do
+            run_measurement(frequency, storages)
           end
-        rescue => e
-          log_exception(Logger::ERROR, e)
-          interruptible_sleep retry_delay
-          retry_delay += frequency if retry_delay < frequency * 10
-          retry
-        rescue Exception => e
-          log_exception(Logger::FATAL, e)
-          raise e
+          interruptible_sleep (frequency - (Time.now - time))
         end
+      rescue MeasurementTimeout => e
+        log(Logger::ERROR, "Measurement timeout of #{timeout} seconds reached.")
+        retry
+      rescue StandardError => e
+        log_exception(Logger::ERROR, e)
+        interruptible_sleep frequency
+        retry
+      rescue Exception => e
+        log_exception(Logger::FATAL, e)
+        raise e
       end
     end
 
-    def interrupt
-      log(Logger::INFO, "Interrupting plugin process #{Process.pid}")
-      @interrupted = true
-      interrupt_sleep
-    end
-
-    def interrupted?
-      @interrupted || Process.getpgid(Process.ppid) != Process.getpgrp
-    end
-
     def frequency
       @frequency
     end
@@ -91,6 +83,17 @@ module Bipbip
 
     private
 
+    def run_measurement(time, storages)
+      data = monitor
+      if data.empty?
+        raise "#{name} #{source_identifier}: Empty data"
+      end
+      log(Logger::DEBUG, "Data: #{data}")
+      storages.each do |storage|
+        storage.store_sample(self, time, data)
+      end
+    end
+
     def log(severity, message)
       Bipbip.logger.add(severity, message, "#{name} #{source_identifier}")
     end

data/lib/bipbip/version.rb

@@ -1,3 +1,3 @@
 module Bipbip
-  VERSION = '0.5.24'
+  VERSION = '0.6.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bipbip
 version: !ruby/object:Gem::Version
-  version: 0.5.24
+  version: 0.6.0
 platform: ruby
 authors:
 - Cargo Media
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-10 00:00:00.000000000 Z
+date: 2015-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: copperegg-revealmetrics
@@ -138,20 +138,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.10'
-- !ruby/object:Gem::Dependency
-  name: process_exists
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.1.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.1.3
 - !ruby/object:Gem::Dependency
   name: rb-inotify
   requirement: !ruby/object:Gem::Requirement
@@ -222,6 +208,7 @@ files:
 - data/php-opcache-status.php
 - lib/bipbip.rb
 - lib/bipbip/agent.rb
+- lib/bipbip/config.rb
 - lib/bipbip/helper.rb
 - lib/bipbip/plugin.rb
 - lib/bipbip/plugin/apache2.rb
@@ -268,7 +255,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: Gather services data and store in CopperEgg