opstat-master 0.3.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4cf77a8d238ca849725328c7688ca572748289e4d390726db0bce00210c74ffc
+  data.tar.gz: f1f3d1e624f127621bc790496757290a8b94591503ffe9a0c1d75ce6b3c014c8
+SHA512:
+  metadata.gz: aff05eab2eab8ca8e223410af0ab91bf54e407ee88f6bdf11e1fad02bee5ed903b0c4835ae159abd989da3a413cc0e3eb0d59a63c65175546325baf075c50f1a
+  data.tar.gz: d782f03e71d3a5e1d96286a2a1f68302312ccccd49131394dab3c298cef560aeb57de8e74a151ca52bc91300a3cf01e6d3dc42eb41d3698fb30f59aee30c4171

data/.gitignore

@@ -0,0 +1 @@
+opstat-master*.gem

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in opstat.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Krzysztof Tomczyk
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Opstat
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'opstat'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install opstat
+
+## Usage
+
+TODO: i Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/opstatd_master.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'daemons'
+require 'opstat-master'
+ 
+ Daemons.run_proc(
+   'opstatd_master', # name of daemon
+ #  :dir_mode => :normal
+   :dir => '/var/run', # directory where pid file will be stored
+   :backtrace => true,
+ #  :monitor => true,
+   :log_dir => '/var/log',
+   :log_output => true
+ ) do
+   
+   Opstat::Master.main_loop
+ end

data/init.d/opstatd_master

@@ -0,0 +1,21 @@
+#!/sbin/runscript
+# Copyright 1999-2013 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Header: $
+
+depend() {
+  need net
+}
+
+#TODO - parameters - log file, config file, etc.
+start() {
+  ebegin Starting opstat master
+  opstatd_master.rb start
+  eend $?
+}
+
+stop() {
+  ebegin Stoping opstat master
+  opstatd_master.rb stop
+  eend $?
+}

data/lib/opstat-master/common.rb

@@ -0,0 +1,28 @@
+module Opstat
+module Common
+  def self.camelize(term)
+    string = term.to_s
+    string = string.sub(/^[a-z\d]*/) { $&.capitalize }
+    string.gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }.gsub('/', '::')
+  end
+
+  def self.constantize(class_name)
+    names = class_name.split('::')
+    names.shift if names.empty? || names.first.empty?
+
+    constant = Object
+    names.each do |n|
+      name = Opstat::Common.camelize(n)
+      constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+    end
+    constant
+  end 
+
+  def constantize(camel_cased_word)
+    Opstat::Common.constantize(camel_cased_word)
+  end
+
+end
+end
+
+#TODO load only if run on main server

data/lib/opstat-master/config/opstat.yml

@@ -0,0 +1,28 @@
+client:
+  send_data_interval: 10
+
+mq:
+  queue_name: opstat
+  queue_address: 10.5.0.10
+
+plugins:
+  cpu:
+    interval: 3
+  load:
+    interval: 5
+#  meminfo:
+#    interval: 4
+#  diskstats:
+#    interval: 5
+#  vmstat:
+#    interval: 3
+  connectionusage:
+    interval: 3
+#  facter:
+#    interval: 21600
+  oracle_tablespaces_sizes:
+    interval: 3600
+    su_user: oracle
+    db_user: db_user
+    db_password: db_password
+

data/lib/opstat-master/config.rb

@@ -0,0 +1,74 @@
+module Opstat
+  # A pretty sucky config class, ripe for refactoring/improving
+  class Config
+    include Singleton
+
+    def initialize
+      @configured = false
+      @config = ''
+      @config_file = ''
+      #TODO set defaults
+    end
+
+    def load_config(config_file)
+      #set_config_defaults(configfile)
+      @config_file = config_file
+      @config = YAML.load_file(config_file)
+      self.set_defaults
+    end
+
+    def set_defaults
+      @config['client']['send_data_interval'] ||= 30
+      @config['client']['log_level'] ||= "WARN"
+    end
+
+    def get_mongo_config_file_path
+      "#{File.dirname(@config_file)}/mongoid.yml"
+    end
+
+    def get_influx_config
+       get('influxdb')
+    end
+
+    def get_mq_config
+       get('mq')
+    end
+
+    def get(key)
+      @config[key]
+    end
+
+    def init_config(options)
+      load_config(options[:config_file])
+      set_defaults
+      @config[:config_file] = options[:config_file]
+    end
+
+  end
+end
+options = {}
+optparse = OptionParser.new do|opts|
+  # Set a banner, displayed at the top
+  # of the help screen.
+  opts.banner = "Usage: command [options]"
+
+  options[:verbose] = false
+    opts.on( '-v', '--verbose', 'Output more information' ) do
+    options[:verbose] = true
+  end
+
+  #TODO required options
+  opts.on( '-c', '--config-file String', :required,  "Config file path" ) do|l|
+    options[:config_file] = l
+  end
+  options[:config_file] ||= '/etc/opstat/opstat.yml'
+
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+end
+optparse.parse!
+
+Opstat::Config.instance.init_config(options)
+ 

data/lib/opstat-master/db/influx.rb

@@ -0,0 +1,16 @@
+module Opstat
+module DB
+  class Influx
+    include Singleton
+    def initialize
+      @config = Opstat::Config.instance.get_influx_config
+      @influxdb = InfluxDB::Client.new @config
+      @influxdb = InfluxDB::Client.new @config['database'], username: @config['username'], password: @config['password'], time_precision: @config['time_precision']
+    end
+
+    def write_point(name, measurement)
+      @influxdb.write_point(name, measurement)
+    end
+  end
+end
+end

data/lib/opstat-master/db/mongo.rb

@@ -0,0 +1,3 @@
+require 'mongoid'
+mongo_config_file = Opstat::Config.instance.get_mongo_config_file_path
+Mongoid.load!(mongo_config_file)

data/lib/opstat-master/logging.rb

@@ -0,0 +1,20 @@
+require 'log4r'
+
+module Opstat
+module Logging
+  def oplogger
+    return @oplogger if @oplogger
+    @oplogger = Log4r::Logger.new self.class.to_s
+    @oplogger.level = @oplogger.levels.index(log_level)
+    outputter = Log4r::Outputter.stdout
+    outputter.formatter = Log4r::PatternFormatter.new(:pattern => "%l - %C - %m")
+    @oplogger.outputters << outputter
+    @oplogger
+  end
+  def log_level
+    @log_level ||= Opstat::Config.instance.get('client')['log_level']
+    @log_level
+  end
+end
+end
+

data/lib/opstat-master/master.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+#TODO don't allow ip_addres of loopback
+
+  module TaskServer
+    extend Opstat::Logging
+    def self.save(queue_data)
+      oplogger.debug queue_data
+
+      hostname = queue_data["hostname"]
+      ip = queue_data['ip_address']
+      client_version = queue_data['version']
+      send_data_interval = queue_data['send_data_interval']
+
+      queue_data["collected_data"].each do |data|
+        host = Host.find_or_create_by(hostname: hostname, ip_address: ip)
+        plugin = Plugin.find_or_create_by(type: data['plugin'], host_id: host.id)
+	host['send_data_interval'] = send_data_interval
+	plugin['interval'] = data['interval']
+	plugin_dead_ratio = 2
+	unless plugin['interval'].nil?
+	  if plugin['interval'] > host['send_data_interval']
+	    plugin_is_alive_interval = data['interval'] * plugin_dead_ratio
+	  else
+	    plugin_is_alive_interval = host['send_data_interval'] * plugin_dead_ratio
+	  end
+	  plugin['is_alive_time'] = Time.now + plugin_is_alive_interval
+	end
+
+        plugin.save
+	  
+        ##TODO reconsider if it is really needed
+      #client_host.touch
+          ##TODO reconsider if it is really needed
+        #  client_host_plugin.touch
+        Opstat::Parsers::Master.instance.parse_and_save(:host => host, :plugin_data => data, :plugin => plugin)
+      end
+    end
+  end
+
+#TODO EM AMQP reconnect
+##TODO - AMQP config data from /etc
+module Opstat
+module Master
+extend Opstat::Logging
+def self.main_loop
+  mq_config = Opstat::Config.instance.get_mq_config
+  Opstat::Parsers::Master.instance.load_parsers
+  oplogger.info 'START'
+  EventMachine::run do
+    AMQP.start(mq_config) do |connection|
+      oplogger.info "Connecting to AMQP broker. Running #{AMQP::VERSION} version of the gem..."
+
+      channel  = AMQP::Channel.new(connection,:auto_recovery => true)
+      ampqqueue    = channel.queue(mq_config['queue_name'],:auto_delete => false, :durable => true)
+      connection.on_tcp_connection_loss do |conn, settings|
+        oplogger.error "[network failure] Trying to reconnect..."
+        conn.reconnect(false, 2)
+      end
+
+      ampqqueue.subscribe(:ack => true) do |metadata,payload|
+        TaskServer::save(JSON.parse(payload))
+        metadata.ack
+      end
+    end
+  end 
+end
+end
+end

data/lib/opstat-master/opstat-master.rb

@@ -0,0 +1,29 @@
+require "opstat-master/version"
+# Try to load rubygems.  Hey rubygems, I hate you.
+begin
+  require 'rubygems'
+rescue LoadError
+end
+require 'yaml'
+require 'log4r'
+require 'eventmachine'
+require 'opstat-master/db/mongo.rb'
+require 'amqp'
+require 'json'
+require 'singleton'
+require 'opstat-master/config.rb'
+require 'opstat-master/common.rb'
+require 'opstat-master/logging.rb'
+require 'opstat-master/parsers.rb'
+
+require 'opstat-master/master.rb'
+# see the bottom of the file for further inclusions
+
+#------------------------------------------------------------
+# the top-level module
+#
+# all this really does is dictate how the whole system behaves, through
+# preferences for things like debugging
+#
+# it's also a place to find top-level commands like 'debug'
+

data/lib/opstat-master/parsers.rb

@@ -0,0 +1,72 @@
+module Opstat
+module Parsers
+  class Master
+    include Opstat::Logging
+    include Singleton
+    def initialize
+      @parsers = {}
+    end
+    def load_parsers
+      @parsers = Opstat::Plugins.load_parsers(oplogger)
+    end
+
+    def parse_and_save(params)
+      plugin = params[:plugin]
+      host = params[:host]
+      data = params[:plugin_data]['data']
+      time = Time.parse(params[:plugin_data]['timestamp'])
+      oplogger.info "Saving parsed data collected on #{time} from #{host.id}(plugin:#{plugin[:type]} #{host[:hostname]}) "
+      begin
+        reports = @parsers[plugin[:type]].parse_data(data: data, time: time)
+      #TODO save errors to db
+      rescue Exception => e
+        oplogger.error "current params #{params}"
+	raise e
+      end
+      if reports.nil? or reports.empty?
+        oplogger.warn "no report data parsed - empty report for #{plugin.type}?"
+        return
+      end
+      reports.each do |report|
+	default_tags = { :host_id => host.id, :plugin_id => plugin.id, :hostname => host[:hostname] }
+	report_data = report.select{|k,v| not k.to_s.starts_with?('OPSTAT_TAG_')}
+	report_tags = report.select{|k,v| k.to_s.starts_with?('OPSTAT_TAG_')}
+	report_tags ||= {}
+	measurement_tags = report_tags.merge(default_tags)
+	measurement = { :values => report_data, :timestamp => time.to_i, :tags => measurement_tags, :name => plugin.type }
+	Opstat::DB::Influx.instance.write_point(plugin.type, measurement)
+      end
+    end
+  end	     
+end
+end
+
+class Report
+  include Mongoid::Document
+  include Mongoid::Attributes::Dynamic
+  store_in collection: "opstat.reports"
+end
+
+class User
+  include Mongoid::Document
+  include Mongoid::Attributes::Dynamic
+  include Mongoid::Timestamps
+  store_in collection: "opstat.users"
+end
+
+class Host
+  include Mongoid::Document
+  include Mongoid::Attributes::Dynamic
+  include Mongoid::Timestamps
+  store_in collection: "opstat.hosts"
+  has_many :plugins
+end
+
+class Plugin
+  include Mongoid::Document
+  include Mongoid::Attributes::Dynamic
+  include Mongoid::Timestamps
+  store_in collection: "opstat.plugins"
+  belongs_to :host
+end
+

data/lib/opstat-master.rb

@@ -0,0 +1,32 @@
+begin
+  require 'rubygems'
+rescue LoadError
+end
+require 'yaml'
+require 'log4r'
+require 'opstat-plugins'
+require 'eventmachine'
+require 'amqp'
+require 'json'
+require 'singleton'
+require 'influxdb'
+
+require 'activemodel-serializers-xml'
+require 'opstat-master/config.rb'
+require 'opstat-master/common.rb'
+require 'opstat-master/db/mongo.rb'
+require 'opstat-master/db/influx.rb'
+require 'opstat-master/logging.rb'
+require 'opstat-master/parsers.rb'
+
+require 'opstat-master/master.rb'
+# see the bottom of the file for further inclusions
+
+#------------------------------------------------------------
+# the top-level module
+#
+# all this really does is dictate how the whole system behaves, through
+# preferences for things like debugging
+#
+# it's also a place to find top-level commands like 'debug'
+

data/opstat-master.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require '../version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "opstat-master"
+  spec.version       = Opstat::VERSION
+  spec.authors       = ["Krzysztof Tomczyk"]
+  spec.email         = ["office@optilabs.eu"]
+  spec.description   = %q{Systems monitoring tool}
+  spec.summary       = %q{Systems monitoring tool that works}
+  spec.homepage      = "http://www.optilabs.eu/opstat"
+  spec.license       = "GPL-3.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "opstat-plugins", Opstat::VERSION
+  spec.add_dependency "eventmachine"
+  spec.add_dependency "activemodel-serializers-xml"
+  spec.add_dependency "json"
+  spec.add_dependency "amqp"
+  spec.add_dependency "daemons"
+  spec.add_dependency "bson_ext"
+  spec.add_dependency "xml-simple"
+  spec.add_dependency "xmlhasher"
+  spec.add_dependency "log4r"
+  spec.add_dependency "influxdb"
+  spec.add_dependency "mongoid", "~> 6"
+end

metadata

@@ -0,0 +1,231 @@
+--- !ruby/object:Gem::Specification
+name: opstat-master
+version: !ruby/object:Gem::Version
+  version: 0.3.8
+platform: ruby
+authors:
+- Krzysztof Tomczyk
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-09-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: opstat-plugins
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.8
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activemodel-serializers-xml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: xml-simple
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: xmlhasher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: log4r
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: influxdb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mongoid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6'
+description: Systems monitoring tool
+email:
+- office@optilabs.eu
+executables:
+- opstatd_master.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README
+- README.md
+- Rakefile
+- bin/opstatd_master.rb
+- init.d/opstatd_master
+- lib/opstat-master.rb
+- lib/opstat-master/common.rb
+- lib/opstat-master/config.rb
+- lib/opstat-master/config/opstat.yml
+- lib/opstat-master/db/influx.rb
+- lib/opstat-master/db/mongo.rb
+- lib/opstat-master/logging.rb
+- lib/opstat-master/master.rb
+- lib/opstat-master/opstat-master.rb
+- lib/opstat-master/parsers.rb
+- opstat-master.gemspec
+homepage: http://www.optilabs.eu/opstat
+licenses:
+- GPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.33
+signing_key: 
+specification_version: 4
+summary: Systems monitoring tool that works
+test_files: []