onesnooper-server 0.0.1

data/lib/onesnooper_server/request_handler.rb

@@ -0,0 +1,85 @@
+# Request handler for validating and dispatching
+# datagram processing on defined types of datagrams.
+# Invalid or unknown datagrams will be processed by
+# the `OnesnooperServer::Datagrams::InvalidDatagram`
+# sub-handler.
+class OnesnooperServer::RequestHandler
+
+  # Validation constant for incoming datagrams
+  MONITORING_DATA_REGEXP = /^MONITOR\s(?<result>[[:alpha:]]+)\s(?<host_id>\d+)\s(?<payload>\S+)$/
+
+  # Registered allowed datagram processing classes & the default fallback
+  DATAGRAMS = {
+    "SUCCESS" => ::OnesnooperServer::Datagrams::SuccessDatagram,
+    "FAILURE" => ::OnesnooperServer::Datagrams::FailureDatagram,
+  }
+  DATAGRAMS.default = ::OnesnooperServer::Datagrams::InvalidDatagram
+
+  # Registered allowed stores for monitoring data
+  STORES = {
+    "mongodb" => ::OnesnooperServer::Stores::MongodbStore,
+    "mysql"   => ::OnesnooperServer::Stores::MysqlStore,
+    "sqlite"  => ::OnesnooperServer::Stores::SqliteStore,
+  }
+  STORES.default = ::OnesnooperServer::Stores::InvalidStore
+
+  # Static parsing method for identifying types of incoming datagrams
+  # and choosing the right processing class for each datagram. Always
+  # returns an instance responding to `run(callback)`.
+  #
+  # @param monitoring_datagram [Object] datagram payload for processing
+  # @param source_ip [String] IP address of the client
+  # @param source_port [String] port number of the client
+  def self.parse(monitoring_datagram, source_ip, source_port)
+    unless valid_data?(monitoring_datagram)
+      ::OnesnooperServer::Log.fatal "[#{self.name}] Dropping invalid monitoring data #{monitoring_datagram.inspect}"
+      return DATAGRAMS.default.new
+    end
+
+    unless valid_peer?(source_ip)
+      ::OnesnooperServer::Log.warn "[#{self.name}] Dropping monitoring data from #{source_ip}, not allowed!"
+      return DATAGRAMS.default.new
+    end
+
+    match_data = monitoring_datagram.match(MONITORING_DATA_REGEXP)
+    return DATAGRAMS.default.new unless match_data
+
+    DATAGRAMS[match_data[:result]].new({
+      host_id: match_data[:host_id],
+      payload: match_data[:payload],
+      stores: store_instances,
+    })
+  end
+
+private
+
+  # Identifies incoming datagrams as valid for further processing.
+  # Datagrams must be non-nil and String-like.
+  #
+  # @param monitoring_datagram [Object] incoming datagram for validation
+  # @return [Boolean] result
+  def self.valid_data?(monitoring_datagram)
+    monitoring_datagram && monitoring_datagram.kind_of?(String)
+  end
+
+  # Matches provided IP address against the list of known
+  # allowed peers.
+  #
+  # @param source_ip [String] IP address to match
+  # @return [Boolean] result
+  def self.valid_peer?(source_ip)
+    ::OnesnooperServer::Settings.allowed_sources.include? source_ip
+  end
+
+  # Retrieves a list of instances for allowed store backends.
+  #
+  # @return [Array] list of store instances
+  def self.store_instances
+    ::OnesnooperServer::Settings.store.collect do |store_name|
+      STORES[store_name].new(
+        ::OnesnooperServer::Settings.stores.respond_to?(store_name) ? ::OnesnooperServer::Settings.stores.send(store_name) : {}
+      )
+    end
+  end
+
+end

data/lib/onesnooper_server/settings.rb

@@ -0,0 +1,12 @@
+# Wraps persistent setting taken from configuration
+# files. This singleton is available throughout the
+# whole application environment.
+class OnesnooperServer::Settings < ::Settingslogic
+  gem_root = File.expand_path '../../..', __FILE__
+
+  source "#{ENV['HOME']}/.onesnooper-server" if File.readable?("#{ENV['HOME']}/.onesnooper-server")
+  source "/etc/onesnooper-server/onesnooper-server.yml" if File.readable?("/etc/onesnooper-server/onesnooper-server.yml")
+  source "#{gem_root}/config/onesnooper-server.yml"
+
+  namespace ENV['RAILS_ENV'] ? ENV['RAILS_ENV'] : 'production'
+end

data/lib/onesnooper_server/sql_store.rb

@@ -0,0 +1,65 @@
+require 'sequel'
+
+# Class wrapping common features of all SQL-based
+# stores. Using 'sequel' to abstract differences.
+class OnesnooperServer::SqlStore < ::OnesnooperServer::Store
+
+  # constant table name
+  SQL_TABLE_NAME = :one_monitoring
+
+  def save!(timestamp, data)
+    ::OnesnooperServer::Log.debug "[#{self.class.name}] Saving #{timestamp.to_s} => #{data.inspect}"
+    fail "DB connection has to be initialized from subclasses, " \
+         "::OnesnooperServer::SqlStore cannot be used directly!" unless @db_conn
+
+    if insert_data = data_in_vm_groups(timestamp, data)
+      @db_conn[SQL_TABLE_NAME].multi_insert(insert_data)
+    else
+      ::OnesnooperServer::Log.warn "[#{self.class.name}] Skipping SQL INSERT for an empty dataset"
+    end
+  end
+
+private
+
+  # Converts parsed datagram payload into table rows
+  # for direct insert into the DB.
+  #
+  # @param timestamp [DateTime] payload time stamp
+  # @param data [Hash] hash-like parsed payload structure
+  # @return [Array, NilClass] array with row hashes
+  def data_in_vm_groups(timestamp, data)
+    data_ary = []
+
+    data['VM'].each do |vm_on_host|
+      next if vm_on_host.blank?
+      data_ary << {
+        :timestamp => timestamp,
+        :vm_id => vm_on_host['ID'],
+        :vm_deploy_id => vm_on_host['DEPLOY_ID'],
+        :vm_netrx => vm_on_host['POLL']['NETRX'],
+        :vm_nettx => vm_on_host['POLL']['NETTX'],
+        :vm_used_cpu => vm_on_host['POLL']['USEDCPU'],
+        :vm_used_memory => vm_on_host['POLL']['USEDMEMORY'],
+        :vm_state => vm_on_host['POLL']['STATE'],
+        :host_name => data['HOSTNAME'],
+        :host_arch => data['ARCH'],
+        :host_model => data['MODELNAME'],
+        :host_hypervisor => data['HYPERVISOR'],
+        :host_ds_total => data['DS_LOCATION_TOTAL_MB'],
+        :host_ds_used => data['DS_LOCATION_USED_MB'],
+        :host_ds_free => data['DS_LOCATION_FREE_MB'],
+        :host_total_cpu => data['TOTALCPU'],
+        :host_cpu_speed => data['CPUSPEED'],
+        :host_used_cpu => data['USEDCPU'],
+        :host_free_cpu => data['FREECPU'],
+        :host_total_memory => data['TOTALMEMORY'],
+        :host_free_memory => data['FREEMEMORY'],
+        :host_used_memory => data['USEDMEMORY'],
+        :one_version => data['VERSION'],
+      }
+    end
+
+    data_ary.empty? ? nil : data_ary
+  end
+
+end

data/lib/onesnooper_server/store.rb

@@ -0,0 +1,23 @@
+# Base class for all backend data stores. Implements key
+# method stubs required for all specific backend data store
+# implementations.
+class OnesnooperServer::Store
+
+  # Initializes data store instance with given parameters.
+  #
+  # @param params [Hash] hash-like structure with parameters
+  def initialize(params = {})
+    @params = params
+  end
+
+  # Saves given data set into the underlying data store.
+  # Behavior is determined by the underlying data store
+  # implementation.
+  #
+  # @param timestamp [DateTime] current time
+  # @param data [Hash] data to be saved in the data store
+  def save!(timestamp, data)
+    fail "This method needs to be implemented in subclasses"
+  end
+
+end

data/lib/onesnooper_server/stores.rb

@@ -0,0 +1,7 @@
+# Module housing available backend implementations for
+# data stores. All stores must implement basic methods
+# outlined in the `OnesnooperServer::Store` class.
+module OnesnooperServer::Stores; end
+
+# Load all available store types
+Dir.glob(File.join(File.dirname(__FILE__), 'stores', "*.rb")) { |store_file| require store_file.chomp('.rb') }

data/lib/onesnooper_server/stores/invalid_store.rb

@@ -0,0 +1,10 @@
+# Placeholder store for invalid configuration handling and
+# testing purposes.
+class OnesnooperServer::Stores::InvalidStore < ::OnesnooperServer::Store
+
+  def save!(timestamp, data)
+    ::OnesnooperServer::Log.fatal "[#{self.class.name}] This is a dummy store, do not use it!"
+    fail "InvalidStore selected as fallback, check your configuration!"
+  end
+
+end

data/lib/onesnooper_server/stores/mongodb_store.rb

@@ -0,0 +1,23 @@
+require 'mongo'
+
+# MongoDB-based store for production deployments and dynamic
+# document structure.
+class OnesnooperServer::Stores::MongodbStore < ::OnesnooperServer::Store
+
+  # constant collection name
+  MONGO_COLL_NAME = 'one_monitoring'
+
+  def initialize(params = {})
+    super
+    @db_conn = ::Mongo::MongoClient.new(params[:host], params[:port])
+    @db_active_db = @db_conn.db(params[:database])
+    @db_coll = @db_active_db.create_collection(MONGO_COLL_NAME)
+  end
+
+  def save!(timestamp, data)
+    ::OnesnooperServer::Log.debug "[#{self.class.name}] Saving #{timestamp.to_s} => #{data.inspect}"
+    data['TIMESTAMP'] = timestamp.to_time.utc
+    @db_coll.insert data
+  end
+
+end

data/lib/onesnooper_server/stores/mysql_store.rb

@@ -0,0 +1,17 @@
+# MySQL-based store for production deployments with static
+# table structure.
+class OnesnooperServer::Stores::MysqlStore < ::OnesnooperServer::SqlStore
+
+  def initialize(params = {})
+    super
+    @db_conn = ::Sequel.connect(
+      :adapter  => 'mysql2',
+      :database => params[:database],
+      :user     => params[:username],
+      :password => params[:password],
+      :host     => params[:host],
+      :port     => params[:port],
+    )
+  end
+
+end

data/lib/onesnooper_server/stores/sqlite_store.rb

@@ -0,0 +1,10 @@
+# Sqlite3-based store for testing and small deployments.
+class OnesnooperServer::Stores::SqliteStore < ::OnesnooperServer::SqlStore
+
+  def initialize(params = {})
+    super
+    @db_path = "sqlite://#{params[:database_file]}"
+    @db_conn = ::Sequel.connect(@db_path)
+  end
+
+end

data/lib/onesnooper_server/udp_handler.rb

@@ -0,0 +1,50 @@
+# Handler for incoming UDP datagrams. Implements required methods
+# for direct use with EventMachine listeners. This handler will not
+# respond to incoming datagrams as its primary purpose is to record
+# mirrored monitoring traffic.
+class OnesnooperServer::UDPHandler < ::EventMachine::Connection
+
+  DATAGRAM_PREFIX = 'MONITOR'
+
+  # Receives data and triggers processing of the given
+  # datagram. Main internal processing triggered from this
+  # method should always happen asynchronously (i.e., using
+  # EventMachine.defer or Deferrable classes).
+  #
+  # @param monitoring_datagram [String] incoming data payload
+  def receive_data(monitoring_datagram)
+    monitoring_datagram.chomp!
+    source_port, source_ip = Socket.unpack_sockaddr_in(get_peername)
+    unless monitoring_datagram.start_with?(DATAGRAM_PREFIX)
+      ::OnesnooperServer::Log.warn "[#{self.class.name}] Discarding datagram from " \
+                                   "#{source_ip}:#{source_port} (not #{DATAGRAM_PREFIX})"
+      return
+    end
+
+    ::OnesnooperServer::Log.debug "[#{self.class.name}] Received #{monitoring_datagram.inspect} " \
+                                  "from #{source_ip}:#{source_port}"
+    ::OnesnooperServer::RequestHandler.parse(
+      monitoring_datagram,
+      source_ip,
+      source_port
+    ).run(callback)
+  end
+
+private
+
+  # Callable processing callback. As this implementation does
+  # not send responses to incoming datagrams, this methods is
+  # for logging and debugging purposes only.
+  def callback
+    def_deferr = ::EventMachine::DefaultDeferrable.new
+    proc_callback = Proc.new { |response| ::OnesnooperServer::Log.debug(
+      "[#{self.class.name}] Handled as: #{response}"
+    ) }
+
+    def_deferr.callback &proc_callback
+    def_deferr.errback &proc_callback
+
+    def_deferr
+  end
+
+end

data/lib/onesnooper_server/version.rb

@@ -0,0 +1,4 @@
+module OnesnooperServer
+  # Static version information
+  VERSION = "0.0.1" unless defined?(::OnesnooperServer::VERSION)
+end

data/migrations/001_create_db.rb

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+
+# -------------------------------------------------------------------------- #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+require 'rubygems'
+require 'sequel'
+
+Sequel.migration do
+  up do
+    create_table(:one_monitoring) do
+      #
+      primary_key :id, :type => Bignum
+      DateTime :timestamp, :null => false, :index => true
+
+      #
+      String :vm_id, :null => false, :index => true
+      String :vm_deploy_id, :null => false, :index => true
+      BigDecimal :vm_netrx, :default => 0
+      BigDecimal :vm_nettx, :default => 0
+      Float :vm_used_cpu, :default => 0.0
+      BigDecimal :vm_used_memory, :default => 0
+      String :vm_state, :null => false, :index => true
+
+      #
+      String :host_name, :null => false, :index => true
+      String :host_arch, :default => 'unknown'
+      String :host_model, :default => 'unknown'
+      String :host_hypervisor, :null => false
+      BigDecimal :host_ds_total, :default => 0
+      BigDecimal :host_ds_used, :default => 0
+      BigDecimal :host_ds_free, :default => 0
+      Integer :host_total_cpu, :null => false
+      Integer :host_cpu_speed, :default => 0
+      Integer :host_used_cpu, :default => 0
+      Integer :host_free_cpu, :default => 0
+      BigDecimal :host_total_memory, :null => false
+      BigDecimal :host_free_memory, :default => 0
+      BigDecimal :host_used_memory, :default => 0
+
+      #
+      String :one_version, :null => false
+    end
+  end
+
+  down do
+    drop_table(:one_monitoring)
+  end
+end

data/onesnooper-server.gemspec

@@ -0,0 +1,41 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'onesnooper_server/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'onesnooper-server'
+  spec.version       = OnesnooperServer::VERSION
+  spec.authors       = ['Boris Parak']
+  spec.email         = ['parak@cesnet.cz']
+  spec.summary       = 'Simple server snooping on and recording OpenNebula\'s VM & Host monitoring traffic'
+  spec.description   = 'Simple server snooping on and recording OpenNebula\'s VM & Host monitoring traffic'
+  spec.homepage      = 'https://github.com/arax/onesnooper-server'
+  spec.license       = 'Apache License, Version 2.0'
+
+  spec.files                 = `git ls-files -z`.split("\x0")
+  spec.executables           = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files            = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths         = ['lib']
+  spec.required_ruby_version = '>= 1.9.3'
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0.0'
+  spec.add_development_dependency 'simplecov', '~> 0.9.0'
+  spec.add_development_dependency 'rubygems-tasks', '~> 0.2.4'
+
+  # internals
+  spec.add_runtime_dependency 'eventmachine', '~> 1.0.4'
+  spec.add_runtime_dependency 'activesupport', '~> 4.2.0'
+  spec.add_runtime_dependency 'settingslogic', '~> 2.0.9'
+
+  # SQL DB connectors
+  spec.add_runtime_dependency 'sequel', '~> 4.18.0'
+  spec.add_runtime_dependency 'sqlite3', '~> 1.3.10'
+  spec.add_runtime_dependency 'mysql2', '~> 0.3.17'
+
+  # NoSQL DB connectors
+  spec.add_runtime_dependency 'mongo', '~> 1.11.1'
+  spec.add_runtime_dependency 'bson_ext', '~> 1.11.1'
+end

data/spec/onesnooper_server_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe OnesnooperServer do
+  subject { onesnooper_server }
+
+  it 'does something'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+require 'rubygems'
+
+# enable coverage reports
+if ENV['COVERAGE']
+  require 'simplecov'
+
+  SimpleCov.add_filter "/spec/"
+  SimpleCov.start
+end
+
+require 'onesnooper-server'
+
+Dir["#{File.dirname(__FILE__)}/helpers/*.rb"].each {|file| require file }