xmonitor 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44a174e8c7490ab651f0823a290e3b4ef94e04a2
-  data.tar.gz: 1016331ffc5863e623e546c5d0cdae0f767cc7e7
+  metadata.gz: 3bae11c8a9e98c0e0747e4bc7c918b17873985a3
+  data.tar.gz: 77956a6841772b2da403235c0376d311272d6ce0
 SHA512:
-  metadata.gz: 4afdce528b340efae2305e444a18d03812b873c799b8110934d9207e69bce9f8c77ad2892234b212f297a2295a1aab5d392f2906ea98be75b6e4bf9d31f0722c
-  data.tar.gz: fc7f46884fcab312fb2e7350e2b62aad76ae4ee3827fe91698dccf8c94f8ff6e5932bfd3241105b3f4c8862830d0a13746fde2587d4cac61fab6d23604896eae
+  metadata.gz: f8daf556fb7562912b9bc01b0e4aad84b3c508836a145b11e3dd9394aad3698d45f36a686f664a35112248bba51f6982ea6bbff204055125ddd046f6222db6a0
+  data.tar.gz: f095fe8d5dbdfd3ecfb7500a4c3c60910a046c529091b7efb0b7858bc9b726de5b1fb6ee06b5142520ac2ed94756591dc1d3ffaa428748345395a3e7b5c51cbb

data/exe/xmonitor_dashboard

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "xmonitor"
+
+Xmonitor::Dashboard.start ARGV

data/lib/xmonitor.rb

@@ -6,10 +6,15 @@ module Xmonitor
   require "yaml"
   require "json"
   require "irb"
+  require "csv"
 
   require "posixpsutil"
   require "aws-sdk-firehose"
+  require "aws-sdk-athena"
+  require "aws-sdk-s3"
 
   require "xmonitor/config"
   require "xmonitor/agent"
+
+  autoload :Dashboard, 'xmonitor/dashboard'
 end

data/lib/xmonitor/agent.rb

@@ -56,6 +56,7 @@ class Xmonitor::Agent
 
   def monitor
     records = grab
+
     raise GrabError if records.empty?
 
     data = records.join(NEW_LINE) + NEW_LINE
@@ -64,7 +65,7 @@ class Xmonitor::Agent
   end
 
   def grab
-    grab_cpu + grab_memory
+    grab_cpu + grab_memory + grab_disks + grab_network
   rescue => e
     raise GrabError.new(e)
   end
@@ -81,6 +82,47 @@ class Xmonitor::Agent
     }
   end
 
+  def grab_disks
+    grab_disks_usage + grab_disks_io_counter
+  end
+
+  def grab_disks_usage
+    devices = PosixPsutil::Disks.disk_partitions.map{|partition| partition.device}
+
+    devices.map{|device|
+      PosixPsutil::Disks.disk_usage(device).each_pair.map{|(k, v)|
+        metric = "#{device}.#{k}"
+        create_record('disks', metric, v)
+      }
+    }.flatten
+  end
+
+  def grab_disks_io_counter
+    PosixPsutil::Disks.disk_io_counters.each_pair.map{|(k, v)|
+      disk = k
+
+      v.each_pair.map{|k2, v2|
+        metric = "#{disk}.#{k2}"
+        create_record('disks', metric, v2)
+      }
+    }.flatten
+  end
+
+  def grab_network
+    grab_net_io_counter
+  end
+
+  def grab_net_io_counter
+    PosixPsutil::Network.net_io_counters(true).each_pair.map{|(k, v)|
+      interface = k
+
+      v.each_pair.map{|k2, v2|
+        metric = "#{interface}.#{k2}"
+        create_record('network', metric, v2)
+      }
+    }.flatten
+  end
+
   def create_record(metric, dimension, value)
     {timestamp: Time.now.strftime(TIME_FORMAT), host: @hostname, metric: metric, dimension: dimension, value: value}.to_json
   end

data/lib/xmonitor/config.rb

@@ -1,18 +1,21 @@
 class Xmonitor::Config
-  attr_reader :region, :stream_name, :access_key_id, :secret_access_key, 
+  attr_reader :region, :stream_name, :access_key_id, :secret_access_key, :athena_database, :athena_s3_bucket, :athena_table_name
 
   KEY_REGION = 'region'
   KEY_STREAM_NAME = 'stream_name'
   KEY_ACCESS_KEY_ID = 'access_key_id'
   KEY_SECRET_ACCESS_KEY = 'secret_access_key'
+  KEY_ATHENA_DATABASE = 'athena_database'
+  KEY_ATHENA_S3_BUCKET = 'athena_s3_bucket'
+  KEY_ATHENA_TABLE_NAME = 'athena_table_name'
 
   def self.from_yaml(path)
-    json = YAML.load(File.read(path))
+    yaml = YAML.load(File.read(path))
 
-    self.new(json[KEY_REGION], json[KEY_STREAM_NAME], json[KEY_ACCESS_KEY_ID], json[KEY_SECRET_ACCESS_KEY])
+    self.new(yaml[KEY_REGION], yaml[KEY_STREAM_NAME], yaml[KEY_ACCESS_KEY_ID], yaml[KEY_SECRET_ACCESS_KEY], yaml[KEY_ATHENA_DATABASE], yaml[KEY_ATHENA_S3_BUCKET], yaml[KEY_ATHENA_TABLE_NAME])
   end
 
   def initialize(*args)
-    @region, @stream_name, @access_key_id, @secret_access_key = *args
+    @region, @stream_name, @access_key_id, @secret_access_key, @athena_database, @athena_s3_bucket, @athena_table_name = *args
   end
 end

data/lib/xmonitor/dashboard.rb

@@ -0,0 +1,79 @@
+class Xmonitor::Dashboard
+  def self.start(argv)
+    config_file = argv[0]
+
+    config = Xmonitor::Config.from_yaml(config_file)
+
+    server = self.new(config)
+
+    server.run
+  end
+
+  def initialize(config)
+    @config = config
+    @logger = Logger.new(STDERR)
+  end
+
+  def run
+    init_aws_config_and_clients
+
+    query_execution_response = start_query_execution(all_metrics_by_hour_query)
+
+    query_execution_id = query_execution_response.query_execution_id
+    @logger.info(query_execution_id: query_execution_id)
+
+    last_get_query_execution_response = wait_for_finish_query_execution(query_execution_id)
+
+    output_location = last_get_query_execution_response.query_execution.result_configuration.output_location
+    @logger.info(output_location: output_location)
+
+    get_s3_object_and_process_body(output_location) do |body|
+      CSV.new(body, headers: true).each{|row|
+        puts [Time.parse(row[0]), row[1], row[2], row[3], row[4].to_f].to_csv
+      }
+    end
+  rescue StandardError => e
+    @logger.error(error: e, backtrace: e.backtrace)
+  end
+
+  def init_aws_config_and_clients
+    Aws.config[:credentials] = Aws::Credentials.new(@config.access_key_id, @config.secret_access_key)
+
+    @athena = Aws::Athena::Client.new(region: @config.region)
+    @s3 = Aws::S3::Client.new(region: @config.region)
+  end
+
+  def all_metrics_by_hour_query
+    'SELECT DATE_FORMAT(timestamp, \'%Y-%m-%d %H:00:00\') AS _timestamp, host AS _host, metric AS _metric, dimension AS _dimension, AVG(value) AS _value FROM "' + @config.athena_database + '"."' + @config.athena_table_name + '" GROUP BY DATE_FORMAT(timestamp, \'%Y-%m-%d %H:00:00\'), host, metric, dimension ORDER BY _host, _metric, _dimension, _timestamp DESC;'
+  end
+
+  def start_query_execution(query_string)
+    @athena.start_query_execution(
+      query_string: query_string,
+      query_execution_context: {database: @config.athena_database},
+      result_configuration: {output_location: "s3://#{@config.athena_s3_bucket}/"}
+    ).tap{|response|
+      @logger.info(response: response)
+    }
+  end
+
+  def wait_for_finish_query_execution(query_execution_id)
+    loop do
+      response = @athena.get_query_execution(query_execution_id: query_execution_id)
+
+      @logger.info(response: response)
+
+      state = response.query_execution.status.state
+      
+      return response unless ['QUEUED', 'RUNNING'].include?(state)
+
+      sleep 1
+    end
+  end
+
+  def get_s3_object_and_process_body(s3_object_arn, &blk)
+    _, bucket, key = s3_object_arn.match(/s3:\/\/(.+?)\/(.+)$/).to_a
+
+    yield @s3.get_object(bucket: bucket, key: key).body
+  end
+end

data/lib/xmonitor/version.rb

@@ -1,3 +1,3 @@
 module Xmonitor
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/xmonitor.gemspec

@@ -25,5 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_runtime_dependency "aws-sdk-firehose", "~> 1.1"
+  spec.add_runtime_dependency "aws-sdk-athena", "~> 1.0"
+  spec.add_runtime_dependency "aws-sdk-s3", "~> 1.8"
   spec.add_runtime_dependency "posixpsutil", "~> 0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: xmonitor
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - xmisao
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-25 00:00:00.000000000 Z
+date: 2017-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-athena
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: posixpsutil
   requirement: !ruby/object:Gem::Requirement
@@ -86,6 +114,7 @@ email:
 - mail@xmisao.com
 executables:
 - xmonitor
+- xmonitor_dashboard
 extensions: []
 extra_rdoc_files: []
 files:
@@ -99,9 +128,11 @@ files:
 - bin/console
 - bin/setup
 - exe/xmonitor
+- exe/xmonitor_dashboard
 - lib/xmonitor.rb
 - lib/xmonitor/agent.rb
 - lib/xmonitor/config.rb
+- lib/xmonitor/dashboard.rb
 - lib/xmonitor/version.rb
 - xmonitor.gemspec
 homepage: https://github.com/xmisao/xmonitor