nephelae 0.1.2 → 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 46ba6a351d50bbdc74cce995595f6df3da22dad2
+  data.tar.gz: eaae6fb957bb66420f0e40ed75b649d888e24215
+SHA512:
+  metadata.gz: 1182630f3252d7f6ec8aa3ff9b4525f9412dcae994c4bcaaa8d8ae48f490a72c6a45fdd2cff281af1a8deef12ad1f2758944fe28c0589d01f4fe636cabd60e3d
+  data.tar.gz: 5835f81f0b2ba3ebd6c3b3c5a25470959f775069129025e1d47a8e63484e108317e87c6288163ca1c7ae7200772e8bf8ede55b516c375a471f1b663fbf7911b5

data/.gitignore

@@ -1,10 +1,12 @@
 *.gem
+.vagrant
 *.rbc
 .bundle
 .config
 Gemfile.lock
 InstalledFiles
 coverage
+nephelae.yml
 doc/
 lib/bundler/man
 pkg

data/.ruby-version

@@ -0,0 +1 @@
+ 2.0.0-p247

data/MIT-LICENCE

@@ -0,0 +1,20 @@
+Copyright (c) 2011-2012 by Made by Many Ltd - http://madebymany.com
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,24 +1,59 @@
 # Nephelae
 
-TODO: Write a gem description
+The Nephelae were the nymph of clouds (http://www.theoi.com/Nymphe/Nephelai.html) carrying water from Earth to Heaven. In this particular case the nephelae carry data from servers to Amazon EC2 Cloud Watch. It was written to send disk space and memory usage of EC2 servers but can be extended to query any metrics and upload them on a regular basis.
+
+For example the Redis plugin will collect the UsedMemory, ConnectedSlaves, ConnectedClients, ChangesSinceLastSave, MasterIOSecondsAgo, MasterLinkStatus and Up of a Redis instance and send this to AWS CloudWatch. We use this to put CloudWatch Alarms on the Up and MasterLinkStatus metrics to notify if Redis goes down or loses connection with the Master
+
+The Nephelae gem contains a daemon process which has a number of plugins to collect stats. The daemon process is configured through a yaml file 
 
 ## Installation
 
-Add this line to your application's Gemfile:
+To install as a gem
 
-    gem 'nephelae'
+    $ gem install nephelae
 
-And then execute:
+## Usage
 
-    $ bundle
+To run the nephelae deamon
 
-Or install it yourself as:
+    $ nephelae start
 
-    $ gem install nephelae
+This will start with default settings looking for a config yaml file in .etc/nephelae.yml. It will use /var/log as a log directory and /var/run for a pid directory
 
-## Usage
+To start in the background (this uses the daemon gem http://daemons.rubyforge.org/)
+
+    $ nephelae start --piddir /home/nephelae/nephelae.pid --loglevel debug
+    
+    
+To start in the foreground
+
+	$ nephelae --foreground
+
+Other command line options
+
+	--loglevel *level* (debug, info, warn, error, fatal) default: warn
+	--config *path*
+	--instance-name *override the instance-id*
+
+## Configuration
+
+The config yaml file is used to define the AWS access keys and plugin settings
+
+An example configuration file
 
-TODO: Write usage instructions here
+```yaml
+:aws_secret_access_key: your_secret_key
+:aws_access_key_id: your_access_key
+:region: ec2_region
+:plugins:
+  :disk_space:
+    :plugin_class: DiskSpace
+    :path: /
+    :schedule: 1m
+  :mem_usage:
+    :plugin_class: MemUsage
+    :schedule: 1m
+```
 
 ## Contributing
 

data/Vagrantfile

@@ -0,0 +1,14 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant::Config.run do |config|
+
+  config.vm.define :precise64 do |precise64|
+    precise64.vm.box = "precise64"
+    precise64.vm.box_url = "http://files.vagrantup.com/precise64.box"
+    precise64.vm.network :bridged, :bridge => "en0"
+    precise64.vm.customize ["modifyvm", :id, "--memory", 512]
+    precise64.vm.share_folder "v-data", "/nephelae", "./"
+  end
+
+end

data/bin/nephelae

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 require 'rubygems'
 require 'daemons'
-require 'nephelae'
+require_relative '../lib/nephelae'
 require 'optparse'
 require 'yaml'
 
@@ -16,6 +16,8 @@ logfile = STDOUT
 loglevel = Logger::WARN
 logdir = '/var/log'
 piddir = '/var/run'
+foreground = false
+instance_name = nil
 
 OptionParser.new do |o|
   o.banner << " command"
@@ -28,6 +30,14 @@ OptionParser.new do |o|
     logdir = v
   end
 
+  o.on("--instance-name INSTANCE_NAME", "The instance name you want metrics bound to") do |v|
+    instance_name = v
+  end
+
+  o.on("--foreground", "Run in foreground") do |v|
+    foreground = v
+  end
+
   o.on("--piddir PIDDIR", "The directory to put pid files") do |v|
     piddir = v
   end
@@ -56,7 +66,11 @@ Nephelae::Logging.logger = log
 File.open( file ) { |yf| settings = YAML::load( yf ) }
 
 new_argv = [cmd]
-Daemons.run_proc('nephelae', {ARGV: new_argv, log_dir: logdir, dir_mode: :normal, dir: piddir, log_output: true}) do
-  r = Nephelae::Runner.new(settings)
-  r.run
+if foreground then
+  Nephelae::Runner.run(instance_name, settings)
+else
+  Daemons.run_proc('nephelae', {ARGV: new_argv, log_dir: logdir, dir_mode: :normal, dir: piddir, log_output: true}) do
+    Nephelae::Runner.run(instance_name, settings)
+  end
 end
+

data/lib/nephelae.rb

@@ -1,17 +1,16 @@
-require "nephelae/version"
-require "nephelae/logging"
-require "nephelae/cloud_watch/simple_aws"
-require "nephelae/cloud_watch/cloud_watch"
-require "nephelae/cloud_watch/metrics"
+require_relative "nephelae/version"
+require_relative "nephelae/logging"
+require_relative "nephelae/cloud_watch/cloud_watch"
+require_relative "nephelae/cloud_watch/metrics"
 
-require "nephelae/plugins/plugin"
-require "nephelae/plugins/disk_space"
-require "nephelae/plugins/mem_usage"
-require "nephelae/plugins/nephelae_process"
-require "nephelae/plugins/passenger_status"
-require "nephelae/plugins/redis"
+require_relative "nephelae/plugins/plugin"
+require_relative "nephelae/plugins/disk_space"
+require_relative "nephelae/plugins/mem_usage"
+require_relative "nephelae/plugins/nephelae_process"
+require_relative "nephelae/plugins/passenger_status"
+require_relative "nephelae/plugins/redis"
 
-require "nephelae/runner"
+require_relative "nephelae/runner"
 
 module Nephelae
   # Your code goes here...

data/lib/nephelae/cloud_watch/cloud_watch.rb

@@ -1,3 +1,6 @@
+require 'aws-sdk'
+require 'excon'
+
 module Nephelae
   class CloudWatch
     include Logging
@@ -5,51 +8,41 @@ module Nephelae
     attr_accessor :aws_access_key_id, :aws_secret_access_key, :aws_session_token, :aws_credentials_expire_at, :url
 
     def initialize(options={})
-      options[:region] ||= 'us-east-1'
-      @host       = options[:host] || "monitoring.#{options[:region]}.amazonaws.com"
-      @path       = options[:path]        || '/'
-      @port       = options[:port]        || 443
-      @scheme     = options[:scheme]      || 'https'
-      @aws_access_key_id      = options[:aws_access_key_id]
-      @aws_secret_access_key  = options[:aws_secret_access_key]
-      @aws_session_token      = options[:aws_session_token]
-      @aws_credentials_expire_at = options[:aws_credentials_expire_at]
-
-      @url = "#{@scheme}://#{@host}:#{@port}#{@path}"
-
-    end
-
-    def request(params)
-      body = AWS.signed_params(
-        params,
-        {
-          :aws_access_key_id  => @aws_access_key_id,
-          :aws_session_token  => @aws_session_token,
-          :aws_secret_access_key => @aws_secret_access_key,
-          :host               => @host,
-          :path               => @path,
-          :port               => @port,
-          :version            => '2010-08-01'
-        }
-      )
-
-      AWS.request(@url, {
-         :body       => body,
-         :expects    => 200,
-         :headers    => { 'Content-Type' => 'application/x-www-form-urlencoded' },
-         :host       => @host,
-         :method     => 'POST'
-       })
+      opts = {
+        access_key_id: options[:aws_access_key_id],
+        secret_access_key: options[:aws_secret_access_key],
+        region: options[:region]
+      }
+
+      AWS.config opts
+      @cw = AWS::CloudWatch.new
     end
 
-    def put_metric(metric)
+    def put_metric(instance_name, metric)
       params = metric.params
+      data = {}
+      data[:namespace] = metric.namespace
+      data[:metric_data] = []
       unless params.nil?
-        cw_output = request(metric.params) unless params.nil?
-        log.info(cw_output)
-        cw_output
+        params.each do |param|
+          Logging.logger.warn "Key : #{param[:name]}, Value : #{param[:value]}"
+          options = {}
+          options[:metric_name] = param[:name]
+          options[:dimensions] = [
+            {name: 'InstanceId', value: instance_name}
+          ]
+          options[:timestamp] = Time.now.iso8601
+          options[:value] = param[:value]
+          options[:unit] = param[:unit]
+          data[:metric_data] << options
+        end
+        @cw.put_metric_data(data)
       end
     end
 
+    def get_instance_id()
+      Excon.get('http://169.254.169.254/latest/meta-data/instance-id').body
+    end
+
   end
 end

data/lib/nephelae/cloud_watch/metrics.rb

@@ -1,3 +1,5 @@
+require_relative '../logging'
+
 module Nephelae
   class Metrics
     attr_accessor :params, :namespace, :instance_id
@@ -6,26 +8,22 @@ module Nephelae
 
     def initialize(namespace, options = {})
       #code to get the instance id from ec2 metadata
-      @instance_id = options[:instance_id] || AWS.get_instance_id
-      @params = {}
-      @params['Action'] = 'PutMetricData'
-      @params['Namespace'] = namespace
+      @instance_id = options[:instance_id]
+      @params = []
+      @namespace = namespace
       @mcount = 0
     end
 
     def append_metric(name, value, options = {})
       dim_count = 1
       @mcount = @mcount + 1
-      params["MetricData.member.#{@mcount}.MetricName"] = name
-      params["MetricData.member.#{@mcount}.Value"] = value
-      params["MetricData.member.#{@mcount}.Unit"] = options[:unit] || 'None'
-      params["MetricData.member.#{@mcount}.Timestamp"] = options[:timestamp] if options[:timestamp]
-
-      if @instance_id
-        params["MetricData.member.#{@mcount}.Dimensions.member.#{dim_count}.Name"] = 'InstanceId'
-        params["MetricData.member.#{@mcount}.Dimensions.member.#{dim_count}.Value"] = @instance_id
-      end
+      param = {}
+      param[:name] = name
+      param[:value] = value
+      param[:unit] = options[:unit] || 'None'
+      param[:timestamp] = options[:timestamp] if options[:timestamp]
 
+      @params << param
     end
 
     def params

data/lib/nephelae/plugins/passenger_status.rb

@@ -13,7 +13,6 @@ module Nephelae
         metrics.append_metric('ActiveInstances', stats[:active], {unit: 'Count'})
         metrics.append_metric('InactiveInstances', stats[:inactive], {unit: 'Count'})
         metrics.append_metric('WaitingOnGlobalQueue', stats[:waiting_on_global_queue], {unit: 'Count'})
-
       end
 
       return metrics

data/lib/nephelae/plugins/redis.rb

@@ -9,7 +9,7 @@ module Nephelae
       if $?.success?
         stats = parse_status(output)
         metrics.append_metric('Up', 1)
-        metrics.append_metric('MasterLinkStatus', (stats[:master] == 'up' ? 1 : 0)) unless stats[:master].nil?
+        metrics.append_metric('MasterLinkStatus', (stats[:master_link_status] == 'up' ? 1 : 0)) unless stats[:master_link_status].nil?
         metrics.append_metric('MasterIOSecondsAgo', stats[:master_last_io_seconds_ago], {unit: 'Seconds'}) unless stats[:master_last_io_seconds_ago].nil?
         metrics.append_metric('ChangesSinceLastSave', stats[:changes_since_last_save], {unit: 'Count'})
         metrics.append_metric('UsedMemory', stats[:used_memory], {unit: 'Bytes'})

data/lib/nephelae/runner.rb

@@ -6,40 +6,50 @@ module Nephelae
 
     attr_accessor :aws_access_key_id, :aws_secret_access_key, :region, :plugins
 
-    def initialize(config = {})
+    def initialize(instance_name, config = {})
       new_conf = config.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+      @instance_name = instance_name
       @aws_access_key_id = new_conf[:aws_access_key_id]
       @aws_secret_access_key = new_conf[:aws_secret_access_key]
       @region = new_conf[:region] || 'us-east-1'
       @plugins = new_conf[:plugins] || default_plugins
       @plugins[:nephelae_process] = {plugin_class: "NephelaeProcess", schedule: "5m"}
     end
-    
+
+    def self.run(instance_name, settings)
+      new(instance_name, settings).run
+    end
+
     def run
       log.warn "starting nephelae"
       EM.run {
-          cloud = CloudWatch.new({aws_access_key_id: @aws_access_key_id, aws_secret_access_key: @aws_secret_access_key, region: @region})
-
-          #make one request to put a cloud watch metric for nephelae being up. hopefully this can make it bork early if anything is wrong
-          cloud.put_metric(NephelaeProcess.new.get_metrics)
-
-          scheduler = Rufus::Scheduler::EmScheduler.start_new
-
-          plugins.each do |name, config|
-            schedule = config.delete(:schedule) || '5m'
-            klass_name = config.delete(:plugin_class)
-            klass = Object.const_get('Nephelae').const_get(klass_name)
-            p = klass.new(config)
-            scheduler.every schedule do
-              begin
-                cloud.put_metric(p.get_metrics)
-              rescue StandardError => e
-                log.error("nephelae plugin #{p.class} failed #{e.message}")
-                log.error(e.backtrace.join("\n"))
-              end
+        cloud = CloudWatch.new({aws_access_key_id: @aws_access_key_id, aws_secret_access_key: @aws_secret_access_key, region: @region})
+        @instance_name ||= cloud.get_instance_id
+
+        log.warn("sending metrics for instance : #{@instance_name}")
+
+        #make one request to put a cloud watch metric for nephelae being up. hopefully this can make it bork early if anything is wrong
+        cloud.put_metric(@instance_name, NephelaeProcess.new.get_metrics)
+
+        scheduler = Rufus::Scheduler.new
+
+        log.warn("setting up schudules")
+        plugins.each do |name, config|
+          schedule = config.delete(:schedule) || '5m'
+          klass_name = config.delete(:plugin_class)
+          klass = Object.const_get('Nephelae').const_get(klass_name)
+          p = klass.new(config)
+          log.warn("setting up schudule for #{name} plugin")
+          scheduler.every schedule do
+            begin
+              log.warn "putting metric plugin  #{name}"
+              cloud.put_metric(@instance_name, p.get_metrics)
+            rescue StandardError => e
+              log.error("nephelae plugin #{p.class} failed #{e.message}")
+              log.error(e.backtrace.join("\n"))
             end
           end
-
+        end
       }
     end
 

data/lib/nephelae/version.rb

@@ -1,3 +1,3 @@
 module Nephelae
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/nephelae.gemspec

@@ -19,6 +19,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency('eventmachine')
   gem.add_dependency('rufus-scheduler')
   gem.add_dependency('daemons')
-  gem.add_dependency('ruby-hmac')
+  gem.add_dependency('aws-sdk')
 
 end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: nephelae
 version: !ruby/object:Gem::Version
-  version: 0.1.2
-  prerelease: 
+  version: 0.2.0
 platform: ruby
 authors:
 - Stuart Eccles
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-28 00:00:00.000000000 Z
+date: 2014-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -30,65 +27,57 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: daemons
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ruby-hmac
+  name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Nephelae is a daemon process that will upload custom cloud watch metrics
@@ -101,15 +90,17 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .ruby-version
 - Gemfile
 - LICENSE
+- MIT-LICENCE
 - README.md
 - Rakefile
+- Vagrantfile
 - bin/nephelae
 - lib/nephelae.rb
 - lib/nephelae/cloud_watch/cloud_watch.rb
 - lib/nephelae/cloud_watch/metrics.rb
-- lib/nephelae/cloud_watch/simple_aws.rb
 - lib/nephelae/logging.rb
 - lib/nephelae/plugins/disk_space.rb
 - lib/nephelae/plugins/mem_usage.rb
@@ -122,26 +113,25 @@ files:
 - nephelae.gemspec
 homepage: https://github.com/madebymany/nephelae
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Poll for server metrics and post them to AWS as custom CloudWatch metrics
 test_files: []

data/lib/nephelae/cloud_watch/simple_aws.rb

@@ -1,56 +0,0 @@
-require 'base64'
-require 'excon'
-require 'openssl'
-
-module Nephelae
-  module AWS
-
-    def self.escape(string)
-      string.gsub(/([^a-zA-Z0-9_.\-~]+)/) {
-        "%" + $1.unpack("H2" * $1.bytesize).join("%").upcase
-      }
-    end
-
-    def self.hmac_sign(string_to_sign, key)
-      digest = OpenSSL::Digest::Digest.new('sha256')
-      OpenSSL::HMAC.digest(digest, key, string_to_sign)
-    end
-
-    def self.signed_params(params, options = {})
-      params.merge!({
-        'AWSAccessKeyId'    => options[:aws_access_key_id],
-        'SignatureMethod'   => 'HmacSHA256',
-        'SignatureVersion'  => '2',
-        'Timestamp'         => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
-        'Version'           => options[:version]
-      })
-
-      params.merge!({
-        'SecurityToken'     => options[:aws_session_token]
-      }) if options[:aws_session_token]
-
-      body = ''
-      for key in params.keys.sort
-        unless (value = params[key]).nil?
-          body << "#{key}=#{escape(value.to_s)}&"
-        end
-      end
-      string_to_sign = "POST\n#{options[:host]}:#{options[:port]}\n#{options[:path]}\n" << body.chop
-      signed_string = hmac_sign(string_to_sign, options[:aws_secret_access_key])
-      body << "Signature=#{escape(Base64.encode64(signed_string).chomp!)}"
-
-      body
-    end
-
-    def self.request(url, params, &block)
-      excon = Excon.new(url, params)
-      excon.request(params, &block)
-    end
-
-    def self.get_instance_id
-      Excon.get('http://169.254.169.254/latest/meta-data/instance-id').body
-      #"i-f746ec92"
-    end
-
-  end
-end