cloudwatch-sender 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d0fad232296a6d86bca73bedbd6fffbde0be70f
-  data.tar.gz: 57292de37c41b742a83c0d84e9dc3121b5fa4403
+  metadata.gz: 0de3762390f886fad8a077d0cecd0e2f1e1ad48f
+  data.tar.gz: cde93ef7efa2f1286dffb59c3139aad87cd222a8
 SHA512:
-  metadata.gz: d34160fa07a75e25f747363046372cea00d0ea7688647f5cfd848579571b7f71d15801ddea989bf2310d8f49335d20511c95ee1b86a5564dadf4d3e9651a574f
-  data.tar.gz: c62d4e97ad2e04ac314ca36f900b21fab82b39eb109aed4123022e6ebd9aabe495adcd239dab42cf2f6b1ab0cf0ea29f539a765bbb515979b6d76a935ee8f73c
+  metadata.gz: 8c6c836c2c5051ddd48438b930279041ac8c5da3a5f48495e866018e58f0d8f81274de1133a1f5582c83ee4b5ff07028a33b9847fdc0309669ab1c4eac59bc1e
+  data.tar.gz: e7b8c961bb55cfdcc3b12f579e145217bddb24ca46de1f8351062ac959de3006c2b48a223e0b746b24ef192658d682d28bb8a81593f53ecd82cb5a0d45ecc276

data/.gitignore

@@ -0,0 +1,38 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+.DS_Store
+
+Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+2.1.3

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in cloudwatch-sender.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 BBC News
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,116 @@
+<h1 align="center">cloudwatch-sender</h1>
+
+<p align="center">
+  <a href="https://rubygems.org/gems/cloudwatch-sender"><img src="https://img.shields.io/gem/v/cloudwatch-sender.svg?style=flat-square"></a>
+</p>
+
+<p align="center">
+  Retrieves metrics from <b>Cloudwatch</b> and sends them to <b>InfluxDB</b>/<b>Graphite</b>.
+</p>
+
+## What is this?
+
+Allows you to send **EC2**, **SQS** and **custom** metrics from Cloudwatch to InfluxDB or Graphite. EC2 metrics are gathered via EC2 tags instead of EC2 Instance IDs, making the tool far more dynamic.
+
+## Installation
+
+Add this line to your Gemfile:
+
+```ruby
+gem "cloudwatch-sender"
+```
+
+Then execute:
+
+```
+bundle
+```
+
+Or install it yourself:
+
+```
+gem install cloudwatch-sender
+```
+
+## Usage
+
+### Command Line
+
+```sh
+cloudwatch-sender send_metrics /path/to/config.yaml $AWS_ACCESS_KEY $AWS_SECRET_KEY $AWS_REGION
+```
+
+If you would like to stream metrics to your endpoint at a set interval, use `continuous`:
+
+```sh
+cloudwatch-sender continuous /path/to/config.yaml $AWS_ACCESS_KEY $AWS_SECRET_KEY $AWS_REGION $INTERVAL
+```
+
+**Note** - the default `$INTERVAL` is 60 seconds.
+
+### Programmatically
+
+```ruby
+require "cloudwatch/sender/cli"
+
+loop do
+  Cloudwatch::Sender::CLI.new.send_metrics(config_path, key_id, access_key, region)
+  sleep 120
+end
+```
+
+```ruby
+require "cloudwatch/sender/cli"
+
+Cloudwatch::Sender::CLI.new.continuous(config_path, key_id, access_key, region, sleep_time)
+```
+
+## Configs
+
+The gem is powered via a YAML config file, see [example.yaml](https://github.com/BBC-News/cloudwatch-sender/blob/master/configs/example.yaml) for an example.
+
+**Note**: take into account how often metrics update for each AWS product:
+
+- **EC2** - every 60 seconds.
+- **SQS** - every 5 minutes.
+- **Custom** - every 60 seconds.
+
+## How it works
+
+The gem extracts metrics for a given set of EC2 instances based on an EC2 tag key/value. For example:
+
+```yaml
+ec2_tag_key:   ProjectName
+ec2_tag_value: bbc_news
+```
+
+As seen in the example above a tag relates to a single project. Thus if that key/value is called, it returns all the instances attached to that project - which is what the gem then uses to gather metrics on each instance.
+
+The gem collects EC2 metrics for the previous 3 minutes and SQS metrics for the previous 20 minutes. Thus running the gem every 60 seconds for EC2 and every 5 minutes for SQS will provide sufficient data. This allows for the gem to remain unaffected by network/database issues.
+
+## Why make this?
+
+We found the existing tools heavily rely upon AWS variables which often become out-of-date, e.g. EC2 Instance ID. This becomes a problem when a component is being regularly re-built and deployed. Thus we made the decision to base cloudwatch-sender on a broder set of criteria, such as EC2 tags.
+
+## TODO
+
+See [open issues](https://github.com/BBC-News/cloudwatch-sender/issues?utf8=%E2%9C%93&q=is%3Aopen+is%3Aissue+label%3Ato-do).
+
+## License
+
+Standard MIT License, see included [license file](https://github.com/BBC-News/cloudwatch-sender/blob/master/LICENSE.txt).
+
+## Authors
+
+- [David Blooman](http://twitter.com/dblooman)
+- [Charlie Revett](http://twitter.com/charlierevett)
+
+## Contributing
+
+1. [Fork it!](https://github.com/bbc-news/cloudwatch-sender/fork)
+2. Create your feature branch: `git checkout -b my-new-feature`
+3. Commit your changes: `git commit -am "Add some feature"`
+4. Push to the branch: `git push origin my-new-feature`
+5. Create a new [Pull Request](https://github.com/BBC-News/cloudwatch-sender/compare).
+
+Please feel free to raise an [issue](https://github.com/BBC-News/cloudwatch-sender/issues/new) if you find a bug or have a feature request.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/cloudwatch-sender

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+begin
+  require "cloudwatch/sender/cli"
+  Cloudwatch::Sender::CLI.start
+rescue Interrupt
+  puts "\nQuitting..."
+  exit 1
+rescue SystemExit => e
+  exit e.status
+end

data/cloudwatch-sender.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "cloudwatch/sender/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "cloudwatch-sender"
+  spec.version       = Cloudwatch::Sender::VERSION
+  spec.authors       = ["DaveBlooman", "Charlie Revett"]
+  spec.email         = ["david.blooman@gmail.com", "charlierevett@gmail.com"]
+  spec.summary       = "Cloudwatch Metrics Sender"
+  spec.description   = "Get metrics from Cloudwatch and send to Graphite/InfluxDB"
+  spec.homepage      = "https://github.com/BBC-News/cloudwatch-sender"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_runtime_dependency "aws-sdk"
+  spec.add_runtime_dependency "thor"
+end

data/configs/example.yaml

@@ -0,0 +1,63 @@
+---
+metric_prefix: "cloudwatch"
+influx_host: "influxdb.domain.com"
+influx_port: 2003
+
+metric_types:
+## EC2 Metrics
+-
+  namespace: "AWS/EC2"
+  ec2_tag_key: "BBCComponent"
+  ec2_tag_value: "newsbeat-composition"
+  metrics:
+    -
+      name: "CPUUtilization"
+      unit: "Percent"
+      statistics:
+        - Average
+    -
+      name: "NetworkIn"
+      unit: "Bytes"
+      statistics:
+        - Average
+    -
+      name: "NetworkOut"
+      unit: "Bytes"
+      statistics:
+        - Average
+
+## Custom Metrics
+-
+  namespace: "BBCApp/newsbeat/live"
+  metrics:
+    -
+      name: "Headline301Redirect"
+      unit: "Count"
+      statistics:
+        - Sum
+    -
+      name: "InvalidHeadline"
+      unit: "Count"
+      statistics:
+        - Sum
+    -
+      name: "LegacyArticle"
+      unit: "Count"
+      statistics:
+        - Sum
+
+## SQS Metrics
+-
+  namespace: "AWS/SQS"
+  queue_name: "live-news-ArticleQueue-N7RAX00100RC"
+  metrics:
+    -
+      name: "ApproximateNumberOfMessagesVisible"
+      unit: "Count"
+      statistics:
+        - Sum
+    -
+      name: "NumberOfMessagesDeleted"
+      unit: "Count"
+      statistics:
+        - Sum

data/lib/cloudwatch/sender/base.rb

@@ -0,0 +1,26 @@
+require "json"
+require "yaml"
+
+module Cloudwatch
+  module Sender
+    class Base
+      def initialize(server, port)
+        @influx_server = server
+        @influx_port   = port
+      end
+
+      def send_tcp(contents)
+        sock = TCPSocket.open(influx_server, influx_port)
+        sock.print contents
+      rescue StandardError => e
+        logger.debug "Error: #{e}"
+      ensure
+        sock.close
+      end
+
+      protected
+
+      attr_reader :influx_server, :influx_port
+    end
+  end
+end

data/lib/cloudwatch/sender/cli.rb

@@ -0,0 +1,47 @@
+require "thor"
+require "logger"
+require "cloudwatch/sender/ec2"
+require "cloudwatch/sender/metric_definition"
+require "cloudwatch/sender/fetcher/base"
+require "cloudwatch/sender/fetcher/ec2"
+require "cloudwatch/sender/fetcher/sqs"
+require "cloudwatch/sender/fetcher/custom"
+
+module Cloudwatch
+  module Sender
+    class CLI < Thor
+      include Thor::Actions
+
+      desc "send_metrics [metrics_file] [key_id] [access_key] [region]", "Gets metrics from Cloudwatch and sends them to influx"
+      def send_metrics(metrics_file, key_id, access_key, region)
+        setup_aws(key_id, access_key, region)
+        MetricDefinition.metric_type load_metrics(metrics_file)
+      end
+
+      desc "continuous [metrics_file] [key_id] [access_key] [region] [sleep time]", "Continuously sends metrics to Influx/Cloudwatch"
+      def continuous(metrics_file, key_id, access_key, region, sleep_time = 60)
+        logger = Logger.new(STDOUT)
+
+        loop do
+          begin
+            send_metrics(metrics_file, key_id, access_key, region)
+            sleep sleep_time.to_i
+          rescue => e
+            logger.debug("Unable to complete operation #{e}")
+          end
+        end
+      end
+
+      no_commands do
+        def load_metrics(metrics_file)
+          YAML.load(File.open(metrics_file))
+        end
+
+        def setup_aws(key_id, access_key, region)
+          Aws.config.update(:region      => region,
+                            :credentials => Aws::Credentials.new(key_id, access_key))
+        end
+      end
+    end
+  end
+end

data/lib/cloudwatch/sender/ec2.rb

@@ -0,0 +1,40 @@
+require "aws-sdk"
+
+module Cloudwatch
+  module Sender
+    class EC2
+      def initialize
+        @ec2 = Aws::EC2::Client.new
+      end
+
+      def list_instances(key, value)
+        list_instance_ids aws_resp(key, value)
+      end
+
+      private
+
+      attr_reader :ec2
+
+      def aws_resp(key, value)
+        ec2.describe_instances(
+          :filters => [
+            {
+              :name   => "tag-key",
+              :values => [key]
+            },
+            {
+              :name   => "tag-value",
+              :values => [value]
+            }
+          ]
+        )
+      end
+
+      def list_instance_ids(instances)
+        instances.reservations.map do |reservations|
+          reservations.instances.map(&:instance_id)
+        end
+      end
+    end
+  end
+end

data/lib/cloudwatch/sender/fetcher/base.rb

@@ -0,0 +1,57 @@
+require "cloudwatch/sender/base"
+
+module Cloudwatch
+  module Sender
+    module Fetcher
+      class Base
+        def initialize(components, namespace)
+          @components = components
+          @namespace  = namespace
+        end
+
+        def retrieve(component_meta)
+          component_meta["metrics"].each do |metric|
+            metric_type(component_meta, metric)
+          end
+        end
+
+        private
+
+        attr_reader :components, :namespace
+
+        def cloudwatch
+          Aws::CloudWatch::Client.new
+        end
+
+        def extract_class_name(component_meta)
+          component_meta["namespace"].split("/").last
+        end
+
+        def fetcher(component_meta)
+          namespace.start_with?("AWS/") ? Object.const_get(class_namespace component_meta)
+                                        : Cloudwatch::Sender::Fetcher::Custom
+        end
+
+        def class_namespace(component_meta)
+          "Cloudwatch::Sender::Fetcher::#{extract_class_name component_meta}"
+        end
+
+        def metric_prefix
+          components["metric_prefix"]
+        end
+
+        def metric_type(component_meta, metric)
+          fetcher(component_meta).new(
+            cloudwatch, sender, metric_prefix
+          ).metrics(component_meta, metric)
+        end
+
+        def sender
+          Cloudwatch::Sender::Base.new(
+            components["influx_host"], components["influx_port"]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/cloudwatch/sender/fetcher/custom.rb

@@ -0,0 +1,46 @@
+module Cloudwatch
+  module Sender
+    module Fetcher
+      class Custom
+        attr_reader :metric_prefix, :cloudwatch, :sender
+
+        def initialize(cloudwatch, sender, metric_prefix)
+          @cloudwatch = cloudwatch
+          @sender = sender
+          @metric_prefix = metric_prefix
+        end
+
+        START_TIME = 180
+
+        def metrics(component_meta, metric)
+          resp = cloudwatch.get_metric_statistics(
+            :namespace   => component_meta["namespace"],
+            :metric_name => metric["name"],
+            :start_time  => Time.now - START_TIME,
+            :end_time    => Time.now,
+            :period      => 60,
+            :statistics  => metric["statistics"],
+            :unit        => metric["unit"]
+          )
+          name = component_meta["namespace"].downcase
+          name_metrics(resp, name, metric["statistics"])
+        end
+
+        private
+
+        def name_metrics(resp, name, statistics)
+          resp.data["datapoints"].each do |data|
+            time = data["timestamp"].to_i
+            check_statistics(name, resp.data["label"], statistics, time, data)
+          end
+        end
+
+        def check_statistics(name, label, statistics, time, data)
+          statistics.each do |stat|
+            sender.send_tcp("#{metric_prefix}.#{name}.#{label.downcase}.#{stat.downcase}" " " "#{data[stat.downcase]}" " "  "#{time}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cloudwatch/sender/fetcher/ec2.rb

@@ -0,0 +1,70 @@
+module Cloudwatch
+  module Sender
+    module Fetcher
+      class EC2
+        def initialize(cloudwatch, sender, metric_prefix)
+          @cloudwatch = cloudwatch
+          @sender = sender
+          @metric_prefix = metric_prefix
+        end
+
+        def metrics(component_meta, metric)
+          ec2_metrics(instance_list(component_meta), component_meta, metric)
+        end
+
+        private
+
+        attr_reader :metric_prefix, :cloudwatch, :sender
+
+        START_TIME = 180
+
+        def ec2_metrics(instance_list, component_meta, metric)
+          instance_list.each do |instance|
+            metric_data = aws_metric_meta(component_meta, metric, instance)
+            resp = cloudwatch.get_metric_statistics metric_data
+            name_metrics(resp, instance, component_meta["ec2_tag_value"], metric["statistics"])
+          end
+        end
+
+        def aws_metric_meta(component_meta, metric, instance)
+          {
+            :namespace   => component_meta["namespace"],
+            :metric_name => metric["name"],
+            :dimensions  => [{ :name => "InstanceId", :value => instance }],
+            :start_time  => Time.now - START_TIME,
+            :end_time    => Time.now,
+            :period      => 60,
+            :statistics  => metric["statistics"],
+            :unit        => metric["unit"]
+          }
+        end
+
+        def ec2
+          Cloudwatch::Sender::EC2.new
+        end
+
+        def instance_list(component_meta)
+          ec2.list_instances(
+            component_meta["ec2_tag_key"], component_meta["ec2_tag_value"]
+          ).flatten
+        end
+
+        def name_metrics(resp, instance, name, statistics)
+          resp.data["datapoints"].each do |data|
+            check_statistics(instance, name, resp.data["label"], statistics, metric_time(data), data)
+          end
+        end
+
+        def metric_time(data)
+          data["timestamp"].to_i
+        end
+
+        def check_statistics(instanceid, name, label, statistics, time, data)
+          statistics.each do |stat|
+            sender.send_tcp("#{metric_prefix}.#{name}.#{instanceid}.#{label.downcase}.#{stat}" " " "#{data[stat.downcase]}" " "  "#{time}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cloudwatch/sender/fetcher/sqs.rb

@@ -0,0 +1,48 @@
+module Cloudwatch
+  module Sender
+    module Fetcher
+      class SQS
+        attr_reader :metric_prefix, :cloudwatch, :sender
+
+        def initialize(cloudwatch, sender, metric_prefix)
+          @cloudwatch = cloudwatch
+          @sender = sender
+          @metric_prefix = metric_prefix
+        end
+
+        START_TIME = 1200
+
+        def metrics(component_meta, metric)
+          resp = cloudwatch.get_metric_statistics(
+            :namespace   => component_meta["namespace"],
+            :metric_name => metric["name"],
+            :dimensions  => [{ :name  => "QueueName",
+                               :value => component_meta["queue_name"] }],
+            :start_time  => Time.now - START_TIME,
+            :end_time    => Time.now,
+            :period      => 60,
+            :statistics  => metric["statistics"],
+            :unit        => metric["unit"]
+          )
+          name = component_meta["namespace"].downcase
+          name_metrics(resp, name, metric["statistics"], component_meta["queue_name"])
+        end
+
+        private
+
+        def name_metrics(resp, name, statistics, queue_name)
+          resp.data["datapoints"].each do |data|
+            time = data["timestamp"].to_i
+            check_statistics(name, resp.data["label"], statistics, time, data, queue_name)
+          end
+        end
+
+        def check_statistics(name, label, statistics, time, data, queue_name)
+          statistics.each do |stat|
+            sender.send_tcp("#{metric_prefix}.#{name}.#{queue_name}.#{label.downcase}.#{stat.downcase}" " " "#{data[stat.downcase]}" " "  "#{time}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cloudwatch/sender/metric_definition.rb

@@ -0,0 +1,12 @@
+module Cloudwatch
+  module Sender
+    class MetricDefinition
+      def self.metric_type(components)
+        components["metric_types"].each do |component_meta|
+          namespace = component_meta["namespace"]
+          Cloudwatch::Sender::Fetcher::Base.new(components, namespace).retrieve component_meta
+        end
+      end
+    end
+  end
+end

data/lib/cloudwatch/sender/version.rb

@@ -0,0 +1,5 @@
+module Cloudwatch
+  module Sender
+    VERSION = "0.1.1"
+  end
+end

data/lib/cloudwatch_sender.rb

@@ -0,0 +1,7 @@
+require "cloudwatch/version"
+
+module Cloudwatch
+  module Sender
+    autoload :CLI, "cloudwatch/cli"
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloudwatch-sender
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - DaveBlooman
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-19 00:00:00.000000000 Z
+date: 2015-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -71,10 +71,30 @@ description: Get metrics from Cloudwatch and send to Graphite/InfluxDB
 email:
 - david.blooman@gmail.com
 - charlierevett@gmail.com
-executables: []
+executables:
+- cloudwatch-sender
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/cloudwatch-sender
+- cloudwatch-sender.gemspec
+- configs/example.yaml
+- lib/cloudwatch/sender/base.rb
+- lib/cloudwatch/sender/cli.rb
+- lib/cloudwatch/sender/ec2.rb
+- lib/cloudwatch/sender/fetcher/base.rb
+- lib/cloudwatch/sender/fetcher/custom.rb
+- lib/cloudwatch/sender/fetcher/ec2.rb
+- lib/cloudwatch/sender/fetcher/sqs.rb
+- lib/cloudwatch/sender/metric_definition.rb
+- lib/cloudwatch/sender/version.rb
+- lib/cloudwatch_sender.rb
 homepage: https://github.com/BBC-News/cloudwatch-sender
 licenses:
 - MIT