cloudwatchtographite 0.0.0.pre1

data/lib/cloudwatchtographite/metricdefinition.rb

@@ -0,0 +1,205 @@
+# _*_ coding: utf-8 _*_
+# == Synopsis
+# CloudwatchToGraphite retrieves metrics from the Amazon CloudWatch APIs
+# and passes them on to a graphite server
+#
+# == Author
+# S. Zachariah Sprackett <zac@sprackett.com>
+#
+# == License
+# The MIT License (MIT)
+#
+# == Copyright
+# Copyright (C) 2013 - S. Zachariah Sprackett <zac@sprackett.com>
+#
+require 'time'
+require 'hashifiable'
+
+module CloudwatchToGraphite
+  UNITS = [
+    'None',
+    'Seconds',
+    'Microseconds',
+    'Milliseconds',
+    'Bytes',
+    'Kilobytes',
+    'Megabytes',
+    'Gigabytes',
+    'Terabytes',
+    'Bits',
+    'Kilobits',
+    'Megabits',
+    'Gigabits',
+    'Terabits',
+    'Gigabytes/Second',
+    'Terabytes/Second',
+    'Bits/Second',
+    'Kilobits/Second',
+    'Megabits/Second',
+    'Gigabits/Second',
+    'Terabits/Second',
+    'Count/Second'
+  ]
+
+  STATISTICS = [
+    'Minimum',
+    'Maximum',
+    'Sum',
+    'Average',
+    'SampleCount'
+  ]
+
+  SETTER_MAPPINGS = {
+    'namespace'  => :Namespace=,
+    'metricname' => :MetricName=,
+    'statistics' => :Statistics=,
+    'starttime'  => :StartTime=,
+    'endtime'    => :EndTime=,
+    'period'     => :Period=,
+    'dimensions' => :Dimensions=,
+    'unit'       => :Unit=
+  }
+
+  # A hashable representation of an AWS CloudWatch metric
+  #
+  class MetricDefinition
+    attr_reader :Namespace, :MetricName, :Statistics, :Unit, :Period
+    extend Hashifiable
+    hashify 'Namespace', 'MetricName', 'Statistics', 'StartTime', \
+      'EndTime', 'Period', 'Dimensions'#, 'Unit'
+
+    def initialize
+      @Unit = UNITS[0]
+      @Dimensions = []
+      @Statistics = []
+      @Period = 60
+    end
+
+    def Namespace=(n)
+      Validator::string_shorter_than(n, 256)
+      @Namespace=n
+    end
+
+    def MetricName=(n)
+      Validator::string_shorter_than(n, 256)
+      @MetricName=n
+    end
+
+    def StartTime=(time)
+      raise ArgumentTypeError unless time.kind_of?(Time)
+      @StartTime=time
+    end
+
+    def StartTime
+      if @StartTime.nil?
+        (Time.now-600).iso8601
+      else
+        @StartTime.iso8601
+      end
+    end
+
+    def EndTime=(time)
+      raise ArgumentTypeError unless time.kind_of?(Time)
+      @EndTime=time
+    end
+
+    def EndTime
+      if @EndTime.nil?
+        Time.now.iso8601
+      else
+        @EndTime.iso8601
+      end
+    end
+
+    def Period=(n)
+      raise ArgumentTypeError unless n.kind_of? Integer
+      @Period = n
+    end
+
+    def Unit=(n)
+      raise ArgumentTypeError unless UNITS.include? n
+      @Unit = n
+    end
+
+    def Dimensions
+      @Dimensions.map(&:to_h)
+    end
+
+    def add_statistic(n)
+      raise ArgumentTypeError unless STATISTICS.include? n
+      if not @Statistics.include? n
+        @Statistics.push(n)
+      end
+    end
+
+    def add_dimension(n)
+      if not n.kind_of?(MetricDimension)
+        raise ArgumentTypeError
+      elsif @Dimensions.length >= 10
+        raise TooManyDimensionError
+      end
+      @Dimensions.push(n)
+    end
+
+    def valid?
+      if @Namespace.nil? or @MetricName.nil? or @Statistics.empty? \
+        or @Dimensions.empty? or @Unit.nil?
+          false
+      else
+        true
+      end
+    end
+
+    def graphite_path(stat)
+     path = "%s.%s.%s.%s" % [
+       self.Namespace,
+       self.MetricName,
+       stat,
+       @Dimensions.join('.')
+     ]
+     path.gsub('/', '.').downcase
+    end
+
+    def self.create_and_fill(definition)
+      md = MetricDefinition.new
+      definition.each_key do |k|
+        populate_metric_definition(
+          md, k, definition[k]
+        )
+      end
+      raise ParseError unless md.valid?
+      return md
+    end
+
+    def self.populate_metric_definition(md, key, value)
+      case key
+      when 'namespace', 'metricname', 'period', 'unit'
+        md.send(SETTER_MAPPINGS[key], value)
+      when 'starttime', 'endtime'
+        begin
+          md.send(SETTER_MAPPINGS[key], Time.parse(value))
+        rescue ArgumentTypeError
+          raise ParseError
+        end
+      when 'statistics'
+        populate_statistics(md, value)
+      when 'dimensions'
+        populate_dimensions_from_hashes(md, value)
+      else
+        raise ParseError
+      end
+    end
+
+    def self.populate_statistics(md, statistics)
+      Array(statistics).each do |stat|
+        md.add_statistic(stat)
+      end
+    end
+
+    def self.populate_dimensions_from_hashes(md, dimensions)
+      Array(dimensions).each do |dimension|
+        md.add_dimension(MetricDimension.create_from_hash(dimension))
+      end
+    end
+  end
+end

data/lib/cloudwatchtographite/metricdimension.rb

@@ -0,0 +1,48 @@
+# _*_ coding: utf-8 _*_
+# == Synopsis
+# CloudwatchToGraphite retrieves metrics from the Amazon CloudWatch APIs
+# and passes them on to a graphite server
+#
+# == Author
+# S. Zachariah Sprackett <zac@sprackett.com>
+#
+# == License
+# The MIT License (MIT)
+#
+# == Copyright
+# Copyright (C) 2013 - S. Zachariah Sprackett <zac@sprackett.com>
+#
+module CloudwatchToGraphite
+  # A hashable representation of an AWS CloudWatch metric dimension
+  class MetricDimension
+    attr_reader :Name, :Value
+    extend Hashifiable
+    hashify 'Name', 'Value'
+
+    def Name=(n)
+      Validator::string_shorter_than(n, 256)
+      @Name=n
+    end
+
+    def Value=(n)
+      Validator::string_shorter_than(n, 256)
+      @Value=n
+    end
+
+    def self.create(name, value)
+      md = MetricDimension.new
+      md.Name = name
+      md.Value = value
+      md
+    end
+
+    def self.create_from_hash(dhash)
+      Validator::hash_with_keys(dhash, ['name', 'value'])
+
+      MetricDimension::create(
+        dhash['name'],
+        dhash['value']
+      )
+    end
+  end
+end

data/lib/cloudwatchtographite/validator.rb

@@ -0,0 +1,47 @@
+# _*_ coding: utf-8 _*_
+# == Synopsis
+# CloudwatchToGraphite retrieves metrics from the Amazon CloudWatch APIs
+# and passes them on to a graphite server
+#
+# == Author
+# S. Zachariah Sprackett <zac@sprackett.com>
+#
+# == License
+# The MIT License (MIT)
+#
+# == Copyright
+# Copyright (C) 2013 - S. Zachariah Sprackett <zac@sprackett.com>
+#
+module CloudwatchToGraphite
+  # static methods to validate arguments
+  class Validator
+    def self.string(n)
+      raise ArgumentTypeError unless n.kind_of?(String)
+    end
+
+    def self.string_shorter_than(n, length)
+      string(n)
+      raise ArgumentLengthError unless n.length < length
+    end
+
+    def self.string_longer_than(n, length)
+      string(n)
+      raise ArgumentLengthError unless n.length > length
+    end
+
+    def self.hash_with_key(h, key)
+      raise ArgumentTypeError unless h.kind_of?(Hash) and h.has_key?(key)
+    end
+
+    def self.hash_with_keys(h, keys)
+      keys.each do |k|
+        hash_with_key(h,k)
+      end
+    end
+
+    def self.hash_with_key_of_type(h, key, type)
+      hash_with_key(h, key)
+      raise ArgumentTypeError unless h[key].kind_of?(type)
+    end
+  end
+end

data/lib/cloudwatchtographite/version.rb

@@ -0,0 +1,25 @@
+# _*_ coding: utf-8 _*_
+# == Synopsis
+# CloudwatchToGraphite retrieves metrics from the Amazon CloudWatch APIs
+# and passes them on to a graphite server
+#
+# == Author
+# S. Zachariah Sprackett <zac@sprackett.com>
+#
+# == License
+# The MIT License (MIT)
+#
+# == Copyright
+# Copyright (C) 2013 - S. Zachariah Sprackett <zac@sprackett.com>
+#
+module CloudwatchToGraphite
+  class VERSION
+    MAJOR = 0
+    MINOR = 0
+    PATCH = 0
+    BUILD = 'pre1'
+
+    STRING = [MAJOR, MINOR, PATCH].compact.join('.') \
+      + (BUILD.empty? ? '' : ".#{BUILD}")
+  end
+end

data/lib/cloudwatchtographite.rb

@@ -0,0 +1,185 @@
+# _*_ coding: utf-8 _*_
+#
+# Author:: S. Zachariah Sprackett <zac@sprackett.com>
+# License:: The MIT License (MIT)
+# Copyright:: Copyright (C) 2013 - S. Zachariah Sprackett <zac@sprackett.com>
+#
+require_relative './cloudwatchtographite/exception'
+require_relative './cloudwatchtographite/version'
+require_relative './cloudwatchtographite/metricdefinition'
+require_relative './cloudwatchtographite/metricdimension'
+require_relative './cloudwatchtographite/loadmetrics'
+require_relative './cloudwatchtographite/validator'
+require 'socket'
+require 'fog'
+require 'pp'
+
+module CloudwatchToGraphite
+  # This class is responsible for retrieving metrics from CloudWatch and
+  # sending the results to a Graphite server.
+  class Base
+    attr_accessor :protocol
+    attr_accessor :graphite_server
+    attr_accessor :graphite_port
+    attr_reader   :carbon_prefix
+
+    # Initialize the CloudwatchToGraphite::Base object.
+    # aws_access_key:: The AWS user key
+    # aws_secret_key:: The AWS secret
+    # region:: The AWS region (eg: us-west-1)
+    # verbose:: boolean to enable verbose output
+    #
+    def initialize(aws_access_key, aws_secret_key, region, verbose=false)
+      @protocol        = 'udp'
+      @carbon_prefix   = 'cloudwatch'
+      @graphite_server = 'localhost'
+      @graphite_port   = 2003
+      @verbose         = verbose
+
+      debug_log "Fog setting up for region #{region}"
+
+      @cloudwatch = Fog::AWS::CloudWatch.new(
+        :aws_access_key_id => aws_access_key,
+        :aws_secret_access_key => aws_secret_key,
+        :region => region
+      )
+    end
+
+    # Send data to a Graphite server via the UDP protocol
+    # contents:: a string or array containing the contents to send
+    #
+    def send_udp(contents)
+      sock = nil
+      contents = contents.join("\n") if contents.kind_of?(Array)
+
+      debug_log "Attempting to send #{contents.length}  bytes " +
+        "to #{@graphite_server}:#{@graphite_port} via udp"
+
+      begin
+        sock = UDPSocket.open
+        sock.send(contents, 0, @graphite_server, @graphite_port)
+        retval = true
+      rescue Exception => e
+        debug_log "Caught exception! [#{e}]"
+        retval = false
+      ensure
+        sock.close if sock
+      end
+      retval
+    end
+
+    # Send data to a Graphite server via the TCP protocol
+    # contents:: a string or array containing the contents to send
+    #
+    def send_tcp(contents)
+      sock = nil
+      contents = contents.join("\n") if contents.kind_of?(Array)
+
+      debug_log "Attempting to send #{contents.length}  bytes " +
+        "to #{@graphite_server}:#{@graphite_port} via tcp"
+
+      retval = false
+      begin
+        sock = TCPSocket.open(@graphite_server, @graphite_port)
+        sock.print(contents)
+        retval = true
+      rescue Exception => e
+        debug_log "Caught exception! [#{e}]"
+      ensure
+        sock.close if sock
+      end
+      retval
+    end
+
+    def retrieve_datapoints(metrics)
+      ret = []
+      Array(metrics).each do |m|
+        begin
+          ret.concat retrieve_one_datapoint(m)
+        rescue Excon::Errors::SocketError, Excon::Errors::BadRequest => e
+          warn "[Error in CloudWatch call] #{e.message}"
+        rescue Excon::Errors::Forbidden
+          warn "[Error in CloudWatch call] permission denied - check keys!"
+        end
+      end
+      ret
+    end
+
+    def retrieve_one_datapoint(metric)
+      debug_log "Sending to CloudWatch:"
+      debug_object metric.to_h
+      data_points = @cloudwatch.get_metric_statistics(
+        metric.to_h
+      ).body['GetMetricStatisticsResult']['Datapoints']
+      debug_log "Received from CloudWatch:"
+      debug_object data_points
+
+      return retrieve_statistics(metric, order_data_points(data_points))
+    end
+
+    def retrieve_statistics(metric, data_points)
+      ret = []
+      metric.Statistics.each do |stat|
+        name = "#{@carbon_prefix}.#{metric.graphite_path(stat)}"
+        data_points.each do |d|
+          ret.push "#{name} #{d[stat]} #{d['Timestamp'].utc.to_i}"
+        end
+      end
+      debug_log "Returning Statistics:"
+      debug_object ret
+      ret
+    end
+
+    def fetch_and_forward(metrics)
+      results = retrieve_datapoints(metrics)
+      if results.length == 0
+        false
+      else
+        case @protocol
+        when 'tcp'
+          send_tcp(results)
+        when 'udp'
+          send_udp(results)
+        else
+          debug_log "Unknown protocol #{@protocol}"
+          raise ProtocolError
+        end
+      end
+    end
+
+    # set the carbon prefix
+    # p:: the string prefix to use
+    def carbon_prefix=(p)
+      Validator::string_longer_than(p, 0)
+      @carbon_prefix=p
+    end
+
+    private
+    def order_data_points(data_points)
+      if data_points.nil?
+        data_points = []
+      else
+        data_points = Array(data_points)
+      end
+
+      if data_points.length == 0
+        warn "No data points!"
+        data_points
+      else
+        data_points = data_points.sort_by {|array| array['Timestamp'] }
+      end
+    end
+
+    def debug_log(s)
+      if @verbose
+        warn s
+      end
+    end
+
+    def debug_object(s)
+      if @verbose
+        warn PP.pp(s, "")
+      end
+    end
+  end
+end

data/spec/cloudwatchtographite_spec.rb

@@ -0,0 +1,28 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe CloudwatchToGraphite::Base do
+  before :each do
+    @base = CloudwatchToGraphite::Base.new 'foo', 'bar', 'us-east-1', false
+  end
+
+  describe ".new" do
+    it "takes four parameters and returns a CloudwatchToGraphite::Base" do
+      @base.should be_an_instance_of CloudwatchToGraphite::Base
+    end
+  end
+
+  describe ".fetch_and_forward" do
+  end
+
+  describe ".carbon_prefix=" do
+    it "allows setting a prefix for carbon" do
+      expect { @base.carbon_prefix = 123 }.to \
+        raise_error(CloudwatchToGraphite::ArgumentTypeError)
+      expect { @base.carbon_prefix = '' }.to \
+        raise_error(CloudwatchToGraphite::ArgumentLengthError)
+      @base.carbon_prefix = "the_prefix"
+      @base.carbon_prefix.should eql "the_prefix"
+    end
+  end
+end
+

data/spec/factories/metricdefinition.rb

@@ -0,0 +1,12 @@
+FactoryGirl.define do
+  factory :metricdefinition, class: CloudwatchToGraphite::MetricDefinition do
+    sequence(:MetricName) { |n| "metricname#{n}" }
+    sequence(:Namespace) { |n| "namespace#{n}" }
+    Period 90
+    Unit 'None'
+    after(:build) do |definition, evaluator|
+      definition.add_statistic('Sum')
+      definition.add_statistic('Average')
+    end
+  end
+end

data/spec/factories/metricdimension.rb

@@ -0,0 +1,6 @@
+FactoryGirl.define do
+  factory :metricdimension, class: CloudwatchToGraphite::MetricDimension do
+    sequence(:Name) { |n| "name#{n}" }
+    sequence(:Value) { |n| "value#{n}" }
+  end
+end

data/spec/loadmetrics_spec.rb

@@ -0,0 +1,27 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe CloudwatchToGraphite::LoadMetrics do
+
+  describe "#load_content" do
+    it "it should not load invalid metrics" do
+      content = {'shmetrics' => 'foo'}
+      expect {
+        CloudwatchToGraphite::LoadMetrics::load_content(content)
+      }.to raise_error(CloudwatchToGraphite::ArgumentTypeError)
+    end
+
+    #it "it should load valid metrics" do
+    #  content = {
+    #    'metrics'    => [ {
+    #      'namespace'  => 'thenamespace',
+    #      'metricname' => 'themetricname',
+    #      'statistics' => 'Average',
+    #      'period'     => 1,
+    #      'dimensions' => [ { 'name' => 'thename', 'value' => 'thevalue' }]
+    #    }]
+    #  }
+    #  metrics = CloudwatchToGraphite::LoadMetrics::load_content(content)
+    #  metrics.should be_an(Array)
+    #end
+  end
+end