resque_to_cloudwatch 1.0.3 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d88766e5181718a770178730716c7d9ad96c7b0b
-  data.tar.gz: 539d25384ef0cb4e83663950c0ffe56bd0e42eb5
+  metadata.gz: 11e5ee66485c3ad30e8ceb3c2c62918025b90433
+  data.tar.gz: c5cdef90cef5a6ac4dc60619add241626425707d
 SHA512:
-  metadata.gz: ce13d441f2406006263875786c3f5a4bf1c0fe3ae4c28e5f30e9fc85739f79faad4b3461ac0ab0ff8a0cea688456d68e6e53e4de2cfcebd5d8e590b0a993a3a8
-  data.tar.gz: 878cd5a3c2dfe5b9a30c97317042fe8c3c8b38115a87b29a6375115ec385e1b9959de3d84178c0a994138c44be545ec0c7304e7c4412796e1c9de3e74406fedf
+  metadata.gz: 0d66e4be36c5b12e0fa3084630c6bfd39accee9d5fbb983ae66aefd2536c99cea7a1199bfb1d4e621fa4490293892ba3437486b94a60c8bb57c3d350d8514e9e
+  data.tar.gz: 956431e69c343e0dabff5e28a128b322269c602862ba764b8b6014595a48ba0f931fe7ec2021e0caee77bd021a85628e58a36794eba8df8fa61cdaed2e25fc5b

data/README.md

@@ -1,7 +1,7 @@
 # resque_to_cloudwatch
 
 This is a gem containing a daemon for submitting Resque queue lengths to AWS
-Cloudwatch.
+Cloudwatch. In addition, it can submit this metric to Graphite.
 
 ## Usage
 
@@ -12,6 +12,8 @@ The daemon lives at `bin/resque_to_cloudwatch`, and takes only a single paramete
 If you don't supply this parameter, the daemon will look for a config file in the 
 current directory.
 
+## Configuration
+
 The config file has the following format (in YAML):
 
     access_key_id: asdfasfasd
@@ -21,4 +23,42 @@ The config file has the following format (in YAML):
     hostname: laptop
     redis_host: some.redis.host
     redis_port: 6379
-    period: 60
+    period: 60
+    namespace: F3D
+
+#### `access_key_id`, `secret_access_key`
+
+Your AWS access key/secret key pair.
+
+#### `region`
+
+Region of Cloudwatch the metric should be submitted to.
+
+#### `project`, `namespace`, `hostname`
+
+These can actually contain any text - they are submitted as "dimensions" along
+with the metric value. A single metric in Cloudwatch is uniquely identified by 
+its name (hardcoded to `jobs_queued` in this gem), and any dimensions it has.
+At Forward3D, we have a number of autoscaling "projects" (codebases), so we use
+the project dimension to represent that. Namespace is more important, as that
+determines how the metric is categorised in the Cloudwatch interface. Hostname
+can be anything - we set it to the hostname of the submitting machine.
+
+#### `redis_host`, `redis_port`
+
+Redis host and port.
+
+#### `enable_graphite`, `graphite_host`, `graphite_port`
+
+This gem can also send the metric it collects to Graphite - set `enable_graphite`
+to `true`, and set the Graphite host and port if you want to the daemon to do this.
+
+#### `period`
+
+The period of the EventMachine loop - how often stats are collected and sent.
+
+## Graphite metric
+
+The Graphite metric name will look like this:
+
+    resque_to_cloudwatch.namespace.hostname.project

data/bin/resque_to_cloudwatch

@@ -18,19 +18,26 @@ end.parse!
 options[:config] ||= "#{File.expand_path("../../config.yaml", __FILE__)}"
 config = ResqueToCloudwatch::Config.new(options[:config])
 collector = ResqueToCloudwatch::Collector.new(config)
-sender = ResqueToCloudwatch::Sender.new(config)
 
+# Initialize senders we're going to use
+senders = []
+senders << ResqueToCloudwatch::CloudwatchSender.new(config)
+senders << ResqueToCloudwatch::GraphiteSender.new(config) if config.enable_graphite
+
+$log.info "Senders in use: #{senders.join(", ")}"
 $log.info "Entering EventMachine loop with period of #{config.period} seconds"
 
 EventMachine.run do
   EventMachine.add_periodic_timer(config.period) do
-    begin
-      sender.send_value(collector.get_queue_length)
-    rescue => e
-      $log.error "Exception sending or collecting data: #{e.message}"
-      e.backtrace.each do |line|
-        $log.error line
+    senders.each do |sender|
+      begin
+        sender.send_value(collector.get_queue_length)
+      rescue => e
+        $log.error "Exception sending or collecting data with sender #{sender}: #{e.message}"
+        e.backtrace.each do |line|
+          $log.error line
+        end
       end
-    end
+    end    
   end
 end

data/config.yaml

@@ -6,3 +6,4 @@ hostname: andy-laptop
 redis_host: some.redis.host
 redis_port: 6379
 period: 60
+namespace: F3D

data/lib/resque_to_cloudwatch/{sender.rb → cloudwatch_sender.rb}

@@ -1,7 +1,7 @@
 require 'aws-sdk'
 
 module ResqueToCloudwatch
-  class Sender
+  class CloudwatchSender
     
     # Pass an instance of CloudwatchToResque::Config
     def initialize(config)
@@ -11,7 +11,7 @@ module ResqueToCloudwatch
     def send_value(value)
       cw = AWS::CloudWatch.new
       cw.client.put_metric_data({
-        :namespace      => 'F3D/resque_queues',
+        :namespace      => "#{@config.namespace}/resque_queues",
         :metric_data    => [
           :metric_name  => "jobs_queued",
           :dimensions   => [
@@ -28,7 +28,17 @@ module ResqueToCloudwatch
           :unit  => 'Count'
         ]
       })
-      $log.info "Sent metric value #{value}"
+      $log.info "CloudwatchSender: Sent metric value #{value}"
     end
+    
+    def inspect
+      to_s
+    end
+    
+    def to_s
+      "CloudwatchSender"
+    end
+    
   end
+  
 end

data/lib/resque_to_cloudwatch/config.rb

@@ -6,17 +6,19 @@ module ResqueToCloudwatch
    
     attr_reader :access_key_id, :secret_access_key, :project, :period, :region
     attr_reader :redis_host, :redis_port, :hostname
+    attr_reader :graphite_host, :graphite_port, :enable_graphite
+    attr_reader :namespace
    
     def initialize(path)
       $log.info "Loading configuration"
       raise "Config file #{path} not found or readable" unless File.exists?(path)
-      @required_opts = %w{access_key_id secret_access_key project period region redis_host redis_port hostname}
+      @required_opts = %w{access_key_id secret_access_key project period region redis_host redis_port hostname namespace}
       @hash = YAML.load_file(path)
       raise "Config file #{path} is empty" unless @hash
       validate_config
-      @required_opts.each do |opt|
-        instance_variable_set("@#{opt}", @hash[opt])
-        $log.info "Config parameter: #{opt} is #{@hash[opt]}"
+      @hash.each_pair do |opt,value|
+        instance_variable_set("@#{opt}", value)
+        $log.info "Config parameter: #{opt} is #{value}"
       end
       
       # Set up AWS credentials
@@ -34,6 +36,12 @@ module ResqueToCloudwatch
         @hash[opt].nil?
       end
       raise "Missing options: #{missing_opts.join(", ")}" unless missing_opts.empty?
+      if @hash["enable_graphite"]
+        raise "Graphite enabled but config missing graphite_host" if @hash["graphite_host"].nil?
+        @hash["graphite_port"] ||= 2003
+      else
+        @hash["enable_graphite"] = false
+      end
     end
     
   end

data/lib/resque_to_cloudwatch/graphite_sender.rb

@@ -0,0 +1,28 @@
+require 'simple-graphite'
+
+module ResqueToCloudwatch
+  class GraphiteSender
+    
+    # Pass an instance of ResqueToCloudwatch::Config
+    def initialize(config)
+      @config = config
+    end
+    
+    def send_value(value)
+      graphite = Graphite.new({:host => @config.graphite_host, :port => @config.graphite_port})
+      graphite.send_metrics({
+        "resque_to_cloudwatch.#{@config.namespace}.#{@config.hostname}.#{@config.project}" => value
+      })
+      $log.info "GraphiteSender: sent metric value #{value}"
+    end
+    
+    def inspect
+      to_s
+    end
+    
+    def to_s
+      "GraphiteSender"
+    end
+    
+  end
+end

data/lib/resque_to_cloudwatch/version.rb

@@ -1,3 +1,3 @@
 module ResqueToCloudwatch
-  VERSION = "1.0.3"
+  VERSION = "1.1.0"
 end

data/lib/resque_to_cloudwatch.rb

@@ -3,5 +3,6 @@ $stdout.sync = true
 $log = Logger.new($stdout)
 
 require_relative "resque_to_cloudwatch/config.rb"
-require_relative "resque_to_cloudwatch/sender.rb"
+require_relative "resque_to_cloudwatch/cloudwatch_sender.rb"
+require_relative "resque_to_cloudwatch/graphite_sender.rb"
 require_relative "resque_to_cloudwatch/collector.rb"

data/resque_to_cloudwatch.gemspec

@@ -24,5 +24,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "aws-sdk", "= 1.24.0"
   spec.add_dependency "eventmachine", "= 1.0.3"
   spec.add_dependency "redis", "= 3.0.6"
+  spec.add_dependency "simple-graphite", "~> 2.1.0"
   
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque_to_cloudwatch
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.0
 platform: ruby
 authors:
 - Andy Sykes
@@ -80,6 +80,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.0.6
+- !ruby/object:Gem::Dependency
+  name: simple-graphite
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.0
 description: Submit Resque queue lengths to Cloudwatch
 email:
 - github@tinycat.co.uk
@@ -98,9 +112,10 @@ files:
 - bin/resque_to_cloudwatch
 - config.yaml
 - lib/resque_to_cloudwatch.rb
+- lib/resque_to_cloudwatch/cloudwatch_sender.rb
 - lib/resque_to_cloudwatch/collector.rb
 - lib/resque_to_cloudwatch/config.rb
-- lib/resque_to_cloudwatch/sender.rb
+- lib/resque_to_cloudwatch/graphite_sender.rb
 - lib/resque_to_cloudwatch/version.rb
 - resque_to_cloudwatch.gemspec
 homepage: https://github.com/forward3d/resque_to_cloudwatch