sidewatch 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4b49f4160ae5ce3ac9bf51f9368f526659f841ba
+  data.tar.gz: 533bfa8a7b43ccb239ed218c1dcfeb26e5b1c3bb
+SHA512:
+  metadata.gz: b76fa21af012ac9750c1397324ee005fb0389626657cacec615e740ee45b4b46679d0251555e29676b2cdc59f2f88eeb4ec7cc97478a45f64c962fb2849c5639
+  data.tar.gz: 058f477685ebe6800eda1394913a626b702a1d07ec85cf0b93be25afdf3f1f477b1aa0b86f5173cad60a5b6aeb1b015444ee6867958514cbca20c5ec0ee7da77

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.log
+*.yml
+*.rbc
+.bundle
+.env
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Jan Hecking
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,35 @@
+# Sidewatch
+
+Sidewatch uploads Sidekiq metrics to AWS CloudWatch in regular intervals.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sidewatch'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidewatch
+
+## Usage
+
+To start sidewatch as a daemon process in the background run
+
+    ruby sidewatch.rb start
+
+To run sidewatch in the foreground, e.g. for testing, run
+
+    ruby sidewatch.rb run
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"

data/bin/sidewatch

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "bundler/setup"
+require "dotenv"
+require "sidewatch"
+
+Bundler.require
+Dotenv.load
+
+pwd = Dir.pwd
+Daemons.run_proc("sidewatch") do
+  Sidewatch::CLI.new(pwd).run
+end

data/lib/sidewatch.rb

@@ -0,0 +1,19 @@
+require "sidewatch/version"
+require "sidewatch/cli"
+require "sidewatch/daemon"
+require "sidewatch/metric"
+require "sidewatch/sidekiq"
+require "sidewatch/cloudwatch"
+
+module Sidewatch
+
+  class << self
+    attr_accessor :config
+    attr_accessor :logger
+  end
+
+  self.config = {}
+  self.logger = Logger.new(STDERR)
+  logger.level = Logger::WARN
+
+end

data/lib/sidewatch/cli.rb

@@ -0,0 +1,66 @@
+require "erb"
+
+module Sidewatch
+  class CLI
+
+    attr_accessor :options
+    attr_accessor :config_file
+
+    def initialize(pwd = Dir.pwd)
+      self.working_dir = pwd
+      parse_options
+      parse_config
+      setup_logger
+    end
+
+    def run
+      Sidewatch::Daemon.new(Sidewatch.config).run
+    end
+
+    private
+
+    attr_accessor :working_dir
+
+    def drop_daemons_args(args)
+      args = args.drop_while{|arg| arg != "--"}
+      args.drop(1)
+    end
+
+    def parse_options(args = ARGV)
+      args = drop_daemons_args(args)
+      options = {}
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: #{__FILE__} [start|stop|run] (-- [options])"
+        opts.on("-c", "--config PATH", "path to YAML Config file") do |file|
+          options[:config] = file
+        end
+        opts.on("-l", "--logfile PATH", "path to writeable logfile") do |file|
+          options[:logfile] = file
+        end
+      end
+      parser.parse(args)
+      self.options = options
+    end
+
+    def parse_config
+      self.config_file = options.fetch(:config){ "/etc/sidewatch.yml" }
+      self.config_file = File.expand_path(config_file, working_dir)
+      if File.exists?(config_file)
+        config = YAML.load(ERB.new(IO.read(config_file)).result)
+        config.merge(options)
+        Sidewatch.config = config.symbolize_keys
+      end
+    end
+
+    def setup_logger
+      logfile =
+        if options[:logfile]
+          File.expand_path(options[:logfile], working_dir)
+        elsif Sidewatch.config[:logfile]
+          File.expand_path(Sidewatch.config[:logfile], File.dirname(config_file))
+        end
+      Sidewatch.logger = Logger.new(logfile) if logfile
+    end
+
+  end
+end

data/lib/sidewatch/cloudwatch.rb

@@ -0,0 +1,49 @@
+require "aws-sdk"
+
+module Sidewatch
+  class Cloudwatch
+
+    attr_accessor :client
+
+    def initialize(config = {})
+      client = config.delete(:client){ Aws::CloudWatch::Client }
+      client = Object.const_get(client) if client.is_a?(String)
+      client = client.new(config) if client.is_a?(Class)
+      self.client = client
+    end
+
+    def send(metrics)
+      metrics.group_by(&:namespace).each do |ns, ns_metrics|
+        data = metric_data(ns, ns_metrics)
+        client.put_metric_data(data)
+      end
+    end
+
+    private
+
+    def metric_data(ns, metrics)
+      {
+        namespace: ns,
+        metric_data: metrics.map{|m|
+          {
+            metric_name: m.name,
+            dimensions: Array(m.dimensions).map{|name, value| {name: name, value: value}},
+            timestamp: m.timestamp,
+            value: m.value,
+            unit: m.unit
+          }
+        }
+      }
+    end
+
+    class MockClient
+      def initialize(config = {})
+        @out = config.fetch(:output){ $stdout }
+      end
+      def put_metric_data(data)
+        @out << data.to_s
+      end
+    end
+
+  end
+end

data/lib/sidewatch/daemon.rb

@@ -0,0 +1,46 @@
+require "daemons"
+
+module Sidewatch
+
+  class Daemon
+
+    attr_accessor :sidekiq
+    attr_accessor :cloudwatch
+    attr_accessor :interval
+
+    def initialize(config = {})
+      sidekiq_config = config.fetch(:sidekiq){ Hash.new }.symbolize_keys
+      self.sidekiq = Sidewatch::Sidekiq.new(sidekiq_config)
+      cloudwatch_config = config.fetch(:cloudwatch){ Hash.new }.symbolize_keys
+      self.cloudwatch = Sidewatch::Cloudwatch.new(cloudwatch_config)
+      self.interval = config.fetch(:interval){ 60 }
+    end
+
+    def run
+      logger.info("Starting #{self.class}...")
+      every(interval) do
+        metrics = sidekiq.metrics
+        logger.info(metrics)
+        cloudwatch.send(metrics)
+      end
+    end
+
+    private
+
+    def every(interval, &block)
+      loop do
+        ts = Time.now
+        block.call
+        remaining = (ts + interval) - Time.now
+        sleep remaining
+      end
+    rescue Interrupt
+      logger.info("Interrupted; exiting.")
+    end
+
+    def logger
+      Sidewatch.logger
+    end
+
+  end
+end

data/lib/sidewatch/metric.rb

@@ -0,0 +1,29 @@
+class Metric
+
+  attr_accessor :namespace
+  attr_accessor :name
+  attr_accessor :value
+  attr_accessor :unit
+  attr_accessor :dimensions
+  attr_accessor :timestamp
+
+  def initialize(attributes = {})
+    attributes.each do |name, value|
+      public_send("#{name}=", value)
+    end
+    self.timestamp ||= Time.now.utc
+  end
+
+  def inspect
+    ts = timestamp.iso8601 if timestamp
+    {
+      namespace: namespace,
+      name: name,
+      value: value,
+      unit: unit,
+      dimensions: dimensions,
+      timestamp: ts
+    }.reject{|_,v| v.nil?}
+  end
+
+end

data/lib/sidewatch/sidekiq.rb

@@ -0,0 +1,72 @@
+require "sidekiq"
+
+module Sidewatch
+  class Sidekiq
+
+    attr_accessor :namespace
+
+    def initialize(config = {})
+      self.namespace = config.fetch(:namespace){ "Sidekiq" }
+      ::Sidekiq.redis = config[:redis].symbolize_keys if config[:redis]
+    end
+
+    def metrics
+      metrics = []
+      metrics += global_metrics
+      metrics += per_queue_metrics
+      metrics
+    end
+
+    def global_metrics
+      stats = sidekiq_stats
+      metrics = []
+      [:failed, :enqueued, :processed, :retry_size, :scheduled_size].each do |stat|
+        name = camelcase(stat)
+        value = stats.send(stat)
+        metrics << Metric.new(namespace: namespace, name: name, value: value, unit: "Count", dimensions: default_dimensions)
+      end
+      metrics
+    end
+
+    def per_queue_metrics
+      metrics = []
+      sidekiq_queues.each do |queue|
+        dimensions = default_dimensions.merge("QueueName" => queue.name)
+        metrics << Metric.new(namespace: namespace, name: "Enqueued", value: queue.size, unit: "Count", dimensions: dimensions)
+        metrics << Metric.new(namespace: namespace, name: "Latency", value: queue.latency, unit: "Count", dimensions: dimensions)
+      end
+      metrics
+    end
+
+    private
+
+    def sidekiq_stats
+      ::Sidekiq::Stats.new
+    end
+
+    def sidekiq_queues
+      ::Sidekiq::Queue.all
+    end
+
+    def redis_uri
+      # is there a simpler way to get this?
+      uri = ::Sidekiq.redis {|redis|
+        redis = redis.redis if redis.is_a?(Redis::Namespace)
+        redis.client.id
+      }
+      uri = URI.parse(uri)
+      uri.password = "*****" if uri.password
+      uri.to_s
+    end
+
+    def default_dimensions
+      { "RedisURI" => redis_uri }
+    end
+
+    def camelcase(term)
+      string = term.to_s
+      string.gsub(/(?:^|_)([a-z\d]*)/){ $1.capitalize }
+    end
+
+  end
+end

data/lib/sidewatch/version.rb

@@ -0,0 +1,3 @@
+module Sidewatch
+  VERSION = "0.0.1"
+end

data/sidewatch.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidewatch/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sidewatch"
+  spec.version       = Sidewatch::VERSION
+  spec.authors       = ["Jan Hecking"]
+  spec.email         = ["jan@travelmob.com"]
+  spec.description   = %q{Uploads Sidekiq metrics to AWS CloudWatch in regular intervals}
+  spec.summary       = %q{Uploads Sidekiq metrics to AWS CloudWatch in regular intervals}
+  spec.homepage      = "https://github.com/jhecking"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake", "~> 10.4"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_runtime_dependency "daemons", "~> 1.2"
+  spec.add_runtime_dependency "sidekiq", "~> 2.17"
+  spec.add_runtime_dependency "aws-sdk", "~> 2.0"
+  spec.add_runtime_dependency "dotenv", "~> 2.0"
+end

data/spec/cloudwatch_spec.rb

@@ -0,0 +1,53 @@
+require "spec_helper"
+
+module Sidewatch
+
+  describe Cloudwatch do
+
+    let(:client){ spy("cloudwatch-client") }
+    subject{ described_class.new(client: client) }
+
+    describe "#send" do
+
+      it "sends the metrics using the cloudwatch client" do
+        metrics = []
+        metrics << Metric.new(namespace: "Clocks", name: "Ticks", value: 1234, unit: "Count", dimensions: { "ClockName" => "cuckoo" }, timestamp: Time.at(1440494300))
+        expected = {
+          namespace: "Clocks",
+          metric_data: [
+            {
+              metric_name: "Ticks",
+              dimensions: [
+                {
+                  name: "ClockName",
+                  value: "cuckoo"
+                }
+              ],
+              timestamp: Time.at(1440494300),
+              value: 1234,
+              unit: "Count"
+            }
+          ]
+        }
+
+        subject.send(metrics)
+
+        expect(client).to have_received(:put_metric_data).with(expected)
+      end
+
+      it "groups the metrics by namespace" do
+        metrics = []
+        metrics << Metric.new(namespace: "Clocks", name: "Ticks", value: 1234, unit: "Count")
+        metrics << Metric.new(namespace: "Events", name: "Failed", value: 55, unit: "Count")
+        received = []
+        allow(client).to receive(:put_metric_data){ |data| received << data }
+
+        subject.send(metrics)
+
+        expect(received).to contain_exactly(hash_including(namespace: "Clocks"), hash_including(namespace: "Events"))
+      end
+
+    end
+
+  end
+end

data/spec/metric_spec.rb

@@ -0,0 +1,34 @@
+require "spec_helper"
+
+module Sidewatch
+
+  describe Metric do
+
+    let(:namespace){ "Clocks" }
+    let(:metric_name){ "Ticks" }
+    let(:metric_value){ 1234 }
+    let(:metric_unit){ "Count" }
+    let(:dimensions){ { "ClockName" => "cuckoo" } }
+    let(:timestamp){ Time.at(1440494300) }
+    subject {
+      described_class.new(
+        name: metric_name,
+        value: metric_value,
+        unit: metric_unit,
+        dimensions: dimensions,
+        timestamp: timestamp,
+      )
+    }
+
+    describe "#initialize" do
+
+      context "missing timestamp" do
+        let(:timestamp){ nil }
+        it "sets a default timestamp" do
+          expect(subject.timestamp).to_not be_nil
+        end
+      end
+
+    end
+  end
+end

data/spec/sidekiq_spec.rb

@@ -0,0 +1,95 @@
+require "spec_helper"
+
+module Sidewatch
+
+  describe Sidekiq do
+
+    before(:each) do
+      allow(::Sidekiq).to receive(:redis=)
+      allow(subject).to receive(:redis_uri){ "redis://localhost:6379/0" }
+    end
+
+    describe "#metrics" do
+
+      it "returns global and per-queue metrics" do
+        global = double("global-metric")
+        expect(subject).to receive(:global_metrics).and_return([global])
+        per_queue = double("queue-metric")
+        expect(subject).to receive(:per_queue_metrics).and_return([per_queue])
+
+        metrics = subject.metrics
+
+        expect(metrics).to contain_exactly(global, per_queue)
+      end
+    end
+
+    describe "#global_metrics" do
+
+      let(:stats){
+        double("stats",
+          failed: 111,
+          enqueued: 222,
+          processed: 333,
+          retry_size: 444,
+          scheduled_size: 555,
+        )
+      }
+      before(:each) do
+        allow(subject).to receive(:sidekiq_stats){ stats }
+      end
+
+      it "produces a metric for failed jobs" do
+        failed = subject.global_metrics.find{|m| m.name == "Failed"}
+        expect(failed.value).to eq 111
+      end
+
+      it "produces a metric for enqueued jobs" do
+        failed = subject.global_metrics.find{|m| m.name == "Enqueued"}
+        expect(failed.value).to eq 222
+      end
+
+      it "produces a metric for processed jobs" do
+        failed = subject.global_metrics.find{|m| m.name == "Processed"}
+        expect(failed.value).to eq 333
+      end
+
+      it "produces a metric for retry queue size" do
+        failed = subject.global_metrics.find{|m| m.name == "RetrySize"}
+        expect(failed.value).to eq 444
+      end
+
+      it "produces a metric for scheduled queue size" do
+        failed = subject.global_metrics.find{|m| m.name == "ScheduledSize"}
+        expect(failed.value).to eq 555
+      end
+
+    end
+
+    describe "#per_queue_metrics" do
+
+      let(:queues){
+        [
+          double("queue1", name: "high-priority", size: 50, latency: 5),
+          double("queue2", name: "low-priority", size: 100, latency: 10),
+        ]
+      }
+      before(:each) do
+        allow(subject).to receive(:sidekiq_queues){ queues }
+      end
+
+      it "produces queue length and latency metrics for each queue" do
+        metrics = subject.per_queue_metrics.map(&:name).uniq
+
+        expect(metrics).to contain_exactly("Enqueued", "Latency")
+      end
+
+      it "produces at least one metric for every queue" do
+        metrics = subject.per_queue_metrics
+        queues = metrics.map{|metric| metric.dimensions["QueueName"]}.uniq
+
+        expect(queues).to contain_exactly("high-priority", "low-priority")
+      end
+
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require "sidewatch"

metadata

@@ -0,0 +1,165 @@
+--- !ruby/object:Gem::Specification
+name: sidewatch
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jan Hecking
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.4'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.17'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.17'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: Uploads Sidekiq metrics to AWS CloudWatch in regular intervals
+email:
+- jan@travelmob.com
+executables:
+- sidewatch
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/sidewatch
+- lib/sidewatch.rb
+- lib/sidewatch/cli.rb
+- lib/sidewatch/cloudwatch.rb
+- lib/sidewatch/daemon.rb
+- lib/sidewatch/metric.rb
+- lib/sidewatch/sidekiq.rb
+- lib/sidewatch/version.rb
+- sidewatch.gemspec
+- spec/cloudwatch_spec.rb
+- spec/metric_spec.rb
+- spec/sidekiq_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/jhecking
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: Uploads Sidekiq metrics to AWS CloudWatch in regular intervals
+test_files:
+- spec/cloudwatch_spec.rb
+- spec/metric_spec.rb
+- spec/sidekiq_spec.rb
+- spec/spec_helper.rb