nightwing 0.0.5 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 255d6f7c87f16dc384c82e377d341eb612394058
-  data.tar.gz: a0dd286e8e277bddf7b83812a08b685f43dab653
+  metadata.gz: 918b61571eb184206d03a99f762982658944339c
+  data.tar.gz: 1e3741e8078e80f055354d80de4fee4a311d72e0
 SHA512:
-  metadata.gz: 20e173c3b908125fecd2dde7ff03892b1fd063e6d5113cb39e878832145f9a6f5648a277705da53b0e2ad56bf764113e727fc8b55f5399247fa7158e597540b3
-  data.tar.gz: 2c67a6b6f6c749437387feec35e220d59b58803f06f170c364df9157fc161196e49051c8ef91aa70beae8b4768d444d011c4c6c62fb76774c39744ec87cf0f19
+  metadata.gz: 06dbfca4a437ba0df72dee941eb0847e5831629b1ca98270712a7bda6d4426c1c65ae3db7359171d2e8d0908687f2494ceb5b016e0ade49c5a4d888b5eda5452
+  data.tar.gz: 1dabeed4b2bfc8f07d514520acb3f50b39a61fae193c0950b8f3b141a475e501324e124a22cd36228397d9a6c77937f3b152350369e9d9bf71998bb7194ed9cf

data/.rubocop.yml

@@ -12,6 +12,8 @@ StringLiterals:
   EnforcedStyle: double_quotes
 Style/FrozenStringLiteralComment:
   Enabled: false
+Style/IfUnlessModifier:
+  Enabled: false
 Style/SignalException:
   Enabled: false
 Style/GuardClause:

data/CHANGELOG.md

@@ -1,4 +1,24 @@
-## Unreleased
+## 0.1.0
+
+- Add new Profiler middleware
+- Add a copy of the MIT license
+- New debug mode
+- Allow users to pass in their own logger object
+
+**New Metrics**
+
+In `Nightwing::Sidekiq::Profiler` middleware:
+
+- `sidekiq.<queue>.<worker>.retried`
+
+In `Nightwing::Sidekiq::Profiler` middleware:
+
+- `sidekiq.<queue>.time`
+- `sidekiq.<queue>.memory_used`
+- `sidekiq.<queue>.gc.count`
+- `sidekiq.<queue>.<worker>.time`
+- `sidekiq.<queue>.<worker>.gc.count`
+- `sidekiq.<queue>.<worker>.memory_used`
 
 ## 0.0.5
 

data/Gemfile.lock

@@ -1,27 +1,40 @@
 PATH
   remote: .
   specs:
-    nightwing (0.0.5)
+    nightwing (0.1.0)
       activesupport (>= 4.1.0)
+      oink (= 0.10.1)
       sidekiq (>= 2.6)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.2.5.1)
+    activemodel (4.2.5)
+      activesupport (= 4.2.5)
+      builder (~> 3.1)
+    activerecord (4.2.5)
+      activemodel (= 4.2.5)
+      activesupport (= 4.2.5)
+      arel (~> 6.0)
+    activesupport (4.2.5)
       i18n (~> 0.7)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
+    arel (6.0.3)
     ast (2.2.0)
     builder (3.2.2)
     concurrent-ruby (1.0.0)
     connection_pool (2.2.0)
     diff-lcs (1.2.5)
+    hodel_3000_compliant_logger (0.1.1)
     i18n (0.7.0)
     json (1.8.3)
     minitest (5.8.4)
+    oink (0.10.1)
+      activerecord
+      hodel_3000_compliant_logger
     parser (2.3.0.2)
       ast (~> 2.2)
     powerpack (0.1.1)

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Teespring, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -14,6 +14,8 @@ gem 'nightwing'
 
 You will need to add the code below to your app. In a typical Rails app, this would go into an initializer.
 
+Please note that you must require your own librato-rack gem and supply it to Nightwing
+
 ```ruby
 # config/initializers/sidekiq.rb
 Sidekiq.configure_server do |config|
@@ -21,14 +23,21 @@ Sidekiq.configure_server do |config|
     chain.add Nightwing::Sidekiq::Stats, client: Librato
     chain.add Nightwing::Sidekiq::QueueStats, client: Librato
     chain.add Nightwing::Sidekiq::WorkerStats, client: Librato
+    chain.add Nightwing::Sidekiq::Profiler, client: Librato
   end
 end
 ```
 
 ### Available options
 
-- `client`: Librato or statsd client. Required.
-- `namespace`: the string each metric is prefixed with. Defaults to `"sidekiq"`.
+| Name      | Description                    | Required? | Default           |
+|-----------|--------------------------------|-----------|-------------------|
+| client    | Librato or statsd client       | yes       | N/A               |
+| namespace | Prefix for each metric         | no        | "sidekiq"         |
+| debug     | Enable for verbose logging     | no        | false             |
+| logger    | Logger instance for debug mode | no        | Nightwing::Logger |
+
+When debug mode is turned on, Nightwing will output the metrics into a parsable format. The output destination is determined by the logger. If no logger is given then we send the debugging output to STDOUT.
 
 ## Metrics
 
@@ -45,6 +54,9 @@ Below are the metrics reported to Librato from the Sidekiq middleware
 - `sidekiq.<queue>.latency`: latency for given queue¹
 - `sidekiq.<queue>.processed`: number of times middleware called for given queue
 - `sidekiq.<queue>.failed`: number of jobs in given queue that raised an error
+- `sidekiq.<queue>.time`: how long jobs took to process (in milliseconds)
+- `sidekiq.<queue>.gc.count`: number of times the Ruby GC kicked off
+- `sidekiq.<queue>.memory_used`: the different in the process memory after jobs were processed (in bytes)
 
 ¹: the difference between now and when the oldest job was enqueued (given in seconds)
 
@@ -53,3 +65,7 @@ Below are the metrics reported to Librato from the Sidekiq middleware
 - `sidekiq.<queue>.<worker>.processed`: number of times middleware called for given worker
 - `sidekiq.<queue>.<worker>.failed`: number of jobs in given worker that raised an error
 - `sidekiq.<queue>.<worker>.finished`: number of successful worker jobs
+- `sidekiq.<queue>.<worker>.time`: how long given worker took to process (in milliseconds)
+- `sidekiq.<queue>.<worker>.retried`: number of times a given worker retried
+- `sidekiq.<queue>.<worker>.gc.count`: number of times the Ruby GC kicked off
+- `sidekiq.<queue>.<worker>.memory_used`: the different in the process memory after jobs were processed (in bytes)

data/lib/nightwing/client_logger.rb

@@ -0,0 +1,17 @@
+module Nightwing
+  class ClientLogger
+    def initialize(client:, logger:)
+      @client = client
+      @logger = logger
+    end
+
+    def method_missing(method, *args, &block)
+      unless @client.respond_to?(method)
+        super
+      end
+
+      @logger.info "method=#{method} metric=#{args[0]} value=#{args.last if args.size > 1}"
+      @client.public_send(method, *args)
+    end
+  end
+end

data/lib/nightwing/{nil_client.rb → debug_client.rb}

@@ -1,5 +1,5 @@
 module Nightwing
-  class NilClient
+  class DebugClient
     def increment(*args)
     end
 

data/lib/nightwing/logger.rb

@@ -0,0 +1,21 @@
+module Nightwing
+  class Logger
+    def debug(msg)
+      print :debug, msg
+    end
+
+    def info(msg)
+      print :info, msg
+    end
+
+    def warn(msg)
+      print :warn, msg
+    end
+
+    private
+
+    def print(level, msg)
+      puts "#{Time.now} nightwing.#{level}: #{msg}"
+    end
+  end
+end

data/lib/nightwing/sidekiq/base.rb

@@ -1,15 +1,29 @@
 require "sidekiq/api"
 require "nightwing/metric"
-require "nightwing/nil_client"
+require "nightwing/debug_client"
+require "nightwing/logger"
+require "nightwing/client_logger"
 
 module Nightwing
   module Sidekiq
     class Base
-      attr_reader :namespace, :client
+      attr_reader :namespace, :logger
 
       def initialize(options = {})
         @namespace = options.fetch(:namespace, "sidekiq")
-        @client = options.fetch(:client, Nightwing::NilClient.new)
+        @client = options.fetch(:client, Nightwing::DebugClient.new)
+        @logger = options.fetch(:logger, Nightwing::Logger.new)
+        @debug = options.fetch(:debug, false)
+      end
+
+      def client
+        @client_proxy ||= begin
+          if @debug
+            Nightwing::ClientLogger.new(client: @client, logger: @logger)
+          else
+            @client
+          end
+        end
       end
 
       private

data/lib/nightwing/sidekiq/profiler.rb

@@ -0,0 +1,38 @@
+require "nightwing/sidekiq/profiler"
+require "oink"
+
+module Nightwing
+  module Sidekiq
+    class Profiler < Base
+      def call(worker, _msg, queue)
+        queue_namespace = metrics.for(queue: queue)
+        worker_namespace = metrics.for(queue: queue, worker: worker.class)
+
+        begin
+          started_at = Time.now
+          initial_gc_count = ::GC.count
+          initial_snapshot = memory_snapshot
+
+          yield
+        ensure
+          memory_delta_in_bytes = memory_snapshot - initial_snapshot
+          total_time = ((Time.now - started_at) * 1_000).round
+          total_gc_count = ::GC.count - initial_gc_count
+
+          client.timing "#{worker_namespace}.time", total_time
+          client.measure "#{worker_namespace}.memory_used", memory_delta_in_bytes
+          client.measure "#{worker_namespace}.gc.count", total_gc_count
+
+          client.timing "#{queue_namespace}.time", total_time
+          client.measure "#{queue_namespace}.memory_used", memory_delta_in_bytes
+          client.measure "#{queue_namespace}.gc.count", total_gc_count
+        end
+      end
+
+      # returns number of bytes used by current process
+      def memory_snapshot
+        Oink::Instrumentation::MemorySnapshot.memory
+      end
+    end
+  end
+end

data/lib/nightwing/sidekiq/queue_stats.rb

@@ -17,7 +17,6 @@ module Nightwing
       # @param [String] queue
       #   The current queue.
       def call(_worker, _msg, queue)
-        exception = nil
         sidekiq_queue = ::Sidekiq::Queue.new(queue)
         queue_namespace = metrics.for(queue: queue)
 
@@ -27,13 +26,9 @@ module Nightwing
 
         begin
           yield
-        rescue => e
-          exception = e
-        end
-
-        if exception
+        rescue
           client.increment "#{queue_namespace}.failed"
-          raise exception
+          raise
         end
       end
     end

data/lib/nightwing/sidekiq/stats.rb

@@ -17,21 +17,15 @@ module Nightwing
       # @param [String] _queue
       #   The current queue.
       def call(_worker, _msg, _queue)
-        exception = nil
-
         client.measure "#{namespace}.retries", retries.size
         client.measure "#{namespace}.scheduled", scheduled.size
         client.increment "#{namespace}.processed"
 
         begin
           yield
-        rescue => e
-          exception = e
-        end
-
-        if exception
+        rescue
           client.increment "#{namespace}.failed"
-          raise exception
+          raise
         end
       end
 

data/lib/nightwing/sidekiq/worker_stats.rb

@@ -3,21 +3,22 @@ require "nightwing/sidekiq/base"
 module Nightwing
   module Sidekiq
     class WorkerStats < Base
-      def call(worker, _msg, queue)
-        exception = nil
+      def call(worker, msg, queue)
         worker_namespace = metrics.for(queue: queue, worker: worker.class)
 
         client.increment "#{worker_namespace}.processed"
 
-        begin
-          yield
-        rescue => e
-          exception = e
+        if msg["retry"]
+          if msg["retry_count"].to_i > 0
+            client.increment "#{worker_namespace}.retried"
+          end
         end
 
-        if exception
+        begin
+          yield
+        rescue
           client.increment "#{worker_namespace}.failed"
-          raise exception
+          raise
         end
 
         client.increment "#{worker_namespace}.finished"

data/lib/nightwing/version.rb

@@ -1,3 +1,3 @@
 module Nightwing
-  VERSION = "0.0.5".freeze
+  VERSION = "0.1.0".freeze
 end

data/lib/nightwing.rb

@@ -2,3 +2,4 @@ require "nightwing/version"
 require "nightwing/sidekiq/stats"
 require "nightwing/sidekiq/queue_stats"
 require "nightwing/sidekiq/worker_stats"
+require "nightwing/sidekiq/profiler"

data/nightwing.gemspec

@@ -14,6 +14,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "sidekiq", ">= 2.6"
   s.add_runtime_dependency "activesupport", ">= 4.1.0"
+  s.add_runtime_dependency "oink", "0.10.1"
 
   s.add_development_dependency "rubocop", "~> 0.36"
   s.add_development_dependency "rspec", "~> 3.4", ">= 3.4.0"

data/spec/nightwing/sidekiq/profiler_spec.rb

@@ -0,0 +1,37 @@
+require "spec_helper"
+
+describe Nightwing::Sidekiq::Profiler do
+  subject { Nightwing::Sidekiq::Profiler.new(client: Nightwing::DebugClient.new) }
+
+  let(:time_in_ms) { 55 }
+  let(:buffer) { 5 }
+
+  describe "#call" do
+    it "records time" do
+      expect(subject.client).to receive(:timing) do |*args|
+        expect(args.first).to eq("sidekiq.default.my_worker.time")
+        expect(args.last).to be_between(0, time_in_ms + buffer)
+      end
+
+      expect(subject.client).to receive(:timing) do |*args|
+        expect(args.first).to eq("sidekiq.default.time")
+        expect(args.last).to be_between(0, time_in_ms + buffer)
+      end
+
+      subject.call(MyWorker.new, nil, "default") do
+        sleep time_in_ms / 1_000 # sleep for ~55ms
+      end
+    end
+
+    it "records memory" do
+      expect(subject.client).to receive(:measure).with("sidekiq.default.my_worker.memory_used", 0).and_call_original
+      expect(subject.client).to receive(:measure).with("sidekiq.default.my_worker.gc.count", 0).and_call_original
+      expect(subject.client).to receive(:measure).with("sidekiq.default.memory_used", 0).and_call_original
+      expect(subject.client).to receive(:measure).with("sidekiq.default.gc.count", 0).and_call_original
+
+      subject.call(MyWorker.new, nil, "default") do
+        # beep
+      end
+    end
+  end
+end

data/spec/nightwing/sidekiq/queue_stats_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 
 describe Nightwing::Sidekiq::QueueStats do
-  subject { Nightwing::Sidekiq::QueueStats.new(client: Nightwing::NilClient.new) }
+  subject { Nightwing::Sidekiq::QueueStats.new(client: Nightwing::DebugClient.new) }
 
   describe "#call" do
     let(:fake_queue) { Struct.new(:size, :latency).new(0, 0) }
@@ -12,9 +12,9 @@ describe Nightwing::Sidekiq::QueueStats do
 
     context "when everything just works" do
       it "increments process count" do
-        expect(subject.client).to receive(:measure).with("sidekiq.default.size", 0)
-        expect(subject.client).to receive(:measure).with("sidekiq.default.latency", 0)
-        expect(subject.client).to receive(:increment).with("sidekiq.default.processed")
+        expect(subject.client).to receive(:measure).with("sidekiq.default.size", 0).and_call_original
+        expect(subject.client).to receive(:measure).with("sidekiq.default.latency", 0).and_call_original
+        expect(subject.client).to receive(:increment).with("sidekiq.default.processed").and_call_original
 
         subject.call(nil, nil, "default") do
           # beep
@@ -24,10 +24,10 @@ describe Nightwing::Sidekiq::QueueStats do
 
     context "when something fails" do
       it "increments process and failure count" do
-        expect(subject.client).to receive(:measure).with("sidekiq.default.size", 0)
-        expect(subject.client).to receive(:measure).with("sidekiq.default.latency", 0)
-        expect(subject.client).to receive(:increment).with("sidekiq.default.processed")
-        expect(subject.client).to receive(:increment).with("sidekiq.default.failed")
+        expect(subject.client).to receive(:measure).with("sidekiq.default.size", 0).and_call_original
+        expect(subject.client).to receive(:measure).with("sidekiq.default.latency", 0).and_call_original
+        expect(subject.client).to receive(:increment).with("sidekiq.default.processed").and_call_original
+        expect(subject.client).to receive(:increment).with("sidekiq.default.failed").and_call_original
 
         expect do
           subject.call(nil, nil, "default") do

data/spec/nightwing/sidekiq/stats_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 
 describe Nightwing::Sidekiq::Stats do
-  subject { Nightwing::Sidekiq::Stats.new(client: Nightwing::NilClient.new) }
+  subject { Nightwing::Sidekiq::Stats.new(client: Nightwing::DebugClient.new) }
 
   before do
     fake_set = Struct.new(:size).new(1)
@@ -12,9 +12,9 @@ describe Nightwing::Sidekiq::Stats do
   describe "#call" do
     context "when everything just works" do
       it "increments process count" do
-        expect(subject.client).to receive(:measure).with("sidekiq.retries", 1)
-        expect(subject.client).to receive(:measure).with("sidekiq.scheduled", 1)
-        expect(subject.client).to receive(:increment).with("sidekiq.processed")
+        expect(subject.client).to receive(:measure).with("sidekiq.retries", 1).and_call_original
+        expect(subject.client).to receive(:measure).with("sidekiq.scheduled", 1).and_call_original
+        expect(subject.client).to receive(:increment).with("sidekiq.processed").and_call_original
 
         subject.call(nil, nil, nil) do
           # beep
@@ -24,10 +24,10 @@ describe Nightwing::Sidekiq::Stats do
 
     context "when something fails" do
       it "increments process and failure count" do
-        expect(subject.client).to receive(:measure).with("sidekiq.retries", 1)
-        expect(subject.client).to receive(:measure).with("sidekiq.scheduled", 1)
-        expect(subject.client).to receive(:increment).with("sidekiq.processed")
-        expect(subject.client).to receive(:increment).with("sidekiq.failed")
+        expect(subject.client).to receive(:measure).with("sidekiq.retries", 1).and_call_original
+        expect(subject.client).to receive(:measure).with("sidekiq.scheduled", 1).and_call_original
+        expect(subject.client).to receive(:increment).with("sidekiq.processed").and_call_original
+        expect(subject.client).to receive(:increment).with("sidekiq.failed").and_call_original
 
         expect do
           subject.call(nil, nil, nil) do

data/spec/nightwing/sidekiq/worker_stats_spec.rb

@@ -1,17 +1,15 @@
 require "spec_helper"
 
 describe Nightwing::Sidekiq::WorkerStats do
-  MyWorker = Class.new
-
-  subject { Nightwing::Sidekiq::WorkerStats.new(client: Nightwing::NilClient.new) }
+  subject { Nightwing::Sidekiq::WorkerStats.new(client: Nightwing::DebugClient.new) }
 
   describe "#call" do
     context "when everything just works" do
       it "increments process count" do
-        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.processed")
-        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.finished")
+        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.processed").and_call_original
+        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.finished").and_call_original
 
-        subject.call(MyWorker.new, nil, "default") do
+        subject.call(MyWorker.new, {}, "default") do
           # beep
         end
       end
@@ -19,15 +17,27 @@ describe Nightwing::Sidekiq::WorkerStats do
 
     context "when something fails" do
       it "increments process and failure count" do
-        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.processed")
-        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.failed")
+        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.processed").and_call_original
+        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.failed").and_call_original
 
         expect do
-          subject.call(MyWorker.new, nil, "default") do
+          subject.call(MyWorker.new, {}, "default") do
             fail "beep"
           end
         end.to raise_error RuntimeError
       end
     end
+
+    context "when being retried" do
+      it "increments process and retried count" do
+        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.processed").and_call_original
+        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.retried").and_call_original
+        expect(subject.client).to receive(:increment).with("sidekiq.default.my_worker.finished").and_call_original
+
+        subject.call(MyWorker.new, { "retry" => 0, "retry_count" => 2 }, "default") do
+          # beep
+        end
+      end
+    end
   end
 end

data/spec/spec_helper.rb

@@ -1,2 +1,4 @@
 require "rspec"
 require "nightwing"
+
+MyWorker = Class.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nightwing
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
 platform: ruby
 authors:
 - Chris Ledet
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.1.0
+- !ruby/object:Gem::Dependency
+  name: oink
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.10.1
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -105,18 +119,23 @@ files:
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
+- LICENSE.txt
 - README.md
 - circle.yml
 - lib/nightwing.rb
+- lib/nightwing/client_logger.rb
+- lib/nightwing/debug_client.rb
+- lib/nightwing/logger.rb
 - lib/nightwing/metric.rb
-- lib/nightwing/nil_client.rb
 - lib/nightwing/sidekiq/base.rb
+- lib/nightwing/sidekiq/profiler.rb
 - lib/nightwing/sidekiq/queue_stats.rb
 - lib/nightwing/sidekiq/stats.rb
 - lib/nightwing/sidekiq/worker_stats.rb
 - lib/nightwing/version.rb
 - nightwing.gemspec
 - spec/nightwing/metric_spec.rb
+- spec/nightwing/sidekiq/profiler_spec.rb
 - spec/nightwing/sidekiq/queue_stats_spec.rb
 - spec/nightwing/sidekiq/stats_spec.rb
 - spec/nightwing/sidekiq/worker_stats_spec.rb