nexia_worker_roulette 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 485a631bf7929866af1715af3649ce0ef5abcd49
-  data.tar.gz: 562bdf170332fe368447e8c2dac02ec9a6fbabd8
+  metadata.gz: 53d36e15910ce2db5cd63152496ee34dfa5c9b0f
+  data.tar.gz: 28376e7730e8e6fbe960f53a9936897307bc1bc8
 SHA512:
-  metadata.gz: 24ad87c7f75d708465ae59e0844f9157feb850bf8c631da32deb47f3f6d480ae80c7a2848a539acf3d25263602ec9c6126418fb31f58a55eedd11f225f2e4a27
-  data.tar.gz: d40ea3cdb9da663ae62d5413a7445b14daf5170f4c42353f77609c569c9351fb4b7e6442074e83a8c64240ef830c4c906f817b2e1bb22186d6713dd13daf0f2a
+  metadata.gz: aef9594153c398471a13a261e673a442d085c670c3782a17c673112ec4ffbfe55cf309fbfce5e746af8a66bbdb8ed629e71ca836d1fb160f6cc9b49b1088761e
+  data.tar.gz: 79cfc9c99a374508c56b2783d180f170671723530eccea31f6de1a8bd8dc7e19160e6b87dd1708b8eb3b68e96e03f32df40d9efa7d8052ca6e5a79ee21e0ec8f

data/lib/worker_roulette/batch_size.rb

@@ -0,0 +1,18 @@
+require_relative "queue_metric_tracker"
+
+module WorkerRoulette
+  class BatchSize
+    include ::QueueMetricTracker
+
+    def track(sender, work_orders, _remaining)
+      return unless enabled?
+
+      batch_size = work_orders.length
+      return if batch_size == 0
+
+      if value = calculate_stats(:batch_size, batch_size)
+        tracker_send(message("batch_size", channel(sender), value))
+      end
+    end
+  end
+end

data/lib/worker_roulette/monkey_patches.rb

@@ -0,0 +1,9 @@
+class String
+  def underscore
+    self.gsub(/::/, '/').
+    gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+    gsub(/([a-z\d])([A-Z])/,'\1_\2').
+    tr("-", "_").
+    downcase
+  end
+end

data/lib/worker_roulette/queue_depth.rb

@@ -0,0 +1,15 @@
+require_relative "queue_metric_tracker"
+
+module WorkerRoulette
+  class QueueDepth
+    include ::QueueMetricTracker
+
+    def track(sender, work_orders, remaining)
+      return unless enabled?
+
+      if value = calculate_stats(:queue_depth, remaining)
+        tracker_send(message("queue_depth", channel(sender), value))
+      end
+    end
+  end
+end

data/lib/worker_roulette/queue_latency.rb

@@ -0,0 +1,32 @@
+module WorkerRoulette
+  module QueueLatency
+    GRANULARITY = 1_000_000
+
+    class Foreman
+      def process(work_order, _channel)
+        work_order['headers'].merge!(
+          "queued_at" => (Time.now.to_f * GRANULARITY).to_i) if work_order.is_a?(Hash) && work_order["headers"]
+        work_order
+      end
+    end
+
+    class Tradesman
+      include QueueMetricTracker
+
+      def process(work_order, channel)
+        send_latency(work_order["headers"]["queued_at"], channel)
+        work_order
+      end
+
+      def send_latency(queued_at, channel)
+        return unless queued_at
+
+        latency_ns = (Time.now.to_f * GRANULARITY).to_i - queued_at
+
+        if value = calculate_stats(:queue_latency, latency_ns / 1000.0)
+          tracker_send(message("queue_latency(ms)", channel, value))
+        end
+      end
+    end
+  end
+end

data/lib/worker_roulette/queue_metric_tracker.rb

@@ -0,0 +1,71 @@
+module QueueMetricTracker
+  def tracker_send(msg)
+    UDPSocket.new.send(msg, 0, config[:metric_host][:host_ip], config[:metric_host][:host_port])
+  end
+
+  def granularity
+    config[:granularity] || 100
+  end
+
+  def calculate_stats(stat_name, value)
+    calculator(stat_name).add(value)
+  end
+
+  def calculator(stat_name)
+    QueueMetricTracker.calculators[stat_name] ||= QueueMetricTracker::StatCalculator.new(granularity)
+  end
+
+  def channel(sender)
+    (sender.split ":").first
+  end
+
+  def config
+    QueueMetricTracker.config
+  end
+
+  def message(label, channel, value)
+    "#{label},server_name=#{config[:server_name]},channel=#{channel} value=#{value} #{(Time.now.to_f * 1_000_000_000).to_i}"
+  end
+
+  def enabled?
+    return false unless config && config[:metrics]
+
+    klass = self.class.to_s.split("::").last.underscore.to_sym
+    config[:metrics][klass] rescue false
+  end
+
+  class << self
+    attr_reader :config, :calculators
+    def configure(options)
+      @calculators = {}
+      @config = {
+        server_name: options[:server_name],
+        granularity: options[:granularity],
+        metric_host: {
+          host_ip:   ip_address(options[:metric_host]),
+          host_port: options[:metric_host_port]
+        }
+      }
+      @config.merge!({
+        metrics: options[:metrics]
+      }) if options[:metrics]
+    end
+
+    def included(tracker)
+      @trackers ||= []
+      @trackers << tracker
+    end
+
+    def track_all(options)
+      @trackers.each do |tracker_class|
+        tracker = tracker_class.new
+        tracker.track(*options) if tracker.respond_to?(:track)
+      end
+    end
+
+    def ip_address(server_name)
+      server_name == "localhost" ? "127.0.0.1" : Resolv.new.getaddress(server_name).to_s
+    end
+
+  end
+end

data/lib/worker_roulette/stat_calculator.rb

@@ -0,0 +1,24 @@
+module QueueMetricTracker
+  class StatCalculator
+    attr_accessor :count, :sum, :granularity
+
+    def initialize(granularity = 100)
+      @granularity = granularity
+      @sum = 0
+      @count = 0
+    end
+
+    def add(value)
+      @sum   += value
+      @count += 1
+
+      if @count == granularity
+        value = @sum / granularity
+        @sum = @count = 0
+        return value
+      end
+
+      return nil
+    end
+  end
+end

data/lib/worker_roulette/tradesman.rb

@@ -134,6 +134,8 @@ module WorkerRoulette
         work_orders     = results[1]
         @remaining_jobs = results[2]
         @last_sender    = sender_key.split(':').last
+
+        QueueMetricTracker.track_all(results) if work_orders.any?
         work = work_orders.map { |wo| preprocess(WorkerRoulette.load(wo), channel) }
         callback.call work if callback
         work

data/lib/worker_roulette/version.rb

@@ -1,3 +1,3 @@
 module WorkerRoulette
-  VERSION = '0.2.2'
+  VERSION = '0.2.3'
 end

data/lib/worker_roulette.rb

@@ -67,21 +67,22 @@ module WorkerRoulette
 
       @preprocessors = []
 
-      configure_latency_tracker(config.delete(:latency_tracker))
+      configure_queue_tracker(config.delete(:metric_tracker))
     end
 
-    def configure_latency_tracker(config)
+    def configure_queue_tracker(config)
       return unless config
 
-      QueueLatencyTracker.configure(
+      QueueMetricTracker.configure(
         {
           server_name: `hostname`.chomp,
-          logstash_server_ip: ip_address(config[:logstash_server_name]),
-          logstash_port: config[:logstash_port]
+          metric_host: config[:metric_host],
+          metric_host_port: config[:metric_host_port],
+          metrics: config[:metrics]
         }
       )
 
-      preprocessors << QueueLatencyTracker
+      preprocessors << QueueLatency
     end
 
     def foreman(sender, namespace = nil)
@@ -112,10 +113,6 @@ module WorkerRoulette
 
     private
 
-    def ip_address(server_name)
-      server_name == "localhost" ? "127.0.0.1" : Resolv::DNS.new.getaddress(server_name).to_s
-    end
-
     def new_redis
       if @evented
         require 'eventmachine'

data/spec/integration/evented_worker_roulette_spec.rb

@@ -12,13 +12,13 @@ module WorkerRoulette
     let(:foreman_work_order)                { Hash["payload" => "foreman"] }
     let(:work_orders_with_headers)          { default_headers.merge({ "payload" => work_orders }) }
     let(:jsonized_work_orders_with_headers) { [WorkerRoulette.dump(work_orders_with_headers)] }
-    let(:latency_tracker)                   {
+    let(:metric_tracker)                   {
         {
-          logstash_server_name: "localhost",
-          logstash_port: 7777
+          metric_host:     "localhost",
+          metric_host_port: 7777
         }
     }
-    let(:worker_roulette)                   { WorkerRoulette.start(evented: true, latency_tracker: latency_tracker) }
+    let(:worker_roulette)                   { WorkerRoulette.start(evented: true, metric_tracker: metric_tracker) }
     let(:redis)                             { Redis.new(worker_roulette.redis_config) }
 
     before do

data/spec/integration/worker_roulette_spec.rb

@@ -14,8 +14,8 @@ module WorkerRoulette
     let(:foreman_work_order)                { Hash['payload' => "foreman"] }
     let(:work_orders_with_headers)          { default_headers.merge({ 'payload' => work_orders }) }
     let(:jsonized_work_orders_with_headers) { [WorkerRoulette.dump(work_orders_with_headers)] }
-    let(:worker_roulette)                   { WorkerRoulette.start(evented: false, latency_tracker: latency_tracker) }
-    let(:latency_tracker)                   { nil }
+    let(:worker_roulette)                   { WorkerRoulette.start(evented: false, metric_tracker: metric_tracker) }
+    let(:metric_tracker)                   { nil }
 
     let(:redis) { Redis.new(worker_roulette.redis_config) }
 
@@ -153,10 +153,10 @@ module WorkerRoulette
       context "when latency tracker is enabled" do
         let(:default_headers) { Hash["headers" => { "sender" => sender, "queued_at" => (queued_at.to_f * 1_000_000).to_i }] }
         let(:queued_at)       { 1234567 }
-        let(:latency_tracker) {
+        let(:metric_tracker) {
           {
-            logstash_server_name: "localhost",
-            logstash_port: 7777
+            metric_host:     "localhost",
+            metric_host_port: 7777
           }
         }
 

data/spec/unit/{queue_latency_tracker_spec.rb → queue_latency_spec.rb}

@@ -1,25 +1,6 @@
 require "spec_helper"
 
-module QueueLatencyTracker
-  describe ".configure" do
-    let(:source_config) { { logstash_server_ip: ip, logstash_port: port, server_name: server_name } }
-    let(:ip)            { "1.2.3.4" }
-    let(:port)          { 123 }
-    let(:server_name)   { "server.example" }
-
-
-    it "stores the configuration" do
-      QueueLatencyTracker.configure(source_config)
-
-      expect(QueueLatencyTracker.config).to eq({
-        logstash: {
-          server_ip: ip,
-          port: port },
-        server_name: server_name
-      })
-    end
-  end
-
+module WorkerRoulette::QueueLatency
   describe Foreman do
     describe "#process" do
       let(:channel)        { "a_channel" }
@@ -39,7 +20,7 @@ module QueueLatencyTracker
   describe Tradesman do
     describe "#process" do
       let(:queued_at)       { 1234567 * GRANULARITY }
-      let(:expected_json)   { %({"server_name":"#{server_name}","queue_latency (ms)":#{latency * 1000},"channel":"#{channel}"}) }
+      let(:host)            { "a_metric_host" }
       let(:ip)              { "1.2.3.4" }
       let(:port)            { 123 }
       let(:latency)         { 123.432 }
@@ -47,15 +28,18 @@ module QueueLatencyTracker
       let(:channel)         { "a_channel" }
       let(:headers)         { { "queued_at" => queued_at } }
       let(:raw_work_order)  { { "headers" => headers, "payload" => "aPayload" } }
-      let(:logstash_config) { { server_ip: ip, port: port } }
-      let(:config)          { { logstash: logstash_config, server_name: server_name } }
+      let(:metric_config)   { { host_ip: ip, host_port: port } }
+      let(:config)          { { metric_host: metric_config, server_name: server_name } }
+      let(:expected_msg)    { "queue_latency(ms),server_name=server.example,channel=a_channel value=123.432 1234690432000000" }
 
-      before { allow(QueueLatencyTracker).to receive(:config).and_return(config) }
+      before { allow(QueueMetricTracker).to receive(:config).and_return(config) }
       before { allow(Time).to receive(:now).and_return(queued_at / GRANULARITY + latency) }
       before { allow_any_instance_of(UDPSocket).to receive(:send) }
+      before { allow_any_instance_of(QueueMetricTracker).to receive(:calculate_stats).and_return(latency) }
+      before { allow(QueueMetricTracker).to receive(:ipaddress).and_return(ip) }
 
       it "passes the right json to logstash_send" do
-        expect_any_instance_of(UDPSocket).to receive(:send).with(expected_json, 0, ip, port)
+        expect_any_instance_of(UDPSocket).to receive(:send).with(expected_msg, 0, ip, port)
 
         subject.process(raw_work_order, channel)
       end

data/spec/unit/queue_metric_tracker_spec.rb

@@ -0,0 +1,71 @@
+require "spec_helper"
+
+module QueueMetricTracker
+  describe QueueMetricTracker do
+    let(:host)          { "localhost" }
+    let(:port)          { 123 }
+    let(:granularity)   { 3 }
+    let(:ip)            { "1.2.3.4" }
+    let(:server_name)   { "server.example" }
+    let(:source_config) { {
+        metric_host: host,
+        metric_host_port: port,
+        server_name: server_name,
+        granularity: granularity
+      }
+    }
+
+    describe ".configure" do
+      it "stores the configuration" do
+        allow(QueueMetricTracker).to receive(:ip_address).and_return(ip)
+        QueueMetricTracker.configure(source_config)
+
+        expect(QueueMetricTracker.config).to eq({
+          metric_host: {
+            host_ip:   ip,
+            host_port: port },
+          server_name: server_name,
+          granularity: granularity
+        })
+      end
+    end
+
+    describe "#enabled?" do
+      let(:config) { {} }
+      subject(:metric) { WorkerRoulette::BatchSize }
+      subject(:metric_object) { metric.new }
+
+      before { QueueMetricTracker.configure(source_config.merge(config)) }
+
+      context "when the config is nil" do
+        it "returns false" do
+          expect(metric_object.enabled?).to be_falsey
+        end
+      end
+
+      context "when the config has no metrics defined" do
+        let(:config) { { metrics: {}} }
+
+        it "returns false" do
+          expect(metric_object.enabled?).to be_falsey
+        end
+      end
+
+      context "when the metric is false" do
+        let(:config) { { metrics: { :batch_size => false } } }
+
+        it "returns false" do
+          expect(metric_object.enabled?).to be_falsey
+        end
+      end
+
+      context "when the metric is true" do
+        let(:config) { { metrics: { :batch_size => true }} }
+
+        it "returns true" do
+          expect(metric_object.enabled?).to be_truthy
+        end
+      end
+    end
+  end
+end

data/spec/unit/readlock_spec.rb

@@ -2,13 +2,13 @@ require "spec_helper"
 
 module WorkerRoulette
   describe "Read Lock" do
-    let(:latency_tracker)          {
+    let(:metric_tracker)          {
         {
-          logstash_server_name: "localhost",
-          logstash_port: 7777
+          metric_host:     "localhost",
+          metric_host_port: 7777
         }
     }
-    let(:worker_roulette)          { WorkerRoulette.start(evented: false, latency_tracker: latency_tracker) }
+    let(:worker_roulette)          { WorkerRoulette.start(evented: false, metric_tracker: metric_tracker) }
     let(:redis)                    { Redis.new(worker_roulette.redis_config) }
     let(:sender)                   { "katie_80" }
     let(:work_orders)              { "hello" }

data/spec/unit/stat_calculator_spec.rb

@@ -0,0 +1,32 @@
+require "spec_helper"
+
+module QueueMetricTracker
+  describe StatCalculator do
+    let(:granularity)         { 10 }
+    let(:default_granularity) { 100 }
+    subject(:calculator) { described_class }
+
+    it "responds to new with a granularity" do
+      expect(calculator.new(granularity).granularity).to eq(granularity)
+    end
+
+    it "granularity defaults to a value" do
+      expect(calculator.new().granularity).to eq(default_granularity)
+    end
+
+    describe "#add" do
+      let(:granularity)         { 3 }
+      let(:value1)              { 4 }
+      let(:value2)              { 3 }
+      let(:value3)              { 8 }
+      let(:average)             { 5 }
+      subject(:calculator)      { described_class.new(granularity) }
+
+      it "calculates average of N values" do
+        expect(subject.add(value1)).to be_nil
+        expect(subject.add(value2)).to be_nil
+        expect(subject.add(value3)).to eq(average)
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nexia_worker_roulette
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Paul Saieg
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-13 00:00:00.000000000 Z
+date: 2015-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oj
@@ -211,10 +211,15 @@ files:
 - README.md
 - Rakefile
 - lib/worker_roulette.rb
+- lib/worker_roulette/batch_size.rb
 - lib/worker_roulette/foreman.rb
 - lib/worker_roulette/lua.rb
+- lib/worker_roulette/monkey_patches.rb
 - lib/worker_roulette/preprocessor.rb
-- lib/worker_roulette/queue_latency_tracker.rb
+- lib/worker_roulette/queue_depth.rb
+- lib/worker_roulette/queue_latency.rb
+- lib/worker_roulette/queue_metric_tracker.rb
+- lib/worker_roulette/stat_calculator.rb
 - lib/worker_roulette/tradesman.rb
 - lib/worker_roulette/version.rb
 - spec/benchmark/irb_demo_runner.rb
@@ -226,8 +231,10 @@ files:
 - spec/unit/evented_readlock_spec.rb
 - spec/unit/lua_spec.rb
 - spec/unit/preprocessor_spec.rb
-- spec/unit/queue_latency_tracker_spec.rb
+- spec/unit/queue_latency_spec.rb
+- spec/unit/queue_metric_tracker_spec.rb
 - spec/unit/readlock_spec.rb
+- spec/unit/stat_calculator_spec.rb
 - worker_roulette.gemspec
 homepage: https://github.com/nexiahome/worker_roulette
 licenses: []
@@ -248,7 +255,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Pub Sub Queue for Redis that ensures ordered processing
@@ -262,5 +269,7 @@ test_files:
 - spec/unit/evented_readlock_spec.rb
 - spec/unit/lua_spec.rb
 - spec/unit/preprocessor_spec.rb
-- spec/unit/queue_latency_tracker_spec.rb
+- spec/unit/queue_latency_spec.rb
+- spec/unit/queue_metric_tracker_spec.rb
 - spec/unit/readlock_spec.rb
+- spec/unit/stat_calculator_spec.rb

data/lib/worker_roulette/queue_latency_tracker.rb

@@ -1,48 +0,0 @@
-module QueueLatencyTracker
-  GRANULARITY = 1_000_000
-
-  class Foreman
-    def process(work_order, _channel)
-      work_order['headers'].merge!(
-        "queued_at" => (Time.now.to_f * GRANULARITY).to_i) if work_order.is_a?(Hash) && work_order["headers"]
-      work_order
-    end
-  end
-
-  class Tradesman
-    def process(work_order, channel)
-      send_latency(work_order["headers"]["queued_at"], channel)
-      work_order
-    end
-
-    def send_latency(queued_at, channel)
-      latency_ns = (Time.now.to_f * GRANULARITY).to_i - queued_at
-      logstash_send(latency_json(latency_ns / 1000.0, channel))
-    end
-
-    def logstash_send(json)
-      UDPSocket.new.send(json, 0, config[:logstash][:server_ip], config[:logstash][:port])
-    end
-
-    def latency_json(latency_ms, channel)
-      %({"server_name":"#{config[:server_name]}","queue_latency (ms)":#{latency_ms},"channel":"#{channel}"})
-    end
-
-    def config
-      QueueLatencyTracker.config
-    end
-  end
-
-  class << self
-    attr_reader :config
-    def configure(config)
-      @config = {
-        logstash: {
-          server_ip: config[:logstash_server_ip],
-          port: config[:logstash_port] },
-        server_name: config[:server_name]
-      }
-    end
-
-  end
-end