recognizer 0.1.6 → 0.2.0

data/README.org

@@ -3,6 +3,8 @@
 
   TCP plain text & AMQP support.
 
+  [[https://secure.travis-ci.org/portertech/recognizer.png]]
+
   [[https://github.com/portertech/recognizer/raw/master/recognizer.gif]]
 * Install
   Executable Java JAR (recommended)

data/lib/recognizer.rb

@@ -1,20 +1,37 @@
 require "recognizer/cli"
 require "recognizer/config"
 require "recognizer/librato"
-require "recognizer/tcp"
-require "recognizer/amqp"
+require "recognizer/inputs/tcp"
+require "recognizer/inputs/amqp"
 
 module Recognizer
   def self.run
-    cli = Recognizer::CLI.new
-    cli_options = cli.read
-    config = Recognizer::Config.new(cli_options)
-    config_options = config.read
-    carbon_queue = Queue.new
-    logger = Logger.new(STDOUT)
-    Recognizer::Librato.new(carbon_queue, logger, config_options)
-    Recognizer::TCP.new(carbon_queue, logger, config_options)
-    Recognizer::AMQP.new(carbon_queue, logger, config_options)
+    logger      = Logger.new(STDOUT)
+    input_queue = Queue.new
+    cli         = Recognizer::CLI.new
+    config      = Recognizer::Config.new(cli.read)
+
+    librato = Recognizer::Librato.new(
+      :logger      => logger,
+      :options     => config.read,
+      :input_queue => input_queue
+    )
+    librato.run
+
+    tcp = Recognizer::Input::TCP.new(
+      :logger      => logger,
+      :options     => config.read,
+      :input_queue => input_queue
+    )
+    tcp.run
+
+    amqp = Recognizer::Input::AMQP.new(
+      :logger      => logger,
+      :options     => config.read,
+      :input_queue => input_queue
+    )
+    amqp.run
+
     loop do
       sleep 30
     end

data/lib/recognizer/cli.rb

@@ -1,4 +1,3 @@
-require "rubygems"
 require "mixlib/cli"
 
 module Recognizer

data/lib/recognizer/config.rb

@@ -1,8 +1,5 @@
-require "rubygems"
 require "json"
 
-require File.join(File.dirname(__FILE__), "patches", "hash")
-
 module Recognizer
   class Config
     def initialize(options={})
@@ -12,17 +9,24 @@ module Recognizer
       if File.readable?(options[:config_file])
         config_file_contents = File.open(options[:config_file], "r").read
         begin
-          @config = JSON.parse(config_file_contents)
+          @config = JSON.parse(config_file_contents, :symbolize_names => true)
         rescue JSON::ParserError => error
           raise "Config file must be valid JSON: #{error}"
         end
       else
         raise "Config file does not exist or is not readable: #{options[:config_file]}"
       end
+      validate
+    end
+
+    def validate
+      unless @config[:librato][:email] && @config[:librato][:api_key]
+        raise "You must provide a Librato Metrics account email and API key"
+      end
     end
 
     def read
-      @config.symbolize_keys
+      @config
     end
   end
 end

data/lib/recognizer/inputs/amqp.rb

@@ -0,0 +1,70 @@
+require "recognizer/patches/openssl"
+require "thread"
+require "bunny"
+
+module Recognizer
+  module Input
+    class AMQP
+      def initialize(options={})
+        @logger      = options[:logger]
+        @options     = options[:options]
+        @input_queue = options[:input_queue]
+
+        Thread.abort_on_exception = true
+      end
+
+      def run
+        if @options.has_key?(:amqp)
+          set_default_options
+          setup_consumer
+        else
+          @logger.warn("AMQP -- Not configured")
+        end
+      end
+
+      private
+
+      def set_default_options
+        @options[:amqp][:exchange]               ||= Hash.new
+        @options[:amqp][:exchange][:name]        ||= "graphite"
+        @options[:amqp][:exchange][:durable]     ||= false
+        @options[:amqp][:exchange][:routing_key] ||= "#"
+        @options[:amqp][:exchange][:type]        ||= (@options[:amqp][:exchange][:type] || "topic").to_sym
+      end
+
+      def setup_consumer
+        amqp = Bunny.new(@options[:amqp].reject { |key, value| key == :exchange })
+        amqp.start
+
+        exchange = amqp.exchange(@options[:amqp][:exchange][:name], {
+          :type    => @options[:amqp][:exchange][:type],
+          :durable => @options[:amqp][:exchange][:durable]
+        })
+
+        queue = amqp.queue("recognizer")
+        queue.bind(exchange, {
+          :key => @options[:amqp][:exchange][:routing_key]
+        })
+
+        Thread.new do
+          @logger.info("AMQP -- Awaiting metrics with impatience ...")
+          queue.subscribe do |message|
+            msg_routing_key = message[:routing_key] || message[:delivery_details][:routing_key]
+            lines           = message[:payload].split("\n")
+            lines.each do |line|
+              line = line.strip
+              case line.split("\s").count
+              when 3
+                @input_queue.push(line)
+              when 2
+                @input_queue.push("#{msg_routing_key} #{line}")
+              else
+                @logger.warn("AMQP -- Received malformed metric :: #{msg_routing_key} :: #{line}")
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/recognizer/inputs/tcp.rb

@@ -0,0 +1,61 @@
+require "thread"
+require "socket"
+
+module Recognizer
+  module Input
+    class TCP
+      def initialize(options={})
+        @logger      = options[:logger]
+        @options     = options[:options]
+        @input_queue = options[:input_queue]
+
+        @options[:tcp] ||= Hash.new
+        @tcp_connections = Queue.new
+
+        Thread.abort_on_exception = true
+      end
+
+      def run
+        setup_server
+        setup_thread_pool
+      end
+
+      private
+
+      def setup_server
+        port       = @options[:tcp][:port] || 2003
+        tcp_server = TCPServer.new("0.0.0.0", port)
+        Thread.new do
+          @logger.info("TCP -- Awaiting metrics with impatience ...")
+          loop do
+            @tcp_connections.push(tcp_server.accept)
+          end
+        end
+      end
+
+      def create_server_thread
+        Thread.new do
+          loop do
+            if connection = @tcp_connections.shift
+              while line = connection.gets
+                line = line.strip
+                if line.split("\s").count == 3
+                  @input_queue.push(line)
+                else
+                  @logger.warn("TCP -- Received malformed metric :: #{line}")
+                end
+              end
+            end
+          end
+        end
+      end
+
+      def setup_thread_pool
+        threads = @options[:tcp][:threads] || 20
+        threads.times do
+          create_server_thread
+        end
+      end
+    end
+  end
+end

data/lib/recognizer/librato.rb

@@ -1,96 +1,124 @@
-require "rubygems"
 require "recognizer/version"
+require "recognizer/patches/float"
+require "recognizer/patches/openssl"
 require "thread"
 require "librato/metrics"
 
-require File.join(File.dirname(__FILE__), "patches", "float")
-
-if RUBY_PLATFORM == "java"
-  require File.join(File.dirname(__FILE__), "patches", "openssl")
-end
-
 module Recognizer
   class Librato
-    def initialize(carbon_queue, logger, options)
-      unless carbon_queue && options.is_a?(Hash)
-        raise "You must provide a thread queue and options"
-      end
-      unless options[:librato][:email] && options[:librato][:api_key]
-        raise "You must provide a Librato Metrics account email and API key"
-      end
+    def initialize(options={})
+      @logger      = options[:logger]
+      @options     = options[:options]
+      @input_queue = options[:input_queue]
 
-      ::Librato::Metrics.authenticate(options[:librato][:email], options[:librato][:api_key])
+      ::Librato::Metrics.authenticate(@options[:librato][:email], @options[:librato][:api_key])
       ::Librato::Metrics.agent_identifier("recognizer", Recognizer::VERSION, "portertech")
-      librato = ::Librato::Metrics::Queue.new
+      @librato_queue = ::Librato::Metrics::Queue.new
+      @librato_mutex = Mutex.new
 
-      mutex = Mutex.new
       Thread.abort_on_exception = true
+    end
 
-      Thread.new do
-        loop do
-          sleep(options[:librato][:flush_interval] || 10)
-          unless librato.empty?
-            logger.info("Attempting to flush metrics to Librato")
-            mutex.synchronize do
-              begin
-                librato.submit
-                logger.info("Successfully flushed metrics to Librato")
-              rescue => error
-                logger.error("Encountered an error when flushing metrics to Librato :: #{error}")
-              end
-            end
-          end
-        end
+    def run
+      setup_publisher
+      setup_consumer
+    end
+
+    def invalid_metric(metric, message)
+      @logger.warn("Invalid metric :: #{metric.inspect} :: #{message}")
+      false
+    end
+
+    def valid_carbon_metric?(carbon_formatted)
+      parts = carbon_formatted.split("\s")
+      if parts[0] !~ /^[A-Za-z0-9\._-]*$/
+        invalid_metric(carbon_formatted, "Metric name must only consist of alpha-numeric characters, periods, underscores, and dashes")
+      elsif parts[1] !~ /^[0-9]*\.?[0-9]*$/
+        invalid_metric(carbon_formatted, "Metric value must be an integer or float")
+      elsif parts[2] !~ /^[0-9]{10}$/
+        invalid_metric(carbon_formatted, "Metric timestamp must be epoch, 10 digits")
+      else
+        true
       end
+    end
 
-      get_source = case options[:librato][:metric_source]
+    def metric_source(path)
+      @metric_source ||= case @options[:librato][:metric_source]
       when String
-        if options[:librato][:metric_source].match("^/.*/$")
-          @source_pattern = Regexp.new(options[:librato][:metric_source].delete("/"))
+        if @options[:librato][:metric_source].match("^/.*/$")
+          @source_pattern = Regexp.new(@options[:librato][:metric_source].delete("/"))
           Proc.new { |path| (matched = path.grep(@source_pattern).first) ? matched : "recognizer" }
         else
-          Proc.new { options[:librato][:metric_source] }
+          Proc.new { @options[:librato][:metric_source] }
         end
       when Integer
-        Proc.new { |path| path.slice(options[:librato][:metric_source]) }
+        Proc.new { |path| path.slice(@options[:librato][:metric_source]) || "recognizer" }
       else
         Proc.new { "recognizer" }
       end
+      @metric_source.call(path)
+    end
 
-      Thread.new do
-        loop do
-          graphite_formated = carbon_queue.pop
-          begin
-            parts = graphite_formated.split("\s")
+    def create_metric(carbon_formatted)
+      if valid_carbon_metric?(carbon_formatted)
+        parts     = carbon_formatted.split("\s")
 
-            unless parts.first =~ /^[A-Za-z0-9\._-]*$/
-              raise "metric name must only consist of alpha-numeric characters, periods, underscores, and dashes"
-            end
-            unless parts.last =~ /^[0-9]{10}$/
-              raise "metric timestamp must be epoch, 10 digits"
-            end
+        path      = parts.shift.split(".")
+        value     = Float(parts.shift).pretty
+        timestamp = Float(parts.shift).pretty
+        source    = metric_source(path)
 
-            path      = parts.shift.split(".")
-            value     = Float(parts.shift).pretty
-            timestamp = Float(parts.shift).pretty
-            source    = get_source.call(path)
+        path.delete(source)
 
-            path.delete(source)
+        name = path.join(".")
 
-            name = path.join(".")
+        metric = {
+          name => {
+            :value => value,
+            :measure_time => timestamp,
+            :source => source
+          }
+        }
 
-            unless name.size <= 63
-              raise "metric name must be 63 or fewer characters"
-            end
+        unless name.size <= 63
+          invalid_metric(metric, "Metric name must be 63 or fewer characters")
+        else
+          metric
+        end
+      else
+        false
+      end
+    end
 
-            metric = {name => {:value => value, :measure_time => timestamp, :source => source}}
+    private
+
+    def setup_publisher
+      Thread.new do
+        loop do
+          sleep(@options[:librato][:flush_interval] || 10)
+          unless @librato_queue.empty?
+            @logger.info("Attempting to flush metrics to Librato")
+            @librato_mutex.synchronize do
+              begin
+                @librato_queue.submit
+                @logger.info("Successfully flushed metrics to Librato")
+              rescue => error
+                @logger.error("Encountered an error when flushing metrics to Librato :: #{error}")
+              end
+            end
+          end
+        end
+      end
+    end
 
-            mutex.synchronize do
-              logger.info("Adding metric to queue :: #{metric.inspect}")
-              librato.add(metric)
+    def setup_consumer
+      Thread.new do
+        loop do
+          if metric = create_metric(@input_queue.shift)
+            @logger.info("Adding metric to Librato queue :: #{metric.inspect}")
+            @librato_mutex.synchronize do
+              @librato_queue.add(metric)
             end
-          rescue => error
-            logger.info("Invalid metric :: #{graphite_formated} :: #{error}")
           end
         end
       end

data/lib/recognizer/patches/openssl.rb

@@ -1,8 +1,10 @@
-require "openssl"
+if RUBY_PLATFORM == "java"
+  require "openssl"
 
-module OpenSSL
-  module SSL
-    remove_const(:VERIFY_PEER)
-    const_set(:VERIFY_PEER, VERIFY_NONE)
+  module OpenSSL
+    module SSL
+      remove_const(:VERIFY_PEER)
+      const_set(:VERIFY_PEER, VERIFY_NONE)
+    end
   end
 end

data/lib/recognizer/version.rb

@@ -1,3 +1,5 @@
 module Recognizer
-  VERSION = "0.1.6"
+  unless defined?(Recognizer::VERSION)
+    VERSION = "0.2.0"
+  end
 end

data/recognizer.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_dependency("json")
-  s.add_dependency("mixlib-cli", ">= 1.1.0")
-  s.add_dependency("bunny", "0.7.9")
-  s.add_dependency("librato-metrics", "0.6.0.pre3")
+  s.add_dependency("mixlib-cli", "1.2.2")
+  s.add_dependency("bunny", "0.8.0")
+  s.add_dependency("librato-metrics", "0.7.0")
 end

metadata

@@ -5,9 +5,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
-  - 6
-  version: 0.1.6
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Sean Porter
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-04-26 00:00:00 -07:00
+date: 2012-08-03 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -38,14 +38,14 @@ dependencies:
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - "="
       - !ruby/object:Gem::Version 
-        hash: 19
+        hash: 27
         segments: 
         - 1
-        - 1
-        - 0
-        version: 1.1.0
+        - 2
+        - 2
+        version: 1.2.2
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -56,12 +56,12 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        hash: 17
+        hash: 63
         segments: 
         - 0
-        - 7
-        - 9
-        version: 0.7.9
+        - 8
+        - 0
+        version: 0.8.0
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
@@ -72,13 +72,12 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        hash: -1876988210
+        hash: 3
         segments: 
         - 0
-        - 6
+        - 7
         - 0
-        - pre3
-        version: 0.6.0.pre3
+        version: 0.7.0
   type: :runtime
   version_requirements: *id004
 description: A drop-in replacement for Graphite Carbon (TCP & AMQP), sending metrics to Librato Metrics.
@@ -92,14 +91,13 @@ extra_rdoc_files: []
 
 files: 
 - bin/recognizer
-- lib/recognizer/amqp.rb
 - lib/recognizer/cli.rb
 - lib/recognizer/config.rb
+- lib/recognizer/inputs/amqp.rb
+- lib/recognizer/inputs/tcp.rb
 - lib/recognizer/librato.rb
 - lib/recognizer/patches/float.rb
-- lib/recognizer/patches/hash.rb
 - lib/recognizer/patches/openssl.rb
-- lib/recognizer/tcp.rb
 - lib/recognizer/version.rb
 - lib/recognizer.rb
 - recognizer.gemspec

data/lib/recognizer/amqp.rb

@@ -1,56 +0,0 @@
-require "rubygems"
-require "thread"
-require "bunny"
-
-if RUBY_PLATFORM == "java"
-  require File.join(File.dirname(__FILE__), "patches", "openssl")
-end
-
-module Recognizer
-  class AMQP
-    def initialize(carbon_queue, logger, options)
-      unless carbon_queue && options.is_a?(Hash)
-        raise "You must provide a thread queue and options"
-      end
-
-      if options.has_key?(:amqp)
-        options[:amqp][:exchange] ||= Hash.new
-
-        exchange_name = options[:amqp][:exchange][:name]        || "graphite"
-        durable       = options[:amqp][:exchange][:durable]     || false
-        routing_key   = options[:amqp][:exchange][:routing_key] || "#"
-        exchange_type = options[:amqp][:exchange][:type]        || :topic
-
-        amqp = Bunny.new(options[:amqp].reject { |key, value| key == :exchange })
-        amqp.start
-
-        exchange = amqp.exchange(exchange_name, :type => exchange_type.to_sym, :durable => durable)
-        queue    = amqp.queue("recognizer")
-        queue.bind(exchange, :key => routing_key)
-
-        Thread.abort_on_exception = true
-
-        Thread.new do
-          logger.info("AMQP -- Awaiting metrics with impatience ...")
-          queue.subscribe do |message|
-            payload         = message[:payload]
-            msg_routing_key = message[:routing_key] || message[:delivery_details][:routing_key]
-
-            lines = payload.split("\n")
-            lines.each do |line|
-              line = line.strip
-              case line.split("\s").count
-              when 3
-                carbon_queue.push(line)
-              when 2
-                carbon_queue.push("#{msg_routing_key} #{line}")
-              end
-            end
-          end
-        end
-      else
-        logger.warn("AMQP -- Not configured")
-      end
-    end
-  end
-end

data/lib/recognizer/patches/hash.rb

@@ -1,10 +0,0 @@
-class Hash
-  def symbolize_keys
-    inject(Hash.new) do |result, (key, value)|
-      new_key = key.is_a?(String) ? key.to_sym : key
-      new_value = value.is_a?(Hash) ? value.symbolize_keys : value
-      result[new_key] = new_value
-      result
-    end
-  end
-end

data/lib/recognizer/tcp.rb

@@ -1,46 +0,0 @@
-require "rubygems"
-require "timeout"
-require "thread"
-require "socket"
-
-module Recognizer
-  class TCP
-    def initialize(carbon_queue, logger, options)
-      unless carbon_queue && options.is_a?(Hash)
-        raise "You must provide a thread queue and options"
-      end
-
-      options[:tcp] ||= Hash.new
-
-      threads = options[:tcp][:threads] || 20
-      port    = options[:tcp][:port]    || 2003
-
-      tcp_server      = TCPServer.new("0.0.0.0", port)
-      tcp_connections = Queue.new
-
-      Thread.abort_on_exception = true
-
-      threads.times do
-        Thread.new do
-          loop do
-            if connection = tcp_connections.shift
-              while line = connection.gets
-                line = line.strip
-                if line.split("\s").count == 3
-                  carbon_queue.push(line)
-                end
-              end
-            end
-          end
-        end
-      end
-
-      Thread.new do
-        logger.info("TCP -- Awaiting metrics with impatience ...")
-        loop do
-          tcp_connections.push(tcp_server.accept)
-        end
-      end
-    end
-  end
-end