recognizer 0.0.10 → 0.1.0

data/README.org

@@ -1,28 +1,40 @@
 * Welcome to Recognizer
-  A Graphite Carbon AMQP impostor, sending metrics to [[https://metrics.librato.com/][Librato Metrics]].
+  A Graphite Carbon impostor, sending metrics to [[https://metrics.librato.com/][Librato Metrics]].
+
+  TCP plain text & AMQP support.
 
   [[https://github.com/portertech/recognizer/raw/master/recognizer.gif]]
 * Install
+  Executable Java JAR
+  : wget https://github.com/downloads/portertech/recognizer/recognizer.jar
+  RubyGems
   : gem install recognizer
 * Configure
   Example: =config.json=
   : {
   :   "librato": {
   :     "email": "email@example.com",
-  :     "api_key": "706325cf16d84d098127e143221dd180706325cf16d84d098127e143221dd180",
-  :     "flush_interval": 5
+  :     "api_key": "706325cf16d84d098127e143221dd180706325cf16d84d098127e143221dd180"
   :   },
   :   "amqp": {
-  :     "host": "hostname",
-  :     "user": "username",
-  :     "pass": "password"
+  :     "host": "localhost"
+  :   },
+  :   "tcp": {
+  :     "port": 2003
   :   }
   : }
 * Usage
+  Executable Java JAR
+  : java -jar recognizer.jar -h
+  RubyGems
+  : recognizer -h
+
   : Usage: recognizer (options)
   :  -c, --config CONFIG              The config file path
   :  -h, --help                       Show this message
 * More
+  By default, Recognizer uses the source: =recognizer=
+
   Example metric path: =production.i-424242.cpu.user=
 
   Extract the metric source from the metric path using a regular expression
@@ -37,6 +49,16 @@
   : {
   :   "librato": {
   :     "metric_source": "example"
-  By default, Recognizer uses the source: =recognizer=
+
+  By default, Recognizer uses =20= threads for the TCP server
+
+  Specify the number of TCP server threads
+  : {
+  :   "tcp": {
+  :     "threads": 30
+
+  By default, Recognizer uses an AMQP queue named: =recognizer=
+
+  By default, Recognizer binds its AMQP queue to the exchange: =graphite=
 * License
   Recognizer is released under the [[https://github.com/portertech/recognizer/raw/master/MIT-LICENSE.txt][MIT license]].

data/lib/recognizer/amqp.rb

@@ -4,46 +4,48 @@ require "bunny"
 
 module Recognizer
   class AMQP
-    def initialize(thread_queue, options)
-      unless thread_queue && options.is_a?(Hash)
+    def initialize(carbon_queue, logger, options)
+      unless carbon_queue && options.is_a?(Hash)
         raise "You must provide a thread queue and options"
       end
 
-      options[:amqp]            ||= Hash.new
-      options[:amqp][:exchange] ||= Hash.new
+      if options.has_key?(:amqp)
+        options[:amqp][:exchange] ||= Hash.new
 
-      exchange_name = options[:amqp][:exchange][:name]        || "graphite"
-      durable       = options[:amqp][:exchange][:durable]     || false
-      routing_key   = options[:amqp][:exchange][:routing_key] || "#"
-      exchange_type = options[:amqp][:exchange][:type]        || :topic
+        exchange_name = options[:amqp][:exchange][:name]        || "graphite"
+        durable       = options[:amqp][:exchange][:durable]     || false
+        routing_key   = options[:amqp][:exchange][:routing_key] || "#"
+        exchange_type = options[:amqp][:exchange][:type]        || :topic
 
-      amqp = Bunny.new(options[:amqp].reject { |key, value| key == :exchange })
-      amqp.start
+        amqp = Bunny.new(options[:amqp].reject { |key, value| key == :exchange })
+        amqp.start
 
-      exchange = amqp.exchange(exchange_name, :type => exchange_type.to_sym, :durable => durable)
-      queue = amqp.queue("recognizer")
-      queue.bind(exchange, :key => routing_key)
+        exchange = amqp.exchange(exchange_name, :type => exchange_type.to_sym, :durable => durable)
+        queue = amqp.queue("recognizer")
+        queue.bind(exchange, :key => routing_key)
 
-      Thread.abort_on_exception = true
+        Thread.abort_on_exception = true
 
-      consumer = Thread.new do
-        puts "Awaiting the metrics with impatience ..."
-        queue.subscribe do |message|
-          payload         = message[:payload]
-          msg_routing_key = message[:routing_key] || message[:delivery_details][:routing_key]
-          lines = payload.split("\n")
-          lines.each do |line|
-            line = line.strip
-            case line.split("\s").count
-            when 3
-              thread_queue.push(line)
-            when 2
-              thread_queue.push("#{msg_routing_key} #{line}")
+        Thread.new do
+          logger.info("AMQP -- Awaiting metrics with impatience ...")
+          queue.subscribe do |message|
+            payload         = message[:payload]
+            msg_routing_key = message[:routing_key] || message[:delivery_details][:routing_key]
+            lines = payload.split("\n")
+            lines.each do |line|
+              line = line.strip
+              case line.split("\s").count
+              when 3
+                carbon_queue.push(line)
+              when 2
+                carbon_queue.push("#{msg_routing_key} #{line}")
+              end
             end
           end
         end
+      else
+        logger.warn("AMQP -- Not configured")
       end
-      consumer.join
     end
   end
 end

data/lib/recognizer/librato.rb

@@ -7,8 +7,8 @@ require File.join(File.dirname(__FILE__), 'patches', 'float')
 
 module Recognizer
   class Librato
-    def initialize(thread_queue, options)
-      unless thread_queue && options.is_a?(Hash)
+    def initialize(carbon_queue, logger, options)
+      unless carbon_queue && options.is_a?(Hash)
         raise "You must provide a thread queue and options"
       end
       unless options[:librato][:email] && options[:librato][:api_key]
@@ -26,11 +26,11 @@ module Recognizer
         loop do
           sleep(options[:librato][:flush_interval] || 10)
           unless librato.empty?
-            puts "Attempting to flush metrics to Librato"
+            logger.info("Attempting to flush metrics to Librato")
             mutex.synchronize do
               librato.submit
             end
-            puts "Successfully flushed metrics to Librato"
+            logger.info("Successfully flushed metrics to Librato")
           end
         end
       end
@@ -51,7 +51,7 @@ module Recognizer
 
       Thread.new do
         loop do
-          graphite_formated = thread_queue.pop
+          graphite_formated = carbon_queue.pop
           begin
             path, value, timestamp = graphite_formated.split(" ").inject([]) do |result, part|
               result << (result.empty? ? part.split(".") : Float(part).pretty)
@@ -61,11 +61,11 @@ module Recognizer
             path.delete(source)
             metric = {path.join(".") => {:value => value, :measure_time => timestamp, :source => source}}
             mutex.synchronize do
-              puts "Adding metric to queue: #{metric.inspect}"
+              logger.info("Adding metric to queue: #{metric.inspect}")
               librato.add(metric)
             end
           rescue ArgumentError
-            puts "Invalid metric: #{graphite_formated}"
+            logger.info("Invalid metric: #{graphite_formated}")
           end
         end
       end

data/lib/recognizer/tcp.rb

@@ -0,0 +1,53 @@
+require "rubygems"
+require "timeout"
+require "thread"
+require "socket"
+
+module Recognizer
+  class TCP
+    def initialize(carbon_queue, logger, options)
+      unless carbon_queue && options.is_a?(Hash)
+        raise "You must provide a thread queue and options"
+      end
+
+      options[:tcp] ||= Hash.new
+
+      threads = options[:tcp][:threads] || 20
+      port    = options[:tcp][:port]    || 2003
+
+      tcp_server = TCPServer.new("0.0.0.0", port)
+      tcp_connections = Queue.new
+
+      Thread.abort_on_exception = true
+
+      threads.times do
+        Thread.new do
+          loop do
+            if connection = tcp_connections.shift
+              begin
+                lines = timeout(12) do
+                  connection.gets.split("\n")
+                end
+                lines.each do |line|
+                  line = line.strip
+                  if line.split("\s").count == 3
+                    carbon_queue.push(line)
+                  end
+                end
+              rescue Timeout::Error
+                connection.close
+              end
+            end
+          end
+        end
+      end
+
+      Thread.new do
+        logger.info("TCP -- Awaiting metrics with impatience ...")
+        loop do
+          tcp_connections.push(tcp_server.accept)
+        end
+      end
+    end
+  end
+end

data/lib/recognizer/version.rb

@@ -1,3 +1,3 @@
 module Recognizer
-  VERSION = "0.0.10"
+  VERSION = "0.1.0"
 end

data/lib/recognizer.rb

@@ -1,6 +1,7 @@
 require "recognizer/cli"
 require "recognizer/config"
 require "recognizer/librato"
+require "recognizer/tcp"
 require "recognizer/amqp"
 
 module Recognizer
@@ -9,8 +10,13 @@ module Recognizer
     cli_options = cli.read
     config = Recognizer::Config.new(cli_options)
     config_options = config.read
-    thread_queue = Queue.new
-    Recognizer::Librato.new(thread_queue, config_options)
-    Recognizer::AMQP.new(thread_queue, config_options)
+    carbon_queue = Queue.new
+    logger = Logger.new(STDOUT)
+    Recognizer::Librato.new(carbon_queue, logger, config_options)
+    Recognizer::TCP.new(carbon_queue, logger, config_options)
+    Recognizer::AMQP.new(carbon_queue, logger, config_options)
+    loop do
+      sleep 30
+    end
   end
 end

data/recognizer.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |s|
   s.authors     = ["Sean Porter"]
   s.email       = ["portertech@gmail.com"]
   s.homepage    = "https://github.com/portertech/recognizer"
-  s.summary     = "A Graphite Carbon AMQP impostor, sending metrics to Librato Metrics."
-  s.description = "A Graphite Carbon AMQP impostor, sending metrics to Librato Metrics."
+  s.summary     = "A Graphite Carbon impostor, sending metrics to Librato Metrics."
+  s.description = "A drop-in replacement for Graphite Carbon (TCP & AMQP), sending metrics to Librato Metrics."
 
   s.rubyforge_project = "recognizer"
 
@@ -17,8 +17,8 @@ Gem::Specification.new do |s|
   s.executables   = Dir.glob("bin/**/*").map { |file| File.basename(file) }
   s.require_paths = ["lib"]
 
-  s.add_dependency("librato-metrics", "0.4.0")
-  s.add_dependency("bunny", "0.7.9")
-  s.add_dependency("mixlib-cli", ">= 1.1.0")
   s.add_dependency("json")
+  s.add_dependency("mixlib-cli", ">= 1.1.0")
+  s.add_dependency("bunny", "0.7.9")
+  s.add_dependency("librato-metrics", "0.4.2")
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: recognizer
 version: !ruby/object:Gem::Version 
-  hash: 11
+  hash: 27
   prerelease: false
   segments: 
   - 0
+  - 1
   - 0
-  - 10
-  version: 0.0.10
+  version: 0.1.0
 platform: ruby
 authors: 
 - Sean Porter
@@ -15,72 +15,72 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-06 00:00:00 -08:00
+date: 2012-03-10 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: librato-metrics
+  name: json
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - "="
+    - - ">="
       - !ruby/object:Gem::Version 
-        hash: 15
+        hash: 3
         segments: 
         - 0
-        - 4
-        - 0
-        version: 0.4.0
+        version: "0"
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: bunny
+  name: mixlib-cli
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - "="
+    - - ">="
       - !ruby/object:Gem::Version 
-        hash: 17
+        hash: 19
         segments: 
+        - 1
+        - 1
         - 0
-        - 7
-        - 9
-        version: 0.7.9
+        version: 1.1.0
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: mixlib-cli
+  name: bunny
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - "="
       - !ruby/object:Gem::Version 
-        hash: 19
+        hash: 17
         segments: 
-        - 1
-        - 1
         - 0
-        version: 1.1.0
+        - 7
+        - 9
+        version: 0.7.9
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: json
+  name: librato-metrics
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - "="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 11
         segments: 
         - 0
-        version: "0"
+        - 4
+        - 2
+        version: 0.4.2
   type: :runtime
   version_requirements: *id004
-description: A Graphite Carbon AMQP impostor, sending metrics to Librato Metrics.
+description: A drop-in replacement for Graphite Carbon (TCP & AMQP), sending metrics to Librato Metrics.
 email: 
 - portertech@gmail.com
 executables: 
@@ -97,6 +97,7 @@ files:
 - lib/recognizer/librato.rb
 - lib/recognizer/patches/float.rb
 - lib/recognizer/patches/hash.rb
+- lib/recognizer/tcp.rb
 - lib/recognizer/version.rb
 - lib/recognizer.rb
 - recognizer.gemspec
@@ -135,6 +136,6 @@ rubyforge_project: recognizer
 rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
-summary: A Graphite Carbon AMQP impostor, sending metrics to Librato Metrics.
+summary: A Graphite Carbon impostor, sending metrics to Librato Metrics.
 test_files: []