recognizer 0.3.0-java

data/MIT-LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Sean Porter
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.org

@@ -0,0 +1,73 @@
+* Welcome to Recognizer
+  A Graphite Carbon impostor, sending metrics to [[https://metrics.librato.com/][Librato Metrics]].
+
+  TCP plain text & AMQP support.
+
+  [[https://secure.travis-ci.org/portertech/recognizer.png]]
+
+  [[https://github.com/portertech/recognizer/raw/master/recognizer.gif]]
+* Install
+  Executable Java JAR (recommended)
+  : wget https://github.com/downloads/portertech/recognizer/recognizer.jar
+  RubyGems
+  : gem install recognizer
+* Configure
+  Example: =config.json=
+  : {
+  :   "librato": {
+  :     "email": "email@example.com",
+  :     "api_key": "706325cf16d84d098127e143221dd180706325cf16d84d098127e143221dd180"
+  :   },
+  :   "amqp": {
+  :     "host": "localhost"
+  :   },
+  :   "tcp": {
+  :     "port": 2003
+  :   }
+  : }
+* Usage
+  Executable Java JAR
+  : java -jar recognizer.jar -h
+  RubyGems
+  : recognizer -h
+
+  : Usage: recognizer (options)
+  :  -c, --config CONFIG              The config file path
+  :  -h, --help                       Show this message
+* More
+***** By default, Recognizer flushes metrics to Librato every =10= seconds
+  Set the interval to flush to Librato
+  : {
+  :   "librato": {
+  :     "flush_interval": 5
+***** By default, Recognizer uses =recognizer= as the metric source
+  Example metric path: =production.i-424242.cpu.user=
+
+  Extract the metric source from the metric path using a regular expression
+  : {
+  :   "librato": {
+  :     "metric_source": "/i-.*/"
+  Or using an index
+  : {
+  :   "librato": {
+  :     "metric_source": 1
+  Or set a static source
+  : {
+  :   "librato": {
+  :     "metric_source": "example"
+***** By default, The Recognizer TCP server uses =20= threads
+  Set the number of threads the TCP server uses
+  : {
+  :   "tcp": {
+  :     "threads": 30
+***** By default, Recognizer binds the AMQP queue =recognizer= to the topic exchange =graphite= with the routing key =#=
+  Use a custom AMQP exchange
+  : {
+  :   "amqp": {
+  :     "exchange": {
+  :       "name": "metrics",
+  :       "type": "topic",
+  :       "durable": true,
+  :       "routing_key": "#"
+* License
+  Recognizer is released under the [[https://github.com/portertech/recognizer/raw/master/MIT-LICENSE.txt][MIT license]].

data/bin/recognizer

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname(__FILE__) + '/../lib' unless $:.include?(File.dirname(__FILE__) + '/../lib/')
+
+require "rubygems"
+require "recognizer"
+
+Recognizer.run

data/lib/recognizer/cli.rb

@@ -0,0 +1,33 @@
+require "mixlib/cli"
+
+module Recognizer
+  class CLI
+    include Mixlib::CLI
+
+    option :config_file,
+      :short => "-c CONFIG",
+      :long  => "--config CONFIG",
+      :description => "The config file path"
+
+    option :verbose,
+      :short => "-v",
+      :long  => "--verbose",
+      :description => "Enable verbose logging",
+      :boolean => true,
+      :default => false
+
+    option :help,
+      :short => "-h",
+      :long => "--help",
+      :description => "Show this message",
+      :on => :tail,
+      :boolean => true,
+      :show_options => true,
+      :exit => 0
+
+    def read
+      parse_options
+      config
+    end
+  end
+end

data/lib/recognizer/config.rb

@@ -0,0 +1,32 @@
+require "json"
+
+module Recognizer
+  class Config
+    def initialize(options={})
+      unless options[:config_file]
+        raise "Missing config file path"
+      end
+      if File.readable?(options[:config_file])
+        config_file_contents = File.open(options[:config_file], "r").read
+        begin
+          @config = JSON.parse(config_file_contents, :symbolize_names => true)
+        rescue JSON::ParserError => error
+          raise "Config file must be valid JSON: #{error}"
+        end
+      else
+        raise "Config file does not exist or is not readable: #{options[:config_file]}"
+      end
+      validate
+    end
+
+    def validate
+      unless @config[:librato][:email] && @config[:librato][:api_key]
+        raise "You must provide a Librato Metrics account email and API key"
+      end
+    end
+
+    def read
+      @config
+    end
+  end
+end

data/lib/recognizer/inputs/amqp.rb

@@ -0,0 +1,69 @@
+require "hot_bunnies"
+
+module Recognizer
+  module Input
+    class AMQP
+      def initialize(options={})
+        @logger      = options[:logger]
+        @options     = options[:options]
+        @input_queue = options[:input_queue]
+
+        @options[:amqp][:exchange]               ||= Hash.new
+        @options[:amqp][:exchange][:name]        ||= "graphite"
+        @options[:amqp][:exchange][:durable]     ||= false
+        @options[:amqp][:exchange][:routing_key] ||= "#"
+        @options[:amqp][:exchange][:type]        ||= (@options[:amqp][:exchange][:type] || "topic").to_sym
+      end
+
+      def run
+        setup_consumer
+      end
+
+      private
+
+      def setup_consumer
+        connection_options = @options[:amqp].reject { |key, value| key == :exchange }
+
+        rabbitmq = HotBunnies.connect(connection_options)
+        amq      = rabbitmq.create_channel
+
+        amq.prefetch = 10
+
+        exchange = amq.exchange(@options[:amqp][:exchange][:name], {
+          :type    => @options[:amqp][:exchange][:type],
+          :durable => @options[:amqp][:exchange][:durable]
+        })
+
+        queue = amq.queue("recognizer")
+        queue.bind(exchange, {
+          :routing_key => @options[:amqp][:exchange][:routing_key]
+        })
+
+        @logger.info("AMQP -- Awaiting metrics with impatience ...")
+
+        subscription = queue.subscribe(:ack => true, :blocking => false) do |header, message|
+          msg_routing_key = header.routing_key
+          lines           = message.split("\n")
+          lines.each do |line|
+            line = line.strip
+            case line.split("\s").count
+            when 3
+              @input_queue.push(line)
+            when 2
+              @input_queue.push("#{msg_routing_key} #{line}")
+            else
+              @logger.warn("AMQP -- Received malformed metric :: #{msg_routing_key} :: #{line}")
+            end
+          end
+          header.ack
+        end
+
+        at_exit do
+          subscription.cancel
+          amq.close
+          rabbitmq.close
+        end
+      end
+    end
+  end
+end

data/lib/recognizer/inputs/tcp.rb

@@ -0,0 +1,61 @@
+require "thread"
+require "socket"
+
+module Recognizer
+  module Input
+    class TCP
+      def initialize(options={})
+        @logger      = options[:logger]
+        @options     = options[:options]
+        @input_queue = options[:input_queue]
+
+        @options[:tcp] ||= Hash.new
+        @tcp_connections = Queue.new
+
+        Thread.abort_on_exception = true
+      end
+
+      def run
+        setup_server
+        setup_thread_pool
+      end
+
+      private
+
+      def setup_server
+        port       = @options[:tcp][:port] || 2003
+        tcp_server = TCPServer.new("0.0.0.0", port)
+        Thread.new do
+          @logger.info("TCP -- Awaiting metrics with impatience ...")
+          loop do
+            @tcp_connections.push(tcp_server.accept)
+          end
+        end
+      end
+
+      def create_server_thread
+        Thread.new do
+          loop do
+            if connection = @tcp_connections.shift
+              while line = connection.gets
+                line = line.strip
+                if line.split("\s").count == 3
+                  @input_queue.push(line)
+                else
+                  @logger.warn("TCP -- Received malformed metric :: #{line}")
+                end
+              end
+            end
+          end
+        end
+      end
+
+      def setup_thread_pool
+        threads = @options[:tcp][:threads] || 20
+        threads.times do
+          create_server_thread
+        end
+      end
+    end
+  end
+end

data/lib/recognizer/librato.rb

@@ -0,0 +1,132 @@
+require "recognizer/version"
+require "thread"
+require "librato/metrics"
+
+module Recognizer
+  class Librato
+    def initialize(options={})
+      @logger      = options[:logger]
+      @options     = options[:options]
+      @input_queue = options[:input_queue]
+
+      ::Librato::Metrics.authenticate(@options[:librato][:email], @options[:librato][:api_key])
+      ::Librato::Metrics.agent_identifier("recognizer", Recognizer::VERSION, "portertech")
+
+      @librato_queue = ::Librato::Metrics::Queue.new
+      @librato_mutex = Mutex.new
+
+      Thread.abort_on_exception = true
+    end
+
+    def run
+      setup_publisher
+      setup_consumer
+    end
+
+    def invalid_metric(plain_text, message)
+      @logger.warn("Invalid metric :: #{plain_text} :: #{message}")
+      false
+    end
+
+    def valid_plain_text?(plain_text)
+      segments = plain_text.split("\s")
+      if segments[0] !~ /^[A-Za-z0-9\._-]*$/
+        message = "Metric name must only consist of alpha-numeric characters, periods, underscores, and dashes"
+        invalid_metric(plain_text, message)
+      elsif segments[1] !~ /^[0-9]*\.?[0-9]*$/
+        invalid_metric(plain_text, "Metric value must be an integer or float")
+      elsif segments[2] !~ /^[0-9]{10}$/
+        invalid_metric(plain_text, "Metric timestamp must be epoch, 10 digits")
+      else
+        true
+      end
+    end
+
+    def extract_metric_source(metric_path)
+      metric_source = @options[:librato][:metric_source]
+      fallback_source = "recognizer"
+      case metric_source
+      when String
+        if metric_source =~ /^\/.*\/$/
+          metric_path.grep(Regexp.new(metric_source.slice(1..-2))).first || fallback_source
+        else
+          metric_source
+        end
+      when Integer
+        metric_path.slice(metric_source) || fallback_source
+      else
+        fallback_source
+      end
+    end
+
+    def pretty_number(number)
+      float = Float(number)
+      float == float.to_i ? float.to_i : float
+    end
+
+    def create_librato_metric(plain_text)
+      if valid_plain_text?(plain_text)
+        segments  = plain_text.split("\s")
+
+        path      = segments.shift.split(".")
+        value     = pretty_number(segments.shift)
+        timestamp = pretty_number(segments.shift)
+        source    = extract_metric_source(path)
+
+        path.delete(source)
+
+        name = path.join(".")
+
+        if name.size <= 63
+          {
+            name => {
+              :value => value,
+              :measure_time => timestamp,
+              :source => source
+            }
+          }
+        else
+          message = "Metric name must be 63 or fewer characters after source extraction"
+          invalid_metric(plain_text, message)
+        end
+      else
+        false
+      end
+    end
+
+    private
+
+    def setup_publisher
+      Thread.new do
+        loop do
+          sleep(@options[:librato][:flush_interval] || 10)
+          unless @librato_queue.empty?
+            @logger.info("Attempting to flush metrics to Librato")
+            @librato_mutex.synchronize do
+              begin
+                metric_count = @librato_queue.size
+                @librato_queue.submit
+                @logger.info("Successfully flushed #{metric_count} metrics to Librato")
+              rescue => error
+                @logger.error("Encountered an error when flushing metrics to Librato :: #{error}")
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def setup_consumer
+      Thread.new do
+        loop do
+          if metric = create_librato_metric(@input_queue.shift)
+            @logger.debug("Adding metric to Librato queue :: #{metric.inspect}")
+            @librato_mutex.synchronize do
+              @librato_queue.add(metric)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/recognizer/version.rb

@@ -0,0 +1,5 @@
+module Recognizer
+  unless defined?(Recognizer::VERSION)
+    VERSION = "0.3.0"
+  end
+end

data/lib/recognizer.rb

@@ -0,0 +1,47 @@
+require "logger"
+require "recognizer/cli"
+require "recognizer/config"
+require "recognizer/librato"
+require "recognizer/inputs/tcp"
+require "recognizer/inputs/amqp"
+
+module Recognizer
+  def self.run
+    cli         = Recognizer::CLI.new
+    config      = Recognizer::Config.new(cli.read)
+    input_queue = Queue.new
+
+    options = cli.read.merge(config.read)
+
+    logger = Logger.new(STDOUT)
+
+    logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO
+
+    librato = Recognizer::Librato.new(
+      :logger      => logger,
+      :options     => options,
+      :input_queue => input_queue
+    )
+    librato.run
+
+    tcp = Recognizer::Input::TCP.new(
+      :logger      => logger,
+      :options     => options,
+      :input_queue => input_queue
+    )
+    tcp.run
+
+    if options.has_key?(:amqp)
+      amqp = Recognizer::Input::AMQP.new(
+        :logger      => logger,
+        :options     => options,
+        :input_queue => input_queue
+      )
+      amqp.run
+    end
+
+    loop do
+      sleep 30
+    end
+  end
+end

data/recognizer.gemspec

@@ -0,0 +1,25 @@
+$:.push File.expand_path("../lib", __FILE__)
+require "recognizer/version"
+
+Gem::Specification.new do |s|
+  s.name        = "recognizer"
+  s.version     = Recognizer::VERSION
+  s.platform    = "java"
+  s.authors     = ["Sean Porter"]
+  s.email       = ["portertech@gmail.com"]
+  s.homepage    = "https://github.com/portertech/recognizer"
+  s.summary     = "A Graphite Carbon impostor, sending metrics to Librato Metrics."
+  s.description = "A drop-in replacement for Graphite Carbon (TCP & AMQP), sending metrics to Librato Metrics."
+
+  s.rubyforge_project = "recognizer"
+
+  s.files         = Dir.glob("{bin,lib}/**/*") + %w[recognizer.gemspec README.org MIT-LICENSE.txt]
+  s.executables   = Dir.glob("bin/**/*").map { |file| File.basename(file) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency("json")
+  s.add_dependency("jruby-openssl", "0.7.7")
+  s.add_dependency("mixlib-cli", "1.2.2")
+  s.add_dependency("hot_bunnies", "1.4.0")
+  s.add_dependency("librato-metrics", "0.7.5")
+end

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: recognizer
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+  prerelease:
+platform: java
+authors:
+- Sean Porter
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2012-11-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MA==
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MA==
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jruby-openssl
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.7
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.7
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: mixlib-cli
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: hot_bunnies
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: librato-metrics
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.5
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.5
+    none: false
+  prerelease: false
+  type: :runtime
+description: A drop-in replacement for Graphite Carbon (TCP & AMQP), sending metrics to Librato Metrics.
+email:
+- portertech@gmail.com
+executables:
+- recognizer
+extensions: []
+extra_rdoc_files: []
+files:
+- !binary |-
+  YmluL3JlY29nbml6ZXI=
+- !binary |-
+  bGliL3JlY29nbml6ZXIucmI=
+- !binary |-
+  bGliL3JlY29nbml6ZXIvdmVyc2lvbi5yYg==
+- !binary |-
+  bGliL3JlY29nbml6ZXIvY2xpLnJi
+- !binary |-
+  bGliL3JlY29nbml6ZXIvY29uZmlnLnJi
+- !binary |-
+  bGliL3JlY29nbml6ZXIvbGlicmF0by5yYg==
+- !binary |-
+  bGliL3JlY29nbml6ZXIvaW5wdXRzL2FtcXAucmI=
+- !binary |-
+  bGliL3JlY29nbml6ZXIvaW5wdXRzL3RjcC5yYg==
+- recognizer.gemspec
+- README.org
+- MIT-LICENSE.txt
+homepage: https://github.com/portertech/recognizer
+licenses: []
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      segments:
+      - 0
+      version: !binary |-
+        MA==
+      hash: 2
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: !binary |-
+        MA==
+  none: false
+requirements: []
+rubyforge_project: recognizer
+rubygems_version: 1.8.24
+signing_key:
+specification_version: 3
+summary: A Graphite Carbon impostor, sending metrics to Librato Metrics.
+test_files: []