recognizer 0.3.0-java

Sign up to get free protection for your applications and to get access to all the features.
data/MIT-LICENSE.txt ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2011 Sean Porter
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.org ADDED
@@ -0,0 +1,73 @@
1
+ * Welcome to Recognizer
2
+ A Graphite Carbon impostor, sending metrics to [[https://metrics.librato.com/][Librato Metrics]].
3
+
4
+ TCP plain text & AMQP support.
5
+
6
+ [[https://secure.travis-ci.org/portertech/recognizer.png]]
7
+
8
+ [[https://github.com/portertech/recognizer/raw/master/recognizer.gif]]
9
+ * Install
10
+ Executable Java JAR (recommended)
11
+ : wget https://github.com/downloads/portertech/recognizer/recognizer.jar
12
+ RubyGems
13
+ : gem install recognizer
14
+ * Configure
15
+ Example: =config.json=
16
+ : {
17
+ : "librato": {
18
+ : "email": "email@example.com",
19
+ : "api_key": "706325cf16d84d098127e143221dd180706325cf16d84d098127e143221dd180"
20
+ : },
21
+ : "amqp": {
22
+ : "host": "localhost"
23
+ : },
24
+ : "tcp": {
25
+ : "port": 2003
26
+ : }
27
+ : }
28
+ * Usage
29
+ Executable Java JAR
30
+ : java -jar recognizer.jar -h
31
+ RubyGems
32
+ : recognizer -h
33
+
34
+ : Usage: recognizer (options)
35
+ : -c, --config CONFIG The config file path
36
+ : -h, --help Show this message
37
+ * More
38
+ ***** By default, Recognizer flushes metrics to Librato every =10= seconds
39
+ Set the interval to flush to Librato
40
+ : {
41
+ : "librato": {
42
+ : "flush_interval": 5
43
+ ***** By default, Recognizer uses =recognizer= as the metric source
44
+ Example metric path: =production.i-424242.cpu.user=
45
+
46
+ Extract the metric source from the metric path using a regular expression
47
+ : {
48
+ : "librato": {
49
+ : "metric_source": "/i-.*/"
50
+ Or using an index
51
+ : {
52
+ : "librato": {
53
+ : "metric_source": 1
54
+ Or set a static source
55
+ : {
56
+ : "librato": {
57
+ : "metric_source": "example"
58
+ ***** By default, The Recognizer TCP server uses =20= threads
59
+ Set the number of threads the TCP server uses
60
+ : {
61
+ : "tcp": {
62
+ : "threads": 30
63
+ ***** By default, Recognizer binds the AMQP queue =recognizer= to the topic exchange =graphite= with the routing key =#=
64
+ Use a custom AMQP exchange
65
+ : {
66
+ : "amqp": {
67
+ : "exchange": {
68
+ : "name": "metrics",
69
+ : "type": "topic",
70
+ : "durable": true,
71
+ : "routing_key": "#"
72
+ * License
73
+ Recognizer is released under the [[https://github.com/portertech/recognizer/raw/master/MIT-LICENSE.txt][MIT license]].
data/bin/recognizer ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $: << File.dirname(__FILE__) + '/../lib' unless $:.include?(File.dirname(__FILE__) + '/../lib/')
4
+
5
+ require "rubygems"
6
+ require "recognizer"
7
+
8
+ Recognizer.run
@@ -0,0 +1,33 @@
1
+ require "mixlib/cli"
2
+
3
+ module Recognizer
4
+ class CLI
5
+ include Mixlib::CLI
6
+
7
+ option :config_file,
8
+ :short => "-c CONFIG",
9
+ :long => "--config CONFIG",
10
+ :description => "The config file path"
11
+
12
+ option :verbose,
13
+ :short => "-v",
14
+ :long => "--verbose",
15
+ :description => "Enable verbose logging",
16
+ :boolean => true,
17
+ :default => false
18
+
19
+ option :help,
20
+ :short => "-h",
21
+ :long => "--help",
22
+ :description => "Show this message",
23
+ :on => :tail,
24
+ :boolean => true,
25
+ :show_options => true,
26
+ :exit => 0
27
+
28
+ def read
29
+ parse_options
30
+ config
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,32 @@
1
+ require "json"
2
+
3
+ module Recognizer
4
+ class Config
5
+ def initialize(options={})
6
+ unless options[:config_file]
7
+ raise "Missing config file path"
8
+ end
9
+ if File.readable?(options[:config_file])
10
+ config_file_contents = File.open(options[:config_file], "r").read
11
+ begin
12
+ @config = JSON.parse(config_file_contents, :symbolize_names => true)
13
+ rescue JSON::ParserError => error
14
+ raise "Config file must be valid JSON: #{error}"
15
+ end
16
+ else
17
+ raise "Config file does not exist or is not readable: #{options[:config_file]}"
18
+ end
19
+ validate
20
+ end
21
+
22
+ def validate
23
+ unless @config[:librato][:email] && @config[:librato][:api_key]
24
+ raise "You must provide a Librato Metrics account email and API key"
25
+ end
26
+ end
27
+
28
+ def read
29
+ @config
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,69 @@
1
+ require "hot_bunnies"
2
+
3
+ module Recognizer
4
+ module Input
5
+ class AMQP
6
+ def initialize(options={})
7
+ @logger = options[:logger]
8
+ @options = options[:options]
9
+ @input_queue = options[:input_queue]
10
+
11
+ @options[:amqp][:exchange] ||= Hash.new
12
+ @options[:amqp][:exchange][:name] ||= "graphite"
13
+ @options[:amqp][:exchange][:durable] ||= false
14
+ @options[:amqp][:exchange][:routing_key] ||= "#"
15
+ @options[:amqp][:exchange][:type] ||= (@options[:amqp][:exchange][:type] || "topic").to_sym
16
+ end
17
+
18
+ def run
19
+ setup_consumer
20
+ end
21
+
22
+ private
23
+
24
+ def setup_consumer
25
+ connection_options = @options[:amqp].reject { |key, value| key == :exchange }
26
+
27
+ rabbitmq = HotBunnies.connect(connection_options)
28
+ amq = rabbitmq.create_channel
29
+
30
+ amq.prefetch = 10
31
+
32
+ exchange = amq.exchange(@options[:amqp][:exchange][:name], {
33
+ :type => @options[:amqp][:exchange][:type],
34
+ :durable => @options[:amqp][:exchange][:durable]
35
+ })
36
+
37
+ queue = amq.queue("recognizer")
38
+ queue.bind(exchange, {
39
+ :routing_key => @options[:amqp][:exchange][:routing_key]
40
+ })
41
+
42
+ @logger.info("AMQP -- Awaiting metrics with impatience ...")
43
+
44
+ subscription = queue.subscribe(:ack => true, :blocking => false) do |header, message|
45
+ msg_routing_key = header.routing_key
46
+ lines = message.split("\n")
47
+ lines.each do |line|
48
+ line = line.strip
49
+ case line.split("\s").count
50
+ when 3
51
+ @input_queue.push(line)
52
+ when 2
53
+ @input_queue.push("#{msg_routing_key} #{line}")
54
+ else
55
+ @logger.warn("AMQP -- Received malformed metric :: #{msg_routing_key} :: #{line}")
56
+ end
57
+ end
58
+ header.ack
59
+ end
60
+
61
+ at_exit do
62
+ subscription.cancel
63
+ amq.close
64
+ rabbitmq.close
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,61 @@
1
+ require "thread"
2
+ require "socket"
3
+
4
+ module Recognizer
5
+ module Input
6
+ class TCP
7
+ def initialize(options={})
8
+ @logger = options[:logger]
9
+ @options = options[:options]
10
+ @input_queue = options[:input_queue]
11
+
12
+ @options[:tcp] ||= Hash.new
13
+ @tcp_connections = Queue.new
14
+
15
+ Thread.abort_on_exception = true
16
+ end
17
+
18
+ def run
19
+ setup_server
20
+ setup_thread_pool
21
+ end
22
+
23
+ private
24
+
25
+ def setup_server
26
+ port = @options[:tcp][:port] || 2003
27
+ tcp_server = TCPServer.new("0.0.0.0", port)
28
+ Thread.new do
29
+ @logger.info("TCP -- Awaiting metrics with impatience ...")
30
+ loop do
31
+ @tcp_connections.push(tcp_server.accept)
32
+ end
33
+ end
34
+ end
35
+
36
+ def create_server_thread
37
+ Thread.new do
38
+ loop do
39
+ if connection = @tcp_connections.shift
40
+ while line = connection.gets
41
+ line = line.strip
42
+ if line.split("\s").count == 3
43
+ @input_queue.push(line)
44
+ else
45
+ @logger.warn("TCP -- Received malformed metric :: #{line}")
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
52
+
53
+ def setup_thread_pool
54
+ threads = @options[:tcp][:threads] || 20
55
+ threads.times do
56
+ create_server_thread
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,132 @@
1
+ require "recognizer/version"
2
+ require "thread"
3
+ require "librato/metrics"
4
+
5
+ module Recognizer
6
+ class Librato
7
+ def initialize(options={})
8
+ @logger = options[:logger]
9
+ @options = options[:options]
10
+ @input_queue = options[:input_queue]
11
+
12
+ ::Librato::Metrics.authenticate(@options[:librato][:email], @options[:librato][:api_key])
13
+ ::Librato::Metrics.agent_identifier("recognizer", Recognizer::VERSION, "portertech")
14
+
15
+ @librato_queue = ::Librato::Metrics::Queue.new
16
+ @librato_mutex = Mutex.new
17
+
18
+ Thread.abort_on_exception = true
19
+ end
20
+
21
+ def run
22
+ setup_publisher
23
+ setup_consumer
24
+ end
25
+
26
+ def invalid_metric(plain_text, message)
27
+ @logger.warn("Invalid metric :: #{plain_text} :: #{message}")
28
+ false
29
+ end
30
+
31
+ def valid_plain_text?(plain_text)
32
+ segments = plain_text.split("\s")
33
+ if segments[0] !~ /^[A-Za-z0-9\._-]*$/
34
+ message = "Metric name must only consist of alpha-numeric characters, periods, underscores, and dashes"
35
+ invalid_metric(plain_text, message)
36
+ elsif segments[1] !~ /^[0-9]*\.?[0-9]*$/
37
+ invalid_metric(plain_text, "Metric value must be an integer or float")
38
+ elsif segments[2] !~ /^[0-9]{10}$/
39
+ invalid_metric(plain_text, "Metric timestamp must be epoch, 10 digits")
40
+ else
41
+ true
42
+ end
43
+ end
44
+
45
+ def extract_metric_source(metric_path)
46
+ metric_source = @options[:librato][:metric_source]
47
+ fallback_source = "recognizer"
48
+ case metric_source
49
+ when String
50
+ if metric_source =~ /^\/.*\/$/
51
+ metric_path.grep(Regexp.new(metric_source.slice(1..-2))).first || fallback_source
52
+ else
53
+ metric_source
54
+ end
55
+ when Integer
56
+ metric_path.slice(metric_source) || fallback_source
57
+ else
58
+ fallback_source
59
+ end
60
+ end
61
+
62
+ def pretty_number(number)
63
+ float = Float(number)
64
+ float == float.to_i ? float.to_i : float
65
+ end
66
+
67
+ def create_librato_metric(plain_text)
68
+ if valid_plain_text?(plain_text)
69
+ segments = plain_text.split("\s")
70
+
71
+ path = segments.shift.split(".")
72
+ value = pretty_number(segments.shift)
73
+ timestamp = pretty_number(segments.shift)
74
+ source = extract_metric_source(path)
75
+
76
+ path.delete(source)
77
+
78
+ name = path.join(".")
79
+
80
+ if name.size <= 63
81
+ {
82
+ name => {
83
+ :value => value,
84
+ :measure_time => timestamp,
85
+ :source => source
86
+ }
87
+ }
88
+ else
89
+ message = "Metric name must be 63 or fewer characters after source extraction"
90
+ invalid_metric(plain_text, message)
91
+ end
92
+ else
93
+ false
94
+ end
95
+ end
96
+
97
+ private
98
+
99
+ def setup_publisher
100
+ Thread.new do
101
+ loop do
102
+ sleep(@options[:librato][:flush_interval] || 10)
103
+ unless @librato_queue.empty?
104
+ @logger.info("Attempting to flush metrics to Librato")
105
+ @librato_mutex.synchronize do
106
+ begin
107
+ metric_count = @librato_queue.size
108
+ @librato_queue.submit
109
+ @logger.info("Successfully flushed #{metric_count} metrics to Librato")
110
+ rescue => error
111
+ @logger.error("Encountered an error when flushing metrics to Librato :: #{error}")
112
+ end
113
+ end
114
+ end
115
+ end
116
+ end
117
+ end
118
+
119
+ def setup_consumer
120
+ Thread.new do
121
+ loop do
122
+ if metric = create_librato_metric(@input_queue.shift)
123
+ @logger.debug("Adding metric to Librato queue :: #{metric.inspect}")
124
+ @librato_mutex.synchronize do
125
+ @librato_queue.add(metric)
126
+ end
127
+ end
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
@@ -0,0 +1,5 @@
1
+ module Recognizer
2
+ unless defined?(Recognizer::VERSION)
3
+ VERSION = "0.3.0"
4
+ end
5
+ end
data/lib/recognizer.rb ADDED
@@ -0,0 +1,47 @@
1
+ require "logger"
2
+ require "recognizer/cli"
3
+ require "recognizer/config"
4
+ require "recognizer/librato"
5
+ require "recognizer/inputs/tcp"
6
+ require "recognizer/inputs/amqp"
7
+
8
+ module Recognizer
9
+ def self.run
10
+ cli = Recognizer::CLI.new
11
+ config = Recognizer::Config.new(cli.read)
12
+ input_queue = Queue.new
13
+
14
+ options = cli.read.merge(config.read)
15
+
16
+ logger = Logger.new(STDOUT)
17
+
18
+ logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO
19
+
20
+ librato = Recognizer::Librato.new(
21
+ :logger => logger,
22
+ :options => options,
23
+ :input_queue => input_queue
24
+ )
25
+ librato.run
26
+
27
+ tcp = Recognizer::Input::TCP.new(
28
+ :logger => logger,
29
+ :options => options,
30
+ :input_queue => input_queue
31
+ )
32
+ tcp.run
33
+
34
+ if options.has_key?(:amqp)
35
+ amqp = Recognizer::Input::AMQP.new(
36
+ :logger => logger,
37
+ :options => options,
38
+ :input_queue => input_queue
39
+ )
40
+ amqp.run
41
+ end
42
+
43
+ loop do
44
+ sleep 30
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,25 @@
1
+ $:.push File.expand_path("../lib", __FILE__)
2
+ require "recognizer/version"
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = "recognizer"
6
+ s.version = Recognizer::VERSION
7
+ s.platform = "java"
8
+ s.authors = ["Sean Porter"]
9
+ s.email = ["portertech@gmail.com"]
10
+ s.homepage = "https://github.com/portertech/recognizer"
11
+ s.summary = "A Graphite Carbon impostor, sending metrics to Librato Metrics."
12
+ s.description = "A drop-in replacement for Graphite Carbon (TCP & AMQP), sending metrics to Librato Metrics."
13
+
14
+ s.rubyforge_project = "recognizer"
15
+
16
+ s.files = Dir.glob("{bin,lib}/**/*") + %w[recognizer.gemspec README.org MIT-LICENSE.txt]
17
+ s.executables = Dir.glob("bin/**/*").map { |file| File.basename(file) }
18
+ s.require_paths = ["lib"]
19
+
20
+ s.add_dependency("json")
21
+ s.add_dependency("jruby-openssl", "0.7.7")
22
+ s.add_dependency("mixlib-cli", "1.2.2")
23
+ s.add_dependency("hot_bunnies", "1.4.0")
24
+ s.add_dependency("librato-metrics", "0.7.5")
25
+ end
metadata ADDED
@@ -0,0 +1,152 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: recognizer
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.3.0
5
+ prerelease:
6
+ platform: java
7
+ authors:
8
+ - Sean Porter
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-11-28 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: json
16
+ version_requirements: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - ! '>='
19
+ - !ruby/object:Gem::Version
20
+ version: !binary |-
21
+ MA==
22
+ none: false
23
+ requirement: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - ! '>='
26
+ - !ruby/object:Gem::Version
27
+ version: !binary |-
28
+ MA==
29
+ none: false
30
+ prerelease: false
31
+ type: :runtime
32
+ - !ruby/object:Gem::Dependency
33
+ name: jruby-openssl
34
+ version_requirements: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - '='
37
+ - !ruby/object:Gem::Version
38
+ version: 0.7.7
39
+ none: false
40
+ requirement: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - '='
43
+ - !ruby/object:Gem::Version
44
+ version: 0.7.7
45
+ none: false
46
+ prerelease: false
47
+ type: :runtime
48
+ - !ruby/object:Gem::Dependency
49
+ name: mixlib-cli
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '='
53
+ - !ruby/object:Gem::Version
54
+ version: 1.2.2
55
+ none: false
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - '='
59
+ - !ruby/object:Gem::Version
60
+ version: 1.2.2
61
+ none: false
62
+ prerelease: false
63
+ type: :runtime
64
+ - !ruby/object:Gem::Dependency
65
+ name: hot_bunnies
66
+ version_requirements: !ruby/object:Gem::Requirement
67
+ requirements:
68
+ - - '='
69
+ - !ruby/object:Gem::Version
70
+ version: 1.4.0
71
+ none: false
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - '='
75
+ - !ruby/object:Gem::Version
76
+ version: 1.4.0
77
+ none: false
78
+ prerelease: false
79
+ type: :runtime
80
+ - !ruby/object:Gem::Dependency
81
+ name: librato-metrics
82
+ version_requirements: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - '='
85
+ - !ruby/object:Gem::Version
86
+ version: 0.7.5
87
+ none: false
88
+ requirement: !ruby/object:Gem::Requirement
89
+ requirements:
90
+ - - '='
91
+ - !ruby/object:Gem::Version
92
+ version: 0.7.5
93
+ none: false
94
+ prerelease: false
95
+ type: :runtime
96
+ description: A drop-in replacement for Graphite Carbon (TCP & AMQP), sending metrics to Librato Metrics.
97
+ email:
98
+ - portertech@gmail.com
99
+ executables:
100
+ - recognizer
101
+ extensions: []
102
+ extra_rdoc_files: []
103
+ files:
104
+ - !binary |-
105
+ YmluL3JlY29nbml6ZXI=
106
+ - !binary |-
107
+ bGliL3JlY29nbml6ZXIucmI=
108
+ - !binary |-
109
+ bGliL3JlY29nbml6ZXIvdmVyc2lvbi5yYg==
110
+ - !binary |-
111
+ bGliL3JlY29nbml6ZXIvY2xpLnJi
112
+ - !binary |-
113
+ bGliL3JlY29nbml6ZXIvY29uZmlnLnJi
114
+ - !binary |-
115
+ bGliL3JlY29nbml6ZXIvbGlicmF0by5yYg==
116
+ - !binary |-
117
+ bGliL3JlY29nbml6ZXIvaW5wdXRzL2FtcXAucmI=
118
+ - !binary |-
119
+ bGliL3JlY29nbml6ZXIvaW5wdXRzL3RjcC5yYg==
120
+ - recognizer.gemspec
121
+ - README.org
122
+ - MIT-LICENSE.txt
123
+ homepage: https://github.com/portertech/recognizer
124
+ licenses: []
125
+ post_install_message:
126
+ rdoc_options: []
127
+ require_paths:
128
+ - lib
129
+ required_ruby_version: !ruby/object:Gem::Requirement
130
+ requirements:
131
+ - - ! '>='
132
+ - !ruby/object:Gem::Version
133
+ segments:
134
+ - 0
135
+ version: !binary |-
136
+ MA==
137
+ hash: 2
138
+ none: false
139
+ required_rubygems_version: !ruby/object:Gem::Requirement
140
+ requirements:
141
+ - - ! '>='
142
+ - !ruby/object:Gem::Version
143
+ version: !binary |-
144
+ MA==
145
+ none: false
146
+ requirements: []
147
+ rubyforge_project: recognizer
148
+ rubygems_version: 1.8.24
149
+ signing_key:
150
+ specification_version: 3
151
+ summary: A Graphite Carbon impostor, sending metrics to Librato Metrics.
152
+ test_files: []