recognizer 0.0.10 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.org +29 -7
- data/lib/recognizer/amqp.rb +30 -28
- data/lib/recognizer/librato.rb +7 -7
- data/lib/recognizer/tcp.rb +53 -0
- data/lib/recognizer/version.rb +1 -1
- data/lib/recognizer.rb +9 -3
- data/recognizer.gemspec +5 -5
- metadata +29 -28
data/README.org
CHANGED
@@ -1,28 +1,40 @@
|
|
1
1
|
* Welcome to Recognizer
|
2
|
-
A Graphite Carbon
|
2
|
+
A Graphite Carbon impostor, sending metrics to [[https://metrics.librato.com/][Librato Metrics]].
|
3
|
+
|
4
|
+
TCP plain text & AMQP support.
|
3
5
|
|
4
6
|
[[https://github.com/portertech/recognizer/raw/master/recognizer.gif]]
|
5
7
|
* Install
|
8
|
+
Executable Java JAR
|
9
|
+
: wget https://github.com/downloads/portertech/recognizer/recognizer.jar
|
10
|
+
RubyGems
|
6
11
|
: gem install recognizer
|
7
12
|
* Configure
|
8
13
|
Example: =config.json=
|
9
14
|
: {
|
10
15
|
: "librato": {
|
11
16
|
: "email": "email@example.com",
|
12
|
-
: "api_key": "706325cf16d84d098127e143221dd180706325cf16d84d098127e143221dd180"
|
13
|
-
: "flush_interval": 5
|
17
|
+
: "api_key": "706325cf16d84d098127e143221dd180706325cf16d84d098127e143221dd180"
|
14
18
|
: },
|
15
19
|
: "amqp": {
|
16
|
-
: "host": "
|
17
|
-
:
|
18
|
-
:
|
20
|
+
: "host": "localhost"
|
21
|
+
: },
|
22
|
+
: "tcp": {
|
23
|
+
: "port": 2003
|
19
24
|
: }
|
20
25
|
: }
|
21
26
|
* Usage
|
27
|
+
Executable Java JAR
|
28
|
+
: java -jar recognizer.jar -h
|
29
|
+
RubyGems
|
30
|
+
: recognizer -h
|
31
|
+
|
22
32
|
: Usage: recognizer (options)
|
23
33
|
: -c, --config CONFIG The config file path
|
24
34
|
: -h, --help Show this message
|
25
35
|
* More
|
36
|
+
By default, Recognizer uses the source: =recognizer=
|
37
|
+
|
26
38
|
Example metric path: =production.i-424242.cpu.user=
|
27
39
|
|
28
40
|
Extract the metric source from the metric path using a regular expression
|
@@ -37,6 +49,16 @@
|
|
37
49
|
: {
|
38
50
|
: "librato": {
|
39
51
|
: "metric_source": "example"
|
40
|
-
|
52
|
+
|
53
|
+
By default, Recognizer uses =20= threads for the TCP server
|
54
|
+
|
55
|
+
Specify the number of TCP server threads
|
56
|
+
: {
|
57
|
+
: "tcp": {
|
58
|
+
: "threads": 30
|
59
|
+
|
60
|
+
By default, Recognizer uses an AMQP queue named: =recognizer=
|
61
|
+
|
62
|
+
By default, Recognizer binds its AMQP queue to the exchange: =graphite=
|
41
63
|
* License
|
42
64
|
Recognizer is released under the [[https://github.com/portertech/recognizer/raw/master/MIT-LICENSE.txt][MIT license]].
|
data/lib/recognizer/amqp.rb
CHANGED
@@ -4,46 +4,48 @@ require "bunny"
|
|
4
4
|
|
5
5
|
module Recognizer
|
6
6
|
class AMQP
|
7
|
-
def initialize(
|
8
|
-
unless
|
7
|
+
def initialize(carbon_queue, logger, options)
|
8
|
+
unless carbon_queue && options.is_a?(Hash)
|
9
9
|
raise "You must provide a thread queue and options"
|
10
10
|
end
|
11
11
|
|
12
|
-
options
|
13
|
-
|
12
|
+
if options.has_key?(:amqp)
|
13
|
+
options[:amqp][:exchange] ||= Hash.new
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
15
|
+
exchange_name = options[:amqp][:exchange][:name] || "graphite"
|
16
|
+
durable = options[:amqp][:exchange][:durable] || false
|
17
|
+
routing_key = options[:amqp][:exchange][:routing_key] || "#"
|
18
|
+
exchange_type = options[:amqp][:exchange][:type] || :topic
|
19
19
|
|
20
|
-
|
21
|
-
|
20
|
+
amqp = Bunny.new(options[:amqp].reject { |key, value| key == :exchange })
|
21
|
+
amqp.start
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
|
23
|
+
exchange = amqp.exchange(exchange_name, :type => exchange_type.to_sym, :durable => durable)
|
24
|
+
queue = amqp.queue("recognizer")
|
25
|
+
queue.bind(exchange, :key => routing_key)
|
26
26
|
|
27
|
-
|
27
|
+
Thread.abort_on_exception = true
|
28
28
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
29
|
+
Thread.new do
|
30
|
+
logger.info("AMQP -- Awaiting metrics with impatience ...")
|
31
|
+
queue.subscribe do |message|
|
32
|
+
payload = message[:payload]
|
33
|
+
msg_routing_key = message[:routing_key] || message[:delivery_details][:routing_key]
|
34
|
+
lines = payload.split("\n")
|
35
|
+
lines.each do |line|
|
36
|
+
line = line.strip
|
37
|
+
case line.split("\s").count
|
38
|
+
when 3
|
39
|
+
carbon_queue.push(line)
|
40
|
+
when 2
|
41
|
+
carbon_queue.push("#{msg_routing_key} #{line}")
|
42
|
+
end
|
42
43
|
end
|
43
44
|
end
|
44
45
|
end
|
46
|
+
else
|
47
|
+
logger.warn("AMQP -- Not configured")
|
45
48
|
end
|
46
|
-
consumer.join
|
47
49
|
end
|
48
50
|
end
|
49
51
|
end
|
data/lib/recognizer/librato.rb
CHANGED
@@ -7,8 +7,8 @@ require File.join(File.dirname(__FILE__), 'patches', 'float')
|
|
7
7
|
|
8
8
|
module Recognizer
|
9
9
|
class Librato
|
10
|
-
def initialize(
|
11
|
-
unless
|
10
|
+
def initialize(carbon_queue, logger, options)
|
11
|
+
unless carbon_queue && options.is_a?(Hash)
|
12
12
|
raise "You must provide a thread queue and options"
|
13
13
|
end
|
14
14
|
unless options[:librato][:email] && options[:librato][:api_key]
|
@@ -26,11 +26,11 @@ module Recognizer
|
|
26
26
|
loop do
|
27
27
|
sleep(options[:librato][:flush_interval] || 10)
|
28
28
|
unless librato.empty?
|
29
|
-
|
29
|
+
logger.info("Attempting to flush metrics to Librato")
|
30
30
|
mutex.synchronize do
|
31
31
|
librato.submit
|
32
32
|
end
|
33
|
-
|
33
|
+
logger.info("Successfully flushed metrics to Librato")
|
34
34
|
end
|
35
35
|
end
|
36
36
|
end
|
@@ -51,7 +51,7 @@ module Recognizer
|
|
51
51
|
|
52
52
|
Thread.new do
|
53
53
|
loop do
|
54
|
-
graphite_formated =
|
54
|
+
graphite_formated = carbon_queue.pop
|
55
55
|
begin
|
56
56
|
path, value, timestamp = graphite_formated.split(" ").inject([]) do |result, part|
|
57
57
|
result << (result.empty? ? part.split(".") : Float(part).pretty)
|
@@ -61,11 +61,11 @@ module Recognizer
|
|
61
61
|
path.delete(source)
|
62
62
|
metric = {path.join(".") => {:value => value, :measure_time => timestamp, :source => source}}
|
63
63
|
mutex.synchronize do
|
64
|
-
|
64
|
+
logger.info("Adding metric to queue: #{metric.inspect}")
|
65
65
|
librato.add(metric)
|
66
66
|
end
|
67
67
|
rescue ArgumentError
|
68
|
-
|
68
|
+
logger.info("Invalid metric: #{graphite_formated}")
|
69
69
|
end
|
70
70
|
end
|
71
71
|
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require "rubygems"
|
2
|
+
require "timeout"
|
3
|
+
require "thread"
|
4
|
+
require "socket"
|
5
|
+
|
6
|
+
module Recognizer
|
7
|
+
class TCP
|
8
|
+
def initialize(carbon_queue, logger, options)
|
9
|
+
unless carbon_queue && options.is_a?(Hash)
|
10
|
+
raise "You must provide a thread queue and options"
|
11
|
+
end
|
12
|
+
|
13
|
+
options[:tcp] ||= Hash.new
|
14
|
+
|
15
|
+
threads = options[:tcp][:threads] || 20
|
16
|
+
port = options[:tcp][:port] || 2003
|
17
|
+
|
18
|
+
tcp_server = TCPServer.new("0.0.0.0", port)
|
19
|
+
tcp_connections = Queue.new
|
20
|
+
|
21
|
+
Thread.abort_on_exception = true
|
22
|
+
|
23
|
+
threads.times do
|
24
|
+
Thread.new do
|
25
|
+
loop do
|
26
|
+
if connection = tcp_connections.shift
|
27
|
+
begin
|
28
|
+
lines = timeout(12) do
|
29
|
+
connection.gets.split("\n")
|
30
|
+
end
|
31
|
+
lines.each do |line|
|
32
|
+
line = line.strip
|
33
|
+
if line.split("\s").count == 3
|
34
|
+
carbon_queue.push(line)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
rescue Timeout::Error
|
38
|
+
connection.close
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
Thread.new do
|
46
|
+
logger.info("TCP -- Awaiting metrics with impatience ...")
|
47
|
+
loop do
|
48
|
+
tcp_connections.push(tcp_server.accept)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
data/lib/recognizer/version.rb
CHANGED
data/lib/recognizer.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require "recognizer/cli"
|
2
2
|
require "recognizer/config"
|
3
3
|
require "recognizer/librato"
|
4
|
+
require "recognizer/tcp"
|
4
5
|
require "recognizer/amqp"
|
5
6
|
|
6
7
|
module Recognizer
|
@@ -9,8 +10,13 @@ module Recognizer
|
|
9
10
|
cli_options = cli.read
|
10
11
|
config = Recognizer::Config.new(cli_options)
|
11
12
|
config_options = config.read
|
12
|
-
|
13
|
-
|
14
|
-
Recognizer::
|
13
|
+
carbon_queue = Queue.new
|
14
|
+
logger = Logger.new(STDOUT)
|
15
|
+
Recognizer::Librato.new(carbon_queue, logger, config_options)
|
16
|
+
Recognizer::TCP.new(carbon_queue, logger, config_options)
|
17
|
+
Recognizer::AMQP.new(carbon_queue, logger, config_options)
|
18
|
+
loop do
|
19
|
+
sleep 30
|
20
|
+
end
|
15
21
|
end
|
16
22
|
end
|
data/recognizer.gemspec
CHANGED
@@ -8,8 +8,8 @@ Gem::Specification.new do |s|
|
|
8
8
|
s.authors = ["Sean Porter"]
|
9
9
|
s.email = ["portertech@gmail.com"]
|
10
10
|
s.homepage = "https://github.com/portertech/recognizer"
|
11
|
-
s.summary = "A Graphite Carbon
|
12
|
-
s.description = "A Graphite Carbon AMQP
|
11
|
+
s.summary = "A Graphite Carbon impostor, sending metrics to Librato Metrics."
|
12
|
+
s.description = "A drop-in replacement for Graphite Carbon (TCP & AMQP), sending metrics to Librato Metrics."
|
13
13
|
|
14
14
|
s.rubyforge_project = "recognizer"
|
15
15
|
|
@@ -17,8 +17,8 @@ Gem::Specification.new do |s|
|
|
17
17
|
s.executables = Dir.glob("bin/**/*").map { |file| File.basename(file) }
|
18
18
|
s.require_paths = ["lib"]
|
19
19
|
|
20
|
-
s.add_dependency("librato-metrics", "0.4.0")
|
21
|
-
s.add_dependency("bunny", "0.7.9")
|
22
|
-
s.add_dependency("mixlib-cli", ">= 1.1.0")
|
23
20
|
s.add_dependency("json")
|
21
|
+
s.add_dependency("mixlib-cli", ">= 1.1.0")
|
22
|
+
s.add_dependency("bunny", "0.7.9")
|
23
|
+
s.add_dependency("librato-metrics", "0.4.2")
|
24
24
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: recognizer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 27
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
+
- 1
|
8
9
|
- 0
|
9
|
-
|
10
|
-
version: 0.0.10
|
10
|
+
version: 0.1.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Sean Porter
|
@@ -15,72 +15,72 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-03-
|
18
|
+
date: 2012-03-10 00:00:00 -08:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
22
|
-
name:
|
22
|
+
name: json
|
23
23
|
prerelease: false
|
24
24
|
requirement: &id001 !ruby/object:Gem::Requirement
|
25
25
|
none: false
|
26
26
|
requirements:
|
27
|
-
- - "
|
27
|
+
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
hash:
|
29
|
+
hash: 3
|
30
30
|
segments:
|
31
31
|
- 0
|
32
|
-
|
33
|
-
- 0
|
34
|
-
version: 0.4.0
|
32
|
+
version: "0"
|
35
33
|
type: :runtime
|
36
34
|
version_requirements: *id001
|
37
35
|
- !ruby/object:Gem::Dependency
|
38
|
-
name:
|
36
|
+
name: mixlib-cli
|
39
37
|
prerelease: false
|
40
38
|
requirement: &id002 !ruby/object:Gem::Requirement
|
41
39
|
none: false
|
42
40
|
requirements:
|
43
|
-
- - "
|
41
|
+
- - ">="
|
44
42
|
- !ruby/object:Gem::Version
|
45
|
-
hash:
|
43
|
+
hash: 19
|
46
44
|
segments:
|
45
|
+
- 1
|
46
|
+
- 1
|
47
47
|
- 0
|
48
|
-
|
49
|
-
- 9
|
50
|
-
version: 0.7.9
|
48
|
+
version: 1.1.0
|
51
49
|
type: :runtime
|
52
50
|
version_requirements: *id002
|
53
51
|
- !ruby/object:Gem::Dependency
|
54
|
-
name:
|
52
|
+
name: bunny
|
55
53
|
prerelease: false
|
56
54
|
requirement: &id003 !ruby/object:Gem::Requirement
|
57
55
|
none: false
|
58
56
|
requirements:
|
59
|
-
- - "
|
57
|
+
- - "="
|
60
58
|
- !ruby/object:Gem::Version
|
61
|
-
hash:
|
59
|
+
hash: 17
|
62
60
|
segments:
|
63
|
-
- 1
|
64
|
-
- 1
|
65
61
|
- 0
|
66
|
-
|
62
|
+
- 7
|
63
|
+
- 9
|
64
|
+
version: 0.7.9
|
67
65
|
type: :runtime
|
68
66
|
version_requirements: *id003
|
69
67
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
68
|
+
name: librato-metrics
|
71
69
|
prerelease: false
|
72
70
|
requirement: &id004 !ruby/object:Gem::Requirement
|
73
71
|
none: false
|
74
72
|
requirements:
|
75
|
-
- - "
|
73
|
+
- - "="
|
76
74
|
- !ruby/object:Gem::Version
|
77
|
-
hash:
|
75
|
+
hash: 11
|
78
76
|
segments:
|
79
77
|
- 0
|
80
|
-
|
78
|
+
- 4
|
79
|
+
- 2
|
80
|
+
version: 0.4.2
|
81
81
|
type: :runtime
|
82
82
|
version_requirements: *id004
|
83
|
-
description: A Graphite Carbon AMQP
|
83
|
+
description: A drop-in replacement for Graphite Carbon (TCP & AMQP), sending metrics to Librato Metrics.
|
84
84
|
email:
|
85
85
|
- portertech@gmail.com
|
86
86
|
executables:
|
@@ -97,6 +97,7 @@ files:
|
|
97
97
|
- lib/recognizer/librato.rb
|
98
98
|
- lib/recognizer/patches/float.rb
|
99
99
|
- lib/recognizer/patches/hash.rb
|
100
|
+
- lib/recognizer/tcp.rb
|
100
101
|
- lib/recognizer/version.rb
|
101
102
|
- lib/recognizer.rb
|
102
103
|
- recognizer.gemspec
|
@@ -135,6 +136,6 @@ rubyforge_project: recognizer
|
|
135
136
|
rubygems_version: 1.3.7
|
136
137
|
signing_key:
|
137
138
|
specification_version: 3
|
138
|
-
summary: A Graphite Carbon
|
139
|
+
summary: A Graphite Carbon impostor, sending metrics to Librato Metrics.
|
139
140
|
test_files: []
|
140
141
|
|