pipeline_toolkit 1.0.6 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +3 -0
- data/bin/msg_generator +3 -6
- data/bin/msg_push +1 -1
- data/lib/pipeline_toolkit.rb +1 -8
- data/lib/pipeline_toolkit/amqp/reader.rb +1 -0
- data/lib/pipeline_toolkit/commands/msg_generator/cli.rb +8 -5
- data/lib/pipeline_toolkit/cucumber.rb +6 -0
- data/lib/pipeline_toolkit/cucumber/amqp.rb +44 -0
- data/lib/pipeline_toolkit/cucumber/amqp_steps.rb +18 -0
- data/lib/pipeline_toolkit/cucumber/machine.rb +57 -0
- data/lib/pipeline_toolkit/cucumber/machine_steps.rb +23 -0
- data/lib/pipeline_toolkit/cucumber/msg_sub_machine.rb +34 -0
- data/lib/pipeline_toolkit/handlers/message_handler.rb +21 -8
- data/lib/pipeline_toolkit/message_coder.rb +1 -1
- data/lib/pipeline_toolkit/message_command.rb +4 -4
- data/lib/pipeline_toolkit/message_generator.rb +11 -5
- data/lib/pipeline_toolkit/message_subscriber.rb +1 -0
- metadata +11 -10
- data/bin/msg_probe +0 -13
- data/lib/pipeline_toolkit/commands/msg_probe/cli.rb +0 -46
- data/lib/pipeline_toolkit/message_probe.rb +0 -114
- data/spec/eventmachine_helper.rb +0 -44
- data/spec/message_subscriber_spec.rb +0 -64
- data/spec/spec_helper.rb +0 -15
data/README.markdown
CHANGED
@@ -94,5 +94,8 @@ msg_sink acknowledges all messages.
|
|
94
94
|
* DNS-SD is broken. See http://github.com/tenderlove/dnssd/issues#issue/3. We need to fix it or kill it.
|
95
95
|
|
96
96
|
* Write units tests.
|
97
|
+
- Make tests re-usable
|
97
98
|
|
98
99
|
* Tidy up logging. Not used consistently throughout code. What do we want to log?
|
100
|
+
|
101
|
+
* Change encoding to use YAJL
|
data/bin/msg_generator
CHANGED
@@ -3,11 +3,8 @@
|
|
3
3
|
# Created on 2010-3-17.
|
4
4
|
# Copyright Visfleet Ltd. (c) 2010. All rights reserved.
|
5
5
|
|
6
|
-
|
7
|
-
|
6
|
+
# Magic line to make bin work from
|
7
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../lib') unless $LOAD_PATH.include?(File.dirname(__FILE__) + '/../lib')
|
8
8
|
require "pipeline_toolkit/commands/msg_generator/cli"
|
9
9
|
|
10
|
-
|
11
|
-
ARGV << "--help" if ARGV.empty?
|
12
|
-
|
13
|
-
Commands::MsgGenerate::CLI.execute(STDOUT, ARGV)
|
10
|
+
Commands::MsgGenerate::CLI.execute(STDOUT, ARGV)
|
data/bin/msg_push
CHANGED
data/lib/pipeline_toolkit.rb
CHANGED
@@ -11,16 +11,9 @@ require 'pipeline_toolkit/handlers/message_handler'
|
|
11
11
|
require "pipeline_toolkit/default_logger"
|
12
12
|
require "pipeline_toolkit/message_coder"
|
13
13
|
require "pipeline_toolkit/message_command"
|
14
|
-
require "pipeline_toolkit/message_probe"
|
15
14
|
require "pipeline_toolkit/message_pusher"
|
16
15
|
require "pipeline_toolkit/message_subscriber"
|
17
16
|
require "pipeline_toolkit/message_sink"
|
18
17
|
require "pipeline_toolkit/open_hash"
|
19
18
|
require "pipeline_toolkit/socket_util"
|
20
|
-
require 'pipeline_toolkit/message_generator'
|
21
|
-
|
22
|
-
require 'pipeline_toolkit/commands/msg_subscribe/cli'
|
23
|
-
require 'pipeline_toolkit/commands/msg_generator/cli'
|
24
|
-
require 'pipeline_toolkit/commands/msg_probe/cli'
|
25
|
-
require 'pipeline_toolkit/commands/msg_push/cli'
|
26
|
-
require 'pipeline_toolkit/commands/msg_sink/cli'
|
19
|
+
require 'pipeline_toolkit/message_generator'
|
@@ -1,20 +1,24 @@
|
|
1
1
|
require 'pp'
|
2
2
|
require 'trollop'
|
3
3
|
require 'eventmachine'
|
4
|
+
require 'pipeline_toolkit'
|
4
5
|
|
5
6
|
Signal.trap('INT') do
|
6
|
-
|
7
|
+
puts "\nStopping"
|
8
|
+
exit(0)
|
7
9
|
# TODO: complete signal trap interrupt
|
8
10
|
end
|
9
11
|
|
10
12
|
Signal.trap('TERM') do
|
11
|
-
|
13
|
+
puts "\nStopping"
|
14
|
+
exit(0)
|
12
15
|
# TODO: complete signal trap terminate
|
13
16
|
end
|
14
17
|
|
15
18
|
module Commands # :nodoc:
|
16
19
|
module MsgGenerate
|
17
20
|
class CLI
|
21
|
+
|
18
22
|
def self.execute(stdout, arguments=[])
|
19
23
|
|
20
24
|
opts = Trollop::options do
|
@@ -33,9 +37,8 @@ Examples:
|
|
33
37
|
Options:
|
34
38
|
EOL
|
35
39
|
opt :msg, "The message to send", :short => "m", :default => '{"id":1}'
|
36
|
-
opt :delay, "Sleep time between sends in seconds", :short => "d", :type => :float
|
37
|
-
|
38
|
-
opt :env, "The environment to run (development, production)", :default => "development", :short => :e
|
40
|
+
opt :delay, "Sleep time between sends in seconds", :short => "d", :type => :float, :default => 1.0
|
41
|
+
opt :limit, "Generate this number of messages and then exit. By default messages will keep generating indefinately", :short => "n", :type => :integer
|
39
42
|
end
|
40
43
|
|
41
44
|
MessageGenerator.new(opts).start
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module PipelineToolkit
|
2
|
+
module Cucumber
|
3
|
+
|
4
|
+
class AMQP
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
end
|
8
|
+
|
9
|
+
def start(host, port)
|
10
|
+
@mq = Bunny.new(:host => host, :port => port)
|
11
|
+
@mq.start
|
12
|
+
end
|
13
|
+
|
14
|
+
def purge_queue(queue)
|
15
|
+
@mq.queue(queue).purge
|
16
|
+
end
|
17
|
+
|
18
|
+
def send_messages(exchange, messages)
|
19
|
+
exchange = @mq.exchange(exchange, :type => :fanout)
|
20
|
+
messages.each do |message|
|
21
|
+
message.default = nil
|
22
|
+
exchange.publish(MessageCoder.encode(message))
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def receive_messages(queue, number, timeout = 5)
|
27
|
+
received_msgs = []
|
28
|
+
Timeout::timeout(timeout) do
|
29
|
+
number.times do
|
30
|
+
message = @mq.queue(queue).pop
|
31
|
+
sleep(0.1)
|
32
|
+
retry if message.payload == :queue_empty
|
33
|
+
received_msgs << MessageCoder.decode(message.payload)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
received_msgs
|
37
|
+
rescue Timeout::Error
|
38
|
+
raise "Timed out waiting for AMQP messages. Received #{received_msgs.size}"
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
Given /^an amqp server running at "(\w+)" on port (\d+)$/ do |host, port|
|
2
|
+
@amqp = PipelineToolkit::Cucumber::AMQP.new
|
3
|
+
@amqp.start(host, port)
|
4
|
+
end
|
5
|
+
|
6
|
+
Given /^amqp queue "(\w+)" is empty$/ do |queue|
|
7
|
+
@amqp.purge_queue(queue)
|
8
|
+
end
|
9
|
+
|
10
|
+
When /^I input these messages to exchange "(\w+)":$/ do |queue, messages|
|
11
|
+
input_messages = messages.hashes
|
12
|
+
@amqp.send_messages(queue, input_messages)
|
13
|
+
end
|
14
|
+
|
15
|
+
Then /^these messages are on queue "(\w+)":$/ do |queue, expected_messages|
|
16
|
+
received_messages = @amqp.receive_messages(queue, expected_messages.rows.size)
|
17
|
+
expected_messages.diff!(received_messages)
|
18
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'pipeline_toolkit'
|
2
|
+
require 'background_process'
|
3
|
+
|
4
|
+
module PipelineToolkit
|
5
|
+
module Cucumber
|
6
|
+
|
7
|
+
# Provides an interface for running and interacting with a machine process.
|
8
|
+
class Machine
|
9
|
+
|
10
|
+
def run(command)
|
11
|
+
@process = BackgroundProcess.run("ruby -rubygems bin/#{command}")
|
12
|
+
# wait for process to start, and check if running
|
13
|
+
sleep(0.5)
|
14
|
+
raise @process.stderr.gets unless @process.running?
|
15
|
+
end
|
16
|
+
|
17
|
+
def kill
|
18
|
+
@process.kill
|
19
|
+
@process.wait
|
20
|
+
end
|
21
|
+
|
22
|
+
def input_messages(messages)
|
23
|
+
messages.each do |message|
|
24
|
+
# NB: required otherwise marshal complains about the Hash's default proc
|
25
|
+
message.default = nil
|
26
|
+
@process.stdin.puts(MessageCoder.encode(message))
|
27
|
+
@process.stdin.flush
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def get_messages(number, timeout = 5)
|
32
|
+
output_msgs = []
|
33
|
+
output_lines(number, timeout) do |line|
|
34
|
+
msg = MessageCoder.decode(line)
|
35
|
+
output_msgs << msg
|
36
|
+
end
|
37
|
+
output_msgs
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def output_lines(number, timeout = 5)
|
43
|
+
count = 0
|
44
|
+
Timeout::timeout(timeout) do
|
45
|
+
while (count < number)
|
46
|
+
line = @process.stdout.gets
|
47
|
+
yield line
|
48
|
+
count += 1
|
49
|
+
end
|
50
|
+
end
|
51
|
+
rescue Timeout::Error
|
52
|
+
raise "Timed out waiting for messages. Received #{count} of #{number}"
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# ---------
|
2
|
+
# Machine lifecycle
|
3
|
+
|
4
|
+
Given /^I run this machine:$/ do |command|
|
5
|
+
@machine = PipelineToolkit::Cucumber::Machine.new
|
6
|
+
@machine.run(command)
|
7
|
+
end
|
8
|
+
|
9
|
+
# Clean up backround processes after each scenario
|
10
|
+
After do
|
11
|
+
@machine.kill unless @machine.nil?
|
12
|
+
end
|
13
|
+
|
14
|
+
When /^I input these messages:$/ do |messages|
|
15
|
+
@machine.input_messages(messages.hashes)
|
16
|
+
end
|
17
|
+
|
18
|
+
Then /^these messages are output:$/ do |expected_msgs|
|
19
|
+
received_messages = @machine.get_messages(expected_msgs.rows.size)
|
20
|
+
# turn everything into str
|
21
|
+
received_messages.map! { |h| h.each { |k,v| h[k] = v.to_s }}
|
22
|
+
expected_msgs.diff!(received_messages)
|
23
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
|
2
|
+
module PipelineToolkit
|
3
|
+
module Cucumber
|
4
|
+
|
5
|
+
# Provides an interface for running and interacting with a machine process.
|
6
|
+
class MsgSubMachine < Machine
|
7
|
+
|
8
|
+
def run(command)
|
9
|
+
super(command)
|
10
|
+
self.setup_pipe
|
11
|
+
end
|
12
|
+
|
13
|
+
def setup_pipe
|
14
|
+
# first message should be system one
|
15
|
+
msg = get_messages(1).first
|
16
|
+
raise "Expecting first message to be system message, got instead: #{msg}" if msg.msg_type != :system
|
17
|
+
@system_setup = msg
|
18
|
+
@ack_pipe = File.open(@system_setup[:sys_pipe], "w")
|
19
|
+
end
|
20
|
+
|
21
|
+
def acknowledge_message(message)
|
22
|
+
ack_message = {:msg_type => :ack, :ack_id => message[:ack_id]}
|
23
|
+
@ack_pipe.syswrite(MessageCoder.encode(ack_message) << "\n")
|
24
|
+
end
|
25
|
+
|
26
|
+
def kill
|
27
|
+
@ack_pipe.close
|
28
|
+
super
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -5,7 +5,9 @@ module Handlers # :nodoc:
|
|
5
5
|
# when it receives a new message from IO watcher
|
6
6
|
#
|
7
7
|
module MessageHandler
|
8
|
-
|
8
|
+
|
9
|
+
MAX_BUFFER = 3_000_000
|
10
|
+
|
9
11
|
attr_reader :options
|
10
12
|
|
11
13
|
##
|
@@ -24,15 +26,26 @@ module Handlers # :nodoc:
|
|
24
26
|
#
|
25
27
|
def notify_readable
|
26
28
|
DefaultLogger.debug("Handlers::MessageHandler#notify_readable") if options[:env] == "development"
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
29
|
+
|
30
|
+
# Grab everything from buffer, in case several messages have built up.
|
31
|
+
# NB: Can't use gets or read because they block when reaching EOF.
|
32
|
+
@buffer ||= BufferedTokenizer.new
|
33
|
+
data = @io.read_nonblock(MAX_BUFFER)
|
34
|
+
|
35
|
+
@buffer.extract(data).each do |line|
|
36
|
+
receive_line(line)
|
33
37
|
end
|
38
|
+
|
34
39
|
rescue StandardError => e # rescued here because main thread does not seem to see it
|
35
40
|
DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
|
36
41
|
end
|
37
|
-
|
42
|
+
|
43
|
+
def receive_line(message_coded)
|
44
|
+
DefaultLogger.debug("Raw message: #{message_coded}") if options[:env] == "development"
|
45
|
+
message = MessageCoder.decode(message_coded.chomp)
|
46
|
+
DefaultLogger.debug("Message: #{message}") if options[:env] == "development"
|
47
|
+
@target.process(message)
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
38
51
|
end
|
@@ -47,7 +47,7 @@ module MessageCommand
|
|
47
47
|
attr_reader :options
|
48
48
|
|
49
49
|
##
|
50
|
-
#
|
50
|
+
# Initializes a new instance. A message command can only
|
51
51
|
# be initialized through the implementation instance.
|
52
52
|
#
|
53
53
|
# @param options<Hash> An options hash.
|
@@ -62,7 +62,7 @@ module MessageCommand
|
|
62
62
|
# and starts watching the STDIN for new messages
|
63
63
|
#
|
64
64
|
def start
|
65
|
-
DefaultLogger.debug("MessageCommand#start") if options[:env] == "development"
|
65
|
+
DefaultLogger.debug("MessageCommand#start") if @options[:env] == "development"
|
66
66
|
|
67
67
|
Signal.trap('INT') { EM.stop }
|
68
68
|
Signal.trap('TERM') { EM.stop }
|
@@ -70,10 +70,10 @@ module MessageCommand
|
|
70
70
|
begin
|
71
71
|
EM.run do
|
72
72
|
conn = EM.watch($stdin, Handlers::MessageHandler, self, options)
|
73
|
+
initialize_machine
|
74
|
+
|
73
75
|
# must call this to setup callback to notify_readable
|
74
76
|
conn.notify_readable = true
|
75
|
-
|
76
|
-
initialize_machine
|
77
77
|
end
|
78
78
|
rescue StandardError => e
|
79
79
|
DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
|
@@ -2,20 +2,26 @@ require 'json'
|
|
2
2
|
|
3
3
|
class MessageGenerator
|
4
4
|
|
5
|
-
def initialize(
|
6
|
-
@delay =
|
7
|
-
@msg = JSON.parse(
|
5
|
+
def initialize(options)
|
6
|
+
@delay = options.delay
|
7
|
+
@msg = JSON.parse(options.msg)
|
8
|
+
@limit = options.limit
|
8
9
|
end
|
9
|
-
|
10
|
+
|
10
11
|
def start
|
12
|
+
@count = 0
|
11
13
|
loop do
|
14
|
+
@count += 1
|
15
|
+
break if @limit && @count > @limit
|
16
|
+
|
12
17
|
self.send_msg(@msg)
|
13
18
|
sleep(@delay) unless @delay.nil?
|
14
19
|
end
|
15
20
|
end
|
16
21
|
|
17
22
|
def send_msg(msg)
|
18
|
-
|
23
|
+
puts MessageCoder.encode(@msg)
|
19
24
|
$stdout.flush
|
20
25
|
end
|
26
|
+
|
21
27
|
end
|
@@ -133,6 +133,7 @@ class MessageSubscriber
|
|
133
133
|
#
|
134
134
|
def process_queue_message(header, body)
|
135
135
|
DefaultLogger.debug("MessageSubscriber#process_queue_message(header, body)")
|
136
|
+
body = MessageCoder.decode(body)
|
136
137
|
unless body.is_a?(Hash)
|
137
138
|
message = { :msg_type => :standard, :raw => body }
|
138
139
|
else
|
metadata
CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
|
|
4
4
|
prerelease: false
|
5
5
|
segments:
|
6
6
|
- 1
|
7
|
+
- 1
|
7
8
|
- 0
|
8
|
-
|
9
|
-
version: 1.0.6
|
9
|
+
version: 1.1.0
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Aisha Fenton
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2010-08-
|
18
|
+
date: 2010-08-20 00:00:00 +12:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -213,7 +213,6 @@ description: |-
|
|
213
213
|
(serialized as YAML) so they can hold any values and change throughout the processing.
|
214
214
|
email: labs@visfleet.com
|
215
215
|
executables:
|
216
|
-
- msg_probe
|
217
216
|
- msg_subscribe
|
218
217
|
- msg_push
|
219
218
|
- msg_sink
|
@@ -229,16 +228,20 @@ files:
|
|
229
228
|
- lib/pipeline_toolkit/amqp/reader.rb
|
230
229
|
- lib/pipeline_toolkit/amqp/writer.rb
|
231
230
|
- lib/pipeline_toolkit/commands/msg_generator/cli.rb
|
232
|
-
- lib/pipeline_toolkit/commands/msg_probe/cli.rb
|
233
231
|
- lib/pipeline_toolkit/commands/msg_push/cli.rb
|
234
232
|
- lib/pipeline_toolkit/commands/msg_sink/cli.rb
|
235
233
|
- lib/pipeline_toolkit/commands/msg_subscribe/cli.rb
|
234
|
+
- lib/pipeline_toolkit/cucumber.rb
|
235
|
+
- lib/pipeline_toolkit/cucumber/amqp.rb
|
236
|
+
- lib/pipeline_toolkit/cucumber/amqp_steps.rb
|
237
|
+
- lib/pipeline_toolkit/cucumber/machine.rb
|
238
|
+
- lib/pipeline_toolkit/cucumber/machine_steps.rb
|
239
|
+
- lib/pipeline_toolkit/cucumber/msg_sub_machine.rb
|
236
240
|
- lib/pipeline_toolkit/default_logger.rb
|
237
241
|
- lib/pipeline_toolkit/handlers/message_handler.rb
|
238
242
|
- lib/pipeline_toolkit/message_coder.rb
|
239
243
|
- lib/pipeline_toolkit/message_command.rb
|
240
244
|
- lib/pipeline_toolkit/message_generator.rb
|
241
|
-
- lib/pipeline_toolkit/message_probe.rb
|
242
245
|
- lib/pipeline_toolkit/message_pusher.rb
|
243
246
|
- lib/pipeline_toolkit/message_sink.rb
|
244
247
|
- lib/pipeline_toolkit/message_subscriber.rb
|
@@ -277,7 +280,5 @@ rubygems_version: 1.3.6
|
|
277
280
|
signing_key:
|
278
281
|
specification_version: 3
|
279
282
|
summary: Toolkit for building processing pipelines using Unix Pipes and AMQP messages
|
280
|
-
test_files:
|
281
|
-
|
282
|
-
- spec/message_subscriber_spec.rb
|
283
|
-
- spec/spec_helper.rb
|
283
|
+
test_files: []
|
284
|
+
|
data/bin/msg_probe
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
#
|
3
|
-
# Created on 2010-3-17.
|
4
|
-
# Copyright Visfleet Ltd. (c) 2010. All rights reserved.
|
5
|
-
|
6
|
-
require 'rubygems'
|
7
|
-
require File.dirname(__FILE__) + '/../lib/pipeline_toolkit'
|
8
|
-
require 'pipeline_toolkit/commands/msg_probe/cli'
|
9
|
-
|
10
|
-
# print an options summary if no args specified
|
11
|
-
ARGV << "--help" if ARGV.empty?
|
12
|
-
|
13
|
-
Commands::MsgProbe::CLI.execute(STDOUT, ARGV)
|
@@ -1,46 +0,0 @@
|
|
1
|
-
require 'pp'
|
2
|
-
require 'trollop'
|
3
|
-
require 'eventmachine'
|
4
|
-
|
5
|
-
Signal.trap('INT') do
|
6
|
-
# puts "\nStopping"
|
7
|
-
# TODO: complete signal trap interrupt
|
8
|
-
end
|
9
|
-
|
10
|
-
Signal.trap('TERM') do
|
11
|
-
# puts "\nStopping"
|
12
|
-
# TODO: complete signal trap terminate
|
13
|
-
end
|
14
|
-
|
15
|
-
module Commands # :nodoc:
|
16
|
-
module MsgProbe
|
17
|
-
class CLI
|
18
|
-
def self.execute(stdout, arguments=[])
|
19
|
-
|
20
|
-
opts = Trollop::options do
|
21
|
-
banner <<-EOL
|
22
|
-
Message Probe
|
23
|
-
------------------
|
24
|
-
TODO: description
|
25
|
-
|
26
|
-
Usage:
|
27
|
-
msg_probe [options]
|
28
|
-
|
29
|
-
Examples:
|
30
|
-
msg_probe --name octane_message_formatter
|
31
|
-
msg_probe --name octane_message_formatter --http_port 10001 --interval 1
|
32
|
-
|
33
|
-
Options:
|
34
|
-
EOL
|
35
|
-
opt :interval, "Time in seconds between updates", :short => "i", :default => 2
|
36
|
-
opt :http_port, "The port the HTTP server runs on. Default is a random port between 10000-11000", :type => :integer
|
37
|
-
opt :name, "The name of the probe. Used in monitoring", :type => :string, :short => 'n'
|
38
|
-
opt :dnssd, "Switches on DNSSD (i.e. Bonjour) for the monitoring interface", :short => 'd'
|
39
|
-
opt :env, "The environment to run (development, production)", :default => "development", :short => :e
|
40
|
-
end
|
41
|
-
|
42
|
-
MessageProbe.new(opts).start
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
@@ -1,114 +0,0 @@
|
|
1
|
-
require 'evma_httpserver'
|
2
|
-
|
3
|
-
class MessageProbe
|
4
|
-
include MessageCommand
|
5
|
-
attr_reader :mps, :uptime, :name
|
6
|
-
|
7
|
-
def initialize(opts)
|
8
|
-
@interval = opts.interval
|
9
|
-
@http_port= opts.http_port || Socket.select_random_port(10_000, 11_000)
|
10
|
-
@start_time = Time.now
|
11
|
-
@name = opts.name
|
12
|
-
self.reset
|
13
|
-
self.init_dnssd if opts.dnssd
|
14
|
-
end
|
15
|
-
|
16
|
-
def init_dnssd
|
17
|
-
require 'dnssd'
|
18
|
-
DNSSD.register!("#{@name} probe", "_http._tcp", nil, @http_port)
|
19
|
-
end
|
20
|
-
|
21
|
-
def initialize_machine
|
22
|
-
EM.start_server('0.0.0.0', @http_port, ProbeHttpRequest, self)
|
23
|
-
EM.add_periodic_timer(@interval) { self.tick }
|
24
|
-
end
|
25
|
-
|
26
|
-
def tick
|
27
|
-
@time_delta = Time.now - @prev_time
|
28
|
-
@mps = @count / @time_delta
|
29
|
-
self.reset
|
30
|
-
end
|
31
|
-
|
32
|
-
def reset
|
33
|
-
@count = 0
|
34
|
-
@prev_time = Time.now
|
35
|
-
end
|
36
|
-
|
37
|
-
def uptime
|
38
|
-
Time.now - @start_time
|
39
|
-
end
|
40
|
-
|
41
|
-
# OPTIMIZE. can improve performance by overriding base process_line method
|
42
|
-
# saving us the unnecessary marshal step.
|
43
|
-
def process_standard(msg)
|
44
|
-
@count += 1
|
45
|
-
msg
|
46
|
-
end
|
47
|
-
|
48
|
-
end
|
49
|
-
|
50
|
-
class ProbeHttpRequest < EM::Connection
|
51
|
-
include EM::HttpServer
|
52
|
-
|
53
|
-
def initialize(probe)
|
54
|
-
@probe = probe
|
55
|
-
end
|
56
|
-
|
57
|
-
def post_init
|
58
|
-
super
|
59
|
-
no_environment_strings
|
60
|
-
end
|
61
|
-
|
62
|
-
def process_http_request
|
63
|
-
response = EM::DelegatedHttpResponse.new(self)
|
64
|
-
response.status = 200
|
65
|
-
response.content_type 'text/html'
|
66
|
-
response.content = <<-EOL
|
67
|
-
<html>
|
68
|
-
<head>
|
69
|
-
<title>Message Probe</title>
|
70
|
-
<style type="text/css">
|
71
|
-
body {
|
72
|
-
background: black;
|
73
|
-
color: #80c0c0;
|
74
|
-
}
|
75
|
-
h1 {
|
76
|
-
font: 12pt Monospace;
|
77
|
-
text-align:center;
|
78
|
-
}
|
79
|
-
table {
|
80
|
-
font: 10pt Monospace;
|
81
|
-
margin-left:auto;
|
82
|
-
margin-right:auto;
|
83
|
-
text-align:right;
|
84
|
-
}
|
85
|
-
.page {
|
86
|
-
position:relative;
|
87
|
-
top: 20%;
|
88
|
-
# border-style:solid;
|
89
|
-
# border-width:5px;
|
90
|
-
width: 30%;
|
91
|
-
margin-left:auto;
|
92
|
-
margin-right:auto;
|
93
|
-
}
|
94
|
-
</style>
|
95
|
-
</head>
|
96
|
-
<body>
|
97
|
-
<div class=page>
|
98
|
-
<h1><span class="name">#{@probe.name}</span></h1>
|
99
|
-
<table>
|
100
|
-
<tr>
|
101
|
-
<td>messages per second:</td><td><span class="mps">#{@probe.mps}</span></td><td></td>
|
102
|
-
</tr>
|
103
|
-
<tr>
|
104
|
-
<td>uptime:</td><td><span class="uptime">#{@probe.uptime.to_i / 60}</span></td><td>mins</td>
|
105
|
-
</tr>
|
106
|
-
</table>
|
107
|
-
</div>
|
108
|
-
</body>
|
109
|
-
</html>
|
110
|
-
EOL
|
111
|
-
|
112
|
-
response.send_response
|
113
|
-
end
|
114
|
-
end
|
data/spec/eventmachine_helper.rb
DELETED
@@ -1,44 +0,0 @@
|
|
1
|
-
require 'fiber'
|
2
|
-
require 'eventmachine'
|
3
|
-
|
4
|
-
module Eventmachine
|
5
|
-
module SpecHelper
|
6
|
-
|
7
|
-
attr_reader :current_fiber
|
8
|
-
|
9
|
-
def em(&block)
|
10
|
-
EventMachine.run do
|
11
|
-
|
12
|
-
exception = nil
|
13
|
-
|
14
|
-
@current_fiber = Fiber.new do
|
15
|
-
begin
|
16
|
-
block.call
|
17
|
-
rescue Exception => exception
|
18
|
-
done
|
19
|
-
end
|
20
|
-
Fiber.yield
|
21
|
-
end
|
22
|
-
|
23
|
-
@current_fiber.resume
|
24
|
-
|
25
|
-
raise exception if exception
|
26
|
-
end
|
27
|
-
|
28
|
-
end
|
29
|
-
|
30
|
-
def finish
|
31
|
-
EventMachine.next_tick do
|
32
|
-
finish_current_fiber
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
private
|
37
|
-
|
38
|
-
def finish_current_fiber
|
39
|
-
EventMachine.stop_event_loop if EventMachine.reactor_running?
|
40
|
-
@current_fiber.resume if @current_fiber.alive?
|
41
|
-
end
|
42
|
-
|
43
|
-
end
|
44
|
-
end
|
@@ -1,64 +0,0 @@
|
|
1
|
-
require "spec_helper"
|
2
|
-
|
3
|
-
|
4
|
-
require 'pipeline_toolkit/amqp/abstract'
|
5
|
-
require 'pipeline_toolkit/amqp/reader'
|
6
|
-
require 'pipeline_toolkit/amqp/writer'
|
7
|
-
require "pipeline_toolkit/message_subscriber"
|
8
|
-
|
9
|
-
describe MessageSubscriber do
|
10
|
-
include Eventmachine::SpecHelper
|
11
|
-
|
12
|
-
# --exchange, -x <s>: The exchange name
|
13
|
-
# --type, -t <s>: The exchange type (direct, fanout or topic)
|
14
|
-
# (default: fanout)
|
15
|
-
# --passive, -s: If set to true, the server will not create the
|
16
|
-
# exchange if it does not already exist.
|
17
|
-
# --durable, -d: If set to true, the exchange will be marked as
|
18
|
-
# durable.
|
19
|
-
# --queue, -q <s>: The destination queue (queue:routing_key). Routing
|
20
|
-
# keys will be ignored if exchange type is not topic.
|
21
|
-
# --ack, -a: If this field is set to false the server does not
|
22
|
-
# expect acknowledgments for messages. (Default:
|
23
|
-
# true)
|
24
|
-
# --host, -h <s>: The AMQP message server host (default: localhost)
|
25
|
-
# --port, -p <i>: The AMQP message server port (default: 5672)
|
26
|
-
# --user, -u <s>: The AMQP message server username (default: guest)
|
27
|
-
# --pass, -w <s>: The AMQP message server username (default: guest)
|
28
|
-
# --vhost, -v <s>: The AMQP message server vhost (default: /)
|
29
|
-
# --name, -n <s>: The name used to describe the entire process chain
|
30
|
-
# --http-port, -o <i>: The port the HTTP monitoring server runs on.
|
31
|
-
# Default is a random port between 10000-11000
|
32
|
-
# --content-type, -c <s>: The type of response we are expecting from the HTTP
|
33
|
-
# monitoring server. (http, xml and json supported)
|
34
|
-
# (default: html)
|
35
|
-
# --dnssd, -b: Switches on DNSSD (i.e. Bonjour) for the monitoring
|
36
|
-
# interface
|
37
|
-
let(:options) do
|
38
|
-
{
|
39
|
-
:exchange => "raw",
|
40
|
-
:type => "topic",
|
41
|
-
:queue => "octane",
|
42
|
-
:ack => false,
|
43
|
-
:host => "localhost",
|
44
|
-
:port => 5672,
|
45
|
-
:user => "guest",
|
46
|
-
:pass => "guest",
|
47
|
-
:vhost => "/"
|
48
|
-
}
|
49
|
-
end
|
50
|
-
|
51
|
-
before(:each) do
|
52
|
-
@msg_subscriber = MessageSubscriber.new(options).start
|
53
|
-
done
|
54
|
-
end
|
55
|
-
|
56
|
-
after(:each) do
|
57
|
-
done
|
58
|
-
end
|
59
|
-
|
60
|
-
it "should not fail" do
|
61
|
-
true.should == true
|
62
|
-
end
|
63
|
-
|
64
|
-
end
|
data/spec/spec_helper.rb
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
2
|
-
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
3
|
-
|
4
|
-
unless RUBY_VERSION.to_f >= 1.9
|
5
|
-
puts "tmu-server only works under Ruby 1.9"
|
6
|
-
exit(1)
|
7
|
-
end
|
8
|
-
|
9
|
-
require 'pipeline_toolkit'
|
10
|
-
require 'rspec/core'
|
11
|
-
require File.dirname(__FILE__) + '/eventmachine_helper'
|
12
|
-
|
13
|
-
Rspec.configure do |config|
|
14
|
-
config.mock_framework = :rspec
|
15
|
-
end
|