pipeline_toolkit 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE.markdown +1 -1
- data/README.markdown +38 -19
- data/bin/msg_generator +3 -4
- data/bin/msg_push +3 -4
- data/bin/msg_sink +3 -7
- data/bin/msg_subscribe +3 -4
- data/lib/pipeline_toolkit.rb +5 -5
- data/lib/pipeline_toolkit/amqp/abstract.rb +81 -78
- data/lib/pipeline_toolkit/amqp/reader.rb +55 -52
- data/lib/pipeline_toolkit/amqp/writer.rb +42 -39
- data/lib/pipeline_toolkit/{commands/msg_generator/cli.rb → cli/msg_generator_cli.rb} +6 -5
- data/lib/pipeline_toolkit/{commands/msg_push/cli.rb → cli/msg_push_cli.rb} +16 -8
- data/lib/pipeline_toolkit/cli/msg_sink_cli.rb +28 -0
- data/lib/pipeline_toolkit/{commands/msg_subscribe/cli.rb → cli/msg_subscribe_cli.rb} +23 -14
- data/lib/pipeline_toolkit/cucumber.rb +5 -3
- data/lib/pipeline_toolkit/cucumber/amqp.rb +1 -1
- data/lib/pipeline_toolkit/cucumber/in_out_process.rb +50 -0
- data/lib/pipeline_toolkit/cucumber/machine.rb +44 -26
- data/lib/pipeline_toolkit/cucumber/machine_steps.rb +15 -9
- data/lib/pipeline_toolkit/cucumber/{msg_sub_machine.rb → msg_sub_process.rb} +5 -5
- data/lib/pipeline_toolkit/handlers/message_handler.rb +43 -39
- data/lib/pipeline_toolkit/message_coder.rb +38 -25
- data/lib/pipeline_toolkit/message_command.rb +167 -157
- data/lib/pipeline_toolkit/message_generator.rb +22 -18
- data/lib/pipeline_toolkit/message_pusher.rb +46 -47
- data/lib/pipeline_toolkit/message_sink.rb +9 -5
- data/lib/pipeline_toolkit/message_subscriber.rb +190 -183
- data/lib/pipeline_toolkit/monitoring/monitor_server.rb +64 -109
- data/lib/pipeline_toolkit/util/hash_ext.rb +8 -0
- data/lib/pipeline_toolkit/util/indifferent_hash.rb +10 -0
- data/lib/pipeline_toolkit/{socket_util.rb → util/socket_util.rb} +1 -1
- metadata +25 -115
- data/lib/pipeline_toolkit/commands/msg_sink/cli.rb +0 -41
- data/lib/pipeline_toolkit/open_hash.rb +0 -24
@@ -2,8 +2,8 @@
|
|
2
2
|
module PipelineToolkit
|
3
3
|
module Cucumber
|
4
4
|
|
5
|
-
# Provides an interface for running and interacting with a
|
6
|
-
class
|
5
|
+
# Provides an interface for running and interacting with a msg_subscribe process.
|
6
|
+
class MsgSubProcess < InOutProcess
|
7
7
|
|
8
8
|
def run(command)
|
9
9
|
super(command)
|
@@ -13,13 +13,13 @@ module PipelineToolkit
|
|
13
13
|
def setup_pipe
|
14
14
|
# first message should be system one
|
15
15
|
msg = get_messages(1).first
|
16
|
-
raise "Expecting first message to be system message, got instead: #{msg}" if msg
|
16
|
+
raise "Expecting first message to be system message, got instead: #{msg.inspect}" if msg[:msg_type] != "system"
|
17
17
|
@system_setup = msg
|
18
|
-
@ack_pipe = File.open(
|
18
|
+
@ack_pipe = File.open(msg[:sys_pipe], "w")
|
19
19
|
end
|
20
20
|
|
21
21
|
def acknowledge_message(message)
|
22
|
-
ack_message = {:msg_type =>
|
22
|
+
ack_message = {:msg_type => "ack", :ack_id => message[:ack_id]}
|
23
23
|
@ack_pipe.syswrite(MessageCoder.encode(ack_message) << "\n")
|
24
24
|
end
|
25
25
|
|
@@ -1,51 +1,55 @@
|
|
1
|
-
|
1
|
+
|
2
|
+
module PipelineToolkit
|
3
|
+
module Handlers # :nodoc:
|
2
4
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
5
|
+
##
|
6
|
+
# The Message Process handler that is called by Event Machine reactor loop
|
7
|
+
# when it receives a new message from IO watcher
|
8
|
+
#
|
9
|
+
module MessageHandler
|
8
10
|
|
9
|
-
|
11
|
+
MAX_BUFFER = 3_000_000
|
10
12
|
|
11
|
-
|
13
|
+
attr_reader :options
|
12
14
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
15
|
+
##
|
16
|
+
# Initialize a new instance
|
17
|
+
#
|
18
|
+
# @param msg_command<MessageCommand> An instance of the object to delegate the message handling to
|
19
|
+
# @param options<Hash> An options hash
|
20
|
+
#
|
21
|
+
def initialize(target, options = {})
|
22
|
+
@options = options
|
23
|
+
@target = target
|
24
|
+
end
|
23
25
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
26
|
+
##
|
27
|
+
# The callback method that EventMachine calls as soon as a new message arrives
|
28
|
+
#
|
29
|
+
def notify_readable
|
30
|
+
DefaultLogger.debug("Handlers::MessageHandler#notify_readable") if options[:env] == "development"
|
29
31
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
32
|
+
# Grab everything from buffer, in case several messages have built up.
|
33
|
+
# NB: Can't use gets or read because they block when reaching EOF.
|
34
|
+
@buffer ||= BufferedTokenizer.new
|
35
|
+
data = @io.read_nonblock(MAX_BUFFER)
|
34
36
|
|
35
|
-
|
36
|
-
|
37
|
-
|
37
|
+
@buffer.extract(data).each do |line|
|
38
|
+
receive_line(line)
|
39
|
+
end
|
38
40
|
|
39
|
-
|
40
|
-
|
41
|
-
|
41
|
+
rescue StandardError => e # rescued here because main thread does not seem to see it
|
42
|
+
DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
|
43
|
+
end
|
42
44
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
45
|
+
def receive_line(message_coded)
|
46
|
+
DefaultLogger.debug("Raw message: #{message_coded}") if options[:env] == "development"
|
47
|
+
message = MessageCoder.decode(message_coded.chomp)
|
48
|
+
DefaultLogger.debug("Message: #{message.inspect}") if options[:env] == "development"
|
49
|
+
@target.process(message)
|
50
|
+
end
|
49
51
|
|
52
|
+
end
|
53
|
+
|
50
54
|
end
|
51
55
|
end
|
@@ -1,29 +1,42 @@
|
|
1
|
-
|
2
|
-
# Encode and decode messages using Marshal
|
3
|
-
#
|
4
|
-
class MessageCoder
|
5
|
-
|
6
|
-
##
|
7
|
-
# Encode the hash message
|
8
|
-
#
|
9
|
-
# @param message<Hash> The message as a hash
|
10
|
-
#
|
11
|
-
def self.encode(message)
|
12
|
-
# NB: Using Marshal here because it's 9-10x faster than to_yaml
|
13
|
-
# See http://gist.github.com/190849
|
14
|
-
str = Marshal.dump(message)
|
15
|
-
str.gsub!("\n", '--\\n')
|
16
|
-
str
|
17
|
-
end
|
1
|
+
require 'msgpack'
|
18
2
|
|
3
|
+
module PipelineToolkit
|
19
4
|
##
|
20
|
-
#
|
5
|
+
# Encode and decode messages using Marshal
|
21
6
|
#
|
22
|
-
|
23
|
-
#
|
24
|
-
def self.decode(str)
|
25
|
-
str.gsub!('--\\n', "\n")
|
26
|
-
Marshal.load(str)
|
27
|
-
end
|
7
|
+
class MessageCoder
|
28
8
|
|
29
|
-
|
9
|
+
##
|
10
|
+
# Encode the hash message
|
11
|
+
#
|
12
|
+
# @param message<Hash> The message as a hash
|
13
|
+
#
|
14
|
+
def self.encode(message)
|
15
|
+
# NB: Using Msgpack because it's 9-10x faster than altnatives
|
16
|
+
# See http://gist.github.com/190849
|
17
|
+
MessagePack.pack(message)
|
18
|
+
# str.gsub!("\n", "?n")
|
19
|
+
# str.gsub!("\r", "?r")
|
20
|
+
end
|
21
|
+
|
22
|
+
##
|
23
|
+
# Decode the binary string into a hash
|
24
|
+
#
|
25
|
+
# @param str<String> Binary string
|
26
|
+
#
|
27
|
+
def self.decode(str)
|
28
|
+
str.chomp!
|
29
|
+
obj = MessagePack.unpack(str)
|
30
|
+
|
31
|
+
obj = case obj
|
32
|
+
when Hash
|
33
|
+
# makes code a little more readable
|
34
|
+
IndifferentHash.new(obj)
|
35
|
+
else
|
36
|
+
obj
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
@@ -1,181 +1,191 @@
|
|
1
|
-
require "rubygems"
|
2
1
|
require "eventmachine"
|
3
2
|
|
4
|
-
|
5
|
-
# Provide abstract base functionality for pipeline machines.
|
6
|
-
#
|
7
|
-
# Required, To implement a your own machine, override the following methods:
|
8
|
-
#
|
9
|
-
# * include MessageCommand
|
10
|
-
# * {#process_standard} Handle the messages
|
11
|
-
#
|
12
|
-
# Optionally, your machine can implement the following:
|
13
|
-
#
|
14
|
-
# * {#initialize_machine} Setup your machine
|
15
|
-
# * {#report_back} Notify on the acknowledgement pipe about your worker
|
16
|
-
#
|
17
|
-
# @example An example of a pipeline machine
|
18
|
-
#
|
19
|
-
# class DefaultMachine
|
20
|
-
# include MessageCommand
|
21
|
-
#
|
22
|
-
# def report_back
|
23
|
-
#
|
24
|
-
# end
|
25
|
-
#
|
26
|
-
# def process_standard(message)
|
27
|
-
# # do some work ie. transform the message or apply some business rules
|
28
|
-
# # raise exception if processing or business rule fails
|
29
|
-
# pass_on(message)
|
30
|
-
# rescue
|
31
|
-
# # handle the storage of the message that could not be processed
|
32
|
-
# # and acknowledge the AMQP server server
|
33
|
-
# acknowledge(message)
|
34
|
-
# end
|
35
|
-
# end
|
36
|
-
#
|
37
|
-
# DefaultMachine.new.start
|
38
|
-
#
|
39
|
-
#
|
40
|
-
# @abstract
|
41
|
-
#
|
42
|
-
module MessageCommand
|
3
|
+
module PipelineToolkit
|
43
4
|
|
44
5
|
##
|
45
|
-
#
|
6
|
+
# Provide abstract base functionality for pipeline machines.
|
46
7
|
#
|
47
|
-
|
48
|
-
|
49
|
-
##
|
50
|
-
# Initializes a new instance. A message command can only
|
51
|
-
# be initialized through the implementation instance.
|
52
|
-
#
|
53
|
-
# @param options<Hash> An options hash.
|
8
|
+
# Required, To implement a your own machine, override the following methods:
|
54
9
|
#
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
#
|
62
|
-
# and starts watching the STDIN for new messages
|
10
|
+
# * include MessageCommand
|
11
|
+
# * {#process_standard} Handle the messages
|
12
|
+
#
|
13
|
+
# Optionally, your machine can implement the following:
|
14
|
+
#
|
15
|
+
# * {#initialize_machine} Setup your machine
|
16
|
+
# * {#description} To change the name that is used to describe your machine in the montitoring interface
|
63
17
|
#
|
64
|
-
|
65
|
-
|
18
|
+
# @example An example of a pipeline machine
|
19
|
+
#
|
20
|
+
# class DefaultMachine
|
21
|
+
# include MessageCommand
|
22
|
+
#
|
23
|
+
# def process_standard(message)
|
24
|
+
# # do some work ie. transform the message or apply some business rules
|
25
|
+
# # raise exception if processing or business rule fails
|
26
|
+
# pass_on(message)
|
27
|
+
# rescue
|
28
|
+
# # handle the storage of the message that could not be processed
|
29
|
+
# # and acknowledge the AMQP server server
|
30
|
+
# acknowledge(message)
|
31
|
+
# end
|
32
|
+
# end
|
33
|
+
#
|
34
|
+
# DefaultMachine.new.start
|
35
|
+
#
|
36
|
+
#
|
37
|
+
# @abstract
|
38
|
+
#
|
39
|
+
module MessageCommand
|
40
|
+
|
41
|
+
##
|
42
|
+
# The options the machine should use
|
43
|
+
#
|
44
|
+
attr_reader :options
|
45
|
+
|
46
|
+
##
|
47
|
+
# Initializes a new instance. A message command can only
|
48
|
+
# be initialized through the implementation instance.
|
49
|
+
#
|
50
|
+
# @param options<Hash> An options hash.
|
51
|
+
#
|
52
|
+
def initialize(options = {})
|
53
|
+
DefaultLogger.init_logger(options)
|
54
|
+
@options = options
|
55
|
+
end
|
56
|
+
|
57
|
+
##
|
58
|
+
# Starts the machine up, ie. creates a new eventmachine reactor loop
|
59
|
+
# and starts watching the STDIN for new messages
|
60
|
+
#
|
61
|
+
def start
|
62
|
+
DefaultLogger.debug("MessageCommand#start") if @options[:env] == "development"
|
66
63
|
|
67
|
-
|
68
|
-
|
64
|
+
Signal.trap('INT') { EM.stop }
|
65
|
+
Signal.trap('TERM') { EM.stop }
|
69
66
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
67
|
+
begin
|
68
|
+
EM.run do
|
69
|
+
conn = EM.watch($stdin, Handlers::MessageHandler, self, options)
|
70
|
+
initialize_machine
|
74
71
|
|
75
|
-
|
76
|
-
|
72
|
+
# must call this to setup callback to notify_readable
|
73
|
+
conn.notify_readable = true
|
74
|
+
end
|
75
|
+
rescue StandardError => e
|
76
|
+
DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
|
77
|
+
raise e
|
78
|
+
ensure
|
79
|
+
shutdown
|
77
80
|
end
|
78
|
-
rescue StandardError => e
|
79
|
-
DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
|
80
|
-
raise e
|
81
|
-
ensure
|
82
|
-
shutdown
|
83
81
|
end
|
84
|
-
end
|
85
82
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
83
|
+
##
|
84
|
+
# Stops the machine
|
85
|
+
#
|
86
|
+
def shutdown
|
87
|
+
DefaultLogger.info("Shutting down #{self.class.name}")
|
88
|
+
DefaultLogger.info "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
|
89
|
+
@ack_pipe.close if @ack_pipe
|
90
|
+
end
|
94
91
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
92
|
+
##
|
93
|
+
# Callback for the Handlers::MessageHandler when it receives a message
|
94
|
+
#
|
95
|
+
# @param message<Hash> The decoded message
|
96
|
+
#
|
97
|
+
def process(message)
|
98
|
+
DefaultLogger.debug("MessageCommand#process(message)") if options[:env] == "development"
|
99
|
+
if message[:msg_type] == "system"
|
100
|
+
process_system(message)
|
101
|
+
else
|
102
|
+
process_standard(message)
|
103
|
+
end
|
106
104
|
end
|
107
|
-
end
|
108
105
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
106
|
+
##
|
107
|
+
# Notify the AMQP server server that we've handled the message
|
108
|
+
#
|
109
|
+
# @param message<Hash> The message to acknowledge
|
110
|
+
#
|
111
|
+
def acknowledge(message)
|
112
|
+
DefaultLogger.debug("MessageCommand#acknowledge(message)") if options[:env] == "development"
|
113
|
+
ack_message = {:msg_type => "ack", :ack_id => message[:ack_id]}
|
114
|
+
write_to_pipe(ack_message, @ack_pipe)
|
115
|
+
end
|
119
116
|
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
117
|
+
##
|
118
|
+
# Pass the message to the next machine in the pipeline by writing the message to the STDOUT.
|
119
|
+
#
|
120
|
+
# @param message<Hash> The message to pass on
|
121
|
+
#
|
122
|
+
def pass_on(message)
|
123
|
+
DefaultLogger.debug("MessageCommand#pass_on(message)") if options[:env] == "development"
|
124
|
+
write_to_pipe(message)
|
125
|
+
end
|
129
126
|
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
127
|
+
##
|
128
|
+
# @abstract Override in the class the MessageCommand is included into. Provides a
|
129
|
+
# chance to initialize any code that needs to take place once the EventMachine loop
|
130
|
+
# has started.
|
131
|
+
#
|
132
|
+
def initialize_machine
|
133
|
+
DefaultLogger.debug("MessageCommand#initialize_machine") if options[:env] == "development"
|
134
|
+
# Implemented in class that includes me
|
135
|
+
end
|
139
136
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
#
|
147
|
-
|
137
|
+
##
|
138
|
+
# @abstract In your machine implementation you need to override the {#process_standard} method.
|
139
|
+
# Processes a message. This method must call {#pass_on} if the message was handled successfully,
|
140
|
+
# or if the handling of the message fails for whatever reason, you need to decide how you are going
|
141
|
+
# to handle the failure of the message (write message to error log | queue | etc.) and then
|
142
|
+
# acknowledge the message by calling {acknowledge}. Acknowledging the message mean that the
|
143
|
+
# message has been dealt with. All messages must be acknowledged by at least one worker in a pipeline.
|
144
|
+
#
|
145
|
+
def process_standard(message)
|
146
|
+
DefaultLogger.debug("MessageCommand#process_standard(message)") if options[:env] == "development"
|
147
|
+
# Implemented in class that includes me
|
148
|
+
pass_on(message)
|
149
|
+
end
|
150
|
+
|
151
|
+
##
|
152
|
+
# A string describing the machine. The description is used to describe the machine in the
|
153
|
+
# monitoring interface. Default implement is to return the name of the Machine's class. This
|
154
|
+
# can be overriden within idividual machines if you wish to change it.
|
155
|
+
#
|
156
|
+
def description
|
157
|
+
# NB: Regex removes modules from description
|
158
|
+
self.class.name.scan(/(\w+)$/).first
|
159
|
+
end
|
148
160
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
161
|
+
##
|
162
|
+
# Process the first system message to notify machines down the pipeline which named pipe to
|
163
|
+
# use for acknowledgements.
|
164
|
+
#
|
165
|
+
# @param message<Hash> The system message
|
166
|
+
#
|
167
|
+
def process_system(message)
|
168
|
+
DefaultLogger.debug("MessageCommand#process_system(message)") if options[:env] == "development"
|
169
|
+
options[:ack] = message[:ack] # inherit setting from upstream
|
170
|
+
@ack_pipe = File.open(message[:sys_pipe], "w") # open ack pipe (needs to already exist)
|
171
|
+
send_description
|
172
|
+
pass_on(message)
|
173
|
+
end
|
174
|
+
|
175
|
+
def write_to_pipe(message, pipe=$stdout)
|
176
|
+
pipe.syswrite(MessageCoder.encode(message) << "\n")
|
177
|
+
end
|
178
|
+
|
179
|
+
##
|
180
|
+
# Can be overriden in the class the MessageCommand is included into. Called
|
181
|
+
# when the acknowledgement named pipe has been established.
|
182
|
+
#
|
183
|
+
def send_description
|
184
|
+
message = { :msg_type => "pipe_desc", :description => self.description }
|
185
|
+
write_to_pipe(message, @ack_pipe)
|
186
|
+
end
|
162
187
|
|
163
|
-
##
|
164
|
-
# Process the first system message to notify machines down the pipeline which named pipe to
|
165
|
-
# use for acknowledgements.
|
166
|
-
#
|
167
|
-
# @param message<Hash> The system message
|
168
|
-
#
|
169
|
-
def process_system(message)
|
170
|
-
DefaultLogger.debug("MessageCommand#process_system(message)") if options[:env] == "development"
|
171
|
-
options[:ack] = message[:ack] # inherit setting from upstream
|
172
|
-
@ack_pipe = File.open(message[:sys_pipe], "w") # open ack pipe (needs to already exist)
|
173
|
-
report_back
|
174
|
-
pass_on(message)
|
175
|
-
end
|
176
188
|
|
177
|
-
def write_to_pipe(message, pipe=$stdout)
|
178
|
-
pipe.syswrite(MessageCoder.encode(message) << "\n")
|
179
189
|
end
|
180
190
|
|
181
|
-
end
|
191
|
+
end
|