pipeline_toolkit 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. data/LICENSE.markdown +1 -1
  2. data/README.markdown +38 -19
  3. data/bin/msg_generator +3 -4
  4. data/bin/msg_push +3 -4
  5. data/bin/msg_sink +3 -7
  6. data/bin/msg_subscribe +3 -4
  7. data/lib/pipeline_toolkit.rb +5 -5
  8. data/lib/pipeline_toolkit/amqp/abstract.rb +81 -78
  9. data/lib/pipeline_toolkit/amqp/reader.rb +55 -52
  10. data/lib/pipeline_toolkit/amqp/writer.rb +42 -39
  11. data/lib/pipeline_toolkit/{commands/msg_generator/cli.rb → cli/msg_generator_cli.rb} +6 -5
  12. data/lib/pipeline_toolkit/{commands/msg_push/cli.rb → cli/msg_push_cli.rb} +16 -8
  13. data/lib/pipeline_toolkit/cli/msg_sink_cli.rb +28 -0
  14. data/lib/pipeline_toolkit/{commands/msg_subscribe/cli.rb → cli/msg_subscribe_cli.rb} +23 -14
  15. data/lib/pipeline_toolkit/cucumber.rb +5 -3
  16. data/lib/pipeline_toolkit/cucumber/amqp.rb +1 -1
  17. data/lib/pipeline_toolkit/cucumber/in_out_process.rb +50 -0
  18. data/lib/pipeline_toolkit/cucumber/machine.rb +44 -26
  19. data/lib/pipeline_toolkit/cucumber/machine_steps.rb +15 -9
  20. data/lib/pipeline_toolkit/cucumber/{msg_sub_machine.rb → msg_sub_process.rb} +5 -5
  21. data/lib/pipeline_toolkit/handlers/message_handler.rb +43 -39
  22. data/lib/pipeline_toolkit/message_coder.rb +38 -25
  23. data/lib/pipeline_toolkit/message_command.rb +167 -157
  24. data/lib/pipeline_toolkit/message_generator.rb +22 -18
  25. data/lib/pipeline_toolkit/message_pusher.rb +46 -47
  26. data/lib/pipeline_toolkit/message_sink.rb +9 -5
  27. data/lib/pipeline_toolkit/message_subscriber.rb +190 -183
  28. data/lib/pipeline_toolkit/monitoring/monitor_server.rb +64 -109
  29. data/lib/pipeline_toolkit/util/hash_ext.rb +8 -0
  30. data/lib/pipeline_toolkit/util/indifferent_hash.rb +10 -0
  31. data/lib/pipeline_toolkit/{socket_util.rb → util/socket_util.rb} +1 -1
  32. metadata +25 -115
  33. data/lib/pipeline_toolkit/commands/msg_sink/cli.rb +0 -41
  34. data/lib/pipeline_toolkit/open_hash.rb +0 -24
@@ -2,8 +2,8 @@
2
2
  module PipelineToolkit
3
3
  module Cucumber
4
4
 
5
- # Provides an interface for running and interacting with a machine process.
6
- class MsgSubMachine < Machine
5
+ # Provides an interface for running and interacting with a msg_subscribe process.
6
+ class MsgSubProcess < InOutProcess
7
7
 
8
8
  def run(command)
9
9
  super(command)
@@ -13,13 +13,13 @@ module PipelineToolkit
13
13
  def setup_pipe
14
14
  # first message should be system one
15
15
  msg = get_messages(1).first
16
- raise "Expecting first message to be system message, got instead: #{msg}" if msg.msg_type != :system
16
+ raise "Expecting first message to be system message, got instead: #{msg.inspect}" if msg[:msg_type] != "system"
17
17
  @system_setup = msg
18
- @ack_pipe = File.open(@system_setup[:sys_pipe], "w")
18
+ @ack_pipe = File.open(msg[:sys_pipe], "w")
19
19
  end
20
20
 
21
21
  def acknowledge_message(message)
22
- ack_message = {:msg_type => :ack, :ack_id => message[:ack_id]}
22
+ ack_message = {:msg_type => "ack", :ack_id => message[:ack_id]}
23
23
  @ack_pipe.syswrite(MessageCoder.encode(ack_message) << "\n")
24
24
  end
25
25
 
@@ -1,51 +1,55 @@
1
- module Handlers # :nodoc:
1
+
2
+ module PipelineToolkit
3
+ module Handlers # :nodoc:
2
4
 
3
- ##
4
- # The Message Process handler that is called by Event Machine reactor loop
5
- # when it receives a new message from IO watcher
6
- #
7
- module MessageHandler
5
+ ##
6
+ # The Message Process handler that is called by Event Machine reactor loop
7
+ # when it receives a new message from IO watcher
8
+ #
9
+ module MessageHandler
8
10
 
9
- MAX_BUFFER = 3_000_000
11
+ MAX_BUFFER = 3_000_000
10
12
 
11
- attr_reader :options
13
+ attr_reader :options
12
14
 
13
- ##
14
- # Initialize a new instance
15
- #
16
- # @param msg_command<MessageCommand> An instance of the object to delegate the message handling to
17
- # @param options<Hash> An options hash
18
- #
19
- def initialize(target, options = {})
20
- @options = options
21
- @target = target
22
- end
15
+ ##
16
+ # Initialize a new instance
17
+ #
18
+ # @param msg_command<MessageCommand> An instance of the object to delegate the message handling to
19
+ # @param options<Hash> An options hash
20
+ #
21
+ def initialize(target, options = {})
22
+ @options = options
23
+ @target = target
24
+ end
23
25
 
24
- ##
25
- # The callback method that EventMachine calls as soon as a new message arrives
26
- #
27
- def notify_readable
28
- DefaultLogger.debug("Handlers::MessageHandler#notify_readable") if options[:env] == "development"
26
+ ##
27
+ # The callback method that EventMachine calls as soon as a new message arrives
28
+ #
29
+ def notify_readable
30
+ DefaultLogger.debug("Handlers::MessageHandler#notify_readable") if options[:env] == "development"
29
31
 
30
- # Grab everything from buffer, in case several messages have built up.
31
- # NB: Can't use gets or read because they block when reaching EOF.
32
- @buffer ||= BufferedTokenizer.new
33
- data = @io.read_nonblock(MAX_BUFFER)
32
+ # Grab everything from buffer, in case several messages have built up.
33
+ # NB: Can't use gets or read because they block when reaching EOF.
34
+ @buffer ||= BufferedTokenizer.new
35
+ data = @io.read_nonblock(MAX_BUFFER)
34
36
 
35
- @buffer.extract(data).each do |line|
36
- receive_line(line)
37
- end
37
+ @buffer.extract(data).each do |line|
38
+ receive_line(line)
39
+ end
38
40
 
39
- rescue StandardError => e # rescued here because main thread does not seem to see it
40
- DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
41
- end
41
+ rescue StandardError => e # rescued here because main thread does not seem to see it
42
+ DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
43
+ end
42
44
 
43
- def receive_line(message_coded)
44
- DefaultLogger.debug("Raw message: #{message_coded}") if options[:env] == "development"
45
- message = MessageCoder.decode(message_coded.chomp)
46
- DefaultLogger.debug("Message: #{message}") if options[:env] == "development"
47
- @target.process(message)
48
- end
45
+ def receive_line(message_coded)
46
+ DefaultLogger.debug("Raw message: #{message_coded}") if options[:env] == "development"
47
+ message = MessageCoder.decode(message_coded.chomp)
48
+ DefaultLogger.debug("Message: #{message.inspect}") if options[:env] == "development"
49
+ @target.process(message)
50
+ end
49
51
 
52
+ end
53
+
50
54
  end
51
55
  end
@@ -1,29 +1,42 @@
1
- ##
2
- # Encode and decode messages using Marshal
3
- #
4
- class MessageCoder
5
-
6
- ##
7
- # Encode the hash message
8
- #
9
- # @param message<Hash> The message as a hash
10
- #
11
- def self.encode(message)
12
- # NB: Using Marshal here because it's 9-10x faster than to_yaml
13
- # See http://gist.github.com/190849
14
- str = Marshal.dump(message)
15
- str.gsub!("\n", '--\\n')
16
- str
17
- end
1
+ require 'msgpack'
18
2
 
3
+ module PipelineToolkit
19
4
  ##
20
- # Decode the binary string into a hash
5
+ # Encode and decode messages using Marshal
21
6
  #
22
- # @param str<String> Binary string
23
- #
24
- def self.decode(str)
25
- str.gsub!('--\\n', "\n")
26
- Marshal.load(str)
27
- end
7
+ class MessageCoder
28
8
 
29
- end
9
+ ##
10
+ # Encode the hash message
11
+ #
12
+ # @param message<Hash> The message as a hash
13
+ #
14
+ def self.encode(message)
15
+ # NB: Using Msgpack because it's 9-10x faster than altnatives
16
+ # See http://gist.github.com/190849
17
+ MessagePack.pack(message)
18
+ # str.gsub!("\n", "?n")
19
+ # str.gsub!("\r", "?r")
20
+ end
21
+
22
+ ##
23
+ # Decode the binary string into a hash
24
+ #
25
+ # @param str<String> Binary string
26
+ #
27
+ def self.decode(str)
28
+ str.chomp!
29
+ obj = MessagePack.unpack(str)
30
+
31
+ obj = case obj
32
+ when Hash
33
+ # makes code a little more readable
34
+ IndifferentHash.new(obj)
35
+ else
36
+ obj
37
+ end
38
+ end
39
+
40
+ end
41
+
42
+ end
@@ -1,181 +1,191 @@
1
- require "rubygems"
2
1
  require "eventmachine"
3
2
 
4
- ##
5
- # Provide abstract base functionality for pipeline machines.
6
- #
7
- # Required, To implement a your own machine, override the following methods:
8
- #
9
- # * include MessageCommand
10
- # * {#process_standard} Handle the messages
11
- #
12
- # Optionally, your machine can implement the following:
13
- #
14
- # * {#initialize_machine} Setup your machine
15
- # * {#report_back} Notify on the acknowledgement pipe about your worker
16
- #
17
- # @example An example of a pipeline machine
18
- #
19
- # class DefaultMachine
20
- # include MessageCommand
21
- #
22
- # def report_back
23
- #
24
- # end
25
- #
26
- # def process_standard(message)
27
- # # do some work ie. transform the message or apply some business rules
28
- # # raise exception if processing or business rule fails
29
- # pass_on(message)
30
- # rescue
31
- # # handle the storage of the message that could not be processed
32
- # # and acknowledge the AMQP server server
33
- # acknowledge(message)
34
- # end
35
- # end
36
- #
37
- # DefaultMachine.new.start
38
- #
39
- #
40
- # @abstract
41
- #
42
- module MessageCommand
3
+ module PipelineToolkit
43
4
 
44
5
  ##
45
- # The options the machine should use
6
+ # Provide abstract base functionality for pipeline machines.
46
7
  #
47
- attr_reader :options
48
-
49
- ##
50
- # Initializes a new instance. A message command can only
51
- # be initialized through the implementation instance.
52
- #
53
- # @param options<Hash> An options hash.
8
+ # Required, To implement a your own machine, override the following methods:
54
9
  #
55
- def initialize(options = {})
56
- DefaultLogger.init_logger(options)
57
- @options = options
58
- end
59
-
60
- ##
61
- # Starts the machine up, ie. creates a new eventmachine reactor loop
62
- # and starts watching the STDIN for new messages
10
+ # * include MessageCommand
11
+ # * {#process_standard} Handle the messages
12
+ #
13
+ # Optionally, your machine can implement the following:
14
+ #
15
+ # * {#initialize_machine} Setup your machine
16
+ # * {#description} To change the name that is used to describe your machine in the montitoring interface
63
17
  #
64
- def start
65
- DefaultLogger.debug("MessageCommand#start") if @options[:env] == "development"
18
+ # @example An example of a pipeline machine
19
+ #
20
+ # class DefaultMachine
21
+ # include MessageCommand
22
+ #
23
+ # def process_standard(message)
24
+ # # do some work ie. transform the message or apply some business rules
25
+ # # raise exception if processing or business rule fails
26
+ # pass_on(message)
27
+ # rescue
28
+ # # handle the storage of the message that could not be processed
29
+ # # and acknowledge the AMQP server server
30
+ # acknowledge(message)
31
+ # end
32
+ # end
33
+ #
34
+ # DefaultMachine.new.start
35
+ #
36
+ #
37
+ # @abstract
38
+ #
39
+ module MessageCommand
40
+
41
+ ##
42
+ # The options the machine should use
43
+ #
44
+ attr_reader :options
45
+
46
+ ##
47
+ # Initializes a new instance. A message command can only
48
+ # be initialized through the implementation instance.
49
+ #
50
+ # @param options<Hash> An options hash.
51
+ #
52
+ def initialize(options = {})
53
+ DefaultLogger.init_logger(options)
54
+ @options = options
55
+ end
56
+
57
+ ##
58
+ # Starts the machine up, ie. creates a new eventmachine reactor loop
59
+ # and starts watching the STDIN for new messages
60
+ #
61
+ def start
62
+ DefaultLogger.debug("MessageCommand#start") if @options[:env] == "development"
66
63
 
67
- Signal.trap('INT') { EM.stop }
68
- Signal.trap('TERM') { EM.stop }
64
+ Signal.trap('INT') { EM.stop }
65
+ Signal.trap('TERM') { EM.stop }
69
66
 
70
- begin
71
- EM.run do
72
- conn = EM.watch($stdin, Handlers::MessageHandler, self, options)
73
- initialize_machine
67
+ begin
68
+ EM.run do
69
+ conn = EM.watch($stdin, Handlers::MessageHandler, self, options)
70
+ initialize_machine
74
71
 
75
- # must call this to setup callback to notify_readable
76
- conn.notify_readable = true
72
+ # must call this to setup callback to notify_readable
73
+ conn.notify_readable = true
74
+ end
75
+ rescue StandardError => e
76
+ DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
77
+ raise e
78
+ ensure
79
+ shutdown
77
80
  end
78
- rescue StandardError => e
79
- DefaultLogger.error("#{e.class.name}: #{e.message}\n" << e.backtrace.join("\n"))
80
- raise e
81
- ensure
82
- shutdown
83
81
  end
84
- end
85
82
 
86
- ##
87
- # Stops the machine
88
- #
89
- def shutdown
90
- DefaultLogger.info("Shutting down #{self.class.name}")
91
- DefaultLogger.info "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
92
- @ack_pipe.close if @ack_pipe
93
- end
83
+ ##
84
+ # Stops the machine
85
+ #
86
+ def shutdown
87
+ DefaultLogger.info("Shutting down #{self.class.name}")
88
+ DefaultLogger.info "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
89
+ @ack_pipe.close if @ack_pipe
90
+ end
94
91
 
95
- ##
96
- # Callback for the Handlers::MessageHandler when it receives a message
97
- #
98
- # @param message<Hash> The decoded message
99
- #
100
- def process(message)
101
- DefaultLogger.debug("MessageCommand#process(message)") if options[:env] == "development"
102
- if message[:msg_type] == :system
103
- process_system(message)
104
- else
105
- process_standard(message)
92
+ ##
93
+ # Callback for the Handlers::MessageHandler when it receives a message
94
+ #
95
+ # @param message<Hash> The decoded message
96
+ #
97
+ def process(message)
98
+ DefaultLogger.debug("MessageCommand#process(message)") if options[:env] == "development"
99
+ if message[:msg_type] == "system"
100
+ process_system(message)
101
+ else
102
+ process_standard(message)
103
+ end
106
104
  end
107
- end
108
105
 
109
- ##
110
- # Notify the AMQP server server that we've handled the message
111
- #
112
- # @param message<Hash> The message to acknowledge
113
- #
114
- def acknowledge(message)
115
- DefaultLogger.debug("MessageCommand#acknowledge(message)") if options[:env] == "development"
116
- ack_message = {:msg_type => :ack, :ack_id => message[:ack_id]}
117
- write_to_pipe(ack_message, @ack_pipe)
118
- end
106
+ ##
107
+ # Notify the AMQP server server that we've handled the message
108
+ #
109
+ # @param message<Hash> The message to acknowledge
110
+ #
111
+ def acknowledge(message)
112
+ DefaultLogger.debug("MessageCommand#acknowledge(message)") if options[:env] == "development"
113
+ ack_message = {:msg_type => "ack", :ack_id => message[:ack_id]}
114
+ write_to_pipe(ack_message, @ack_pipe)
115
+ end
119
116
 
120
- ##
121
- # Pass the message to the next machine in the pipeline by writing the message to the STDOUT.
122
- #
123
- # @param message<Hash> The message to pass on
124
- #
125
- def pass_on(message)
126
- DefaultLogger.debug("MessageCommand#pass_on(message)") if options[:env] == "development"
127
- write_to_pipe(message)
128
- end
117
+ ##
118
+ # Pass the message to the next machine in the pipeline by writing the message to the STDOUT.
119
+ #
120
+ # @param message<Hash> The message to pass on
121
+ #
122
+ def pass_on(message)
123
+ DefaultLogger.debug("MessageCommand#pass_on(message)") if options[:env] == "development"
124
+ write_to_pipe(message)
125
+ end
129
126
 
130
- ##
131
- # @abstract Override in the class the MessageCommand is included into. Provides a
132
- # chance to initialize any code that needs to take place once the EventMachine loop
133
- # has started.
134
- #
135
- def initialize_machine
136
- DefaultLogger.debug("MessageCommand#initialize_machine") if options[:env] == "development"
137
- # Implemented in class that includes me
138
- end
127
+ ##
128
+ # @abstract Override in the class the MessageCommand is included into. Provides a
129
+ # chance to initialize any code that needs to take place once the EventMachine loop
130
+ # has started.
131
+ #
132
+ def initialize_machine
133
+ DefaultLogger.debug("MessageCommand#initialize_machine") if options[:env] == "development"
134
+ # Implemented in class that includes me
135
+ end
139
136
 
140
- ##
141
- # @abstract Can be overriden in the class the MessageCommand is included into. Called
142
- # when the acknowledgement named pipe has been established.
143
- #
144
- def report_back
145
- DefaultLogger.debug("MessageCommand#report_back") if options[:env] == "development"
146
- # Implemented in class that includes me
147
- end
137
+ ##
138
+ # @abstract In your machine implementation you need to override the {#process_standard} method.
139
+ # Processes a message. This method must call {#pass_on} if the message was handled successfully,
140
+ # or if the handling of the message fails for whatever reason, you need to decide how you are going
141
+ # to handle the failure of the message (write message to error log | queue | etc.) and then
142
+ # acknowledge the message by calling {acknowledge}. Acknowledging the message mean that the
143
+ # message has been dealt with. All messages must be acknowledged by at least one worker in a pipeline.
144
+ #
145
+ def process_standard(message)
146
+ DefaultLogger.debug("MessageCommand#process_standard(message)") if options[:env] == "development"
147
+ # Implemented in class that includes me
148
+ pass_on(message)
149
+ end
150
+
151
+ ##
152
+ # A string describing the machine. The description is used to describe the machine in the
153
+ # monitoring interface. Default implement is to return the name of the Machine's class. This
154
+ # can be overriden within idividual machines if you wish to change it.
155
+ #
156
+ def description
157
+ # NB: Regex removes modules from description
158
+ self.class.name.scan(/(\w+)$/).first
159
+ end
148
160
 
149
- ##
150
- # @abstract In your machine implementation you need to override the {#process_standard} method.
151
- # Processes a message. This method must call {#pass_on} if the message was handled successfully,
152
- # or if the handling of the message fails for whatever reason, you need to decide how you are going
153
- # to handle the failure of the message (write message to error log | queue | etc.) and then
154
- # acknowledge the message by calling {acknowledge}. Acknowledging the message mean that the
155
- # message has been dealt with. All messages must be acknowledged by at least one worker in a pipeline.
156
- #
157
- def process_standard(message)
158
- DefaultLogger.debug("MessageCommand#process_standard(message)") if options[:env] == "development"
159
- # Implemented in class that includes me
160
- pass_on(message)
161
- end
161
+ ##
162
+ # Process the first system message to notify machines down the pipeline which named pipe to
163
+ # use for acknowledgements.
164
+ #
165
+ # @param message<Hash> The system message
166
+ #
167
+ def process_system(message)
168
+ DefaultLogger.debug("MessageCommand#process_system(message)") if options[:env] == "development"
169
+ options[:ack] = message[:ack] # inherit setting from upstream
170
+ @ack_pipe = File.open(message[:sys_pipe], "w") # open ack pipe (needs to already exist)
171
+ send_description
172
+ pass_on(message)
173
+ end
174
+
175
+ def write_to_pipe(message, pipe=$stdout)
176
+ pipe.syswrite(MessageCoder.encode(message) << "\n")
177
+ end
178
+
179
+ ##
180
+ # Can be overriden in the class the MessageCommand is included into. Called
181
+ # when the acknowledgement named pipe has been established.
182
+ #
183
+ def send_description
184
+ message = { :msg_type => "pipe_desc", :description => self.description }
185
+ write_to_pipe(message, @ack_pipe)
186
+ end
162
187
 
163
- ##
164
- # Process the first system message to notify machines down the pipeline which named pipe to
165
- # use for acknowledgements.
166
- #
167
- # @param message<Hash> The system message
168
- #
169
- def process_system(message)
170
- DefaultLogger.debug("MessageCommand#process_system(message)") if options[:env] == "development"
171
- options[:ack] = message[:ack] # inherit setting from upstream
172
- @ack_pipe = File.open(message[:sys_pipe], "w") # open ack pipe (needs to already exist)
173
- report_back
174
- pass_on(message)
175
- end
176
188
 
177
- def write_to_pipe(message, pipe=$stdout)
178
- pipe.syswrite(MessageCoder.encode(message) << "\n")
179
189
  end
180
190
 
181
- end
191
+ end