pipeline_toolkit 1.0.4 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. data/LICENSE.markdown +20 -0
  2. data/README.markdown +98 -0
  3. data/bin/msg_generator +13 -0
  4. data/bin/msg_probe +13 -0
  5. data/bin/msg_push +13 -0
  6. data/bin/msg_sink +14 -0
  7. data/bin/msg_subscribe +13 -0
  8. data/lib/pipeline_toolkit.rb +17 -0
  9. data/lib/pipeline_toolkit/amqp/abstract.rb +95 -0
  10. data/lib/pipeline_toolkit/amqp/reader.rb +64 -0
  11. data/lib/pipeline_toolkit/amqp/writer.rb +54 -0
  12. data/lib/pipeline_toolkit/commands/msg_generator/cli.rb +45 -0
  13. data/lib/pipeline_toolkit/commands/msg_probe/cli.rb +46 -0
  14. data/lib/pipeline_toolkit/commands/msg_push/cli.rb +58 -0
  15. data/lib/pipeline_toolkit/commands/msg_sink/cli.rb +41 -0
  16. data/lib/pipeline_toolkit/commands/msg_subscribe/cli.rb +58 -0
  17. data/lib/pipeline_toolkit/default_logger.rb +126 -11
  18. data/lib/pipeline_toolkit/handlers/message_handler.rb +38 -0
  19. data/lib/pipeline_toolkit/message_coder.rb +18 -8
  20. data/lib/pipeline_toolkit/message_command.rb +138 -61
  21. data/lib/pipeline_toolkit/message_generator.rb +21 -0
  22. data/lib/pipeline_toolkit/message_probe.rb +6 -6
  23. data/lib/pipeline_toolkit/message_pusher.rb +51 -54
  24. data/lib/pipeline_toolkit/message_sink.rb +1 -1
  25. data/lib/pipeline_toolkit/message_subscriber.rb +182 -201
  26. data/lib/pipeline_toolkit/monitoring/monitor_server.rb +124 -0
  27. data/spec/eventmachine_helper.rb +44 -0
  28. data/spec/message_subscriber_spec.rb +64 -0
  29. data/spec/spec_helper.rb +15 -0
  30. metadata +202 -47
  31. data/.gitignore +0 -5
  32. data/README.rdoc +0 -70
  33. data/Rakefile +0 -40
  34. data/VERSION +0 -1
  35. data/bin/msg_generator.rb +0 -0
  36. data/bin/msg_probe.rb +0 -15
  37. data/bin/msg_push.rb +0 -25
  38. data/bin/msg_sink.rb +0 -11
  39. data/bin/msg_subscribe.rb +0 -27
  40. data/monitor/munin.rb +0 -91
  41. data/pipeline_toolkit.gemspec +0 -72
@@ -0,0 +1,124 @@
1
+ # encoding: utf-8
2
+ require 'erb'
3
+ require 'evma_httpserver'
4
+
5
+ module Monitoring # :nodoc:
6
+
7
+ ##
8
+ # The http server to monitor the pipeline and report the results.
9
+ #
10
+ class MonitorServer < EM::Connection
11
+ include EM::HttpServer
12
+
13
+ ##
14
+ # Initialize a new intance
15
+ #
16
+ # @param message_subscriber<MessageSubscriber> The message subscriber
17
+ #
18
+ def initialize(message_subscriber, options = {})
19
+ DefaultLogger.init_logger(options)
20
+
21
+ @message_subscriber = message_subscriber
22
+ # @template = File.open("../lib/pipeline_toolkit/monitoring/index.rhtml","r:utf-8")
23
+ end
24
+
25
+ def post_init
26
+ super
27
+ no_environment_strings
28
+ end
29
+
30
+ def process_http_request
31
+ response = EM::DelegatedHttpResponse.new(self)
32
+ response.status = 200
33
+ response.content_type 'text/html'
34
+ # if ((@http_content_type =~ content_type_regex) == 0)
35
+ #
36
+ # end
37
+ response.content = html_template
38
+
39
+ response.send_response
40
+ end
41
+
42
+ private
43
+
44
+ def html_template
45
+ <<-EOL
46
+ <!DOCTYPE HTML>
47
+ <html lang="en-NZ">
48
+ <head>
49
+ <title>Message Probe</title>
50
+ <meta charset="UTF-8">
51
+ <style type="text/css">
52
+ body {
53
+ background: black;
54
+ color: #80c0c0;
55
+ }
56
+ h1 {
57
+ font: 12pt Monospace;
58
+ text-align:center;
59
+ }
60
+ table {
61
+ font: 10pt Monospace;
62
+ margin-left:auto;
63
+ margin-right:auto;
64
+ text-align:right;
65
+ }
66
+ .page {
67
+ position:relative;
68
+ top: 20%;
69
+ # border-style:solid;
70
+ # border-width:5px;
71
+ width: 400px;
72
+ margin-left:auto;
73
+ margin-right:auto;
74
+ }
75
+ </style>
76
+ </head>
77
+ <body>
78
+ <div class=page>
79
+ <h1>
80
+ <span class="name">#{@message_subscriber.name}</span>
81
+ </h1>
82
+ <table>
83
+ <tr>
84
+ <td>Structure:</td>
85
+ <td>
86
+ <span class="queue_in">#{@message_subscriber.queue_name}</span> -> <span class="queues_out">#{@message_subscriber.queues_out}</span>
87
+ </td>
88
+ </tr>
89
+ <tr>
90
+ <td>Throughput:</td>
91
+ <td>
92
+ <span class="mps">#{@message_subscriber.mps.to_i}</span>
93
+ </td>
94
+ </tr>
95
+ <tr>
96
+ <td>Uptime:</td>
97
+ <td>
98
+ <span class="uptime">#{(Time.now - @message_subscriber.start_time).to_i / 60}mins</span>
99
+ </td>
100
+ </tr>
101
+ </table>
102
+ </div>
103
+ </body>
104
+ </html>
105
+ EOL
106
+ end
107
+
108
+ ##
109
+ # Build a regex to check the content_type request header
110
+ #
111
+ def content_type_regex
112
+ case options[:content_type]
113
+ when "json"
114
+ @content_type_regex = /^(text|application)\/json/
115
+ when "xml"
116
+ @content_type_regex = /^(text|application)\/xml/
117
+ else
118
+ @content_type_regex = /^(text|application)\/html/
119
+ end
120
+ end
121
+
122
+ end
123
+
124
+ end
@@ -0,0 +1,44 @@
1
+ require 'fiber'
2
+ require 'eventmachine'
3
+
4
+ module Eventmachine
5
+ module SpecHelper
6
+
7
+ attr_reader :current_fiber
8
+
9
+ def em(&block)
10
+ EventMachine.run do
11
+
12
+ exception = nil
13
+
14
+ @current_fiber = Fiber.new do
15
+ begin
16
+ block.call
17
+ rescue Exception => exception
18
+ done
19
+ end
20
+ Fiber.yield
21
+ end
22
+
23
+ @current_fiber.resume
24
+
25
+ raise exception if exception
26
+ end
27
+
28
+ end
29
+
30
+ def finish
31
+ EventMachine.next_tick do
32
+ finish_current_fiber
33
+ end
34
+ end
35
+
36
+ private
37
+
38
+ def finish_current_fiber
39
+ EventMachine.stop_event_loop if EventMachine.reactor_running?
40
+ @current_fiber.resume if @current_fiber.alive?
41
+ end
42
+
43
+ end
44
+ end
@@ -0,0 +1,64 @@
1
+ require "spec_helper"
2
+
3
+
4
+ require 'pipeline_toolkit/amqp/abstract'
5
+ require 'pipeline_toolkit/amqp/reader'
6
+ require 'pipeline_toolkit/amqp/writer'
7
+ require "pipeline_toolkit/message_subscriber"
8
+
9
+ describe MessageSubscriber do
10
+ include Eventmachine::SpecHelper
11
+
12
+ # --exchange, -x <s>: The exchange name
13
+ # --type, -t <s>: The exchange type (direct, fanout or topic)
14
+ # (default: fanout)
15
+ # --passive, -s: If set to true, the server will not create the
16
+ # exchange if it does not already exist.
17
+ # --durable, -d: If set to true, the exchange will be marked as
18
+ # durable.
19
+ # --queue, -q <s>: The destination queue (queue:routing_key). Routing
20
+ # keys will be ignored if exchange type is not topic.
21
+ # --ack, -a: If this field is set to false the server does not
22
+ # expect acknowledgments for messages. (Default:
23
+ # true)
24
+ # --host, -h <s>: The AMQP message server host (default: localhost)
25
+ # --port, -p <i>: The AMQP message server port (default: 5672)
26
+ # --user, -u <s>: The AMQP message server username (default: guest)
27
+ # --pass, -w <s>: The AMQP message server username (default: guest)
28
+ # --vhost, -v <s>: The AMQP message server vhost (default: /)
29
+ # --name, -n <s>: The name used to describe the entire process chain
30
+ # --http-port, -o <i>: The port the HTTP monitoring server runs on.
31
+ # Default is a random port between 10000-11000
32
+ # --content-type, -c <s>: The type of response we are expecting from the HTTP
33
+ # monitoring server. (http, xml and json supported)
34
+ # (default: html)
35
+ # --dnssd, -b: Switches on DNSSD (i.e. Bonjour) for the monitoring
36
+ # interface
37
+ let(:options) do
38
+ {
39
+ :exchange => "raw",
40
+ :type => "topic",
41
+ :queue => "octane",
42
+ :ack => false,
43
+ :host => "localhost",
44
+ :port => 5672,
45
+ :user => "guest",
46
+ :pass => "guest",
47
+ :vhost => "/"
48
+ }
49
+ end
50
+
51
+ before(:each) do
52
+ @msg_subscriber = MessageSubscriber.new(options).start
53
+ done
54
+ end
55
+
56
+ after(:each) do
57
+ done
58
+ end
59
+
60
+ it "should not fail" do
61
+ true.should == true
62
+ end
63
+
64
+ end
@@ -0,0 +1,15 @@
1
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
2
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
3
+
4
+ unless RUBY_VERSION.to_f >= 1.9
5
+ puts "tmu-server only works under Ruby 1.9"
6
+ exit(1)
7
+ end
8
+
9
+ require 'pipeline_toolkit'
10
+ require 'rspec/core'
11
+ require File.dirname(__FILE__) + '/eventmachine_helper'
12
+
13
+ Rspec.configure do |config|
14
+ config.mock_framework = :rspec
15
+ end
metadata CHANGED
@@ -1,101 +1,252 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pipeline_toolkit
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.4
4
+ prerelease: false
5
+ segments:
6
+ - 1
7
+ - 0
8
+ - 6
9
+ version: 1.0.6
5
10
  platform: ruby
6
11
  authors:
7
12
  - Aisha Fenton
13
+ - Marcus Wyatt
8
14
  autorequire:
9
15
  bindir: bin
10
16
  cert_chain: []
11
17
 
12
- date: 2010-01-15 00:00:00 +13:00
18
+ date: 2010-08-01 00:00:00 +12:00
13
19
  default_executable:
14
20
  dependencies:
15
21
  - !ruby/object:Gem::Dependency
16
22
  name: amqp
17
- type: :runtime
18
- version_requirement:
19
- version_requirements: !ruby/object:Gem::Requirement
23
+ prerelease: false
24
+ requirement: &id001 !ruby/object:Gem::Requirement
20
25
  requirements:
21
26
  - - ">="
22
27
  - !ruby/object:Gem::Version
23
- version: 0.6.5
24
- version:
28
+ segments:
29
+ - 0
30
+ - 6
31
+ - 7
32
+ version: 0.6.7
33
+ type: :runtime
34
+ version_requirements: *id001
25
35
  - !ruby/object:Gem::Dependency
26
36
  name: trollop
27
- type: :runtime
28
- version_requirement:
29
- version_requirements: !ruby/object:Gem::Requirement
37
+ prerelease: false
38
+ requirement: &id002 !ruby/object:Gem::Requirement
30
39
  requirements:
31
40
  - - ">="
32
41
  - !ruby/object:Gem::Version
33
- version: "1.14"
34
- version:
42
+ segments:
43
+ - 1
44
+ - 15
45
+ version: "1.15"
46
+ type: :runtime
47
+ version_requirements: *id002
35
48
  - !ruby/object:Gem::Dependency
36
49
  name: eventmachine
37
- type: :runtime
38
- version_requirement:
39
- version_requirements: !ruby/object:Gem::Requirement
50
+ prerelease: false
51
+ requirement: &id003 !ruby/object:Gem::Requirement
40
52
  requirements:
41
53
  - - ">="
42
54
  - !ruby/object:Gem::Version
55
+ segments:
56
+ - 0
57
+ - 12
58
+ - 10
43
59
  version: 0.12.10
44
- version:
60
+ type: :runtime
61
+ version_requirements: *id003
45
62
  - !ruby/object:Gem::Dependency
46
63
  name: eventmachine_httpserver
47
- type: :runtime
48
- version_requirement:
49
- version_requirements: !ruby/object:Gem::Requirement
64
+ prerelease: false
65
+ requirement: &id004 !ruby/object:Gem::Requirement
50
66
  requirements:
51
67
  - - ">="
52
68
  - !ruby/object:Gem::Version
69
+ segments:
70
+ - 0
71
+ - 2
72
+ - 0
53
73
  version: 0.2.0
54
- version:
74
+ type: :runtime
75
+ version_requirements: *id004
55
76
  - !ruby/object:Gem::Dependency
56
77
  name: SyslogLogger
78
+ prerelease: false
79
+ requirement: &id005 !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - "="
82
+ - !ruby/object:Gem::Version
83
+ segments:
84
+ - 1
85
+ - 4
86
+ - 0
87
+ version: 1.4.0
57
88
  type: :runtime
58
- version_requirement:
59
- version_requirements: !ruby/object:Gem::Requirement
89
+ version_requirements: *id005
90
+ - !ruby/object:Gem::Dependency
91
+ name: msgpack
92
+ prerelease: false
93
+ requirement: &id006 !ruby/object:Gem::Requirement
60
94
  requirements:
61
95
  - - ">="
62
96
  - !ruby/object:Gem::Version
63
- version: 1.4.0
64
- version:
65
- description:
97
+ segments:
98
+ - 0
99
+ - 3
100
+ - 3
101
+ version: 0.3.3
102
+ type: :runtime
103
+ version_requirements: *id006
104
+ - !ruby/object:Gem::Dependency
105
+ name: rspec
106
+ prerelease: false
107
+ requirement: &id007 !ruby/object:Gem::Requirement
108
+ requirements:
109
+ - - ">="
110
+ - !ruby/object:Gem::Version
111
+ segments:
112
+ - 2
113
+ - 0
114
+ - 0
115
+ - beta
116
+ - 2
117
+ version: 2.0.0.beta.2
118
+ type: :development
119
+ version_requirements: *id007
120
+ - !ruby/object:Gem::Dependency
121
+ name: rspec-core
122
+ prerelease: false
123
+ requirement: &id008 !ruby/object:Gem::Requirement
124
+ requirements:
125
+ - - ">="
126
+ - !ruby/object:Gem::Version
127
+ segments:
128
+ - 2
129
+ - 0
130
+ - 0
131
+ - beta
132
+ - 2
133
+ version: 2.0.0.beta.2
134
+ type: :development
135
+ version_requirements: *id008
136
+ - !ruby/object:Gem::Dependency
137
+ name: rspec-expectations
138
+ prerelease: false
139
+ requirement: &id009 !ruby/object:Gem::Requirement
140
+ requirements:
141
+ - - ">="
142
+ - !ruby/object:Gem::Version
143
+ segments:
144
+ - 2
145
+ - 0
146
+ - 0
147
+ - beta
148
+ - 2
149
+ version: 2.0.0.beta.2
150
+ type: :development
151
+ version_requirements: *id009
152
+ - !ruby/object:Gem::Dependency
153
+ name: rspec-mocks
154
+ prerelease: false
155
+ requirement: &id010 !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ segments:
160
+ - 2
161
+ - 0
162
+ - 0
163
+ - beta
164
+ - 2
165
+ version: 2.0.0.beta.2
166
+ type: :development
167
+ version_requirements: *id010
168
+ - !ruby/object:Gem::Dependency
169
+ name: yard
170
+ prerelease: false
171
+ requirement: &id011 !ruby/object:Gem::Requirement
172
+ requirements:
173
+ - - ">="
174
+ - !ruby/object:Gem::Version
175
+ segments:
176
+ - 0
177
+ - 5
178
+ - 3
179
+ version: 0.5.3
180
+ type: :development
181
+ version_requirements: *id011
182
+ - !ruby/object:Gem::Dependency
183
+ name: cucumber
184
+ prerelease: false
185
+ requirement: &id012 !ruby/object:Gem::Requirement
186
+ requirements:
187
+ - - ">="
188
+ - !ruby/object:Gem::Version
189
+ segments:
190
+ - 0
191
+ - 6
192
+ - 3
193
+ version: 0.6.3
194
+ type: :development
195
+ version_requirements: *id012
196
+ - !ruby/object:Gem::Dependency
197
+ name: aruba
198
+ prerelease: false
199
+ requirement: &id013 !ruby/object:Gem::Requirement
200
+ requirements:
201
+ - - ">="
202
+ - !ruby/object:Gem::Version
203
+ segments:
204
+ - 0
205
+ - 1
206
+ - 7
207
+ version: 0.1.7
208
+ type: :development
209
+ version_requirements: *id013
210
+ description: |-
211
+ Command line tools for processing messages by constructing a pipeline of workers.
212
+ AMQP and Unix pipes are used to construct the pipeline. Messages are simple Hashes
213
+ (serialized as YAML) so they can hold any values and change throughout the processing.
66
214
  email: labs@visfleet.com
67
215
  executables:
68
- - msg_probe.rb
69
- - msg_subscribe.rb
70
- - msg_push.rb
71
- - msg_sink.rb
72
- - msg_generator.rb
216
+ - msg_probe
217
+ - msg_subscribe
218
+ - msg_push
219
+ - msg_sink
220
+ - msg_generator
73
221
  extensions: []
74
222
 
75
223
  extra_rdoc_files:
76
- - README.rdoc
224
+ - LICENSE.markdown
225
+ - README.markdown
77
226
  files:
78
- - .gitignore
79
- - README.rdoc
80
- - Rakefile
81
- - VERSION
82
- - bin/msg_generator.rb
83
- - bin/msg_probe.rb
84
- - bin/msg_push.rb
85
- - bin/msg_sink.rb
86
- - bin/msg_subscribe.rb
87
227
  - lib/pipeline_toolkit.rb
228
+ - lib/pipeline_toolkit/amqp/abstract.rb
229
+ - lib/pipeline_toolkit/amqp/reader.rb
230
+ - lib/pipeline_toolkit/amqp/writer.rb
231
+ - lib/pipeline_toolkit/commands/msg_generator/cli.rb
232
+ - lib/pipeline_toolkit/commands/msg_probe/cli.rb
233
+ - lib/pipeline_toolkit/commands/msg_push/cli.rb
234
+ - lib/pipeline_toolkit/commands/msg_sink/cli.rb
235
+ - lib/pipeline_toolkit/commands/msg_subscribe/cli.rb
88
236
  - lib/pipeline_toolkit/default_logger.rb
237
+ - lib/pipeline_toolkit/handlers/message_handler.rb
89
238
  - lib/pipeline_toolkit/message_coder.rb
90
239
  - lib/pipeline_toolkit/message_command.rb
240
+ - lib/pipeline_toolkit/message_generator.rb
91
241
  - lib/pipeline_toolkit/message_probe.rb
92
242
  - lib/pipeline_toolkit/message_pusher.rb
93
243
  - lib/pipeline_toolkit/message_sink.rb
94
244
  - lib/pipeline_toolkit/message_subscriber.rb
245
+ - lib/pipeline_toolkit/monitoring/monitor_server.rb
95
246
  - lib/pipeline_toolkit/open_hash.rb
96
247
  - lib/pipeline_toolkit/socket_util.rb
97
- - monitor/munin.rb
98
- - pipeline_toolkit.gemspec
248
+ - LICENSE.markdown
249
+ - README.markdown
99
250
  has_rdoc: true
100
251
  homepage: http://github.com/visfleet/pipeline_toolkit
101
252
  licenses: []
@@ -109,20 +260,24 @@ required_ruby_version: !ruby/object:Gem::Requirement
109
260
  requirements:
110
261
  - - ">="
111
262
  - !ruby/object:Gem::Version
263
+ segments:
264
+ - 0
112
265
  version: "0"
113
- version:
114
266
  required_rubygems_version: !ruby/object:Gem::Requirement
115
267
  requirements:
116
268
  - - ">="
117
269
  - !ruby/object:Gem::Version
270
+ segments:
271
+ - 0
118
272
  version: "0"
119
- version:
120
273
  requirements: []
121
274
 
122
275
  rubyforge_project:
123
- rubygems_version: 1.3.5
276
+ rubygems_version: 1.3.6
124
277
  signing_key:
125
278
  specification_version: 3
126
279
  summary: Toolkit for building processing pipelines using Unix Pipes and AMQP messages
127
- test_files: []
128
-
280
+ test_files:
281
+ - spec/eventmachine_helper.rb
282
+ - spec/message_subscriber_spec.rb
283
+ - spec/spec_helper.rb