pipeline_toolkit 1.0.4 → 1.0.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. data/LICENSE.markdown +20 -0
  2. data/README.markdown +98 -0
  3. data/bin/msg_generator +13 -0
  4. data/bin/msg_probe +13 -0
  5. data/bin/msg_push +13 -0
  6. data/bin/msg_sink +14 -0
  7. data/bin/msg_subscribe +13 -0
  8. data/lib/pipeline_toolkit.rb +17 -0
  9. data/lib/pipeline_toolkit/amqp/abstract.rb +95 -0
  10. data/lib/pipeline_toolkit/amqp/reader.rb +64 -0
  11. data/lib/pipeline_toolkit/amqp/writer.rb +54 -0
  12. data/lib/pipeline_toolkit/commands/msg_generator/cli.rb +45 -0
  13. data/lib/pipeline_toolkit/commands/msg_probe/cli.rb +46 -0
  14. data/lib/pipeline_toolkit/commands/msg_push/cli.rb +58 -0
  15. data/lib/pipeline_toolkit/commands/msg_sink/cli.rb +41 -0
  16. data/lib/pipeline_toolkit/commands/msg_subscribe/cli.rb +58 -0
  17. data/lib/pipeline_toolkit/default_logger.rb +126 -11
  18. data/lib/pipeline_toolkit/handlers/message_handler.rb +38 -0
  19. data/lib/pipeline_toolkit/message_coder.rb +18 -8
  20. data/lib/pipeline_toolkit/message_command.rb +138 -61
  21. data/lib/pipeline_toolkit/message_generator.rb +21 -0
  22. data/lib/pipeline_toolkit/message_probe.rb +6 -6
  23. data/lib/pipeline_toolkit/message_pusher.rb +51 -54
  24. data/lib/pipeline_toolkit/message_sink.rb +1 -1
  25. data/lib/pipeline_toolkit/message_subscriber.rb +182 -201
  26. data/lib/pipeline_toolkit/monitoring/monitor_server.rb +124 -0
  27. data/spec/eventmachine_helper.rb +44 -0
  28. data/spec/message_subscriber_spec.rb +64 -0
  29. data/spec/spec_helper.rb +15 -0
  30. metadata +202 -47
  31. data/.gitignore +0 -5
  32. data/README.rdoc +0 -70
  33. data/Rakefile +0 -40
  34. data/VERSION +0 -1
  35. data/bin/msg_generator.rb +0 -0
  36. data/bin/msg_probe.rb +0 -15
  37. data/bin/msg_push.rb +0 -25
  38. data/bin/msg_sink.rb +0 -11
  39. data/bin/msg_subscribe.rb +0 -27
  40. data/monitor/munin.rb +0 -91
  41. data/pipeline_toolkit.gemspec +0 -72
@@ -0,0 +1,124 @@
1
+ # encoding: utf-8
2
+ require 'erb'
3
+ require 'evma_httpserver'
4
+
5
+ module Monitoring # :nodoc:
6
+
7
+ ##
8
+ # The http server to monitor the pipeline and report the results.
9
+ #
10
+ class MonitorServer < EM::Connection
11
+ include EM::HttpServer
12
+
13
+ ##
14
+ # Initialize a new intance
15
+ #
16
+ # @param message_subscriber<MessageSubscriber> The message subscriber
17
+ #
18
+ def initialize(message_subscriber, options = {})
19
+ DefaultLogger.init_logger(options)
20
+
21
+ @message_subscriber = message_subscriber
22
+ # @template = File.open("../lib/pipeline_toolkit/monitoring/index.rhtml","r:utf-8")
23
+ end
24
+
25
+ def post_init
26
+ super
27
+ no_environment_strings
28
+ end
29
+
30
+ def process_http_request
31
+ response = EM::DelegatedHttpResponse.new(self)
32
+ response.status = 200
33
+ response.content_type 'text/html'
34
+ # if ((@http_content_type =~ content_type_regex) == 0)
35
+ #
36
+ # end
37
+ response.content = html_template
38
+
39
+ response.send_response
40
+ end
41
+
42
+ private
43
+
44
+ def html_template
45
+ <<-EOL
46
+ <!DOCTYPE HTML>
47
+ <html lang="en-NZ">
48
+ <head>
49
+ <title>Message Probe</title>
50
+ <meta charset="UTF-8">
51
+ <style type="text/css">
52
+ body {
53
+ background: black;
54
+ color: #80c0c0;
55
+ }
56
+ h1 {
57
+ font: 12pt Monospace;
58
+ text-align:center;
59
+ }
60
+ table {
61
+ font: 10pt Monospace;
62
+ margin-left:auto;
63
+ margin-right:auto;
64
+ text-align:right;
65
+ }
66
+ .page {
67
+ position:relative;
68
+ top: 20%;
69
+ # border-style:solid;
70
+ # border-width:5px;
71
+ width: 400px;
72
+ margin-left:auto;
73
+ margin-right:auto;
74
+ }
75
+ </style>
76
+ </head>
77
+ <body>
78
+ <div class=page>
79
+ <h1>
80
+ <span class="name">#{@message_subscriber.name}</span>
81
+ </h1>
82
+ <table>
83
+ <tr>
84
+ <td>Structure:</td>
85
+ <td>
86
+ <span class="queue_in">#{@message_subscriber.queue_name}</span> -> <span class="queues_out">#{@message_subscriber.queues_out}</span>
87
+ </td>
88
+ </tr>
89
+ <tr>
90
+ <td>Throughput:</td>
91
+ <td>
92
+ <span class="mps">#{@message_subscriber.mps.to_i}</span>
93
+ </td>
94
+ </tr>
95
+ <tr>
96
+ <td>Uptime:</td>
97
+ <td>
98
+ <span class="uptime">#{(Time.now - @message_subscriber.start_time).to_i / 60}mins</span>
99
+ </td>
100
+ </tr>
101
+ </table>
102
+ </div>
103
+ </body>
104
+ </html>
105
+ EOL
106
+ end
107
+
108
+ ##
109
+ # Build a regex to check the content_type request header
110
+ #
111
+ def content_type_regex
112
+ case options[:content_type]
113
+ when "json"
114
+ @content_type_regex = /^(text|application)\/json/
115
+ when "xml"
116
+ @content_type_regex = /^(text|application)\/xml/
117
+ else
118
+ @content_type_regex = /^(text|application)\/html/
119
+ end
120
+ end
121
+
122
+ end
123
+
124
+ end
@@ -0,0 +1,44 @@
1
+ require 'fiber'
2
+ require 'eventmachine'
3
+
4
+ module Eventmachine
5
+ module SpecHelper
6
+
7
+ attr_reader :current_fiber
8
+
9
+ def em(&block)
10
+ EventMachine.run do
11
+
12
+ exception = nil
13
+
14
+ @current_fiber = Fiber.new do
15
+ begin
16
+ block.call
17
+ rescue Exception => exception
18
+ done
19
+ end
20
+ Fiber.yield
21
+ end
22
+
23
+ @current_fiber.resume
24
+
25
+ raise exception if exception
26
+ end
27
+
28
+ end
29
+
30
+ def finish
31
+ EventMachine.next_tick do
32
+ finish_current_fiber
33
+ end
34
+ end
35
+
36
+ private
37
+
38
+ def finish_current_fiber
39
+ EventMachine.stop_event_loop if EventMachine.reactor_running?
40
+ @current_fiber.resume if @current_fiber.alive?
41
+ end
42
+
43
+ end
44
+ end
@@ -0,0 +1,64 @@
1
+ require "spec_helper"
2
+
3
+
4
+ require 'pipeline_toolkit/amqp/abstract'
5
+ require 'pipeline_toolkit/amqp/reader'
6
+ require 'pipeline_toolkit/amqp/writer'
7
+ require "pipeline_toolkit/message_subscriber"
8
+
9
+ describe MessageSubscriber do
10
+ include Eventmachine::SpecHelper
11
+
12
+ # --exchange, -x <s>: The exchange name
13
+ # --type, -t <s>: The exchange type (direct, fanout or topic)
14
+ # (default: fanout)
15
+ # --passive, -s: If set to true, the server will not create the
16
+ # exchange if it does not already exist.
17
+ # --durable, -d: If set to true, the exchange will be marked as
18
+ # durable.
19
+ # --queue, -q <s>: The destination queue (queue:routing_key). Routing
20
+ # keys will be ignored if exchange type is not topic.
21
+ # --ack, -a: If this field is set to false the server does not
22
+ # expect acknowledgments for messages. (Default:
23
+ # true)
24
+ # --host, -h <s>: The AMQP message server host (default: localhost)
25
+ # --port, -p <i>: The AMQP message server port (default: 5672)
26
+ # --user, -u <s>: The AMQP message server username (default: guest)
27
+ # --pass, -w <s>: The AMQP message server username (default: guest)
28
+ # --vhost, -v <s>: The AMQP message server vhost (default: /)
29
+ # --name, -n <s>: The name used to describe the entire process chain
30
+ # --http-port, -o <i>: The port the HTTP monitoring server runs on.
31
+ # Default is a random port between 10000-11000
32
+ # --content-type, -c <s>: The type of response we are expecting from the HTTP
33
+ # monitoring server. (http, xml and json supported)
34
+ # (default: html)
35
+ # --dnssd, -b: Switches on DNSSD (i.e. Bonjour) for the monitoring
36
+ # interface
37
+ let(:options) do
38
+ {
39
+ :exchange => "raw",
40
+ :type => "topic",
41
+ :queue => "octane",
42
+ :ack => false,
43
+ :host => "localhost",
44
+ :port => 5672,
45
+ :user => "guest",
46
+ :pass => "guest",
47
+ :vhost => "/"
48
+ }
49
+ end
50
+
51
+ before(:each) do
52
+ @msg_subscriber = MessageSubscriber.new(options).start
53
+ done
54
+ end
55
+
56
+ after(:each) do
57
+ done
58
+ end
59
+
60
+ it "should not fail" do
61
+ true.should == true
62
+ end
63
+
64
+ end
@@ -0,0 +1,15 @@
1
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
2
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
3
+
4
+ unless RUBY_VERSION.to_f >= 1.9
5
+ puts "tmu-server only works under Ruby 1.9"
6
+ exit(1)
7
+ end
8
+
9
+ require 'pipeline_toolkit'
10
+ require 'rspec/core'
11
+ require File.dirname(__FILE__) + '/eventmachine_helper'
12
+
13
+ Rspec.configure do |config|
14
+ config.mock_framework = :rspec
15
+ end
metadata CHANGED
@@ -1,101 +1,252 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pipeline_toolkit
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.4
4
+ prerelease: false
5
+ segments:
6
+ - 1
7
+ - 0
8
+ - 6
9
+ version: 1.0.6
5
10
  platform: ruby
6
11
  authors:
7
12
  - Aisha Fenton
13
+ - Marcus Wyatt
8
14
  autorequire:
9
15
  bindir: bin
10
16
  cert_chain: []
11
17
 
12
- date: 2010-01-15 00:00:00 +13:00
18
+ date: 2010-08-01 00:00:00 +12:00
13
19
  default_executable:
14
20
  dependencies:
15
21
  - !ruby/object:Gem::Dependency
16
22
  name: amqp
17
- type: :runtime
18
- version_requirement:
19
- version_requirements: !ruby/object:Gem::Requirement
23
+ prerelease: false
24
+ requirement: &id001 !ruby/object:Gem::Requirement
20
25
  requirements:
21
26
  - - ">="
22
27
  - !ruby/object:Gem::Version
23
- version: 0.6.5
24
- version:
28
+ segments:
29
+ - 0
30
+ - 6
31
+ - 7
32
+ version: 0.6.7
33
+ type: :runtime
34
+ version_requirements: *id001
25
35
  - !ruby/object:Gem::Dependency
26
36
  name: trollop
27
- type: :runtime
28
- version_requirement:
29
- version_requirements: !ruby/object:Gem::Requirement
37
+ prerelease: false
38
+ requirement: &id002 !ruby/object:Gem::Requirement
30
39
  requirements:
31
40
  - - ">="
32
41
  - !ruby/object:Gem::Version
33
- version: "1.14"
34
- version:
42
+ segments:
43
+ - 1
44
+ - 15
45
+ version: "1.15"
46
+ type: :runtime
47
+ version_requirements: *id002
35
48
  - !ruby/object:Gem::Dependency
36
49
  name: eventmachine
37
- type: :runtime
38
- version_requirement:
39
- version_requirements: !ruby/object:Gem::Requirement
50
+ prerelease: false
51
+ requirement: &id003 !ruby/object:Gem::Requirement
40
52
  requirements:
41
53
  - - ">="
42
54
  - !ruby/object:Gem::Version
55
+ segments:
56
+ - 0
57
+ - 12
58
+ - 10
43
59
  version: 0.12.10
44
- version:
60
+ type: :runtime
61
+ version_requirements: *id003
45
62
  - !ruby/object:Gem::Dependency
46
63
  name: eventmachine_httpserver
47
- type: :runtime
48
- version_requirement:
49
- version_requirements: !ruby/object:Gem::Requirement
64
+ prerelease: false
65
+ requirement: &id004 !ruby/object:Gem::Requirement
50
66
  requirements:
51
67
  - - ">="
52
68
  - !ruby/object:Gem::Version
69
+ segments:
70
+ - 0
71
+ - 2
72
+ - 0
53
73
  version: 0.2.0
54
- version:
74
+ type: :runtime
75
+ version_requirements: *id004
55
76
  - !ruby/object:Gem::Dependency
56
77
  name: SyslogLogger
78
+ prerelease: false
79
+ requirement: &id005 !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - "="
82
+ - !ruby/object:Gem::Version
83
+ segments:
84
+ - 1
85
+ - 4
86
+ - 0
87
+ version: 1.4.0
57
88
  type: :runtime
58
- version_requirement:
59
- version_requirements: !ruby/object:Gem::Requirement
89
+ version_requirements: *id005
90
+ - !ruby/object:Gem::Dependency
91
+ name: msgpack
92
+ prerelease: false
93
+ requirement: &id006 !ruby/object:Gem::Requirement
60
94
  requirements:
61
95
  - - ">="
62
96
  - !ruby/object:Gem::Version
63
- version: 1.4.0
64
- version:
65
- description:
97
+ segments:
98
+ - 0
99
+ - 3
100
+ - 3
101
+ version: 0.3.3
102
+ type: :runtime
103
+ version_requirements: *id006
104
+ - !ruby/object:Gem::Dependency
105
+ name: rspec
106
+ prerelease: false
107
+ requirement: &id007 !ruby/object:Gem::Requirement
108
+ requirements:
109
+ - - ">="
110
+ - !ruby/object:Gem::Version
111
+ segments:
112
+ - 2
113
+ - 0
114
+ - 0
115
+ - beta
116
+ - 2
117
+ version: 2.0.0.beta.2
118
+ type: :development
119
+ version_requirements: *id007
120
+ - !ruby/object:Gem::Dependency
121
+ name: rspec-core
122
+ prerelease: false
123
+ requirement: &id008 !ruby/object:Gem::Requirement
124
+ requirements:
125
+ - - ">="
126
+ - !ruby/object:Gem::Version
127
+ segments:
128
+ - 2
129
+ - 0
130
+ - 0
131
+ - beta
132
+ - 2
133
+ version: 2.0.0.beta.2
134
+ type: :development
135
+ version_requirements: *id008
136
+ - !ruby/object:Gem::Dependency
137
+ name: rspec-expectations
138
+ prerelease: false
139
+ requirement: &id009 !ruby/object:Gem::Requirement
140
+ requirements:
141
+ - - ">="
142
+ - !ruby/object:Gem::Version
143
+ segments:
144
+ - 2
145
+ - 0
146
+ - 0
147
+ - beta
148
+ - 2
149
+ version: 2.0.0.beta.2
150
+ type: :development
151
+ version_requirements: *id009
152
+ - !ruby/object:Gem::Dependency
153
+ name: rspec-mocks
154
+ prerelease: false
155
+ requirement: &id010 !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ segments:
160
+ - 2
161
+ - 0
162
+ - 0
163
+ - beta
164
+ - 2
165
+ version: 2.0.0.beta.2
166
+ type: :development
167
+ version_requirements: *id010
168
+ - !ruby/object:Gem::Dependency
169
+ name: yard
170
+ prerelease: false
171
+ requirement: &id011 !ruby/object:Gem::Requirement
172
+ requirements:
173
+ - - ">="
174
+ - !ruby/object:Gem::Version
175
+ segments:
176
+ - 0
177
+ - 5
178
+ - 3
179
+ version: 0.5.3
180
+ type: :development
181
+ version_requirements: *id011
182
+ - !ruby/object:Gem::Dependency
183
+ name: cucumber
184
+ prerelease: false
185
+ requirement: &id012 !ruby/object:Gem::Requirement
186
+ requirements:
187
+ - - ">="
188
+ - !ruby/object:Gem::Version
189
+ segments:
190
+ - 0
191
+ - 6
192
+ - 3
193
+ version: 0.6.3
194
+ type: :development
195
+ version_requirements: *id012
196
+ - !ruby/object:Gem::Dependency
197
+ name: aruba
198
+ prerelease: false
199
+ requirement: &id013 !ruby/object:Gem::Requirement
200
+ requirements:
201
+ - - ">="
202
+ - !ruby/object:Gem::Version
203
+ segments:
204
+ - 0
205
+ - 1
206
+ - 7
207
+ version: 0.1.7
208
+ type: :development
209
+ version_requirements: *id013
210
+ description: |-
211
+ Command line tools for processing messages by constructing a pipeline of workers.
212
+ AMQP and Unix pipes are used to construct the pipeline. Messages are simple Hashes
213
+ (serialized as YAML) so they can hold any values and change throughout the processing.
66
214
  email: labs@visfleet.com
67
215
  executables:
68
- - msg_probe.rb
69
- - msg_subscribe.rb
70
- - msg_push.rb
71
- - msg_sink.rb
72
- - msg_generator.rb
216
+ - msg_probe
217
+ - msg_subscribe
218
+ - msg_push
219
+ - msg_sink
220
+ - msg_generator
73
221
  extensions: []
74
222
 
75
223
  extra_rdoc_files:
76
- - README.rdoc
224
+ - LICENSE.markdown
225
+ - README.markdown
77
226
  files:
78
- - .gitignore
79
- - README.rdoc
80
- - Rakefile
81
- - VERSION
82
- - bin/msg_generator.rb
83
- - bin/msg_probe.rb
84
- - bin/msg_push.rb
85
- - bin/msg_sink.rb
86
- - bin/msg_subscribe.rb
87
227
  - lib/pipeline_toolkit.rb
228
+ - lib/pipeline_toolkit/amqp/abstract.rb
229
+ - lib/pipeline_toolkit/amqp/reader.rb
230
+ - lib/pipeline_toolkit/amqp/writer.rb
231
+ - lib/pipeline_toolkit/commands/msg_generator/cli.rb
232
+ - lib/pipeline_toolkit/commands/msg_probe/cli.rb
233
+ - lib/pipeline_toolkit/commands/msg_push/cli.rb
234
+ - lib/pipeline_toolkit/commands/msg_sink/cli.rb
235
+ - lib/pipeline_toolkit/commands/msg_subscribe/cli.rb
88
236
  - lib/pipeline_toolkit/default_logger.rb
237
+ - lib/pipeline_toolkit/handlers/message_handler.rb
89
238
  - lib/pipeline_toolkit/message_coder.rb
90
239
  - lib/pipeline_toolkit/message_command.rb
240
+ - lib/pipeline_toolkit/message_generator.rb
91
241
  - lib/pipeline_toolkit/message_probe.rb
92
242
  - lib/pipeline_toolkit/message_pusher.rb
93
243
  - lib/pipeline_toolkit/message_sink.rb
94
244
  - lib/pipeline_toolkit/message_subscriber.rb
245
+ - lib/pipeline_toolkit/monitoring/monitor_server.rb
95
246
  - lib/pipeline_toolkit/open_hash.rb
96
247
  - lib/pipeline_toolkit/socket_util.rb
97
- - monitor/munin.rb
98
- - pipeline_toolkit.gemspec
248
+ - LICENSE.markdown
249
+ - README.markdown
99
250
  has_rdoc: true
100
251
  homepage: http://github.com/visfleet/pipeline_toolkit
101
252
  licenses: []
@@ -109,20 +260,24 @@ required_ruby_version: !ruby/object:Gem::Requirement
109
260
  requirements:
110
261
  - - ">="
111
262
  - !ruby/object:Gem::Version
263
+ segments:
264
+ - 0
112
265
  version: "0"
113
- version:
114
266
  required_rubygems_version: !ruby/object:Gem::Requirement
115
267
  requirements:
116
268
  - - ">="
117
269
  - !ruby/object:Gem::Version
270
+ segments:
271
+ - 0
118
272
  version: "0"
119
- version:
120
273
  requirements: []
121
274
 
122
275
  rubyforge_project:
123
- rubygems_version: 1.3.5
276
+ rubygems_version: 1.3.6
124
277
  signing_key:
125
278
  specification_version: 3
126
279
  summary: Toolkit for building processing pipelines using Unix Pipes and AMQP messages
127
- test_files: []
128
-
280
+ test_files:
281
+ - spec/eventmachine_helper.rb
282
+ - spec/message_subscriber_spec.rb
283
+ - spec/spec_helper.rb