ap4r 0.3.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +17 -7
- data/MIT-LICENSE +1 -1
- data/README +4 -145
- data/Rakefile +50 -4
- data/lib/ap4r.rb +1 -1
- data/lib/ap4r/carrier.rb +107 -0
- data/lib/ap4r/dispatcher.rb +326 -0
- data/lib/ap4r/message_store_ext.rb +9 -5
- data/lib/ap4r/mongrel.rb +9 -4
- data/{script → lib/ap4r}/mongrel_ap4r.rb +1 -2
- data/lib/ap4r/multi_queue.rb +2 -1
- data/lib/ap4r/queue_manager_ext.rb +86 -200
- data/lib/ap4r/queue_manager_ext_debug.rb +15 -7
- data/lib/ap4r/retention_history.rb +3 -2
- data/lib/ap4r/script/base.rb +1 -1
- data/lib/ap4r/script/queue_manager_control.rb +1 -1
- data/lib/ap4r/script/setup.rb +1 -1
- data/lib/ap4r/script/workspace_generator.rb +2 -2
- data/lib/ap4r/start_with_log4r.rb +4 -1
- data/lib/ap4r/store_and_forward.rb +10 -6
- data/lib/ap4r/stored_message.rb +4 -3
- data/lib/ap4r/util/irm.rb +3 -3
- data/lib/ap4r/util/queue_client.rb +1 -1
- data/lib/ap4r/version.rb +2 -2
- data/rails_plugin/ap4r/lib/async_controller.rb +52 -41
- data/script/mongrel_ap4r +4 -0
- metadata +34 -8
- data/lib/ap4r/util/loc.rb +0 -12
- data/lib/ap4r/xxx_create_table_for_saf.rb +0 -21
- data/rails_plugin/ap4r/init.rb +0 -10
data/CHANGELOG
CHANGED
@@ -1,22 +1,32 @@
|
|
1
1
|
== 0.3.x
|
2
2
|
|
3
|
+
=== 0.3.2 (June 7th, 2007)
|
4
|
+
* Fixed: util/loc.rb doesn't work.
|
5
|
+
* Changed: Argument order of async_dispatch has changed, backward INCOMPATIBLE.
|
6
|
+
* Added: Dynamic configuration with ERb.
|
7
|
+
* Added: Script to run AP4R on Mongrel.
|
8
|
+
* Changed: How to plugin and main API names have changed.
|
9
|
+
* Added: Support of several Content-type on asynchronous call.
|
10
|
+
* Added: Block style for async_to.
|
11
|
+
* Added: Url rewrite filter.
|
12
|
+
|
3
13
|
=== 0.3.1 (April 24th, 2007)
|
4
14
|
|
5
|
-
* Changed: @delete_mode of AsyncController to @@saf_delete_mode with accessor
|
6
|
-
* Changed: default value of dispatch_mode, from :XMLRPC to :HTTP
|
7
|
-
* Added: Bootstrap script to let ap4r run on mongrel, experimental yet
|
15
|
+
* Changed: @delete_mode of AsyncController to @@saf_delete_mode with accessor.
|
16
|
+
* Changed: default value of dispatch_mode, from :XMLRPC to :HTTP.
|
17
|
+
* Added: Bootstrap script to let ap4r run on mongrel, experimental yet.
|
8
18
|
|
9
19
|
=== 0.3.0 (April 6th, 2007)
|
10
20
|
|
11
|
-
* Changed: name space from "AP4R" to "Ap4r"
|
12
|
-
* Added: support the latest version for
|
21
|
+
* Changed: name space from "AP4R" to "Ap4r".
|
22
|
+
* Added: support the latest version for Ruby(1.8.6) and RubyGems(0.9.2) and Rails(1.2.3) .
|
13
23
|
|
14
24
|
== 0.2.x
|
15
25
|
|
16
26
|
=== 0.2.0 (October 17th, 2006)
|
17
27
|
|
18
|
-
* Added: Protocols to invoke asynchronous logics
|
19
|
-
* Added: At-lease-once QoS
|
28
|
+
* Added: Protocols to invoke asynchronous logics.
|
29
|
+
* Added: At-lease-once QoS.
|
20
30
|
|
21
31
|
== 0.1.x
|
22
32
|
|
data/MIT-LICENSE
CHANGED
data/README
CHANGED
@@ -4,7 +4,9 @@ AP4R, Asynchronous Processing for Ruby, is the implementation of reliable asynch
|
|
4
4
|
Using asynchronous processing, we can cut down turn-around-time of web applications by queuing, or can utilize more machine power by load-balancing.
|
5
5
|
Also AP4R nicely ties with your Ruby on Rails applications. See Hello World sample application from rubyforge.
|
6
6
|
|
7
|
-
|
7
|
+
For more information, please step in AP4R homepage!
|
8
|
+
|
9
|
+
http://ap4r.rubyforge.org/wiki/wiki.pl
|
8
10
|
|
9
11
|
== Features
|
10
12
|
|
@@ -30,149 +32,6 @@ Use RubyGems command.
|
|
30
32
|
|
31
33
|
$ sudo gem install ap4r --include-dependencies
|
32
34
|
|
33
|
-
|
34
|
-
== Working directory
|
35
|
-
|
36
|
-
Create your working directory (ex. my_work) wherever you want.
|
37
|
-
|
38
|
-
$ ap4r_setup my_work
|
39
|
-
$ cd my_work
|
40
|
-
|
41
|
-
Its structure is as follows.
|
42
|
-
|
43
|
-
my_work
|
44
|
-
+-- config
|
45
|
-
+-- log
|
46
|
-
+-- script
|
47
|
-
+-- tmp
|
48
|
-
|
49
|
-
== Message Persistence
|
50
|
-
|
51
|
-
AP4R uses reliable-msg for message persistence. It enables disk or RDBMS persistence. See references for details of reliable-msg. Install MySQL if you choose MySQL persistence. We recommend to install MySQL/Ruby library to connect to MySQL. For convinience, Ruby/MySQL bundled in the ActiveRecord gem can be used.
|
52
|
-
Create a table in your database. (If you use topics via reliable-msg, create another slightly different table.)
|
53
|
-
|
54
|
-
CREATE TABLE `reliable_msg_queues` (
|
55
|
-
`id` varchar(255) NOT NULL default '',
|
56
|
-
`queue` varchar(255) NOT NULL default '',
|
57
|
-
`headers` text NOT NULL,
|
58
|
-
`object` blob NOT NULL,
|
59
|
-
PRIMARY KEY (`id`)
|
60
|
-
) ENGINE=InnoDB DEFAULT CHARSET=binary;
|
61
|
-
|
62
|
-
== Configuration file
|
63
|
-
|
64
|
-
A command ap4r_setup have created a template configuration file (my_work/config/queues.cfg).
|
65
|
-
|
66
|
-
---
|
67
|
-
store:
|
68
|
-
type: mysql
|
69
|
-
host: localhost
|
70
|
-
database: test
|
71
|
-
username: test
|
72
|
-
password:
|
73
|
-
drb:
|
74
|
-
host:
|
75
|
-
port: 6438
|
76
|
-
acl: allow 127.0.0.1
|
77
|
-
dispatchers:
|
78
|
-
-
|
79
|
-
targets: queue.*
|
80
|
-
threads: 1
|
81
|
-
#carriers:
|
82
|
-
# -
|
83
|
-
# source_uri: druby://another.host.local:6438
|
84
|
-
# threads: 1
|
85
|
-
|
86
|
-
queues.cfg has four parts.
|
87
|
-
|
88
|
-
* persistent role (store: )
|
89
|
-
* Set database or file information.
|
90
|
-
|
91
|
-
* message listener (drb:)
|
92
|
-
* Set IP, and port number which are used by clients.
|
93
|
-
* acl = access control list, exmaple below.
|
94
|
-
|
95
|
-
allow 127.0.0.1 allow 10.0.0.0/8 deny 192.168.0.1
|
96
|
-
|
97
|
-
* asynchronous process invokers (dispatchers:)
|
98
|
-
* Dispatchers handle messages in queues specified by targets,
|
99
|
-
* with as many threads as specified by threads.
|
100
|
-
* Each thread waits (is blocked) until the invocation returns.
|
101
|
-
|
102
|
-
* message routing (carriers:) # EXPERIMENTAL
|
103
|
-
* Carriers get messages from an AP4R server specified by source_uri,
|
104
|
-
* with as many threads as specified by threads,
|
105
|
-
* and put messages to a local AP4R server.
|
106
|
-
|
107
|
-
== Monitoring
|
108
|
-
|
109
|
-
Future plan: connections with monitoring tools such as Cacti and ZABBIX.
|
110
|
-
|
111
|
-
* Cacti
|
112
|
-
* http://www.cacti.net/
|
113
|
-
* ZABBIX
|
114
|
-
* http://www.zabbix.org/
|
115
|
-
|
116
|
-
== Sample - HelloWorld -
|
117
|
-
|
118
|
-
There is an asynchronous application sample with file persistence, where a synchronous logic outputs "Hello" to a file, and an asynchronous logic appends "World".
|
119
|
-
Once the synchronous logic has done, a client (a web browser) displays response, and there is only "Hello" in a file.
|
120
|
-
Since the asynchronous logic sleeps ten seconds and appends to the file, wait briefly and you can see whole "HelloWorld" in the file
|
121
|
-
|
122
|
-
* Install Ruby, Rails and AP4R and configure AP4R in reference above sections
|
123
|
-
* No need for MySQL.
|
124
|
-
|
125
|
-
* Make sample application available
|
126
|
-
* If you have an SVN client,
|
127
|
-
|
128
|
-
$ svn co svn://rubyforge.org/var/ap4r/tags/ap4r-0.2.0/sample [some directory]
|
129
|
-
|
130
|
-
* Unless
|
131
|
-
1. download the sample from RubyForge
|
132
|
-
http://rubyforge.org/projects/ap4r/
|
133
|
-
filename: HelloWorld.tar.gz
|
134
|
-
|
135
|
-
1. and extract to an appropricate directory(ex. HelloWorld ).
|
136
|
-
There are app/, components/,... directories under HelloWorld.
|
137
|
-
|
138
|
-
* Start WEBRick.
|
139
|
-
* In a command line
|
140
|
-
|
141
|
-
$ cd HelloWorld
|
142
|
-
$ ruby script\server
|
143
|
-
|
144
|
-
* If you see Welcome screen at http://localhost:3000, it's ok.
|
145
|
-
|
146
|
-
* Start AP4R.
|
147
|
-
* In another command line,
|
148
|
-
* start AP4R with specifying the configuraion file.
|
149
|
-
|
150
|
-
$ cd my_work
|
151
|
-
$ ruby script/start -c config/queues_disk.cfg
|
152
|
-
|
153
|
-
* Execute a synchronous logic.
|
154
|
-
* http://localhost:3000/sync_hello/execute
|
155
|
-
* A file HelloWorld.txt is creted under HelloWorld/ which contains a word "Hello".
|
156
|
-
* execute_via_soap, execute_via_http and execute_with_saf actions are also available.
|
157
|
-
|
158
|
-
* Confirm the execution of an asynchronous logic.
|
159
|
-
* Wait ten seconds,
|
160
|
-
* and look into the file HelloWorld.txt again, there must be "HelloWorld".
|
161
|
-
|
162
|
-
* Think of application to your real application
|
163
|
-
* This mechanism can work in general applications.
|
164
|
-
* An order acceptance logic instead of printing "Hello".
|
165
|
-
* Asynchronous logic of auto shipping order or accounting instead of appending "World".
|
166
|
-
|
167
|
-
== Future Plan
|
168
|
-
|
169
|
-
* only-once QoS
|
170
|
-
* DLQ recovery
|
171
|
-
* flow volume control, load balancing
|
172
|
-
* 7x24 support (e.g. rolling database tables)
|
173
|
-
* monitoring (e.g. thread status, web frontend)
|
174
|
-
* Coordination with Ruby on Rails, such as development/testing lifesycle support.
|
175
|
-
|
176
35
|
== References
|
177
36
|
|
178
37
|
* Ruby Homepage
|
@@ -187,7 +46,7 @@ Since the asynchronous logic sleeps ten seconds and appends to the file, wait br
|
|
187
46
|
== Licence
|
188
47
|
|
189
48
|
This licence is licensed under the MIT license.
|
190
|
-
Copyright(c)
|
49
|
+
Copyright(c) 2007 Future Architect Inc.
|
191
50
|
|
192
51
|
== Authors
|
193
52
|
|
data/Rakefile
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# Author:: Kiwamu Kato
|
2
|
-
# Copyright:: Copyright (c)
|
2
|
+
# Copyright:: Copyright (c) 2007 Future Architect Inc.
|
3
3
|
# Licence:: MIT Licence
|
4
4
|
|
5
5
|
require 'rake'
|
@@ -59,6 +59,9 @@ spec = Gem::Specification.new do |s|
|
|
59
59
|
EOF
|
60
60
|
|
61
61
|
s.add_dependency(%q<reliable-msg>, ["= 1.1.0"])
|
62
|
+
s.add_dependency(%q<rake>)
|
63
|
+
s.add_dependency(%q<activesupport>)
|
64
|
+
s.add_dependency(%q<mongrel>)
|
62
65
|
|
63
66
|
s.has_rdoc = true
|
64
67
|
s.extra_rdoc_files = ["README", "CHANGELOG", 'rails_plugin']
|
@@ -94,6 +97,32 @@ Rake::RDocTask.new { |rdoc|
|
|
94
97
|
rdoc.rdoc_files.include('rails_plugin/**/*.rb')
|
95
98
|
}
|
96
99
|
|
100
|
+
# Spec tasks ----------------------------------------------------------------
|
101
|
+
require 'spec/rake/spectask'
|
102
|
+
|
103
|
+
namespace :spec do
|
104
|
+
%w(local).each do |flavor|
|
105
|
+
desc "Run #{flavor} examples"
|
106
|
+
Spec::Rake::SpecTask.new(flavor) do |t|
|
107
|
+
t.spec_files = FileList["spec/#{flavor}/**/*.rb"]
|
108
|
+
end
|
109
|
+
|
110
|
+
namespace :coverage do
|
111
|
+
desc "Run #{flavor} examples with RCov"
|
112
|
+
Spec::Rake::SpecTask.new(flavor) do |t|
|
113
|
+
t.spec_files = FileList["spec/#{flavor}/**/*.rb"]
|
114
|
+
t.rcov = true
|
115
|
+
excludes = %w(^spec\/)
|
116
|
+
if ENV['GEM_HOME']
|
117
|
+
excludes << Regexp.escape(ENV['GEM_HOME'])
|
118
|
+
end
|
119
|
+
t.rcov_opts = ["--exclude" , excludes.join(","),
|
120
|
+
"--text-summary"]
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
97
126
|
# AP4R release ----------------------------------------------------------------
|
98
127
|
|
99
128
|
desc "Make gem"
|
@@ -103,7 +132,9 @@ task :copy_to_ap4r_from_sample do
|
|
103
132
|
FileUtils.cp(HELLO_WORLD_DIR + '/db/migrate/001_create_table_for_saf.rb', './lib/ap4r/xxx_create_table_for_saf.rb')
|
104
133
|
|
105
134
|
FileUtils.cp(HELLO_WORLD_DIR + '/vendor/plugins/ap4r/init.rb', './rails_plugin/ap4r/init.rb')
|
106
|
-
FileUtils.cp(HELLO_WORLD_DIR + '/vendor/plugins/ap4r/lib/
|
135
|
+
FileUtils.cp(HELLO_WORLD_DIR + '/vendor/plugins/ap4r/lib/async_helper.rb', './rails_plugin/ap4r/lib/async_helper.rb')
|
136
|
+
FileUtils.cp(HELLO_WORLD_DIR + '/vendor/plugins/ap4r/lib/ap4r_client.rb', './rails_plugin/ap4r/lib/ap4r_client.rb')
|
137
|
+
FileUtils.cp(HELLO_WORLD_DIR + '/vendor/plugins/ap4r/lib/message_builder.rb', './rails_plugin/ap4r/lib/message_builder.rb')
|
107
138
|
end
|
108
139
|
|
109
140
|
desc "Make sample tgz"
|
@@ -137,9 +168,9 @@ task :copy_sample do
|
|
137
168
|
end
|
138
169
|
|
139
170
|
task :execute_migration do
|
140
|
-
Dir.chdir('temp/')
|
171
|
+
Dir.chdir('temp/HelloWorld')
|
141
172
|
`rake db:migrate`
|
142
|
-
Dir.chdir('
|
173
|
+
Dir.chdir('../../')
|
143
174
|
end
|
144
175
|
|
145
176
|
task :make_tgz do
|
@@ -157,3 +188,18 @@ task :copy_to_release_dir do
|
|
157
188
|
}
|
158
189
|
end
|
159
190
|
|
191
|
+
# AP4R misc tools ----------------------------------------------------------------
|
192
|
+
|
193
|
+
desc "display code statistics"
|
194
|
+
task :stats do
|
195
|
+
require 'rubygems'
|
196
|
+
require 'active_support'
|
197
|
+
require 'code_statistics'
|
198
|
+
CodeStatistics::TEST_TYPES.concat(["Local specs"])
|
199
|
+
CodeStatistics.new(
|
200
|
+
["Core Sources", "lib"],
|
201
|
+
["Rails plugin", "rails_plugin"],
|
202
|
+
["Scripts", "script"],
|
203
|
+
["Local specs", "spec/local"]
|
204
|
+
).to_s
|
205
|
+
end
|
data/lib/ap4r.rb
CHANGED
data/lib/ap4r/carrier.rb
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
# Author:: Shunichi Shinohara
|
2
|
+
# Copyright:: Copyright (c) 2007 Future Architect Inc.
|
3
|
+
# Licence:: MIT Licence
|
4
|
+
|
5
|
+
require 'yaml'
|
6
|
+
require 'thread'
|
7
|
+
require 'pp'
|
8
|
+
require 'active_support'
|
9
|
+
|
10
|
+
require 'reliable-msg'
|
11
|
+
|
12
|
+
module Ap4r
|
13
|
+
|
14
|
+
# This class aims to balance loads of several reliable-msg servers.
|
15
|
+
# Only P2P channells (queues) are considered so far.
|
16
|
+
# Now reliable-msg can not be accessed from remote (means "not localhost").
|
17
|
+
# Alpha status now.
|
18
|
+
#--
|
19
|
+
# TODO: refactoring with dispatcher.rb, around thread group, etc. 2007/05/09 by shino
|
20
|
+
class Carriers
|
21
|
+
|
22
|
+
def initialize(queue_manager, config, logger, dispatchers)
|
23
|
+
@qm = queue_manager
|
24
|
+
@config = config
|
25
|
+
@logger = logger
|
26
|
+
@group = ThreadGroup.new
|
27
|
+
@dispatchers = dispatchers
|
28
|
+
end
|
29
|
+
|
30
|
+
def start
|
31
|
+
return unless @config
|
32
|
+
@logger.info{ "ready to start carrires with config #{@config.to_yaml}" }
|
33
|
+
@config.each { |remote|
|
34
|
+
remote["threads"].times { |index|
|
35
|
+
Thread.fork(@group, remote, index){|group, remote, index|
|
36
|
+
carrier_loop(group, remote, index)
|
37
|
+
}
|
38
|
+
}
|
39
|
+
}
|
40
|
+
@logger.info{"queue manager has forked all carriers"}
|
41
|
+
end
|
42
|
+
|
43
|
+
def stop
|
44
|
+
@logger.info{"stop_carriers #{@group}"}
|
45
|
+
return unless @group
|
46
|
+
@group.list.each{|d| d[:dying] = true}
|
47
|
+
@group.list.each{|d| d.join }
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
def carrier_loop(group, remote, index)
|
52
|
+
# TODO: refactor structure, 2006/10/06 shino
|
53
|
+
group.add Thread.current
|
54
|
+
@logger.info{ "starting a carrier (index #{index}) for the queue manager #{remote['source_uri']}" }
|
55
|
+
uri = remote['source_uri']
|
56
|
+
until Thread.current[:dying]
|
57
|
+
begin
|
58
|
+
sleep 0.1
|
59
|
+
# TODO check :dying flag here and break, 2006/09/01 shino
|
60
|
+
# TODO cache DRbObject if necessary, 2006/09/01 shino
|
61
|
+
remote_qm = DRb::DRbObject.new_with_uri(uri)
|
62
|
+
queue_name = remote_qm.stale_queue dispatch_targets
|
63
|
+
next unless queue_name
|
64
|
+
|
65
|
+
@logger.debug{ "stale queue name : #{queue_name}" }
|
66
|
+
q = ReliableMsg::Queue.new queue_name, :drb_uri => uri
|
67
|
+
q.get { |m|
|
68
|
+
unless m
|
69
|
+
@logger.debug{ "carrier strikes at the air (T_T)" }
|
70
|
+
next
|
71
|
+
end
|
72
|
+
# @logger.debug{ "carrier gets a message\n#{m.to_yaml}" }
|
73
|
+
|
74
|
+
# TODO: decide the better one, and delete another, 2006/09/01 shino
|
75
|
+
# TODO: or switchable implementation in versions, 2006/10/16 shino
|
76
|
+
|
77
|
+
# version 1: use thread fork so queue manager use a different tx
|
78
|
+
# TODO probably should have a thread as an instance variable or in a thread local, 2006/09/01 shino
|
79
|
+
# Thread.fork(m) {|m|
|
80
|
+
# local_queue = ReliableMsg::Queue.new queue_name
|
81
|
+
# local_queue.put m.object
|
82
|
+
# }.join
|
83
|
+
|
84
|
+
#version 2: store tx and set nil, and resotre tx after putting a message
|
85
|
+
begin
|
86
|
+
tx = Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX]
|
87
|
+
Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = nil
|
88
|
+
# @logger.debug{ "before tx: #{tx}" }
|
89
|
+
ReliableMsg::Queue.new(queue_name).put(m.object)
|
90
|
+
ensure
|
91
|
+
Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = tx
|
92
|
+
# @logger.debug{ "after tx: #{Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX]}" }
|
93
|
+
end
|
94
|
+
}
|
95
|
+
rescue Exception => ex
|
96
|
+
@logger.warn "error in remote-get/local-put #{ex}\n#{ex.backtrace.join("\n\t")}\n"
|
97
|
+
end
|
98
|
+
end
|
99
|
+
@logger.info{"ends a carrier (index #{index}) for the queue manager #{remote['uri']}"}
|
100
|
+
end
|
101
|
+
|
102
|
+
def dispatch_targets
|
103
|
+
@dispatchers.targets
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,326 @@
|
|
1
|
+
# Author:: Shunichi Shinohara
|
2
|
+
# Copyright:: Copyright (c) 2007 Future Architect Inc.
|
3
|
+
# Licence:: MIT Licence
|
4
|
+
|
5
|
+
require 'yaml'
|
6
|
+
require 'thread'
|
7
|
+
require 'pp'
|
8
|
+
require 'active_support'
|
9
|
+
|
10
|
+
require 'uri'
|
11
|
+
require 'net/http'
|
12
|
+
require 'xmlrpc/client'
|
13
|
+
require 'soap/wsdlDriver'
|
14
|
+
|
15
|
+
module Ap4r
|
16
|
+
|
17
|
+
# Represents a group of dispatchers.
|
18
|
+
# Responsibilities are follows:
|
19
|
+
# - polls target queues,
|
20
|
+
# - gets messages from queues, and
|
21
|
+
# - calls a <tt>Dispatchers::Base</tt>'s instance.
|
22
|
+
class Dispatchers
|
23
|
+
|
24
|
+
@@sleep_inverval = 0.1
|
25
|
+
|
26
|
+
@@logger = nil
|
27
|
+
def self.logger
|
28
|
+
@@logger
|
29
|
+
end
|
30
|
+
|
31
|
+
# storage for <tt>Dispatchers::Base</tt>'s subclasses.
|
32
|
+
@@subclasses = {}
|
33
|
+
|
34
|
+
# Stores +klass+ for a dispatch mode +mode+
|
35
|
+
# Each +klass+ is used to create instances to handle dispatching messages.
|
36
|
+
def self.register_dispatcher_class(mode, klass)
|
37
|
+
@@subclasses[mode] = klass
|
38
|
+
end
|
39
|
+
|
40
|
+
# Sum of each dispatcher's target queues.
|
41
|
+
def targets
|
42
|
+
@dispatch_targets
|
43
|
+
end
|
44
|
+
|
45
|
+
def initialize(queue_manager, config, logger_obj)
|
46
|
+
@qm = queue_manager
|
47
|
+
@config = config # (typically) dispatcher section of queues.cfg
|
48
|
+
@@logger ||= logger_obj
|
49
|
+
raise "no configuration specified" unless @config
|
50
|
+
@group = ThreadGroup.new
|
51
|
+
# TODO: needs refinement 2007/05/30 by shino
|
52
|
+
@dispatch_targets = ""
|
53
|
+
end
|
54
|
+
|
55
|
+
# Starts every dispatcher.
|
56
|
+
# If an exception is detected, this method raise it through with logging.
|
57
|
+
#
|
58
|
+
def start
|
59
|
+
begin
|
60
|
+
logger.info{ "about to start dispatchers with config\n#{@config.to_yaml}" }
|
61
|
+
@config.each{ |conf|
|
62
|
+
conf["threads"].to_i.times { |index|
|
63
|
+
Thread.fork(@group, conf, index){|group, conf, index|
|
64
|
+
dispatching_loop(group, conf, index)
|
65
|
+
}
|
66
|
+
}
|
67
|
+
@dispatch_targets.concat(conf["targets"]).concat(';')
|
68
|
+
logger.debug{ "dispatch targets are : #{@dispatch_targets}" }
|
69
|
+
}
|
70
|
+
logger.info "queue manager has forked dispatchers"
|
71
|
+
rescue Exception => err
|
72
|
+
logger.warn{"Error in starting dipatchers #{err}"}
|
73
|
+
logger.warn{err.backtrace.join("\n")}
|
74
|
+
raise err
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
# Stops every dispatcher.
|
79
|
+
# Current implementation makes just dying flags up.
|
80
|
+
# Some threads don't stop quickly in some cases such as blocking at socket read.
|
81
|
+
#--
|
82
|
+
# TODO: needs forced mode? 2007/05/09 by shino
|
83
|
+
def stop
|
84
|
+
logger.info{"stop_dispatchers #{@group}"}
|
85
|
+
return unless @group
|
86
|
+
@group.list.each {|d| d[:dying] = true}
|
87
|
+
@group.list.each {|d| d.join }
|
88
|
+
@dispatch_targets = ""
|
89
|
+
end
|
90
|
+
|
91
|
+
private
|
92
|
+
|
93
|
+
# Creates and returns an appropriate instace.
|
94
|
+
# If no class for +dispatch_mode+, raises an exception.
|
95
|
+
def get_dispather_instance(dispatch_mode, message, conf_per_targets)
|
96
|
+
klass = @@subclasses[dispatch_mode]
|
97
|
+
raise "undefined dispatch mode #{m.headers[:mode]}" unless klass
|
98
|
+
klass.new(message, conf_per_targets)
|
99
|
+
end
|
100
|
+
|
101
|
+
# Defines the general structure for each dispatcher thread
|
102
|
+
# from begging to end.
|
103
|
+
def dispatching_loop(group, conf, index)
|
104
|
+
group.add(Thread.current)
|
105
|
+
mq = ::ReliableMsg::MultiQueue.new(conf["targets"])
|
106
|
+
logger.info{ "start dispatcher: targets= #{mq}, index= #{index})" }
|
107
|
+
until Thread.current[:dying]
|
108
|
+
# TODO: change sleep interval depending on last result? 2007/05/09 by shino
|
109
|
+
sleep @@sleep_inverval
|
110
|
+
# logger.debug{ "try dispatch #{mq} #{mq.name}" }
|
111
|
+
# TODO: needs timeout?, 2006/10/16 shino
|
112
|
+
begin
|
113
|
+
mq.get{|m|
|
114
|
+
unless m
|
115
|
+
logger.debug{"message is nul"}
|
116
|
+
break
|
117
|
+
end
|
118
|
+
logger.debug{"dispatcher get message\n#{m.to_yaml}"}
|
119
|
+
response = get_dispather_instance(m.headers[:dispatch_mode], m, conf).call
|
120
|
+
logger.debug{"dispatcher get response\n#{response.to_yaml}"}
|
121
|
+
}
|
122
|
+
rescue Exception => err
|
123
|
+
logger.warn("dispatch err #{err.inspect}")
|
124
|
+
logger.warn(err.backtrace.join("\n"))
|
125
|
+
end
|
126
|
+
end
|
127
|
+
logger.info{"end dispatcher #{mq} (index #{index})"}
|
128
|
+
end
|
129
|
+
|
130
|
+
def logger
|
131
|
+
@@logger
|
132
|
+
end
|
133
|
+
|
134
|
+
|
135
|
+
# A base class for dispathcer classes associated with each <tt>dispatch_mode</tt>.
|
136
|
+
# Responsibilities of subclasses are to implement following methods, only +invoke+
|
137
|
+
# is mandatory and others are optional (no operations by default).
|
138
|
+
# * +modify_message+ to preprocess a message, e.g. rewirte URL or discard message.
|
139
|
+
# * +invoke+ to execute main logic, e.g. HTTP POST call. *mandatory*
|
140
|
+
# * +validate_response+ to judge whether +invoke+ finished successfully.
|
141
|
+
# * +response+ to return the result of +invoke+ process.
|
142
|
+
class Base
|
143
|
+
|
144
|
+
# Difine a constant +DISPATCH_MODE+ to value 'mode_symbol' and
|
145
|
+
# add self to a Converters list.
|
146
|
+
def self.dispatch_mode(mode_symbol)
|
147
|
+
self.const_set(:DISPATCH_MODE, mode_symbol)
|
148
|
+
::Ap4r::Dispatchers.register_dispatcher_class(mode_symbol, self)
|
149
|
+
end
|
150
|
+
|
151
|
+
# Takes
|
152
|
+
# * +message+: from a queue
|
153
|
+
# * +conf+: configuration from dispatchers section
|
154
|
+
#--
|
155
|
+
# TODO: Subclass should have +conf+ instead of instance? 2007/06/06 by shino
|
156
|
+
def initialize(message, conf)
|
157
|
+
@message = message
|
158
|
+
@conf = conf
|
159
|
+
end
|
160
|
+
|
161
|
+
# Entry facade for each message processing.
|
162
|
+
# Modifies, invokes (maybe remote), validates, and responds.
|
163
|
+
def call
|
164
|
+
# TODO: rename to more appropriate one 2007/05/10 by shino
|
165
|
+
self.modify_message
|
166
|
+
logger.debug{"Ap4r::Dispatcher after modification\n#{@message.to_yaml}"}
|
167
|
+
self.invoke
|
168
|
+
self.validate_response
|
169
|
+
self.response
|
170
|
+
end
|
171
|
+
|
172
|
+
# Modifies message.
|
173
|
+
# Now only URL modification is implemented.
|
174
|
+
def modify_message
|
175
|
+
modification_rules = @conf["modify_rules"]
|
176
|
+
return unless modification_rules
|
177
|
+
modify_url(modification_rules)
|
178
|
+
end
|
179
|
+
|
180
|
+
# Main logic of message processing.
|
181
|
+
# Maybe calls remote, e.g. HTTP request.
|
182
|
+
def invoke
|
183
|
+
# TODO: rename to more appropriate one 2007/05/10 by shino
|
184
|
+
raise 'must be implemented in subclasses'
|
185
|
+
end
|
186
|
+
|
187
|
+
def validate_response
|
188
|
+
# nop
|
189
|
+
end
|
190
|
+
|
191
|
+
# Returns response.
|
192
|
+
# The return value is also the return value of +call+ method.
|
193
|
+
# By default impl, the instance variable <tt>@response</tt> is used.
|
194
|
+
def response
|
195
|
+
@response
|
196
|
+
end
|
197
|
+
|
198
|
+
private
|
199
|
+
def logger
|
200
|
+
::Ap4r::Dispatchers.logger
|
201
|
+
end
|
202
|
+
|
203
|
+
# Modifies <tt>:target_url</tt> according to a rule.
|
204
|
+
# TODO: +proc+ in configuration is eval'ed every time. 2007/06/06 by shino
|
205
|
+
def modify_url(modification_rules)
|
206
|
+
proc_for_url = modification_rules["url"]
|
207
|
+
return unless proc_for_url
|
208
|
+
|
209
|
+
url = URI.parse(@message.headers[:target_url])
|
210
|
+
eval(proc_for_url).call(url)
|
211
|
+
@message.headers[:target_url] = url.to_s
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
# Dispatches via a raw HTTP protocol.
|
216
|
+
# Current implementation uses only a POST method, irrespective of
|
217
|
+
# <tt>options[:target_method]</tt>.
|
218
|
+
#
|
219
|
+
# Determination of "success" is two fold:
|
220
|
+
# * status code should be exactly 200, other codes (including 201-2xx) are
|
221
|
+
# treated as error, and
|
222
|
+
# * body should include a string "true"
|
223
|
+
#
|
224
|
+
class Http < Base
|
225
|
+
dispatch_mode :HTTP
|
226
|
+
|
227
|
+
def invoke
|
228
|
+
# TODO: should be added some request headers 2006/10/12 shino
|
229
|
+
# e.g. X-Ap4r-Version, Accept(need it?)
|
230
|
+
# TODO: Now supports POST only, 2006/10/12 shino
|
231
|
+
@response = nil
|
232
|
+
uri = URI.parse(@message[:target_url])
|
233
|
+
headers = make_header
|
234
|
+
|
235
|
+
Net::HTTP.start(uri.host, uri.port) do |http|
|
236
|
+
@response, = http.post(uri.path, @message.object, headers)
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
def make_header
|
241
|
+
headers = { }
|
242
|
+
@message.headers.map do |k,v|
|
243
|
+
s = StringScanner.new(k.to_s)
|
244
|
+
s.scan(/\Ahttp_header_/)
|
245
|
+
headers[s.post_match] = v if s.post_match
|
246
|
+
end
|
247
|
+
headers
|
248
|
+
end
|
249
|
+
|
250
|
+
def validate_response
|
251
|
+
logger.debug{"response status [#{@response.code} #{@response.message}]"}
|
252
|
+
validate_response_status(Net::HTTPOK)
|
253
|
+
validate_response_body(/true/)
|
254
|
+
end
|
255
|
+
|
256
|
+
# Checks whether the response status is a kind of +status_kind+.
|
257
|
+
# +status_kind+ should be one of <tt>Net::HTTPRespose</tt>'s subclasses.
|
258
|
+
def validate_response_status(status_kind)
|
259
|
+
#TODO: make the difinition of success variable, 2006/10/13 shino
|
260
|
+
unless @response.kind_of?(status_kind)
|
261
|
+
error_message = "HTTP Response FAILURE, " +
|
262
|
+
"status [#{@response.code} #{@response.message}]"
|
263
|
+
logger.error(error_message)
|
264
|
+
logger.info{@response.to_yaml}
|
265
|
+
#TODO: must create AP4R specific Exception class, 2006/10/12 shino
|
266
|
+
raise StandardError.new(error_message)
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
# Checks whether the response body includes +pattern+.
|
271
|
+
# +pattern+ should be a regular expression.
|
272
|
+
def validate_response_body(pattern)
|
273
|
+
unless @response.body =~ pattern
|
274
|
+
error_message = "HTTP Response FAILURE, status" +
|
275
|
+
" [#{@response.code} #{@response.message}], body [#{@response.body}]"
|
276
|
+
#TODO: Refactor error logging, 2006/10/13 shino
|
277
|
+
logger.error(error_message)
|
278
|
+
logger.info{@response.to_yaml}
|
279
|
+
#TODO: must create AP4R specific Exception class, 2006/10/12 shino
|
280
|
+
raise StandardError.new(error_message)
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
end
|
285
|
+
|
286
|
+
# Dispatches via XML-RPC protocol.
|
287
|
+
# Uses +XMLRPC+ library.
|
288
|
+
#
|
289
|
+
# The call result is judged as
|
290
|
+
# * "failure" if the first element of <tt>XMLRPC::Client#call2</tt> result
|
291
|
+
# is false, and
|
292
|
+
# * "success" otherwise.
|
293
|
+
#
|
294
|
+
class XmlRpc < Base
|
295
|
+
dispatch_mode :XMLRPC
|
296
|
+
|
297
|
+
def invoke
|
298
|
+
endpoint = @message[:target_url]
|
299
|
+
client = XMLRPC::Client.new2(endpoint)
|
300
|
+
@success, @response = client.call2(@message[:target_action], @message.object)
|
301
|
+
end
|
302
|
+
|
303
|
+
def validate_response
|
304
|
+
raise @response unless @success
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
# Dispatches via SOAP protocol.
|
309
|
+
# Uses +SOAP+ library.
|
310
|
+
#
|
311
|
+
# The call result is judged as
|
312
|
+
# * "success" if <tt>SOAP::WSDLDrive#send</tt> finishes without an exception, and
|
313
|
+
# * "failuar" otherwise.
|
314
|
+
#
|
315
|
+
class SOAP < Base
|
316
|
+
dispatch_mode :SOAP
|
317
|
+
|
318
|
+
def invoke
|
319
|
+
# TODO: nice to cache drivers probably 2007/05/09 by shino
|
320
|
+
driver = ::SOAP::WSDLDriverFactory.new(@message[:target_url]).create_rpc_driver
|
321
|
+
driver.send(@message[:target_action], @message.object)
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
end
|
326
|
+
end
|