ap4r 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +17 -7
- data/MIT-LICENSE +1 -1
- data/README +4 -145
- data/Rakefile +50 -4
- data/lib/ap4r.rb +1 -1
- data/lib/ap4r/carrier.rb +107 -0
- data/lib/ap4r/dispatcher.rb +326 -0
- data/lib/ap4r/message_store_ext.rb +9 -5
- data/lib/ap4r/mongrel.rb +9 -4
- data/{script → lib/ap4r}/mongrel_ap4r.rb +1 -2
- data/lib/ap4r/multi_queue.rb +2 -1
- data/lib/ap4r/queue_manager_ext.rb +86 -200
- data/lib/ap4r/queue_manager_ext_debug.rb +15 -7
- data/lib/ap4r/retention_history.rb +3 -2
- data/lib/ap4r/script/base.rb +1 -1
- data/lib/ap4r/script/queue_manager_control.rb +1 -1
- data/lib/ap4r/script/setup.rb +1 -1
- data/lib/ap4r/script/workspace_generator.rb +2 -2
- data/lib/ap4r/start_with_log4r.rb +4 -1
- data/lib/ap4r/store_and_forward.rb +10 -6
- data/lib/ap4r/stored_message.rb +4 -3
- data/lib/ap4r/util/irm.rb +3 -3
- data/lib/ap4r/util/queue_client.rb +1 -1
- data/lib/ap4r/version.rb +2 -2
- data/rails_plugin/ap4r/lib/async_controller.rb +52 -41
- data/script/mongrel_ap4r +4 -0
- metadata +34 -8
- data/lib/ap4r/util/loc.rb +0 -12
- data/lib/ap4r/xxx_create_table_for_saf.rb +0 -21
- data/rails_plugin/ap4r/init.rb +0 -10
data/CHANGELOG
CHANGED
@@ -1,22 +1,32 @@
|
|
1
1
|
== 0.3.x
|
2
2
|
|
3
|
+
=== 0.3.2 (June 7th, 2007)
|
4
|
+
* Fixed: util/loc.rb doesn't work.
|
5
|
+
* Changed: Argument order of async_dispatch has changed, backward INCOMPATIBLE.
|
6
|
+
* Added: Dynamic configuration with ERb.
|
7
|
+
* Added: Script to run AP4R on Mongrel.
|
8
|
+
* Changed: How to plugin and main API names have changed.
|
9
|
+
* Added: Support of several Content-type on asynchronous call.
|
10
|
+
* Added: Block style for async_to.
|
11
|
+
* Added: Url rewrite filter.
|
12
|
+
|
3
13
|
=== 0.3.1 (April 24th, 2007)
|
4
14
|
|
5
|
-
* Changed: @delete_mode of AsyncController to @@saf_delete_mode with accessor
|
6
|
-
* Changed: default value of dispatch_mode, from :XMLRPC to :HTTP
|
7
|
-
* Added: Bootstrap script to let ap4r run on mongrel, experimental yet
|
15
|
+
* Changed: @delete_mode of AsyncController to @@saf_delete_mode with accessor.
|
16
|
+
* Changed: default value of dispatch_mode, from :XMLRPC to :HTTP.
|
17
|
+
* Added: Bootstrap script to let ap4r run on mongrel, experimental yet.
|
8
18
|
|
9
19
|
=== 0.3.0 (April 6th, 2007)
|
10
20
|
|
11
|
-
* Changed: name space from "AP4R" to "Ap4r"
|
12
|
-
* Added: support the latest version for
|
21
|
+
* Changed: name space from "AP4R" to "Ap4r".
|
22
|
+
* Added: support the latest version for Ruby(1.8.6) and RubyGems(0.9.2) and Rails(1.2.3) .
|
13
23
|
|
14
24
|
== 0.2.x
|
15
25
|
|
16
26
|
=== 0.2.0 (October 17th, 2006)
|
17
27
|
|
18
|
-
* Added: Protocols to invoke asynchronous logics
|
19
|
-
* Added: At-lease-once QoS
|
28
|
+
* Added: Protocols to invoke asynchronous logics.
|
29
|
+
* Added: At-lease-once QoS.
|
20
30
|
|
21
31
|
== 0.1.x
|
22
32
|
|
data/MIT-LICENSE
CHANGED
data/README
CHANGED
@@ -4,7 +4,9 @@ AP4R, Asynchronous Processing for Ruby, is the implementation of reliable asynch
|
|
4
4
|
Using asynchronous processing, we can cut down turn-around-time of web applications by queuing, or can utilize more machine power by load-balancing.
|
5
5
|
Also AP4R nicely ties with your Ruby on Rails applications. See Hello World sample application from rubyforge.
|
6
6
|
|
7
|
-
|
7
|
+
For more information, please step in AP4R homepage!
|
8
|
+
|
9
|
+
http://ap4r.rubyforge.org/wiki/wiki.pl
|
8
10
|
|
9
11
|
== Features
|
10
12
|
|
@@ -30,149 +32,6 @@ Use RubyGems command.
|
|
30
32
|
|
31
33
|
$ sudo gem install ap4r --include-dependencies
|
32
34
|
|
33
|
-
|
34
|
-
== Working directory
|
35
|
-
|
36
|
-
Create your working directory (ex. my_work) wherever you want.
|
37
|
-
|
38
|
-
$ ap4r_setup my_work
|
39
|
-
$ cd my_work
|
40
|
-
|
41
|
-
Its structure is as follows.
|
42
|
-
|
43
|
-
my_work
|
44
|
-
+-- config
|
45
|
-
+-- log
|
46
|
-
+-- script
|
47
|
-
+-- tmp
|
48
|
-
|
49
|
-
== Message Persistence
|
50
|
-
|
51
|
-
AP4R uses reliable-msg for message persistence. It enables disk or RDBMS persistence. See references for details of reliable-msg. Install MySQL if you choose MySQL persistence. We recommend to install MySQL/Ruby library to connect to MySQL. For convinience, Ruby/MySQL bundled in the ActiveRecord gem can be used.
|
52
|
-
Create a table in your database. (If you use topics via reliable-msg, create another slightly different table.)
|
53
|
-
|
54
|
-
CREATE TABLE `reliable_msg_queues` (
|
55
|
-
`id` varchar(255) NOT NULL default '',
|
56
|
-
`queue` varchar(255) NOT NULL default '',
|
57
|
-
`headers` text NOT NULL,
|
58
|
-
`object` blob NOT NULL,
|
59
|
-
PRIMARY KEY (`id`)
|
60
|
-
) ENGINE=InnoDB DEFAULT CHARSET=binary;
|
61
|
-
|
62
|
-
== Configuration file
|
63
|
-
|
64
|
-
A command ap4r_setup have created a template configuration file (my_work/config/queues.cfg).
|
65
|
-
|
66
|
-
---
|
67
|
-
store:
|
68
|
-
type: mysql
|
69
|
-
host: localhost
|
70
|
-
database: test
|
71
|
-
username: test
|
72
|
-
password:
|
73
|
-
drb:
|
74
|
-
host:
|
75
|
-
port: 6438
|
76
|
-
acl: allow 127.0.0.1
|
77
|
-
dispatchers:
|
78
|
-
-
|
79
|
-
targets: queue.*
|
80
|
-
threads: 1
|
81
|
-
#carriers:
|
82
|
-
# -
|
83
|
-
# source_uri: druby://another.host.local:6438
|
84
|
-
# threads: 1
|
85
|
-
|
86
|
-
queues.cfg has four parts.
|
87
|
-
|
88
|
-
* persistent role (store: )
|
89
|
-
* Set database or file information.
|
90
|
-
|
91
|
-
* message listener (drb:)
|
92
|
-
* Set IP, and port number which are used by clients.
|
93
|
-
* acl = access control list, exmaple below.
|
94
|
-
|
95
|
-
allow 127.0.0.1 allow 10.0.0.0/8 deny 192.168.0.1
|
96
|
-
|
97
|
-
* asynchronous process invokers (dispatchers:)
|
98
|
-
* Dispatchers handle messages in queues specified by targets,
|
99
|
-
* with as many threads as specified by threads.
|
100
|
-
* Each thread waits (is blocked) until the invocation returns.
|
101
|
-
|
102
|
-
* message routing (carriers:) # EXPERIMENTAL
|
103
|
-
* Carriers get messages from an AP4R server specified by source_uri,
|
104
|
-
* with as many threads as specified by threads,
|
105
|
-
* and put messages to a local AP4R server.
|
106
|
-
|
107
|
-
== Monitoring
|
108
|
-
|
109
|
-
Future plan: connections with monitoring tools such as Cacti and ZABBIX.
|
110
|
-
|
111
|
-
* Cacti
|
112
|
-
* http://www.cacti.net/
|
113
|
-
* ZABBIX
|
114
|
-
* http://www.zabbix.org/
|
115
|
-
|
116
|
-
== Sample - HelloWorld -
|
117
|
-
|
118
|
-
There is an asynchronous application sample with file persistence, where a synchronous logic outputs "Hello" to a file, and an asynchronous logic appends "World".
|
119
|
-
Once the synchronous logic has done, a client (a web browser) displays response, and there is only "Hello" in a file.
|
120
|
-
Since the asynchronous logic sleeps ten seconds and appends to the file, wait briefly and you can see whole "HelloWorld" in the file
|
121
|
-
|
122
|
-
* Install Ruby, Rails and AP4R and configure AP4R in reference above sections
|
123
|
-
* No need for MySQL.
|
124
|
-
|
125
|
-
* Make sample application available
|
126
|
-
* If you have an SVN client,
|
127
|
-
|
128
|
-
$ svn co svn://rubyforge.org/var/ap4r/tags/ap4r-0.2.0/sample [some directory]
|
129
|
-
|
130
|
-
* Unless
|
131
|
-
1. download the sample from RubyForge
|
132
|
-
http://rubyforge.org/projects/ap4r/
|
133
|
-
filename: HelloWorld.tar.gz
|
134
|
-
|
135
|
-
1. and extract to an appropricate directory(ex. HelloWorld ).
|
136
|
-
There are app/, components/,... directories under HelloWorld.
|
137
|
-
|
138
|
-
* Start WEBRick.
|
139
|
-
* In a command line
|
140
|
-
|
141
|
-
$ cd HelloWorld
|
142
|
-
$ ruby script\server
|
143
|
-
|
144
|
-
* If you see Welcome screen at http://localhost:3000, it's ok.
|
145
|
-
|
146
|
-
* Start AP4R.
|
147
|
-
* In another command line,
|
148
|
-
* start AP4R with specifying the configuraion file.
|
149
|
-
|
150
|
-
$ cd my_work
|
151
|
-
$ ruby script/start -c config/queues_disk.cfg
|
152
|
-
|
153
|
-
* Execute a synchronous logic.
|
154
|
-
* http://localhost:3000/sync_hello/execute
|
155
|
-
* A file HelloWorld.txt is creted under HelloWorld/ which contains a word "Hello".
|
156
|
-
* execute_via_soap, execute_via_http and execute_with_saf actions are also available.
|
157
|
-
|
158
|
-
* Confirm the execution of an asynchronous logic.
|
159
|
-
* Wait ten seconds,
|
160
|
-
* and look into the file HelloWorld.txt again, there must be "HelloWorld".
|
161
|
-
|
162
|
-
* Think of application to your real application
|
163
|
-
* This mechanism can work in general applications.
|
164
|
-
* An order acceptance logic instead of printing "Hello".
|
165
|
-
* Asynchronous logic of auto shipping order or accounting instead of appending "World".
|
166
|
-
|
167
|
-
== Future Plan
|
168
|
-
|
169
|
-
* only-once QoS
|
170
|
-
* DLQ recovery
|
171
|
-
* flow volume control, load balancing
|
172
|
-
* 7x24 support (e.g. rolling database tables)
|
173
|
-
* monitoring (e.g. thread status, web frontend)
|
174
|
-
* Coordination with Ruby on Rails, such as development/testing lifesycle support.
|
175
|
-
|
176
35
|
== References
|
177
36
|
|
178
37
|
* Ruby Homepage
|
@@ -187,7 +46,7 @@ Since the asynchronous logic sleeps ten seconds and appends to the file, wait br
|
|
187
46
|
== Licence
|
188
47
|
|
189
48
|
This licence is licensed under the MIT license.
|
190
|
-
Copyright(c)
|
49
|
+
Copyright(c) 2007 Future Architect Inc.
|
191
50
|
|
192
51
|
== Authors
|
193
52
|
|
data/Rakefile
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# Author:: Kiwamu Kato
|
2
|
-
# Copyright:: Copyright (c)
|
2
|
+
# Copyright:: Copyright (c) 2007 Future Architect Inc.
|
3
3
|
# Licence:: MIT Licence
|
4
4
|
|
5
5
|
require 'rake'
|
@@ -59,6 +59,9 @@ spec = Gem::Specification.new do |s|
|
|
59
59
|
EOF
|
60
60
|
|
61
61
|
s.add_dependency(%q<reliable-msg>, ["= 1.1.0"])
|
62
|
+
s.add_dependency(%q<rake>)
|
63
|
+
s.add_dependency(%q<activesupport>)
|
64
|
+
s.add_dependency(%q<mongrel>)
|
62
65
|
|
63
66
|
s.has_rdoc = true
|
64
67
|
s.extra_rdoc_files = ["README", "CHANGELOG", 'rails_plugin']
|
@@ -94,6 +97,32 @@ Rake::RDocTask.new { |rdoc|
|
|
94
97
|
rdoc.rdoc_files.include('rails_plugin/**/*.rb')
|
95
98
|
}
|
96
99
|
|
100
|
+
# Spec tasks ----------------------------------------------------------------
|
101
|
+
require 'spec/rake/spectask'
|
102
|
+
|
103
|
+
namespace :spec do
|
104
|
+
%w(local).each do |flavor|
|
105
|
+
desc "Run #{flavor} examples"
|
106
|
+
Spec::Rake::SpecTask.new(flavor) do |t|
|
107
|
+
t.spec_files = FileList["spec/#{flavor}/**/*.rb"]
|
108
|
+
end
|
109
|
+
|
110
|
+
namespace :coverage do
|
111
|
+
desc "Run #{flavor} examples with RCov"
|
112
|
+
Spec::Rake::SpecTask.new(flavor) do |t|
|
113
|
+
t.spec_files = FileList["spec/#{flavor}/**/*.rb"]
|
114
|
+
t.rcov = true
|
115
|
+
excludes = %w(^spec\/)
|
116
|
+
if ENV['GEM_HOME']
|
117
|
+
excludes << Regexp.escape(ENV['GEM_HOME'])
|
118
|
+
end
|
119
|
+
t.rcov_opts = ["--exclude" , excludes.join(","),
|
120
|
+
"--text-summary"]
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
97
126
|
# AP4R release ----------------------------------------------------------------
|
98
127
|
|
99
128
|
desc "Make gem"
|
@@ -103,7 +132,9 @@ task :copy_to_ap4r_from_sample do
|
|
103
132
|
FileUtils.cp(HELLO_WORLD_DIR + '/db/migrate/001_create_table_for_saf.rb', './lib/ap4r/xxx_create_table_for_saf.rb')
|
104
133
|
|
105
134
|
FileUtils.cp(HELLO_WORLD_DIR + '/vendor/plugins/ap4r/init.rb', './rails_plugin/ap4r/init.rb')
|
106
|
-
FileUtils.cp(HELLO_WORLD_DIR + '/vendor/plugins/ap4r/lib/
|
135
|
+
FileUtils.cp(HELLO_WORLD_DIR + '/vendor/plugins/ap4r/lib/async_helper.rb', './rails_plugin/ap4r/lib/async_helper.rb')
|
136
|
+
FileUtils.cp(HELLO_WORLD_DIR + '/vendor/plugins/ap4r/lib/ap4r_client.rb', './rails_plugin/ap4r/lib/ap4r_client.rb')
|
137
|
+
FileUtils.cp(HELLO_WORLD_DIR + '/vendor/plugins/ap4r/lib/message_builder.rb', './rails_plugin/ap4r/lib/message_builder.rb')
|
107
138
|
end
|
108
139
|
|
109
140
|
desc "Make sample tgz"
|
@@ -137,9 +168,9 @@ task :copy_sample do
|
|
137
168
|
end
|
138
169
|
|
139
170
|
task :execute_migration do
|
140
|
-
Dir.chdir('temp/')
|
171
|
+
Dir.chdir('temp/HelloWorld')
|
141
172
|
`rake db:migrate`
|
142
|
-
Dir.chdir('
|
173
|
+
Dir.chdir('../../')
|
143
174
|
end
|
144
175
|
|
145
176
|
task :make_tgz do
|
@@ -157,3 +188,18 @@ task :copy_to_release_dir do
|
|
157
188
|
}
|
158
189
|
end
|
159
190
|
|
191
|
+
# AP4R misc tools ----------------------------------------------------------------
|
192
|
+
|
193
|
+
desc "display code statistics"
|
194
|
+
task :stats do
|
195
|
+
require 'rubygems'
|
196
|
+
require 'active_support'
|
197
|
+
require 'code_statistics'
|
198
|
+
CodeStatistics::TEST_TYPES.concat(["Local specs"])
|
199
|
+
CodeStatistics.new(
|
200
|
+
["Core Sources", "lib"],
|
201
|
+
["Rails plugin", "rails_plugin"],
|
202
|
+
["Scripts", "script"],
|
203
|
+
["Local specs", "spec/local"]
|
204
|
+
).to_s
|
205
|
+
end
|
data/lib/ap4r.rb
CHANGED
data/lib/ap4r/carrier.rb
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
# Author:: Shunichi Shinohara
|
2
|
+
# Copyright:: Copyright (c) 2007 Future Architect Inc.
|
3
|
+
# Licence:: MIT Licence
|
4
|
+
|
5
|
+
require 'yaml'
|
6
|
+
require 'thread'
|
7
|
+
require 'pp'
|
8
|
+
require 'active_support'
|
9
|
+
|
10
|
+
require 'reliable-msg'
|
11
|
+
|
12
|
+
module Ap4r
|
13
|
+
|
14
|
+
# This class aims to balance loads of several reliable-msg servers.
|
15
|
+
# Only P2P channells (queues) are considered so far.
|
16
|
+
# Now reliable-msg can not be accessed from remote (means "not localhost").
|
17
|
+
# Alpha status now.
|
18
|
+
#--
|
19
|
+
# TODO: refactoring with dispatcher.rb, around thread group, etc. 2007/05/09 by shino
|
20
|
+
class Carriers
|
21
|
+
|
22
|
+
def initialize(queue_manager, config, logger, dispatchers)
|
23
|
+
@qm = queue_manager
|
24
|
+
@config = config
|
25
|
+
@logger = logger
|
26
|
+
@group = ThreadGroup.new
|
27
|
+
@dispatchers = dispatchers
|
28
|
+
end
|
29
|
+
|
30
|
+
def start
|
31
|
+
return unless @config
|
32
|
+
@logger.info{ "ready to start carrires with config #{@config.to_yaml}" }
|
33
|
+
@config.each { |remote|
|
34
|
+
remote["threads"].times { |index|
|
35
|
+
Thread.fork(@group, remote, index){|group, remote, index|
|
36
|
+
carrier_loop(group, remote, index)
|
37
|
+
}
|
38
|
+
}
|
39
|
+
}
|
40
|
+
@logger.info{"queue manager has forked all carriers"}
|
41
|
+
end
|
42
|
+
|
43
|
+
def stop
|
44
|
+
@logger.info{"stop_carriers #{@group}"}
|
45
|
+
return unless @group
|
46
|
+
@group.list.each{|d| d[:dying] = true}
|
47
|
+
@group.list.each{|d| d.join }
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
def carrier_loop(group, remote, index)
|
52
|
+
# TODO: refactor structure, 2006/10/06 shino
|
53
|
+
group.add Thread.current
|
54
|
+
@logger.info{ "starting a carrier (index #{index}) for the queue manager #{remote['source_uri']}" }
|
55
|
+
uri = remote['source_uri']
|
56
|
+
until Thread.current[:dying]
|
57
|
+
begin
|
58
|
+
sleep 0.1
|
59
|
+
# TODO check :dying flag here and break, 2006/09/01 shino
|
60
|
+
# TODO cache DRbObject if necessary, 2006/09/01 shino
|
61
|
+
remote_qm = DRb::DRbObject.new_with_uri(uri)
|
62
|
+
queue_name = remote_qm.stale_queue dispatch_targets
|
63
|
+
next unless queue_name
|
64
|
+
|
65
|
+
@logger.debug{ "stale queue name : #{queue_name}" }
|
66
|
+
q = ReliableMsg::Queue.new queue_name, :drb_uri => uri
|
67
|
+
q.get { |m|
|
68
|
+
unless m
|
69
|
+
@logger.debug{ "carrier strikes at the air (T_T)" }
|
70
|
+
next
|
71
|
+
end
|
72
|
+
# @logger.debug{ "carrier gets a message\n#{m.to_yaml}" }
|
73
|
+
|
74
|
+
# TODO: decide the better one, and delete another, 2006/09/01 shino
|
75
|
+
# TODO: or switchable implementation in versions, 2006/10/16 shino
|
76
|
+
|
77
|
+
# version 1: use thread fork so queue manager use a different tx
|
78
|
+
# TODO probably should have a thread as an instance variable or in a thread local, 2006/09/01 shino
|
79
|
+
# Thread.fork(m) {|m|
|
80
|
+
# local_queue = ReliableMsg::Queue.new queue_name
|
81
|
+
# local_queue.put m.object
|
82
|
+
# }.join
|
83
|
+
|
84
|
+
#version 2: store tx and set nil, and resotre tx after putting a message
|
85
|
+
begin
|
86
|
+
tx = Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX]
|
87
|
+
Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = nil
|
88
|
+
# @logger.debug{ "before tx: #{tx}" }
|
89
|
+
ReliableMsg::Queue.new(queue_name).put(m.object)
|
90
|
+
ensure
|
91
|
+
Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = tx
|
92
|
+
# @logger.debug{ "after tx: #{Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX]}" }
|
93
|
+
end
|
94
|
+
}
|
95
|
+
rescue Exception => ex
|
96
|
+
@logger.warn "error in remote-get/local-put #{ex}\n#{ex.backtrace.join("\n\t")}\n"
|
97
|
+
end
|
98
|
+
end
|
99
|
+
@logger.info{"ends a carrier (index #{index}) for the queue manager #{remote['uri']}"}
|
100
|
+
end
|
101
|
+
|
102
|
+
def dispatch_targets
|
103
|
+
@dispatchers.targets
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,326 @@
|
|
1
|
+
# Author:: Shunichi Shinohara
|
2
|
+
# Copyright:: Copyright (c) 2007 Future Architect Inc.
|
3
|
+
# Licence:: MIT Licence
|
4
|
+
|
5
|
+
require 'yaml'
|
6
|
+
require 'thread'
|
7
|
+
require 'pp'
|
8
|
+
require 'active_support'
|
9
|
+
|
10
|
+
require 'uri'
|
11
|
+
require 'net/http'
|
12
|
+
require 'xmlrpc/client'
|
13
|
+
require 'soap/wsdlDriver'
|
14
|
+
|
15
|
+
module Ap4r
|
16
|
+
|
17
|
+
# Represents a group of dispatchers.
|
18
|
+
# Responsibilities are follows:
|
19
|
+
# - polls target queues,
|
20
|
+
# - gets messages from queues, and
|
21
|
+
# - calls a <tt>Dispatchers::Base</tt>'s instance.
|
22
|
+
class Dispatchers
|
23
|
+
|
24
|
+
@@sleep_inverval = 0.1
|
25
|
+
|
26
|
+
@@logger = nil
|
27
|
+
def self.logger
|
28
|
+
@@logger
|
29
|
+
end
|
30
|
+
|
31
|
+
# storage for <tt>Dispatchers::Base</tt>'s subclasses.
|
32
|
+
@@subclasses = {}
|
33
|
+
|
34
|
+
# Stores +klass+ for a dispatch mode +mode+
|
35
|
+
# Each +klass+ is used to create instances to handle dispatching messages.
|
36
|
+
def self.register_dispatcher_class(mode, klass)
|
37
|
+
@@subclasses[mode] = klass
|
38
|
+
end
|
39
|
+
|
40
|
+
# Sum of each dispatcher's target queues.
|
41
|
+
def targets
|
42
|
+
@dispatch_targets
|
43
|
+
end
|
44
|
+
|
45
|
+
def initialize(queue_manager, config, logger_obj)
|
46
|
+
@qm = queue_manager
|
47
|
+
@config = config # (typically) dispatcher section of queues.cfg
|
48
|
+
@@logger ||= logger_obj
|
49
|
+
raise "no configuration specified" unless @config
|
50
|
+
@group = ThreadGroup.new
|
51
|
+
# TODO: needs refinement 2007/05/30 by shino
|
52
|
+
@dispatch_targets = ""
|
53
|
+
end
|
54
|
+
|
55
|
+
# Starts every dispatcher.
|
56
|
+
# If an exception is detected, this method raise it through with logging.
|
57
|
+
#
|
58
|
+
def start
|
59
|
+
begin
|
60
|
+
logger.info{ "about to start dispatchers with config\n#{@config.to_yaml}" }
|
61
|
+
@config.each{ |conf|
|
62
|
+
conf["threads"].to_i.times { |index|
|
63
|
+
Thread.fork(@group, conf, index){|group, conf, index|
|
64
|
+
dispatching_loop(group, conf, index)
|
65
|
+
}
|
66
|
+
}
|
67
|
+
@dispatch_targets.concat(conf["targets"]).concat(';')
|
68
|
+
logger.debug{ "dispatch targets are : #{@dispatch_targets}" }
|
69
|
+
}
|
70
|
+
logger.info "queue manager has forked dispatchers"
|
71
|
+
rescue Exception => err
|
72
|
+
logger.warn{"Error in starting dipatchers #{err}"}
|
73
|
+
logger.warn{err.backtrace.join("\n")}
|
74
|
+
raise err
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
# Stops every dispatcher.
|
79
|
+
# Current implementation makes just dying flags up.
|
80
|
+
# Some threads don't stop quickly in some cases such as blocking at socket read.
|
81
|
+
#--
|
82
|
+
# TODO: needs forced mode? 2007/05/09 by shino
|
83
|
+
def stop
|
84
|
+
logger.info{"stop_dispatchers #{@group}"}
|
85
|
+
return unless @group
|
86
|
+
@group.list.each {|d| d[:dying] = true}
|
87
|
+
@group.list.each {|d| d.join }
|
88
|
+
@dispatch_targets = ""
|
89
|
+
end
|
90
|
+
|
91
|
+
private
|
92
|
+
|
93
|
+
# Creates and returns an appropriate instace.
|
94
|
+
# If no class for +dispatch_mode+, raises an exception.
|
95
|
+
def get_dispather_instance(dispatch_mode, message, conf_per_targets)
|
96
|
+
klass = @@subclasses[dispatch_mode]
|
97
|
+
raise "undefined dispatch mode #{m.headers[:mode]}" unless klass
|
98
|
+
klass.new(message, conf_per_targets)
|
99
|
+
end
|
100
|
+
|
101
|
+
# Defines the general structure for each dispatcher thread
|
102
|
+
# from begging to end.
|
103
|
+
def dispatching_loop(group, conf, index)
|
104
|
+
group.add(Thread.current)
|
105
|
+
mq = ::ReliableMsg::MultiQueue.new(conf["targets"])
|
106
|
+
logger.info{ "start dispatcher: targets= #{mq}, index= #{index})" }
|
107
|
+
until Thread.current[:dying]
|
108
|
+
# TODO: change sleep interval depending on last result? 2007/05/09 by shino
|
109
|
+
sleep @@sleep_inverval
|
110
|
+
# logger.debug{ "try dispatch #{mq} #{mq.name}" }
|
111
|
+
# TODO: needs timeout?, 2006/10/16 shino
|
112
|
+
begin
|
113
|
+
mq.get{|m|
|
114
|
+
unless m
|
115
|
+
logger.debug{"message is nul"}
|
116
|
+
break
|
117
|
+
end
|
118
|
+
logger.debug{"dispatcher get message\n#{m.to_yaml}"}
|
119
|
+
response = get_dispather_instance(m.headers[:dispatch_mode], m, conf).call
|
120
|
+
logger.debug{"dispatcher get response\n#{response.to_yaml}"}
|
121
|
+
}
|
122
|
+
rescue Exception => err
|
123
|
+
logger.warn("dispatch err #{err.inspect}")
|
124
|
+
logger.warn(err.backtrace.join("\n"))
|
125
|
+
end
|
126
|
+
end
|
127
|
+
logger.info{"end dispatcher #{mq} (index #{index})"}
|
128
|
+
end
|
129
|
+
|
130
|
+
def logger
|
131
|
+
@@logger
|
132
|
+
end
|
133
|
+
|
134
|
+
|
135
|
+
# A base class for dispathcer classes associated with each <tt>dispatch_mode</tt>.
|
136
|
+
# Responsibilities of subclasses are to implement following methods, only +invoke+
|
137
|
+
# is mandatory and others are optional (no operations by default).
|
138
|
+
# * +modify_message+ to preprocess a message, e.g. rewirte URL or discard message.
|
139
|
+
# * +invoke+ to execute main logic, e.g. HTTP POST call. *mandatory*
|
140
|
+
# * +validate_response+ to judge whether +invoke+ finished successfully.
|
141
|
+
# * +response+ to return the result of +invoke+ process.
|
142
|
+
class Base
|
143
|
+
|
144
|
+
# Difine a constant +DISPATCH_MODE+ to value 'mode_symbol' and
|
145
|
+
# add self to a Converters list.
|
146
|
+
def self.dispatch_mode(mode_symbol)
|
147
|
+
self.const_set(:DISPATCH_MODE, mode_symbol)
|
148
|
+
::Ap4r::Dispatchers.register_dispatcher_class(mode_symbol, self)
|
149
|
+
end
|
150
|
+
|
151
|
+
# Takes
|
152
|
+
# * +message+: from a queue
|
153
|
+
# * +conf+: configuration from dispatchers section
|
154
|
+
#--
|
155
|
+
# TODO: Subclass should have +conf+ instead of instance? 2007/06/06 by shino
|
156
|
+
def initialize(message, conf)
|
157
|
+
@message = message
|
158
|
+
@conf = conf
|
159
|
+
end
|
160
|
+
|
161
|
+
# Entry facade for each message processing.
|
162
|
+
# Modifies, invokes (maybe remote), validates, and responds.
|
163
|
+
def call
|
164
|
+
# TODO: rename to more appropriate one 2007/05/10 by shino
|
165
|
+
self.modify_message
|
166
|
+
logger.debug{"Ap4r::Dispatcher after modification\n#{@message.to_yaml}"}
|
167
|
+
self.invoke
|
168
|
+
self.validate_response
|
169
|
+
self.response
|
170
|
+
end
|
171
|
+
|
172
|
+
# Modifies message.
|
173
|
+
# Now only URL modification is implemented.
|
174
|
+
def modify_message
|
175
|
+
modification_rules = @conf["modify_rules"]
|
176
|
+
return unless modification_rules
|
177
|
+
modify_url(modification_rules)
|
178
|
+
end
|
179
|
+
|
180
|
+
# Main logic of message processing.
|
181
|
+
# Maybe calls remote, e.g. HTTP request.
|
182
|
+
def invoke
|
183
|
+
# TODO: rename to more appropriate one 2007/05/10 by shino
|
184
|
+
raise 'must be implemented in subclasses'
|
185
|
+
end
|
186
|
+
|
187
|
+
def validate_response
|
188
|
+
# nop
|
189
|
+
end
|
190
|
+
|
191
|
+
# Returns response.
|
192
|
+
# The return value is also the return value of +call+ method.
|
193
|
+
# By default impl, the instance variable <tt>@response</tt> is used.
|
194
|
+
def response
|
195
|
+
@response
|
196
|
+
end
|
197
|
+
|
198
|
+
private
|
199
|
+
def logger
|
200
|
+
::Ap4r::Dispatchers.logger
|
201
|
+
end
|
202
|
+
|
203
|
+
# Modifies <tt>:target_url</tt> according to a rule.
|
204
|
+
# TODO: +proc+ in configuration is eval'ed every time. 2007/06/06 by shino
|
205
|
+
def modify_url(modification_rules)
|
206
|
+
proc_for_url = modification_rules["url"]
|
207
|
+
return unless proc_for_url
|
208
|
+
|
209
|
+
url = URI.parse(@message.headers[:target_url])
|
210
|
+
eval(proc_for_url).call(url)
|
211
|
+
@message.headers[:target_url] = url.to_s
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
# Dispatches via a raw HTTP protocol.
|
216
|
+
# Current implementation uses only a POST method, irrespective of
|
217
|
+
# <tt>options[:target_method]</tt>.
|
218
|
+
#
|
219
|
+
# Determination of "success" is two fold:
|
220
|
+
# * status code should be exactly 200, other codes (including 201-2xx) are
|
221
|
+
# treated as error, and
|
222
|
+
# * body should include a string "true"
|
223
|
+
#
|
224
|
+
class Http < Base
|
225
|
+
dispatch_mode :HTTP
|
226
|
+
|
227
|
+
def invoke
|
228
|
+
# TODO: should be added some request headers 2006/10/12 shino
|
229
|
+
# e.g. X-Ap4r-Version, Accept(need it?)
|
230
|
+
# TODO: Now supports POST only, 2006/10/12 shino
|
231
|
+
@response = nil
|
232
|
+
uri = URI.parse(@message[:target_url])
|
233
|
+
headers = make_header
|
234
|
+
|
235
|
+
Net::HTTP.start(uri.host, uri.port) do |http|
|
236
|
+
@response, = http.post(uri.path, @message.object, headers)
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
def make_header
|
241
|
+
headers = { }
|
242
|
+
@message.headers.map do |k,v|
|
243
|
+
s = StringScanner.new(k.to_s)
|
244
|
+
s.scan(/\Ahttp_header_/)
|
245
|
+
headers[s.post_match] = v if s.post_match
|
246
|
+
end
|
247
|
+
headers
|
248
|
+
end
|
249
|
+
|
250
|
+
def validate_response
|
251
|
+
logger.debug{"response status [#{@response.code} #{@response.message}]"}
|
252
|
+
validate_response_status(Net::HTTPOK)
|
253
|
+
validate_response_body(/true/)
|
254
|
+
end
|
255
|
+
|
256
|
+
# Checks whether the response status is a kind of +status_kind+.
|
257
|
+
# +status_kind+ should be one of <tt>Net::HTTPRespose</tt>'s subclasses.
|
258
|
+
def validate_response_status(status_kind)
|
259
|
+
#TODO: make the difinition of success variable, 2006/10/13 shino
|
260
|
+
unless @response.kind_of?(status_kind)
|
261
|
+
error_message = "HTTP Response FAILURE, " +
|
262
|
+
"status [#{@response.code} #{@response.message}]"
|
263
|
+
logger.error(error_message)
|
264
|
+
logger.info{@response.to_yaml}
|
265
|
+
#TODO: must create AP4R specific Exception class, 2006/10/12 shino
|
266
|
+
raise StandardError.new(error_message)
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
# Checks whether the response body includes +pattern+.
|
271
|
+
# +pattern+ should be a regular expression.
|
272
|
+
def validate_response_body(pattern)
|
273
|
+
unless @response.body =~ pattern
|
274
|
+
error_message = "HTTP Response FAILURE, status" +
|
275
|
+
" [#{@response.code} #{@response.message}], body [#{@response.body}]"
|
276
|
+
#TODO: Refactor error logging, 2006/10/13 shino
|
277
|
+
logger.error(error_message)
|
278
|
+
logger.info{@response.to_yaml}
|
279
|
+
#TODO: must create AP4R specific Exception class, 2006/10/12 shino
|
280
|
+
raise StandardError.new(error_message)
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
end
|
285
|
+
|
286
|
+
# Dispatches via XML-RPC protocol.
|
287
|
+
# Uses +XMLRPC+ library.
|
288
|
+
#
|
289
|
+
# The call result is judged as
|
290
|
+
# * "failure" if the first element of <tt>XMLRPC::Client#call2</tt> result
|
291
|
+
# is false, and
|
292
|
+
# * "success" otherwise.
|
293
|
+
#
|
294
|
+
class XmlRpc < Base
|
295
|
+
dispatch_mode :XMLRPC
|
296
|
+
|
297
|
+
def invoke
|
298
|
+
endpoint = @message[:target_url]
|
299
|
+
client = XMLRPC::Client.new2(endpoint)
|
300
|
+
@success, @response = client.call2(@message[:target_action], @message.object)
|
301
|
+
end
|
302
|
+
|
303
|
+
def validate_response
|
304
|
+
raise @response unless @success
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
# Dispatches via SOAP protocol.
|
309
|
+
# Uses +SOAP+ library.
|
310
|
+
#
|
311
|
+
# The call result is judged as
|
312
|
+
# * "success" if <tt>SOAP::WSDLDrive#send</tt> finishes without an exception, and
|
313
|
+
# * "failuar" otherwise.
|
314
|
+
#
|
315
|
+
class SOAP < Base
|
316
|
+
dispatch_mode :SOAP
|
317
|
+
|
318
|
+
def invoke
|
319
|
+
# TODO: nice to cache drivers probably 2007/05/09 by shino
|
320
|
+
driver = ::SOAP::WSDLDriverFactory.new(@message[:target_url]).create_rpc_driver
|
321
|
+
driver.send(@message[:target_action], @message.object)
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
end
|
326
|
+
end
|