ap4r 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +6 -0
- data/MIT-LICENSE +21 -0
- data/README +52 -0
- data/Rakefile +25 -0
- data/bin/ap4r +0 -0
- data/bin/irm.cmd +2 -0
- data/bin/loop.cmd +3 -0
- data/bin/loop.rb +8 -0
- data/bin/queues.cmd +2 -0
- data/bin/queues.rb +3 -0
- data/bin/stop.cmd +1 -0
- data/configs/log4r.yaml +61 -0
- data/configs/queues.cfg +20 -0
- data/configs/queues_disk.cfg +15 -0
- data/configs/queues_mysql.cfg +15 -0
- data/lib/ap4r.rb +22 -0
- data/lib/ap4r/message_store_ext.rb +29 -0
- data/lib/ap4r/multi_queue.rb +21 -0
- data/lib/ap4r/queue_manager_ext.rb +175 -0
- data/lib/ap4r/queue_manager_ext_debug.rb +48 -0
- data/lib/ap4r/retention_history.rb +57 -0
- data/lib/ap4r/start_with_log4r.rb +30 -0
- data/lib/ap4r/utils/irm.rb +109 -0
- data/lib/ap4r/utils/loc.rb +8 -0
- metadata +79 -0
data/MIT-LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
Copyright (c) 2006 ???
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
21
|
+
|
data/README
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
We are pleased to announce the FIRST release of AP4R.
|
2
|
+
|
3
|
+
http://rubyforge.org/projects/ap4r/
|
4
|
+
|
5
|
+
|
6
|
+
== DESCRIPTION
|
7
|
+
|
8
|
+
AP4R, Asynchronous Processing for Ruby, is the implementation of
|
9
|
+
reliable asynchous message processing.
|
10
|
+
|
11
|
+
|
12
|
+
Using asynchronous processing, we can cut down turn-around-time
|
13
|
+
of web applications by queuing, or can utilize more machine power
|
14
|
+
by load-balancing.
|
15
|
+
|
16
|
+
AP4R provides message queuing, and message dispatching.
|
17
|
+
Also AP4R nicely ties with your Ruby on Rails applications.
|
18
|
+
|
19
|
+
|
20
|
+
== INSTALLATION
|
21
|
+
|
22
|
+
sudo gem install ap4r
|
23
|
+
|
24
|
+
|
25
|
+
== FEATURES
|
26
|
+
|
27
|
+
* Business logics can be implemented as simple Web apps, or ruby code,
|
28
|
+
whether it's called asynchronously or synchronously.
|
29
|
+
|
30
|
+
* Asynchronous messaging are reliable
|
31
|
+
by PersistRDBMS (now MySQL only) or file persistence.
|
32
|
+
|
33
|
+
* Load balancing over multiple AP4R processes on single/multiple servers
|
34
|
+
is supported.
|
35
|
+
|
36
|
+
* Asynchronous processing is via various protocols, such as XML-RPC, SOAP,
|
37
|
+
HTTP PUT, and more. (now implemented just as XML-RPC)
|
38
|
+
|
39
|
+
|
40
|
+
== CHANGES
|
41
|
+
|
42
|
+
This is the first release!
|
43
|
+
|
44
|
+
|
45
|
+
== ACKNOWLEDGEMENT
|
46
|
+
|
47
|
+
K.K. and S.S. are on the payroll of Future System Consulting Corp. Japan.
|
48
|
+
|
49
|
+
|
50
|
+
--
|
51
|
+
Kato, Kiwamu kato.kiwamu@future.co.jp
|
52
|
+
Shinohara, Shunichi shinohara.shunichi@future.co.jp
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
require 'rake/testtask'
|
4
|
+
require 'rake/rdoctask'
|
5
|
+
require 'rake/packagetask'
|
6
|
+
require 'rake/gempackagetask'
|
7
|
+
require 'rake/contrib/rubyforgepublisher'
|
8
|
+
|
9
|
+
Gem::Specification.new do |s|
|
10
|
+
s.name = %q{orca}
|
11
|
+
s.version = "0.1.0"
|
12
|
+
s.date = %q{2006-05-18}
|
13
|
+
s.summary = %q{Messageing library with Rails for asynchronous process.}
|
14
|
+
s.email = %q{kato.kiwamu@future.co.jp}
|
15
|
+
s.homepage = %q{http://orca2.rtfa.local/orca/}
|
16
|
+
s.rubyforge_project = %q{}
|
17
|
+
s.description = %q{Messageing library with Rails for asynchronous proces.}
|
18
|
+
s.autorequire = %q{orca}
|
19
|
+
s.has_rdoc = true
|
20
|
+
s.authors = ["Shino,Kato"]
|
21
|
+
s.files = Dir.glob("{script,lib,logs,utils}/**/*") << "CHANGELOG" << "log4r.yaml" << "MIT-LICENSE" << "queues.cfg" << "queues1.cfg" << "queues2.cfg" << "Rakefile" << "README"
|
22
|
+
s.requirements = ["none"]
|
23
|
+
end
|
24
|
+
|
25
|
+
|
data/bin/ap4r
ADDED
File without changes
|
data/bin/irm.cmd
ADDED
data/bin/loop.cmd
ADDED
data/bin/loop.rb
ADDED
data/bin/queues.cmd
ADDED
data/bin/queues.rb
ADDED
data/bin/stop.cmd
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
./queues manager stop
|
data/configs/log4r.yaml
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
purpose : Test
|
2
|
+
description: This is the YAML doc
|
3
|
+
this yaml has many many unnecessary setting
|
4
|
+
for testing/explanation only.
|
5
|
+
edit properly for normal use.
|
6
|
+
|
7
|
+
---
|
8
|
+
|
9
|
+
log4r_config:
|
10
|
+
# define all pre config ...
|
11
|
+
pre_config:
|
12
|
+
global:
|
13
|
+
level: DEBUG
|
14
|
+
root :
|
15
|
+
level: DEBUG
|
16
|
+
parameters:
|
17
|
+
- name : x
|
18
|
+
value : aaa
|
19
|
+
- name : y
|
20
|
+
value : bbb
|
21
|
+
|
22
|
+
# define all loggers ...
|
23
|
+
loggers:
|
24
|
+
- name : qm_logger
|
25
|
+
level : DEBUG
|
26
|
+
additive : 'false'
|
27
|
+
trace : 'false'
|
28
|
+
outputters:
|
29
|
+
- stderr
|
30
|
+
- logfile
|
31
|
+
|
32
|
+
- name : yourlogger #not used yet...
|
33
|
+
level : INFO
|
34
|
+
outputters:
|
35
|
+
- stderr
|
36
|
+
- logfile
|
37
|
+
|
38
|
+
# define all outputters (incl. formatters)
|
39
|
+
outputters:
|
40
|
+
- type : StderrOutputter
|
41
|
+
name : stderr
|
42
|
+
level : DEBUG
|
43
|
+
only_at :
|
44
|
+
- INFO
|
45
|
+
- WARN
|
46
|
+
- FATAL
|
47
|
+
formatter:
|
48
|
+
date_pattern: '%y%m%d %H:%M:%S'
|
49
|
+
pattern : '%d %l: %m '
|
50
|
+
type : PatternFormatter
|
51
|
+
|
52
|
+
- type : DateFileOutputter
|
53
|
+
name : logfile
|
54
|
+
level : DEBUG
|
55
|
+
date_pattern: '%Y%m%d'
|
56
|
+
trunc : 'false'
|
57
|
+
dirname : "#{HOME}/logs"
|
58
|
+
formatter :
|
59
|
+
date_pattern: '%y%m%d %H:%M:%S'
|
60
|
+
pattern : '%d %l: %m'
|
61
|
+
type : PatternFormatter
|
data/configs/queues.cfg
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
---
|
2
|
+
store:
|
3
|
+
type: mysql
|
4
|
+
host: localhost
|
5
|
+
database: test
|
6
|
+
username: test
|
7
|
+
password:
|
8
|
+
drb:
|
9
|
+
host:
|
10
|
+
port: 6438
|
11
|
+
acl: allow 127.0.0.1
|
12
|
+
dispatchers:
|
13
|
+
-
|
14
|
+
targets: queue.*
|
15
|
+
threads: 1
|
16
|
+
#carriers:
|
17
|
+
# -
|
18
|
+
# source_uri: druby://another.host.local:6438
|
19
|
+
# threads: 1
|
20
|
+
|
data/lib/ap4r.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
version = "> 0"
|
3
|
+
if ARGV.size > 0 && ARGV[0][0]==95 && ARGV[0][-1]==95
|
4
|
+
if Gem::Version.correct?(ARGV[0][1..-2])
|
5
|
+
version = ARGV[0][1..-2]
|
6
|
+
ARGV.shift
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
require_gem 'reliable-msg', version
|
11
|
+
|
12
|
+
hack = true
|
13
|
+
debug_hack = true
|
14
|
+
|
15
|
+
if hack
|
16
|
+
require "ap4r/queue_manager_ext"
|
17
|
+
end
|
18
|
+
|
19
|
+
if debug_hack
|
20
|
+
require "ap4r/queue_manager_ext_debug"
|
21
|
+
end
|
22
|
+
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module ReliableMsg
|
2
|
+
module MessageStore
|
3
|
+
class Base
|
4
|
+
|
5
|
+
def stale_queue targets
|
6
|
+
queue_names = targets.split(/[\s;]/).
|
7
|
+
select{|queue| !queue.empty? }.
|
8
|
+
map{|queue| queue.strip! || queue }
|
9
|
+
|
10
|
+
queue_and_created = @queues.select{|queue, messages|
|
11
|
+
messages && (messages.size > 0) &&
|
12
|
+
queue_names.any?{|mq|
|
13
|
+
if mq[-1] == '*'[0]
|
14
|
+
queue[0..(mq.size-2)] == mq[0..-2]
|
15
|
+
else
|
16
|
+
queue == mq
|
17
|
+
end
|
18
|
+
}
|
19
|
+
}.map{|queue, messages|
|
20
|
+
[queue, messages[0][:created]]
|
21
|
+
}.min {|q1, q2|
|
22
|
+
q1[1] <=> q2[1]
|
23
|
+
}
|
24
|
+
queue_and_created ? queue_and_created[0] : nil
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module ReliableMsg
|
2
|
+
class MultiQueue < Client
|
3
|
+
def initialize multi_queue, options = nil
|
4
|
+
@multi_queue = multi_queue
|
5
|
+
@options = options
|
6
|
+
end
|
7
|
+
|
8
|
+
def get selector = nil, &block
|
9
|
+
queue_name = repeated {|qm|
|
10
|
+
qm.stale_queue @multi_queue
|
11
|
+
}
|
12
|
+
return nil unless queue_name
|
13
|
+
queue = Queue.new queue_name, @options
|
14
|
+
queue.get selector, &block
|
15
|
+
end
|
16
|
+
|
17
|
+
def name
|
18
|
+
@multi_queue
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,175 @@
|
|
1
|
+
require 'ap4r/message_store_ext'
|
2
|
+
require 'ap4r/multi_queue'
|
3
|
+
require 'ap4r/retention_history'
|
4
|
+
|
5
|
+
require 'soap/wsdlDriver'
|
6
|
+
require 'xmlrpc/client'
|
7
|
+
require 'active_support'
|
8
|
+
|
9
|
+
require 'yaml'
|
10
|
+
require 'thread'
|
11
|
+
require 'pp'
|
12
|
+
|
13
|
+
module ReliableMsg
|
14
|
+
module LifecycleListener
|
15
|
+
|
16
|
+
end
|
17
|
+
|
18
|
+
class QueueManager
|
19
|
+
def stale_queue multi_queue
|
20
|
+
@store.stale_queue multi_queue
|
21
|
+
end
|
22
|
+
|
23
|
+
alias :start_original :start
|
24
|
+
alias :stop_original :stop
|
25
|
+
alias :initialize_original :initialize
|
26
|
+
|
27
|
+
def initialize options = nil
|
28
|
+
initialize_original options
|
29
|
+
@global_lock ||= Mutex.new
|
30
|
+
@lifecycle_listeners = []
|
31
|
+
RetentionHistory.new(self, @logger, @config)
|
32
|
+
end
|
33
|
+
|
34
|
+
def add_lifecycle_listener listener, iv_name, attr_mode = 'reader'
|
35
|
+
@lifecycle_listeners << listener
|
36
|
+
instance_variable_set "@#{iv_name}".to_sym, listener
|
37
|
+
self.class.class_eval("attr_#{attr_mode} :#{iv_name}")
|
38
|
+
end
|
39
|
+
|
40
|
+
def start
|
41
|
+
@global_lock.synchronize do
|
42
|
+
return if @@active == self
|
43
|
+
@dispatch_targets = ''
|
44
|
+
start_original
|
45
|
+
start_dispatchers
|
46
|
+
start_carriers
|
47
|
+
@lifecycle_listeners.each {|l| l.start }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def stop
|
52
|
+
@global_lock.synchronize do
|
53
|
+
return unless @@active == self
|
54
|
+
@lifecycle_listeners.each {|l| l.stop }
|
55
|
+
stop_carriers
|
56
|
+
stop_dispatchers
|
57
|
+
stop_original
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def start_dispatchers
|
62
|
+
return unless @config.disps
|
63
|
+
@logger.info{ "about to start dispatchers with config #{@config.disps.to_yaml}" }
|
64
|
+
@disps = ThreadGroup.new
|
65
|
+
@config.dispatchers.each{ |conf|
|
66
|
+
conf["threads"].to_i.times { |index|
|
67
|
+
Thread.fork(@disps, conf["targets"], index, @logger){|group, targets, index, logger|
|
68
|
+
group.add Thread.current
|
69
|
+
mq = MultiQueue.new targets
|
70
|
+
logger.info{ "start dispatcher: targets= #{mq} (index #{index})" }
|
71
|
+
until Thread.current[:dying]
|
72
|
+
sleep 0.1
|
73
|
+
# logger.debug{ "try dispatch #{mq} #{mq.name}" }
|
74
|
+
begin
|
75
|
+
mq.get{|m|
|
76
|
+
logger.debug{"dispatcher get message\n#{m.to_yaml}"} if m
|
77
|
+
# version 1 SOAP
|
78
|
+
# driver = SOAP::WSDLDriverFactory.new(m[:target_url]).create_rpc_driver
|
79
|
+
# logger.debug(driver)
|
80
|
+
# response = driver.send(m[:target_action], m.object)
|
81
|
+
# version 2 XML-RPC
|
82
|
+
endpoint = m[:target_url]
|
83
|
+
client = XMLRPC::Client.new2(endpoint)
|
84
|
+
success, response = client.call2(m[:target_action], m.object)
|
85
|
+
logger.debug{"dispatcher get response\n#{response.to_yaml}"}
|
86
|
+
raise response unless success
|
87
|
+
}
|
88
|
+
rescue Exception => err
|
89
|
+
logger.warn("dispatch err #{err.inspect}")
|
90
|
+
logger.warn(err.backtrace.join("\n"))
|
91
|
+
end
|
92
|
+
end
|
93
|
+
logger.info "end dispatcher #{mq} (index #{index})"
|
94
|
+
}
|
95
|
+
}
|
96
|
+
@dispatch_targets.concat(conf["targets"]).concat(';')
|
97
|
+
@logger.info{ "dispatch queues : #{@dispatch_targets}" }
|
98
|
+
}
|
99
|
+
@logger.info "queue manager has forked dispatchers"
|
100
|
+
end
|
101
|
+
|
102
|
+
def stop_dispatchers
|
103
|
+
return unless @disps
|
104
|
+
@disps.list.each {|d| d[:dying] = true}
|
105
|
+
@disps.list.each {|d| d.join }
|
106
|
+
end
|
107
|
+
|
108
|
+
def start_carriers
|
109
|
+
conf = @config.carriers
|
110
|
+
return unless conf
|
111
|
+
|
112
|
+
@logger.info{ "ready to start carrires with config #{conf.to_yaml}" }
|
113
|
+
@carriers = ThreadGroup.new
|
114
|
+
conf.each { |remote|
|
115
|
+
remote["threads"].times { |index|
|
116
|
+
Thread.fork(@carriers, remote, index, @logger){|group, remote, index, logger|
|
117
|
+
group.add Thread.current
|
118
|
+
logger.info{ "starting a carrier (index #{index}) for the queue manager #{remote['source_uri']}" }
|
119
|
+
uri = remote['source_uri']
|
120
|
+
until Thread.current[:dying]
|
121
|
+
begin
|
122
|
+
sleep 0.1
|
123
|
+
#TODO check :dying flag here and break
|
124
|
+
#TODO cache DRbObject if necessary
|
125
|
+
remote_qm = DRb::DRbObject.new_with_uri(uri)
|
126
|
+
queue_name = remote_qm.stale_queue @dispatch_targets
|
127
|
+
next unless queue_name
|
128
|
+
|
129
|
+
logger.debug{ "stale queue name : #{queue_name}" }
|
130
|
+
q = ReliableMsg::Queue.new queue_name, :drb_uri => uri
|
131
|
+
q.get { |m|
|
132
|
+
unless m
|
133
|
+
logger.debug{ "carrier strikes at the air (T_T)" }
|
134
|
+
next
|
135
|
+
end
|
136
|
+
#logger.debug{ "carrier gets a message\n#{m.to_yaml}" }
|
137
|
+
|
138
|
+
#version 1: use thread fork so queue manager use a different tx
|
139
|
+
# TODO probably should have a thread as an instance variable or in a thread local
|
140
|
+
#Thread.fork(m) {|m|
|
141
|
+
# local_queue = ReliableMsg::Queue.new queue_name
|
142
|
+
# local_queue.put m.object
|
143
|
+
#}.join
|
144
|
+
|
145
|
+
#version 2: store tx and set nil, and resotre tx after putting a message
|
146
|
+
begin
|
147
|
+
tx = Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX]
|
148
|
+
Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = nil
|
149
|
+
#logger.debug{ "before tx: #{tx}" }
|
150
|
+
ReliableMsg::Queue.new(queue_name).put(m.object)
|
151
|
+
ensure
|
152
|
+
Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX] = tx
|
153
|
+
#logger.debug{ "after tx: #{Thread.current[ReliableMsg::Client::THREAD_CURRENT_TX]}" }
|
154
|
+
end
|
155
|
+
}
|
156
|
+
rescue Exception => ex
|
157
|
+
logger.warn "error in remote-get/local-put #{ex}\n#{ex.backtrace.join("\n\t")}\n"
|
158
|
+
end
|
159
|
+
end
|
160
|
+
logger.info "ends carrier (index #{index}) for the queue manager #{remote['uri']}"
|
161
|
+
}
|
162
|
+
}
|
163
|
+
}
|
164
|
+
@logger.info "queue manager has forked carriers"
|
165
|
+
end
|
166
|
+
|
167
|
+
def stop_carriers
|
168
|
+
@logger.info "stop_carriers #{@carriers}"
|
169
|
+
return unless @carriers
|
170
|
+
@carriers.list.each {|d| d[:dying] = true}
|
171
|
+
@carriers.list.each {|d| d.join }
|
172
|
+
end
|
173
|
+
|
174
|
+
end
|
175
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'drb/drb'
|
2
|
+
|
3
|
+
module ReliableMsg
|
4
|
+
class QueueManager
|
5
|
+
attr_reader :store, :transactions, :mutex, :config
|
6
|
+
|
7
|
+
def eval_to_inspect code, inspect_mode = :inspect
|
8
|
+
# TODO: too sloppy implementation
|
9
|
+
result = Thread.new(code, inspect_mode){ |c, mode|
|
10
|
+
$SAFE = 4
|
11
|
+
result = self.instance_eval(c)
|
12
|
+
}.value
|
13
|
+
case inspect_mode
|
14
|
+
when :inspect
|
15
|
+
result.inspect
|
16
|
+
when :yaml
|
17
|
+
result.to_yaml
|
18
|
+
when :json
|
19
|
+
result.to_json
|
20
|
+
when :xml
|
21
|
+
result.to_xml
|
22
|
+
else
|
23
|
+
result.inspect
|
24
|
+
end
|
25
|
+
end
|
26
|
+
alias e2i eval_to_inspect
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
module MessageStore
|
31
|
+
class Base
|
32
|
+
include DRbUndumped
|
33
|
+
attr_reader :mutex, :queues, :topics, :cache
|
34
|
+
|
35
|
+
alias activate_original activate
|
36
|
+
|
37
|
+
def activate
|
38
|
+
activate_original
|
39
|
+
@mutex.extend DRbUndumped
|
40
|
+
# @queues.extend DRbUndumped
|
41
|
+
# @topics.extend DRbUndumped
|
42
|
+
# @cache.extend DRbUndumped
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'active_support'
|
2
|
+
|
3
|
+
module ReliableMsg
|
4
|
+
class RetentionHistory
|
5
|
+
include DRbUndumped
|
6
|
+
|
7
|
+
LOOP_INTERVAL = 3.seconds
|
8
|
+
SHELF_LIFE = 1.hour
|
9
|
+
# SHELF_LIFE = 10.seconds
|
10
|
+
attr_reader :data
|
11
|
+
|
12
|
+
def initialize(qm, logger, config)
|
13
|
+
@data = {}
|
14
|
+
@qm = qm
|
15
|
+
@logger = logger
|
16
|
+
@config = config
|
17
|
+
@qm.add_lifecycle_listener(self, 'retention')
|
18
|
+
end
|
19
|
+
|
20
|
+
def start
|
21
|
+
@collector = Thread.start do
|
22
|
+
loop do
|
23
|
+
begin
|
24
|
+
sleep LOOP_INTERVAL
|
25
|
+
collect
|
26
|
+
sweep
|
27
|
+
rescue Exception => ex
|
28
|
+
@logger.warn("message retention history (collect) #{ex.inspect}")
|
29
|
+
@logger.warn(ex.backtrace.join("\n\t"))
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def stop
|
36
|
+
@collector.terminate
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
def collect
|
41
|
+
@qm.store.queues.each {|name, messages|
|
42
|
+
new_data(name, messages.size)
|
43
|
+
}
|
44
|
+
end
|
45
|
+
|
46
|
+
def new_data(queue_name, count)
|
47
|
+
(@data[queue_name.to_sym] ||= []) << [Time.now, count]
|
48
|
+
end
|
49
|
+
|
50
|
+
def sweep
|
51
|
+
limit = Time.now - SHELF_LIFE
|
52
|
+
@data.each do |q, time_count|
|
53
|
+
time_count.delete_if {|t,c| t < limit }
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'log4r'
|
3
|
+
require 'log4r/yamlconfigurator'
|
4
|
+
# we use various outputters, so require them, otherwise config chokes
|
5
|
+
require 'log4r/outputter/datefileoutputter'
|
6
|
+
require 'log4r/outputter/emailoutputter'
|
7
|
+
include Log4r
|
8
|
+
|
9
|
+
cfg = YamlConfigurator
|
10
|
+
cfg['HOME'] = '.'
|
11
|
+
|
12
|
+
cfg.load_yaml_file('log4r.yaml')
|
13
|
+
|
14
|
+
log4r_logger = Log4r::Logger.get 'mylogger'
|
15
|
+
#log4r_logger.outputters = Log4r::Outputter.stdout
|
16
|
+
|
17
|
+
|
18
|
+
require 'ap4r'
|
19
|
+
manager = ReliableMsg::QueueManager.new( {:config => 'queues.cfg',
|
20
|
+
:logger => log4r_logger })
|
21
|
+
manager.start
|
22
|
+
|
23
|
+
begin
|
24
|
+
while manager.alive?
|
25
|
+
sleep 3
|
26
|
+
end
|
27
|
+
rescue Interrupt
|
28
|
+
manager.stop
|
29
|
+
end
|
30
|
+
|
@@ -0,0 +1,109 @@
|
|
1
|
+
# interactive reliable message
|
2
|
+
# version 0.1.1
|
3
|
+
# written by shino
|
4
|
+
|
5
|
+
require 'reliable-msg-hack'
|
6
|
+
|
7
|
+
require 'pathname'
|
8
|
+
require 'uri'
|
9
|
+
|
10
|
+
$KCODE = 'u'
|
11
|
+
|
12
|
+
$config = 'queues.cfg'
|
13
|
+
$drb_host = 'localhost'
|
14
|
+
$drb_port = 6438
|
15
|
+
|
16
|
+
module Kernel
|
17
|
+
DEFAULT_QUEUE_PREFIX = 'queue.test.'
|
18
|
+
DEFAULT_QUEUE_SUFFIX = 'default'
|
19
|
+
DEFAULT_MULTI_QUEUE = DEFAULT_QUEUE_PREFIX + '*'
|
20
|
+
|
21
|
+
def irm_queue_manager config = $config
|
22
|
+
raise "NOT exist : #{config}" unless Pathname.new(config).exist?
|
23
|
+
DRb::DRbObject.new_with_uri irm_drb_uri
|
24
|
+
end
|
25
|
+
|
26
|
+
def irm_queue_manager_stop config = $config
|
27
|
+
manager = irm_queue_manager config
|
28
|
+
begin
|
29
|
+
manager.stop
|
30
|
+
rescue DRb::DRbConnError => error
|
31
|
+
error.message
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
|
36
|
+
def irm_list_queues
|
37
|
+
qm.store.queues.keys
|
38
|
+
end
|
39
|
+
|
40
|
+
def irm_list_messages suffix = DEFAULT_QUEUE_SUFFIX,
|
41
|
+
prefix = DEFAULT_QUEUE_PREFIX
|
42
|
+
qm.store.queues[prefix.to_s + suffix.to_s]
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
def irm_make_queue suffix = DEFAULT_QUEUE_SUFFIX,
|
47
|
+
prefix = DEFAULT_QUEUE_PREFIX
|
48
|
+
ReliableMsg::Queue.new prefix.to_s + suffix.to_s, :drb_uri => irm_drb_uri
|
49
|
+
end
|
50
|
+
|
51
|
+
def irm_queue_get suffix = DEFAULT_QUEUE_SUFFIX, selector = nil,
|
52
|
+
prefix = DEFAULT_QUEUE_PREFIX, &block
|
53
|
+
q = irm_make_queue suffix, prefix
|
54
|
+
q.get selector, &block
|
55
|
+
end
|
56
|
+
|
57
|
+
def irm_queue_put suffix = DEFAULT_QUEUE_SUFFIX,
|
58
|
+
message = nil, prefix = DEFAULT_QUEUE_PREFIX,
|
59
|
+
headers = nil
|
60
|
+
unless message
|
61
|
+
t = Time.now
|
62
|
+
message = sprintf("test message %s,%s",
|
63
|
+
t.strftime("%Y/%m/%d %H:%M:%S"), t.usec)
|
64
|
+
end
|
65
|
+
q = irm_make_queue suffix, prefix
|
66
|
+
q.put message, headers
|
67
|
+
end
|
68
|
+
|
69
|
+
def irm_make_multi_queue multi_queue = DEFAULT_MULTI_QUEUE
|
70
|
+
ReliableMsg::MultiQueue.new multi_queue.to_s
|
71
|
+
end
|
72
|
+
|
73
|
+
def irm_multi_queue_get selector = nil,
|
74
|
+
multi_queue = DEFAULT_MULTI_QUEUE, &block
|
75
|
+
mq = irm_make_multi_queue multi_queue, :drb_uri => irm_drb_uri
|
76
|
+
mq.get selector, &block
|
77
|
+
end
|
78
|
+
|
79
|
+
def irm_set_config arg
|
80
|
+
if arg.is_a? Integer
|
81
|
+
$config = "queues#{arg.to_s}.cfg"
|
82
|
+
else
|
83
|
+
$config = arg.to_s
|
84
|
+
end
|
85
|
+
config = ReliableMsg::Config.new($config)
|
86
|
+
config.load_no_create
|
87
|
+
$drb_port = config.drb['port']
|
88
|
+
end
|
89
|
+
|
90
|
+
def irm_drb_uri
|
91
|
+
"druby://#{$drb_host}:#{$drb_port}"
|
92
|
+
end
|
93
|
+
|
94
|
+
alias qm irm_queue_manager
|
95
|
+
alias qm_stop irm_queue_manager_stop
|
96
|
+
|
97
|
+
alias lsq irm_list_queues
|
98
|
+
alias lsm irm_list_messages
|
99
|
+
|
100
|
+
alias mkq irm_make_queue
|
101
|
+
alias qg irm_queue_get
|
102
|
+
alias qp irm_queue_put
|
103
|
+
alias mkmq irm_make_multi_queue
|
104
|
+
alias mqg irm_multi_queue_get
|
105
|
+
|
106
|
+
alias cfg irm_set_config
|
107
|
+
alias uri irm_drb_uri
|
108
|
+
end
|
109
|
+
|
metadata
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.8.11
|
3
|
+
specification_version: 1
|
4
|
+
name: ap4r
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: 0.1.0
|
7
|
+
date: 2006-09-01 00:00:00 +09:00
|
8
|
+
summary: Asynchronous Processing for Ruby.
|
9
|
+
require_paths:
|
10
|
+
- lib
|
11
|
+
email: kato.kiwamu@future.co.jp, shinohara.shunichi@future.co.jp
|
12
|
+
homepage: http://rubyforge.org/projects/ap4r/
|
13
|
+
rubyforge_project: ap4r
|
14
|
+
description: Asynchronous Processing for Ruby.
|
15
|
+
autorequire: ap4r.rb
|
16
|
+
default_executable:
|
17
|
+
- ap4r
|
18
|
+
bindir: bin
|
19
|
+
has_rdoc: true
|
20
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
21
|
+
requirements:
|
22
|
+
- - ">"
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: 0.0.0
|
25
|
+
version:
|
26
|
+
platform: ruby
|
27
|
+
signing_key:
|
28
|
+
cert_chain:
|
29
|
+
authors:
|
30
|
+
- Shunichi Shinohara, Kiwamu Kato
|
31
|
+
files:
|
32
|
+
- bin/ap4r
|
33
|
+
- bin/irm.cmd
|
34
|
+
- bin/loop.cmd
|
35
|
+
- bin/loop.rb
|
36
|
+
- bin/queues.cmd
|
37
|
+
- bin/queues.rb
|
38
|
+
- bin/stop.cmd
|
39
|
+
- configs/log4r.yaml
|
40
|
+
- configs/queues.cfg
|
41
|
+
- configs/queues_disk.cfg
|
42
|
+
- configs/queues_mysql.cfg
|
43
|
+
- lib/ap4r
|
44
|
+
- lib/ap4r.rb
|
45
|
+
- lib/ap4r/message_store_ext.rb
|
46
|
+
- lib/ap4r/multi_queue.rb
|
47
|
+
- lib/ap4r/queue_manager_ext.rb
|
48
|
+
- lib/ap4r/queue_manager_ext_debug.rb
|
49
|
+
- lib/ap4r/retention_history.rb
|
50
|
+
- lib/ap4r/start_with_log4r.rb
|
51
|
+
- lib/ap4r/utils
|
52
|
+
- lib/ap4r/utils/irm.rb
|
53
|
+
- lib/ap4r/utils/loc.rb
|
54
|
+
- CHANGELOG
|
55
|
+
- MIT-LICENSE
|
56
|
+
- Rakefile
|
57
|
+
- README
|
58
|
+
test_files: []
|
59
|
+
|
60
|
+
rdoc_options: []
|
61
|
+
|
62
|
+
extra_rdoc_files: []
|
63
|
+
|
64
|
+
executables:
|
65
|
+
- ap4r
|
66
|
+
extensions: []
|
67
|
+
|
68
|
+
requirements:
|
69
|
+
- none
|
70
|
+
dependencies:
|
71
|
+
- !ruby/object:Gem::Dependency
|
72
|
+
name: reliable-msg
|
73
|
+
version_requirement:
|
74
|
+
version_requirements: !ruby/object:Gem::Version::Requirement
|
75
|
+
requirements:
|
76
|
+
- - "="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: 1.1.0
|
79
|
+
version:
|