josip-backgroundrb_merb 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +4 -0
- data/README +45 -0
- data/Rakefile +44 -0
- data/TODO +8 -0
- data/config/backgroundrb.yml +11 -0
- data/generators/worker/USAGE +16 -0
- data/generators/worker/templates/app/workers/%worker_file_name%_worker.rb +7 -0
- data/generators/worker/templates/spec/workers/%worker_file_name%_spec.rb +9 -0
- data/generators/worker/worker_generator.rb +36 -0
- data/lib/backgroundrb_merb.rb +19 -0
- data/lib/backgroundrb_merb/bdrb_config.rb +47 -0
- data/lib/backgroundrb_merb/bdrb_conn_error.rb +9 -0
- data/lib/backgroundrb_merb/merb_worker_proxy.rb +57 -0
- data/lib/backgroundrb_merb/merbtasks.rb +91 -0
- data/lib/backgroundrb_merb/worker_proxy.rb +174 -0
- data/server/backgroundrb_server.rb +7 -0
- data/server/lib/cron_trigger.rb +195 -0
- data/server/lib/invalid_dump_error.rb +4 -0
- data/server/lib/log_worker.rb +25 -0
- data/server/lib/master_worker.rb +301 -0
- data/server/lib/meta_worker.rb +360 -0
- data/server/lib/trigger.rb +34 -0
- metadata +110 -0
@@ -0,0 +1,360 @@
|
|
1
|
+
module BackgrounDRbMerb
|
2
|
+
# this class is a dummy class that implements things required for passing data to
|
3
|
+
# actual logger worker
|
4
|
+
class PacketLogger
|
5
|
+
def initialize(worker,log_flag = true)
|
6
|
+
@log_flag = log_flag
|
7
|
+
@worker = worker
|
8
|
+
end
|
9
|
+
def info(p_data)
|
10
|
+
return unless @log_flag
|
11
|
+
@worker.send_request(:worker => :log_worker, :data => p_data)
|
12
|
+
end
|
13
|
+
|
14
|
+
def debug(p_data)
|
15
|
+
return unless @log_flag
|
16
|
+
@worker.send_request(:worker => :log_worker, :data => p_data)
|
17
|
+
end
|
18
|
+
|
19
|
+
def error(p_data)
|
20
|
+
return unless @log_flag
|
21
|
+
@worker.send_request(:worker => :log_worker, :data => p_data)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class WorkData
|
26
|
+
attr_accessor :data,:block
|
27
|
+
def initialize(args,&block)
|
28
|
+
@data = args
|
29
|
+
@block = block
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class ThreadPool
|
34
|
+
attr_accessor :size,:threads,:work_queue,:logger
|
35
|
+
def initialize(size,logger)
|
36
|
+
@logger = logger
|
37
|
+
@size = size
|
38
|
+
@threads = []
|
39
|
+
@work_queue = Queue.new
|
40
|
+
@running_tasks = Queue.new
|
41
|
+
@size.times { add_thread }
|
42
|
+
end
|
43
|
+
|
44
|
+
# can be used to make a call in threaded manner
|
45
|
+
# passed block runs in a thread from thread pool
|
46
|
+
# for example in a worker method you can do:
|
47
|
+
# def fetch_url(url)
|
48
|
+
# puts "fetching url #{url}"
|
49
|
+
# thread_pool.defer(url) do |url|
|
50
|
+
# begin
|
51
|
+
# data = Net::HTTP.get(url,'/')
|
52
|
+
# File.open("#{RAILS_ROOT}/log/pages.txt","w") do |fl|
|
53
|
+
# fl.puts(data)
|
54
|
+
# end
|
55
|
+
# rescue
|
56
|
+
# logger.info "Error downloading page"
|
57
|
+
# end
|
58
|
+
# end
|
59
|
+
# end
|
60
|
+
# you can invoke above method from rails as:
|
61
|
+
# MiddleMan.ask_work(:worker => :rss_worker, :worker_method => :fetch_url, :data => "www.example.com")
|
62
|
+
# assuming method is defined in rss_worker
|
63
|
+
|
64
|
+
def defer(*args,&block)
|
65
|
+
@work_queue << WorkData.new(args,&block)
|
66
|
+
end
|
67
|
+
|
68
|
+
def add_thread
|
69
|
+
@threads << Thread.new do
|
70
|
+
while true
|
71
|
+
task = @work_queue.pop
|
72
|
+
@running_tasks << task
|
73
|
+
block_arity = task.block.arity
|
74
|
+
|
75
|
+
begin
|
76
|
+
ActiveRecord::Base.verify_active_connections!
|
77
|
+
block_arity == 0 ? task.block.call : task.block.call(*(task.data))
|
78
|
+
rescue
|
79
|
+
logger.info($!.to_s)
|
80
|
+
logger.info($!.backtrace.join("\n"))
|
81
|
+
end
|
82
|
+
@running_tasks.pop
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
# method ensures exclusive run of deferred tasks for 2 seconds, so as they do get a chance to run.
|
88
|
+
def exclusive_run
|
89
|
+
if @running_tasks.empty? && @work_queue.empty?
|
90
|
+
return
|
91
|
+
else
|
92
|
+
# puts "going to sleep for a while"
|
93
|
+
sleep(0.05)
|
94
|
+
return
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# == MetaWorker class
|
100
|
+
# BackgrounDRb workers are asynchrounous reactors which work using events
|
101
|
+
# You are free to use threads in your workers, but be reasonable with them.
|
102
|
+
# Following methods are available to all workers from parent classes.
|
103
|
+
# * BackgrounDRb::MetaWorker#connect
|
104
|
+
#
|
105
|
+
# Above method connects to an external tcp server and integrates the connection
|
106
|
+
# within reactor loop of worker. For example:
|
107
|
+
#
|
108
|
+
# class TimeClient
|
109
|
+
# def receive_data(p_data)
|
110
|
+
# worker.get_external_data(p_data)
|
111
|
+
# end
|
112
|
+
#
|
113
|
+
# def post_init
|
114
|
+
# p "***************** : connection completed"
|
115
|
+
# end
|
116
|
+
# end
|
117
|
+
#
|
118
|
+
# class FooWorker < BackgrounDRb::MetaWorker
|
119
|
+
# set_worker_name :foo_worker
|
120
|
+
# def create(args = nil)
|
121
|
+
# external_connection = nil
|
122
|
+
# connect("localhost",11009,TimeClient) { |conn| external_connection = conn }
|
123
|
+
# end
|
124
|
+
#
|
125
|
+
# def get_external_data(p_data)
|
126
|
+
# puts "And external data is : #{p_data}"
|
127
|
+
# end
|
128
|
+
# end
|
129
|
+
# * BackgrounDRb::MetaWorker#start_server
|
130
|
+
#
|
131
|
+
# Above method allows you to start a tcp server from your worker, all the
|
132
|
+
# accepted connections are integrated with event loop of worker
|
133
|
+
# class TimeServer
|
134
|
+
#
|
135
|
+
# def receive_data(p_data)
|
136
|
+
# end
|
137
|
+
#
|
138
|
+
# def post_init
|
139
|
+
# add_periodic_timer(2) { say_hello_world }
|
140
|
+
# end
|
141
|
+
#
|
142
|
+
# def connection_completed
|
143
|
+
# end
|
144
|
+
#
|
145
|
+
# def say_hello_world
|
146
|
+
# p "***************** : invoking hello world #{Time.now}"
|
147
|
+
# send_data("Hello World\n")
|
148
|
+
# end
|
149
|
+
# end
|
150
|
+
#
|
151
|
+
# class ServerWorker < BackgrounDRb::MetaWorker
|
152
|
+
# set_worker_name :server_worker
|
153
|
+
# def create(args = nil)
|
154
|
+
# # start the server when worker starts
|
155
|
+
# start_server("0.0.0.0",11009,TimeServer) do |client_connection|
|
156
|
+
# client_connection.say_hello_world
|
157
|
+
# end
|
158
|
+
# end
|
159
|
+
# end
|
160
|
+
|
161
|
+
class MetaWorker < Packet::Worker
|
162
|
+
attr_accessor :config_file, :my_schedule, :run_time, :trigger_type, :trigger
|
163
|
+
attr_accessor :logger, :thread_pool
|
164
|
+
iattr_accessor :pool_size
|
165
|
+
iattr_accessor :reload_flag
|
166
|
+
|
167
|
+
@pool_size = nil
|
168
|
+
@reload_flag = false
|
169
|
+
|
170
|
+
def self.pool_size(size = nil)
|
171
|
+
@pool_size = size if size
|
172
|
+
@pool_size
|
173
|
+
end
|
174
|
+
|
175
|
+
def self.reload_on_schedule(flag = nil)
|
176
|
+
if flag
|
177
|
+
self.no_auto_load = true
|
178
|
+
self.reload_flag = true
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
# does initialization of worker stuff and invokes create method in
|
183
|
+
# user defined worker class
|
184
|
+
def worker_init
|
185
|
+
@config_file = BackgrounDRbMerb::Config.read_config("#{Merb.root}/config/backgroundrb.yml")
|
186
|
+
log_flag = @config_file[:backgroundrb][:debug_log].nil? ? true : @config_file[:backgroundrb][:debug_log]
|
187
|
+
load_merb_env
|
188
|
+
@logger = PacketLogger.new(self,log_flag)
|
189
|
+
@thread_pool = ThreadPool.new(pool_size || 20,@logger)
|
190
|
+
|
191
|
+
if(worker_options && worker_options[:schedule] && no_auto_load)
|
192
|
+
load_schedule_from_args
|
193
|
+
elsif(@config_file[:schedules] && @config_file[:schedules][worker_name.to_sym])
|
194
|
+
@my_schedule = @config_file[:schedules][worker_name.to_sym]
|
195
|
+
new_load_schedule if @my_schedule
|
196
|
+
end
|
197
|
+
if respond_to?(:create)
|
198
|
+
create_arity = method(:create).arity
|
199
|
+
(create_arity == 0) ? create : create(worker_options[:data])
|
200
|
+
end
|
201
|
+
@logger.info "#{worker_name} started"
|
202
|
+
@logger.info "Schedules for worker loaded"
|
203
|
+
end
|
204
|
+
|
205
|
+
# loads workers schedule from options supplied from rails
|
206
|
+
# a user may pass trigger arguments to dynamically define the schedule
|
207
|
+
def load_schedule_from_args
|
208
|
+
@my_schedule = worker_options[:schedule]
|
209
|
+
new_load_schedule if @my_schedule
|
210
|
+
end
|
211
|
+
|
212
|
+
# receives requests/responses from master process or other workers
|
213
|
+
def receive_data p_data
|
214
|
+
if p_data[:data][:worker_method] == :exit
|
215
|
+
exit
|
216
|
+
return
|
217
|
+
end
|
218
|
+
case p_data[:type]
|
219
|
+
when :request: process_request(p_data)
|
220
|
+
when :response: process_response(p_data)
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
# method is responsible for invoking appropriate method in user
|
225
|
+
def process_request(p_data)
|
226
|
+
user_input = p_data[:data]
|
227
|
+
logger.info "#{user_input[:worker_method]} #{user_input[:data]}"
|
228
|
+
if (user_input[:worker_method]).nil? or !respond_to?(user_input[:worker_method])
|
229
|
+
logger.info "Undefined method #{user_input[:worker_method]} called on worker #{worker_name}"
|
230
|
+
return
|
231
|
+
end
|
232
|
+
called_method_arity = self.method(user_input[:worker_method]).arity
|
233
|
+
result = nil
|
234
|
+
if called_method_arity != 0
|
235
|
+
result = self.send(user_input[:worker_method],user_input[:data])
|
236
|
+
else
|
237
|
+
result = self.send(user_input[:worker_method])
|
238
|
+
end
|
239
|
+
if p_data[:result]
|
240
|
+
result = "dummy_result" unless result
|
241
|
+
send_response(p_data,result) if can_dump?(result)
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
def can_dump?(p_object)
|
246
|
+
begin
|
247
|
+
Marshal.dump(p_object)
|
248
|
+
return true
|
249
|
+
rescue TypeError
|
250
|
+
return false
|
251
|
+
rescue
|
252
|
+
return false
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
# new experimental scheduler
|
257
|
+
def new_load_schedule
|
258
|
+
@worker_method_triggers = { }
|
259
|
+
@my_schedule.each do |key,value|
|
260
|
+
case value[:trigger_args]
|
261
|
+
when String
|
262
|
+
cron_args = value[:trigger_args] || "0 0 0 0 0"
|
263
|
+
trigger = BackgrounDRbMerb::CronTrigger.new(cron_args)
|
264
|
+
when Hash
|
265
|
+
trigger = BackgrounDRbMerb::Trigger.new(value[:trigger_args])
|
266
|
+
end
|
267
|
+
@worker_method_triggers[key] = { :trigger => trigger,:data => value[:data],:runtime => trigger.fire_after_time(Time.now).to_i }
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
# probably this method should be made thread safe, so as a method needs to have a
|
272
|
+
# lock or something before it can use the method
|
273
|
+
def register_status p_data
|
274
|
+
status = {:type => :status,:data => p_data}
|
275
|
+
begin
|
276
|
+
send_data(status)
|
277
|
+
rescue TypeError => e
|
278
|
+
status = {:type => :status,:data => "invalid_status_dump_check_log"}
|
279
|
+
send_data(status)
|
280
|
+
logger.info(e.to_s)
|
281
|
+
logger.info(e.backtrace.join("\n"))
|
282
|
+
rescue
|
283
|
+
status = {:type => :status,:data => "invalid_status_dump_check_log"}
|
284
|
+
send_data(status)
|
285
|
+
logger.info($!.to_s)
|
286
|
+
logger.info($!.backtrace.join("\n"))
|
287
|
+
end
|
288
|
+
end
|
289
|
+
|
290
|
+
def send_response input,output
|
291
|
+
input[:data] = output
|
292
|
+
input[:type] = :response
|
293
|
+
|
294
|
+
begin
|
295
|
+
send_data(input)
|
296
|
+
rescue TypeError => e
|
297
|
+
logger.info(e.to_s)
|
298
|
+
logger.info(e.backtrace.join("\n"))
|
299
|
+
input[:data] = "invalid_result_dump_check_log"
|
300
|
+
send_data(input)
|
301
|
+
rescue
|
302
|
+
logger.info($!.to_s)
|
303
|
+
logger.info($!.backtrace.join("\n"))
|
304
|
+
input[:data] = "invalid_result_dump_check_log"
|
305
|
+
send_data(input)
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
def unbind; end
|
310
|
+
|
311
|
+
def connection_completed; end
|
312
|
+
|
313
|
+
def check_for_timer_events
|
314
|
+
begin
|
315
|
+
ActiveRecord::Base.verify_active_connections! if defined?(ActiveRecord)
|
316
|
+
super
|
317
|
+
rescue
|
318
|
+
logger.info($!.to_s)
|
319
|
+
logger.info($!.backtrace.join("\n"))
|
320
|
+
end
|
321
|
+
|
322
|
+
return if @worker_method_triggers.nil? or @worker_method_triggers.empty?
|
323
|
+
@worker_method_triggers.delete_if { |key,value| value[:trigger].respond_to?(:end_time) && value[:trigger].end_time <= Time.now }
|
324
|
+
|
325
|
+
@worker_method_triggers.each do |key,value|
|
326
|
+
time_now = Time.now.to_i
|
327
|
+
if value[:runtime] < time_now
|
328
|
+
begin
|
329
|
+
(t_data = value[:data]) ? send(key,t_data) : send(key)
|
330
|
+
rescue
|
331
|
+
# logger.info($!.to_s)
|
332
|
+
# logger.info($!.backtrace.join("\n"))
|
333
|
+
puts $!
|
334
|
+
puts $!.backtrace
|
335
|
+
end
|
336
|
+
t_time = value[:trigger].fire_after_time(Time.now)
|
337
|
+
value[:runtime] = t_time.to_i
|
338
|
+
end
|
339
|
+
end
|
340
|
+
end
|
341
|
+
|
342
|
+
# method would allow user threads to run exclusively for a while
|
343
|
+
def run_user_threads
|
344
|
+
@thread_pool.exclusive_run
|
345
|
+
end
|
346
|
+
|
347
|
+
private
|
348
|
+
def load_merb_env
|
349
|
+
return unless File.exists?(Merb.root / "config" / "database.yml")
|
350
|
+
|
351
|
+
db_config_file = YAML.load(ERB.new(IO.read("#{Merb.root}/config/database.yml")).result)
|
352
|
+
run_env = @config_file[:backgroundrb][:environment] || 'development'
|
353
|
+
|
354
|
+
if defined?(ActiveRecord) then
|
355
|
+
ActiveRecord::Base.establish_connection(db_config_file[run_env.to_sym])
|
356
|
+
ActiveRecord::Base.allow_concurrency = true
|
357
|
+
end
|
358
|
+
end
|
359
|
+
end # end of class MetaWorker
|
360
|
+
end # end of module BackgrounDRb
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module BackgrounDRbMerb
|
2
|
+
class Trigger
|
3
|
+
|
4
|
+
attr_accessor :start_time, :end_time, :repeat_interval
|
5
|
+
|
6
|
+
def initialize(opts={})
|
7
|
+
@start_time = Time.parse(opts[:start])
|
8
|
+
@end_time = Time.parse(opts[:end])
|
9
|
+
@repeat_interval = opts[:repeat_interval].to_i
|
10
|
+
end
|
11
|
+
|
12
|
+
def fire_after_time(time)
|
13
|
+
@start_time = time if not @start_time
|
14
|
+
|
15
|
+
# Support UNIX at-style scheduling, by just specifying a start
|
16
|
+
# time.
|
17
|
+
if @end_time.nil? and @repeat_interval.nil?
|
18
|
+
@end_time = start_time + 1
|
19
|
+
@repeat_interval = 1
|
20
|
+
end
|
21
|
+
|
22
|
+
case
|
23
|
+
when @end_time && time > @end_time
|
24
|
+
nil
|
25
|
+
when time < @start_time
|
26
|
+
@start_time
|
27
|
+
when @repeat_interval != nil && @repeat_interval > 0
|
28
|
+
time + @repeat_interval - ((time - @start_time) % @repeat_interval)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
metadata
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: josip-backgroundrb_merb
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Josip Lisec
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2008-05-14 00:00:00 -07:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: merb
|
17
|
+
version_requirement:
|
18
|
+
version_requirements: !ruby/object:Gem::Requirement
|
19
|
+
requirements:
|
20
|
+
- - ">="
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 0.9.2
|
23
|
+
version:
|
24
|
+
- !ruby/object:Gem::Dependency
|
25
|
+
name: packet
|
26
|
+
version_requirement:
|
27
|
+
version_requirements: !ruby/object:Gem::Requirement
|
28
|
+
requirements:
|
29
|
+
- - ">="
|
30
|
+
- !ruby/object:Gem::Version
|
31
|
+
version: 0.1.5
|
32
|
+
version:
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: chronic
|
35
|
+
version_requirement:
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.2.3
|
41
|
+
version:
|
42
|
+
description: BackgroundRB for Merb
|
43
|
+
email: josiplisec@gmail.com
|
44
|
+
executables: []
|
45
|
+
|
46
|
+
extensions: []
|
47
|
+
|
48
|
+
extra_rdoc_files:
|
49
|
+
- README
|
50
|
+
- LICENSE
|
51
|
+
- TODO
|
52
|
+
files:
|
53
|
+
- LICENSE
|
54
|
+
- README
|
55
|
+
- Rakefile
|
56
|
+
- TODO
|
57
|
+
- config/backgroundrb.yml
|
58
|
+
- generators/worker
|
59
|
+
- generators/worker/USAGE
|
60
|
+
- generators/worker/worker_generator.rb
|
61
|
+
- generators/worker/templates
|
62
|
+
- generators/worker/templates/app
|
63
|
+
- generators/worker/templates/app/workers
|
64
|
+
- generators/worker/templates/app/workers/%worker_file_name%_worker.rb
|
65
|
+
- generators/worker/templates/spec
|
66
|
+
- generators/worker/templates/spec/workers
|
67
|
+
- generators/worker/templates/spec/workers/%worker_file_name%_spec.rb
|
68
|
+
- lib/backgroundrb_merb
|
69
|
+
- lib/backgroundrb_merb/merb_worker_proxy.rb
|
70
|
+
- lib/backgroundrb_merb/bdrb_config.rb
|
71
|
+
- lib/backgroundrb_merb/merbtasks.rb
|
72
|
+
- lib/backgroundrb_merb/worker_proxy.rb
|
73
|
+
- lib/backgroundrb_merb/bdrb_conn_error.rb
|
74
|
+
- lib/backgroundrb_merb.rb
|
75
|
+
- server/lib
|
76
|
+
- server/lib/log_worker.rb
|
77
|
+
- server/lib/master_worker.rb
|
78
|
+
- server/lib/invalid_dump_error.rb
|
79
|
+
- server/lib/meta_worker.rb
|
80
|
+
- server/lib/trigger.rb
|
81
|
+
- server/lib/cron_trigger.rb
|
82
|
+
- server/backgroundrb_server.rb
|
83
|
+
has_rdoc: true
|
84
|
+
homepage: http://github.com/josip/backgroundrb_merb/tree/master
|
85
|
+
post_install_message:
|
86
|
+
rdoc_options: []
|
87
|
+
|
88
|
+
require_paths:
|
89
|
+
- lib
|
90
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: "0"
|
95
|
+
version:
|
96
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
97
|
+
requirements:
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: "0"
|
101
|
+
version:
|
102
|
+
requirements: []
|
103
|
+
|
104
|
+
rubyforge_project:
|
105
|
+
rubygems_version: 1.0.1
|
106
|
+
signing_key:
|
107
|
+
specification_version: 2
|
108
|
+
summary: BackgroundRB for Merb
|
109
|
+
test_files: []
|
110
|
+
|