josip-backgroundrb_merb 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,360 @@
1
+ module BackgrounDRbMerb
2
+ # this class is a dummy class that implements things required for passing data to
3
+ # actual logger worker
4
+ class PacketLogger
5
+ def initialize(worker,log_flag = true)
6
+ @log_flag = log_flag
7
+ @worker = worker
8
+ end
9
+ def info(p_data)
10
+ return unless @log_flag
11
+ @worker.send_request(:worker => :log_worker, :data => p_data)
12
+ end
13
+
14
+ def debug(p_data)
15
+ return unless @log_flag
16
+ @worker.send_request(:worker => :log_worker, :data => p_data)
17
+ end
18
+
19
+ def error(p_data)
20
+ return unless @log_flag
21
+ @worker.send_request(:worker => :log_worker, :data => p_data)
22
+ end
23
+ end
24
+
25
+ class WorkData
26
+ attr_accessor :data,:block
27
+ def initialize(args,&block)
28
+ @data = args
29
+ @block = block
30
+ end
31
+ end
32
+
33
+ class ThreadPool
34
+ attr_accessor :size,:threads,:work_queue,:logger
35
+ def initialize(size,logger)
36
+ @logger = logger
37
+ @size = size
38
+ @threads = []
39
+ @work_queue = Queue.new
40
+ @running_tasks = Queue.new
41
+ @size.times { add_thread }
42
+ end
43
+
44
+ # can be used to make a call in threaded manner
45
+ # passed block runs in a thread from thread pool
46
+ # for example in a worker method you can do:
47
+ # def fetch_url(url)
48
+ # puts "fetching url #{url}"
49
+ # thread_pool.defer(url) do |url|
50
+ # begin
51
+ # data = Net::HTTP.get(url,'/')
52
+ # File.open("#{RAILS_ROOT}/log/pages.txt","w") do |fl|
53
+ # fl.puts(data)
54
+ # end
55
+ # rescue
56
+ # logger.info "Error downloading page"
57
+ # end
58
+ # end
59
+ # end
60
+ # you can invoke above method from rails as:
61
+ # MiddleMan.ask_work(:worker => :rss_worker, :worker_method => :fetch_url, :data => "www.example.com")
62
+ # assuming method is defined in rss_worker
63
+
64
+ def defer(*args,&block)
65
+ @work_queue << WorkData.new(args,&block)
66
+ end
67
+
68
+ def add_thread
69
+ @threads << Thread.new do
70
+ while true
71
+ task = @work_queue.pop
72
+ @running_tasks << task
73
+ block_arity = task.block.arity
74
+
75
+ begin
76
+ ActiveRecord::Base.verify_active_connections!
77
+ block_arity == 0 ? task.block.call : task.block.call(*(task.data))
78
+ rescue
79
+ logger.info($!.to_s)
80
+ logger.info($!.backtrace.join("\n"))
81
+ end
82
+ @running_tasks.pop
83
+ end
84
+ end
85
+ end
86
+
87
+ # method ensures exclusive run of deferred tasks for 2 seconds, so as they do get a chance to run.
88
+ def exclusive_run
89
+ if @running_tasks.empty? && @work_queue.empty?
90
+ return
91
+ else
92
+ # puts "going to sleep for a while"
93
+ sleep(0.05)
94
+ return
95
+ end
96
+ end
97
+ end
98
+
99
+ # == MetaWorker class
100
+ # BackgrounDRb workers are asynchrounous reactors which work using events
101
+ # You are free to use threads in your workers, but be reasonable with them.
102
+ # Following methods are available to all workers from parent classes.
103
+ # * BackgrounDRb::MetaWorker#connect
104
+ #
105
+ # Above method connects to an external tcp server and integrates the connection
106
+ # within reactor loop of worker. For example:
107
+ #
108
+ # class TimeClient
109
+ # def receive_data(p_data)
110
+ # worker.get_external_data(p_data)
111
+ # end
112
+ #
113
+ # def post_init
114
+ # p "***************** : connection completed"
115
+ # end
116
+ # end
117
+ #
118
+ # class FooWorker < BackgrounDRb::MetaWorker
119
+ # set_worker_name :foo_worker
120
+ # def create(args = nil)
121
+ # external_connection = nil
122
+ # connect("localhost",11009,TimeClient) { |conn| external_connection = conn }
123
+ # end
124
+ #
125
+ # def get_external_data(p_data)
126
+ # puts "And external data is : #{p_data}"
127
+ # end
128
+ # end
129
+ # * BackgrounDRb::MetaWorker#start_server
130
+ #
131
+ # Above method allows you to start a tcp server from your worker, all the
132
+ # accepted connections are integrated with event loop of worker
133
+ # class TimeServer
134
+ #
135
+ # def receive_data(p_data)
136
+ # end
137
+ #
138
+ # def post_init
139
+ # add_periodic_timer(2) { say_hello_world }
140
+ # end
141
+ #
142
+ # def connection_completed
143
+ # end
144
+ #
145
+ # def say_hello_world
146
+ # p "***************** : invoking hello world #{Time.now}"
147
+ # send_data("Hello World\n")
148
+ # end
149
+ # end
150
+ #
151
+ # class ServerWorker < BackgrounDRb::MetaWorker
152
+ # set_worker_name :server_worker
153
+ # def create(args = nil)
154
+ # # start the server when worker starts
155
+ # start_server("0.0.0.0",11009,TimeServer) do |client_connection|
156
+ # client_connection.say_hello_world
157
+ # end
158
+ # end
159
+ # end
160
+
161
+ class MetaWorker < Packet::Worker
162
+ attr_accessor :config_file, :my_schedule, :run_time, :trigger_type, :trigger
163
+ attr_accessor :logger, :thread_pool
164
+ iattr_accessor :pool_size
165
+ iattr_accessor :reload_flag
166
+
167
+ @pool_size = nil
168
+ @reload_flag = false
169
+
170
+ def self.pool_size(size = nil)
171
+ @pool_size = size if size
172
+ @pool_size
173
+ end
174
+
175
+ def self.reload_on_schedule(flag = nil)
176
+ if flag
177
+ self.no_auto_load = true
178
+ self.reload_flag = true
179
+ end
180
+ end
181
+
182
+ # does initialization of worker stuff and invokes create method in
183
+ # user defined worker class
184
+ def worker_init
185
+ @config_file = BackgrounDRbMerb::Config.read_config("#{Merb.root}/config/backgroundrb.yml")
186
+ log_flag = @config_file[:backgroundrb][:debug_log].nil? ? true : @config_file[:backgroundrb][:debug_log]
187
+ load_merb_env
188
+ @logger = PacketLogger.new(self,log_flag)
189
+ @thread_pool = ThreadPool.new(pool_size || 20,@logger)
190
+
191
+ if(worker_options && worker_options[:schedule] && no_auto_load)
192
+ load_schedule_from_args
193
+ elsif(@config_file[:schedules] && @config_file[:schedules][worker_name.to_sym])
194
+ @my_schedule = @config_file[:schedules][worker_name.to_sym]
195
+ new_load_schedule if @my_schedule
196
+ end
197
+ if respond_to?(:create)
198
+ create_arity = method(:create).arity
199
+ (create_arity == 0) ? create : create(worker_options[:data])
200
+ end
201
+ @logger.info "#{worker_name} started"
202
+ @logger.info "Schedules for worker loaded"
203
+ end
204
+
205
+ # loads workers schedule from options supplied from rails
206
+ # a user may pass trigger arguments to dynamically define the schedule
207
+ def load_schedule_from_args
208
+ @my_schedule = worker_options[:schedule]
209
+ new_load_schedule if @my_schedule
210
+ end
211
+
212
+ # receives requests/responses from master process or other workers
213
+ def receive_data p_data
214
+ if p_data[:data][:worker_method] == :exit
215
+ exit
216
+ return
217
+ end
218
+ case p_data[:type]
219
+ when :request: process_request(p_data)
220
+ when :response: process_response(p_data)
221
+ end
222
+ end
223
+
224
+ # method is responsible for invoking appropriate method in user
225
+ def process_request(p_data)
226
+ user_input = p_data[:data]
227
+ logger.info "#{user_input[:worker_method]} #{user_input[:data]}"
228
+ if (user_input[:worker_method]).nil? or !respond_to?(user_input[:worker_method])
229
+ logger.info "Undefined method #{user_input[:worker_method]} called on worker #{worker_name}"
230
+ return
231
+ end
232
+ called_method_arity = self.method(user_input[:worker_method]).arity
233
+ result = nil
234
+ if called_method_arity != 0
235
+ result = self.send(user_input[:worker_method],user_input[:data])
236
+ else
237
+ result = self.send(user_input[:worker_method])
238
+ end
239
+ if p_data[:result]
240
+ result = "dummy_result" unless result
241
+ send_response(p_data,result) if can_dump?(result)
242
+ end
243
+ end
244
+
245
+ def can_dump?(p_object)
246
+ begin
247
+ Marshal.dump(p_object)
248
+ return true
249
+ rescue TypeError
250
+ return false
251
+ rescue
252
+ return false
253
+ end
254
+ end
255
+
256
+ # new experimental scheduler
257
+ def new_load_schedule
258
+ @worker_method_triggers = { }
259
+ @my_schedule.each do |key,value|
260
+ case value[:trigger_args]
261
+ when String
262
+ cron_args = value[:trigger_args] || "0 0 0 0 0"
263
+ trigger = BackgrounDRbMerb::CronTrigger.new(cron_args)
264
+ when Hash
265
+ trigger = BackgrounDRbMerb::Trigger.new(value[:trigger_args])
266
+ end
267
+ @worker_method_triggers[key] = { :trigger => trigger,:data => value[:data],:runtime => trigger.fire_after_time(Time.now).to_i }
268
+ end
269
+ end
270
+
271
+ # probably this method should be made thread safe, so as a method needs to have a
272
+ # lock or something before it can use the method
273
+ def register_status p_data
274
+ status = {:type => :status,:data => p_data}
275
+ begin
276
+ send_data(status)
277
+ rescue TypeError => e
278
+ status = {:type => :status,:data => "invalid_status_dump_check_log"}
279
+ send_data(status)
280
+ logger.info(e.to_s)
281
+ logger.info(e.backtrace.join("\n"))
282
+ rescue
283
+ status = {:type => :status,:data => "invalid_status_dump_check_log"}
284
+ send_data(status)
285
+ logger.info($!.to_s)
286
+ logger.info($!.backtrace.join("\n"))
287
+ end
288
+ end
289
+
290
+ def send_response input,output
291
+ input[:data] = output
292
+ input[:type] = :response
293
+
294
+ begin
295
+ send_data(input)
296
+ rescue TypeError => e
297
+ logger.info(e.to_s)
298
+ logger.info(e.backtrace.join("\n"))
299
+ input[:data] = "invalid_result_dump_check_log"
300
+ send_data(input)
301
+ rescue
302
+ logger.info($!.to_s)
303
+ logger.info($!.backtrace.join("\n"))
304
+ input[:data] = "invalid_result_dump_check_log"
305
+ send_data(input)
306
+ end
307
+ end
308
+
309
+ def unbind; end
310
+
311
+ def connection_completed; end
312
+
313
+ def check_for_timer_events
314
+ begin
315
+ ActiveRecord::Base.verify_active_connections! if defined?(ActiveRecord)
316
+ super
317
+ rescue
318
+ logger.info($!.to_s)
319
+ logger.info($!.backtrace.join("\n"))
320
+ end
321
+
322
+ return if @worker_method_triggers.nil? or @worker_method_triggers.empty?
323
+ @worker_method_triggers.delete_if { |key,value| value[:trigger].respond_to?(:end_time) && value[:trigger].end_time <= Time.now }
324
+
325
+ @worker_method_triggers.each do |key,value|
326
+ time_now = Time.now.to_i
327
+ if value[:runtime] < time_now
328
+ begin
329
+ (t_data = value[:data]) ? send(key,t_data) : send(key)
330
+ rescue
331
+ # logger.info($!.to_s)
332
+ # logger.info($!.backtrace.join("\n"))
333
+ puts $!
334
+ puts $!.backtrace
335
+ end
336
+ t_time = value[:trigger].fire_after_time(Time.now)
337
+ value[:runtime] = t_time.to_i
338
+ end
339
+ end
340
+ end
341
+
342
+ # method would allow user threads to run exclusively for a while
343
+ def run_user_threads
344
+ @thread_pool.exclusive_run
345
+ end
346
+
347
+ private
348
+ def load_merb_env
349
+ return unless File.exists?(Merb.root / "config" / "database.yml")
350
+
351
+ db_config_file = YAML.load(ERB.new(IO.read("#{Merb.root}/config/database.yml")).result)
352
+ run_env = @config_file[:backgroundrb][:environment] || 'development'
353
+
354
+ if defined?(ActiveRecord) then
355
+ ActiveRecord::Base.establish_connection(db_config_file[run_env.to_sym])
356
+ ActiveRecord::Base.allow_concurrency = true
357
+ end
358
+ end
359
+ end # end of class MetaWorker
360
+ end # end of module BackgrounDRb
@@ -0,0 +1,34 @@
1
+ module BackgrounDRbMerb
2
+ class Trigger
3
+
4
+ attr_accessor :start_time, :end_time, :repeat_interval
5
+
6
+ def initialize(opts={})
7
+ @start_time = Time.parse(opts[:start])
8
+ @end_time = Time.parse(opts[:end])
9
+ @repeat_interval = opts[:repeat_interval].to_i
10
+ end
11
+
12
+ def fire_after_time(time)
13
+ @start_time = time if not @start_time
14
+
15
+ # Support UNIX at-style scheduling, by just specifying a start
16
+ # time.
17
+ if @end_time.nil? and @repeat_interval.nil?
18
+ @end_time = start_time + 1
19
+ @repeat_interval = 1
20
+ end
21
+
22
+ case
23
+ when @end_time && time > @end_time
24
+ nil
25
+ when time < @start_time
26
+ @start_time
27
+ when @repeat_interval != nil && @repeat_interval > 0
28
+ time + @repeat_interval - ((time - @start_time) % @repeat_interval)
29
+ end
30
+ end
31
+
32
+ end
33
+
34
+ end
metadata ADDED
@@ -0,0 +1,110 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: josip-backgroundrb_merb
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.3
5
+ platform: ruby
6
+ authors:
7
+ - Josip Lisec
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2008-05-14 00:00:00 -07:00
13
+ default_executable:
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: merb
17
+ version_requirement:
18
+ version_requirements: !ruby/object:Gem::Requirement
19
+ requirements:
20
+ - - ">="
21
+ - !ruby/object:Gem::Version
22
+ version: 0.9.2
23
+ version:
24
+ - !ruby/object:Gem::Dependency
25
+ name: packet
26
+ version_requirement:
27
+ version_requirements: !ruby/object:Gem::Requirement
28
+ requirements:
29
+ - - ">="
30
+ - !ruby/object:Gem::Version
31
+ version: 0.1.5
32
+ version:
33
+ - !ruby/object:Gem::Dependency
34
+ name: chronic
35
+ version_requirement:
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: 0.2.3
41
+ version:
42
+ description: BackgroundRB for Merb
43
+ email: josiplisec@gmail.com
44
+ executables: []
45
+
46
+ extensions: []
47
+
48
+ extra_rdoc_files:
49
+ - README
50
+ - LICENSE
51
+ - TODO
52
+ files:
53
+ - LICENSE
54
+ - README
55
+ - Rakefile
56
+ - TODO
57
+ - config/backgroundrb.yml
58
+ - generators/worker
59
+ - generators/worker/USAGE
60
+ - generators/worker/worker_generator.rb
61
+ - generators/worker/templates
62
+ - generators/worker/templates/app
63
+ - generators/worker/templates/app/workers
64
+ - generators/worker/templates/app/workers/%worker_file_name%_worker.rb
65
+ - generators/worker/templates/spec
66
+ - generators/worker/templates/spec/workers
67
+ - generators/worker/templates/spec/workers/%worker_file_name%_spec.rb
68
+ - lib/backgroundrb_merb
69
+ - lib/backgroundrb_merb/merb_worker_proxy.rb
70
+ - lib/backgroundrb_merb/bdrb_config.rb
71
+ - lib/backgroundrb_merb/merbtasks.rb
72
+ - lib/backgroundrb_merb/worker_proxy.rb
73
+ - lib/backgroundrb_merb/bdrb_conn_error.rb
74
+ - lib/backgroundrb_merb.rb
75
+ - server/lib
76
+ - server/lib/log_worker.rb
77
+ - server/lib/master_worker.rb
78
+ - server/lib/invalid_dump_error.rb
79
+ - server/lib/meta_worker.rb
80
+ - server/lib/trigger.rb
81
+ - server/lib/cron_trigger.rb
82
+ - server/backgroundrb_server.rb
83
+ has_rdoc: true
84
+ homepage: http://github.com/josip/backgroundrb_merb/tree/master
85
+ post_install_message:
86
+ rdoc_options: []
87
+
88
+ require_paths:
89
+ - lib
90
+ required_ruby_version: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: "0"
95
+ version:
96
+ required_rubygems_version: !ruby/object:Gem::Requirement
97
+ requirements:
98
+ - - ">="
99
+ - !ruby/object:Gem::Version
100
+ version: "0"
101
+ version:
102
+ requirements: []
103
+
104
+ rubyforge_project:
105
+ rubygems_version: 1.0.1
106
+ signing_key:
107
+ specification_version: 2
108
+ summary: BackgroundRB for Merb
109
+ test_files: []
110
+