creeper 1.0.6 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
    
        data/lib/creeper.rb
    CHANGED
    
    | 
         @@ -27,8 +27,8 @@ module Creeper 
     | 
|
| 
       27 
27 
     | 
    
         
             
              ## default configuration ##
         
     | 
| 
       28 
28 
     | 
    
         | 
| 
       29 
29 
     | 
    
         
             
              @beanstalk_url   = ENV['BEANSTALK_URL'] || 'beanstalk://127.0.0.1/'
         
     | 
| 
       30 
     | 
    
         
            -
              @ 
     | 
| 
       31 
     | 
    
         
            -
              @ 
     | 
| 
      
 30 
     | 
    
         
            +
              @err_logger      = ::Logger.new($stderr)
         
     | 
| 
      
 31 
     | 
    
         
            +
              @out_logger      = ::Logger.new($stdout)
         
     | 
| 
       32 
32 
     | 
    
         
             
              @patience_soft   = 60
         
     | 
| 
       33 
33 
     | 
    
         
             
              @patience_hard   = 30
         
     | 
| 
       34 
34 
     | 
    
         
             
              @pool_size       = 2
         
     | 
| 
         @@ -101,7 +101,7 @@ module Creeper 
     | 
|
| 
       101 
101 
     | 
    
         
             
                def work(jobs = nil, size = nil)
         
     | 
| 
       102 
102 
     | 
    
         
             
                  require 'creeper/worker'
         
     | 
| 
       103 
103 
     | 
    
         | 
| 
       104 
     | 
    
         
            -
                  Creeper.pool_size = size || Creeper. 
     | 
| 
      
 104 
     | 
    
         
            +
                  Creeper.pool_size = size || Creeper.pool_
         
     | 
| 
       105 
105 
     | 
    
         | 
| 
       106 
106 
     | 
    
         
             
                  Creeper::Worker.work(jobs, Creeper.pool_size)
         
     | 
| 
       107 
107 
     | 
    
         
             
                end
         
     | 
| 
         @@ -213,8 +213,8 @@ module Creeper 
     | 
|
| 
       213 
213 
     | 
    
         
             
                ## queue ##
         
     | 
| 
       214 
214 
     | 
    
         | 
| 
       215 
215 
     | 
    
         
             
                def enqueue(job, data = {}, options = {})
         
     | 
| 
       216 
     | 
    
         
            -
                  #  
     | 
| 
       217 
     | 
    
         
            -
                   
     | 
| 
      
 216 
     | 
    
         
            +
                  # OutLogger.debug "#{Thread.current[:actor].inspect} Enqueueing #{job.inspect}, #{data.inspect}"#\n#{Celluloid::Actor.all.pretty_inspect}"
         
     | 
| 
      
 217 
     | 
    
         
            +
                  OutLogger.debug "[#{Thread.current[:actor] ? Thread.current[:actor].subject.number : nil}] Enqueueing #{job.inspect}, #{data.inspect}" if $DEBUG
         
     | 
| 
       218 
218 
     | 
    
         
             
                  enqueue!(job, data, options)
         
     | 
| 
       219 
219 
     | 
    
         
             
                rescue Beanstalk::NotConnected => e
         
     | 
| 
       220 
220 
     | 
    
         
             
                  disconnected(self, :enqueue, job, data, options)
         
     | 
| 
         @@ -236,8 +236,7 @@ module Creeper 
     | 
|
| 
       236 
236 
     | 
    
         
             
                def error_work(worker, data, name, job)
         
     | 
| 
       237 
237 
     | 
    
         
             
                  (worker.stopped_at = Time.now).tap do |stopped_at|
         
     | 
| 
       238 
238 
     | 
    
         
             
                    error_message = "#{worker.prefix} Error after #{worker.time_in_milliseconds}ms #{worker.dump(job, name, data)}"
         
     | 
| 
       239 
     | 
    
         
            -
                     
     | 
| 
       240 
     | 
    
         
            -
                    Logger.error error_message
         
     | 
| 
      
 239 
     | 
    
         
            +
                    OutLogger.error error_message
         
     | 
| 
       241 
240 
     | 
    
         
             
                  end
         
     | 
| 
       242 
241 
     | 
    
         
             
                end
         
     | 
| 
       243 
242 
     | 
    
         | 
| 
         @@ -264,19 +263,19 @@ module Creeper 
     | 
|
| 
       264 
263 
     | 
    
         | 
| 
       265 
264 
     | 
    
         
             
                def start_work(worker, data, name, job)
         
     | 
| 
       266 
265 
     | 
    
         
             
                  (worker.started_at = Time.now).tap do |started_at|
         
     | 
| 
       267 
     | 
    
         
            -
                     
     | 
| 
      
 266 
     | 
    
         
            +
                    OutLogger.info "#{worker.prefix} Working #{worker.dump(job, name, data)}"
         
     | 
| 
       268 
267 
     | 
    
         
             
                  end
         
     | 
| 
       269 
268 
     | 
    
         
             
                end
         
     | 
| 
       270 
269 
     | 
    
         | 
| 
       271 
270 
     | 
    
         
             
                def stop_work(worker, data, name, job)
         
     | 
| 
       272 
271 
     | 
    
         
             
                  (worker.stopped_at = Time.now).tap do |stopped_at|
         
     | 
| 
       273 
     | 
    
         
            -
                     
     | 
| 
      
 272 
     | 
    
         
            +
                    OutLogger.info "#{worker.prefix} Finished in #{worker.time_in_milliseconds}ms #{worker.dump(job, name, data)}"
         
     | 
| 
       274 
273 
     | 
    
         
             
                  end
         
     | 
| 
       275 
274 
     | 
    
         
             
                end
         
     | 
| 
       276 
275 
     | 
    
         | 
| 
       277 
276 
     | 
    
         
             
                def unregister_worker(worker, reason = nil)
         
     | 
| 
       278 
277 
     | 
    
         
             
                  reason ||= 'Stopping'
         
     | 
| 
       279 
     | 
    
         
            -
                   
     | 
| 
      
 278 
     | 
    
         
            +
                  OutLogger.info "#{worker.prefix} #{reason}"
         
     | 
| 
       280 
279 
     | 
    
         
             
                  lock.synchronize do
         
     | 
| 
       281 
280 
     | 
    
         
             
                    WORKERS.delete(worker.number)
         
     | 
| 
       282 
281 
     | 
    
         
             
                  end
         
     | 
| 
         @@ -296,7 +295,7 @@ module Creeper 
     | 
|
| 
       296 
295 
     | 
    
         
             
                  Thread.current[:beanstalk_connection_retries] ||= 0
         
     | 
| 
       297 
296 
     | 
    
         | 
| 
       298 
297 
     | 
    
         
             
                  if Thread.current[:beanstalk_connection_retries] >= retry_count
         
     | 
| 
       299 
     | 
    
         
            -
                     
     | 
| 
      
 298 
     | 
    
         
            +
                    OutLogger.error "Unable to connect to beanstalk after #{Thread.current[:beanstalk_connection_retries]} attempts"
         
     | 
| 
       300 
299 
     | 
    
         
             
                    Thread.current[:beanstalk_connection_retries] = 0
         
     | 
| 
       301 
300 
     | 
    
         
             
                    return false
         
     | 
| 
       302 
301 
     | 
    
         
             
                  end
         
     | 
| 
         @@ -313,7 +312,7 @@ module Creeper 
     | 
|
| 
       313 
312 
     | 
    
         
             
                def soft_shutdown_workers(timeout)
         
     | 
| 
       314 
313 
     | 
    
         
             
                  Timeout.timeout(timeout) do
         
     | 
| 
       315 
314 
     | 
    
         
             
                    actors = Celluloid::Actor.all
         
     | 
| 
       316 
     | 
    
         
            -
                     
     | 
| 
      
 315 
     | 
    
         
            +
                    OutLogger.info "Gracefully stopping #{actors.size} actors..." if actors.size > 0
         
     | 
| 
       317 
316 
     | 
    
         | 
| 
       318 
317 
     | 
    
         
             
                    # Attempt to shut down the supervision tree, if available
         
     | 
| 
       319 
318 
     | 
    
         
             
                    Celluloid::Supervisor.root.terminate if Celluloid::Supervisor.root
         
     | 
| 
         @@ -336,14 +335,14 @@ module Creeper 
     | 
|
| 
       336 
335 
     | 
    
         
             
                      end
         
     | 
| 
       337 
336 
     | 
    
         
             
                    end
         
     | 
| 
       338 
337 
     | 
    
         | 
| 
       339 
     | 
    
         
            -
                     
     | 
| 
      
 338 
     | 
    
         
            +
                    OutLogger.info "Graceful stop completed cleanly"
         
     | 
| 
       340 
339 
     | 
    
         
             
                  end
         
     | 
| 
       341 
340 
     | 
    
         
             
                end
         
     | 
| 
       342 
341 
     | 
    
         | 
| 
       343 
342 
     | 
    
         
             
                def hard_shutdown_workers(timeout)
         
     | 
| 
       344 
343 
     | 
    
         
             
                  Timeout.timeout(timeout) do
         
     | 
| 
       345 
344 
     | 
    
         
             
                    actors = Celluloid::Actor.all
         
     | 
| 
       346 
     | 
    
         
            -
                     
     | 
| 
      
 345 
     | 
    
         
            +
                    OutLogger.info "Terminating #{actors.size} actors..." if actors.size > 0
         
     | 
| 
       347 
346 
     | 
    
         | 
| 
       348 
347 
     | 
    
         
             
                    # Attempt to shut down the supervision tree, if available
         
     | 
| 
       349 
348 
     | 
    
         
             
                    Celluloid::Supervisor.root.terminate if Celluloid::Supervisor.root
         
     | 
| 
         @@ -363,13 +362,13 @@ module Creeper 
     | 
|
| 
       363 
362 
     | 
    
         
             
                      end
         
     | 
| 
       364 
363 
     | 
    
         
             
                    end
         
     | 
| 
       365 
364 
     | 
    
         | 
| 
       366 
     | 
    
         
            -
                     
     | 
| 
      
 365 
     | 
    
         
            +
                    OutLogger.info "Termination completed cleanly"
         
     | 
| 
       367 
366 
     | 
    
         
             
                  end
         
     | 
| 
       368 
367 
     | 
    
         
             
                end
         
     | 
| 
       369 
368 
     | 
    
         | 
| 
       370 
369 
     | 
    
         
             
                def kill_shutdown_workers
         
     | 
| 
       371 
370 
     | 
    
         
             
                  actors = Celluloid::Actor.all
         
     | 
| 
       372 
     | 
    
         
            -
                   
     | 
| 
      
 371 
     | 
    
         
            +
                  OutLogger.info "Killing #{actors.size} actors..." if actors.size > 0
         
     | 
| 
       373 
372 
     | 
    
         | 
| 
       374 
373 
     | 
    
         
             
                  # Attempt to shut down the supervision tree, if available
         
     | 
| 
       375 
374 
     | 
    
         
             
                  Celluloid::Supervisor.root.kill if Celluloid::Supervisor.root
         
     | 
| 
         @@ -383,7 +382,7 @@ module Creeper 
     | 
|
| 
       383 
382 
     | 
    
         
             
                    end
         
     | 
| 
       384 
383 
     | 
    
         
             
                  end
         
     | 
| 
       385 
384 
     | 
    
         | 
| 
       386 
     | 
    
         
            -
                   
     | 
| 
      
 385 
     | 
    
         
            +
                  OutLogger.info "Killing completed cleanly"
         
     | 
| 
       387 
386 
     | 
    
         
             
                end
         
     | 
| 
       388 
387 
     | 
    
         | 
| 
       389 
388 
     | 
    
         
             
                def pool_managers
         
     | 
| 
         @@ -407,5 +406,5 @@ module Creeper 
     | 
|
| 
       407 
406 
     | 
    
         
             
            end
         
     | 
| 
       408 
407 
     | 
    
         | 
| 
       409 
408 
     | 
    
         
             
            require 'creeper/creep'
         
     | 
| 
       410 
     | 
    
         
            -
            require 'creeper/ 
     | 
| 
       411 
     | 
    
         
            -
            require 'creeper/ 
     | 
| 
      
 409 
     | 
    
         
            +
            require 'creeper/err_logger'
         
     | 
| 
      
 410 
     | 
    
         
            +
            require 'creeper/out_logger'
         
     | 
| 
         @@ -1,25 +1,25 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            module Creeper
         
     | 
| 
       2 
     | 
    
         
            -
              module  
     | 
| 
      
 2 
     | 
    
         
            +
              module ErrLogger
         
     | 
| 
       3 
3 
     | 
    
         
             
                module_function
         
     | 
| 
       4 
4 
     | 
    
         | 
| 
       5 
5 
     | 
    
         
             
                # Send a debug message
         
     | 
| 
       6 
6 
     | 
    
         
             
                def debug(string)
         
     | 
| 
       7 
     | 
    
         
            -
                  Creeper. 
     | 
| 
      
 7 
     | 
    
         
            +
                  Creeper.err_logger.debug(string) if Creeper.err_logger
         
     | 
| 
       8 
8 
     | 
    
         
             
                end
         
     | 
| 
       9 
9 
     | 
    
         | 
| 
       10 
10 
     | 
    
         
             
                # Send a info message
         
     | 
| 
       11 
11 
     | 
    
         
             
                def info(string)
         
     | 
| 
       12 
     | 
    
         
            -
                  Creeper. 
     | 
| 
      
 12 
     | 
    
         
            +
                  Creeper.err_logger.info(string) if Creeper.err_logger
         
     | 
| 
       13 
13 
     | 
    
         
             
                end
         
     | 
| 
       14 
14 
     | 
    
         | 
| 
       15 
15 
     | 
    
         
             
                # Send a warning message
         
     | 
| 
       16 
16 
     | 
    
         
             
                def warn(string)
         
     | 
| 
       17 
     | 
    
         
            -
                  Creeper. 
     | 
| 
      
 17 
     | 
    
         
            +
                  Creeper.err_logger.warn(string) if Creeper.err_logger
         
     | 
| 
       18 
18 
     | 
    
         
             
                end
         
     | 
| 
       19 
19 
     | 
    
         | 
| 
       20 
20 
     | 
    
         
             
                # Send an error message
         
     | 
| 
       21 
21 
     | 
    
         
             
                def error(string)
         
     | 
| 
       22 
     | 
    
         
            -
                  Creeper. 
     | 
| 
      
 22 
     | 
    
         
            +
                  Creeper.err_logger.error(string) if Creeper.err_logger
         
     | 
| 
       23 
23 
     | 
    
         
             
                end
         
     | 
| 
       24 
24 
     | 
    
         | 
| 
       25 
25 
     | 
    
         
             
                # Handle a crash
         
     | 
| 
         @@ -1,25 +1,26 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            module Creeper
         
     | 
| 
       2 
     | 
    
         
            -
              module  
     | 
| 
       3 
     | 
    
         
            -
             
     | 
| 
      
 2 
     | 
    
         
            +
              module OutLogger
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
                extend self
         
     | 
| 
       4 
5 
     | 
    
         | 
| 
       5 
6 
     | 
    
         
             
                # Send a debug message
         
     | 
| 
       6 
7 
     | 
    
         
             
                def debug(string)
         
     | 
| 
       7 
     | 
    
         
            -
                  Creeper. 
     | 
| 
      
 8 
     | 
    
         
            +
                  Creeper.out_logger.debug(string) if Creeper.out_logger
         
     | 
| 
       8 
9 
     | 
    
         
             
                end
         
     | 
| 
       9 
10 
     | 
    
         | 
| 
       10 
11 
     | 
    
         
             
                # Send a info message
         
     | 
| 
       11 
12 
     | 
    
         
             
                def info(string)
         
     | 
| 
       12 
     | 
    
         
            -
                  Creeper. 
     | 
| 
      
 13 
     | 
    
         
            +
                  Creeper.out_logger.info(string) if Creeper.out_logger
         
     | 
| 
       13 
14 
     | 
    
         
             
                end
         
     | 
| 
       14 
15 
     | 
    
         | 
| 
       15 
16 
     | 
    
         
             
                # Send a warning message
         
     | 
| 
       16 
17 
     | 
    
         
             
                def warn(string)
         
     | 
| 
       17 
     | 
    
         
            -
                  Creeper. 
     | 
| 
      
 18 
     | 
    
         
            +
                  Creeper.out_logger.warn(string) if Creeper.out_logger
         
     | 
| 
       18 
19 
     | 
    
         
             
                end
         
     | 
| 
       19 
20 
     | 
    
         | 
| 
       20 
21 
     | 
    
         
             
                # Send an error message
         
     | 
| 
       21 
22 
     | 
    
         
             
                def error(string)
         
     | 
| 
       22 
     | 
    
         
            -
                  Creeper. 
     | 
| 
      
 23 
     | 
    
         
            +
                  Creeper.out_logger.error(string) if Creeper.out_logger
         
     | 
| 
       23 
24 
     | 
    
         
             
                end
         
     | 
| 
       24 
25 
     | 
    
         | 
| 
       25 
26 
     | 
    
         
             
                # Handle a crash
         
     | 
| 
         @@ -33,5 +34,6 @@ module Creeper 
     | 
|
| 
       33 
34 
     | 
    
         
             
                  str = "#{exception.class}: #{exception.to_s}\n"
         
     | 
| 
       34 
35 
     | 
    
         
             
                  str << exception.backtrace.join("\n")
         
     | 
| 
       35 
36 
     | 
    
         
             
                end
         
     | 
| 
      
 37 
     | 
    
         
            +
             
     | 
| 
       36 
38 
     | 
    
         
             
              end
         
     | 
| 
       37 
39 
     | 
    
         
             
            end
         
     | 
    
        data/lib/creeper/version.rb
    CHANGED
    
    
    
        data/lib/creeper/worker.rb
    CHANGED
    
    | 
         @@ -97,7 +97,7 @@ module Creeper 
     | 
|
| 
       97 
97 
     | 
    
         
             
                  @jobs = self.class.jobs_for(jobs)
         
     | 
| 
       98 
98 
     | 
    
         | 
| 
       99 
99 
     | 
    
         
             
                  Creeper.register_worker(self)
         
     | 
| 
       100 
     | 
    
         
            -
                   
     | 
| 
      
 100 
     | 
    
         
            +
                  OutLogger.info "#{prefix} Working #{self.jobs.size} jobs: [ #{self.jobs.join(' ')} ]"
         
     | 
| 
       101 
101 
     | 
    
         
             
                end
         
     | 
| 
       102 
102 
     | 
    
         | 
| 
       103 
103 
     | 
    
         
             
                def dump(job, name = nil, data = nil)
         
     | 
| 
         @@ -175,7 +175,7 @@ module Creeper 
     | 
|
| 
       175 
175 
     | 
    
         
             
                  begin
         
     | 
| 
       176 
176 
     | 
    
         
             
                    job = reserve Creeper.reserve_timeout
         
     | 
| 
       177 
177 
     | 
    
         
             
                  rescue Beanstalk::TimedOut
         
     | 
| 
       178 
     | 
    
         
            -
                     
     | 
| 
      
 178 
     | 
    
         
            +
                    OutLogger.debug "#{prefix} Back to the unemployment line" if $DEBUG
         
     | 
| 
       179 
179 
     | 
    
         
             
                    return false
         
     | 
| 
       180 
180 
     | 
    
         
             
                  end
         
     | 
| 
       181 
181 
     | 
    
         | 
| 
         @@ -183,7 +183,7 @@ module Creeper 
     | 
|
| 
       183 
183 
     | 
    
         | 
| 
       184 
184 
     | 
    
         
             
                  Thread.current[:creeper_working] = true
         
     | 
| 
       185 
185 
     | 
    
         | 
| 
       186 
     | 
    
         
            -
                   
     | 
| 
      
 186 
     | 
    
         
            +
                  OutLogger.debug "#{prefix} Got #{job.inspect}" if $DEBUG
         
     | 
| 
       187 
187 
     | 
    
         | 
| 
       188 
188 
     | 
    
         
             
                  work! job # asynchronously go to work
         
     | 
| 
       189 
189 
     | 
    
         
             
                rescue SystemExit => e
         
     | 
| 
         @@ -205,7 +205,7 @@ module Creeper 
     | 
|
| 
       205 
205 
     | 
    
         
             
                        finalizer.call
         
     | 
| 
       206 
206 
     | 
    
         
             
                      end
         
     | 
| 
       207 
207 
     | 
    
         
             
                    rescue => e
         
     | 
| 
       208 
     | 
    
         
            -
                       
     | 
| 
      
 208 
     | 
    
         
            +
                      OutLogger.crash "#{prefix} finalizer error", e
         
     | 
| 
       209 
209 
     | 
    
         
             
                    end
         
     | 
| 
       210 
210 
     | 
    
         
             
                  end
         
     | 
| 
       211 
211 
     | 
    
         | 
    
        metadata
    CHANGED
    
    | 
         @@ -1,7 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            --- !ruby/object:Gem::Specification
         
     | 
| 
       2 
2 
     | 
    
         
             
            name: creeper
         
     | 
| 
       3 
3 
     | 
    
         
             
            version: !ruby/object:Gem::Version
         
     | 
| 
       4 
     | 
    
         
            -
              version: 1.0. 
     | 
| 
      
 4 
     | 
    
         
            +
              version: 1.0.7
         
     | 
| 
       5 
5 
     | 
    
         
             
              prerelease: 
         
     | 
| 
       6 
6 
     | 
    
         
             
            platform: ruby
         
     | 
| 
       7 
7 
     | 
    
         
             
            authors:
         
     | 
| 
         @@ -129,9 +129,9 @@ files: 
     | 
|
| 
       129 
129 
     | 
    
         
             
            - lib/creeper.rb
         
     | 
| 
       130 
130 
     | 
    
         
             
            - lib/creeper/celluloid_ext.rb
         
     | 
| 
       131 
131 
     | 
    
         
             
            - lib/creeper/creep.rb
         
     | 
| 
       132 
     | 
    
         
            -
            - lib/creeper/ 
     | 
| 
      
 132 
     | 
    
         
            +
            - lib/creeper/err_logger.rb
         
     | 
| 
       133 
133 
     | 
    
         
             
            - lib/creeper/launcher.rb
         
     | 
| 
       134 
     | 
    
         
            -
            - lib/creeper/ 
     | 
| 
      
 134 
     | 
    
         
            +
            - lib/creeper/out_logger.rb
         
     | 
| 
       135 
135 
     | 
    
         
             
            - lib/creeper/version.rb
         
     | 
| 
       136 
136 
     | 
    
         
             
            - lib/creeper/worker.rb
         
     | 
| 
       137 
137 
     | 
    
         
             
            - spec/lib/creeper/session_spec.rb
         
     |