jober 0.0.1 → 0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 658e5f757cdbd193692c730c37853625a6ece891
-  data.tar.gz: 949f6f7ee9ede001aa2ebb14693ee4e537b94737
+  metadata.gz: b38442d4fe738b159e68bb21a37d42d6343058ab
+  data.tar.gz: 3d7ea0e3cd642cfa93545dd486e296c3b4f2a150
 SHA512:
-  metadata.gz: 4c5152af3de453ca2f1dd7d0be59f44c52bff556b23dee3204140f18708133223da49f744f2f83acde333ee113c18f99190c65649829a3deee00a5c624a865b6
-  data.tar.gz: 47fa1ca5099ad4294d673cd2ce54b7347432ca75a3ed5e15e9c968b9208e093b5250ebefd0f1b2a80ce949b89b0261094d06a27ad32a31bf51731539b7797472
+  metadata.gz: c8bb7b73c5725f34b19934b602026f91914b9cf666a5688ebcf249ea8fb4fd7ac9b0598700fba3e44a830476d335f9c2260c087fd5e79f7aa1be2653d3b65980
+  data.tar.gz: 867f8e4a94b2f591c6ebc2f7fd9ff0f251c6ff0fadd8db3ae9ff7c5f8984f34ca06fd71856b87bced9b6ebf786c444af2d762c54c803e988c716d319fd399cfa

data/.gitignore

@@ -15,3 +15,4 @@ mkmf.log
 *.log
 *.swp
 todo
+*.db

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+services:
+  - redis-server
+rvm:
+  - "ree"
+  - "1.8.7"
+  - "1.9.3"
+  - "2.0.0"
+  - "2.1"
+  - "2.2"
+  - "rbx"
+  - "jruby"

data/Rakefile

@@ -1,2 +1,8 @@
 require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
 
+task :default => :spec
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.verbose = false
+end

data/bench/queue.rb

@@ -0,0 +1,25 @@
+require 'rubygems'
+require 'bundler/setup'
+Bundler.require
+
+$summary = 0
+
+class Bench < Jober::Queue
+  def perform(x, y)
+    $summary += x + y
+  end
+end
+
+Jober.logger = Logger.new nil
+
+t = Time.now
+50000.times { |i| Bench.enqueue(i, -i / 2.0) }
+puts Time.now - t
+puts Bench.len
+
+t = Time.now
+Bench.new.execute
+puts Time.now - t
+
+puts $summary
+puts Bench.len

data/bin/jober

@@ -1,10 +1,55 @@
 #!/usr/bin/env ruby
 $:.unshift File.expand_path(File.join(File.dirname(__FILE__), %w[.. lib]))
-require "Jober"
+require "jober"
 
-task_name = ARGV[0]
-recursive = true
+require 'optparse'
 
-while true
-  
+options = {}
+
+OptionParser.new do |opts|
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+
+  opts.on( '-c', '--class CLASS', 'run for class' ) do |klass|
+    options[:klass] = klass
+  end
+
+  opts.on( '-r', '--require PATH', 'require some path, usually config/environment' ) do |req|
+    options[:require] = req
+  end
+
+  opts.on( '-o', '--once', 'run once, instead of recursive' ) do |o|
+    options[:once] = o
+  end
+
+  opts.on( '-i', '--worker_id', 'worker_id for multiple workers' ) do |o|
+    options[:worker_id] = o
+  end
+
+  opts.on( '-w', '--workers_count', 'workers_count for multiple workers' ) do |o|
+    options[:workers_count] = o
+  end
+
+end.parse!
+
+if req = options[:require]
+  require File.expand_path(req)
+end
+
+klass_name = options[:klass]
+klass = Jober.find_class(klass_name)
+
+unless klass
+  puts "not found class #{klass_name}"
+  exit(1)
+end
+
+inst = klass.new(options)
+
+if options[:once]
+  inst.execute
+else
+  inst.run_loop
 end

data/examples/classes.rb

@@ -1,8 +1,9 @@
 require 'bundler/setup'
 Bundler.require :default
 
+Jober.default_interval = 10
+
 class A < Jober::Task
-  every 10
   def perform
     10.times do |i|
       info "enqueue to b #{i}"
@@ -12,7 +13,6 @@ class A < Jober::Task
 end
 
 class B < Jober::Queue
-  every 10
   def perform(x)
     10.times do |i|
       info "enqueue to c #{x} #{i}"
@@ -22,7 +22,6 @@ class B < Jober::Queue
 end
 
 class C < Jober::Queue
-  every 10
   def perform(x, i)
     10.times do |j|
       info "enqueue to d #{x} #{i} #{j}"
@@ -32,7 +31,6 @@ class C < Jober::Queue
 end
 
 class D < Jober::QueueBatch
-  every 10
   batch_size 200
 
   def perform(*batch)
@@ -41,7 +39,7 @@ class D < Jober::QueueBatch
 end
 
 class E < Jober::Task
-  5.times { every 5 }
+  workers 5
 
   def perform
     puts "start e :)"
@@ -50,15 +48,12 @@ class E < Jober::Task
 end
 
 class G < Jober::Task
-  every 10
-
   def perform
     "asdfsdf" + 1
   end
 end
 
 class F < Jober::Task
-  every 5
   def perform
     sleep 100
   end

data/jober.gemspec

@@ -23,4 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "activerecord"
+  spec.add_development_dependency "sqlite3"
 end

data/lib/jober.rb

@@ -1,39 +1,48 @@
-require "Jober/version"
+require "jober/version"
 
 require 'redis'
 require 'logger'
 
 module Jober
-  autoload :Manager,      'Jober/manager'
-  autoload :AbstractTask, 'Jober/abstract_task'
-  autoload :Task,         'Jober/task'
-  autoload :Queue,        'Jober/queue'
-  autoload :QueueBatch,   'Jober/queue_batch'
-  autoload :UniqueQueue,  'Jober/unique_queue'
-  autoload :Logger,       'Jober/logger'
-  autoload :SharedObject, 'Jober/shared_object'
+  autoload :Manager,          'jober/manager'
+  autoload :AbstractTask,     'jober/abstract_task'
+  autoload :Task,             'jober/task'
+  autoload :Queue,            'jober/queue'
+  autoload :QueueBatch,       'jober/queue_batch'
+  autoload :UniqueQueue,      'jober/unique_queue'
+  autoload :Logger,           'jober/logger'
+  autoload :SharedObject,     'jober/shared_object'
+  autoload :UniqueQueueBatch, 'jober/unique_queue_batch'
+  autoload :ARLoop,           'jober/ar_loop'
 
   class << self
-    attr_accessor :redis_config
-    attr_reader :classes
-
     def logger
-      ::Logger.new(STDOUT)
+      @logger ||= ::Logger.new(STDOUT)
+    end
+
+    def logger=(l)
+      @logger = l
     end
 
     def redis
-      Thread.current[:__redis__] ||= Redis.new(redis_config || {})
+      @redis ||= Redis.new
     end
 
-    def reset_redis
-      Thread.current[:__redis__] = nil
+    def redis=(r)
+      @redis = r
+    end
+
+    def internal_classes_names
+      @internal_classes_names ||= (%w{Manager AbstractTask Task Queue} +
+        %w{QueueBatch UniqueQueue UniqueQueueBatch Logger SharedObject ARLoop}).map { |k| "Jober::#{k}" }
+    end
+
+    def classes
+      @classes ||= []
     end
 
     def add_class(klass)
-      unless klass.to_s.start_with?('Jober::')
-        @classes ||= []
-        @classes << klass
-      end
+      classes << klass unless internal_classes_names.include?(klass.to_s)
     end
 
     def after_fork(&block)
@@ -44,6 +53,10 @@ module Jober
       @after_fork.call if @after_fork
     end
 
+    def reset_redis
+      redis.client.reconnect
+    end
+
     def dump(obj)
       Marshal.dump(obj)
     end
@@ -52,8 +65,13 @@ module Jober
       Marshal.load(obj)
     end
 
+    def dump_args(*args)
+      dump(args)
+    end
+
     def exception(ex)
       # redefine me
+      logger.error "#{ex.message} #{ex.backtrace}"
     end
 
     def underscore(str)
@@ -71,7 +89,7 @@ module Jober
       h = {}
       @classes.each do |klass|
         next unless klass.ancestors.include?(Jober::Queue)
-        h[klass.short_name] = klass.len
+        h[klass.queue_name_base] = klass.len
       end
       h
     end
@@ -79,13 +97,59 @@ module Jober
     def stats
       h = {}
       @classes.each do |klass|
-        start = Jober.redis.get("Jober:stats:#{klass.short_name}:start")
-        start = Time.at(start.to_i) if start
-        _end = Jober.redis.get("Jober:stats:#{klass.short_name}:end")
-        _end = Time.at(_end.to_i) if _end
-        h[klass.short_name] = {:start => start, :end => _end, :duration => (_end && start && _end >= start) ? (_end - start) : nil }
+        started = klass.read_timestamp(:started)
+        finished = klass.read_timestamp(:finished)
+        crashed = klass.read_timestamp(:crashed)
+        h[klass.short_name] = {
+          :started => started,
+          :finished => finished,
+          :crashed => crashed,
+          :duration => (finished && started && finished >= started) ? (finished - started) : nil
+        }
       end
       h
     end
+
+    attr_accessor :namespace
+
+    def key(k)
+      "Jober:#{@namespace}:#{k}"
+    end
+
+    def find_class(klass_name)
+      names = classes.map(&:to_s)
+      return eval(klass_name) if names.include?(klass_name)
+      klass_name = "Jober::#{klass_name}"
+      return eval(klass_name) if names.include?(klass_name)
+      klass_name = "Jobs::#{klass_name}"
+      return eval(klass_name) if names.include?(klass_name)
+    end
+
+    def skip_delay!
+      classes.each &:skip_delay!
+    end
+
+    def default_interval
+      @default_interval ||= 5 * 60
+    end
+
+    def default_interval=(di)
+      @default_interval = di
+    end
+
+    def enqueue(queue_name, *args)
+      Jober.redis.rpush(queue_name, Jober.dump_args(*args))
+    end
+
+    def catch(&block)
+      yield
+    rescue Object => ex
+      Jober.exception(ex)
+      nil
+    end
   end
 end
+
+# just empty module for store tasks
+module Jobs
+end

data/lib/jober/abstract_task.rb

@@ -1,13 +1,23 @@
+require 'timeout'
+
 class Jober::AbstractTask
   include Jober::Logger
 
   class << self
-    def every(interval, method = :perform)
-      workers << [interval, method]
+    def interval(interval)
+      @interval = interval
+    end
+
+    def get_interval
+      @interval || Jober.default_interval
+    end
+
+    def workers(n)
+      @workers = n
     end
 
-    def workers
-      @workers ||= []
+    def get_workers
+      @workers || 1
     end
 
     attr_accessor :short_name
@@ -17,30 +27,110 @@ class Jober::AbstractTask
 
   def self.inherited(base)
     Jober.add_class(base)
+    base.interval(self.get_interval)
   end
 
-  def initialize
+  # opts:
+  #   :worker_id
+  #   :workers_count
+  #   :skip_delay
+  def initialize(opts = {})
+    @opts = opts
     @stopped = false
     trap("QUIT") { @stopped = true }
     trap("INT")  { @stopped = true }
+    @worker_id = opts[:worker_id] || 0
+    @workers_count = opts[:workers_count] || 1
+    @skip_delay = opts[:skip_delay]
   end
 
-  def execute(method = :perform)
-    info "=> starting"
+  def execute
+    info "=> start"
     @start_at = Time.now
-    write_timestamp(:start)
-    run(method)
-    write_timestamp(:end)
-    info "<= end of #{method} in #{Time.now - @start_at}"
+    self.class.write_timestamp(:started)
+    run
+    self.class.write_timestamp(:finished)
+    self.class.del_timestamp(:crashed)
+    info "<= end (in #{Time.now - @start_at})"
     self
+  rescue Object
+    self.class.write_timestamp(:crashed)
+    raise
+  end
+
+  def run_loop
+    info { "running loop" }
+
+    # wait until interval + last end
+    if self.class.get_workers <= 1 &&
+        (finished = self.class.read_timestamp(:finished)) &&
+        (Time.now - finished < self.class.get_interval) &&
+        !self.class.pop_skip_delay_flag! &&
+        !@skip_delay
+
+      sleeping(self.class.get_interval - (Time.now - finished))
+    end
+
+    # main loop
+    loop do
+      break if stopped
+      execute
+      break if stopped
+      sleeping
+      break if stopped
+    end
+
+    info { "quit loop" }
+  end
+
+  def sleeping(int = self.class.get_interval)
+    info { "sleeping for %.1fm ..." % [int / 60.0] }
+    Timeout.timeout(int.to_f) do
+      loop do
+        sleep 0.3
+        return if stopped
+      end
+    end
+  rescue Timeout::Error
   end
 
 private
 
-  def write_timestamp(type)
-    Jober.redis.set("Jober:stats:#{self.class.short_name}:#{type}", Time.now.to_i.to_s)
-  rescue Object => ex
-    error "#{ex.inspect} #{ex.backtrace}"
+  def self.timestamp_key(type)
+    Jober.key("stats:#{short_name}:#{type}")
+  end
+
+  def self.pop_skip_delay_flag!
+    Jober.catch do
+      res = Jober.redis.get(timestamp_key(:skip))
+      Jober.redis.del(timestamp_key(:skip)) if res
+      !!res
+    end
+  end
+
+  def self.skip_delay!
+    Jober.catch do
+      Jober.redis.set(timestamp_key(:skip), '1')
+    end
+  end
+
+  def self.read_timestamp(type)
+    Jober.catch do
+      res = Jober.redis.get(timestamp_key(type))
+      Time.at(res.to_i) if res
+    end
+  end
+
+  def self.write_timestamp(type)
+    Jober.catch do
+      Jober.redis.set(timestamp_key(type), Time.now.to_i.to_s)
+    end
+  end
+
+  def self.del_timestamp(type)
+    Jober.catch do
+      Jober.redis.del(timestamp_key(type))
+    end
   end
 
 end