jober 0.0.1 → 0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 658e5f757cdbd193692c730c37853625a6ece891
4
- data.tar.gz: 949f6f7ee9ede001aa2ebb14693ee4e537b94737
3
+ metadata.gz: b38442d4fe738b159e68bb21a37d42d6343058ab
4
+ data.tar.gz: 3d7ea0e3cd642cfa93545dd486e296c3b4f2a150
5
5
  SHA512:
6
- metadata.gz: 4c5152af3de453ca2f1dd7d0be59f44c52bff556b23dee3204140f18708133223da49f744f2f83acde333ee113c18f99190c65649829a3deee00a5c624a865b6
7
- data.tar.gz: 47fa1ca5099ad4294d673cd2ce54b7347432ca75a3ed5e15e9c968b9208e093b5250ebefd0f1b2a80ce949b89b0261094d06a27ad32a31bf51731539b7797472
6
+ metadata.gz: c8bb7b73c5725f34b19934b602026f91914b9cf666a5688ebcf249ea8fb4fd7ac9b0598700fba3e44a830476d335f9c2260c087fd5e79f7aa1be2653d3b65980
7
+ data.tar.gz: 867f8e4a94b2f591c6ebc2f7fd9ff0f251c6ff0fadd8db3ae9ff7c5f8984f34ca06fd71856b87bced9b6ebf786c444af2d762c54c803e988c716d319fd399cfa
data/.gitignore CHANGED
@@ -15,3 +15,4 @@ mkmf.log
15
15
  *.log
16
16
  *.swp
17
17
  todo
18
+ *.db
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --color
2
+ --format progress
@@ -0,0 +1,12 @@
1
+ language: ruby
2
+ services:
3
+ - redis-server
4
+ rvm:
5
+ - "ree"
6
+ - "1.8.7"
7
+ - "1.9.3"
8
+ - "2.0.0"
9
+ - "2.1"
10
+ - "2.2"
11
+ - "rbx"
12
+ - "jruby"
data/Rakefile CHANGED
@@ -1,2 +1,8 @@
1
1
  require "bundler/gem_tasks"
2
+ require 'rspec/core/rake_task'
2
3
 
4
+ task :default => :spec
5
+
6
+ RSpec::Core::RakeTask.new(:spec) do |t|
7
+ t.verbose = false
8
+ end
@@ -0,0 +1,25 @@
1
+ require 'rubygems'
2
+ require 'bundler/setup'
3
+ Bundler.require
4
+
5
+ $summary = 0
6
+
7
+ class Bench < Jober::Queue
8
+ def perform(x, y)
9
+ $summary += x + y
10
+ end
11
+ end
12
+
13
+ Jober.logger = Logger.new nil
14
+
15
+ t = Time.now
16
+ 50000.times { |i| Bench.enqueue(i, -i / 2.0) }
17
+ puts Time.now - t
18
+ puts Bench.len
19
+
20
+ t = Time.now
21
+ Bench.new.execute
22
+ puts Time.now - t
23
+
24
+ puts $summary
25
+ puts Bench.len
data/bin/jober CHANGED
@@ -1,10 +1,55 @@
1
1
  #!/usr/bin/env ruby
2
2
  $:.unshift File.expand_path(File.join(File.dirname(__FILE__), %w[.. lib]))
3
- require "Jober"
3
+ require "jober"
4
4
 
5
- task_name = ARGV[0]
6
- recursive = true
5
+ require 'optparse'
7
6
 
8
- while true
9
-
7
+ options = {}
8
+
9
+ OptionParser.new do |opts|
10
+ opts.on( '-h', '--help', 'Display this screen' ) do
11
+ puts opts
12
+ exit
13
+ end
14
+
15
+ opts.on( '-c', '--class CLASS', 'run for class' ) do |klass|
16
+ options[:klass] = klass
17
+ end
18
+
19
+ opts.on( '-r', '--require PATH', 'require some path, usually config/environment' ) do |req|
20
+ options[:require] = req
21
+ end
22
+
23
+ opts.on( '-o', '--once', 'run once, instead of recursive' ) do |o|
24
+ options[:once] = o
25
+ end
26
+
27
+ opts.on( '-i', '--worker_id', 'worker_id for multiple workers' ) do |o|
28
+ options[:worker_id] = o
29
+ end
30
+
31
+ opts.on( '-w', '--workers_count', 'workers_count for multiple workers' ) do |o|
32
+ options[:workers_count] = o
33
+ end
34
+
35
+ end.parse!
36
+
37
+ if req = options[:require]
38
+ require File.expand_path(req)
39
+ end
40
+
41
+ klass_name = options[:klass]
42
+ klass = Jober.find_class(klass_name)
43
+
44
+ unless klass
45
+ puts "not found class #{klass_name}"
46
+ exit(1)
47
+ end
48
+
49
+ inst = klass.new(options)
50
+
51
+ if options[:once]
52
+ inst.execute
53
+ else
54
+ inst.run_loop
10
55
  end
File without changes
@@ -1,8 +1,9 @@
1
1
  require 'bundler/setup'
2
2
  Bundler.require :default
3
3
 
4
+ Jober.default_interval = 10
5
+
4
6
  class A < Jober::Task
5
- every 10
6
7
  def perform
7
8
  10.times do |i|
8
9
  info "enqueue to b #{i}"
@@ -12,7 +13,6 @@ class A < Jober::Task
12
13
  end
13
14
 
14
15
  class B < Jober::Queue
15
- every 10
16
16
  def perform(x)
17
17
  10.times do |i|
18
18
  info "enqueue to c #{x} #{i}"
@@ -22,7 +22,6 @@ class B < Jober::Queue
22
22
  end
23
23
 
24
24
  class C < Jober::Queue
25
- every 10
26
25
  def perform(x, i)
27
26
  10.times do |j|
28
27
  info "enqueue to d #{x} #{i} #{j}"
@@ -32,7 +31,6 @@ class C < Jober::Queue
32
31
  end
33
32
 
34
33
  class D < Jober::QueueBatch
35
- every 10
36
34
  batch_size 200
37
35
 
38
36
  def perform(*batch)
@@ -41,7 +39,7 @@ class D < Jober::QueueBatch
41
39
  end
42
40
 
43
41
  class E < Jober::Task
44
- 5.times { every 5 }
42
+ workers 5
45
43
 
46
44
  def perform
47
45
  puts "start e :)"
@@ -50,15 +48,12 @@ class E < Jober::Task
50
48
  end
51
49
 
52
50
  class G < Jober::Task
53
- every 10
54
-
55
51
  def perform
56
52
  "asdfsdf" + 1
57
53
  end
58
54
  end
59
55
 
60
56
  class F < Jober::Task
61
- every 5
62
57
  def perform
63
58
  sleep 100
64
59
  end
@@ -23,4 +23,6 @@ Gem::Specification.new do |spec|
23
23
  spec.add_development_dependency "bundler", "~> 1.7"
24
24
  spec.add_development_dependency "rake", "~> 10.0"
25
25
  spec.add_development_dependency "rspec"
26
+ spec.add_development_dependency "activerecord"
27
+ spec.add_development_dependency "sqlite3"
26
28
  end
@@ -1,39 +1,48 @@
1
- require "Jober/version"
1
+ require "jober/version"
2
2
 
3
3
  require 'redis'
4
4
  require 'logger'
5
5
 
6
6
  module Jober
7
- autoload :Manager, 'Jober/manager'
8
- autoload :AbstractTask, 'Jober/abstract_task'
9
- autoload :Task, 'Jober/task'
10
- autoload :Queue, 'Jober/queue'
11
- autoload :QueueBatch, 'Jober/queue_batch'
12
- autoload :UniqueQueue, 'Jober/unique_queue'
13
- autoload :Logger, 'Jober/logger'
14
- autoload :SharedObject, 'Jober/shared_object'
7
+ autoload :Manager, 'jober/manager'
8
+ autoload :AbstractTask, 'jober/abstract_task'
9
+ autoload :Task, 'jober/task'
10
+ autoload :Queue, 'jober/queue'
11
+ autoload :QueueBatch, 'jober/queue_batch'
12
+ autoload :UniqueQueue, 'jober/unique_queue'
13
+ autoload :Logger, 'jober/logger'
14
+ autoload :SharedObject, 'jober/shared_object'
15
+ autoload :UniqueQueueBatch, 'jober/unique_queue_batch'
16
+ autoload :ARLoop, 'jober/ar_loop'
15
17
 
16
18
  class << self
17
- attr_accessor :redis_config
18
- attr_reader :classes
19
-
20
19
  def logger
21
- ::Logger.new(STDOUT)
20
+ @logger ||= ::Logger.new(STDOUT)
21
+ end
22
+
23
+ def logger=(l)
24
+ @logger = l
22
25
  end
23
26
 
24
27
  def redis
25
- Thread.current[:__redis__] ||= Redis.new(redis_config || {})
28
+ @redis ||= Redis.new
26
29
  end
27
30
 
28
- def reset_redis
29
- Thread.current[:__redis__] = nil
31
+ def redis=(r)
32
+ @redis = r
33
+ end
34
+
35
+ def internal_classes_names
36
+ @internal_classes_names ||= (%w{Manager AbstractTask Task Queue} +
37
+ %w{QueueBatch UniqueQueue UniqueQueueBatch Logger SharedObject ARLoop}).map { |k| "Jober::#{k}" }
38
+ end
39
+
40
+ def classes
41
+ @classes ||= []
30
42
  end
31
43
 
32
44
  def add_class(klass)
33
- unless klass.to_s.start_with?('Jober::')
34
- @classes ||= []
35
- @classes << klass
36
- end
45
+ classes << klass unless internal_classes_names.include?(klass.to_s)
37
46
  end
38
47
 
39
48
  def after_fork(&block)
@@ -44,6 +53,10 @@ module Jober
44
53
  @after_fork.call if @after_fork
45
54
  end
46
55
 
56
+ def reset_redis
57
+ redis.client.reconnect
58
+ end
59
+
47
60
  def dump(obj)
48
61
  Marshal.dump(obj)
49
62
  end
@@ -52,8 +65,13 @@ module Jober
52
65
  Marshal.load(obj)
53
66
  end
54
67
 
68
+ def dump_args(*args)
69
+ dump(args)
70
+ end
71
+
55
72
  def exception(ex)
56
73
  # redefine me
74
+ logger.error "#{ex.message} #{ex.backtrace}"
57
75
  end
58
76
 
59
77
  def underscore(str)
@@ -71,7 +89,7 @@ module Jober
71
89
  h = {}
72
90
  @classes.each do |klass|
73
91
  next unless klass.ancestors.include?(Jober::Queue)
74
- h[klass.short_name] = klass.len
92
+ h[klass.queue_name_base] = klass.len
75
93
  end
76
94
  h
77
95
  end
@@ -79,13 +97,59 @@ module Jober
79
97
  def stats
80
98
  h = {}
81
99
  @classes.each do |klass|
82
- start = Jober.redis.get("Jober:stats:#{klass.short_name}:start")
83
- start = Time.at(start.to_i) if start
84
- _end = Jober.redis.get("Jober:stats:#{klass.short_name}:end")
85
- _end = Time.at(_end.to_i) if _end
86
- h[klass.short_name] = {:start => start, :end => _end, :duration => (_end && start && _end >= start) ? (_end - start) : nil }
100
+ started = klass.read_timestamp(:started)
101
+ finished = klass.read_timestamp(:finished)
102
+ crashed = klass.read_timestamp(:crashed)
103
+ h[klass.short_name] = {
104
+ :started => started,
105
+ :finished => finished,
106
+ :crashed => crashed,
107
+ :duration => (finished && started && finished >= started) ? (finished - started) : nil
108
+ }
87
109
  end
88
110
  h
89
111
  end
112
+
113
+ attr_accessor :namespace
114
+
115
+ def key(k)
116
+ "Jober:#{@namespace}:#{k}"
117
+ end
118
+
119
+ def find_class(klass_name)
120
+ names = classes.map(&:to_s)
121
+ return eval(klass_name) if names.include?(klass_name)
122
+ klass_name = "Jober::#{klass_name}"
123
+ return eval(klass_name) if names.include?(klass_name)
124
+ klass_name = "Jobs::#{klass_name}"
125
+ return eval(klass_name) if names.include?(klass_name)
126
+ end
127
+
128
+ def skip_delay!
129
+ classes.each &:skip_delay!
130
+ end
131
+
132
+ def default_interval
133
+ @default_interval ||= 5 * 60
134
+ end
135
+
136
+ def default_interval=(di)
137
+ @default_interval = di
138
+ end
139
+
140
+ def enqueue(queue_name, *args)
141
+ Jober.redis.rpush(queue_name, Jober.dump_args(*args))
142
+ end
143
+
144
+ def catch(&block)
145
+ yield
146
+ rescue Object => ex
147
+ Jober.exception(ex)
148
+ nil
149
+ end
90
150
  end
91
151
  end
152
+
153
+ # just empty module for store tasks
154
+ module Jobs
155
+ end
@@ -1,13 +1,23 @@
1
+ require 'timeout'
2
+
1
3
  class Jober::AbstractTask
2
4
  include Jober::Logger
3
5
 
4
6
  class << self
5
- def every(interval, method = :perform)
6
- workers << [interval, method]
7
+ def interval(interval)
8
+ @interval = interval
9
+ end
10
+
11
+ def get_interval
12
+ @interval || Jober.default_interval
13
+ end
14
+
15
+ def workers(n)
16
+ @workers = n
7
17
  end
8
18
 
9
- def workers
10
- @workers ||= []
19
+ def get_workers
20
+ @workers || 1
11
21
  end
12
22
 
13
23
  attr_accessor :short_name
@@ -17,30 +27,110 @@ class Jober::AbstractTask
17
27
 
18
28
  def self.inherited(base)
19
29
  Jober.add_class(base)
30
+ base.interval(self.get_interval)
20
31
  end
21
32
 
22
- def initialize
33
+ # opts:
34
+ # :worker_id
35
+ # :workers_count
36
+ # :skip_delay
37
+ def initialize(opts = {})
38
+ @opts = opts
23
39
  @stopped = false
24
40
  trap("QUIT") { @stopped = true }
25
41
  trap("INT") { @stopped = true }
42
+ @worker_id = opts[:worker_id] || 0
43
+ @workers_count = opts[:workers_count] || 1
44
+ @skip_delay = opts[:skip_delay]
26
45
  end
27
46
 
28
- def execute(method = :perform)
29
- info "=> starting"
47
+ def execute
48
+ info "=> start"
30
49
  @start_at = Time.now
31
- write_timestamp(:start)
32
- run(method)
33
- write_timestamp(:end)
34
- info "<= end of #{method} in #{Time.now - @start_at}"
50
+ self.class.write_timestamp(:started)
51
+ run
52
+ self.class.write_timestamp(:finished)
53
+ self.class.del_timestamp(:crashed)
54
+ info "<= end (in #{Time.now - @start_at})"
35
55
  self
56
+ rescue Object
57
+ self.class.write_timestamp(:crashed)
58
+ raise
59
+ end
60
+
61
+ def run_loop
62
+ info { "running loop" }
63
+
64
+ # wait until interval + last end
65
+ if self.class.get_workers <= 1 &&
66
+ (finished = self.class.read_timestamp(:finished)) &&
67
+ (Time.now - finished < self.class.get_interval) &&
68
+ !self.class.pop_skip_delay_flag! &&
69
+ !@skip_delay
70
+
71
+ sleeping(self.class.get_interval - (Time.now - finished))
72
+ end
73
+
74
+ # main loop
75
+ loop do
76
+ break if stopped
77
+ execute
78
+ break if stopped
79
+ sleeping
80
+ break if stopped
81
+ end
82
+
83
+ info { "quit loop" }
84
+ end
85
+
86
+ def sleeping(int = self.class.get_interval)
87
+ info { "sleeping for %.1fm ..." % [int / 60.0] }
88
+ Timeout.timeout(int.to_f) do
89
+ loop do
90
+ sleep 0.3
91
+ return if stopped
92
+ end
93
+ end
94
+ rescue Timeout::Error
36
95
  end
37
96
 
38
97
  private
39
98
 
40
- def write_timestamp(type)
41
- Jober.redis.set("Jober:stats:#{self.class.short_name}:#{type}", Time.now.to_i.to_s)
42
- rescue Object => ex
43
- error "#{ex.inspect} #{ex.backtrace}"
99
+ def self.timestamp_key(type)
100
+ Jober.key("stats:#{short_name}:#{type}")
101
+ end
102
+
103
+ def self.pop_skip_delay_flag!
104
+ Jober.catch do
105
+ res = Jober.redis.get(timestamp_key(:skip))
106
+ Jober.redis.del(timestamp_key(:skip)) if res
107
+ !!res
108
+ end
109
+ end
110
+
111
+ def self.skip_delay!
112
+ Jober.catch do
113
+ Jober.redis.set(timestamp_key(:skip), '1')
114
+ end
115
+ end
116
+
117
+ def self.read_timestamp(type)
118
+ Jober.catch do
119
+ res = Jober.redis.get(timestamp_key(type))
120
+ Time.at(res.to_i) if res
121
+ end
122
+ end
123
+
124
+ def self.write_timestamp(type)
125
+ Jober.catch do
126
+ Jober.redis.set(timestamp_key(type), Time.now.to_i.to_s)
127
+ end
128
+ end
129
+
130
+ def self.del_timestamp(type)
131
+ Jober.catch do
132
+ Jober.redis.del(timestamp_key(type))
133
+ end
44
134
  end
45
135
 
46
136
  end