periodically 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -5
- data/lib/periodically.rb +5 -2
- data/lib/periodically/debug.rb +2 -2
- data/lib/periodically/error_counts.rb +30 -0
- data/lib/periodically/error_messages.rb +27 -0
- data/lib/periodically/job.rb +10 -50
- data/lib/periodically/locks.rb +48 -0
- data/periodically.gemspec +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ac09a96f805828da8099e478e54d2aefed9678bb2027385c3e0992a46d27b969
|
4
|
+
data.tar.gz: '08ab0c0976bc83a740b520a0b1d8d0b17c90b405087c9a158950fa8611d02b64'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9a437e37fecd4fad68edf42263a9d0b086c8b60afbe6d949074f11d89d17adc7704126d5a78359a29b106a453fc35bf46eb979dda77077bfe20851b599296300
|
7
|
+
data.tar.gz: a085ecdec1308cd7a276ac9c4a0c2a47038c45943456d2a7f7c73d9a4a4f03e987430f119ce1971f1a3b6a8a6b5d8810fbb83a17d28a81f179f8670aff3fcbf8
|
data/README.md
CHANGED
@@ -1,9 +1,8 @@
|
|
1
1
|
# periodically
|
2
2
|
|
3
|
-
Redis-backed Ruby library for tasks that need to run once in a while. "Once in a while" is defined more accurately by a custom lambda
|
4
|
-
block by the library user.
|
3
|
+
Redis-backed Ruby library for tasks that need to run once in a while. "Once in a while" is intentionally vague and should be defined more accurately by a custom lambda block by the library user.
|
5
4
|
|
6
|
-
Since task execution is done in a single
|
5
|
+
Since task execution is done in a single thread using non-accurate and non-time-based conditions, periodically is best for
|
7
6
|
infrequent and noncritical jobs, such as weekly syncs.
|
8
7
|
|
9
8
|
Example usecases:
|
@@ -32,7 +31,7 @@ Periodically.start
|
|
32
31
|
|
33
32
|
In Rails, Periodically jobs are only registered when the class is loaded.
|
34
33
|
In production mode Rails (by default) eagerly loads all classes, meaning that everything is fine.
|
35
|
-
However, in development mode you might want to
|
34
|
+
However, in development mode you might want to disable eager mode with `config.eager_load = false`
|
36
35
|
|
37
36
|
**Utilize Periodically in e.g. a Model**
|
38
37
|
|
@@ -79,7 +78,7 @@ include Periodically::Model
|
|
79
78
|
|
80
79
|
# Enqueue a Periodically job
|
81
80
|
periodically :update_method, # call instance method "update_method" for found instances
|
82
|
-
on: -> { where("last_synced < ?", 7.days.ago) }, #
|
81
|
+
on: -> { Item.where("last_synced < ?", 7.days.ago) }, # (Optional) Iterator/Array of instances to update. Empty array to skip update
|
83
82
|
min_class_interval: 5.minutes, # (Optional) The minimum interval between calls to this specific class (TODO not implemented)
|
84
83
|
max_retries: 25, # (Optional) Maximum number of retries. Periodically uses exponential backoff (TODO not implemented)
|
85
84
|
instance_id: -> { cache_key_with_version }, # (Optional) Returns this instance's unique identifying key. Used for e.g. deferring jobs and marking them as erroring (TODO not implemented)
|
data/lib/periodically.rb
CHANGED
@@ -2,11 +2,14 @@
|
|
2
2
|
|
3
3
|
require "logger"
|
4
4
|
|
5
|
-
require "periodically/job"
|
6
5
|
require "periodically/debug"
|
7
6
|
require "periodically/defer"
|
8
|
-
require "periodically/
|
7
|
+
require "periodically/error_counts"
|
8
|
+
require "periodically/error_messages"
|
9
|
+
require "periodically/job"
|
10
|
+
require "periodically/locks"
|
9
11
|
require "periodically/model"
|
12
|
+
require "periodically/redis"
|
10
13
|
|
11
14
|
module Periodically
|
12
15
|
@@registered = []
|
data/lib/periodically/debug.rb
CHANGED
@@ -4,11 +4,11 @@ module Periodically
|
|
4
4
|
module Debug
|
5
5
|
def self.total_debug_dump
|
6
6
|
error_counts = Periodically.redis do |conn|
|
7
|
-
keys = conn.
|
7
|
+
keys = conn.keys("errors:*").to_a
|
8
8
|
if keys.length == 0
|
9
9
|
{}
|
10
10
|
else
|
11
|
-
values = conn.mget(keys)
|
11
|
+
values = conn.mget(keys).map(&:to_i)
|
12
12
|
Hash[keys.zip(values)]
|
13
13
|
end
|
14
14
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Periodically
|
4
|
+
module ErrorCounts
|
5
|
+
def self.get(key)
|
6
|
+
Periodically.redis {|conn| conn.get(error_count_key(key))}.to_i
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.increment(key)
|
10
|
+
Periodically.redis {|conn| conn.incr(error_count_key(key))}
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.reset(key)
|
14
|
+
Periodically.redis {|conn| conn.del(error_count_key(key))}
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.reset_all
|
18
|
+
Periodically.redis do |conn|
|
19
|
+
keys = conn.keys("errors:*")
|
20
|
+
conn.del(keys)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def self.error_count_key(key)
|
27
|
+
"errors:#{key}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Periodically
|
4
|
+
module ErrorMessages
|
5
|
+
def self.store(key, err)
|
6
|
+
stored_error = "#{err.message}\n#{err.backtrace.join("\n")}"
|
7
|
+
Periodically.redis {|conn| conn.set(error_message_key(key), stored_error)}
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.get_latest(key)
|
11
|
+
Periodically.redis {|conn| conn.get(error_message_key(key))}
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.clear_all
|
15
|
+
Periodically.redis do |conn|
|
16
|
+
keys = conn.keys("error_messages:*")
|
17
|
+
conn.del(keys)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def self.error_message_key(key)
|
24
|
+
"error_messages:#{key}"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/periodically/job.rb
CHANGED
@@ -13,11 +13,11 @@ module Periodically
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def poll_next_instance
|
16
|
-
return if
|
16
|
+
return if Periodically::Locks.locked?(@job_key, @class_key)
|
17
17
|
|
18
18
|
where = @opts[:on].call()
|
19
19
|
where.to_a.find do |obj|
|
20
|
-
!
|
20
|
+
!Periodically::Locks.locked?(instance_key(obj))
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
@@ -26,23 +26,22 @@ module Periodically
|
|
26
26
|
begin
|
27
27
|
instance.send(@method)
|
28
28
|
rescue => e
|
29
|
-
|
30
|
-
Periodically.
|
31
|
-
|
32
|
-
|
33
|
-
store_instance_error(instance, stored_error)
|
29
|
+
Periodically.logger.error("Job instance[#{instance}] execution raised an exception\n#{e.message}")
|
30
|
+
new_error_count = Periodically::ErrorCounts.increment(instance_key(instance))
|
31
|
+
Periodically::Locks.lock(instance_key(instance), DEFAULT_ERROR_DELAY.call(new_error_count))
|
32
|
+
Periodically::ErrorMessages.store(instance_key(instance), e)
|
34
33
|
return
|
35
34
|
end
|
36
35
|
|
37
36
|
if return_value.is_a?(Periodically::Defer::DeferInstance)
|
38
|
-
|
37
|
+
Periodically::Locks.lock(instance_key(instance), return_value.duration)
|
39
38
|
elsif return_value.is_a?(Periodically::Defer::DeferJob)
|
40
|
-
|
39
|
+
Periodically::Locks.lock(@job_key, return_value.duration)
|
41
40
|
elsif return_value.is_a?(Periodically::Defer::DeferClass)
|
42
|
-
|
41
|
+
Periodically::Locks.lock(@class_key, return_value.duration)
|
43
42
|
end
|
44
43
|
|
45
|
-
|
44
|
+
Periodically::ErrorCounts.reset(instance_key(instance))
|
46
45
|
end
|
47
46
|
|
48
47
|
private
|
@@ -50,44 +49,5 @@ module Periodically
|
|
50
49
|
def instance_key(instance)
|
51
50
|
"#{@job_key}/#{instance.id}"
|
52
51
|
end
|
53
|
-
|
54
|
-
def increase_instance_error_count(instance)
|
55
|
-
error_count_key = "errors:#{instance_key(instance)}"
|
56
|
-
Periodically.redis {|conn| conn.incr(error_count_key)}
|
57
|
-
end
|
58
|
-
|
59
|
-
def clear_instance_error_count(instance)
|
60
|
-
error_count_key = "errors:#{instance_key(instance)}"
|
61
|
-
Periodically.redis {|conn| conn.del(error_count_key)}
|
62
|
-
end
|
63
|
-
|
64
|
-
def store_instance_error(instance, error)
|
65
|
-
error_message_key = "errormessages:#{instance_key(instance)}"
|
66
|
-
Periodically.redis {|conn| conn.set(error_message_key, error)}
|
67
|
-
end
|
68
|
-
|
69
|
-
def job_or_class_locked?
|
70
|
-
Periodically.redis do |conn|
|
71
|
-
# TODO can this be optimized with exists?(Array)
|
72
|
-
conn.exists("locks:#{@job_key}") || conn.exists("locks:#{@class_key}")
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def instance_locked?(instance)
|
77
|
-
Periodically.redis {|conn| conn.exists("locks:#{instance_key(instance)}")}
|
78
|
-
end
|
79
|
-
|
80
|
-
def lock_instance(instance, seconds)
|
81
|
-
Job.lock_key("locks:#{instance_key(instance)}", seconds)
|
82
|
-
end
|
83
|
-
|
84
|
-
def self.lock_key(key, seconds)
|
85
|
-
Periodically.redis do |conn|
|
86
|
-
conn.multi do |multi|
|
87
|
-
multi.set(key, "1")
|
88
|
-
multi.expire(key, seconds)
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
52
|
end
|
93
53
|
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Periodically
|
4
|
+
module Locks
|
5
|
+
def self.locked?(*keys)
|
6
|
+
return false if keys.empty?
|
7
|
+
lkeys = keys.map { |key| lock_key(key) }
|
8
|
+
|
9
|
+
Periodically.redis do |conn|
|
10
|
+
lkeys.any? { |lkey| conn.exists(lkey) }
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.lock(key, seconds)
|
15
|
+
lkey = lock_key(key)
|
16
|
+
|
17
|
+
Periodically.redis do |conn|
|
18
|
+
conn.multi do |multi|
|
19
|
+
multi.set(lkey, "1")
|
20
|
+
multi.expire(lkey, seconds)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.remaining(key)
|
26
|
+
lkey = lock_key(key)
|
27
|
+
|
28
|
+
Periodically.redis do |conn|
|
29
|
+
remaining = conn.ttl(lkey)
|
30
|
+
return nil if remaining == -1
|
31
|
+
remaining
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.reset_all
|
36
|
+
Periodically.redis do |conn|
|
37
|
+
keys = conn.keys("locks:*")
|
38
|
+
conn.del(keys)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def self.lock_key(key)
|
45
|
+
"locks:#{key}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/periodically.gemspec
CHANGED
@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
|
|
4
4
|
|
5
5
|
gem.files = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
|
6
6
|
gem.name = "periodically"
|
7
|
-
gem.version = "0.0.
|
7
|
+
gem.version = "0.0.8"
|
8
8
|
gem.required_ruby_version = ">= 2.5.0"
|
9
9
|
|
10
10
|
gem.add_dependency "redis", ">= 4.1.0"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: periodically
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- wyozi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-01-
|
11
|
+
date: 2020-01-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -68,7 +68,10 @@ files:
|
|
68
68
|
- lib/periodically/cli.rb
|
69
69
|
- lib/periodically/debug.rb
|
70
70
|
- lib/periodically/defer.rb
|
71
|
+
- lib/periodically/error_counts.rb
|
72
|
+
- lib/periodically/error_messages.rb
|
71
73
|
- lib/periodically/job.rb
|
74
|
+
- lib/periodically/locks.rb
|
72
75
|
- lib/periodically/model.rb
|
73
76
|
- lib/periodically/redis.rb
|
74
77
|
- periodically.gemspec
|