async-job 0.7.1 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/lib/async/job/buffer.rb +8 -0
- data/lib/async/job/builder.rb +18 -28
- data/lib/async/job/processor/aggregate.rb +93 -0
- data/lib/async/job/processor/delayed.rb +29 -0
- data/lib/async/job/processor/generic.rb +28 -0
- data/lib/async/job/processor/inline.rb +45 -0
- data/lib/async/job/{backend/redis/delayed_queue.rb → processor/redis/delayed_jobs.rb} +5 -7
- data/lib/async/job/{backend → processor}/redis/job_store.rb +1 -1
- data/lib/async/job/{backend/redis/processing_queue.rb → processor/redis/processing_list.rb} +8 -10
- data/lib/async/job/{backend/redis/ready_queue.rb → processor/redis/ready_list.rb} +2 -2
- data/lib/async/job/processor/redis/server.rb +104 -0
- data/lib/async/job/{backend → processor}/redis.rb +1 -1
- data/lib/async/job/{backend/inline.rb → processor.rb} +4 -6
- data/lib/async/job/queue.rb +32 -0
- data/lib/async/job/version.rb +1 -1
- data/lib/async/job.rb +1 -1
- data/readme.md +1 -1
- data.tar.gz.sig +0 -0
- metadata +14 -13
- metadata.gz.sig +0 -0
- data/lib/async/job/backend/aggregate/server.rb +0 -85
- data/lib/async/job/backend/aggregate.rb +0 -18
- data/lib/async/job/backend/inline/server.rb +0 -37
- data/lib/async/job/backend/redis/server.rb +0 -84
- data/lib/async/job/backend.rb +0 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7381f7520a5ee41df3c93fc4f55c647952f10066868fcb22a196fb6ee807a1dd
|
4
|
+
data.tar.gz: a063193f2ff8a71fe798aadc3ce07d392fd2c53cadd5523cb20813fff9da2bdc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 56f1a056d2cbc2dab6a97226c07b6d2acec56de40bd95c302b652e4689c3fa08986a7b6f4e4c53a0ca15555c61695da6ef2219f8585d1c1006c01cacfffac50b
|
7
|
+
data.tar.gz: b0c62cf9b609beee163c948ff9fc933a1d82b3f0f611e555ddc4dadd4909ab3e0791e474ae7cdf35cb1081d12d38a5f73f37148945e2658bec07af31388b8480
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/lib/async/job/buffer.rb
CHANGED
data/lib/async/job/builder.rb
CHANGED
@@ -3,11 +3,11 @@
|
|
3
3
|
# Released under the MIT License.
|
4
4
|
# Copyright, 2024, by Samuel Williams.
|
5
5
|
|
6
|
+
require_relative 'queue'
|
7
|
+
|
6
8
|
module Async
|
7
9
|
module Job
|
8
10
|
class Builder
|
9
|
-
Pipeline = Struct.new(:producer, :consumer, :delegate)
|
10
|
-
|
11
11
|
def self.build(delegate = nil, &block)
|
12
12
|
builder = self.new(delegate)
|
13
13
|
|
@@ -16,58 +16,48 @@ module Async
|
|
16
16
|
return builder.build
|
17
17
|
end
|
18
18
|
|
19
|
+
# @parameter delegate [Object] The initial delegate that will be wrapped by the queue.
|
19
20
|
def initialize(delegate = nil)
|
21
|
+
# The client side middleware, in the order they should be applied to a job:
|
20
22
|
@enqueue = []
|
23
|
+
|
24
|
+
# The server side middleware, in the order they should be applied to a job:
|
21
25
|
@dequeue = []
|
22
|
-
@delegate = delegate
|
23
26
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
def enqueue(middleware)
|
28
|
-
@enqueue << middleware
|
27
|
+
# The output delegate, if any:
|
28
|
+
@delegate = delegate
|
29
29
|
end
|
30
30
|
|
31
|
-
def
|
32
|
-
|
33
|
-
# The queue itself is instantiated with the delegate.
|
34
|
-
@queue = ->(delegate){queue.new(delegate, *arguments, **options)}
|
31
|
+
def enqueue(middleware, ...)
|
32
|
+
@enqueue << ->(delegate){middleware.new(delegate, ...)}
|
35
33
|
end
|
36
34
|
|
37
|
-
def dequeue(middleware)
|
38
|
-
@dequeue << middleware
|
35
|
+
def dequeue(middleware, ...)
|
36
|
+
@dequeue << ->(delegate){middleware.new(delegate, ...)}
|
39
37
|
end
|
40
38
|
|
41
39
|
def delegate(delegate)
|
42
40
|
@delegate = delegate
|
43
41
|
end
|
44
42
|
|
45
|
-
def build(&block)
|
46
|
-
# To construct the queue, we need the delegate.
|
47
|
-
delegate = @delegate
|
48
|
-
|
43
|
+
def build(delegate = @delegate, &block)
|
49
44
|
# We then wrap the delegate with the middleware in reverse order:
|
50
45
|
@dequeue.reverse_each do |middleware|
|
51
|
-
delegate = middleware.
|
46
|
+
delegate = middleware.call(delegate)
|
52
47
|
end
|
53
48
|
|
54
|
-
|
55
|
-
if @queue
|
56
|
-
producer = consumer = @queue.call(delegate)
|
57
|
-
else
|
58
|
-
producer = consumer = delegate
|
59
|
-
end
|
49
|
+
client = server = delegate
|
60
50
|
|
61
51
|
# We now construct the queue producer:
|
62
52
|
@enqueue.reverse_each do |middleware|
|
63
|
-
|
53
|
+
client = middleware.call(client)
|
64
54
|
end
|
65
55
|
|
66
56
|
if block_given?
|
67
|
-
|
57
|
+
client = yield(client) || client
|
68
58
|
end
|
69
59
|
|
70
|
-
return
|
60
|
+
return Queue.new(client, server, @delegate)
|
71
61
|
end
|
72
62
|
end
|
73
63
|
end
|
@@ -0,0 +1,93 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
require_relative 'generic'
|
7
|
+
|
8
|
+
require 'console/event/failure'
|
9
|
+
|
10
|
+
module Async
|
11
|
+
module Job
|
12
|
+
module Processor
|
13
|
+
class Aggregate < Generic
|
14
|
+
def initialize(delegate, parent: nil)
|
15
|
+
super(delegate)
|
16
|
+
|
17
|
+
@task = nil
|
18
|
+
@ready = Async::Condition.new
|
19
|
+
|
20
|
+
@pending = Array.new
|
21
|
+
@processing = Array.new
|
22
|
+
end
|
23
|
+
|
24
|
+
def flush(jobs)
|
25
|
+
while job = jobs.shift
|
26
|
+
@delegate.call(job)
|
27
|
+
end
|
28
|
+
rescue => error
|
29
|
+
Console::Event::Failure.for(error).emit(self, "Could not flush #{jobs.size} jobs.")
|
30
|
+
end
|
31
|
+
|
32
|
+
def run(task)
|
33
|
+
while true
|
34
|
+
while @pending.empty?
|
35
|
+
@ready.wait
|
36
|
+
end
|
37
|
+
|
38
|
+
task.defer_stop do
|
39
|
+
# Swap the buffers:
|
40
|
+
@pending, @processing = @processing, @pending
|
41
|
+
|
42
|
+
flush(@processing)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# Start the background processing task if it is not already running.
|
48
|
+
#
|
49
|
+
# @return [Boolean] true if the task was started, false if it was already running.
|
50
|
+
protected def start!(parent: Async::Task.current)
|
51
|
+
return false if @task
|
52
|
+
|
53
|
+
# We are creating a task:
|
54
|
+
@task = true
|
55
|
+
|
56
|
+
parent.async(transient: true, annotation: self.class.name) do |task|
|
57
|
+
@task = task
|
58
|
+
|
59
|
+
run(task)
|
60
|
+
ensure
|
61
|
+
# Ensure that all jobs are flushed before we exit:
|
62
|
+
flush(@processing) if @processing.any?
|
63
|
+
flush(@pending) if @pending.any?
|
64
|
+
@task = nil
|
65
|
+
end
|
66
|
+
|
67
|
+
return true
|
68
|
+
end
|
69
|
+
|
70
|
+
# Enqueue a job into the pending buffer.
|
71
|
+
#
|
72
|
+
# Start the background processing task if it is not already running.
|
73
|
+
def call(job)
|
74
|
+
@pending << job
|
75
|
+
|
76
|
+
start! or @ready.signal
|
77
|
+
end
|
78
|
+
|
79
|
+
def start
|
80
|
+
super
|
81
|
+
|
82
|
+
self.start!
|
83
|
+
end
|
84
|
+
|
85
|
+
def stop
|
86
|
+
@task&.stop
|
87
|
+
|
88
|
+
super
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
require_relative 'generic'
|
7
|
+
|
8
|
+
require 'console/event/failure'
|
9
|
+
|
10
|
+
module Async
|
11
|
+
module Job
|
12
|
+
module Processor
|
13
|
+
# Add a small processing delay to each job.
|
14
|
+
class Delayed < Generic
|
15
|
+
def initialize(delegate, delay: 0.1)
|
16
|
+
super(delegate)
|
17
|
+
|
18
|
+
@delay = delay
|
19
|
+
end
|
20
|
+
|
21
|
+
def call(job)
|
22
|
+
sleep(@delay)
|
23
|
+
|
24
|
+
super
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
module Async
|
7
|
+
module Job
|
8
|
+
module Processor
|
9
|
+
class Generic
|
10
|
+
def initialize(delegate = nil)
|
11
|
+
@delegate = delegate
|
12
|
+
end
|
13
|
+
|
14
|
+
def call(job)
|
15
|
+
@delegate.call(job)
|
16
|
+
end
|
17
|
+
|
18
|
+
def start
|
19
|
+
@delegate.start
|
20
|
+
end
|
21
|
+
|
22
|
+
def stop
|
23
|
+
@delegate.stop
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
require_relative '../coder'
|
7
|
+
require_relative 'generic'
|
8
|
+
|
9
|
+
require 'async/idler'
|
10
|
+
|
11
|
+
module Async
|
12
|
+
module Job
|
13
|
+
module Processor
|
14
|
+
class Inline < Generic
|
15
|
+
def initialize(delegate, parent: nil)
|
16
|
+
super(delegate)
|
17
|
+
|
18
|
+
@parent = parent || Async::Idler.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def call(job)
|
22
|
+
scheduled_at = Coder::Time(job["scheduled_at"])
|
23
|
+
|
24
|
+
@parent.async do
|
25
|
+
if scheduled_at
|
26
|
+
sleep(scheduled_at - Time.now)
|
27
|
+
end
|
28
|
+
|
29
|
+
@delegate.call(job)
|
30
|
+
rescue => error
|
31
|
+
Console.error(self, error)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def start
|
36
|
+
@delegate&.start
|
37
|
+
end
|
38
|
+
|
39
|
+
def stop
|
40
|
+
@delegate&.stop
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -5,9 +5,9 @@
|
|
5
5
|
|
6
6
|
module Async
|
7
7
|
module Job
|
8
|
-
module
|
8
|
+
module Processor
|
9
9
|
module Redis
|
10
|
-
class
|
10
|
+
class DelayedJobs
|
11
11
|
ADD = <<~LUA
|
12
12
|
redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
|
13
13
|
redis.call('ZADD', KEYS[2], ARGV[3], ARGV[1])
|
@@ -30,15 +30,13 @@ module Async
|
|
30
30
|
@move = @client.script(:load, MOVE)
|
31
31
|
end
|
32
32
|
|
33
|
-
def start(
|
34
|
-
Console.info(self, "Starting delayed queue...")
|
33
|
+
def start(ready_list, resolution: 10, parent: Async::Task.current)
|
35
34
|
parent.async do
|
36
35
|
while true
|
37
|
-
|
38
|
-
count = move(destination: ready_queue.key)
|
36
|
+
count = move(destination: ready_list.key)
|
39
37
|
|
40
38
|
if count > 0
|
41
|
-
Console.
|
39
|
+
Console.debug(self, "Moved #{count} delayed jobs to ready list.")
|
42
40
|
end
|
43
41
|
|
44
42
|
sleep(resolution)
|
@@ -5,9 +5,9 @@
|
|
5
5
|
|
6
6
|
module Async
|
7
7
|
module Job
|
8
|
-
module
|
8
|
+
module Processor
|
9
9
|
module Redis
|
10
|
-
class
|
10
|
+
class ProcessingList
|
11
11
|
REQUEUE = <<~LUA
|
12
12
|
local cursor = "0"
|
13
13
|
local count = 0
|
@@ -50,12 +50,12 @@ module Async
|
|
50
50
|
redis.call('HDEL', KEYS[2], ARGV[1])
|
51
51
|
LUA
|
52
52
|
|
53
|
-
def initialize(client, key, id,
|
53
|
+
def initialize(client, key, id, ready_list, job_store)
|
54
54
|
@client = client
|
55
55
|
@key = key
|
56
56
|
@id = id
|
57
57
|
|
58
|
-
@
|
58
|
+
@ready_list = ready_list
|
59
59
|
@job_store = job_store
|
60
60
|
|
61
61
|
@pending_key = "#{@key}:#{@id}:pending"
|
@@ -69,7 +69,7 @@ module Async
|
|
69
69
|
attr :key
|
70
70
|
|
71
71
|
def fetch
|
72
|
-
@client.brpoplpush(@
|
72
|
+
@client.brpoplpush(@ready_list.key, @pending_key, 0)
|
73
73
|
end
|
74
74
|
|
75
75
|
def complete(id)
|
@@ -79,22 +79,20 @@ module Async
|
|
79
79
|
|
80
80
|
def retry(id)
|
81
81
|
Console.warn(self, "Retrying job: #{id}")
|
82
|
-
@client.evalsha(@retry, 2, @pending_key, @
|
82
|
+
@client.evalsha(@retry, 2, @pending_key, @ready_list.key, id)
|
83
83
|
end
|
84
84
|
|
85
85
|
def start(delay: 5, factor: 2, parent: Async::Task.current)
|
86
86
|
heartbeat_key = "#{@key}:#{@id}"
|
87
87
|
start_time = Time.now.to_f
|
88
88
|
|
89
|
-
Console.info(self, "Starting processing queue...", key: @key, id: @id, heartbeat_key: heartbeat_key, delay: delay, factor: factor)
|
90
|
-
|
91
89
|
parent.async do
|
92
90
|
while true
|
93
91
|
uptime = (Time.now.to_f - start_time).round(2)
|
94
92
|
@client.set(heartbeat_key, JSON.dump(uptime: uptime), seconds: delay*factor)
|
95
93
|
|
96
|
-
#
|
97
|
-
count = @client.evalsha(@requeue, 2, @key, @
|
94
|
+
# REQUEUE any jobs that have been abandoned:
|
95
|
+
count = @client.evalsha(@requeue, 2, @key, @ready_list.key)
|
98
96
|
if count > 0
|
99
97
|
Console.warn(self, "Requeued #{count} abandoned jobs.")
|
100
98
|
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
require_relative 'delayed_jobs'
|
7
|
+
require_relative 'job_store'
|
8
|
+
require_relative 'processing_list'
|
9
|
+
require_relative 'ready_list'
|
10
|
+
require_relative '../../coder'
|
11
|
+
require_relative '../generic'
|
12
|
+
|
13
|
+
require 'securerandom'
|
14
|
+
require 'async/idler'
|
15
|
+
|
16
|
+
module Async
|
17
|
+
module Job
|
18
|
+
module Processor
|
19
|
+
module Redis
|
20
|
+
class Server < Generic
|
21
|
+
def initialize(delegate, client, prefix: 'async-job', coder: Coder::DEFAULT, resolution: 10, parent: nil)
|
22
|
+
super(delegate)
|
23
|
+
|
24
|
+
@id = SecureRandom.uuid
|
25
|
+
@client = client
|
26
|
+
@prefix = prefix
|
27
|
+
@coder = coder
|
28
|
+
@resolution = resolution
|
29
|
+
|
30
|
+
@job_store = JobStore.new(@client, "#{@prefix}:jobs")
|
31
|
+
@delayed_jobs = DelayedJobs.new(@client, "#{@prefix}:delayed")
|
32
|
+
@ready_list = ReadyList.new(@client, "#{@prefix}:ready")
|
33
|
+
@processing_list = ProcessingList.new(@client, "#{@prefix}:processing", @id, @ready_list, @job_store)
|
34
|
+
|
35
|
+
@parent = parent || Async::Idler.new
|
36
|
+
end
|
37
|
+
|
38
|
+
def start!
|
39
|
+
return false if @task
|
40
|
+
|
41
|
+
@task = true
|
42
|
+
|
43
|
+
@parent.async(transient: true, annotation: self.class.name) do |task|
|
44
|
+
@task = task
|
45
|
+
|
46
|
+
while true
|
47
|
+
self.dequeue(task)
|
48
|
+
end
|
49
|
+
ensure
|
50
|
+
@task = nil
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def start
|
55
|
+
super
|
56
|
+
|
57
|
+
# Start the delayed processor, which will move jobs to the ready processor when they are ready:
|
58
|
+
@delayed_jobs.start(@ready_list, resolution: @resolution)
|
59
|
+
|
60
|
+
# Start the processing processor, which will move jobs to the ready processor when they are abandoned:
|
61
|
+
@processing_list.start
|
62
|
+
|
63
|
+
self.start!
|
64
|
+
end
|
65
|
+
|
66
|
+
def stop
|
67
|
+
@task&.stop
|
68
|
+
|
69
|
+
super
|
70
|
+
end
|
71
|
+
|
72
|
+
def call(job)
|
73
|
+
scheduled_at = Coder::Time(job["scheduled_at"])
|
74
|
+
|
75
|
+
if scheduled_at
|
76
|
+
@delayed_jobs.add(@coder.dump(job), scheduled_at, @job_store)
|
77
|
+
else
|
78
|
+
@ready_list.add(@coder.dump(job), @job_store)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
protected
|
83
|
+
|
84
|
+
def dequeue(parent)
|
85
|
+
_id = @processing_list.fetch
|
86
|
+
|
87
|
+
parent.async do
|
88
|
+
id = _id; _id = nil
|
89
|
+
|
90
|
+
job = @coder.load(@job_store.get(id))
|
91
|
+
@delegate.call(job)
|
92
|
+
@processing_list.complete(id)
|
93
|
+
rescue => error
|
94
|
+
@processing_list.retry(id)
|
95
|
+
Console.error(self, error)
|
96
|
+
end
|
97
|
+
ensure
|
98
|
+
@processing_list.retry(_id) if _id
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -3,15 +3,13 @@
|
|
3
3
|
# Released under the MIT License.
|
4
4
|
# Copyright, 2024, by Samuel Williams.
|
5
5
|
|
6
|
-
require_relative 'inline
|
6
|
+
require_relative 'processor/inline'
|
7
7
|
|
8
8
|
module Async
|
9
9
|
module Job
|
10
|
-
module
|
11
|
-
|
12
|
-
|
13
|
-
return Server.new(delegate)
|
14
|
-
end
|
10
|
+
module Processor
|
11
|
+
def self.new(processor: Inline, **options)
|
12
|
+
processor.new(**options)
|
15
13
|
end
|
16
14
|
end
|
17
15
|
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
module Async
|
7
|
+
module Job
|
8
|
+
class Queue
|
9
|
+
def initialize(client, server, delegate)
|
10
|
+
@client = client
|
11
|
+
@server = server
|
12
|
+
@delegate = delegate
|
13
|
+
end
|
14
|
+
|
15
|
+
attr :client
|
16
|
+
attr :server
|
17
|
+
attr :delegate
|
18
|
+
|
19
|
+
def call(...)
|
20
|
+
@client.call(...)
|
21
|
+
end
|
22
|
+
|
23
|
+
def start
|
24
|
+
@server.start
|
25
|
+
end
|
26
|
+
|
27
|
+
def stop
|
28
|
+
@server.stop
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
data/lib/async/job/version.rb
CHANGED
data/lib/async/job.rb
CHANGED
data/readme.md
CHANGED
@@ -10,7 +10,7 @@ Please see the [project documentation](https://socketry.github.io/async-job/) fo
|
|
10
10
|
|
11
11
|
- [Getting Started](https://socketry.github.io/async-job/guides/getting-started/index) - This guide gives you an overview of the `async-job` gem and explains the core concepts.
|
12
12
|
|
13
|
-
- [Redis
|
13
|
+
- [Redis Processor](https://socketry.github.io/async-job/guides/redis-processor/index) - This guide gives a brief overview of the implementation of the Redis queue.
|
14
14
|
|
15
15
|
## See Also
|
16
16
|
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: async-job
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Samuel Williams
|
@@ -75,23 +75,24 @@ extensions: []
|
|
75
75
|
extra_rdoc_files: []
|
76
76
|
files:
|
77
77
|
- lib/async/job.rb
|
78
|
-
- lib/async/job/backend.rb
|
79
|
-
- lib/async/job/backend/aggregate.rb
|
80
|
-
- lib/async/job/backend/aggregate/server.rb
|
81
|
-
- lib/async/job/backend/inline.rb
|
82
|
-
- lib/async/job/backend/inline/server.rb
|
83
|
-
- lib/async/job/backend/redis.rb
|
84
|
-
- lib/async/job/backend/redis/delayed_queue.rb
|
85
|
-
- lib/async/job/backend/redis/job_store.rb
|
86
|
-
- lib/async/job/backend/redis/processing_queue.rb
|
87
|
-
- lib/async/job/backend/redis/ready_queue.rb
|
88
|
-
- lib/async/job/backend/redis/server.rb
|
89
78
|
- lib/async/job/buffer.rb
|
90
79
|
- lib/async/job/builder.rb
|
91
80
|
- lib/async/job/coder.rb
|
92
81
|
- lib/async/job/coder/marshal.rb
|
93
82
|
- lib/async/job/coder/message_pack.rb
|
94
83
|
- lib/async/job/generic.rb
|
84
|
+
- lib/async/job/processor.rb
|
85
|
+
- lib/async/job/processor/aggregate.rb
|
86
|
+
- lib/async/job/processor/delayed.rb
|
87
|
+
- lib/async/job/processor/generic.rb
|
88
|
+
- lib/async/job/processor/inline.rb
|
89
|
+
- lib/async/job/processor/redis.rb
|
90
|
+
- lib/async/job/processor/redis/delayed_jobs.rb
|
91
|
+
- lib/async/job/processor/redis/job_store.rb
|
92
|
+
- lib/async/job/processor/redis/processing_list.rb
|
93
|
+
- lib/async/job/processor/redis/ready_list.rb
|
94
|
+
- lib/async/job/processor/redis/server.rb
|
95
|
+
- lib/async/job/queue.rb
|
95
96
|
- lib/async/job/version.rb
|
96
97
|
- license.md
|
97
98
|
- readme.md
|
@@ -116,7 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
116
117
|
- !ruby/object:Gem::Version
|
117
118
|
version: '0'
|
118
119
|
requirements: []
|
119
|
-
rubygems_version: 3.5.
|
120
|
+
rubygems_version: 3.5.11
|
120
121
|
signing_key:
|
121
122
|
specification_version: 4
|
122
123
|
summary: A asynchronous job queue for Ruby.
|
metadata.gz.sig
CHANGED
Binary file
|
@@ -1,85 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Released under the MIT License.
|
4
|
-
# Copyright, 2024, by Samuel Williams.
|
5
|
-
|
6
|
-
require 'console/event/failure'
|
7
|
-
|
8
|
-
module Async
|
9
|
-
module Job
|
10
|
-
module Backend
|
11
|
-
module Aggregate
|
12
|
-
class Server
|
13
|
-
def initialize(delegate, parent: nil)
|
14
|
-
@delegate = delegate
|
15
|
-
|
16
|
-
@task = nil
|
17
|
-
@ready = Async::Condition.new
|
18
|
-
|
19
|
-
@pending = Array.new
|
20
|
-
@processing = Array.new
|
21
|
-
end
|
22
|
-
|
23
|
-
def flush(jobs)
|
24
|
-
while job = jobs.shift
|
25
|
-
@delegate.call(job)
|
26
|
-
end
|
27
|
-
rescue => error
|
28
|
-
Console::Event::Failure.for(error).emit(self, "Could not flush #{jobs.size} jobs.")
|
29
|
-
end
|
30
|
-
|
31
|
-
def run(task)
|
32
|
-
while true
|
33
|
-
while @pending.empty?
|
34
|
-
@ready.wait
|
35
|
-
end
|
36
|
-
|
37
|
-
task.defer_stop do
|
38
|
-
# Swap the buffers:
|
39
|
-
@pending, @processing = @processing, @pending
|
40
|
-
|
41
|
-
flush(@processing)
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
# Start the background processing task if it is not already running.
|
47
|
-
#
|
48
|
-
# @return [Boolean] true if the task was started, false if it was already running.
|
49
|
-
protected def start!(parent: Async::Task.current)
|
50
|
-
return false if @task
|
51
|
-
|
52
|
-
# We are creating a task:
|
53
|
-
@task = true
|
54
|
-
|
55
|
-
parent.async(transient: true, annotation: self.class.name) do |task|
|
56
|
-
@task = task
|
57
|
-
|
58
|
-
run(task)
|
59
|
-
ensure
|
60
|
-
# Ensure that all jobs are flushed before we exit:
|
61
|
-
flush(@processing) if @processing.any?
|
62
|
-
flush(@pending) if @pending.any?
|
63
|
-
@task = nil
|
64
|
-
end
|
65
|
-
|
66
|
-
return true
|
67
|
-
end
|
68
|
-
|
69
|
-
# Enqueue a job into the pending buffer.
|
70
|
-
#
|
71
|
-
# Start the background processing task if it is not already running.
|
72
|
-
def call(job)
|
73
|
-
@pending << job
|
74
|
-
|
75
|
-
start! or @ready.signal
|
76
|
-
end
|
77
|
-
|
78
|
-
def stop
|
79
|
-
@task&.stop
|
80
|
-
end
|
81
|
-
end
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
85
|
-
end
|
@@ -1,18 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Released under the MIT License.
|
4
|
-
# Copyright, 2024, by Samuel Williams.
|
5
|
-
|
6
|
-
require_relative 'aggregate/server'
|
7
|
-
|
8
|
-
module Async
|
9
|
-
module Job
|
10
|
-
module Backend
|
11
|
-
module Aggregate
|
12
|
-
def self.new(delegate)
|
13
|
-
return Server.new(delegate)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
@@ -1,37 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Released under the MIT License.
|
4
|
-
# Copyright, 2024, by Samuel Williams.
|
5
|
-
|
6
|
-
require_relative '../../coder'
|
7
|
-
|
8
|
-
require 'async/idler'
|
9
|
-
|
10
|
-
module Async
|
11
|
-
module Job
|
12
|
-
module Backend
|
13
|
-
module Inline
|
14
|
-
class Server
|
15
|
-
def initialize(delegate, parent: nil)
|
16
|
-
@delegate = delegate
|
17
|
-
@parent = parent || Async::Idler.new
|
18
|
-
end
|
19
|
-
|
20
|
-
def call(job)
|
21
|
-
scheduled_at = Coder::Time(job["scheduled_at"])
|
22
|
-
|
23
|
-
@parent.async do
|
24
|
-
if scheduled_at
|
25
|
-
sleep(scheduled_at - Time.now)
|
26
|
-
end
|
27
|
-
|
28
|
-
@delegate.call(job)
|
29
|
-
rescue => error
|
30
|
-
Console.error(self, error)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
@@ -1,84 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Released under the MIT License.
|
4
|
-
# Copyright, 2024, by Samuel Williams.
|
5
|
-
|
6
|
-
require_relative 'delayed_queue'
|
7
|
-
require_relative 'job_store'
|
8
|
-
require_relative 'processing_queue'
|
9
|
-
require_relative 'ready_queue'
|
10
|
-
require_relative '../../coder'
|
11
|
-
|
12
|
-
require 'securerandom'
|
13
|
-
require 'async/idler'
|
14
|
-
|
15
|
-
module Async
|
16
|
-
module Job
|
17
|
-
module Backend
|
18
|
-
module Redis
|
19
|
-
class Server
|
20
|
-
def initialize(delegate, client, prefix: 'async-job', coder: Coder::DEFAULT, resolution: 10, parent: nil)
|
21
|
-
@delegate = delegate
|
22
|
-
|
23
|
-
@id = SecureRandom.uuid
|
24
|
-
@client = client
|
25
|
-
@prefix = prefix
|
26
|
-
@coder = coder
|
27
|
-
@resolution = resolution
|
28
|
-
|
29
|
-
@job_store = JobStore.new(@client, "#{@prefix}:jobs")
|
30
|
-
|
31
|
-
@delayed_queue = DelayedQueue.new(@client, "#{@prefix}:delayed")
|
32
|
-
@ready_queue = ReadyQueue.new(@client, "#{@prefix}:ready")
|
33
|
-
|
34
|
-
@processing_queue = ProcessingQueue.new(@client, "#{@prefix}:processing", @id, @ready_queue, @job_store)
|
35
|
-
|
36
|
-
@parent = parent || Async::Idler.new
|
37
|
-
end
|
38
|
-
|
39
|
-
def start
|
40
|
-
Console.info(self, "Starting server...")
|
41
|
-
# Start the delayed queue, which will move jobs to the ready queue when they are ready:
|
42
|
-
@delayed_queue.start(@ready_queue, resolution: @resolution)
|
43
|
-
|
44
|
-
# Start the processing queue, which will move jobs to the ready queue when they are abandoned:
|
45
|
-
@processing_queue.start
|
46
|
-
|
47
|
-
while true
|
48
|
-
self.dequeue(@parent)
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
def call(job)
|
53
|
-
scheduled_at = Coder::Time(job["scheduled_at"])
|
54
|
-
|
55
|
-
if scheduled_at
|
56
|
-
@delayed_queue.add(@coder.dump(job), scheduled_at, @job_store)
|
57
|
-
else
|
58
|
-
@ready_queue.add(@coder.dump(job), @job_store)
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
protected
|
63
|
-
|
64
|
-
def dequeue(parent)
|
65
|
-
_id = @processing_queue.fetch
|
66
|
-
|
67
|
-
parent.async do
|
68
|
-
id = _id; _id = nil
|
69
|
-
|
70
|
-
job = @coder.load(@job_store.get(id))
|
71
|
-
@delegate.call(job)
|
72
|
-
@processing_queue.complete(id)
|
73
|
-
rescue => error
|
74
|
-
@processing_queue.retry(id)
|
75
|
-
Console.error(self, error)
|
76
|
-
end
|
77
|
-
ensure
|
78
|
-
@processing_queue.retry(_id) if _id
|
79
|
-
end
|
80
|
-
end
|
81
|
-
end
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
data/lib/async/job/backend.rb
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Released under the MIT License.
|
4
|
-
# Copyright, 2024, by Samuel Williams.
|
5
|
-
|
6
|
-
require_relative 'backend/redis'
|
7
|
-
|
8
|
-
module Async
|
9
|
-
module Job
|
10
|
-
module Backend
|
11
|
-
def self.new(backend: Async::Job::Backend::Redis, **options)
|
12
|
-
backend.new(**options)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|