async-job 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/lib/async/job/builder.rb +18 -28
- data/lib/async/job/processor/aggregate.rb +93 -0
- data/lib/async/job/processor/delayed.rb +29 -0
- data/lib/async/job/processor/generic.rb +28 -0
- data/lib/async/job/processor/inline.rb +45 -0
- data/lib/async/job/{backend/redis/delayed_queue.rb → processor/redis/delayed_jobs.rb} +5 -7
- data/lib/async/job/{backend → processor}/redis/job_store.rb +1 -1
- data/lib/async/job/{backend/redis/processing_queue.rb → processor/redis/processing_list.rb} +8 -10
- data/lib/async/job/{backend/redis/ready_queue.rb → processor/redis/ready_list.rb} +2 -2
- data/lib/async/job/{backend → processor}/redis/server.rb +23 -23
- data/lib/async/job/{backend → processor}/redis.rb +1 -1
- data/lib/async/job/{backend/inline.rb → processor.rb} +4 -6
- data/lib/async/job/queue.rb +32 -0
- data/lib/async/job/version.rb +1 -1
- data/lib/async/job.rb +1 -1
- data/readme.md +1 -1
- data.tar.gz.sig +0 -0
- metadata +13 -12
- metadata.gz.sig +0 -0
- data/lib/async/job/backend/aggregate/server.rb +0 -91
- data/lib/async/job/backend/aggregate.rb +0 -18
- data/lib/async/job/backend/inline/server.rb +0 -45
- data/lib/async/job/backend.rb +0 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7381f7520a5ee41df3c93fc4f55c647952f10066868fcb22a196fb6ee807a1dd
|
4
|
+
data.tar.gz: a063193f2ff8a71fe798aadc3ce07d392fd2c53cadd5523cb20813fff9da2bdc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 56f1a056d2cbc2dab6a97226c07b6d2acec56de40bd95c302b652e4689c3fa08986a7b6f4e4c53a0ca15555c61695da6ef2219f8585d1c1006c01cacfffac50b
|
7
|
+
data.tar.gz: b0c62cf9b609beee163c948ff9fc933a1d82b3f0f611e555ddc4dadd4909ab3e0791e474ae7cdf35cb1081d12d38a5f73f37148945e2658bec07af31388b8480
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/lib/async/job/builder.rb
CHANGED
@@ -3,11 +3,11 @@
|
|
3
3
|
# Released under the MIT License.
|
4
4
|
# Copyright, 2024, by Samuel Williams.
|
5
5
|
|
6
|
+
require_relative 'queue'
|
7
|
+
|
6
8
|
module Async
|
7
9
|
module Job
|
8
10
|
class Builder
|
9
|
-
Pipeline = Struct.new(:producer, :consumer, :delegate)
|
10
|
-
|
11
11
|
def self.build(delegate = nil, &block)
|
12
12
|
builder = self.new(delegate)
|
13
13
|
|
@@ -16,58 +16,48 @@ module Async
|
|
16
16
|
return builder.build
|
17
17
|
end
|
18
18
|
|
19
|
+
# @parameter delegate [Object] The initial delegate that will be wrapped by the queue.
|
19
20
|
def initialize(delegate = nil)
|
21
|
+
# The client side middleware, in the order they should be applied to a job:
|
20
22
|
@enqueue = []
|
23
|
+
|
24
|
+
# The server side middleware, in the order they should be applied to a job:
|
21
25
|
@dequeue = []
|
22
|
-
@delegate = delegate
|
23
26
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
def enqueue(middleware)
|
28
|
-
@enqueue << middleware
|
27
|
+
# The output delegate, if any:
|
28
|
+
@delegate = delegate
|
29
29
|
end
|
30
30
|
|
31
|
-
def
|
32
|
-
|
33
|
-
# The queue itself is instantiated with the delegate.
|
34
|
-
@queue = ->(delegate){queue.new(delegate, *arguments, **options)}
|
31
|
+
def enqueue(middleware, ...)
|
32
|
+
@enqueue << ->(delegate){middleware.new(delegate, ...)}
|
35
33
|
end
|
36
34
|
|
37
|
-
def dequeue(middleware)
|
38
|
-
@dequeue << middleware
|
35
|
+
def dequeue(middleware, ...)
|
36
|
+
@dequeue << ->(delegate){middleware.new(delegate, ...)}
|
39
37
|
end
|
40
38
|
|
41
39
|
def delegate(delegate)
|
42
40
|
@delegate = delegate
|
43
41
|
end
|
44
42
|
|
45
|
-
def build(&block)
|
46
|
-
# To construct the queue, we need the delegate.
|
47
|
-
delegate = @delegate
|
48
|
-
|
43
|
+
def build(delegate = @delegate, &block)
|
49
44
|
# We then wrap the delegate with the middleware in reverse order:
|
50
45
|
@dequeue.reverse_each do |middleware|
|
51
|
-
delegate = middleware.
|
46
|
+
delegate = middleware.call(delegate)
|
52
47
|
end
|
53
48
|
|
54
|
-
|
55
|
-
if @queue
|
56
|
-
producer = consumer = @queue.call(delegate)
|
57
|
-
else
|
58
|
-
producer = consumer = delegate
|
59
|
-
end
|
49
|
+
client = server = delegate
|
60
50
|
|
61
51
|
# We now construct the queue producer:
|
62
52
|
@enqueue.reverse_each do |middleware|
|
63
|
-
|
53
|
+
client = middleware.call(client)
|
64
54
|
end
|
65
55
|
|
66
56
|
if block_given?
|
67
|
-
|
57
|
+
client = yield(client) || client
|
68
58
|
end
|
69
59
|
|
70
|
-
return
|
60
|
+
return Queue.new(client, server, @delegate)
|
71
61
|
end
|
72
62
|
end
|
73
63
|
end
|
@@ -0,0 +1,93 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
require_relative 'generic'
|
7
|
+
|
8
|
+
require 'console/event/failure'
|
9
|
+
|
10
|
+
module Async
|
11
|
+
module Job
|
12
|
+
module Processor
|
13
|
+
class Aggregate < Generic
|
14
|
+
def initialize(delegate, parent: nil)
|
15
|
+
super(delegate)
|
16
|
+
|
17
|
+
@task = nil
|
18
|
+
@ready = Async::Condition.new
|
19
|
+
|
20
|
+
@pending = Array.new
|
21
|
+
@processing = Array.new
|
22
|
+
end
|
23
|
+
|
24
|
+
def flush(jobs)
|
25
|
+
while job = jobs.shift
|
26
|
+
@delegate.call(job)
|
27
|
+
end
|
28
|
+
rescue => error
|
29
|
+
Console::Event::Failure.for(error).emit(self, "Could not flush #{jobs.size} jobs.")
|
30
|
+
end
|
31
|
+
|
32
|
+
def run(task)
|
33
|
+
while true
|
34
|
+
while @pending.empty?
|
35
|
+
@ready.wait
|
36
|
+
end
|
37
|
+
|
38
|
+
task.defer_stop do
|
39
|
+
# Swap the buffers:
|
40
|
+
@pending, @processing = @processing, @pending
|
41
|
+
|
42
|
+
flush(@processing)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# Start the background processing task if it is not already running.
|
48
|
+
#
|
49
|
+
# @return [Boolean] true if the task was started, false if it was already running.
|
50
|
+
protected def start!(parent: Async::Task.current)
|
51
|
+
return false if @task
|
52
|
+
|
53
|
+
# We are creating a task:
|
54
|
+
@task = true
|
55
|
+
|
56
|
+
parent.async(transient: true, annotation: self.class.name) do |task|
|
57
|
+
@task = task
|
58
|
+
|
59
|
+
run(task)
|
60
|
+
ensure
|
61
|
+
# Ensure that all jobs are flushed before we exit:
|
62
|
+
flush(@processing) if @processing.any?
|
63
|
+
flush(@pending) if @pending.any?
|
64
|
+
@task = nil
|
65
|
+
end
|
66
|
+
|
67
|
+
return true
|
68
|
+
end
|
69
|
+
|
70
|
+
# Enqueue a job into the pending buffer.
|
71
|
+
#
|
72
|
+
# Start the background processing task if it is not already running.
|
73
|
+
def call(job)
|
74
|
+
@pending << job
|
75
|
+
|
76
|
+
start! or @ready.signal
|
77
|
+
end
|
78
|
+
|
79
|
+
def start
|
80
|
+
super
|
81
|
+
|
82
|
+
self.start!
|
83
|
+
end
|
84
|
+
|
85
|
+
def stop
|
86
|
+
@task&.stop
|
87
|
+
|
88
|
+
super
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
require_relative 'generic'
|
7
|
+
|
8
|
+
require 'console/event/failure'
|
9
|
+
|
10
|
+
module Async
|
11
|
+
module Job
|
12
|
+
module Processor
|
13
|
+
# Add a small processing delay to each job.
|
14
|
+
class Delayed < Generic
|
15
|
+
def initialize(delegate, delay: 0.1)
|
16
|
+
super(delegate)
|
17
|
+
|
18
|
+
@delay = delay
|
19
|
+
end
|
20
|
+
|
21
|
+
def call(job)
|
22
|
+
sleep(@delay)
|
23
|
+
|
24
|
+
super
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
module Async
|
7
|
+
module Job
|
8
|
+
module Processor
|
9
|
+
class Generic
|
10
|
+
def initialize(delegate = nil)
|
11
|
+
@delegate = delegate
|
12
|
+
end
|
13
|
+
|
14
|
+
def call(job)
|
15
|
+
@delegate.call(job)
|
16
|
+
end
|
17
|
+
|
18
|
+
def start
|
19
|
+
@delegate.start
|
20
|
+
end
|
21
|
+
|
22
|
+
def stop
|
23
|
+
@delegate.stop
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
require_relative '../coder'
|
7
|
+
require_relative 'generic'
|
8
|
+
|
9
|
+
require 'async/idler'
|
10
|
+
|
11
|
+
module Async
|
12
|
+
module Job
|
13
|
+
module Processor
|
14
|
+
class Inline < Generic
|
15
|
+
def initialize(delegate, parent: nil)
|
16
|
+
super(delegate)
|
17
|
+
|
18
|
+
@parent = parent || Async::Idler.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def call(job)
|
22
|
+
scheduled_at = Coder::Time(job["scheduled_at"])
|
23
|
+
|
24
|
+
@parent.async do
|
25
|
+
if scheduled_at
|
26
|
+
sleep(scheduled_at - Time.now)
|
27
|
+
end
|
28
|
+
|
29
|
+
@delegate.call(job)
|
30
|
+
rescue => error
|
31
|
+
Console.error(self, error)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def start
|
36
|
+
@delegate&.start
|
37
|
+
end
|
38
|
+
|
39
|
+
def stop
|
40
|
+
@delegate&.stop
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -5,9 +5,9 @@
|
|
5
5
|
|
6
6
|
module Async
|
7
7
|
module Job
|
8
|
-
module
|
8
|
+
module Processor
|
9
9
|
module Redis
|
10
|
-
class
|
10
|
+
class DelayedJobs
|
11
11
|
ADD = <<~LUA
|
12
12
|
redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
|
13
13
|
redis.call('ZADD', KEYS[2], ARGV[3], ARGV[1])
|
@@ -30,15 +30,13 @@ module Async
|
|
30
30
|
@move = @client.script(:load, MOVE)
|
31
31
|
end
|
32
32
|
|
33
|
-
def start(
|
34
|
-
Console.info(self, "Starting delayed queue...")
|
33
|
+
def start(ready_list, resolution: 10, parent: Async::Task.current)
|
35
34
|
parent.async do
|
36
35
|
while true
|
37
|
-
|
38
|
-
count = move(destination: ready_queue.key)
|
36
|
+
count = move(destination: ready_list.key)
|
39
37
|
|
40
38
|
if count > 0
|
41
|
-
Console.
|
39
|
+
Console.debug(self, "Moved #{count} delayed jobs to ready list.")
|
42
40
|
end
|
43
41
|
|
44
42
|
sleep(resolution)
|
@@ -5,9 +5,9 @@
|
|
5
5
|
|
6
6
|
module Async
|
7
7
|
module Job
|
8
|
-
module
|
8
|
+
module Processor
|
9
9
|
module Redis
|
10
|
-
class
|
10
|
+
class ProcessingList
|
11
11
|
REQUEUE = <<~LUA
|
12
12
|
local cursor = "0"
|
13
13
|
local count = 0
|
@@ -50,12 +50,12 @@ module Async
|
|
50
50
|
redis.call('HDEL', KEYS[2], ARGV[1])
|
51
51
|
LUA
|
52
52
|
|
53
|
-
def initialize(client, key, id,
|
53
|
+
def initialize(client, key, id, ready_list, job_store)
|
54
54
|
@client = client
|
55
55
|
@key = key
|
56
56
|
@id = id
|
57
57
|
|
58
|
-
@
|
58
|
+
@ready_list = ready_list
|
59
59
|
@job_store = job_store
|
60
60
|
|
61
61
|
@pending_key = "#{@key}:#{@id}:pending"
|
@@ -69,7 +69,7 @@ module Async
|
|
69
69
|
attr :key
|
70
70
|
|
71
71
|
def fetch
|
72
|
-
@client.brpoplpush(@
|
72
|
+
@client.brpoplpush(@ready_list.key, @pending_key, 0)
|
73
73
|
end
|
74
74
|
|
75
75
|
def complete(id)
|
@@ -79,22 +79,20 @@ module Async
|
|
79
79
|
|
80
80
|
def retry(id)
|
81
81
|
Console.warn(self, "Retrying job: #{id}")
|
82
|
-
@client.evalsha(@retry, 2, @pending_key, @
|
82
|
+
@client.evalsha(@retry, 2, @pending_key, @ready_list.key, id)
|
83
83
|
end
|
84
84
|
|
85
85
|
def start(delay: 5, factor: 2, parent: Async::Task.current)
|
86
86
|
heartbeat_key = "#{@key}:#{@id}"
|
87
87
|
start_time = Time.now.to_f
|
88
88
|
|
89
|
-
Console.info(self, "Starting processing queue...", key: @key, id: @id, heartbeat_key: heartbeat_key, delay: delay, factor: factor)
|
90
|
-
|
91
89
|
parent.async do
|
92
90
|
while true
|
93
91
|
uptime = (Time.now.to_f - start_time).round(2)
|
94
92
|
@client.set(heartbeat_key, JSON.dump(uptime: uptime), seconds: delay*factor)
|
95
93
|
|
96
|
-
#
|
97
|
-
count = @client.evalsha(@requeue, 2, @key, @
|
94
|
+
# REQUEUE any jobs that have been abandoned:
|
95
|
+
count = @client.evalsha(@requeue, 2, @key, @ready_list.key)
|
98
96
|
if count > 0
|
99
97
|
Console.warn(self, "Requeued #{count} abandoned jobs.")
|
100
98
|
end
|
@@ -3,22 +3,23 @@
|
|
3
3
|
# Released under the MIT License.
|
4
4
|
# Copyright, 2024, by Samuel Williams.
|
5
5
|
|
6
|
-
require_relative '
|
6
|
+
require_relative 'delayed_jobs'
|
7
7
|
require_relative 'job_store'
|
8
|
-
require_relative '
|
9
|
-
require_relative '
|
8
|
+
require_relative 'processing_list'
|
9
|
+
require_relative 'ready_list'
|
10
10
|
require_relative '../../coder'
|
11
|
+
require_relative '../generic'
|
11
12
|
|
12
13
|
require 'securerandom'
|
13
14
|
require 'async/idler'
|
14
15
|
|
15
16
|
module Async
|
16
17
|
module Job
|
17
|
-
module
|
18
|
+
module Processor
|
18
19
|
module Redis
|
19
|
-
class Server
|
20
|
+
class Server < Generic
|
20
21
|
def initialize(delegate, client, prefix: 'async-job', coder: Coder::DEFAULT, resolution: 10, parent: nil)
|
21
|
-
|
22
|
+
super(delegate)
|
22
23
|
|
23
24
|
@id = SecureRandom.uuid
|
24
25
|
@client = client
|
@@ -27,11 +28,9 @@ module Async
|
|
27
28
|
@resolution = resolution
|
28
29
|
|
29
30
|
@job_store = JobStore.new(@client, "#{@prefix}:jobs")
|
30
|
-
|
31
|
-
@
|
32
|
-
@
|
33
|
-
|
34
|
-
@processing_queue = ProcessingQueue.new(@client, "#{@prefix}:processing", @id, @ready_queue, @job_store)
|
31
|
+
@delayed_jobs = DelayedJobs.new(@client, "#{@prefix}:delayed")
|
32
|
+
@ready_list = ReadyList.new(@client, "#{@prefix}:ready")
|
33
|
+
@processing_list = ProcessingList.new(@client, "#{@prefix}:processing", @id, @ready_list, @job_store)
|
35
34
|
|
36
35
|
@parent = parent || Async::Idler.new
|
37
36
|
end
|
@@ -53,49 +52,50 @@ module Async
|
|
53
52
|
end
|
54
53
|
|
55
54
|
def start
|
56
|
-
|
55
|
+
super
|
57
56
|
|
58
|
-
# Start the delayed
|
59
|
-
@
|
57
|
+
# Start the delayed processor, which will move jobs to the ready processor when they are ready:
|
58
|
+
@delayed_jobs.start(@ready_list, resolution: @resolution)
|
60
59
|
|
61
|
-
# Start the processing
|
62
|
-
@
|
60
|
+
# Start the processing processor, which will move jobs to the ready processor when they are abandoned:
|
61
|
+
@processing_list.start
|
63
62
|
|
64
63
|
self.start!
|
65
64
|
end
|
66
65
|
|
67
66
|
def stop
|
68
67
|
@task&.stop
|
69
|
-
|
68
|
+
|
69
|
+
super
|
70
70
|
end
|
71
71
|
|
72
72
|
def call(job)
|
73
73
|
scheduled_at = Coder::Time(job["scheduled_at"])
|
74
74
|
|
75
75
|
if scheduled_at
|
76
|
-
@
|
76
|
+
@delayed_jobs.add(@coder.dump(job), scheduled_at, @job_store)
|
77
77
|
else
|
78
|
-
@
|
78
|
+
@ready_list.add(@coder.dump(job), @job_store)
|
79
79
|
end
|
80
80
|
end
|
81
81
|
|
82
82
|
protected
|
83
83
|
|
84
84
|
def dequeue(parent)
|
85
|
-
_id = @
|
85
|
+
_id = @processing_list.fetch
|
86
86
|
|
87
87
|
parent.async do
|
88
88
|
id = _id; _id = nil
|
89
89
|
|
90
90
|
job = @coder.load(@job_store.get(id))
|
91
91
|
@delegate.call(job)
|
92
|
-
@
|
92
|
+
@processing_list.complete(id)
|
93
93
|
rescue => error
|
94
|
-
@
|
94
|
+
@processing_list.retry(id)
|
95
95
|
Console.error(self, error)
|
96
96
|
end
|
97
97
|
ensure
|
98
|
-
@
|
98
|
+
@processing_list.retry(_id) if _id
|
99
99
|
end
|
100
100
|
end
|
101
101
|
end
|
@@ -3,15 +3,13 @@
|
|
3
3
|
# Released under the MIT License.
|
4
4
|
# Copyright, 2024, by Samuel Williams.
|
5
5
|
|
6
|
-
require_relative 'inline
|
6
|
+
require_relative 'processor/inline'
|
7
7
|
|
8
8
|
module Async
|
9
9
|
module Job
|
10
|
-
module
|
11
|
-
|
12
|
-
|
13
|
-
return Server.new(delegate)
|
14
|
-
end
|
10
|
+
module Processor
|
11
|
+
def self.new(processor: Inline, **options)
|
12
|
+
processor.new(**options)
|
15
13
|
end
|
16
14
|
end
|
17
15
|
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Released under the MIT License.
|
4
|
+
# Copyright, 2024, by Samuel Williams.
|
5
|
+
|
6
|
+
module Async
|
7
|
+
module Job
|
8
|
+
class Queue
|
9
|
+
def initialize(client, server, delegate)
|
10
|
+
@client = client
|
11
|
+
@server = server
|
12
|
+
@delegate = delegate
|
13
|
+
end
|
14
|
+
|
15
|
+
attr :client
|
16
|
+
attr :server
|
17
|
+
attr :delegate
|
18
|
+
|
19
|
+
def call(...)
|
20
|
+
@client.call(...)
|
21
|
+
end
|
22
|
+
|
23
|
+
def start
|
24
|
+
@server.start
|
25
|
+
end
|
26
|
+
|
27
|
+
def stop
|
28
|
+
@server.stop
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
data/lib/async/job/version.rb
CHANGED
data/lib/async/job.rb
CHANGED
data/readme.md
CHANGED
@@ -10,7 +10,7 @@ Please see the [project documentation](https://socketry.github.io/async-job/) fo
|
|
10
10
|
|
11
11
|
- [Getting Started](https://socketry.github.io/async-job/guides/getting-started/index) - This guide gives you an overview of the `async-job` gem and explains the core concepts.
|
12
12
|
|
13
|
-
- [Redis
|
13
|
+
- [Redis Processor](https://socketry.github.io/async-job/guides/redis-processor/index) - This guide gives a brief overview of the implementation of the Redis queue.
|
14
14
|
|
15
15
|
## See Also
|
16
16
|
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: async-job
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Samuel Williams
|
@@ -75,23 +75,24 @@ extensions: []
|
|
75
75
|
extra_rdoc_files: []
|
76
76
|
files:
|
77
77
|
- lib/async/job.rb
|
78
|
-
- lib/async/job/backend.rb
|
79
|
-
- lib/async/job/backend/aggregate.rb
|
80
|
-
- lib/async/job/backend/aggregate/server.rb
|
81
|
-
- lib/async/job/backend/inline.rb
|
82
|
-
- lib/async/job/backend/inline/server.rb
|
83
|
-
- lib/async/job/backend/redis.rb
|
84
|
-
- lib/async/job/backend/redis/delayed_queue.rb
|
85
|
-
- lib/async/job/backend/redis/job_store.rb
|
86
|
-
- lib/async/job/backend/redis/processing_queue.rb
|
87
|
-
- lib/async/job/backend/redis/ready_queue.rb
|
88
|
-
- lib/async/job/backend/redis/server.rb
|
89
78
|
- lib/async/job/buffer.rb
|
90
79
|
- lib/async/job/builder.rb
|
91
80
|
- lib/async/job/coder.rb
|
92
81
|
- lib/async/job/coder/marshal.rb
|
93
82
|
- lib/async/job/coder/message_pack.rb
|
94
83
|
- lib/async/job/generic.rb
|
84
|
+
- lib/async/job/processor.rb
|
85
|
+
- lib/async/job/processor/aggregate.rb
|
86
|
+
- lib/async/job/processor/delayed.rb
|
87
|
+
- lib/async/job/processor/generic.rb
|
88
|
+
- lib/async/job/processor/inline.rb
|
89
|
+
- lib/async/job/processor/redis.rb
|
90
|
+
- lib/async/job/processor/redis/delayed_jobs.rb
|
91
|
+
- lib/async/job/processor/redis/job_store.rb
|
92
|
+
- lib/async/job/processor/redis/processing_list.rb
|
93
|
+
- lib/async/job/processor/redis/ready_list.rb
|
94
|
+
- lib/async/job/processor/redis/server.rb
|
95
|
+
- lib/async/job/queue.rb
|
95
96
|
- lib/async/job/version.rb
|
96
97
|
- license.md
|
97
98
|
- readme.md
|
metadata.gz.sig
CHANGED
Binary file
|
@@ -1,91 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Released under the MIT License.
|
4
|
-
# Copyright, 2024, by Samuel Williams.
|
5
|
-
|
6
|
-
require 'console/event/failure'
|
7
|
-
|
8
|
-
module Async
|
9
|
-
module Job
|
10
|
-
module Backend
|
11
|
-
module Aggregate
|
12
|
-
class Server
|
13
|
-
def initialize(delegate, parent: nil)
|
14
|
-
@delegate = delegate
|
15
|
-
|
16
|
-
@task = nil
|
17
|
-
@ready = Async::Condition.new
|
18
|
-
|
19
|
-
@pending = Array.new
|
20
|
-
@processing = Array.new
|
21
|
-
end
|
22
|
-
|
23
|
-
def flush(jobs)
|
24
|
-
while job = jobs.shift
|
25
|
-
@delegate.call(job)
|
26
|
-
end
|
27
|
-
rescue => error
|
28
|
-
Console::Event::Failure.for(error).emit(self, "Could not flush #{jobs.size} jobs.")
|
29
|
-
end
|
30
|
-
|
31
|
-
def run(task)
|
32
|
-
while true
|
33
|
-
while @pending.empty?
|
34
|
-
@ready.wait
|
35
|
-
end
|
36
|
-
|
37
|
-
task.defer_stop do
|
38
|
-
# Swap the buffers:
|
39
|
-
@pending, @processing = @processing, @pending
|
40
|
-
|
41
|
-
flush(@processing)
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
# Start the background processing task if it is not already running.
|
47
|
-
#
|
48
|
-
# @return [Boolean] true if the task was started, false if it was already running.
|
49
|
-
protected def start!(parent: Async::Task.current)
|
50
|
-
return false if @task
|
51
|
-
|
52
|
-
# We are creating a task:
|
53
|
-
@task = true
|
54
|
-
|
55
|
-
parent.async(transient: true, annotation: self.class.name) do |task|
|
56
|
-
@task = task
|
57
|
-
|
58
|
-
run(task)
|
59
|
-
ensure
|
60
|
-
# Ensure that all jobs are flushed before we exit:
|
61
|
-
flush(@processing) if @processing.any?
|
62
|
-
flush(@pending) if @pending.any?
|
63
|
-
@task = nil
|
64
|
-
end
|
65
|
-
|
66
|
-
return true
|
67
|
-
end
|
68
|
-
|
69
|
-
# Enqueue a job into the pending buffer.
|
70
|
-
#
|
71
|
-
# Start the background processing task if it is not already running.
|
72
|
-
def call(job)
|
73
|
-
@pending << job
|
74
|
-
|
75
|
-
start! or @ready.signal
|
76
|
-
end
|
77
|
-
|
78
|
-
def start
|
79
|
-
@delegate&.start
|
80
|
-
self.start!
|
81
|
-
end
|
82
|
-
|
83
|
-
def stop
|
84
|
-
@task&.stop
|
85
|
-
@delegate&.stop
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
@@ -1,18 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Released under the MIT License.
|
4
|
-
# Copyright, 2024, by Samuel Williams.
|
5
|
-
|
6
|
-
require_relative 'aggregate/server'
|
7
|
-
|
8
|
-
module Async
|
9
|
-
module Job
|
10
|
-
module Backend
|
11
|
-
module Aggregate
|
12
|
-
def self.new(delegate)
|
13
|
-
return Server.new(delegate)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
@@ -1,45 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Released under the MIT License.
|
4
|
-
# Copyright, 2024, by Samuel Williams.
|
5
|
-
|
6
|
-
require_relative '../../coder'
|
7
|
-
|
8
|
-
require 'async/idler'
|
9
|
-
|
10
|
-
module Async
|
11
|
-
module Job
|
12
|
-
module Backend
|
13
|
-
module Inline
|
14
|
-
class Server
|
15
|
-
def initialize(delegate, parent: nil)
|
16
|
-
@delegate = delegate
|
17
|
-
@parent = parent || Async::Idler.new
|
18
|
-
end
|
19
|
-
|
20
|
-
def call(job)
|
21
|
-
scheduled_at = Coder::Time(job["scheduled_at"])
|
22
|
-
|
23
|
-
@parent.async do
|
24
|
-
if scheduled_at
|
25
|
-
sleep(scheduled_at - Time.now)
|
26
|
-
end
|
27
|
-
|
28
|
-
@delegate.call(job)
|
29
|
-
rescue => error
|
30
|
-
Console.error(self, error)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
def start
|
35
|
-
@delegate&.start
|
36
|
-
end
|
37
|
-
|
38
|
-
def stop
|
39
|
-
@delegate&.stop
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
data/lib/async/job/backend.rb
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Released under the MIT License.
|
4
|
-
# Copyright, 2024, by Samuel Williams.
|
5
|
-
|
6
|
-
require_relative 'backend/redis'
|
7
|
-
|
8
|
-
module Async
|
9
|
-
module Job
|
10
|
-
module Backend
|
11
|
-
def self.new(backend: Async::Job::Backend::Redis, **options)
|
12
|
-
backend.new(**options)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|