async-job 0.7.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5168ca3167c401c68ffca433ef65962420641cd4826e1d1fb9e4700bf5c9a5c8
4
- data.tar.gz: da6da3de6436cc9c1ade6523610033c683bafff75aa84640b12826b7d95cacfe
3
+ metadata.gz: 7381f7520a5ee41df3c93fc4f55c647952f10066868fcb22a196fb6ee807a1dd
4
+ data.tar.gz: a063193f2ff8a71fe798aadc3ce07d392fd2c53cadd5523cb20813fff9da2bdc
5
5
  SHA512:
6
- metadata.gz: 3fb44c033b61704b4b5cb2d0b6ad2351d71c35d2f63c96efca439478c75e404e80e31caa91fe291d07e537bb3bd47d1114153071d49f78c7a33fc14e6ab6cab2
7
- data.tar.gz: 8371e91c27a9701cf34dacf575596c9af03dbbaa188837df5a6e90d9110f80d54056cd757b399a0e715f2a6268e128c7ee346dd2634fae40bd83f031f3bf7ad5
6
+ metadata.gz: 56f1a056d2cbc2dab6a97226c07b6d2acec56de40bd95c302b652e4689c3fa08986a7b6f4e4c53a0ca15555c61695da6ef2219f8585d1c1006c01cacfffac50b
7
+ data.tar.gz: b0c62cf9b609beee163c948ff9fc933a1d82b3f0f611e555ddc4dadd4909ab3e0791e474ae7cdf35cb1081d12d38a5f73f37148945e2658bec07af31388b8480
checksums.yaml.gz.sig CHANGED
Binary file
@@ -27,6 +27,14 @@ module Async
27
27
  def pop
28
28
  @jobs.dequeue
29
29
  end
30
+
31
+ def start
32
+ @delegate&.start
33
+ end
34
+
35
+ def stop
36
+ @delegate&.stop
37
+ end
30
38
  end
31
39
  end
32
40
  end
@@ -3,11 +3,11 @@
3
3
  # Released under the MIT License.
4
4
  # Copyright, 2024, by Samuel Williams.
5
5
 
6
+ require_relative 'queue'
7
+
6
8
  module Async
7
9
  module Job
8
10
  class Builder
9
- Pipeline = Struct.new(:producer, :consumer, :delegate)
10
-
11
11
  def self.build(delegate = nil, &block)
12
12
  builder = self.new(delegate)
13
13
 
@@ -16,58 +16,48 @@ module Async
16
16
  return builder.build
17
17
  end
18
18
 
19
+ # @parameter delegate [Object] The initial delegate that will be wrapped by the queue.
19
20
  def initialize(delegate = nil)
21
+ # The client side middleware, in the order they should be applied to a job:
20
22
  @enqueue = []
23
+
24
+ # The server side middleware, in the order they should be applied to a job:
21
25
  @dequeue = []
22
- @delegate = delegate
23
26
 
24
- @queue = nil
25
- end
26
-
27
- def enqueue(middleware)
28
- @enqueue << middleware
27
+ # The output delegate, if any:
28
+ @delegate = delegate
29
29
  end
30
30
 
31
- def queue(queue, *arguments, **options)
32
- # The delegate is the output side of the queue, e.g. a Redis server delegate or similar wrapper.
33
- # The queue itself is instantiated with the delegate.
34
- @queue = ->(delegate){queue.new(delegate, *arguments, **options)}
31
+ def enqueue(middleware, ...)
32
+ @enqueue << ->(delegate){middleware.new(delegate, ...)}
35
33
  end
36
34
 
37
- def dequeue(middleware)
38
- @dequeue << middleware
35
+ def dequeue(middleware, ...)
36
+ @dequeue << ->(delegate){middleware.new(delegate, ...)}
39
37
  end
40
38
 
41
39
  def delegate(delegate)
42
40
  @delegate = delegate
43
41
  end
44
42
 
45
- def build(&block)
46
- # To construct the queue, we need the delegate.
47
- delegate = @delegate
48
-
43
+ def build(delegate = @delegate, &block)
49
44
  # We then wrap the delegate with the middleware in reverse order:
50
45
  @dequeue.reverse_each do |middleware|
51
- delegate = middleware.new(delegate)
46
+ delegate = middleware.call(delegate)
52
47
  end
53
48
 
54
- # We can now construct the queue with the delegate:
55
- if @queue
56
- producer = consumer = @queue.call(delegate)
57
- else
58
- producer = consumer = delegate
59
- end
49
+ client = server = delegate
60
50
 
61
51
  # We now construct the queue producer:
62
52
  @enqueue.reverse_each do |middleware|
63
- producer = middleware.new(producer)
53
+ client = middleware.call(client)
64
54
  end
65
55
 
66
56
  if block_given?
67
- producer = yield(producer) || producer
57
+ client = yield(client) || client
68
58
  end
69
59
 
70
- return Pipeline.new(producer, consumer, @delegate)
60
+ return Queue.new(client, server, @delegate)
71
61
  end
72
62
  end
73
63
  end
@@ -0,0 +1,93 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Released under the MIT License.
4
+ # Copyright, 2024, by Samuel Williams.
5
+
6
+ require_relative 'generic'
7
+
8
+ require 'console/event/failure'
9
+
10
+ module Async
11
+ module Job
12
+ module Processor
13
+ class Aggregate < Generic
14
+ def initialize(delegate, parent: nil)
15
+ super(delegate)
16
+
17
+ @task = nil
18
+ @ready = Async::Condition.new
19
+
20
+ @pending = Array.new
21
+ @processing = Array.new
22
+ end
23
+
24
+ def flush(jobs)
25
+ while job = jobs.shift
26
+ @delegate.call(job)
27
+ end
28
+ rescue => error
29
+ Console::Event::Failure.for(error).emit(self, "Could not flush #{jobs.size} jobs.")
30
+ end
31
+
32
+ def run(task)
33
+ while true
34
+ while @pending.empty?
35
+ @ready.wait
36
+ end
37
+
38
+ task.defer_stop do
39
+ # Swap the buffers:
40
+ @pending, @processing = @processing, @pending
41
+
42
+ flush(@processing)
43
+ end
44
+ end
45
+ end
46
+
47
+ # Start the background processing task if it is not already running.
48
+ #
49
+ # @return [Boolean] true if the task was started, false if it was already running.
50
+ protected def start!(parent: Async::Task.current)
51
+ return false if @task
52
+
53
+ # We are creating a task:
54
+ @task = true
55
+
56
+ parent.async(transient: true, annotation: self.class.name) do |task|
57
+ @task = task
58
+
59
+ run(task)
60
+ ensure
61
+ # Ensure that all jobs are flushed before we exit:
62
+ flush(@processing) if @processing.any?
63
+ flush(@pending) if @pending.any?
64
+ @task = nil
65
+ end
66
+
67
+ return true
68
+ end
69
+
70
+ # Enqueue a job into the pending buffer.
71
+ #
72
+ # Start the background processing task if it is not already running.
73
+ def call(job)
74
+ @pending << job
75
+
76
+ start! or @ready.signal
77
+ end
78
+
79
+ def start
80
+ super
81
+
82
+ self.start!
83
+ end
84
+
85
+ def stop
86
+ @task&.stop
87
+
88
+ super
89
+ end
90
+ end
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Released under the MIT License.
4
+ # Copyright, 2024, by Samuel Williams.
5
+
6
+ require_relative 'generic'
7
+
8
+ require 'console/event/failure'
9
+
10
+ module Async
11
+ module Job
12
+ module Processor
13
+ # Add a small processing delay to each job.
14
+ class Delayed < Generic
15
+ def initialize(delegate, delay: 0.1)
16
+ super(delegate)
17
+
18
+ @delay = delay
19
+ end
20
+
21
+ def call(job)
22
+ sleep(@delay)
23
+
24
+ super
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Released under the MIT License.
4
+ # Copyright, 2024, by Samuel Williams.
5
+
6
+ module Async
7
+ module Job
8
+ module Processor
9
+ class Generic
10
+ def initialize(delegate = nil)
11
+ @delegate = delegate
12
+ end
13
+
14
+ def call(job)
15
+ @delegate.call(job)
16
+ end
17
+
18
+ def start
19
+ @delegate.start
20
+ end
21
+
22
+ def stop
23
+ @delegate.stop
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Released under the MIT License.
4
+ # Copyright, 2024, by Samuel Williams.
5
+
6
+ require_relative '../coder'
7
+ require_relative 'generic'
8
+
9
+ require 'async/idler'
10
+
11
+ module Async
12
+ module Job
13
+ module Processor
14
+ class Inline < Generic
15
+ def initialize(delegate, parent: nil)
16
+ super(delegate)
17
+
18
+ @parent = parent || Async::Idler.new
19
+ end
20
+
21
+ def call(job)
22
+ scheduled_at = Coder::Time(job["scheduled_at"])
23
+
24
+ @parent.async do
25
+ if scheduled_at
26
+ sleep(scheduled_at - Time.now)
27
+ end
28
+
29
+ @delegate.call(job)
30
+ rescue => error
31
+ Console.error(self, error)
32
+ end
33
+ end
34
+
35
+ def start
36
+ @delegate&.start
37
+ end
38
+
39
+ def stop
40
+ @delegate&.stop
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -5,9 +5,9 @@
5
5
 
6
6
  module Async
7
7
  module Job
8
- module Backend
8
+ module Processor
9
9
  module Redis
10
- class DelayedQueue
10
+ class DelayedJobs
11
11
  ADD = <<~LUA
12
12
  redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
13
13
  redis.call('ZADD', KEYS[2], ARGV[3], ARGV[1])
@@ -30,15 +30,13 @@ module Async
30
30
  @move = @client.script(:load, MOVE)
31
31
  end
32
32
 
33
- def start(ready_queue, resolution: 10, parent: Async::Task.current)
34
- Console.info(self, "Starting delayed queue...")
33
+ def start(ready_list, resolution: 10, parent: Async::Task.current)
35
34
  parent.async do
36
35
  while true
37
- Console.debug(self, "Checking for delayed jobs...")
38
- count = move(destination: ready_queue.key)
36
+ count = move(destination: ready_list.key)
39
37
 
40
38
  if count > 0
41
- Console.info(self, "Moved #{count} delayed jobs to ready queue.")
39
+ Console.debug(self, "Moved #{count} delayed jobs to ready list.")
42
40
  end
43
41
 
44
42
  sleep(resolution)
@@ -5,7 +5,7 @@
5
5
 
6
6
  module Async
7
7
  module Job
8
- module Backend
8
+ module Processor
9
9
  module Redis
10
10
  class JobStore
11
11
  def initialize(client, key)
@@ -5,9 +5,9 @@
5
5
 
6
6
  module Async
7
7
  module Job
8
- module Backend
8
+ module Processor
9
9
  module Redis
10
- class ProcessingQueue
10
+ class ProcessingList
11
11
  REQUEUE = <<~LUA
12
12
  local cursor = "0"
13
13
  local count = 0
@@ -50,12 +50,12 @@ module Async
50
50
  redis.call('HDEL', KEYS[2], ARGV[1])
51
51
  LUA
52
52
 
53
- def initialize(client, key, id, ready_queue, job_store)
53
+ def initialize(client, key, id, ready_list, job_store)
54
54
  @client = client
55
55
  @key = key
56
56
  @id = id
57
57
 
58
- @ready_queue = ready_queue
58
+ @ready_list = ready_list
59
59
  @job_store = job_store
60
60
 
61
61
  @pending_key = "#{@key}:#{@id}:pending"
@@ -69,7 +69,7 @@ module Async
69
69
  attr :key
70
70
 
71
71
  def fetch
72
- @client.brpoplpush(@ready_queue.key, @pending_key, 0)
72
+ @client.brpoplpush(@ready_list.key, @pending_key, 0)
73
73
  end
74
74
 
75
75
  def complete(id)
@@ -79,22 +79,20 @@ module Async
79
79
 
80
80
  def retry(id)
81
81
  Console.warn(self, "Retrying job: #{id}")
82
- @client.evalsha(@retry, 2, @pending_key, @ready_queue.key, id)
82
+ @client.evalsha(@retry, 2, @pending_key, @ready_list.key, id)
83
83
  end
84
84
 
85
85
  def start(delay: 5, factor: 2, parent: Async::Task.current)
86
86
  heartbeat_key = "#{@key}:#{@id}"
87
87
  start_time = Time.now.to_f
88
88
 
89
- Console.info(self, "Starting processing queue...", key: @key, id: @id, heartbeat_key: heartbeat_key, delay: delay, factor: factor)
90
-
91
89
  parent.async do
92
90
  while true
93
91
  uptime = (Time.now.to_f - start_time).round(2)
94
92
  @client.set(heartbeat_key, JSON.dump(uptime: uptime), seconds: delay*factor)
95
93
 
96
- # Requeue any jobs that have been abandoned:
97
- count = @client.evalsha(@requeue, 2, @key, @ready_queue.key)
94
+ # REQUEUE any jobs that have been abandoned:
95
+ count = @client.evalsha(@requeue, 2, @key, @ready_list.key)
98
96
  if count > 0
99
97
  Console.warn(self, "Requeued #{count} abandoned jobs.")
100
98
  end
@@ -5,9 +5,9 @@
5
5
 
6
6
  module Async
7
7
  module Job
8
- module Backend
8
+ module Processor
9
9
  module Redis
10
- class ReadyQueue
10
+ class ReadyList
11
11
  ADD = <<~LUA
12
12
  redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
13
13
  redis.call('LPUSH', KEYS[2], ARGV[1])
@@ -0,0 +1,104 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Released under the MIT License.
4
+ # Copyright, 2024, by Samuel Williams.
5
+
6
+ require_relative 'delayed_jobs'
7
+ require_relative 'job_store'
8
+ require_relative 'processing_list'
9
+ require_relative 'ready_list'
10
+ require_relative '../../coder'
11
+ require_relative '../generic'
12
+
13
+ require 'securerandom'
14
+ require 'async/idler'
15
+
16
+ module Async
17
+ module Job
18
+ module Processor
19
+ module Redis
20
+ class Server < Generic
21
+ def initialize(delegate, client, prefix: 'async-job', coder: Coder::DEFAULT, resolution: 10, parent: nil)
22
+ super(delegate)
23
+
24
+ @id = SecureRandom.uuid
25
+ @client = client
26
+ @prefix = prefix
27
+ @coder = coder
28
+ @resolution = resolution
29
+
30
+ @job_store = JobStore.new(@client, "#{@prefix}:jobs")
31
+ @delayed_jobs = DelayedJobs.new(@client, "#{@prefix}:delayed")
32
+ @ready_list = ReadyList.new(@client, "#{@prefix}:ready")
33
+ @processing_list = ProcessingList.new(@client, "#{@prefix}:processing", @id, @ready_list, @job_store)
34
+
35
+ @parent = parent || Async::Idler.new
36
+ end
37
+
38
+ def start!
39
+ return false if @task
40
+
41
+ @task = true
42
+
43
+ @parent.async(transient: true, annotation: self.class.name) do |task|
44
+ @task = task
45
+
46
+ while true
47
+ self.dequeue(task)
48
+ end
49
+ ensure
50
+ @task = nil
51
+ end
52
+ end
53
+
54
+ def start
55
+ super
56
+
57
+ # Start the delayed processor, which will move jobs to the ready processor when they are ready:
58
+ @delayed_jobs.start(@ready_list, resolution: @resolution)
59
+
60
+ # Start the processing processor, which will move jobs to the ready processor when they are abandoned:
61
+ @processing_list.start
62
+
63
+ self.start!
64
+ end
65
+
66
+ def stop
67
+ @task&.stop
68
+
69
+ super
70
+ end
71
+
72
+ def call(job)
73
+ scheduled_at = Coder::Time(job["scheduled_at"])
74
+
75
+ if scheduled_at
76
+ @delayed_jobs.add(@coder.dump(job), scheduled_at, @job_store)
77
+ else
78
+ @ready_list.add(@coder.dump(job), @job_store)
79
+ end
80
+ end
81
+
82
+ protected
83
+
84
+ def dequeue(parent)
85
+ _id = @processing_list.fetch
86
+
87
+ parent.async do
88
+ id = _id; _id = nil
89
+
90
+ job = @coder.load(@job_store.get(id))
91
+ @delegate.call(job)
92
+ @processing_list.complete(id)
93
+ rescue => error
94
+ @processing_list.retry(id)
95
+ Console.error(self, error)
96
+ end
97
+ ensure
98
+ @processing_list.retry(_id) if _id
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
@@ -8,7 +8,7 @@ require 'async/redis/client'
8
8
 
9
9
  module Async
10
10
  module Job
11
- module Backend
11
+ module Processor
12
12
  module Redis
13
13
  def self.new(delegate, endpoint: Async::Redis.local_endpoint, **options)
14
14
  client = Async::Redis::Client.new(endpoint)
@@ -3,15 +3,13 @@
3
3
  # Released under the MIT License.
4
4
  # Copyright, 2024, by Samuel Williams.
5
5
 
6
- require_relative 'inline/server'
6
+ require_relative 'processor/inline'
7
7
 
8
8
  module Async
9
9
  module Job
10
- module Backend
11
- module Inline
12
- def self.new(delegate)
13
- return Server.new(delegate)
14
- end
10
+ module Processor
11
+ def self.new(processor: Inline, **options)
12
+ processor.new(**options)
15
13
  end
16
14
  end
17
15
  end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Released under the MIT License.
4
+ # Copyright, 2024, by Samuel Williams.
5
+
6
+ module Async
7
+ module Job
8
+ class Queue
9
+ def initialize(client, server, delegate)
10
+ @client = client
11
+ @server = server
12
+ @delegate = delegate
13
+ end
14
+
15
+ attr :client
16
+ attr :server
17
+ attr :delegate
18
+
19
+ def call(...)
20
+ @client.call(...)
21
+ end
22
+
23
+ def start
24
+ @server.start
25
+ end
26
+
27
+ def stop
28
+ @server.stop
29
+ end
30
+ end
31
+ end
32
+ end
@@ -5,6 +5,6 @@
5
5
 
6
6
  module Async
7
7
  module Job
8
- VERSION = "0.7.1"
8
+ VERSION = "0.9.0"
9
9
  end
10
10
  end
data/lib/async/job.rb CHANGED
@@ -4,5 +4,5 @@
4
4
  # Copyright, 2024, by Samuel Williams.
5
5
 
6
6
  require_relative 'job/version'
7
- require_relative 'job/backend'
7
+ require_relative 'job/queue'
8
8
  require_relative 'job/builder'
data/readme.md CHANGED
@@ -10,7 +10,7 @@ Please see the [project documentation](https://socketry.github.io/async-job/) fo
10
10
 
11
11
  - [Getting Started](https://socketry.github.io/async-job/guides/getting-started/index) - This guide gives you an overview of the `async-job` gem and explains the core concepts.
12
12
 
13
- - [Redis Backend](https://socketry.github.io/async-job/guides/redis-backend/index) - This guide gives a brief overview of the implementation of the Redis backend.
13
+ - [Redis Processor](https://socketry.github.io/async-job/guides/redis-processor/index) - This guide gives a brief overview of the implementation of the Redis queue.
14
14
 
15
15
  ## See Also
16
16
 
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: async-job
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.1
4
+ version: 0.9.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Samuel Williams
@@ -75,23 +75,24 @@ extensions: []
75
75
  extra_rdoc_files: []
76
76
  files:
77
77
  - lib/async/job.rb
78
- - lib/async/job/backend.rb
79
- - lib/async/job/backend/aggregate.rb
80
- - lib/async/job/backend/aggregate/server.rb
81
- - lib/async/job/backend/inline.rb
82
- - lib/async/job/backend/inline/server.rb
83
- - lib/async/job/backend/redis.rb
84
- - lib/async/job/backend/redis/delayed_queue.rb
85
- - lib/async/job/backend/redis/job_store.rb
86
- - lib/async/job/backend/redis/processing_queue.rb
87
- - lib/async/job/backend/redis/ready_queue.rb
88
- - lib/async/job/backend/redis/server.rb
89
78
  - lib/async/job/buffer.rb
90
79
  - lib/async/job/builder.rb
91
80
  - lib/async/job/coder.rb
92
81
  - lib/async/job/coder/marshal.rb
93
82
  - lib/async/job/coder/message_pack.rb
94
83
  - lib/async/job/generic.rb
84
+ - lib/async/job/processor.rb
85
+ - lib/async/job/processor/aggregate.rb
86
+ - lib/async/job/processor/delayed.rb
87
+ - lib/async/job/processor/generic.rb
88
+ - lib/async/job/processor/inline.rb
89
+ - lib/async/job/processor/redis.rb
90
+ - lib/async/job/processor/redis/delayed_jobs.rb
91
+ - lib/async/job/processor/redis/job_store.rb
92
+ - lib/async/job/processor/redis/processing_list.rb
93
+ - lib/async/job/processor/redis/ready_list.rb
94
+ - lib/async/job/processor/redis/server.rb
95
+ - lib/async/job/queue.rb
95
96
  - lib/async/job/version.rb
96
97
  - license.md
97
98
  - readme.md
@@ -116,7 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
116
117
  - !ruby/object:Gem::Version
117
118
  version: '0'
118
119
  requirements: []
119
- rubygems_version: 3.5.13
120
+ rubygems_version: 3.5.11
120
121
  signing_key:
121
122
  specification_version: 4
122
123
  summary: A asynchronous job queue for Ruby.
metadata.gz.sig CHANGED
Binary file
@@ -1,85 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Released under the MIT License.
4
- # Copyright, 2024, by Samuel Williams.
5
-
6
- require 'console/event/failure'
7
-
8
- module Async
9
- module Job
10
- module Backend
11
- module Aggregate
12
- class Server
13
- def initialize(delegate, parent: nil)
14
- @delegate = delegate
15
-
16
- @task = nil
17
- @ready = Async::Condition.new
18
-
19
- @pending = Array.new
20
- @processing = Array.new
21
- end
22
-
23
- def flush(jobs)
24
- while job = jobs.shift
25
- @delegate.call(job)
26
- end
27
- rescue => error
28
- Console::Event::Failure.for(error).emit(self, "Could not flush #{jobs.size} jobs.")
29
- end
30
-
31
- def run(task)
32
- while true
33
- while @pending.empty?
34
- @ready.wait
35
- end
36
-
37
- task.defer_stop do
38
- # Swap the buffers:
39
- @pending, @processing = @processing, @pending
40
-
41
- flush(@processing)
42
- end
43
- end
44
- end
45
-
46
- # Start the background processing task if it is not already running.
47
- #
48
- # @return [Boolean] true if the task was started, false if it was already running.
49
- protected def start!(parent: Async::Task.current)
50
- return false if @task
51
-
52
- # We are creating a task:
53
- @task = true
54
-
55
- parent.async(transient: true, annotation: self.class.name) do |task|
56
- @task = task
57
-
58
- run(task)
59
- ensure
60
- # Ensure that all jobs are flushed before we exit:
61
- flush(@processing) if @processing.any?
62
- flush(@pending) if @pending.any?
63
- @task = nil
64
- end
65
-
66
- return true
67
- end
68
-
69
- # Enqueue a job into the pending buffer.
70
- #
71
- # Start the background processing task if it is not already running.
72
- def call(job)
73
- @pending << job
74
-
75
- start! or @ready.signal
76
- end
77
-
78
- def stop
79
- @task&.stop
80
- end
81
- end
82
- end
83
- end
84
- end
85
- end
@@ -1,18 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Released under the MIT License.
4
- # Copyright, 2024, by Samuel Williams.
5
-
6
- require_relative 'aggregate/server'
7
-
8
- module Async
9
- module Job
10
- module Backend
11
- module Aggregate
12
- def self.new(delegate)
13
- return Server.new(delegate)
14
- end
15
- end
16
- end
17
- end
18
- end
@@ -1,37 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Released under the MIT License.
4
- # Copyright, 2024, by Samuel Williams.
5
-
6
- require_relative '../../coder'
7
-
8
- require 'async/idler'
9
-
10
- module Async
11
- module Job
12
- module Backend
13
- module Inline
14
- class Server
15
- def initialize(delegate, parent: nil)
16
- @delegate = delegate
17
- @parent = parent || Async::Idler.new
18
- end
19
-
20
- def call(job)
21
- scheduled_at = Coder::Time(job["scheduled_at"])
22
-
23
- @parent.async do
24
- if scheduled_at
25
- sleep(scheduled_at - Time.now)
26
- end
27
-
28
- @delegate.call(job)
29
- rescue => error
30
- Console.error(self, error)
31
- end
32
- end
33
- end
34
- end
35
- end
36
- end
37
- end
@@ -1,84 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Released under the MIT License.
4
- # Copyright, 2024, by Samuel Williams.
5
-
6
- require_relative 'delayed_queue'
7
- require_relative 'job_store'
8
- require_relative 'processing_queue'
9
- require_relative 'ready_queue'
10
- require_relative '../../coder'
11
-
12
- require 'securerandom'
13
- require 'async/idler'
14
-
15
- module Async
16
- module Job
17
- module Backend
18
- module Redis
19
- class Server
20
- def initialize(delegate, client, prefix: 'async-job', coder: Coder::DEFAULT, resolution: 10, parent: nil)
21
- @delegate = delegate
22
-
23
- @id = SecureRandom.uuid
24
- @client = client
25
- @prefix = prefix
26
- @coder = coder
27
- @resolution = resolution
28
-
29
- @job_store = JobStore.new(@client, "#{@prefix}:jobs")
30
-
31
- @delayed_queue = DelayedQueue.new(@client, "#{@prefix}:delayed")
32
- @ready_queue = ReadyQueue.new(@client, "#{@prefix}:ready")
33
-
34
- @processing_queue = ProcessingQueue.new(@client, "#{@prefix}:processing", @id, @ready_queue, @job_store)
35
-
36
- @parent = parent || Async::Idler.new
37
- end
38
-
39
- def start
40
- Console.info(self, "Starting server...")
41
- # Start the delayed queue, which will move jobs to the ready queue when they are ready:
42
- @delayed_queue.start(@ready_queue, resolution: @resolution)
43
-
44
- # Start the processing queue, which will move jobs to the ready queue when they are abandoned:
45
- @processing_queue.start
46
-
47
- while true
48
- self.dequeue(@parent)
49
- end
50
- end
51
-
52
- def call(job)
53
- scheduled_at = Coder::Time(job["scheduled_at"])
54
-
55
- if scheduled_at
56
- @delayed_queue.add(@coder.dump(job), scheduled_at, @job_store)
57
- else
58
- @ready_queue.add(@coder.dump(job), @job_store)
59
- end
60
- end
61
-
62
- protected
63
-
64
- def dequeue(parent)
65
- _id = @processing_queue.fetch
66
-
67
- parent.async do
68
- id = _id; _id = nil
69
-
70
- job = @coder.load(@job_store.get(id))
71
- @delegate.call(job)
72
- @processing_queue.complete(id)
73
- rescue => error
74
- @processing_queue.retry(id)
75
- Console.error(self, error)
76
- end
77
- ensure
78
- @processing_queue.retry(_id) if _id
79
- end
80
- end
81
- end
82
- end
83
- end
84
- end
@@ -1,16 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Released under the MIT License.
4
- # Copyright, 2024, by Samuel Williams.
5
-
6
- require_relative 'backend/redis'
7
-
8
- module Async
9
- module Job
10
- module Backend
11
- def self.new(backend: Async::Job::Backend::Redis, **options)
12
- backend.new(**options)
13
- end
14
- end
15
- end
16
- end