async-job 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71fdcce6742cea5cfaeac68e800c56f1dbfa5ea6849503544c35912675874838
-  data.tar.gz: a607a76328c14cee24618479efd1d6924adc5634d7a89d21df0ccccbea78b1d8
+  metadata.gz: 7381f7520a5ee41df3c93fc4f55c647952f10066868fcb22a196fb6ee807a1dd
+  data.tar.gz: a063193f2ff8a71fe798aadc3ce07d392fd2c53cadd5523cb20813fff9da2bdc
 SHA512:
-  metadata.gz: 7bc8ac06036c862eb827af97ef43764dd44036dc14cdb5f988a294fc2dbbac5f441b32d71cd6551aa3000c190de82105fcce65771ced7b67d08a8f24e598c872
-  data.tar.gz: 145098c24c1a115d018a4ecc8a1796aa295c4cfb85b8099c30c2e82c9a89fd31780e525015e799a12be807e748c824f299ac88677afdf0764f7b1594094b758e
+  metadata.gz: 56f1a056d2cbc2dab6a97226c07b6d2acec56de40bd95c302b652e4689c3fa08986a7b6f4e4c53a0ca15555c61695da6ef2219f8585d1c1006c01cacfffac50b
+  data.tar.gz: b0c62cf9b609beee163c948ff9fc933a1d82b3f0f611e555ddc4dadd4909ab3e0791e474ae7cdf35cb1081d12d38a5f73f37148945e2658bec07af31388b8480

data/lib/async/job/builder.rb

@@ -3,11 +3,11 @@
 # Released under the MIT License.
 # Copyright, 2024, by Samuel Williams.
 
+require_relative 'queue'
+
 module Async
 	module Job
 		class Builder
-			Pipeline = Struct.new(:producer, :consumer, :delegate)
-			
 			def self.build(delegate = nil, &block)
 				builder = self.new(delegate)
 				
@@ -16,58 +16,48 @@ module Async
 				return builder.build
 			end
 			
+			# @parameter delegate [Object] The initial delegate that will be wrapped by the queue.
 			def initialize(delegate = nil)
+				# The client side middleware, in the order they should be applied to a job:
 				@enqueue = []
+				
+				# The server side middleware, in the order they should be applied to a job:
 				@dequeue = []
-				@delegate = delegate
 				
-				@queue = nil
-			end
-			
-			def enqueue(middleware)
-				@enqueue << middleware
+				# The output delegate, if any:
+				@delegate = delegate
 			end
 			
-			def queue(queue, *arguments, **options)
-				# The delegate is the output side of the queue, e.g. a Redis server delegate or similar wrapper.
-				# The queue itself is instantiated with the delegate.
-				@queue = ->(delegate){queue.new(delegate, *arguments, **options)}
+			def enqueue(middleware, ...)
+				@enqueue << ->(delegate){middleware.new(delegate, ...)}
 			end
 			
-			def dequeue(middleware)
-				@dequeue << middleware
+			def dequeue(middleware, ...)
+				@dequeue << ->(delegate){middleware.new(delegate, ...)}
 			end
 			
 			def delegate(delegate)
 				@delegate = delegate
 			end
 			
-			def build(&block)
-				# To construct the queue, we need the delegate.
-				delegate = @delegate
-				
+			def build(delegate = @delegate, &block)
 				# We then wrap the delegate with the middleware in reverse order:
 				@dequeue.reverse_each do |middleware|
-					delegate = middleware.new(delegate)
+					delegate = middleware.call(delegate)
 				end
 				
-				# We can now construct the queue with the delegate:
-				if @queue
-					producer = consumer = @queue.call(delegate)
-				else
-					producer = consumer = delegate
-				end
+				client = server = delegate
 				
 				# We now construct the queue producer:
 				@enqueue.reverse_each do |middleware|
-					producer = middleware.new(producer)
+					client = middleware.call(client)
 				end
 				
 				if block_given?
-					producer = yield(producer) || producer
+					client = yield(client) || client
 				end
 				
-				return Pipeline.new(producer, consumer, @delegate)
+				return Queue.new(client, server, @delegate)
 			end
 		end
 	end

data/lib/async/job/processor/aggregate.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+require_relative 'generic'
+
+require 'console/event/failure'
+
+module Async
+	module Job
+		module Processor
+			class Aggregate < Generic
+				def initialize(delegate, parent: nil)
+					super(delegate)
+					
+					@task = nil
+					@ready = Async::Condition.new
+					
+					@pending = Array.new
+					@processing = Array.new
+				end
+				
+				def flush(jobs)
+					while job = jobs.shift
+						@delegate.call(job)
+					end
+				rescue => error
+					Console::Event::Failure.for(error).emit(self, "Could not flush #{jobs.size} jobs.")
+				end
+				
+				def run(task)
+					while true
+						while @pending.empty?
+							@ready.wait
+						end
+						
+						task.defer_stop do
+							# Swap the buffers:
+							@pending, @processing = @processing, @pending
+							
+							flush(@processing)
+						end
+					end
+				end
+				
+				# Start the background processing task if it is not already running.
+				#
+				# @return [Boolean] true if the task was started, false if it was already running.
+				protected def start!(parent: Async::Task.current)
+					return false if @task
+					
+					# We are creating a task:
+					@task = true
+					
+					parent.async(transient: true, annotation: self.class.name) do |task|
+						@task = task
+						
+						run(task)
+					ensure
+						# Ensure that all jobs are flushed before we exit:
+						flush(@processing) if @processing.any?
+						flush(@pending) if @pending.any?
+						@task = nil
+					end
+					
+					return true
+				end
+				
+				# Enqueue a job into the pending buffer.
+				#
+				# Start the background processing task if it is not already running.
+				def call(job)
+					@pending << job
+					
+					start! or @ready.signal
+				end
+				
+				def start
+					super
+					
+					self.start!
+				end
+				
+				def stop
+					@task&.stop
+					
+					super
+				end
+			end
+		end
+	end
+end

data/lib/async/job/processor/delayed.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+require_relative 'generic'
+
+require 'console/event/failure'
+
+module Async
+	module Job
+		module Processor
+			# Add a small processing delay to each job.
+			class Delayed < Generic
+				def initialize(delegate, delay: 0.1)
+					super(delegate)
+					
+					@delay = delay
+				end
+				
+				def call(job)
+					sleep(@delay)
+					
+					super
+				end
+			end
+		end
+	end
+end

data/lib/async/job/processor/generic.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		module Processor
+			class Generic
+				def initialize(delegate = nil)
+					@delegate = delegate
+				end
+				
+				def call(job)
+					@delegate.call(job)
+				end
+				
+				def start
+					@delegate.start
+				end
+				
+				def stop
+					@delegate.stop
+				end
+			end
+		end
+	end
+end

data/lib/async/job/processor/inline.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+require_relative '../coder'
+require_relative 'generic'
+
+require 'async/idler'
+
+module Async
+	module Job
+		module Processor
+			class Inline < Generic
+				def initialize(delegate, parent: nil)
+					super(delegate)
+					
+					@parent = parent || Async::Idler.new
+				end
+				
+				def call(job)
+					scheduled_at = Coder::Time(job["scheduled_at"])
+					
+					@parent.async do
+						if scheduled_at
+							sleep(scheduled_at - Time.now)
+						end
+						
+						@delegate.call(job)
+					rescue => error
+						Console.error(self, error)
+					end
+				end
+				
+				def start
+					@delegate&.start
+				end
+				
+				def stop
+					@delegate&.stop
+				end
+			end
+		end
+	end
+end

data/lib/async/job/{backend/redis/delayed_queue.rb → processor/redis/delayed_jobs.rb}

@@ -5,9 +5,9 @@
 
 module Async
 	module Job
-		module Backend
+		module Processor
 			module Redis
-				class DelayedQueue
+				class DelayedJobs
 					ADD = <<~LUA
 						redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
 						redis.call('ZADD', KEYS[2], ARGV[3], ARGV[1])
@@ -30,15 +30,13 @@ module Async
 						@move = @client.script(:load, MOVE)
 					end
 					
-					def start(ready_queue, resolution: 10, parent: Async::Task.current)
-						Console.info(self, "Starting delayed queue...")
+					def start(ready_list, resolution: 10, parent: Async::Task.current)
 						parent.async do
 							while true
-								Console.debug(self, "Checking for delayed jobs...")
-								count = move(destination: ready_queue.key)
+								count = move(destination: ready_list.key)
 								
 								if count > 0
-									Console.info(self, "Moved #{count} delayed jobs to ready queue.")
+									Console.debug(self, "Moved #{count} delayed jobs to ready list.")
 								end
 								
 								sleep(resolution)

data/lib/async/job/{backend → processor}/redis/job_store.rb

@@ -5,7 +5,7 @@
 
 module Async
 	module Job
-		module Backend
+		module Processor
 			module Redis
 				class JobStore
 					def initialize(client, key)

data/lib/async/job/{backend/redis/processing_queue.rb → processor/redis/processing_list.rb}

@@ -5,9 +5,9 @@
 
 module Async
 	module Job
-		module Backend
+		module Processor
 			module Redis
-				class ProcessingQueue
+				class ProcessingList
 					REQUEUE = <<~LUA
 						local cursor = "0"
 						local count = 0
@@ -50,12 +50,12 @@ module Async
 						redis.call('HDEL', KEYS[2], ARGV[1])
 					LUA
 					
-					def initialize(client, key, id, ready_queue, job_store)
+					def initialize(client, key, id, ready_list, job_store)
 						@client = client
 						@key = key
 						@id = id
 						
-						@ready_queue = ready_queue
+						@ready_list = ready_list
 						@job_store = job_store
 						
 						@pending_key = "#{@key}:#{@id}:pending"
@@ -69,7 +69,7 @@ module Async
 					attr :key
 					
 					def fetch
-						@client.brpoplpush(@ready_queue.key, @pending_key, 0)
+						@client.brpoplpush(@ready_list.key, @pending_key, 0)
 					end
 					
 					def complete(id)
@@ -79,22 +79,20 @@ module Async
 					
 					def retry(id)
 						Console.warn(self, "Retrying job: #{id}")
-						@client.evalsha(@retry, 2, @pending_key, @ready_queue.key, id)
+						@client.evalsha(@retry, 2, @pending_key, @ready_list.key, id)
 					end
 					
 					def start(delay: 5, factor: 2, parent: Async::Task.current)
 						heartbeat_key = "#{@key}:#{@id}"
 						start_time = Time.now.to_f
 						
-						Console.info(self, "Starting processing queue...", key: @key, id: @id, heartbeat_key: heartbeat_key, delay: delay, factor: factor)
-						
 						parent.async do
 							while true
 								uptime = (Time.now.to_f - start_time).round(2)
 								@client.set(heartbeat_key, JSON.dump(uptime: uptime), seconds: delay*factor)
 								
-								# Requeue any jobs that have been abandoned:
-								count = @client.evalsha(@requeue, 2, @key, @ready_queue.key)
+								# REQUEUE any jobs that have been abandoned:
+								count = @client.evalsha(@requeue, 2, @key, @ready_list.key)
 								if count > 0
 									Console.warn(self, "Requeued #{count} abandoned jobs.")
 								end

data/lib/async/job/{backend/redis/ready_queue.rb → processor/redis/ready_list.rb}

@@ -5,9 +5,9 @@
 
 module Async
 	module Job
-		module Backend
+		module Processor
 			module Redis
-				class ReadyQueue
+				class ReadyList
 					ADD = <<~LUA
 						redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
 						redis.call('LPUSH', KEYS[2], ARGV[1])

data/lib/async/job/{backend → processor}/redis/server.rb

@@ -3,22 +3,23 @@
 # Released under the MIT License.
 # Copyright, 2024, by Samuel Williams.
 
-require_relative 'delayed_queue'
+require_relative 'delayed_jobs'
 require_relative 'job_store'
-require_relative 'processing_queue'
-require_relative 'ready_queue'
+require_relative 'processing_list'
+require_relative 'ready_list'
 require_relative '../../coder'
+require_relative '../generic'
 
 require 'securerandom'
 require 'async/idler'
 
 module Async
 	module Job
-		module Backend
+		module Processor
 			module Redis
-				class Server
+				class Server < Generic
 					def initialize(delegate, client, prefix: 'async-job', coder: Coder::DEFAULT, resolution: 10, parent: nil)
-						@delegate = delegate
+						super(delegate)
 						
 						@id = SecureRandom.uuid
 						@client = client
@@ -27,11 +28,9 @@ module Async
 						@resolution = resolution
 						
 						@job_store = JobStore.new(@client, "#{@prefix}:jobs")
-						
-						@delayed_queue = DelayedQueue.new(@client, "#{@prefix}:delayed")
-						@ready_queue = ReadyQueue.new(@client, "#{@prefix}:ready")
-						
-						@processing_queue = ProcessingQueue.new(@client, "#{@prefix}:processing", @id, @ready_queue, @job_store)
+						@delayed_jobs = DelayedJobs.new(@client, "#{@prefix}:delayed")
+						@ready_list = ReadyList.new(@client, "#{@prefix}:ready")
+						@processing_list = ProcessingList.new(@client, "#{@prefix}:processing", @id, @ready_list, @job_store)
 						
 						@parent = parent || Async::Idler.new
 					end
@@ -53,49 +52,50 @@ module Async
 					end
 					
 					def start
-						@delegate&.start
+						super
 						
-						# Start the delayed queue, which will move jobs to the ready queue when they are ready:
-						@delayed_queue.start(@ready_queue, resolution: @resolution)
+						# Start the delayed processor, which will move jobs to the ready processor when they are ready:
+						@delayed_jobs.start(@ready_list, resolution: @resolution)
 						
-						# Start the processing queue, which will move jobs to the ready queue when they are abandoned:
-						@processing_queue.start
+						# Start the processing processor, which will move jobs to the ready processor when they are abandoned:
+						@processing_list.start
 						
 						self.start!
 					end
 					
 					def stop
 						@task&.stop
-						@delegate&.stop
+						
+						super
 					end
 					
 					def call(job)
 						scheduled_at = Coder::Time(job["scheduled_at"])
 						
 						if scheduled_at
-							@delayed_queue.add(@coder.dump(job), scheduled_at, @job_store)
+							@delayed_jobs.add(@coder.dump(job), scheduled_at, @job_store)
 						else
-							@ready_queue.add(@coder.dump(job), @job_store)
+							@ready_list.add(@coder.dump(job), @job_store)
 						end
 					end
 					
 					protected
 					
 					def dequeue(parent)
-						_id = @processing_queue.fetch
+						_id = @processing_list.fetch
 						
 						parent.async do
 							id = _id; _id = nil
 							
 							job = @coder.load(@job_store.get(id))
 							@delegate.call(job)
-							@processing_queue.complete(id)
+							@processing_list.complete(id)
 						rescue => error
-							@processing_queue.retry(id)
+							@processing_list.retry(id)
 							Console.error(self, error)
 						end
 					ensure
-						@processing_queue.retry(_id) if _id
+						@processing_list.retry(_id) if _id
 					end
 				end
 			end

data/lib/async/job/{backend → processor}/redis.rb

@@ -8,7 +8,7 @@ require 'async/redis/client'
 
 module Async
 	module Job
-		module Backend
+		module Processor
 			module Redis
 				def self.new(delegate, endpoint: Async::Redis.local_endpoint, **options)
 					client = Async::Redis::Client.new(endpoint)

data/lib/async/job/{backend/inline.rb → processor.rb}

@@ -3,15 +3,13 @@
 # Released under the MIT License.
 # Copyright, 2024, by Samuel Williams.
 
-require_relative 'inline/server'
+require_relative 'processor/inline'
 
 module Async
 	module Job
-		module Backend
-			module Inline
-				def self.new(delegate)
-					return Server.new(delegate)
-				end
+		module Processor
+			def self.new(processor: Inline, **options)
+				processor.new(**options)
 			end
 		end
 	end

data/lib/async/job/queue.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		class Queue
+			def initialize(client, server, delegate)
+				@client = client
+				@server = server
+				@delegate = delegate
+			end
+			
+			attr :client
+			attr :server
+			attr :delegate
+			
+			def call(...)
+				@client.call(...)
+			end
+			
+			def start
+				@server.start
+			end
+			
+			def stop
+				@server.stop
+			end
+		end
+	end
+end

data/lib/async/job/version.rb

@@ -5,6 +5,6 @@
 
 module Async
 	module Job
-		VERSION = "0.8.0"
+		VERSION = "0.9.0"
 	end
 end

data/lib/async/job.rb

@@ -4,5 +4,5 @@
 # Copyright, 2024, by Samuel Williams.
 
 require_relative 'job/version'
-require_relative 'job/backend'
+require_relative 'job/queue'
 require_relative 'job/builder'

data/readme.md

@@ -10,7 +10,7 @@ Please see the [project documentation](https://socketry.github.io/async-job/) fo
 
   - [Getting Started](https://socketry.github.io/async-job/guides/getting-started/index) - This guide gives you an overview of the `async-job` gem and explains the core concepts.
 
-  - [Redis Backend](https://socketry.github.io/async-job/guides/redis-backend/index) - This guide gives a brief overview of the implementation of the Redis backend.
+  - [Redis Processor](https://socketry.github.io/async-job/guides/redis-processor/index) - This guide gives a brief overview of the implementation of the Redis queue.
 
 ## See Also
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: async-job
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Samuel Williams
@@ -75,23 +75,24 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/async/job.rb
-- lib/async/job/backend.rb
-- lib/async/job/backend/aggregate.rb
-- lib/async/job/backend/aggregate/server.rb
-- lib/async/job/backend/inline.rb
-- lib/async/job/backend/inline/server.rb
-- lib/async/job/backend/redis.rb
-- lib/async/job/backend/redis/delayed_queue.rb
-- lib/async/job/backend/redis/job_store.rb
-- lib/async/job/backend/redis/processing_queue.rb
-- lib/async/job/backend/redis/ready_queue.rb
-- lib/async/job/backend/redis/server.rb
 - lib/async/job/buffer.rb
 - lib/async/job/builder.rb
 - lib/async/job/coder.rb
 - lib/async/job/coder/marshal.rb
 - lib/async/job/coder/message_pack.rb
 - lib/async/job/generic.rb
+- lib/async/job/processor.rb
+- lib/async/job/processor/aggregate.rb
+- lib/async/job/processor/delayed.rb
+- lib/async/job/processor/generic.rb
+- lib/async/job/processor/inline.rb
+- lib/async/job/processor/redis.rb
+- lib/async/job/processor/redis/delayed_jobs.rb
+- lib/async/job/processor/redis/job_store.rb
+- lib/async/job/processor/redis/processing_list.rb
+- lib/async/job/processor/redis/ready_list.rb
+- lib/async/job/processor/redis/server.rb
+- lib/async/job/queue.rb
 - lib/async/job/version.rb
 - license.md
 - readme.md

data/lib/async/job/backend/aggregate/server.rb

@@ -1,91 +0,0 @@
-# frozen_string_literal: true
-
-# Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
-
-require 'console/event/failure'
-
-module Async
-	module Job
-		module Backend
-			module Aggregate
-				class Server
-					def initialize(delegate, parent: nil)
-						@delegate = delegate
-						
-						@task = nil
-						@ready = Async::Condition.new
-						
-						@pending = Array.new
-						@processing = Array.new
-					end
-					
-					def flush(jobs)
-						while job = jobs.shift
-							@delegate.call(job)
-						end
-					rescue => error
-						Console::Event::Failure.for(error).emit(self, "Could not flush #{jobs.size} jobs.")
-					end
-					
-					def run(task)
-						while true
-							while @pending.empty?
-								@ready.wait
-							end
-							
-							task.defer_stop do
-								# Swap the buffers:
-								@pending, @processing = @processing, @pending
-								
-								flush(@processing)
-							end
-						end
-					end
-					
-					# Start the background processing task if it is not already running.
-					#
-					# @return [Boolean] true if the task was started, false if it was already running.
-					protected def start!(parent: Async::Task.current)
-						return false if @task
-						
-						# We are creating a task:
-						@task = true
-						
-						parent.async(transient: true, annotation: self.class.name) do |task|
-							@task = task
-							
-							run(task)
-						ensure
-							# Ensure that all jobs are flushed before we exit:
-							flush(@processing) if @processing.any?
-							flush(@pending) if @pending.any?
-							@task = nil
-						end
-						
-						return true
-					end
-					
-					# Enqueue a job into the pending buffer.
-					#
-					# Start the background processing task if it is not already running.
-					def call(job)
-						@pending << job
-						
-						start! or @ready.signal
-					end
-					
-					def start
-						@delegate&.start
-						self.start!
-					end
-					
-					def stop
-						@task&.stop
-						@delegate&.stop
-					end
-				end
-			end
-		end
-	end
-end

data/lib/async/job/backend/aggregate.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-# Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
-
-require_relative 'aggregate/server'
-
-module Async
-	module Job
-		module Backend
-			module Aggregate
-				def self.new(delegate)
-					return Server.new(delegate)
-				end
-			end
-		end
-	end
-end

data/lib/async/job/backend/inline/server.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-# Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
-
-require_relative '../../coder'
-
-require 'async/idler'
-
-module Async
-	module Job
-		module Backend
-			module Inline
-				class Server
-					def initialize(delegate, parent: nil)
-						@delegate = delegate
-						@parent = parent || Async::Idler.new
-					end
-					
-					def call(job)
-						scheduled_at = Coder::Time(job["scheduled_at"])
-						
-						@parent.async do
-							if scheduled_at
-								sleep(scheduled_at - Time.now)
-							end
-							
-							@delegate.call(job)
-						rescue => error
-							Console.error(self, error)
-						end
-					end
-					
-					def start
-						@delegate&.start
-					end
-					
-					def stop
-						@delegate&.stop
-					end
-				end
-			end
-		end
-	end
-end

data/lib/async/job/backend.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-# Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
-
-require_relative 'backend/redis'
-
-module Async
-	module Job
-		module Backend
-			def self.new(backend: Async::Job::Backend::Redis, **options)
-				backend.new(**options)
-			end
-		end
-	end
-end