async-job-processor-redis 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '095272ad3206a878ec049e641b3632181be1f62ca055398230e14dc3fd242d14'
-  data.tar.gz: b10ab3013a961aca3cbecaa8f51265f4aa8a20aa3eb43031b6a94079d23553f8
+  metadata.gz: 5dd229182e417f662b414408d5b49a3c789791f63d7178d2d3acd4c30345b097
+  data.tar.gz: 2f28fd28ebbe51ea344810a039aa2985f68bf4b3934240dabb5d5949e9ffa06c
 SHA512:
-  metadata.gz: '082e58c7f2f95eff36c022f68ce14a0a1b00e01463bd4f3397a94422b4a4f9011b106d09b21462ce5aeab8dd711af85e8168d1c4423357137fc564720ecb997f'
-  data.tar.gz: e3bbfd7290feda5b88c941fef9bd4e7ea191c8a800dd97f330f3d1c7032efb869aa73d1a08419b957a541fb78e3d482f6efb816e327c659fd1c26c433cd22479
+  metadata.gz: 8818c38d052f06d17c8ada64d537ac996c3bb62bfbbb11c6965e7efa55f4c08b25f19ac9d50c44799b0dd3adcc7b8592ab573cf9793e0fab4500a09c3cfe560a
+  data.tar.gz: 9f8a5d182e4f9e7fff41bd01a328a6c50a1349a6dc9136e5fb1566d549e5558e624ccb9442d56e15a32779b5f6b0de4c67b2da619ebc6d8815f58e9183636cc0

data/lib/async/job/processor/redis/delayed_jobs.rb

@@ -7,6 +7,9 @@ module Async
 	module Job
 		module Processor
 			module Redis
+				# Manages delayed job scheduling using Redis sorted sets.
+				# Jobs are stored with their execution timestamps and automatically moved
+				# to the ready queue when their scheduled time arrives.
 				class DelayedJobs
 					ADD = <<~LUA
 						redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
@@ -22,6 +25,9 @@ module Async
 						return #jobs
 					LUA
 					
+					# Initialize a new delayed jobs manager.
+					# @parameter client [Async::Redis::Client] The Redis client instance.
+					# @parameter key [String] The Redis key for the delayed jobs sorted set.
 					def initialize(client, key)
 						@client = client
 						@key = key
@@ -30,6 +36,16 @@ module Async
 						@move = @client.script(:load, MOVE)
 					end
 					
+					# @returns [Integer] The number of jobs currently in the delayed queue.
+					def size
+						@client.zcard(@key)
+					end
+					
+					# Start the background task that moves ready delayed jobs to the ready queue.
+					# @parameter ready_list [ReadyList] The ready list to move jobs to.
+					# @parameter resolution [Integer] The check interval in seconds.
+					# @parameter parent [Async::Task] The parent task to run the background loop in.
+					# @returns [Async::Task] The background processing task.
 					def start(ready_list, resolution: 10, parent: Async::Task.current)
 						parent.async do
 							while true
@@ -44,8 +60,14 @@ module Async
 						end
 					end
 					
+					# @attribute [String] The Redis key for this delayed jobs queue.
 					attr :key
 					
+					# Add a job to the delayed queue with a specified execution time.
+					# @parameter job [String] The serialized job data.
+					# @parameter timestamp [Time] When the job should be executed.
+					# @parameter job_store [JobStore] The job store to save the job data.
+					# @returns [String] The unique job ID.
 					def add(job, timestamp, job_store)
 						id = SecureRandom.uuid
 						
@@ -54,7 +76,11 @@ module Async
 						return id
 					end
 					
-					def move(destination:, now: Time.now.to_i)
+					# Move jobs that are ready to be processed from the delayed queue to the destination.
+					# @parameter destination [String] The Redis key of the destination queue.
+					# @parameter now [Integer] The current timestamp to check against.
+					# @returns [Integer] The number of jobs moved.
+					def move(destination:, now: Time.now.to_f)
 						@client.evalsha(@move, 2, @key, destination, now)
 					end
 				end

data/lib/async/job/processor/redis/job_store.rb

@@ -7,14 +7,23 @@ module Async
 	module Job
 		module Processor
 			module Redis
+				# Stores job data using Redis hashes.
+				# Provides persistent storage for job payloads indexed by job ID.
 				class JobStore
+					# Initialize a new job store.
+					# @parameter client [Async::Redis::Client] The Redis client instance.
+					# @parameter key [String] The Redis key for the job data hash.
 					def initialize(client, key)
 						@client = client
 						@key = key
 					end
 					
+					# @attribute [String] The Redis key for this job store.
 					attr :key
 					
+					# Retrieve job data by ID.
+					# @parameter id [String] The job ID to retrieve.
+					# @returns [String, nil] The serialized job data, or nil if not found.
 					def get(id)
 						@client.hget(@key, id)
 					end

data/lib/async/job/processor/redis/processing_list.rb

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
 # Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
+# Copyright, 2024-2025, by Samuel Williams.
 
 module Async
 	module Job
 		module Processor
 			module Redis
+				# Manages jobs currently being processed and handles abandoned job recovery.
+				# Maintains heartbeats for active workers and automatically requeues
+				# jobs from workers that have stopped responding.
 				class ProcessingList
 					REQUEUE = <<~LUA
 						local cursor = "0"
@@ -50,6 +53,12 @@ module Async
 						redis.call('HDEL', KEYS[2], ARGV[1])
 					LUA
 					
+					# Initialize a new processing list manager.
+					# @parameter client [Async::Redis::Client] The Redis client instance.
+					# @parameter key [String] The base Redis key for processing data.
+					# @parameter id [String] The unique server/worker ID.
+					# @parameter ready_list [ReadyList] The ready job queue.
+					# @parameter job_store [JobStore] The job data store.
 					def initialize(client, key, id, ready_list, job_store)
 						@client = client
 						@key = key
@@ -64,36 +73,78 @@ module Async
 						@requeue = @client.script(:load, REQUEUE)
 						@retry = @client.script(:load, RETRY)
 						@complete = @client.script(:load, COMPLETE)
+						
+						@complete_count = 0
 					end
 					
+					# @attribute [String] The base Redis key for this processing list.
 					attr :key
 					
+					# @attribute [String] The Redis key for this worker's heartbeat.
+					attr :heartbeat_key
+					
+					# @attribute [Integer] The total count of all jobs completed by this worker.
+					attr :complete_count
+					
+					# @returns [Integer] The number of jobs currently being processed by this worker.
+					def size
+						@client.llen(@pending_key)
+					end
+					
+					# Fetch the next job from the ready queue, moving it to this worker's pending list.
+					# This is a blocking operation that waits until a job is available.
+					# @returns [String, nil] The job ID, or nil if no job is available.
 					def fetch
 						@client.brpoplpush(@ready_list.key, @pending_key, 0)
 					end
 					
+					# Mark a job as completed, removing it from the pending list and job store.
+					# @parameter id [String] The job ID to complete.
 					def complete(id)
+						@complete_count += 1
+						
 						@client.evalsha(@complete, 2, @pending_key, @job_store.key, id)
 					end
 					
+					# Retry a failed job by moving it back to the ready queue.
+					# @parameter id [String] The job ID to retry.
 					def retry(id)
 						Console.warn(self, "Retrying job: #{id}")
 						@client.evalsha(@retry, 2, @pending_key, @ready_list.key, id)
 					end
 					
+					# Update heartbeat and requeue any abandoned jobs from inactive workers.
+					# @parameter start_time [Float] The start time for calculating uptime.
+					# @parameter delay [Numeric] The heartbeat update interval.
+					# @parameter factor [Numeric] The heartbeat expiration factor.
+					# @returns [Integer] The number of jobs requeued from abandoned workers.
+					def requeue(start_time, delay, factor)
+						uptime = (Time.now.to_f - start_time).round(2)
+						expiry = (delay*factor).ceil
+						@client.set(@heartbeat_key, JSON.dump(uptime: uptime), seconds: expiry)
+						
+						# Requeue any jobs that have been abandoned:
+						count = @client.evalsha(@requeue, 2, @key, @ready_list.key)
+						
+						return count
+					end
+					
+					# Start the background heartbeat and abandoned job recovery task.
+					# @parameter delay [Integer] The heartbeat update interval in seconds.
+					# @parameter factor [Integer] The heartbeat expiration factor.
+					# @parameter parent [Async::Task] The parent task to run the background loop in.
+					# @returns [Async::Task] The background processing task.
 					def start(delay: 5, factor: 2, parent: Async::Task.current)
-						heartbeat_key = "#{@key}:#{@id}"
 						start_time = Time.now.to_f
 						
-						parent.async do
+						parent.async do |task|
 							while true
-								uptime = (Time.now.to_f - start_time).round(2)
-								@client.set(heartbeat_key, JSON.dump(uptime: uptime), seconds: delay*factor)
-								
-								# Requeue any jobs that have been abandoned:
-								count = @client.evalsha(@requeue, 2, @key, @ready_list.key)
-								if count > 0
-									Console.warn(self, "Requeued #{count} abandoned jobs.")
+								task.defer_stop do
+									count = self.requeue(start_time, delay, factor)
+									
+									if count > 0
+										Console.warn(self, "Requeued #{count} abandoned jobs.")
+									end
 								end
 								
 								sleep(delay)

data/lib/async/job/processor/redis/ready_list.rb

@@ -7,12 +7,17 @@ module Async
 	module Job
 		module Processor
 			module Redis
+				# Manages the queue of jobs ready for immediate processing.
+				# Jobs are stored in Redis lists with FIFO (first-in, first-out) ordering.
 				class ReadyList
 					ADD = <<~LUA
 						redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
 						redis.call('LPUSH', KEYS[2], ARGV[1])
 					LUA
 					
+					# Initialize a new ready list manager.
+					# @parameter client [Async::Redis::Client] The Redis client instance.
+					# @parameter key [String] The Redis key for the ready job list.
 					def initialize(client, key)
 						@client = client
 						@key = key
@@ -20,8 +25,18 @@ module Async
 						@add = @client.script(:load, ADD)
 					end
 					
+					# @attribute [String] The Redis key for this ready list.
 					attr :key
 					
+					# @returns [Integer] The number of jobs currently in the ready list.
+					def size
+						@client.llen(@key)
+					end
+					
+					# Add a new job to the ready queue.
+					# @parameter job [String] The serialized job data.
+					# @parameter job_store [JobStore] The job store to save the job data.
+					# @returns [String] The unique job ID.
 					def add(job, job_store)
 						id = SecureRandom.uuid
 						

data/lib/async/job/processor/redis/server.rb

@@ -1,25 +1,35 @@
 # frozen_string_literal: true
 
 # Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
+# Copyright, 2024-2025, by Samuel Williams.
 
-require 'async/idler'
-require 'async/job/coder'
-require 'async/job/processor/generic'
+require "async/idler"
+require "async/job/coder"
+require "async/job/processor/generic"
 
-require 'securerandom'
+require "securerandom"
 
-require_relative 'delayed_jobs'
-require_relative 'job_store'
-require_relative 'processing_list'
-require_relative 'ready_list'
+require_relative "delayed_jobs"
+require_relative "job_store"
+require_relative "processing_list"
+require_relative "ready_list"
 
 module Async
 	module Job
 		module Processor
 			module Redis
+				# Redis-backed job processor server.
+				# Manages job queues using Redis for distributed job processing across multiple workers.
+				# Handles immediate jobs, delayed jobs, and job retry/recovery mechanisms.
 				class Server < Generic
-					def initialize(delegate, client, prefix: 'async-job', coder: Coder::DEFAULT, resolution: 10, parent: nil)
+					# Initialize a new Redis job processor server.
+					# @parameter delegate [Object] The delegate object that will process jobs.
+					# @parameter client [Async::Redis::Client] The Redis client instance.
+					# @parameter prefix [String] The Redis key prefix for job data.
+					# @parameter coder [Async::Job::Coder] The job serialization codec.
+					# @parameter resolution [Integer] The resolution in seconds for delayed job processing.
+					# @parameter parent [Async::Task] The parent task for background processing.
+					def initialize(delegate, client, prefix: "async-job", coder: Coder::DEFAULT, resolution: 10, parent: nil)
 						super(delegate)
 						
 						@id = SecureRandom.uuid
@@ -36,6 +46,8 @@ module Async
 						@parent = parent || Async::Idler.new
 					end
 					
+					# Start the job processing loop immediately.
+					# @returns [Async::Task | false] The processing task or false if already started.
 					def start!
 						return false if @task
 						
@@ -52,6 +64,8 @@ module Async
 						end
 					end
 					
+					# Start the server and all background processing tasks.
+					# Initializes delayed job processing, abandoned job recovery, and the main processing loop.
 					def start
 						super
 						
@@ -64,12 +78,31 @@ module Async
 						self.start!
 					end
 					
+					# Stop the server and all background processing tasks.
 					def stop
 						@task&.stop
 						
 						super
 					end
 					
+					# Generates a human-readable string representing the current statistics.
+					#
+					# e.g. `R=3.42K D=1.23K P=7/2.34K``
+					#
+					# This can be interpreted as:
+					#
+					# - R: Number of jobs in the ready list
+					# - D: Number of jobs in the delayed queue
+					# - P: Number of jobs currently being processed / total number of completed jobs.
+					#
+					# @returns [String] A string representing the current statistics.
+					def status_string
+						"R=#{format_count(@ready_list.size)} D=#{format_count(@delayed_jobs.size)} P=#{format_count(@processing_list.size)}/#{format_count(@processing_list.complete_count)}"
+					end
+					
+					# Submit a new job for processing.
+					# Jobs with a scheduled_at time are queued for delayed processing, while immediate jobs are added to the ready queue.
+					# @parameter job [Hash] The job data to process.
 					def call(job)
 						scheduled_at = Coder::Time(job["scheduled_at"])
 						
@@ -97,12 +130,24 @@ module Async
 							@delegate.call(job)
 							@processing_list.complete(id)
 						rescue => error
-							Console::Event::Failure.for(error).emit(self, "Job failed with error!", id: id)
+							Console.error(self, "Job failed with error!", id: id, exception: error)
 							@processing_list.retry(id)
 						end
 					ensure
 						@processing_list.retry(_id) if _id
 					end
+					
+					private
+					
+					def format_count(value)
+						if value > 1_000_000
+							"#{(value/1_000_000.0).round(2)}M"
+						elsif value > 1_000
+							"#{(value/1_000.0).round(2)}K"
+						else
+							value
+						end
+					end
 				end
 			end
 		end

data/lib/async/job/processor/redis/version.rb

@@ -7,7 +7,7 @@ module Async
 	module Job
 		module Processor
 			module Redis
-				VERSION = "0.1.0"
+				VERSION = "0.3.0"
 			end
 		end
 	end

data/lib/async/job/processor/redis.rb

@@ -3,13 +3,23 @@
 # Released under the MIT License.
 # Copyright, 2024, by Samuel Williams.
 
-require_relative 'redis/server'
-require 'async/redis/client'
+require_relative "redis/server"
+require "async/redis/client"
 
+# @namespace
 module Async
+	# @namespace
 	module Job
+		# @namespace
 		module Processor
+			# Redis-based job processor implementation.
+			# Provides distributed job processing capabilities using Redis as the backend.
 			module Redis
+				# Create a new Redis job processor server.
+				# @parameter delegate [Object] The delegate object that will process jobs.
+				# @parameter endpoint [Async::Redis::Endpoint] The Redis endpoint to connect to.
+				# @parameter options [Hash] Additional options passed to the server.
+				# @returns [Server] A new Redis job processor server instance.
 				def self.new(delegate, endpoint: Async::Redis.local_endpoint, **options)
 					client = Async::Redis::Client.new(endpoint)
 					return Server.new(delegate, client, **options)

data/license.md

@@ -1,6 +1,6 @@
 # MIT License
 
-Copyright, 2024, by Samuel Williams.  
+Copyright, 2024-2025, by Samuel Williams.  
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/readme.md

@@ -12,8 +12,26 @@ Please see the [project documentation](https://socketry.github.io/async-job-proc
 
   - [Redis Queue](https://socketry.github.io/async-job-processor-redis/guides/redis-queue/index) - This guide gives a brief overview of the implementation of the Redis queue.
 
+## Releases
+
+Please see the [project releases](https://socketry.github.io/async-job-processor-redis/releases/index) for all releases.
+
+### v0.3.0
+
+  - Add `Async::Job::Processor::Redis::Server#status_string` method to return a string with the current job counts.
+
+### v0.2.0
+
+  - Achieve 100% documentation coverage.
+  - Achieve 100% test coverage.
+
+### v0.1.0
+
+  - Initial release of async-job-processor-redis, migrated from the async-job project.
+
 ## See Also
 
+  - [async-job](https://github.com/socketry/async-job) - Asynchronous job processing framework.
   - [async-job-adapter-active\_job](https://github.com/socketry/async-job-adapter-active_job) - ActiveJob adapter for `async-job`.
 
 ## Contributing

data/releases.md

@@ -0,0 +1,14 @@
+# Releases
+
+## v0.3.0
+
+  - Add `Async::Job::Processor::Redis::Server#status_string` method to return a string with the current job counts.
+
+## v0.2.0
+
+  - Achieve 100% documentation coverage.
+  - Achieve 100% test coverage.
+
+## v0.1.0
+
+  - Initial release of async-job-processor-redis, migrated from the async-job project.

metadata

@@ -1,11 +1,10 @@
 --- !ruby/object:Gem::Specification
 name: async-job-processor-redis
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Samuel Williams
-autorequire:
 bindir: bin
 cert_chain:
 - |
@@ -37,7 +36,7 @@ cert_chain:
   Q2K9NVun/S785AP05vKkXZEFYxqG6EW012U4oLcFl5MySFajYXRYbuUpH6AY+HP8
   voD0MPg1DssDLKwXyt1eKD/+Fq0bFWhwVM/1XiAXL7lyYUyOq24KHgQ2Csg=
   -----END CERTIFICATE-----
-date: 2024-08-14 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async-job
@@ -67,8 +66,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
-email:
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -82,13 +79,12 @@ files:
 - lib/async/job/processor/redis/version.rb
 - license.md
 - readme.md
-homepage:
+- releases.md
 licenses:
 - MIT
 metadata:
   documentation_uri: https://socketry.github.io/async-job-processor-redis/
   source_code_uri: https://github.com/socketry/async-job-processor-redis
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -96,15 +92,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: A asynchronous job queue for Ruby.
 test_files: []