async-job-processor-redis 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf246106a0b45eade2e2316e3d198a80101b89e83619ae6ebf753a1a153a853c
-  data.tar.gz: c298e2a87023a73a2014693d09685320ba20b875b1d872eea764ba3d97054cf8
+  metadata.gz: 19f21b96ed37f056471ebbb1935460d1be5a2714b4d1e50f9c82f8587587be8c
+  data.tar.gz: 8e1d7ba47c50236eeeb43e2b1ac38cdeba3cc1812ec028999aae7ccd87c91c66
 SHA512:
-  metadata.gz: c343b68ed4fdefa6473737c65ea4e5d4303573eb60dca28a9f9d7ac36fcb5a35fdf7c37b91d74900aedb335d62701497398a6336073886305998ca3d6668f3c4
-  data.tar.gz: 17c8c94cdd5a3c08bd09d10c77e129bd9bcffc475f444411afa79fcd8f94f3bce6e80e03c391cb4fb2aadf3c729916fb1a8c1ed9ba44db786bc86c0e0d573300
+  metadata.gz: a7600f75d5a7a9b5b0f0d93e8896889b80853a92b59cfc02413ea12c9d02034108c27961d4878dffe4dddcb42eafd9d25825b985842f03a6514e5bd90004b3da
+  data.tar.gz: 50fa8c45039c3505aea3f8e3dc6df6a1c442994d5ca4e7eb083d851af7d3bfe9e59a23b74a89e955360bdb6b8d951e14ba226364ebdd78a9f862b40e457a17c8

data/context/getting-started.md

@@ -0,0 +1,41 @@
+# Getting Started
+
+This guide gives you an overview of the `async-job-processor-redis` gem.
+
+## Installation
+
+Add the gem to your project:
+
+``` shell
+$ bundle add async-job-processor-redis
+```
+
+## Usage
+
+Here is a full example of the job queue:
+
+``` ruby
+require "async"
+require "async/job"
+require "async/job/processor/redis"
+
+Async do
+	buffer = Async::Job::Buffer.new
+	
+	queue = Async::Job::Builder.build(buffer) do
+		dequeue Async::Job::Processor::Redis
+	end
+	
+	# Run the server:
+	server = Async{queue.start}
+	
+	# Enqueue a job:
+	queue.call({message: "Hello, World!"})
+	
+	# Wait for the job to complete:
+	job = buffer.pop
+	pp job: job
+	
+	server.stop
+end
+```

data/context/index.yaml

@@ -0,0 +1,16 @@
+# Automatically generated context index for Utopia::Project guides.
+# Do not edit then files in this directory directly, instead edit the guides and then run `bake utopia:project:agent:context:update`.
+---
+description: A asynchronous job queue for Ruby.
+metadata:
+  documentation_uri: https://socketry.github.io/async-job-processor-redis/
+  source_code_uri: https://github.com/socketry/async-job-processor-redis
+files:
+- path: getting-started.md
+  title: Getting Started
+  description: This guide gives you an overview of the `async-job-processor-redis`
+    gem.
+- path: redis-queue.md
+  title: Redis Queue
+  description: This guide gives a brief overview of the implementation of the Redis
+    queue.

data/context/redis-queue.md

@@ -0,0 +1,19 @@
+# Redis Queue
+
+This guide gives a brief overview of the implementation of the Redis queue.
+
+## Overview
+
+The Redis queue plays a pivotal role in facilitating a sophisticated and reliable job queue architecture, designed to handle diverse processing needs with efficiency and resilience. The architecture is thoughtfully split into three distinct components, each serving a critical function in the lifecycle of a job: the ready queue, the delayed queue, and the processing queue.
+
+## Ready Queue
+
+The ready queue is where jobs that are immediately available for processing are stored. When a job is submitted and is ready to be executed without any delay, it is placed into this queue. Worker processes constantly listen for new jobs on the ready queue, dequeuing and executing them as soon as they become available. This queue operates on a FIFO (First In, First Out) basis, ensuring that jobs are processed in the order they were received.
+
+## Delayed Queue
+
+The delayed queue holds jobs that are not meant to be executed immediately but at a specified future time. This functionality is crucial for tasks that need to be executed at a later stage, such as scheduled notifications or time-dependent processes. Jobs in the delayed queue are sorted according to their execution time. When possible, they are moved to the ready queue to be executed by the next available worker. This transition is managed through Redis's sorted sets, allowing efficient retrieval and management of timed events.
+
+## Processing Queue
+
+Once a job is dequeued from the ready queue, it enters the processing queue, signifying that it is currently being executed by a worker. The processing queue is crucial for tracking the progress of jobs and for ensuring that jobs can be retried or recovered in case of worker failure. Each worker emits a heartbeat, and if a worker fails to emit a heartbeat within a specified time, any jobs associated with that worker are automatically moved back to the ready queue for reprocessing.

data/lib/async/job/processor/redis/delayed_jobs.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
+# Copyright, 2024-2025, by Samuel Williams.
 
 module Async
 	module Job
@@ -36,6 +36,11 @@ module Async
 						@move = @client.script(:load, MOVE)
 					end
 					
+					# @returns [Integer] The number of jobs currently in the delayed queue.
+					def size
+						@client.zcard(@key)
+					end
+					
 					# Start the background task that moves ready delayed jobs to the ready queue.
 					# @parameter ready_list [ReadyList] The ready list to move jobs to.
 					# @parameter resolution [Integer] The check interval in seconds.

data/lib/async/job/processor/redis/job_store.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
+# Copyright, 2024-2025, by Samuel Williams.
 
 module Async
 	module Job

data/lib/async/job/processor/redis/processing_list.rb

@@ -73,6 +73,8 @@ module Async
 						@requeue = @client.script(:load, REQUEUE)
 						@retry = @client.script(:load, RETRY)
 						@complete = @client.script(:load, COMPLETE)
+						
+						@complete_count = 0
 					end
 					
 					# @attribute [String] The base Redis key for this processing list.
@@ -81,6 +83,14 @@ module Async
 					# @attribute [String] The Redis key for this worker's heartbeat.
 					attr :heartbeat_key
 					
+					# @attribute [Integer] The total count of all jobs completed by this worker.
+					attr :complete_count
+					
+					# @returns [Integer] The number of jobs currently being processed by this worker.
+					def size
+						@client.llen(@pending_key)
+					end
+					
 					# Fetch the next job from the ready queue, moving it to this worker's pending list.
 					# This is a blocking operation that waits until a job is available.
 					# @returns [String, nil] The job ID, or nil if no job is available.
@@ -91,6 +101,8 @@ module Async
 					# Mark a job as completed, removing it from the pending list and job store.
 					# @parameter id [String] The job ID to complete.
 					def complete(id)
+						@complete_count += 1
+						
 						@client.evalsha(@complete, 2, @pending_key, @job_store.key, id)
 					end
 					

data/lib/async/job/processor/redis/ready_list.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
+# Copyright, 2024-2025, by Samuel Williams.
 
 module Async
 	module Job
@@ -28,6 +28,11 @@ module Async
 					# @attribute [String] The Redis key for this ready list.
 					attr :key
 					
+					# @returns [Integer] The number of jobs currently in the ready list.
+					def size
+						@client.llen(@key)
+					end
+					
 					# Add a new job to the ready queue.
 					# @parameter job [String] The serialized job data.
 					# @parameter job_store [JobStore] The job store to save the job data.

data/lib/async/job/processor/redis/server.rb

@@ -85,6 +85,21 @@ module Async
 						super
 					end
 					
+					# Generates a human-readable string representing the current statistics.
+					#
+					# e.g. `R=3.42K D=1.23K P=7/2.34K``
+					#
+					# This can be interpreted as:
+					#
+					# - R: Number of jobs in the ready list
+					# - D: Number of jobs in the delayed queue
+					# - P: Number of jobs currently being processed / total number of completed jobs.
+					#
+					# @returns [String] A string representing the current statistics.
+					def status_string
+						"R=#{format_count(@ready_list.size)} D=#{format_count(@delayed_jobs.size)} P=#{format_count(@processing_list.size)}/#{format_count(@processing_list.complete_count)}"
+					end
+					
 					# Submit a new job for processing.
 					# Jobs with a scheduled_at time are queued for delayed processing, while immediate jobs are added to the ready queue.
 					# @parameter job [Hash] The job data to process.
@@ -121,6 +136,18 @@ module Async
 					ensure
 						@processing_list.retry(_id) if _id
 					end
+					
+					private
+					
+					def format_count(value)
+						if value > 1_000_000
+							"#{(value/1_000_000.0).round(2)}M"
+						elsif value > 1_000
+							"#{(value/1_000.0).round(2)}K"
+						else
+							value
+						end
+					end
 				end
 			end
 		end

data/lib/async/job/processor/redis/version.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
 # Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
+# Copyright, 2024-2025, by Samuel Williams.
 
 module Async
 	module Job
 		module Processor
 			module Redis
-				VERSION = "0.2.0"
+				VERSION = "0.3.1"
 			end
 		end
 	end

data/lib/async/job/processor/redis.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
+# Copyright, 2024-2025, by Samuel Williams.
 
 require_relative "redis/server"
 require "async/redis/client"

data/readme.md

@@ -16,6 +16,10 @@ Please see the [project documentation](https://socketry.github.io/async-job-proc
 
 Please see the [project releases](https://socketry.github.io/async-job-processor-redis/releases/index) for all releases.
 
+### v0.3.0
+
+  - Add `Async::Job::Processor::Redis::Server#status_string` method to return a string with the current job counts.
+
 ### v0.2.0
 
   - Achieve 100% documentation coverage.

data/releases.md

@@ -1,5 +1,9 @@
 # Releases
 
+## v0.3.0
+
+  - Add `Async::Job::Processor::Redis::Server#status_string` method to return a string with the current job counts.
+
 ## v0.2.0
 
   - Achieve 100% documentation coverage.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: async-job-processor-redis
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Samuel Williams
@@ -70,6 +70,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- context/getting-started.md
+- context/index.yaml
+- context/redis-queue.md
 - lib/async/job/processor/redis.rb
 - lib/async/job/processor/redis/delayed_jobs.rb
 - lib/async/job/processor/redis/job_store.rb
@@ -99,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: A asynchronous job queue for Ruby.
 test_files: []