async-job 0.9.1 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68846d22341ca9edf95a092dcb92f065282cc2f51a5bf8b2e7849665a557d5ff
-  data.tar.gz: f625a33575bed354ae7b00995af8091160dc58259089241fe58d5d58209feb29
+  metadata.gz: 225fd8c2429fb1d8cdcee418e89c4c8ecb09da8fbda4b46331973a92ae186e75
+  data.tar.gz: 2c555bc6168739cf76188a536fd57e0b3b89d900d82ca2a661e03ada8e1078cf
 SHA512:
-  metadata.gz: 67d0c64d4245fd92b9b82f95d46d0f17fef1963f38971619c1b750ea41902810fe5a354027ebf7cd217929167fbec0fe084c4dadc0910bfa7bd15a626444abd7
-  data.tar.gz: 5ee76e100e7232c9d0ca6b5d7fdc42507ffae422879e30d2ba2b48e67e3795e4295911c22daf4e3e28832d4bad77dbf3b35561dcad330a3abba90a878059490e
+  metadata.gz: 42a545a3279caf7c5ed5de25593993119375693184452eaf9a1794dedd197a27a0859e91325dbe0831c0734646f05a4d915dc479dfe2fcc850d577d39ed32683
+  data.tar.gz: '0063657068e88a81be3f7978ae4963dba29ffa0536bd48db202906a0a28608dc2971fc269a04873c1552de043e90364baf9ef5986718139d2a4a839c695cda5c'

data/lib/async/job/builder.rb

@@ -40,7 +40,7 @@ module Async
 				@delegate = delegate
 			end
 			
-			def build(delegate = @delegate, &block)
+			def build(delegate = @delegate)
 				# We then wrap the delegate with the middleware in reverse order:
 				@dequeue.reverse_each do |middleware|
 					delegate = middleware.call(delegate)
@@ -53,10 +53,6 @@ module Async
 					client = middleware.call(client)
 				end
 				
-				if block_given?
-					client = yield(client) || client
-				end
-				
 				return Queue.new(client, server, @delegate)
 			end
 		end

data/lib/async/job/version.rb

@@ -5,6 +5,6 @@
 
 module Async
 	module Job
-		VERSION = "0.9.1"
+		VERSION = "0.10.0"
 	end
 end

data/readme.md

@@ -10,10 +10,13 @@ Please see the [project documentation](https://socketry.github.io/async-job/) fo
 
   - [Getting Started](https://socketry.github.io/async-job/guides/getting-started/index) - This guide gives you an overview of the `async-job` gem and explains the core concepts.
 
-  - [Redis Processor](https://socketry.github.io/async-job/guides/redis-processor/index) - This guide gives a brief overview of the implementation of the Redis queue.
+  - [Inline Queue](https://socketry.github.io/async-job/guides/inline-queue/index) - This guide explains how to use the Inline queue to execute background jobs.
+
+  - [Aggregate Queue](https://socketry.github.io/async-job/guides/aggregate-queue/index) - This guide explains how to use the Aggregate queue to reduce input latency and improve the performance of your application.
 
 ## See Also
 
+  - [async-job-processor-redis](https://github.com/socketry/async-job-processor-redis) - Redis processor for `async-job` (similar to Sidekiq).
   - [async-job-adapter-active\_job](https://github.com/socketry/async-job-adapter-active_job) - ActiveJob adapter for `async-job`.
 
 ## Contributing

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: async-job
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.10.0
 platform: ruby
 authors:
 - Samuel Williams
@@ -38,7 +38,7 @@ cert_chain:
   Q2K9NVun/S785AP05vKkXZEFYxqG6EW012U4oLcFl5MySFajYXRYbuUpH6AY+HP8
   voD0MPg1DssDLKwXyt1eKD/+Fq0bFWhwVM/1XiAXL7lyYUyOq24KHgQ2Csg=
   -----END CERTIFICATE-----
-date: 2024-08-09 00:00:00.000000000 Z
+date: 2024-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async
@@ -54,20 +54,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.9'
-- !ruby/object:Gem::Dependency
-  name: async-redis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description:
 email:
 executables: []
@@ -86,12 +72,6 @@ files:
 - lib/async/job/processor/delayed.rb
 - lib/async/job/processor/generic.rb
 - lib/async/job/processor/inline.rb
-- lib/async/job/processor/redis.rb
-- lib/async/job/processor/redis/delayed_jobs.rb
-- lib/async/job/processor/redis/job_store.rb
-- lib/async/job/processor/redis/processing_list.rb
-- lib/async/job/processor/redis/ready_list.rb
-- lib/async/job/processor/redis/server.rb
 - lib/async/job/queue.rb
 - lib/async/job/version.rb
 - license.md
@@ -120,5 +100,5 @@ requirements: []
 rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
-summary: A asynchronous job queue for Ruby.
+summary: An asynchronous job queue for Ruby.
 test_files: []

data/lib/async/job/processor/redis/delayed_jobs.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-# Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
-
-module Async
-	module Job
-		module Processor
-			module Redis
-				class DelayedJobs
-					ADD = <<~LUA
-						redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
-						redis.call('ZADD', KEYS[2], ARGV[3], ARGV[1])
-					LUA
-					
-					MOVE = <<~LUA
-						local jobs = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1])
-						redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, ARGV[1])
-						if #jobs > 0 then
-							redis.call('LPUSH', KEYS[2], unpack(jobs))
-						end
-						return #jobs
-					LUA
-					
-					def initialize(client, key)
-						@client = client
-						@key = key
-						
-						@add = @client.script(:load, ADD)
-						@move = @client.script(:load, MOVE)
-					end
-					
-					def start(ready_list, resolution: 10, parent: Async::Task.current)
-						parent.async do
-							while true
-								count = move(destination: ready_list.key)
-								
-								if count > 0
-									Console.debug(self, "Moved #{count} delayed jobs to ready list.")
-								end
-								
-								sleep(resolution)
-							end
-						end
-					end
-					
-					attr :key
-					
-					def add(job, timestamp, job_store)
-						id = SecureRandom.uuid
-						
-						@client.evalsha(@add, 2, job_store.key, @key, id, job, timestamp.to_f)
-						
-						return id
-					end
-					
-					def move(destination:, now: Time.now.to_i)
-						@client.evalsha(@move, 2, @key, destination, now)
-					end
-				end
-			end
-		end
-	end
-end

data/lib/async/job/processor/redis/job_store.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-# Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
-
-module Async
-	module Job
-		module Processor
-			module Redis
-				class JobStore
-					def initialize(client, key)
-						@client = client
-						@key = key
-					end
-					
-					attr :key
-					
-					def get(id)
-						@client.hget(@key, id)
-					end
-				end
-			end
-		end
-	end
-end

data/lib/async/job/processor/redis/processing_list.rb

@@ -1,107 +0,0 @@
-# frozen_string_literal: true
-
-# Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
-
-module Async
-	module Job
-		module Processor
-			module Redis
-				class ProcessingList
-					REQUEUE = <<~LUA
-						local cursor = "0"
-						local count = 0
-						
-						repeat
-							-- Scan through all known server id -> job id mappings and requeue any jobs that have been abandoned:
-							local result = redis.call('SCAN', cursor, 'MATCH', KEYS[1]..':*:pending')
-							cursor = result[1]
-							for _, pending_key in pairs(result[2]) do
-								-- Check if the server is still active:
-								local server_key = KEYS[1]..":"..pending_key:match("([^:]+):pending")
-								local state = redis.call('GET', server_key)
-								if state == false then
-									while true do
-										-- Requeue any pending jobs:
-										local result = redis.call('RPOPLPUSH', pending_key, KEYS[2])
-										
-										if result == false then
-											-- Delete the pending list:
-											redis.call('DEL', pending_key)
-											break
-										end
-										
-										count = count + 1
-									end
-								end
-							end
-						until cursor == "0"
-						
-						return count
-					LUA
-					
-					RETRY = <<~LUA
-						redis.call('LREM', KEYS[1], 1, ARGV[1])
-						redis.call('LPUSH', KEYS[2], ARGV[1])
-					LUA
-					
-					COMPLETE = <<~LUA
-						redis.call('LREM', KEYS[1], 1, ARGV[1])
-						redis.call('HDEL', KEYS[2], ARGV[1])
-					LUA
-					
-					def initialize(client, key, id, ready_list, job_store)
-						@client = client
-						@key = key
-						@id = id
-						
-						@ready_list = ready_list
-						@job_store = job_store
-						
-						@pending_key = "#{@key}:#{@id}:pending"
-						@heartbeat_key = "#{@key}:#{@id}"
-						
-						@requeue = @client.script(:load, REQUEUE)
-						@retry = @client.script(:load, RETRY)
-						@complete = @client.script(:load, COMPLETE)
-					end
-					
-					attr :key
-					
-					def fetch
-						@client.brpoplpush(@ready_list.key, @pending_key, 0)
-					end
-					
-					def complete(id)
-						@client.evalsha(@complete, 2, @pending_key, @job_store.key, id)
-					end
-					
-					def retry(id)
-						Console.warn(self, "Retrying job: #{id}")
-						@client.evalsha(@retry, 2, @pending_key, @ready_list.key, id)
-					end
-					
-					def start(delay: 5, factor: 2, parent: Async::Task.current)
-						heartbeat_key = "#{@key}:#{@id}"
-						start_time = Time.now.to_f
-						
-						parent.async do
-							while true
-								uptime = (Time.now.to_f - start_time).round(2)
-								@client.set(heartbeat_key, JSON.dump(uptime: uptime), seconds: delay*factor)
-								
-								# Requeue any jobs that have been abandoned:
-								count = @client.evalsha(@requeue, 2, @key, @ready_list.key)
-								if count > 0
-									Console.warn(self, "Requeued #{count} abandoned jobs.")
-								end
-								
-								sleep(delay)
-							end
-						end
-					end
-				end
-			end
-		end
-	end
-end

data/lib/async/job/processor/redis/ready_list.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-# Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
-
-module Async
-	module Job
-		module Processor
-			module Redis
-				class ReadyList
-					ADD = <<~LUA
-						redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
-						redis.call('LPUSH', KEYS[2], ARGV[1])
-					LUA
-					
-					def initialize(client, key)
-						@client = client
-						@key = key
-						
-						@add = @client.script(:load, ADD)
-					end
-					
-					attr :key
-					
-					def add(job, job_store)
-						id = SecureRandom.uuid
-						
-						@client.evalsha(@add, 2, job_store.key, @key, id, job)
-						
-						return id
-					end
-				end
-			end
-		end
-	end
-end

data/lib/async/job/processor/redis/server.rb

@@ -1,104 +0,0 @@
-# frozen_string_literal: true
-
-# Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
-
-require_relative 'delayed_jobs'
-require_relative 'job_store'
-require_relative 'processing_list'
-require_relative 'ready_list'
-require_relative '../../coder'
-require_relative '../generic'
-
-require 'securerandom'
-require 'async/idler'
-
-module Async
-	module Job
-		module Processor
-			module Redis
-				class Server < Generic
-					def initialize(delegate, client, prefix: 'async-job', coder: Coder::DEFAULT, resolution: 10, parent: nil)
-						super(delegate)
-						
-						@id = SecureRandom.uuid
-						@client = client
-						@prefix = prefix
-						@coder = coder
-						@resolution = resolution
-						
-						@job_store = JobStore.new(@client, "#{@prefix}:jobs")
-						@delayed_jobs = DelayedJobs.new(@client, "#{@prefix}:delayed")
-						@ready_list = ReadyList.new(@client, "#{@prefix}:ready")
-						@processing_list = ProcessingList.new(@client, "#{@prefix}:processing", @id, @ready_list, @job_store)
-						
-						@parent = parent || Async::Idler.new
-					end
-					
-					def start!
-						return false if @task
-						
-						@task = true
-						
-						@parent.async(transient: true, annotation: self.class.name) do |task|
-							@task = task
-							
-							while true
-								self.dequeue(task)
-							end
-						ensure
-							@task = nil
-						end
-					end
-					
-					def start
-						super
-						
-						# Start the delayed processor, which will move jobs to the ready processor when they are ready:
-						@delayed_jobs.start(@ready_list, resolution: @resolution)
-						
-						# Start the processing processor, which will move jobs to the ready processor when they are abandoned:
-						@processing_list.start
-						
-						self.start!
-					end
-					
-					def stop
-						@task&.stop
-						
-						super
-					end
-					
-					def call(job)
-						scheduled_at = Coder::Time(job["scheduled_at"])
-						
-						if scheduled_at
-							@delayed_jobs.add(@coder.dump(job), scheduled_at, @job_store)
-						else
-							@ready_list.add(@coder.dump(job), @job_store)
-						end
-					end
-					
-					protected
-					
-					def dequeue(parent)
-						_id = @processing_list.fetch
-						
-						parent.async do
-							id = _id; _id = nil
-							
-							job = @coder.load(@job_store.get(id))
-							@delegate.call(job)
-							@processing_list.complete(id)
-						rescue => error
-							Console::Event::Failure.for(error).emit(self, "Job failed with error!", id: id)
-							@processing_list.fail(id)
-						end
-					ensure
-						@processing_list.retry(_id) if _id
-					end
-				end
-			end
-		end
-	end
-end

data/lib/async/job/processor/redis.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-# Released under the MIT License.
-# Copyright, 2024, by Samuel Williams.
-
-require_relative 'redis/server'
-require 'async/redis/client'
-
-module Async
-	module Job
-		module Processor
-			module Redis
-				def self.new(delegate, endpoint: Async::Redis.local_endpoint, **options)
-					client = Async::Redis::Client.new(endpoint)
-					return Server.new(delegate, client, **options)
-				end
-			end
-		end
-	end
-end