async-job-processor-redis 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '095272ad3206a878ec049e641b3632181be1f62ca055398230e14dc3fd242d14'
+  data.tar.gz: b10ab3013a961aca3cbecaa8f51265f4aa8a20aa3eb43031b6a94079d23553f8
+SHA512:
+  metadata.gz: '082e58c7f2f95eff36c022f68ce14a0a1b00e01463bd4f3397a94422b4a4f9011b106d09b21462ce5aeab8dd711af85e8168d1c4423357137fc564720ecb997f'
+  data.tar.gz: e3bbfd7290feda5b88c941fef9bd4e7ea191c8a800dd97f330f3d1c7032efb869aa73d1a08419b957a541fb78e3d482f6efb816e327c659fd1c26c433cd22479

data/lib/async/job/processor/redis/delayed_jobs.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		module Processor
+			module Redis
+				class DelayedJobs
+					ADD = <<~LUA
+						redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+						redis.call('ZADD', KEYS[2], ARGV[3], ARGV[1])
+					LUA
+					
+					MOVE = <<~LUA
+						local jobs = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1])
+						redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, ARGV[1])
+						if #jobs > 0 then
+							redis.call('LPUSH', KEYS[2], unpack(jobs))
+						end
+						return #jobs
+					LUA
+					
+					def initialize(client, key)
+						@client = client
+						@key = key
+						
+						@add = @client.script(:load, ADD)
+						@move = @client.script(:load, MOVE)
+					end
+					
+					def start(ready_list, resolution: 10, parent: Async::Task.current)
+						parent.async do
+							while true
+								count = move(destination: ready_list.key)
+								
+								if count > 0
+									Console.debug(self, "Moved #{count} delayed jobs to ready list.")
+								end
+								
+								sleep(resolution)
+							end
+						end
+					end
+					
+					attr :key
+					
+					def add(job, timestamp, job_store)
+						id = SecureRandom.uuid
+						
+						@client.evalsha(@add, 2, job_store.key, @key, id, job, timestamp.to_f)
+						
+						return id
+					end
+					
+					def move(destination:, now: Time.now.to_i)
+						@client.evalsha(@move, 2, @key, destination, now)
+					end
+				end
+			end
+		end
+	end
+end

data/lib/async/job/processor/redis/job_store.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		module Processor
+			module Redis
+				class JobStore
+					def initialize(client, key)
+						@client = client
+						@key = key
+					end
+					
+					attr :key
+					
+					def get(id)
+						@client.hget(@key, id)
+					end
+				end
+			end
+		end
+	end
+end

data/lib/async/job/processor/redis/processing_list.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		module Processor
+			module Redis
+				class ProcessingList
+					REQUEUE = <<~LUA
+						local cursor = "0"
+						local count = 0
+						
+						repeat
+							-- Scan through all known server id -> job id mappings and requeue any jobs that have been abandoned:
+							local result = redis.call('SCAN', cursor, 'MATCH', KEYS[1]..':*:pending')
+							cursor = result[1]
+							for _, pending_key in pairs(result[2]) do
+								-- Check if the server is still active:
+								local server_key = KEYS[1]..":"..pending_key:match("([^:]+):pending")
+								local state = redis.call('GET', server_key)
+								if state == false then
+									while true do
+										-- Requeue any pending jobs:
+										local result = redis.call('RPOPLPUSH', pending_key, KEYS[2])
+										
+										if result == false then
+											-- Delete the pending list:
+											redis.call('DEL', pending_key)
+											break
+										end
+										
+										count = count + 1
+									end
+								end
+							end
+						until cursor == "0"
+						
+						return count
+					LUA
+					
+					RETRY = <<~LUA
+						redis.call('LREM', KEYS[1], 1, ARGV[1])
+						redis.call('LPUSH', KEYS[2], ARGV[1])
+					LUA
+					
+					COMPLETE = <<~LUA
+						redis.call('LREM', KEYS[1], 1, ARGV[1])
+						redis.call('HDEL', KEYS[2], ARGV[1])
+					LUA
+					
+					def initialize(client, key, id, ready_list, job_store)
+						@client = client
+						@key = key
+						@id = id
+						
+						@ready_list = ready_list
+						@job_store = job_store
+						
+						@pending_key = "#{@key}:#{@id}:pending"
+						@heartbeat_key = "#{@key}:#{@id}"
+						
+						@requeue = @client.script(:load, REQUEUE)
+						@retry = @client.script(:load, RETRY)
+						@complete = @client.script(:load, COMPLETE)
+					end
+					
+					attr :key
+					
+					def fetch
+						@client.brpoplpush(@ready_list.key, @pending_key, 0)
+					end
+					
+					def complete(id)
+						@client.evalsha(@complete, 2, @pending_key, @job_store.key, id)
+					end
+					
+					def retry(id)
+						Console.warn(self, "Retrying job: #{id}")
+						@client.evalsha(@retry, 2, @pending_key, @ready_list.key, id)
+					end
+					
+					def start(delay: 5, factor: 2, parent: Async::Task.current)
+						heartbeat_key = "#{@key}:#{@id}"
+						start_time = Time.now.to_f
+						
+						parent.async do
+							while true
+								uptime = (Time.now.to_f - start_time).round(2)
+								@client.set(heartbeat_key, JSON.dump(uptime: uptime), seconds: delay*factor)
+								
+								# Requeue any jobs that have been abandoned:
+								count = @client.evalsha(@requeue, 2, @key, @ready_list.key)
+								if count > 0
+									Console.warn(self, "Requeued #{count} abandoned jobs.")
+								end
+								
+								sleep(delay)
+							end
+						end
+					end
+				end
+			end
+		end
+	end
+end

data/lib/async/job/processor/redis/ready_list.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		module Processor
+			module Redis
+				class ReadyList
+					ADD = <<~LUA
+						redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+						redis.call('LPUSH', KEYS[2], ARGV[1])
+					LUA
+					
+					def initialize(client, key)
+						@client = client
+						@key = key
+						
+						@add = @client.script(:load, ADD)
+					end
+					
+					attr :key
+					
+					def add(job, job_store)
+						id = SecureRandom.uuid
+						
+						@client.evalsha(@add, 2, job_store.key, @key, id, job)
+						
+						return id
+					end
+				end
+			end
+		end
+	end
+end

data/lib/async/job/processor/redis/server.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+require 'async/idler'
+require 'async/job/coder'
+require 'async/job/processor/generic'
+
+require 'securerandom'
+
+require_relative 'delayed_jobs'
+require_relative 'job_store'
+require_relative 'processing_list'
+require_relative 'ready_list'
+
+module Async
+	module Job
+		module Processor
+			module Redis
+				class Server < Generic
+					def initialize(delegate, client, prefix: 'async-job', coder: Coder::DEFAULT, resolution: 10, parent: nil)
+						super(delegate)
+						
+						@id = SecureRandom.uuid
+						@client = client
+						@prefix = prefix
+						@coder = coder
+						@resolution = resolution
+						
+						@job_store = JobStore.new(@client, "#{@prefix}:jobs")
+						@delayed_jobs = DelayedJobs.new(@client, "#{@prefix}:delayed")
+						@ready_list = ReadyList.new(@client, "#{@prefix}:ready")
+						@processing_list = ProcessingList.new(@client, "#{@prefix}:processing", @id, @ready_list, @job_store)
+						
+						@parent = parent || Async::Idler.new
+					end
+					
+					def start!
+						return false if @task
+						
+						@task = true
+						
+						@parent.async(transient: true, annotation: self.class.name) do |task|
+							@task = task
+							
+							while true
+								self.dequeue(task)
+							end
+						ensure
+							@task = nil
+						end
+					end
+					
+					def start
+						super
+						
+						# Start the delayed processor, which will move jobs to the ready processor when they are ready:
+						@delayed_jobs.start(@ready_list, resolution: @resolution)
+						
+						# Start the processing processor, which will move jobs to the ready processor when they are abandoned:
+						@processing_list.start
+						
+						self.start!
+					end
+					
+					def stop
+						@task&.stop
+						
+						super
+					end
+					
+					def call(job)
+						scheduled_at = Coder::Time(job["scheduled_at"])
+						
+						if scheduled_at
+							@delayed_jobs.add(@coder.dump(job), scheduled_at, @job_store)
+						else
+							@ready_list.add(@coder.dump(job), @job_store)
+						end
+					end
+					
+					protected
+					
+					# Dequeue a job from the ready list and process it.
+					#
+					# If the job fails for any reason, it will be retried.
+					#
+					# If you do not desire this behavior, you should catch exceptions in the delegate.
+					def dequeue(parent)
+						_id = @processing_list.fetch
+						
+						parent.async do
+							id = _id; _id = nil
+							
+							job = @coder.load(@job_store.get(id))
+							@delegate.call(job)
+							@processing_list.complete(id)
+						rescue => error
+							Console::Event::Failure.for(error).emit(self, "Job failed with error!", id: id)
+							@processing_list.retry(id)
+						end
+					ensure
+						@processing_list.retry(_id) if _id
+					end
+				end
+			end
+		end
+	end
+end

data/lib/async/job/processor/redis/version.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		module Processor
+			module Redis
+				VERSION = "0.1.0"
+			end
+		end
+	end
+end

data/lib/async/job/processor/redis.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+require_relative 'redis/server'
+require 'async/redis/client'
+
+module Async
+	module Job
+		module Processor
+			module Redis
+				def self.new(delegate, endpoint: Async::Redis.local_endpoint, **options)
+					client = Async::Redis::Client.new(endpoint)
+					return Server.new(delegate, client, **options)
+				end
+			end
+		end
+	end
+end

data/license.md

@@ -0,0 +1,21 @@
+# MIT License
+
+Copyright, 2024, by Samuel Williams.  
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/readme.md

@@ -0,0 +1,35 @@
+# Async::Job::Processor::Redis
+
+Provides an asynchronous job server.
+
+[![Development Status](https://github.com/socketry/async-job-processor-redis/workflows/Test/badge.svg)](https://github.com/socketry/async-job-processor-redis/actions?workflow=Test)
+
+## Usage
+
+Please see the [project documentation](https://socketry.github.io/async-job-processor-redis/) for more details.
+
+  - [Getting Started](https://socketry.github.io/async-job-processor-redis/guides/getting-started/index) - This guide gives you an overview of the `async-job-processor-redis` gem.
+
+  - [Redis Queue](https://socketry.github.io/async-job-processor-redis/guides/redis-queue/index) - This guide gives a brief overview of the implementation of the Redis queue.
+
+## See Also
+
+  - [async-job-adapter-active\_job](https://github.com/socketry/async-job-adapter-active_job) - ActiveJob adapter for `async-job`.
+
+## Contributing
+
+We welcome contributions to this project.
+
+1.  Fork it.
+2.  Create your feature branch (`git checkout -b my-new-feature`).
+3.  Commit your changes (`git commit -am 'Add some feature'`).
+4.  Push to the branch (`git push origin my-new-feature`).
+5.  Create new Pull Request.
+
+### Developer Certificate of Origin
+
+In order to protect users of this project, we require all contributors to comply with the [Developer Certificate of Origin](https://developercertificate.org/). This ensures that all contributions are properly licensed and attributed.
+
+### Community Guidelines
+
+This project is best served by a collaborative and respectful environment. Treat each other professionally, respect differing viewpoints, and engage constructively. Harassment, discrimination, or harmful behavior is not tolerated. Communicate clearly, listen actively, and support one another. If any issues arise, please inform the project maintainers.

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: async-job-processor-redis
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Samuel Williams
+autorequire:
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIE2DCCA0CgAwIBAgIBATANBgkqhkiG9w0BAQsFADBhMRgwFgYDVQQDDA9zYW11
+  ZWwud2lsbGlhbXMxHTAbBgoJkiaJk/IsZAEZFg1vcmlvbnRyYW5zZmVyMRIwEAYK
+  CZImiZPyLGQBGRYCY28xEjAQBgoJkiaJk/IsZAEZFgJuejAeFw0yMjA4MDYwNDUz
+  MjRaFw0zMjA4MDMwNDUzMjRaMGExGDAWBgNVBAMMD3NhbXVlbC53aWxsaWFtczEd
+  MBsGCgmSJomT8ixkARkWDW9yaW9udHJhbnNmZXIxEjAQBgoJkiaJk/IsZAEZFgJj
+  bzESMBAGCgmSJomT8ixkARkWAm56MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIB
+  igKCAYEAomvSopQXQ24+9DBB6I6jxRI2auu3VVb4nOjmmHq7XWM4u3HL+pni63X2
+  9qZdoq9xt7H+RPbwL28LDpDNflYQXoOhoVhQ37Pjn9YDjl8/4/9xa9+NUpl9XDIW
+  sGkaOY0eqsQm1pEWkHJr3zn/fxoKPZPfaJOglovdxf7dgsHz67Xgd/ka+Wo1YqoE
+  e5AUKRwUuvaUaumAKgPH+4E4oiLXI4T1Ff5Q7xxv6yXvHuYtlMHhYfgNn8iiW8WN
+  XibYXPNP7NtieSQqwR/xM6IRSoyXKuS+ZNGDPUUGk8RoiV/xvVN4LrVm9upSc0ss
+  RZ6qwOQmXCo/lLcDUxJAgG95cPw//sI00tZan75VgsGzSWAOdjQpFM0l4dxvKwHn
+  tUeT3ZsAgt0JnGqNm2Bkz81kG4A2hSyFZTFA8vZGhp+hz+8Q573tAR89y9YJBdYM
+  zp0FM4zwMNEUwgfRzv1tEVVUEXmoFCyhzonUUw4nE4CFu/sE3ffhjKcXcY//qiSW
+  xm4erY3XAgMBAAGjgZowgZcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAwHQYDVR0O
+  BBYEFO9t7XWuFf2SKLmuijgqR4sGDlRsMC4GA1UdEQQnMCWBI3NhbXVlbC53aWxs
+  aWFtc0BvcmlvbnRyYW5zZmVyLmNvLm56MC4GA1UdEgQnMCWBI3NhbXVlbC53aWxs
+  aWFtc0BvcmlvbnRyYW5zZmVyLmNvLm56MA0GCSqGSIb3DQEBCwUAA4IBgQB5sxkE
+  cBsSYwK6fYpM+hA5B5yZY2+L0Z+27jF1pWGgbhPH8/FjjBLVn+VFok3CDpRqwXCl
+  xCO40JEkKdznNy2avOMra6PFiQyOE74kCtv7P+Fdc+FhgqI5lMon6tt9rNeXmnW/
+  c1NaMRdxy999hmRGzUSFjozcCwxpy/LwabxtdXwXgSay4mQ32EDjqR1TixS1+smp
+  8C/NCWgpIfzpHGJsjvmH2wAfKtTTqB9CVKLCWEnCHyCaRVuKkrKjqhYCdmMBqCws
+  JkxfQWC+jBVeG9ZtPhQgZpfhvh+6hMhraUYRQ6XGyvBqEUe+yo6DKIT3MtGE2+CP
+  eX9i9ZWBydWb8/rvmwmX2kkcBbX0hZS1rcR593hGc61JR6lvkGYQ2MYskBveyaxt
+  Q2K9NVun/S785AP05vKkXZEFYxqG6EW012U4oLcFl5MySFajYXRYbuUpH6AY+HP8
+  voD0MPg1DssDLKwXyt1eKD/+Fq0bFWhwVM/1XiAXL7lyYUyOq24KHgQ2Csg=
+  -----END CERTIFICATE-----
+date: 2024-08-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: async-job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: async-redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/async/job/processor/redis.rb
+- lib/async/job/processor/redis/delayed_jobs.rb
+- lib/async/job/processor/redis/job_store.rb
+- lib/async/job/processor/redis/processing_list.rb
+- lib/async/job/processor/redis/ready_list.rb
+- lib/async/job/processor/redis/server.rb
+- lib/async/job/processor/redis/version.rb
+- license.md
+- readme.md
+homepage:
+licenses:
+- MIT
+metadata:
+  documentation_uri: https://socketry.github.io/async-job-processor-redis/
+  source_code_uri: https://github.com/socketry/async-job-processor-redis
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.11
+signing_key:
+specification_version: 4
+summary: A asynchronous job queue for Ruby.
+test_files: []