async-job 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2fa7b9ed6bf57940243caa6fae0e8d72a9e5c6eaf38f13311b800abb80850c09
+  data.tar.gz: 9dce5abfa5b06e6e00592e6ca474241ab92806d65b70c94aab0d0763e5196a19
+SHA512:
+  metadata.gz: 79557b886d6fadc8ae47644c36916c556091f4eac1c78f5d4b3271163a00fa08d5e68931d867fe7ea1871f6257cbd9b00fd413561ecd6a3a244e101a2d2685bf
+  data.tar.gz: 0cbce58e990cff1881f29c4c91f7db93aae228d44d85ecde799b22d88ab5824df5dbabd5c11045dddba01c37d0b644914169adc8c6ed86b38fa7b353492763a7

data/lib/async/job/backend/redis/delayed_queue.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		module Backend
+			module Redis
+				class DelayedQueue
+					ADD = <<~LUA
+						redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+						redis.call('ZADD', KEYS[2], ARGV[3], ARGV[1])
+					LUA
+					
+					MOVE = <<~LUA
+						local jobs = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1])
+						redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, ARGV[1])
+						if #jobs > 0 then
+							redis.call('LPUSH', KEYS[2], unpack(jobs))
+						end
+					LUA
+					
+					def initialize(client, key)
+						@client = client
+						@key = key
+						
+						@add = @client.script(:load, ADD)
+						@move = @client.script(:load, MOVE)
+					end
+					
+					def start(ready_queue, resolution: 10, parent: Async::Task.current)
+						parent.async do
+							while true
+								move(destination: ready_queue.key)
+								sleep(resolution)
+							end
+						end
+					end
+					
+					attr :key
+					
+					def add(job, job_store)
+						@client.evalsha(@add, 2, job_store.key, @key, job.id, job.serialize, job.perform_at.to_f)
+					end
+					
+					def move(destination:, now: Time.now.to_i)
+						@client.evalsha(@move, 2, @key, destination, now)
+					end
+				end
+			end
+		end
+	end
+end

data/lib/async/job/backend/redis/job_store.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		module Backend
+			module Redis
+				class JobStore
+					def initialize(client, key)
+						@client = client
+						@key = key
+					end
+					
+					attr :key
+					
+					def get(id)
+						@client.hget(@key, id)
+					end
+				end
+			end
+		end
+	end
+end

data/lib/async/job/backend/redis/processing_queue.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		module Backend
+			module Redis
+				class ProcessingQueue
+					REQUEUE = <<~LUA
+						local cursor = "0"
+						local count = 0
+						
+						repeat
+							-- Scan through all known server id -> job id mappings and requeue any jobs that have been abandoned:
+							local result = redis.call('SCAN', cursor, 'MATCH', KEYS[1]..':*:pending')
+							cursor = result[1]
+							for _, pending_key in pairs(result[2]) do
+								-- Check if the server is still active:
+								local server_key = KEYS[1]..":"..pending_key:match("([^:]+):pending")
+								local state = redis.call('GET', server_key)
+								if state == false then
+									while true do
+										-- Requeue any pending jobs:
+										local result = redis.call('RPOPLPUSH', pending_key, KEYS[2])
+										
+										if result == false then
+											-- Delete the pending list:
+											redis.call('DEL', pending_key)
+											break
+										end
+										
+										count = count + 1
+									end
+								end
+							end
+						until cursor == "0"
+						
+						return count
+					LUA
+					
+					RETRY = <<~LUA
+						redis.call('LREM', KEYS[1], 1, ARGV[1])
+						redis.call('LPUSH', KEYS[2], ARGV[1])
+					LUA
+					
+					COMPLETE = <<~LUA
+						redis.call('LREM', KEYS[1], 1, ARGV[1])
+						redis.call('HDEL', KEYS[2], ARGV[1])
+					LUA
+					
+					def initialize(client, key, id, ready_queue, job_store)
+						@client = client
+						@key = key
+						@id = id
+						
+						@ready_queue = ready_queue
+						@job_store = job_store
+						
+						@pending_key = "#{@key}:#{@id}:pending"
+						@heartbeat_key = "#{@key}:#{@id}"
+						
+						@requeue = @client.script(:load, REQUEUE)
+						@retry = @client.script(:load, RETRY)
+						@complete = @client.script(:load, COMPLETE)
+					end
+					
+					attr :key
+					
+					def fetch
+						Console.info(self, "Fetching job...")
+						id = @client.brpoplpush(@ready_queue.key, @pending_key, 0)
+						Console.info(self, "Fetching job: #{id}")
+						return id
+					end
+					
+					def complete(id)
+						Console.info(self, "Completing job: #{id}")
+						@client.evalsha(@complete, 2, @pending_key, @job_store.key, id)
+					end
+					
+					def retry(id)
+						Console.warn(self, "Retrying job: #{id}")
+						@client.evalsha(@retry, 2, @pending_key, @ready_queue.key, id)
+					end
+					
+					def start(delay: 5, factor: 2, parent: Async::Task.current)
+						heartbeat_key = "#{@key}:#{@id}"
+						
+						start_time = Time.now.to_f
+						
+						parent.async do
+							while true
+								uptime = (Time.now.to_f - start_time).round(2)
+								@client.set(heartbeat_key, JSON.dump(uptime: uptime), seconds: delay*factor)
+								
+								# Requeue any jobs that have been abandoned:
+								count = @client.evalsha(@requeue, 2, @key, @ready_queue.key)
+								if count > 0
+									Console.warn(self, "Requeued #{count} abandoned jobs.")
+								end
+								
+								sleep(delay)
+							end
+						end
+					end
+				end
+			end
+		end
+	end
+end

data/lib/async/job/backend/redis/ready_queue.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		module Backend
+			module Redis
+				class ReadyQueue
+					ADD = <<~LUA
+						redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+						redis.call('LPUSH', KEYS[2], ARGV[1])
+					LUA
+					
+					def initialize(client, key)
+						@client = client
+						@key = key
+						
+						@add = @client.script(:load, ADD)
+					end
+					
+					attr :key
+					
+					def add(job, job_store)
+						@client.evalsha(@add, 2, job_store.key, @key, job.id, job.serialize)
+					end
+				end
+			end
+		end
+	end
+end

data/lib/async/job/backend/redis/server.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+require_relative 'delayed_queue'
+require_relative 'job_store'
+require_relative 'processing_queue'
+require_relative 'ready_queue'
+
+require 'securerandom'
+
+module Async
+	module Job
+		module Backend
+			module Redis
+				class Server
+					def initialize(client, prefix)
+						@id = SecureRandom.uuid
+						@client = client
+						@prefix = prefix
+						
+						@job_store = JobStore.new(@client, "#{@prefix}:jobs")
+						
+						@delayed_queue = DelayedQueue.new(@client, "#{@prefix}:delayed")
+						@ready_queue = ReadyQueue.new(@client, "#{@prefix}:ready")
+						
+						@processing_queue = ProcessingQueue.new(@client, "#{@prefix}:processing", @id, @ready_queue, @job_store)
+					end
+					
+					def start
+						# Start the delayed queue, which will move jobs to the ready queue when they are ready:
+						@delayed_queue.start(@ready_queue)
+						
+						# Start the processing queue, which will move jobs to the ready queue when they are abandoned:
+						@processing_queue.start
+					end
+					
+					def enqueue(job)
+						if perform_at = job.perform_at and perform_at > Time.now.to_f
+							# If the job is delayed, add it to the delayed queue:
+							@delayed_queue.add(job, @job_store)
+						else
+							# If the job is ready to be processed now, add it to the ready queue:
+							@ready_queue.add(job, @job_store)
+						end
+					end
+					
+					def each(&block)
+						while id = @processing_queue.fetch
+							begin
+								job = @job_store.get(id)
+								yield id, job
+								@processing_queue.complete(id)
+							rescue => error
+								@processing_queue.retry(id)
+								raise
+							end
+						end
+					end
+				end
+			end
+		end
+	end
+end

data/lib/async/job/generic.rb

@@ -0,0 +1,30 @@
+module Async
+	module Job
+		class Generic
+			def self.enqueue(...)
+				self.new(...).enqueue
+			
+			def initialize(id, perform_at: nil)
+				@id = id
+				@perform_at = perform_at
+			end
+			
+			attr :id
+			attr :perform_at
+			
+			def serialize
+				raise NotImplementedError
+			end
+			
+			def call
+				raise NotImplementedError
+			end
+		end
+	end
+end
+
+
+server.enqueue(job)
+	-> job.serialize
+
+Server.enqueue(job)

data/lib/async/job/version.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+module Async
+	module Job
+		VERSION = "0.0.0"
+	end
+end

data/lib/async/job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2024, by Samuel Williams.
+
+require_relative 'job/version'
+require_relative 'job/backend/redis/server'

data/license.md

@@ -0,0 +1,21 @@
+# MIT License
+
+Copyright, 2024, by Samuel Williams.  
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/readme.md

@@ -0,0 +1,27 @@
+# Async::Job
+
+Provides an asynchronous job server.
+
+[![Development Status](https://github.com/socketry/async-job/workflows/Test/badge.svg)](https://github.com/socketry/async-job/actions?workflow=Test)
+
+## Usage
+
+The current implementation is incomplete, but with `redis` running on localhost, you can run several instances of `client.rb` and `server.rb` to show the general operation.
+
+## Contributing
+
+We welcome contributions to this project.
+
+1.  Fork it.
+2.  Create your feature branch (`git checkout -b my-new-feature`).
+3.  Commit your changes (`git commit -am 'Add some feature'`).
+4.  Push to the branch (`git push origin my-new-feature`).
+5.  Create new Pull Request.
+
+### Developer Certificate of Origin
+
+This project uses the [Developer Certificate of Origin](https://developercertificate.org/). All contributors to this project must agree to this document to have their contributions accepted.
+
+### Contributor Covenant
+
+This project is governed by the [Contributor Covenant](https://www.contributor-covenant.org/). All contributors and participants agree to abide by its terms.

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: async-job
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Samuel Williams
+autorequire:
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIE2DCCA0CgAwIBAgIBATANBgkqhkiG9w0BAQsFADBhMRgwFgYDVQQDDA9zYW11
+  ZWwud2lsbGlhbXMxHTAbBgoJkiaJk/IsZAEZFg1vcmlvbnRyYW5zZmVyMRIwEAYK
+  CZImiZPyLGQBGRYCY28xEjAQBgoJkiaJk/IsZAEZFgJuejAeFw0yMjA4MDYwNDUz
+  MjRaFw0zMjA4MDMwNDUzMjRaMGExGDAWBgNVBAMMD3NhbXVlbC53aWxsaWFtczEd
+  MBsGCgmSJomT8ixkARkWDW9yaW9udHJhbnNmZXIxEjAQBgoJkiaJk/IsZAEZFgJj
+  bzESMBAGCgmSJomT8ixkARkWAm56MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIB
+  igKCAYEAomvSopQXQ24+9DBB6I6jxRI2auu3VVb4nOjmmHq7XWM4u3HL+pni63X2
+  9qZdoq9xt7H+RPbwL28LDpDNflYQXoOhoVhQ37Pjn9YDjl8/4/9xa9+NUpl9XDIW
+  sGkaOY0eqsQm1pEWkHJr3zn/fxoKPZPfaJOglovdxf7dgsHz67Xgd/ka+Wo1YqoE
+  e5AUKRwUuvaUaumAKgPH+4E4oiLXI4T1Ff5Q7xxv6yXvHuYtlMHhYfgNn8iiW8WN
+  XibYXPNP7NtieSQqwR/xM6IRSoyXKuS+ZNGDPUUGk8RoiV/xvVN4LrVm9upSc0ss
+  RZ6qwOQmXCo/lLcDUxJAgG95cPw//sI00tZan75VgsGzSWAOdjQpFM0l4dxvKwHn
+  tUeT3ZsAgt0JnGqNm2Bkz81kG4A2hSyFZTFA8vZGhp+hz+8Q573tAR89y9YJBdYM
+  zp0FM4zwMNEUwgfRzv1tEVVUEXmoFCyhzonUUw4nE4CFu/sE3ffhjKcXcY//qiSW
+  xm4erY3XAgMBAAGjgZowgZcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAwHQYDVR0O
+  BBYEFO9t7XWuFf2SKLmuijgqR4sGDlRsMC4GA1UdEQQnMCWBI3NhbXVlbC53aWxs
+  aWFtc0BvcmlvbnRyYW5zZmVyLmNvLm56MC4GA1UdEgQnMCWBI3NhbXVlbC53aWxs
+  aWFtc0BvcmlvbnRyYW5zZmVyLmNvLm56MA0GCSqGSIb3DQEBCwUAA4IBgQB5sxkE
+  cBsSYwK6fYpM+hA5B5yZY2+L0Z+27jF1pWGgbhPH8/FjjBLVn+VFok3CDpRqwXCl
+  xCO40JEkKdznNy2avOMra6PFiQyOE74kCtv7P+Fdc+FhgqI5lMon6tt9rNeXmnW/
+  c1NaMRdxy999hmRGzUSFjozcCwxpy/LwabxtdXwXgSay4mQ32EDjqR1TixS1+smp
+  8C/NCWgpIfzpHGJsjvmH2wAfKtTTqB9CVKLCWEnCHyCaRVuKkrKjqhYCdmMBqCws
+  JkxfQWC+jBVeG9ZtPhQgZpfhvh+6hMhraUYRQ6XGyvBqEUe+yo6DKIT3MtGE2+CP
+  eX9i9ZWBydWb8/rvmwmX2kkcBbX0hZS1rcR593hGc61JR6lvkGYQ2MYskBveyaxt
+  Q2K9NVun/S785AP05vKkXZEFYxqG6EW012U4oLcFl5MySFajYXRYbuUpH6AY+HP8
+  voD0MPg1DssDLKwXyt1eKD/+Fq0bFWhwVM/1XiAXL7lyYUyOq24KHgQ2Csg=
+  -----END CERTIFICATE-----
+date: 2024-02-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: async-redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/async/job.rb
+- lib/async/job/backend/redis/delayed_queue.rb
+- lib/async/job/backend/redis/job_store.rb
+- lib/async/job/backend/redis/processing_queue.rb
+- lib/async/job/backend/redis/ready_queue.rb
+- lib/async/job/backend/redis/server.rb
+- lib/async/job/generic.rb
+- lib/async/job/version.rb
+- license.md
+- readme.md
+homepage:
+licenses:
+- MIT
+metadata:
+  documentation_uri: https://socketry.github.io/async-job
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.3
+signing_key:
+specification_version: 4
+summary: A asynchronous job queue for Ruby.
+test_files: []

metadata.gz.sig

@@ -0,0 +1,3 @@
+m���A������>������4̌E�<k��
J�V]�];hy%�]��;/Xևu1����@����Y������^A��
+?~�G���&8���Zw2k�^�74��3 �r(�)���A��}L�Uߺ��$��E�Z��;�HO�#X�Q�������D�R�=������	f����M�Nxe��	HT�Ҥ�:�@_����@S5��>��ކ���{�� b>Ph0h]�}�ᑻ��]b鈆��2}̅�� �/h��TXJ���
+|Zt��,l�$t�V���	���8��&�)�d�38�pPS�U�Xp���^��5�l�b9,JOj�*�j6D�'