async-job-processor-redis 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '095272ad3206a878ec049e641b3632181be1f62ca055398230e14dc3fd242d14'
4
- data.tar.gz: b10ab3013a961aca3cbecaa8f51265f4aa8a20aa3eb43031b6a94079d23553f8
3
+ metadata.gz: cf246106a0b45eade2e2316e3d198a80101b89e83619ae6ebf753a1a153a853c
4
+ data.tar.gz: c298e2a87023a73a2014693d09685320ba20b875b1d872eea764ba3d97054cf8
5
5
  SHA512:
6
- metadata.gz: '082e58c7f2f95eff36c022f68ce14a0a1b00e01463bd4f3397a94422b4a4f9011b106d09b21462ce5aeab8dd711af85e8168d1c4423357137fc564720ecb997f'
7
- data.tar.gz: e3bbfd7290feda5b88c941fef9bd4e7ea191c8a800dd97f330f3d1c7032efb869aa73d1a08419b957a541fb78e3d482f6efb816e327c659fd1c26c433cd22479
6
+ metadata.gz: c343b68ed4fdefa6473737c65ea4e5d4303573eb60dca28a9f9d7ac36fcb5a35fdf7c37b91d74900aedb335d62701497398a6336073886305998ca3d6668f3c4
7
+ data.tar.gz: 17c8c94cdd5a3c08bd09d10c77e129bd9bcffc475f444411afa79fcd8f94f3bce6e80e03c391cb4fb2aadf3c729916fb1a8c1ed9ba44db786bc86c0e0d573300
checksums.yaml.gz.sig CHANGED
Binary file
@@ -7,6 +7,9 @@ module Async
7
7
  module Job
8
8
  module Processor
9
9
  module Redis
10
+ # Manages delayed job scheduling using Redis sorted sets.
11
+ # Jobs are stored with their execution timestamps and automatically moved
12
+ # to the ready queue when their scheduled time arrives.
10
13
  class DelayedJobs
11
14
  ADD = <<~LUA
12
15
  redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
@@ -22,6 +25,9 @@ module Async
22
25
  return #jobs
23
26
  LUA
24
27
 
28
+ # Initialize a new delayed jobs manager.
29
+ # @parameter client [Async::Redis::Client] The Redis client instance.
30
+ # @parameter key [String] The Redis key for the delayed jobs sorted set.
25
31
  def initialize(client, key)
26
32
  @client = client
27
33
  @key = key
@@ -30,6 +36,11 @@ module Async
30
36
  @move = @client.script(:load, MOVE)
31
37
  end
32
38
 
39
+ # Start the background task that moves ready delayed jobs to the ready queue.
40
+ # @parameter ready_list [ReadyList] The ready list to move jobs to.
41
+ # @parameter resolution [Integer] The check interval in seconds.
42
+ # @parameter parent [Async::Task] The parent task to run the background loop in.
43
+ # @returns [Async::Task] The background processing task.
33
44
  def start(ready_list, resolution: 10, parent: Async::Task.current)
34
45
  parent.async do
35
46
  while true
@@ -44,8 +55,14 @@ module Async
44
55
  end
45
56
  end
46
57
 
58
+ # @attribute [String] The Redis key for this delayed jobs queue.
47
59
  attr :key
48
60
 
61
+ # Add a job to the delayed queue with a specified execution time.
62
+ # @parameter job [String] The serialized job data.
63
+ # @parameter timestamp [Time] When the job should be executed.
64
+ # @parameter job_store [JobStore] The job store to save the job data.
65
+ # @returns [String] The unique job ID.
49
66
  def add(job, timestamp, job_store)
50
67
  id = SecureRandom.uuid
51
68
 
@@ -54,7 +71,11 @@ module Async
54
71
  return id
55
72
  end
56
73
 
57
- def move(destination:, now: Time.now.to_i)
74
+ # Move jobs that are ready to be processed from the delayed queue to the destination.
75
+ # @parameter destination [String] The Redis key of the destination queue.
76
+ # @parameter now [Integer] The current timestamp to check against.
77
+ # @returns [Integer] The number of jobs moved.
78
+ def move(destination:, now: Time.now.to_f)
58
79
  @client.evalsha(@move, 2, @key, destination, now)
59
80
  end
60
81
  end
@@ -7,14 +7,23 @@ module Async
7
7
  module Job
8
8
  module Processor
9
9
  module Redis
10
+ # Stores job data using Redis hashes.
11
+ # Provides persistent storage for job payloads indexed by job ID.
10
12
  class JobStore
13
+ # Initialize a new job store.
14
+ # @parameter client [Async::Redis::Client] The Redis client instance.
15
+ # @parameter key [String] The Redis key for the job data hash.
11
16
  def initialize(client, key)
12
17
  @client = client
13
18
  @key = key
14
19
  end
15
20
 
21
+ # @attribute [String] The Redis key for this job store.
16
22
  attr :key
17
23
 
24
+ # Retrieve job data by ID.
25
+ # @parameter id [String] The job ID to retrieve.
26
+ # @returns [String, nil] The serialized job data, or nil if not found.
18
27
  def get(id)
19
28
  @client.hget(@key, id)
20
29
  end
@@ -1,12 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  # Released under the MIT License.
4
- # Copyright, 2024, by Samuel Williams.
4
+ # Copyright, 2024-2025, by Samuel Williams.
5
5
 
6
6
  module Async
7
7
  module Job
8
8
  module Processor
9
9
  module Redis
10
+ # Manages jobs currently being processed and handles abandoned job recovery.
11
+ # Maintains heartbeats for active workers and automatically requeues
12
+ # jobs from workers that have stopped responding.
10
13
  class ProcessingList
11
14
  REQUEUE = <<~LUA
12
15
  local cursor = "0"
@@ -50,6 +53,12 @@ module Async
50
53
  redis.call('HDEL', KEYS[2], ARGV[1])
51
54
  LUA
52
55
 
56
+ # Initialize a new processing list manager.
57
+ # @parameter client [Async::Redis::Client] The Redis client instance.
58
+ # @parameter key [String] The base Redis key for processing data.
59
+ # @parameter id [String] The unique server/worker ID.
60
+ # @parameter ready_list [ReadyList] The ready job queue.
61
+ # @parameter job_store [JobStore] The job data store.
53
62
  def initialize(client, key, id, ready_list, job_store)
54
63
  @client = client
55
64
  @key = key
@@ -66,34 +75,64 @@ module Async
66
75
  @complete = @client.script(:load, COMPLETE)
67
76
  end
68
77
 
78
+ # @attribute [String] The base Redis key for this processing list.
69
79
  attr :key
70
80
 
81
+ # @attribute [String] The Redis key for this worker's heartbeat.
82
+ attr :heartbeat_key
83
+
84
+ # Fetch the next job from the ready queue, moving it to this worker's pending list.
85
+ # This is a blocking operation that waits until a job is available.
86
+ # @returns [String, nil] The job ID, or nil if no job is available.
71
87
  def fetch
72
88
  @client.brpoplpush(@ready_list.key, @pending_key, 0)
73
89
  end
74
90
 
91
+ # Mark a job as completed, removing it from the pending list and job store.
92
+ # @parameter id [String] The job ID to complete.
75
93
  def complete(id)
76
94
  @client.evalsha(@complete, 2, @pending_key, @job_store.key, id)
77
95
  end
78
96
 
97
+ # Retry a failed job by moving it back to the ready queue.
98
+ # @parameter id [String] The job ID to retry.
79
99
  def retry(id)
80
100
  Console.warn(self, "Retrying job: #{id}")
81
101
  @client.evalsha(@retry, 2, @pending_key, @ready_list.key, id)
82
102
  end
83
103
 
104
+ # Update heartbeat and requeue any abandoned jobs from inactive workers.
105
+ # @parameter start_time [Float] The start time for calculating uptime.
106
+ # @parameter delay [Numeric] The heartbeat update interval.
107
+ # @parameter factor [Numeric] The heartbeat expiration factor.
108
+ # @returns [Integer] The number of jobs requeued from abandoned workers.
109
+ def requeue(start_time, delay, factor)
110
+ uptime = (Time.now.to_f - start_time).round(2)
111
+ expiry = (delay*factor).ceil
112
+ @client.set(@heartbeat_key, JSON.dump(uptime: uptime), seconds: expiry)
113
+
114
+ # Requeue any jobs that have been abandoned:
115
+ count = @client.evalsha(@requeue, 2, @key, @ready_list.key)
116
+
117
+ return count
118
+ end
119
+
120
+ # Start the background heartbeat and abandoned job recovery task.
121
+ # @parameter delay [Integer] The heartbeat update interval in seconds.
122
+ # @parameter factor [Integer] The heartbeat expiration factor.
123
+ # @parameter parent [Async::Task] The parent task to run the background loop in.
124
+ # @returns [Async::Task] The background processing task.
84
125
  def start(delay: 5, factor: 2, parent: Async::Task.current)
85
- heartbeat_key = "#{@key}:#{@id}"
86
126
  start_time = Time.now.to_f
87
127
 
88
- parent.async do
128
+ parent.async do |task|
89
129
  while true
90
- uptime = (Time.now.to_f - start_time).round(2)
91
- @client.set(heartbeat_key, JSON.dump(uptime: uptime), seconds: delay*factor)
92
-
93
- # Requeue any jobs that have been abandoned:
94
- count = @client.evalsha(@requeue, 2, @key, @ready_list.key)
95
- if count > 0
96
- Console.warn(self, "Requeued #{count} abandoned jobs.")
130
+ task.defer_stop do
131
+ count = self.requeue(start_time, delay, factor)
132
+
133
+ if count > 0
134
+ Console.warn(self, "Requeued #{count} abandoned jobs.")
135
+ end
97
136
  end
98
137
 
99
138
  sleep(delay)
@@ -7,12 +7,17 @@ module Async
7
7
  module Job
8
8
  module Processor
9
9
  module Redis
10
+ # Manages the queue of jobs ready for immediate processing.
11
+ # Jobs are stored in Redis lists with FIFO (first-in, first-out) ordering.
10
12
  class ReadyList
11
13
  ADD = <<~LUA
12
14
  redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
13
15
  redis.call('LPUSH', KEYS[2], ARGV[1])
14
16
  LUA
15
17
 
18
+ # Initialize a new ready list manager.
19
+ # @parameter client [Async::Redis::Client] The Redis client instance.
20
+ # @parameter key [String] The Redis key for the ready job list.
16
21
  def initialize(client, key)
17
22
  @client = client
18
23
  @key = key
@@ -20,8 +25,13 @@ module Async
20
25
  @add = @client.script(:load, ADD)
21
26
  end
22
27
 
28
+ # @attribute [String] The Redis key for this ready list.
23
29
  attr :key
24
30
 
31
+ # Add a new job to the ready queue.
32
+ # @parameter job [String] The serialized job data.
33
+ # @parameter job_store [JobStore] The job store to save the job data.
34
+ # @returns [String] The unique job ID.
25
35
  def add(job, job_store)
26
36
  id = SecureRandom.uuid
27
37
 
@@ -1,25 +1,35 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  # Released under the MIT License.
4
- # Copyright, 2024, by Samuel Williams.
4
+ # Copyright, 2024-2025, by Samuel Williams.
5
5
 
6
- require 'async/idler'
7
- require 'async/job/coder'
8
- require 'async/job/processor/generic'
6
+ require "async/idler"
7
+ require "async/job/coder"
8
+ require "async/job/processor/generic"
9
9
 
10
- require 'securerandom'
10
+ require "securerandom"
11
11
 
12
- require_relative 'delayed_jobs'
13
- require_relative 'job_store'
14
- require_relative 'processing_list'
15
- require_relative 'ready_list'
12
+ require_relative "delayed_jobs"
13
+ require_relative "job_store"
14
+ require_relative "processing_list"
15
+ require_relative "ready_list"
16
16
 
17
17
  module Async
18
18
  module Job
19
19
  module Processor
20
20
  module Redis
21
+ # Redis-backed job processor server.
22
+ # Manages job queues using Redis for distributed job processing across multiple workers.
23
+ # Handles immediate jobs, delayed jobs, and job retry/recovery mechanisms.
21
24
  class Server < Generic
22
- def initialize(delegate, client, prefix: 'async-job', coder: Coder::DEFAULT, resolution: 10, parent: nil)
25
+ # Initialize a new Redis job processor server.
26
+ # @parameter delegate [Object] The delegate object that will process jobs.
27
+ # @parameter client [Async::Redis::Client] The Redis client instance.
28
+ # @parameter prefix [String] The Redis key prefix for job data.
29
+ # @parameter coder [Async::Job::Coder] The job serialization codec.
30
+ # @parameter resolution [Integer] The resolution in seconds for delayed job processing.
31
+ # @parameter parent [Async::Task] The parent task for background processing.
32
+ def initialize(delegate, client, prefix: "async-job", coder: Coder::DEFAULT, resolution: 10, parent: nil)
23
33
  super(delegate)
24
34
 
25
35
  @id = SecureRandom.uuid
@@ -36,6 +46,8 @@ module Async
36
46
  @parent = parent || Async::Idler.new
37
47
  end
38
48
 
49
+ # Start the job processing loop immediately.
50
+ # @returns [Async::Task | false] The processing task or false if already started.
39
51
  def start!
40
52
  return false if @task
41
53
 
@@ -52,6 +64,8 @@ module Async
52
64
  end
53
65
  end
54
66
 
67
+ # Start the server and all background processing tasks.
68
+ # Initializes delayed job processing, abandoned job recovery, and the main processing loop.
55
69
  def start
56
70
  super
57
71
 
@@ -64,12 +78,16 @@ module Async
64
78
  self.start!
65
79
  end
66
80
 
81
+ # Stop the server and all background processing tasks.
67
82
  def stop
68
83
  @task&.stop
69
84
 
70
85
  super
71
86
  end
72
87
 
88
+ # Submit a new job for processing.
89
+ # Jobs with a scheduled_at time are queued for delayed processing, while immediate jobs are added to the ready queue.
90
+ # @parameter job [Hash] The job data to process.
73
91
  def call(job)
74
92
  scheduled_at = Coder::Time(job["scheduled_at"])
75
93
 
@@ -97,7 +115,7 @@ module Async
97
115
  @delegate.call(job)
98
116
  @processing_list.complete(id)
99
117
  rescue => error
100
- Console::Event::Failure.for(error).emit(self, "Job failed with error!", id: id)
118
+ Console.error(self, "Job failed with error!", id: id, exception: error)
101
119
  @processing_list.retry(id)
102
120
  end
103
121
  ensure
@@ -7,7 +7,7 @@ module Async
7
7
  module Job
8
8
  module Processor
9
9
  module Redis
10
- VERSION = "0.1.0"
10
+ VERSION = "0.2.0"
11
11
  end
12
12
  end
13
13
  end
@@ -3,13 +3,23 @@
3
3
  # Released under the MIT License.
4
4
  # Copyright, 2024, by Samuel Williams.
5
5
 
6
- require_relative 'redis/server'
7
- require 'async/redis/client'
6
+ require_relative "redis/server"
7
+ require "async/redis/client"
8
8
 
9
+ # @namespace
9
10
  module Async
11
+ # @namespace
10
12
  module Job
13
+ # @namespace
11
14
  module Processor
15
+ # Redis-based job processor implementation.
16
+ # Provides distributed job processing capabilities using Redis as the backend.
12
17
  module Redis
18
+ # Create a new Redis job processor server.
19
+ # @parameter delegate [Object] The delegate object that will process jobs.
20
+ # @parameter endpoint [Async::Redis::Endpoint] The Redis endpoint to connect to.
21
+ # @parameter options [Hash] Additional options passed to the server.
22
+ # @returns [Server] A new Redis job processor server instance.
13
23
  def self.new(delegate, endpoint: Async::Redis.local_endpoint, **options)
14
24
  client = Async::Redis::Client.new(endpoint)
15
25
  return Server.new(delegate, client, **options)
data/license.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # MIT License
2
2
 
3
- Copyright, 2024, by Samuel Williams.
3
+ Copyright, 2024-2025, by Samuel Williams.
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
data/readme.md CHANGED
@@ -12,8 +12,22 @@ Please see the [project documentation](https://socketry.github.io/async-job-proc
12
12
 
13
13
  - [Redis Queue](https://socketry.github.io/async-job-processor-redis/guides/redis-queue/index) - This guide gives a brief overview of the implementation of the Redis queue.
14
14
 
15
+ ## Releases
16
+
17
+ Please see the [project releases](https://socketry.github.io/async-job-processor-redis/releases/index) for all releases.
18
+
19
+ ### v0.2.0
20
+
21
+ - Achieve 100% documentation coverage.
22
+ - Achieve 100% test coverage.
23
+
24
+ ### v0.1.0
25
+
26
+ - Initial release of async-job-processor-redis, migrated from the async-job project.
27
+
15
28
  ## See Also
16
29
 
30
+ - [async-job](https://github.com/socketry/async-job) - Asynchronous job processing framework.
17
31
  - [async-job-adapter-active\_job](https://github.com/socketry/async-job-adapter-active_job) - ActiveJob adapter for `async-job`.
18
32
 
19
33
  ## Contributing
data/releases.md ADDED
@@ -0,0 +1,10 @@
1
+ # Releases
2
+
3
+ ## v0.2.0
4
+
5
+ - Achieve 100% documentation coverage.
6
+ - Achieve 100% test coverage.
7
+
8
+ ## v0.1.0
9
+
10
+ - Initial release of async-job-processor-redis, migrated from the async-job project.
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,11 +1,10 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: async-job-processor-redis
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Samuel Williams
8
- autorequire:
9
8
  bindir: bin
10
9
  cert_chain:
11
10
  - |
@@ -37,7 +36,7 @@ cert_chain:
37
36
  Q2K9NVun/S785AP05vKkXZEFYxqG6EW012U4oLcFl5MySFajYXRYbuUpH6AY+HP8
38
37
  voD0MPg1DssDLKwXyt1eKD/+Fq0bFWhwVM/1XiAXL7lyYUyOq24KHgQ2Csg=
39
38
  -----END CERTIFICATE-----
40
- date: 2024-08-14 00:00:00.000000000 Z
39
+ date: 1980-01-02 00:00:00.000000000 Z
41
40
  dependencies:
42
41
  - !ruby/object:Gem::Dependency
43
42
  name: async-job
@@ -67,8 +66,6 @@ dependencies:
67
66
  - - ">="
68
67
  - !ruby/object:Gem::Version
69
68
  version: '0'
70
- description:
71
- email:
72
69
  executables: []
73
70
  extensions: []
74
71
  extra_rdoc_files: []
@@ -82,13 +79,12 @@ files:
82
79
  - lib/async/job/processor/redis/version.rb
83
80
  - license.md
84
81
  - readme.md
85
- homepage:
82
+ - releases.md
86
83
  licenses:
87
84
  - MIT
88
85
  metadata:
89
86
  documentation_uri: https://socketry.github.io/async-job-processor-redis/
90
87
  source_code_uri: https://github.com/socketry/async-job-processor-redis
91
- post_install_message:
92
88
  rdoc_options: []
93
89
  require_paths:
94
90
  - lib
@@ -96,15 +92,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
96
92
  requirements:
97
93
  - - ">="
98
94
  - !ruby/object:Gem::Version
99
- version: '3.1'
95
+ version: '3.2'
100
96
  required_rubygems_version: !ruby/object:Gem::Requirement
101
97
  requirements:
102
98
  - - ">="
103
99
  - !ruby/object:Gem::Version
104
100
  version: '0'
105
101
  requirements: []
106
- rubygems_version: 3.5.11
107
- signing_key:
102
+ rubygems_version: 3.6.7
108
103
  specification_version: 4
109
104
  summary: A asynchronous job queue for Ruby.
110
105
  test_files: []
metadata.gz.sig CHANGED
Binary file