cloudtasker 0.2.0 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +4 -0
- data/.rubocop.yml +5 -0
- data/.travis.yml +10 -1
- data/Appraisals +25 -0
- data/CHANGELOG.md +29 -0
- data/Gemfile.lock +27 -4
- data/README.md +571 -6
- data/Rakefile +6 -0
- data/app/controllers/cloudtasker/application_controller.rb +2 -0
- data/app/controllers/cloudtasker/worker_controller.rb +24 -2
- data/cloudtasker.gemspec +5 -3
- data/docs/BATCH_JOBS.md +66 -0
- data/docs/CRON_JOBS.md +65 -0
- data/docs/UNIQUE_JOBS.md +127 -0
- data/exe/cloudtasker +15 -0
- data/gemfiles/.bundle/config +2 -0
- data/gemfiles/google_cloud_tasks_1.0.gemfile +9 -0
- data/gemfiles/google_cloud_tasks_1.0.gemfile.lock +263 -0
- data/gemfiles/google_cloud_tasks_1.1.gemfile +9 -0
- data/gemfiles/google_cloud_tasks_1.1.gemfile.lock +263 -0
- data/gemfiles/google_cloud_tasks_1.2.gemfile +9 -0
- data/gemfiles/google_cloud_tasks_1.2.gemfile.lock +263 -0
- data/gemfiles/google_cloud_tasks_1.3.gemfile +9 -0
- data/gemfiles/google_cloud_tasks_1.3.gemfile.lock +264 -0
- data/gemfiles/rails_4.0.gemfile +10 -0
- data/gemfiles/rails_4.1.gemfile +9 -0
- data/gemfiles/rails_4.2.gemfile +9 -0
- data/gemfiles/rails_5.0.gemfile +9 -0
- data/gemfiles/rails_5.1.gemfile +9 -0
- data/gemfiles/rails_5.2.gemfile +9 -0
- data/gemfiles/rails_5.2.gemfile.lock +247 -0
- data/gemfiles/rails_6.0.gemfile +9 -0
- data/gemfiles/rails_6.0.gemfile.lock +263 -0
- data/lib/cloudtasker.rb +19 -1
- data/lib/cloudtasker/backend/google_cloud_task.rb +139 -0
- data/lib/cloudtasker/backend/memory_task.rb +190 -0
- data/lib/cloudtasker/backend/redis_task.rb +249 -0
- data/lib/cloudtasker/batch/batch_progress.rb +19 -1
- data/lib/cloudtasker/batch/job.rb +85 -23
- data/lib/cloudtasker/cli.rb +194 -0
- data/lib/cloudtasker/cloud_task.rb +91 -0
- data/lib/cloudtasker/config.rb +64 -2
- data/lib/cloudtasker/cron/job.rb +2 -2
- data/lib/cloudtasker/cron/schedule.rb +25 -11
- data/lib/cloudtasker/dead_worker_error.rb +6 -0
- data/lib/cloudtasker/local_server.rb +74 -0
- data/lib/cloudtasker/railtie.rb +10 -0
- data/lib/cloudtasker/redis_client.rb +2 -2
- data/lib/cloudtasker/testing.rb +133 -0
- data/lib/cloudtasker/unique_job/job.rb +1 -1
- data/lib/cloudtasker/unique_job/lock/base_lock.rb +1 -1
- data/lib/cloudtasker/unique_job/lock/until_executed.rb +3 -1
- data/lib/cloudtasker/unique_job/lock/while_executing.rb +3 -1
- data/lib/cloudtasker/version.rb +1 -1
- data/lib/cloudtasker/worker.rb +61 -17
- data/lib/cloudtasker/{task.rb → worker_handler.rb} +10 -77
- data/lib/cloudtasker/worker_logger.rb +155 -0
- data/lib/tasks/setup_queue.rake +10 -0
- metadata +70 -6
@@ -0,0 +1,190 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'cloudtasker/redis_client'
|
4
|
+
|
5
|
+
module Cloudtasker
|
6
|
+
module Backend
|
7
|
+
# Manage local tasks pushed to memory.
|
8
|
+
# Used for testing.
|
9
|
+
class MemoryTask
|
10
|
+
attr_reader :id, :http_request, :schedule_time
|
11
|
+
|
12
|
+
#
|
13
|
+
# Return the task queue. A worker class name
|
14
|
+
#
|
15
|
+
# @return [Array<Hash>] <description>
|
16
|
+
#
|
17
|
+
def self.queue
|
18
|
+
@queue ||= []
|
19
|
+
end
|
20
|
+
|
21
|
+
#
|
22
|
+
# Return the workers currently in the queue.
|
23
|
+
#
|
24
|
+
# @param [String] worker_class_name Filter jobs on worker class name.
|
25
|
+
#
|
26
|
+
# @return [Array<Cloudtasker::Worker] The list of workers
|
27
|
+
#
|
28
|
+
def self.jobs(worker_class_name = nil)
|
29
|
+
all(worker_class_name).map(&:worker)
|
30
|
+
end
|
31
|
+
|
32
|
+
#
|
33
|
+
# Run all Tasks in the queue. Optionally filter which tasks to run based
|
34
|
+
# on the worker class name.
|
35
|
+
#
|
36
|
+
# @param [String] worker_class_name Run tasks for a specific worker class name.
|
37
|
+
#
|
38
|
+
# @return [Array<any>] The return values of the workers perform method.
|
39
|
+
#
|
40
|
+
def self.drain(worker_class_name = nil)
|
41
|
+
all(worker_class_name).map(&:execute)
|
42
|
+
end
|
43
|
+
|
44
|
+
#
|
45
|
+
# Return all enqueued tasks. A worker class name can be specified
|
46
|
+
# to filter the returned results.
|
47
|
+
#
|
48
|
+
# @param [String] worker_class_name Filter tasks on worker class name.
|
49
|
+
#
|
50
|
+
# @return [Array<Cloudtasker::Backend::MemoryTask>] All the tasks
|
51
|
+
#
|
52
|
+
def self.all(worker_class_name = nil)
|
53
|
+
list = queue
|
54
|
+
list = list.select { |e| e.worker_class_name == worker_class_name } if worker_class_name
|
55
|
+
list
|
56
|
+
end
|
57
|
+
|
58
|
+
#
|
59
|
+
# Push a job to the queue.
|
60
|
+
#
|
61
|
+
# @param [Hash] payload The Cloud Task payload.
|
62
|
+
#
|
63
|
+
def self.create(payload)
|
64
|
+
id = payload[:id] || SecureRandom.uuid
|
65
|
+
payload = payload.merge(schedule_time: payload[:schedule_time].to_i)
|
66
|
+
|
67
|
+
# Save task
|
68
|
+
task = new(payload.merge(id: id))
|
69
|
+
queue << task
|
70
|
+
|
71
|
+
# Execute task immediately if in testing and inline mode enabled
|
72
|
+
task.execute if defined?(Cloudtasker::Testing) && Cloudtasker::Testing.inline?
|
73
|
+
|
74
|
+
task
|
75
|
+
end
|
76
|
+
|
77
|
+
#
|
78
|
+
# Get a task by id.
|
79
|
+
#
|
80
|
+
# @param [String] id The id of the task.
|
81
|
+
#
|
82
|
+
# @return [Cloudtasker::Backend::MemoryTask, nil] The task.
|
83
|
+
#
|
84
|
+
def self.find(id)
|
85
|
+
queue.find { |e| e.id == id }
|
86
|
+
end
|
87
|
+
|
88
|
+
#
|
89
|
+
# Delete a task by id.
|
90
|
+
#
|
91
|
+
# @param [String] id The task id.
|
92
|
+
#
|
93
|
+
def self.delete(id)
|
94
|
+
queue.reject! { |e| e.id == id }
|
95
|
+
end
|
96
|
+
|
97
|
+
#
|
98
|
+
# Clear the queue.
|
99
|
+
#
|
100
|
+
# @param [String] worker_class_name Filter jobs on worker class name.
|
101
|
+
#
|
102
|
+
# @return [Array<Cloudtasker::Backend::MemoryTask>] The updated queue
|
103
|
+
#
|
104
|
+
def self.clear(worker_class_name = nil)
|
105
|
+
if worker_class_name
|
106
|
+
queue.reject! { |e| e.worker_class_name == worker_class_name }
|
107
|
+
else
|
108
|
+
queue.clear
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
#
|
113
|
+
# Build a new instance of the class.
|
114
|
+
#
|
115
|
+
# @param [String] id The ID of the task.
|
116
|
+
# @param [Hash] http_request The HTTP request content.
|
117
|
+
# @param [Integer] schedule_time When to run the task (Unix timestamp)
|
118
|
+
#
|
119
|
+
def initialize(id:, http_request:, schedule_time: nil)
|
120
|
+
@id = id
|
121
|
+
@http_request = http_request
|
122
|
+
@schedule_time = Time.at(schedule_time || 0)
|
123
|
+
end
|
124
|
+
|
125
|
+
#
|
126
|
+
# Return task payload.
|
127
|
+
#
|
128
|
+
# @return [Hash] The task payload.
|
129
|
+
#
|
130
|
+
def payload
|
131
|
+
@payload ||= JSON.parse(http_request.dig(:body), symbolize_names: true)
|
132
|
+
end
|
133
|
+
|
134
|
+
#
|
135
|
+
# Return the worker class from the task payload.
|
136
|
+
#
|
137
|
+
# @return [String] The task worker class name.
|
138
|
+
#
|
139
|
+
def worker_class_name
|
140
|
+
payload[:worker]
|
141
|
+
end
|
142
|
+
|
143
|
+
#
|
144
|
+
# Return a hash description of the task.
|
145
|
+
#
|
146
|
+
# @return [Hash] A hash description of the task.
|
147
|
+
#
|
148
|
+
def to_h
|
149
|
+
{
|
150
|
+
id: id,
|
151
|
+
http_request: http_request,
|
152
|
+
schedule_time: schedule_time.to_i
|
153
|
+
}
|
154
|
+
end
|
155
|
+
|
156
|
+
#
|
157
|
+
# Return the worker attached to this task.
|
158
|
+
#
|
159
|
+
# @return [Cloudtasker::Worker] The task worker.
|
160
|
+
#
|
161
|
+
def worker
|
162
|
+
@worker ||= Worker.from_hash(payload)
|
163
|
+
end
|
164
|
+
|
165
|
+
#
|
166
|
+
# Execute the task.
|
167
|
+
#
|
168
|
+
# @return [Any] The return value of the worker perform method.
|
169
|
+
#
|
170
|
+
def execute
|
171
|
+
resp = worker.execute
|
172
|
+
self.class.delete(id)
|
173
|
+
resp
|
174
|
+
rescue StandardError
|
175
|
+
worker.job_retries += 1
|
176
|
+
end
|
177
|
+
|
178
|
+
#
|
179
|
+
# Equality operator.
|
180
|
+
#
|
181
|
+
# @param [Any] other The object to compare.
|
182
|
+
#
|
183
|
+
# @return [Boolean] True if the object is equal.
|
184
|
+
#
|
185
|
+
def ==(other)
|
186
|
+
other.is_a?(self.class) && other.id == id
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
@@ -0,0 +1,249 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'cloudtasker/redis_client'
|
4
|
+
require 'net/http'
|
5
|
+
|
6
|
+
module Cloudtasker
|
7
|
+
module Backend
|
8
|
+
# Manage local tasks pushed to Redis
|
9
|
+
class RedisTask
|
10
|
+
attr_reader :id, :http_request, :schedule_time, :retries
|
11
|
+
|
12
|
+
RETRY_INTERVAL = 20 # seconds
|
13
|
+
|
14
|
+
#
|
15
|
+
# Return the cloudtasker redis client
|
16
|
+
#
|
17
|
+
# @return [Class] The redis client.
|
18
|
+
#
|
19
|
+
def self.redis
|
20
|
+
RedisClient
|
21
|
+
end
|
22
|
+
|
23
|
+
#
|
24
|
+
# Return a namespaced key.
|
25
|
+
#
|
26
|
+
# @param [String, Symbol] val The key to namespace
|
27
|
+
#
|
28
|
+
# @return [String] The namespaced key.
|
29
|
+
#
|
30
|
+
def self.key(val)
|
31
|
+
return nil if val.nil?
|
32
|
+
|
33
|
+
[to_s.underscore, val.to_s].join('/')
|
34
|
+
end
|
35
|
+
|
36
|
+
#
|
37
|
+
# Return all tasks stored in Redis.
|
38
|
+
#
|
39
|
+
# @return [Array<Cloudtasker::Backend::RedisTask>] All the tasks.
|
40
|
+
#
|
41
|
+
def self.all
|
42
|
+
redis.search(key('*')).map do |gid|
|
43
|
+
payload = redis.fetch(gid)
|
44
|
+
new(payload.merge(id: gid.sub(key(''), '')))
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
#
|
49
|
+
# Reeturn all tasks ready to process.
|
50
|
+
#
|
51
|
+
# @return [Array<Cloudtasker::Backend::RedisTask>] All the tasks ready to process.
|
52
|
+
#
|
53
|
+
def self.ready_to_process
|
54
|
+
all.select { |e| e.schedule_time <= Time.now }
|
55
|
+
end
|
56
|
+
|
57
|
+
#
|
58
|
+
# Retrieve and remove a task from the queue.
|
59
|
+
#
|
60
|
+
# @return [Cloudtasker::Backend::RedisTask] A task ready to process.
|
61
|
+
#
|
62
|
+
def self.pop
|
63
|
+
redis.with_lock('cloudtasker/server') do
|
64
|
+
ready_to_process.first&.tap(&:destroy)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
#
|
69
|
+
# Push a job to the queue.
|
70
|
+
#
|
71
|
+
# @param [Hash] payload The Cloud Task payload.
|
72
|
+
#
|
73
|
+
def self.create(payload)
|
74
|
+
id = SecureRandom.uuid
|
75
|
+
payload = payload.merge(schedule_time: payload[:schedule_time].to_i)
|
76
|
+
|
77
|
+
# Save job
|
78
|
+
redis.write(key(id), payload)
|
79
|
+
new(payload.merge(id: id))
|
80
|
+
end
|
81
|
+
|
82
|
+
#
|
83
|
+
# Get a task by id.
|
84
|
+
#
|
85
|
+
# @param [String] id The id of the task.
|
86
|
+
#
|
87
|
+
# @return [Cloudtasker::Backend::RedisTask, nil] The task.
|
88
|
+
#
|
89
|
+
def self.find(id)
|
90
|
+
gid = key(id)
|
91
|
+
return nil unless (payload = redis.fetch(gid))
|
92
|
+
|
93
|
+
new(payload.merge(id: id))
|
94
|
+
end
|
95
|
+
|
96
|
+
#
|
97
|
+
# Delete a task by id.
|
98
|
+
#
|
99
|
+
# @param [String] id The task id.
|
100
|
+
#
|
101
|
+
def self.delete(id)
|
102
|
+
redis.del(key(id))
|
103
|
+
end
|
104
|
+
|
105
|
+
#
|
106
|
+
# Build a new instance of the class.
|
107
|
+
#
|
108
|
+
# @param [String] id The ID of the task.
|
109
|
+
# @param [Hash] http_request The HTTP request content.
|
110
|
+
# @param [Integer] schedule_time When to run the task (Unix timestamp)
|
111
|
+
# @param [Integer] retries The number of times the job failed.
|
112
|
+
#
|
113
|
+
def initialize(id:, http_request:, schedule_time: nil, retries: 0)
|
114
|
+
@id = id
|
115
|
+
@http_request = http_request
|
116
|
+
@schedule_time = Time.at(schedule_time || 0)
|
117
|
+
@retries = retries || 0
|
118
|
+
end
|
119
|
+
|
120
|
+
#
|
121
|
+
# Return the redis client.
|
122
|
+
#
|
123
|
+
# @return [Class] The RedisClient.
|
124
|
+
#
|
125
|
+
def redis
|
126
|
+
self.class.redis
|
127
|
+
end
|
128
|
+
|
129
|
+
#
|
130
|
+
# Return a hash description of the task.
|
131
|
+
#
|
132
|
+
# @return [Hash] A hash description of the task.
|
133
|
+
#
|
134
|
+
def to_h
|
135
|
+
{
|
136
|
+
id: id,
|
137
|
+
http_request: http_request,
|
138
|
+
schedule_time: schedule_time.to_i,
|
139
|
+
retries: retries
|
140
|
+
}
|
141
|
+
end
|
142
|
+
|
143
|
+
#
|
144
|
+
# Return the namespaced task id
|
145
|
+
#
|
146
|
+
# @return [<Type>] The namespaced task id
|
147
|
+
#
|
148
|
+
def gid
|
149
|
+
self.class.key(id)
|
150
|
+
end
|
151
|
+
|
152
|
+
#
|
153
|
+
# Retry the task later.
|
154
|
+
#
|
155
|
+
# @param [Integer] interval The delay in seconds before retrying the task
|
156
|
+
#
|
157
|
+
def retry_later(interval, is_error: true)
|
158
|
+
redis.write(gid,
|
159
|
+
retries: is_error ? retries + 1 : retries,
|
160
|
+
http_request: http_request,
|
161
|
+
schedule_time: (Time.now + interval).to_i)
|
162
|
+
end
|
163
|
+
|
164
|
+
#
|
165
|
+
# Remove the task from the queue.
|
166
|
+
#
|
167
|
+
def destroy
|
168
|
+
redis.del(gid)
|
169
|
+
end
|
170
|
+
|
171
|
+
#
|
172
|
+
# Deliver the task to the processing endpoint.
|
173
|
+
#
|
174
|
+
def deliver
|
175
|
+
Cloudtasker.logger.info(format_log_message('Processing task...'))
|
176
|
+
|
177
|
+
# Send request
|
178
|
+
resp = http_client.request(request_content)
|
179
|
+
|
180
|
+
# Delete task if successful
|
181
|
+
if resp.code.to_s =~ /20\d/
|
182
|
+
destroy
|
183
|
+
Cloudtasker.logger.info(format_log_message('Task handled successfully'))
|
184
|
+
else
|
185
|
+
retry_later(RETRY_INTERVAL)
|
186
|
+
Cloudtasker.logger.info(format_log_message("Task failure - Retry in #{RETRY_INTERVAL} seconds..."))
|
187
|
+
end
|
188
|
+
|
189
|
+
resp
|
190
|
+
end
|
191
|
+
|
192
|
+
#
|
193
|
+
# Equality operator.
|
194
|
+
#
|
195
|
+
# @param [Any] other The object to compare.
|
196
|
+
#
|
197
|
+
# @return [Boolean] True if the object is equal.
|
198
|
+
#
|
199
|
+
def ==(other)
|
200
|
+
other.is_a?(self.class) && other.id == id
|
201
|
+
end
|
202
|
+
|
203
|
+
private
|
204
|
+
|
205
|
+
#
|
206
|
+
# Format a log message
|
207
|
+
#
|
208
|
+
# @param [String] msg The message to log.
|
209
|
+
#
|
210
|
+
# @return [String] The formatted message
|
211
|
+
#
|
212
|
+
def format_log_message(msg)
|
213
|
+
"[Cloudtasker/Server][#{id}] #{msg}"
|
214
|
+
end
|
215
|
+
|
216
|
+
#
|
217
|
+
# Return the HTTP client.
|
218
|
+
#
|
219
|
+
# @return [Net::HTTP] The http_client.
|
220
|
+
#
|
221
|
+
def http_client
|
222
|
+
@http_client ||=
|
223
|
+
begin
|
224
|
+
uri = URI(http_request[:url])
|
225
|
+
Net::HTTP.new(uri.host, uri.port).tap { |e| e.read_timeout = 60 * 10 }
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
#
|
230
|
+
# Return the HTTP request to send
|
231
|
+
#
|
232
|
+
# @return [Net::HTTP::Post] The http request
|
233
|
+
#
|
234
|
+
def request_content
|
235
|
+
@request_content ||= begin
|
236
|
+
uri = URI(http_request[:url])
|
237
|
+
req = Net::HTTP::Post.new(uri.path, http_request[:headers])
|
238
|
+
|
239
|
+
# Add retries header
|
240
|
+
req['X-CloudTasks-TaskExecutionCount'] = retries
|
241
|
+
|
242
|
+
# Set job payload
|
243
|
+
req.body = http_request[:body]
|
244
|
+
req
|
245
|
+
end
|
246
|
+
end
|
247
|
+
end
|
248
|
+
end
|
249
|
+
end
|
@@ -53,13 +53,31 @@ module Cloudtasker
|
|
53
53
|
@processing ||= count('processing')
|
54
54
|
end
|
55
55
|
|
56
|
+
#
|
57
|
+
# Return the number of jobs with errors.
|
58
|
+
#
|
59
|
+
# @return [Integer] The number of errored jobs.
|
60
|
+
#
|
61
|
+
def errored
|
62
|
+
@errored ||= count('errored')
|
63
|
+
end
|
64
|
+
|
65
|
+
#
|
66
|
+
# Return the number of dead jobs.
|
67
|
+
#
|
68
|
+
# @return [Integer] The number of dead jobs.
|
69
|
+
#
|
70
|
+
def dead
|
71
|
+
@dead ||= count('dead')
|
72
|
+
end
|
73
|
+
|
56
74
|
#
|
57
75
|
# Return the number of jobs not completed yet.
|
58
76
|
#
|
59
77
|
# @return [Integer] The number of jobs pending.
|
60
78
|
#
|
61
79
|
def pending
|
62
|
-
total - completed
|
80
|
+
total - completed - dead
|
63
81
|
end
|
64
82
|
|
65
83
|
#
|