cloudtasker 0.2.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -0
  3. data/.rubocop.yml +5 -0
  4. data/.travis.yml +10 -1
  5. data/Appraisals +25 -0
  6. data/CHANGELOG.md +29 -0
  7. data/Gemfile.lock +27 -4
  8. data/README.md +571 -6
  9. data/Rakefile +6 -0
  10. data/app/controllers/cloudtasker/application_controller.rb +2 -0
  11. data/app/controllers/cloudtasker/worker_controller.rb +24 -2
  12. data/cloudtasker.gemspec +5 -3
  13. data/docs/BATCH_JOBS.md +66 -0
  14. data/docs/CRON_JOBS.md +65 -0
  15. data/docs/UNIQUE_JOBS.md +127 -0
  16. data/exe/cloudtasker +15 -0
  17. data/gemfiles/.bundle/config +2 -0
  18. data/gemfiles/google_cloud_tasks_1.0.gemfile +9 -0
  19. data/gemfiles/google_cloud_tasks_1.0.gemfile.lock +263 -0
  20. data/gemfiles/google_cloud_tasks_1.1.gemfile +9 -0
  21. data/gemfiles/google_cloud_tasks_1.1.gemfile.lock +263 -0
  22. data/gemfiles/google_cloud_tasks_1.2.gemfile +9 -0
  23. data/gemfiles/google_cloud_tasks_1.2.gemfile.lock +263 -0
  24. data/gemfiles/google_cloud_tasks_1.3.gemfile +9 -0
  25. data/gemfiles/google_cloud_tasks_1.3.gemfile.lock +264 -0
  26. data/gemfiles/rails_4.0.gemfile +10 -0
  27. data/gemfiles/rails_4.1.gemfile +9 -0
  28. data/gemfiles/rails_4.2.gemfile +9 -0
  29. data/gemfiles/rails_5.0.gemfile +9 -0
  30. data/gemfiles/rails_5.1.gemfile +9 -0
  31. data/gemfiles/rails_5.2.gemfile +9 -0
  32. data/gemfiles/rails_5.2.gemfile.lock +247 -0
  33. data/gemfiles/rails_6.0.gemfile +9 -0
  34. data/gemfiles/rails_6.0.gemfile.lock +263 -0
  35. data/lib/cloudtasker.rb +19 -1
  36. data/lib/cloudtasker/backend/google_cloud_task.rb +139 -0
  37. data/lib/cloudtasker/backend/memory_task.rb +190 -0
  38. data/lib/cloudtasker/backend/redis_task.rb +249 -0
  39. data/lib/cloudtasker/batch/batch_progress.rb +19 -1
  40. data/lib/cloudtasker/batch/job.rb +85 -23
  41. data/lib/cloudtasker/cli.rb +194 -0
  42. data/lib/cloudtasker/cloud_task.rb +91 -0
  43. data/lib/cloudtasker/config.rb +64 -2
  44. data/lib/cloudtasker/cron/job.rb +2 -2
  45. data/lib/cloudtasker/cron/schedule.rb +25 -11
  46. data/lib/cloudtasker/dead_worker_error.rb +6 -0
  47. data/lib/cloudtasker/local_server.rb +74 -0
  48. data/lib/cloudtasker/railtie.rb +10 -0
  49. data/lib/cloudtasker/redis_client.rb +2 -2
  50. data/lib/cloudtasker/testing.rb +133 -0
  51. data/lib/cloudtasker/unique_job/job.rb +1 -1
  52. data/lib/cloudtasker/unique_job/lock/base_lock.rb +1 -1
  53. data/lib/cloudtasker/unique_job/lock/until_executed.rb +3 -1
  54. data/lib/cloudtasker/unique_job/lock/while_executing.rb +3 -1
  55. data/lib/cloudtasker/version.rb +1 -1
  56. data/lib/cloudtasker/worker.rb +61 -17
  57. data/lib/cloudtasker/{task.rb → worker_handler.rb} +10 -77
  58. data/lib/cloudtasker/worker_logger.rb +155 -0
  59. data/lib/tasks/setup_queue.rake +10 -0
  60. metadata +70 -6
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This file was generated by Appraisal
4
+
5
+ source 'https://rubygems.org'
6
+
7
+ gem 'rails', '6.0'
8
+
9
+ gemspec path: '../'
@@ -0,0 +1,263 @@
1
+ PATH
2
+ remote: ..
3
+ specs:
4
+ cloudtasker (0.2.0)
5
+ activesupport
6
+ fugit
7
+ google-cloud-tasks
8
+ jwt
9
+ redis
10
+
11
+ GEM
12
+ remote: https://rubygems.org/
13
+ specs:
14
+ actioncable (6.0.0)
15
+ actionpack (= 6.0.0)
16
+ nio4r (~> 2.0)
17
+ websocket-driver (>= 0.6.1)
18
+ actionmailbox (6.0.0)
19
+ actionpack (= 6.0.0)
20
+ activejob (= 6.0.0)
21
+ activerecord (= 6.0.0)
22
+ activestorage (= 6.0.0)
23
+ activesupport (= 6.0.0)
24
+ mail (>= 2.7.1)
25
+ actionmailer (6.0.0)
26
+ actionpack (= 6.0.0)
27
+ actionview (= 6.0.0)
28
+ activejob (= 6.0.0)
29
+ mail (~> 2.5, >= 2.5.4)
30
+ rails-dom-testing (~> 2.0)
31
+ actionpack (6.0.0)
32
+ actionview (= 6.0.0)
33
+ activesupport (= 6.0.0)
34
+ rack (~> 2.0)
35
+ rack-test (>= 0.6.3)
36
+ rails-dom-testing (~> 2.0)
37
+ rails-html-sanitizer (~> 1.0, >= 1.2.0)
38
+ actiontext (6.0.0)
39
+ actionpack (= 6.0.0)
40
+ activerecord (= 6.0.0)
41
+ activestorage (= 6.0.0)
42
+ activesupport (= 6.0.0)
43
+ nokogiri (>= 1.8.5)
44
+ actionview (6.0.0)
45
+ activesupport (= 6.0.0)
46
+ builder (~> 3.1)
47
+ erubi (~> 1.4)
48
+ rails-dom-testing (~> 2.0)
49
+ rails-html-sanitizer (~> 1.1, >= 1.2.0)
50
+ activejob (6.0.0)
51
+ activesupport (= 6.0.0)
52
+ globalid (>= 0.3.6)
53
+ activemodel (6.0.0)
54
+ activesupport (= 6.0.0)
55
+ activerecord (6.0.0)
56
+ activemodel (= 6.0.0)
57
+ activesupport (= 6.0.0)
58
+ activestorage (6.0.0)
59
+ actionpack (= 6.0.0)
60
+ activejob (= 6.0.0)
61
+ activerecord (= 6.0.0)
62
+ marcel (~> 0.3.1)
63
+ activesupport (6.0.0)
64
+ concurrent-ruby (~> 1.0, >= 1.0.2)
65
+ i18n (>= 0.7, < 2)
66
+ minitest (~> 5.1)
67
+ tzinfo (~> 1.1)
68
+ zeitwerk (~> 2.1, >= 2.1.8)
69
+ addressable (2.7.0)
70
+ public_suffix (>= 2.0.2, < 5.0)
71
+ appraisal (2.2.0)
72
+ bundler
73
+ rake
74
+ thor (>= 0.14.0)
75
+ ast (2.4.0)
76
+ builder (3.2.3)
77
+ concurrent-ruby (1.1.5)
78
+ crack (0.4.3)
79
+ safe_yaml (~> 1.0.0)
80
+ crass (1.0.5)
81
+ diff-lcs (1.3)
82
+ erubi (1.9.0)
83
+ et-orbi (1.2.2)
84
+ tzinfo
85
+ faraday (0.17.0)
86
+ multipart-post (>= 1.2, < 3)
87
+ fugit (1.3.3)
88
+ et-orbi (~> 1.1, >= 1.1.8)
89
+ raabro (~> 1.1)
90
+ globalid (0.4.2)
91
+ activesupport (>= 4.2.0)
92
+ google-cloud-tasks (1.3.1)
93
+ google-gax (~> 1.8)
94
+ googleapis-common-protos (>= 1.3.9, < 2.0)
95
+ googleapis-common-protos-types (>= 1.0.4, < 2.0)
96
+ grpc-google-iam-v1 (~> 0.6.9)
97
+ google-gax (1.8.1)
98
+ google-protobuf (~> 3.9)
99
+ googleapis-common-protos (>= 1.3.9, < 2.0)
100
+ googleauth (~> 0.9)
101
+ grpc (~> 1.24)
102
+ rly (~> 0.2.3)
103
+ google-protobuf (3.10.1-universal-darwin)
104
+ googleapis-common-protos (1.3.9)
105
+ google-protobuf (~> 3.0)
106
+ googleapis-common-protos-types (~> 1.0)
107
+ grpc (~> 1.0)
108
+ googleapis-common-protos-types (1.0.4)
109
+ google-protobuf (~> 3.0)
110
+ googleauth (0.10.0)
111
+ faraday (~> 0.12)
112
+ jwt (>= 1.4, < 3.0)
113
+ memoist (~> 0.16)
114
+ multi_json (~> 1.11)
115
+ os (>= 0.9, < 2.0)
116
+ signet (~> 0.12)
117
+ grpc (1.25.0-universal-darwin)
118
+ google-protobuf (~> 3.8)
119
+ googleapis-common-protos-types (~> 1.0)
120
+ grpc-google-iam-v1 (0.6.9)
121
+ googleapis-common-protos (>= 1.3.1, < 2.0)
122
+ grpc (~> 1.0)
123
+ hashdiff (1.0.0)
124
+ i18n (1.7.0)
125
+ concurrent-ruby (~> 1.0)
126
+ jaro_winkler (1.5.4)
127
+ jwt (2.2.1)
128
+ loofah (2.3.1)
129
+ crass (~> 1.0.2)
130
+ nokogiri (>= 1.5.9)
131
+ mail (2.7.1)
132
+ mini_mime (>= 0.1.1)
133
+ marcel (0.3.3)
134
+ mimemagic (~> 0.3.2)
135
+ memoist (0.16.1)
136
+ method_source (0.9.2)
137
+ mimemagic (0.3.3)
138
+ mini_mime (1.0.2)
139
+ mini_portile2 (2.4.0)
140
+ minitest (5.13.0)
141
+ multi_json (1.14.1)
142
+ multipart-post (2.1.1)
143
+ nio4r (2.5.2)
144
+ nokogiri (1.10.5)
145
+ mini_portile2 (~> 2.4.0)
146
+ os (1.0.1)
147
+ parallel (1.19.0)
148
+ parser (2.6.5.0)
149
+ ast (~> 2.4.0)
150
+ public_suffix (4.0.1)
151
+ raabro (1.1.6)
152
+ rack (2.0.7)
153
+ rack-test (1.1.0)
154
+ rack (>= 1.0, < 3)
155
+ rails (6.0.0)
156
+ actioncable (= 6.0.0)
157
+ actionmailbox (= 6.0.0)
158
+ actionmailer (= 6.0.0)
159
+ actionpack (= 6.0.0)
160
+ actiontext (= 6.0.0)
161
+ actionview (= 6.0.0)
162
+ activejob (= 6.0.0)
163
+ activemodel (= 6.0.0)
164
+ activerecord (= 6.0.0)
165
+ activestorage (= 6.0.0)
166
+ activesupport (= 6.0.0)
167
+ bundler (>= 1.3.0)
168
+ railties (= 6.0.0)
169
+ sprockets-rails (>= 2.0.0)
170
+ rails-dom-testing (2.0.3)
171
+ activesupport (>= 4.2.0)
172
+ nokogiri (>= 1.6)
173
+ rails-html-sanitizer (1.3.0)
174
+ loofah (~> 2.3)
175
+ railties (6.0.0)
176
+ actionpack (= 6.0.0)
177
+ activesupport (= 6.0.0)
178
+ method_source
179
+ rake (>= 0.8.7)
180
+ thor (>= 0.20.3, < 2.0)
181
+ rainbow (3.0.0)
182
+ rake (10.5.0)
183
+ redis (4.1.3)
184
+ rly (0.2.3)
185
+ rspec (3.9.0)
186
+ rspec-core (~> 3.9.0)
187
+ rspec-expectations (~> 3.9.0)
188
+ rspec-mocks (~> 3.9.0)
189
+ rspec-core (3.9.0)
190
+ rspec-support (~> 3.9.0)
191
+ rspec-expectations (3.9.0)
192
+ diff-lcs (>= 1.2.0, < 2.0)
193
+ rspec-support (~> 3.9.0)
194
+ rspec-mocks (3.9.0)
195
+ diff-lcs (>= 1.2.0, < 2.0)
196
+ rspec-support (~> 3.9.0)
197
+ rspec-rails (3.9.0)
198
+ actionpack (>= 3.0)
199
+ activesupport (>= 3.0)
200
+ railties (>= 3.0)
201
+ rspec-core (~> 3.9.0)
202
+ rspec-expectations (~> 3.9.0)
203
+ rspec-mocks (~> 3.9.0)
204
+ rspec-support (~> 3.9.0)
205
+ rspec-support (3.9.0)
206
+ rubocop (0.76.0)
207
+ jaro_winkler (~> 1.5.1)
208
+ parallel (~> 1.10)
209
+ parser (>= 2.6)
210
+ rainbow (>= 2.2.2, < 4.0)
211
+ ruby-progressbar (~> 1.7)
212
+ unicode-display_width (>= 1.4.0, < 1.7)
213
+ rubocop-rspec (1.36.0)
214
+ rubocop (>= 0.68.1)
215
+ ruby-progressbar (1.10.1)
216
+ safe_yaml (1.0.5)
217
+ signet (0.12.0)
218
+ addressable (~> 2.3)
219
+ faraday (~> 0.9)
220
+ jwt (>= 1.5, < 3.0)
221
+ multi_json (~> 1.10)
222
+ sprockets (4.0.0)
223
+ concurrent-ruby (~> 1.0)
224
+ rack (> 1, < 3)
225
+ sprockets-rails (3.2.1)
226
+ actionpack (>= 4.0)
227
+ activesupport (>= 4.0)
228
+ sprockets (>= 3.0.0)
229
+ sqlite3 (1.4.1)
230
+ thor (0.20.3)
231
+ thread_safe (0.3.6)
232
+ timecop (0.9.1)
233
+ tzinfo (1.2.5)
234
+ thread_safe (~> 0.1)
235
+ unicode-display_width (1.6.0)
236
+ webmock (3.7.6)
237
+ addressable (>= 2.3.6)
238
+ crack (>= 0.3.2)
239
+ hashdiff (>= 0.4.0, < 2.0.0)
240
+ websocket-driver (0.7.1)
241
+ websocket-extensions (>= 0.1.0)
242
+ websocket-extensions (0.1.4)
243
+ zeitwerk (2.2.1)
244
+
245
+ PLATFORMS
246
+ ruby
247
+
248
+ DEPENDENCIES
249
+ appraisal
250
+ bundler (~> 2.0)
251
+ cloudtasker!
252
+ rails (= 6.0)
253
+ rake (~> 10.0)
254
+ rspec (~> 3.0)
255
+ rspec-rails
256
+ rubocop (= 0.76.0)
257
+ rubocop-rspec
258
+ sqlite3
259
+ timecop
260
+ webmock
261
+
262
+ BUNDLED WITH
263
+ 2.0.2
@@ -6,11 +6,14 @@ require 'cloudtasker/version'
6
6
  require 'cloudtasker/config'
7
7
 
8
8
  require 'cloudtasker/authentication_error'
9
+ require 'cloudtasker/dead_worker_error'
9
10
  require 'cloudtasker/invalid_worker_error'
10
11
 
11
12
  require 'cloudtasker/middleware/chain'
12
13
  require 'cloudtasker/authenticator'
13
- require 'cloudtasker/task'
14
+ require 'cloudtasker/cloud_task'
15
+ require 'cloudtasker/worker_logger'
16
+ require 'cloudtasker/worker_handler'
14
17
  require 'cloudtasker/meta_store'
15
18
  require 'cloudtasker/worker'
16
19
 
@@ -25,9 +28,24 @@ module Cloudtasker
25
28
  yield(config)
26
29
  end
27
30
 
31
+ #
32
+ # Return the Cloudtasker configuration.
33
+ #
34
+ # @return [Cloudtasker::Config] The Cloudtasker configuration.
35
+ #
28
36
  def self.config
29
37
  @config ||= Config.new
30
38
  end
39
+
40
+ #
41
+ # Return the Cloudtasker logger.
42
+ #
43
+ # @return [Logger] The Cloudtasker logger.
44
+ #
45
+ def self.logger
46
+ config.logger
47
+ end
31
48
  end
32
49
 
50
+ require 'cloudtasker/railtie' if defined?(Rails)
33
51
  require 'cloudtasker/engine' if defined?(::Rails::Engine)
@@ -0,0 +1,139 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cloudtasker
4
+ module Backend
5
+ # Manage tasks pushed to GCP Cloud Task
6
+ class GoogleCloudTask
7
+ attr_accessor :gcp_task
8
+
9
+ #
10
+ # Create the queue configured in Cloudtasker if it does not already exist.
11
+ #
12
+ # @return [Google::Cloud::Tasks::V2beta3::Queue] The queue
13
+ #
14
+ def self.setup_queue
15
+ client.get_queue(queue_path)
16
+ rescue Google::Gax::RetryError
17
+ client.create_queue(
18
+ client.location_path(config.gcp_project_id, config.gcp_location_id),
19
+ name: queue_path,
20
+ retry_config: { max_attempts: -1 }
21
+ )
22
+ end
23
+
24
+ #
25
+ # Return the Google Cloud Task client.
26
+ #
27
+ # @return [Google::Cloud::Tasks] The Google Cloud Task client.
28
+ #
29
+ def self.client
30
+ @client ||= ::Google::Cloud::Tasks.new(version: :v2beta3)
31
+ end
32
+
33
+ #
34
+ # Return the cloudtasker configuration. See Cloudtasker#configure.
35
+ #
36
+ # @return [Cloudtasker::Config] The library configuration.
37
+ #
38
+ def self.config
39
+ Cloudtasker.config
40
+ end
41
+
42
+ #
43
+ # Return the fully qualified path for the Cloud Task queue.
44
+ #
45
+ # @return [String] The queue path.
46
+ #
47
+ def self.queue_path
48
+ client.queue_path(
49
+ config.gcp_project_id,
50
+ config.gcp_location_id,
51
+ config.gcp_queue_id
52
+ )
53
+ end
54
+
55
+ #
56
+ # Return a protobuf timestamp specifying how to wait
57
+ # before running a task.
58
+ #
59
+ # @param [Integer, nil] schedule_time A unix timestamp.
60
+ #
61
+ # @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
62
+ #
63
+ def self.format_schedule_time(schedule_time)
64
+ return nil unless schedule_time
65
+
66
+ # Generate protobuf timestamp
67
+ Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
68
+ end
69
+
70
+ #
71
+ # Find a task by id.
72
+ #
73
+ # @param [String] id The task id.
74
+ #
75
+ # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The retrieved task.
76
+ #
77
+ def self.find(id)
78
+ resp = client.get_task(id)
79
+ resp ? new(resp) : nil
80
+ rescue Google::Gax::RetryError
81
+ nil
82
+ end
83
+
84
+ #
85
+ # Create a new task.
86
+ #
87
+ # @param [Hash] payload The task payload.
88
+ #
89
+ # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The created task.
90
+ #
91
+ def self.create(payload)
92
+ # Format payload
93
+ payload = payload.merge(
94
+ schedule_time: format_schedule_time(payload[:schedule_time])
95
+ ).compact
96
+
97
+ # Create task
98
+ resp = client.create_task(queue_path, payload)
99
+ resp ? new(resp) : nil
100
+ rescue Google::Gax::RetryError
101
+ nil
102
+ end
103
+
104
+ #
105
+ # Delete a task by id.
106
+ #
107
+ # @param [String] id The id of the task.
108
+ #
109
+ def self.delete(id)
110
+ client.delete_task(id)
111
+ rescue Google::Gax::RetryError
112
+ nil
113
+ end
114
+
115
+ #
116
+ # Build a new instance of the class.
117
+ #
118
+ # @param [Google::Cloud::Tasks::V2beta3::Task] resp The GCP Cloud Task response
119
+ #
120
+ def initialize(gcp_task)
121
+ @gcp_task = gcp_task
122
+ end
123
+
124
+ #
125
+ # Return a hash description of the task.
126
+ #
127
+ # @return [Hash] A hash description of the task.
128
+ #
129
+ def to_h
130
+ {
131
+ id: gcp_task.name,
132
+ http_request: gcp_task.to_h[:http_request],
133
+ schedule_time: gcp_task.to_h.dig(:schedule_time, :seconds).to_i,
134
+ retries: gcp_task.to_h[:response_count]
135
+ }
136
+ end
137
+ end
138
+ end
139
+ end