cloudtasker 0.2.0 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -0
  3. data/.rubocop.yml +5 -0
  4. data/.travis.yml +10 -1
  5. data/Appraisals +25 -0
  6. data/Gemfile.lock +10 -4
  7. data/README.md +550 -4
  8. data/app/controllers/cloudtasker/application_controller.rb +2 -0
  9. data/app/controllers/cloudtasker/worker_controller.rb +22 -2
  10. data/cloudtasker.gemspec +4 -3
  11. data/docs/BATCH_JOBS.md +66 -0
  12. data/docs/CRON_JOBS.md +63 -0
  13. data/docs/UNIQUE_JOBS.md +127 -0
  14. data/exe/cloudtasker +15 -0
  15. data/gemfiles/.bundle/config +2 -0
  16. data/gemfiles/google_cloud_tasks_1.0.gemfile +9 -0
  17. data/gemfiles/google_cloud_tasks_1.0.gemfile.lock +263 -0
  18. data/gemfiles/google_cloud_tasks_1.1.gemfile +9 -0
  19. data/gemfiles/google_cloud_tasks_1.1.gemfile.lock +263 -0
  20. data/gemfiles/google_cloud_tasks_1.2.gemfile +9 -0
  21. data/gemfiles/google_cloud_tasks_1.2.gemfile.lock +263 -0
  22. data/gemfiles/google_cloud_tasks_1.3.gemfile +9 -0
  23. data/gemfiles/google_cloud_tasks_1.3.gemfile.lock +264 -0
  24. data/gemfiles/rails_4.0.gemfile +10 -0
  25. data/gemfiles/rails_4.1.gemfile +9 -0
  26. data/gemfiles/rails_4.2.gemfile +9 -0
  27. data/gemfiles/rails_5.0.gemfile +9 -0
  28. data/gemfiles/rails_5.1.gemfile +9 -0
  29. data/gemfiles/rails_5.2.gemfile +9 -0
  30. data/gemfiles/rails_5.2.gemfile.lock +247 -0
  31. data/gemfiles/rails_6.0.gemfile +9 -0
  32. data/gemfiles/rails_6.0.gemfile.lock +263 -0
  33. data/lib/cloudtasker.rb +19 -1
  34. data/lib/cloudtasker/backend/google_cloud_task.rb +139 -0
  35. data/lib/cloudtasker/backend/memory_task.rb +190 -0
  36. data/lib/cloudtasker/backend/redis_task.rb +248 -0
  37. data/lib/cloudtasker/batch/batch_progress.rb +19 -1
  38. data/lib/cloudtasker/batch/job.rb +81 -20
  39. data/lib/cloudtasker/cli.rb +194 -0
  40. data/lib/cloudtasker/cloud_task.rb +91 -0
  41. data/lib/cloudtasker/config.rb +64 -2
  42. data/lib/cloudtasker/cron/job.rb +2 -2
  43. data/lib/cloudtasker/cron/schedule.rb +15 -5
  44. data/lib/cloudtasker/dead_worker_error.rb +6 -0
  45. data/lib/cloudtasker/local_server.rb +74 -0
  46. data/lib/cloudtasker/railtie.rb +10 -0
  47. data/lib/cloudtasker/testing.rb +133 -0
  48. data/lib/cloudtasker/unique_job/job.rb +1 -1
  49. data/lib/cloudtasker/unique_job/lock/base_lock.rb +1 -1
  50. data/lib/cloudtasker/unique_job/lock/until_executed.rb +3 -1
  51. data/lib/cloudtasker/unique_job/lock/while_executing.rb +3 -1
  52. data/lib/cloudtasker/version.rb +1 -1
  53. data/lib/cloudtasker/worker.rb +59 -16
  54. data/lib/cloudtasker/{task.rb → worker_handler.rb} +10 -77
  55. data/lib/cloudtasker/worker_logger.rb +155 -0
  56. data/lib/tasks/setup_queue.rake +10 -0
  57. metadata +55 -6
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This file was generated by Appraisal
4
+
5
+ source 'https://rubygems.org'
6
+
7
+ gem 'rails', '6.0'
8
+
9
+ gemspec path: '../'
@@ -0,0 +1,263 @@
1
+ PATH
2
+ remote: ..
3
+ specs:
4
+ cloudtasker (0.2.0)
5
+ activesupport
6
+ fugit
7
+ google-cloud-tasks
8
+ jwt
9
+ redis
10
+
11
+ GEM
12
+ remote: https://rubygems.org/
13
+ specs:
14
+ actioncable (6.0.0)
15
+ actionpack (= 6.0.0)
16
+ nio4r (~> 2.0)
17
+ websocket-driver (>= 0.6.1)
18
+ actionmailbox (6.0.0)
19
+ actionpack (= 6.0.0)
20
+ activejob (= 6.0.0)
21
+ activerecord (= 6.0.0)
22
+ activestorage (= 6.0.0)
23
+ activesupport (= 6.0.0)
24
+ mail (>= 2.7.1)
25
+ actionmailer (6.0.0)
26
+ actionpack (= 6.0.0)
27
+ actionview (= 6.0.0)
28
+ activejob (= 6.0.0)
29
+ mail (~> 2.5, >= 2.5.4)
30
+ rails-dom-testing (~> 2.0)
31
+ actionpack (6.0.0)
32
+ actionview (= 6.0.0)
33
+ activesupport (= 6.0.0)
34
+ rack (~> 2.0)
35
+ rack-test (>= 0.6.3)
36
+ rails-dom-testing (~> 2.0)
37
+ rails-html-sanitizer (~> 1.0, >= 1.2.0)
38
+ actiontext (6.0.0)
39
+ actionpack (= 6.0.0)
40
+ activerecord (= 6.0.0)
41
+ activestorage (= 6.0.0)
42
+ activesupport (= 6.0.0)
43
+ nokogiri (>= 1.8.5)
44
+ actionview (6.0.0)
45
+ activesupport (= 6.0.0)
46
+ builder (~> 3.1)
47
+ erubi (~> 1.4)
48
+ rails-dom-testing (~> 2.0)
49
+ rails-html-sanitizer (~> 1.1, >= 1.2.0)
50
+ activejob (6.0.0)
51
+ activesupport (= 6.0.0)
52
+ globalid (>= 0.3.6)
53
+ activemodel (6.0.0)
54
+ activesupport (= 6.0.0)
55
+ activerecord (6.0.0)
56
+ activemodel (= 6.0.0)
57
+ activesupport (= 6.0.0)
58
+ activestorage (6.0.0)
59
+ actionpack (= 6.0.0)
60
+ activejob (= 6.0.0)
61
+ activerecord (= 6.0.0)
62
+ marcel (~> 0.3.1)
63
+ activesupport (6.0.0)
64
+ concurrent-ruby (~> 1.0, >= 1.0.2)
65
+ i18n (>= 0.7, < 2)
66
+ minitest (~> 5.1)
67
+ tzinfo (~> 1.1)
68
+ zeitwerk (~> 2.1, >= 2.1.8)
69
+ addressable (2.7.0)
70
+ public_suffix (>= 2.0.2, < 5.0)
71
+ appraisal (2.2.0)
72
+ bundler
73
+ rake
74
+ thor (>= 0.14.0)
75
+ ast (2.4.0)
76
+ builder (3.2.3)
77
+ concurrent-ruby (1.1.5)
78
+ crack (0.4.3)
79
+ safe_yaml (~> 1.0.0)
80
+ crass (1.0.5)
81
+ diff-lcs (1.3)
82
+ erubi (1.9.0)
83
+ et-orbi (1.2.2)
84
+ tzinfo
85
+ faraday (0.17.0)
86
+ multipart-post (>= 1.2, < 3)
87
+ fugit (1.3.3)
88
+ et-orbi (~> 1.1, >= 1.1.8)
89
+ raabro (~> 1.1)
90
+ globalid (0.4.2)
91
+ activesupport (>= 4.2.0)
92
+ google-cloud-tasks (1.3.1)
93
+ google-gax (~> 1.8)
94
+ googleapis-common-protos (>= 1.3.9, < 2.0)
95
+ googleapis-common-protos-types (>= 1.0.4, < 2.0)
96
+ grpc-google-iam-v1 (~> 0.6.9)
97
+ google-gax (1.8.1)
98
+ google-protobuf (~> 3.9)
99
+ googleapis-common-protos (>= 1.3.9, < 2.0)
100
+ googleauth (~> 0.9)
101
+ grpc (~> 1.24)
102
+ rly (~> 0.2.3)
103
+ google-protobuf (3.10.1-universal-darwin)
104
+ googleapis-common-protos (1.3.9)
105
+ google-protobuf (~> 3.0)
106
+ googleapis-common-protos-types (~> 1.0)
107
+ grpc (~> 1.0)
108
+ googleapis-common-protos-types (1.0.4)
109
+ google-protobuf (~> 3.0)
110
+ googleauth (0.10.0)
111
+ faraday (~> 0.12)
112
+ jwt (>= 1.4, < 3.0)
113
+ memoist (~> 0.16)
114
+ multi_json (~> 1.11)
115
+ os (>= 0.9, < 2.0)
116
+ signet (~> 0.12)
117
+ grpc (1.25.0-universal-darwin)
118
+ google-protobuf (~> 3.8)
119
+ googleapis-common-protos-types (~> 1.0)
120
+ grpc-google-iam-v1 (0.6.9)
121
+ googleapis-common-protos (>= 1.3.1, < 2.0)
122
+ grpc (~> 1.0)
123
+ hashdiff (1.0.0)
124
+ i18n (1.7.0)
125
+ concurrent-ruby (~> 1.0)
126
+ jaro_winkler (1.5.4)
127
+ jwt (2.2.1)
128
+ loofah (2.3.1)
129
+ crass (~> 1.0.2)
130
+ nokogiri (>= 1.5.9)
131
+ mail (2.7.1)
132
+ mini_mime (>= 0.1.1)
133
+ marcel (0.3.3)
134
+ mimemagic (~> 0.3.2)
135
+ memoist (0.16.1)
136
+ method_source (0.9.2)
137
+ mimemagic (0.3.3)
138
+ mini_mime (1.0.2)
139
+ mini_portile2 (2.4.0)
140
+ minitest (5.13.0)
141
+ multi_json (1.14.1)
142
+ multipart-post (2.1.1)
143
+ nio4r (2.5.2)
144
+ nokogiri (1.10.5)
145
+ mini_portile2 (~> 2.4.0)
146
+ os (1.0.1)
147
+ parallel (1.19.0)
148
+ parser (2.6.5.0)
149
+ ast (~> 2.4.0)
150
+ public_suffix (4.0.1)
151
+ raabro (1.1.6)
152
+ rack (2.0.7)
153
+ rack-test (1.1.0)
154
+ rack (>= 1.0, < 3)
155
+ rails (6.0.0)
156
+ actioncable (= 6.0.0)
157
+ actionmailbox (= 6.0.0)
158
+ actionmailer (= 6.0.0)
159
+ actionpack (= 6.0.0)
160
+ actiontext (= 6.0.0)
161
+ actionview (= 6.0.0)
162
+ activejob (= 6.0.0)
163
+ activemodel (= 6.0.0)
164
+ activerecord (= 6.0.0)
165
+ activestorage (= 6.0.0)
166
+ activesupport (= 6.0.0)
167
+ bundler (>= 1.3.0)
168
+ railties (= 6.0.0)
169
+ sprockets-rails (>= 2.0.0)
170
+ rails-dom-testing (2.0.3)
171
+ activesupport (>= 4.2.0)
172
+ nokogiri (>= 1.6)
173
+ rails-html-sanitizer (1.3.0)
174
+ loofah (~> 2.3)
175
+ railties (6.0.0)
176
+ actionpack (= 6.0.0)
177
+ activesupport (= 6.0.0)
178
+ method_source
179
+ rake (>= 0.8.7)
180
+ thor (>= 0.20.3, < 2.0)
181
+ rainbow (3.0.0)
182
+ rake (10.5.0)
183
+ redis (4.1.3)
184
+ rly (0.2.3)
185
+ rspec (3.9.0)
186
+ rspec-core (~> 3.9.0)
187
+ rspec-expectations (~> 3.9.0)
188
+ rspec-mocks (~> 3.9.0)
189
+ rspec-core (3.9.0)
190
+ rspec-support (~> 3.9.0)
191
+ rspec-expectations (3.9.0)
192
+ diff-lcs (>= 1.2.0, < 2.0)
193
+ rspec-support (~> 3.9.0)
194
+ rspec-mocks (3.9.0)
195
+ diff-lcs (>= 1.2.0, < 2.0)
196
+ rspec-support (~> 3.9.0)
197
+ rspec-rails (3.9.0)
198
+ actionpack (>= 3.0)
199
+ activesupport (>= 3.0)
200
+ railties (>= 3.0)
201
+ rspec-core (~> 3.9.0)
202
+ rspec-expectations (~> 3.9.0)
203
+ rspec-mocks (~> 3.9.0)
204
+ rspec-support (~> 3.9.0)
205
+ rspec-support (3.9.0)
206
+ rubocop (0.76.0)
207
+ jaro_winkler (~> 1.5.1)
208
+ parallel (~> 1.10)
209
+ parser (>= 2.6)
210
+ rainbow (>= 2.2.2, < 4.0)
211
+ ruby-progressbar (~> 1.7)
212
+ unicode-display_width (>= 1.4.0, < 1.7)
213
+ rubocop-rspec (1.36.0)
214
+ rubocop (>= 0.68.1)
215
+ ruby-progressbar (1.10.1)
216
+ safe_yaml (1.0.5)
217
+ signet (0.12.0)
218
+ addressable (~> 2.3)
219
+ faraday (~> 0.9)
220
+ jwt (>= 1.5, < 3.0)
221
+ multi_json (~> 1.10)
222
+ sprockets (4.0.0)
223
+ concurrent-ruby (~> 1.0)
224
+ rack (> 1, < 3)
225
+ sprockets-rails (3.2.1)
226
+ actionpack (>= 4.0)
227
+ activesupport (>= 4.0)
228
+ sprockets (>= 3.0.0)
229
+ sqlite3 (1.4.1)
230
+ thor (0.20.3)
231
+ thread_safe (0.3.6)
232
+ timecop (0.9.1)
233
+ tzinfo (1.2.5)
234
+ thread_safe (~> 0.1)
235
+ unicode-display_width (1.6.0)
236
+ webmock (3.7.6)
237
+ addressable (>= 2.3.6)
238
+ crack (>= 0.3.2)
239
+ hashdiff (>= 0.4.0, < 2.0.0)
240
+ websocket-driver (0.7.1)
241
+ websocket-extensions (>= 0.1.0)
242
+ websocket-extensions (0.1.4)
243
+ zeitwerk (2.2.1)
244
+
245
+ PLATFORMS
246
+ ruby
247
+
248
+ DEPENDENCIES
249
+ appraisal
250
+ bundler (~> 2.0)
251
+ cloudtasker!
252
+ rails (= 6.0)
253
+ rake (~> 10.0)
254
+ rspec (~> 3.0)
255
+ rspec-rails
256
+ rubocop (= 0.76.0)
257
+ rubocop-rspec
258
+ sqlite3
259
+ timecop
260
+ webmock
261
+
262
+ BUNDLED WITH
263
+ 2.0.2
@@ -6,11 +6,14 @@ require 'cloudtasker/version'
6
6
  require 'cloudtasker/config'
7
7
 
8
8
  require 'cloudtasker/authentication_error'
9
+ require 'cloudtasker/dead_worker_error'
9
10
  require 'cloudtasker/invalid_worker_error'
10
11
 
11
12
  require 'cloudtasker/middleware/chain'
12
13
  require 'cloudtasker/authenticator'
13
- require 'cloudtasker/task'
14
+ require 'cloudtasker/cloud_task'
15
+ require 'cloudtasker/worker_logger'
16
+ require 'cloudtasker/worker_handler'
14
17
  require 'cloudtasker/meta_store'
15
18
  require 'cloudtasker/worker'
16
19
 
@@ -25,9 +28,24 @@ module Cloudtasker
25
28
  yield(config)
26
29
  end
27
30
 
31
+ #
32
+ # Return the Cloudtasker configuration.
33
+ #
34
+ # @return [Cloudtasker::Config] The Cloudtasker configuration.
35
+ #
28
36
  def self.config
29
37
  @config ||= Config.new
30
38
  end
39
+
40
+ #
41
+ # Return the Cloudtasker logger.
42
+ #
43
+ # @return [Logger] The Cloudtasker logger.
44
+ #
45
+ def self.logger
46
+ config.logger
47
+ end
31
48
  end
32
49
 
50
+ require 'cloudtasker/railtie' if defined?(Rails)
33
51
  require 'cloudtasker/engine' if defined?(::Rails::Engine)
@@ -0,0 +1,139 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cloudtasker
4
+ module Backend
5
+ # Manage tasks pushed to GCP Cloud Task
6
+ class GoogleCloudTask
7
+ attr_accessor :gcp_task
8
+
9
+ #
10
+ # Create the queue configured in Cloudtasker if it does not already exist.
11
+ #
12
+ # @return [Google::Cloud::Tasks::V2beta3::Queue] The queue
13
+ #
14
+ def self.setup_queue
15
+ client.get_queue(queue_path)
16
+ rescue Google::Gax::RetryError
17
+ client.create_queue(
18
+ client.location_path(config.gcp_project_id, config.gcp_location_id),
19
+ name: queue_path,
20
+ retry_config: { max_attempts: -1 }
21
+ )
22
+ end
23
+
24
+ #
25
+ # Return the Google Cloud Task client.
26
+ #
27
+ # @return [Google::Cloud::Tasks] The Google Cloud Task client.
28
+ #
29
+ def self.client
30
+ @client ||= ::Google::Cloud::Tasks.new(version: :v2beta3)
31
+ end
32
+
33
+ #
34
+ # Return the cloudtasker configuration. See Cloudtasker#configure.
35
+ #
36
+ # @return [Cloudtasker::Config] The library configuration.
37
+ #
38
+ def self.config
39
+ Cloudtasker.config
40
+ end
41
+
42
+ #
43
+ # Return the fully qualified path for the Cloud Task queue.
44
+ #
45
+ # @return [String] The queue path.
46
+ #
47
+ def self.queue_path
48
+ client.queue_path(
49
+ config.gcp_project_id,
50
+ config.gcp_location_id,
51
+ config.gcp_queue_id
52
+ )
53
+ end
54
+
55
+ #
56
+ # Return a protobuf timestamp specifying how to wait
57
+ # before running a task.
58
+ #
59
+ # @param [Integer, nil] schedule_time A unix timestamp.
60
+ #
61
+ # @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
62
+ #
63
+ def self.format_schedule_time(schedule_time)
64
+ return nil unless schedule_time
65
+
66
+ # Generate protobuf timestamp
67
+ Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
68
+ end
69
+
70
+ #
71
+ # Find a task by id.
72
+ #
73
+ # @param [String] id The task id.
74
+ #
75
+ # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The retrieved task.
76
+ #
77
+ def self.find(id)
78
+ resp = client.get_task(id)
79
+ resp ? new(resp) : nil
80
+ rescue Google::Gax::RetryError
81
+ nil
82
+ end
83
+
84
+ #
85
+ # Create a new task.
86
+ #
87
+ # @param [Hash] payload The task payload.
88
+ #
89
+ # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The created task.
90
+ #
91
+ def self.create(payload)
92
+ # Format payload
93
+ payload = payload.merge(
94
+ schedule_time: format_schedule_time(payload[:schedule_time])
95
+ ).compact
96
+
97
+ # Create task
98
+ resp = client.create_task(queue_path, payload)
99
+ resp ? new(resp) : nil
100
+ rescue Google::Gax::RetryError
101
+ nil
102
+ end
103
+
104
+ #
105
+ # Delete a task by id.
106
+ #
107
+ # @param [String] id The id of the task.
108
+ #
109
+ def self.delete(id)
110
+ client.delete_task(id)
111
+ rescue Google::Gax::RetryError
112
+ nil
113
+ end
114
+
115
+ #
116
+ # Build a new instance of the class.
117
+ #
118
+ # @param [Google::Cloud::Tasks::V2beta3::Task] resp The GCP Cloud Task response
119
+ #
120
+ def initialize(gcp_task)
121
+ @gcp_task = gcp_task
122
+ end
123
+
124
+ #
125
+ # Return a hash description of the task.
126
+ #
127
+ # @return [Hash] A hash description of the task.
128
+ #
129
+ def to_h
130
+ {
131
+ id: gcp_task.name,
132
+ http_request: gcp_task.to_h[:http_request],
133
+ schedule_time: gcp_task.to_h.dig(:schedule_time, :seconds).to_i,
134
+ retries: gcp_task.to_h[:response_count]
135
+ }
136
+ end
137
+ end
138
+ end
139
+ end