cloudtasker 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +12 -0
  3. data/.rspec +3 -0
  4. data/.rubocop.yml +27 -0
  5. data/.travis.yml +7 -0
  6. data/CODE_OF_CONDUCT.md +74 -0
  7. data/Gemfile +6 -0
  8. data/Gemfile.lock +247 -0
  9. data/LICENSE.txt +21 -0
  10. data/README.md +43 -0
  11. data/Rakefile +8 -0
  12. data/app/controllers/cloudtasker/application_controller.rb +6 -0
  13. data/app/controllers/cloudtasker/worker_controller.rb +38 -0
  14. data/bin/console +15 -0
  15. data/bin/setup +8 -0
  16. data/cloudtasker.gemspec +48 -0
  17. data/config/routes.rb +5 -0
  18. data/lib/cloudtasker.rb +31 -0
  19. data/lib/cloudtasker/authentication_error.rb +6 -0
  20. data/lib/cloudtasker/authenticator.rb +55 -0
  21. data/lib/cloudtasker/batch.rb +5 -0
  22. data/lib/cloudtasker/batch/batch_progress.rb +97 -0
  23. data/lib/cloudtasker/batch/config.rb +11 -0
  24. data/lib/cloudtasker/batch/extension/worker.rb +13 -0
  25. data/lib/cloudtasker/batch/job.rb +320 -0
  26. data/lib/cloudtasker/batch/middleware.rb +24 -0
  27. data/lib/cloudtasker/batch/middleware/server.rb +14 -0
  28. data/lib/cloudtasker/config.rb +122 -0
  29. data/lib/cloudtasker/cron.rb +5 -0
  30. data/lib/cloudtasker/cron/config.rb +11 -0
  31. data/lib/cloudtasker/cron/job.rb +207 -0
  32. data/lib/cloudtasker/cron/middleware.rb +21 -0
  33. data/lib/cloudtasker/cron/middleware/server.rb +14 -0
  34. data/lib/cloudtasker/cron/schedule.rb +227 -0
  35. data/lib/cloudtasker/engine.rb +20 -0
  36. data/lib/cloudtasker/invalid_worker_error.rb +6 -0
  37. data/lib/cloudtasker/meta_store.rb +86 -0
  38. data/lib/cloudtasker/middleware/chain.rb +250 -0
  39. data/lib/cloudtasker/redis_client.rb +115 -0
  40. data/lib/cloudtasker/task.rb +175 -0
  41. data/lib/cloudtasker/unique_job.rb +5 -0
  42. data/lib/cloudtasker/unique_job/config.rb +10 -0
  43. data/lib/cloudtasker/unique_job/conflict_strategy/base_strategy.rb +37 -0
  44. data/lib/cloudtasker/unique_job/conflict_strategy/raise.rb +28 -0
  45. data/lib/cloudtasker/unique_job/conflict_strategy/reject.rb +11 -0
  46. data/lib/cloudtasker/unique_job/conflict_strategy/reschedule.rb +30 -0
  47. data/lib/cloudtasker/unique_job/job.rb +136 -0
  48. data/lib/cloudtasker/unique_job/lock/base_lock.rb +70 -0
  49. data/lib/cloudtasker/unique_job/lock/no_op.rb +11 -0
  50. data/lib/cloudtasker/unique_job/lock/until_executed.rb +34 -0
  51. data/lib/cloudtasker/unique_job/lock/until_executing.rb +30 -0
  52. data/lib/cloudtasker/unique_job/lock/while_executing.rb +23 -0
  53. data/lib/cloudtasker/unique_job/lock_error.rb +8 -0
  54. data/lib/cloudtasker/unique_job/middleware.rb +36 -0
  55. data/lib/cloudtasker/unique_job/middleware/client.rb +14 -0
  56. data/lib/cloudtasker/unique_job/middleware/server.rb +14 -0
  57. data/lib/cloudtasker/version.rb +5 -0
  58. data/lib/cloudtasker/worker.rb +211 -0
  59. metadata +286 -0
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cloudtasker/redis_client'
4
+
5
+ require_relative 'extension/worker'
6
+ require_relative 'config'
7
+ require_relative 'batch_progress'
8
+ require_relative 'job'
9
+
10
+ require_relative 'middleware/server'
11
+
12
+ module Cloudtasker
13
+ module Batch
14
+ # Registration module
15
+ module Middleware
16
+ def self.configure
17
+ Cloudtasker.configure do |config|
18
+ config.server_middleware { |c| c.add(Middleware::Server) }
19
+ end
20
+ Cloudtasker::Worker.include(Extension::Worker)
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cloudtasker
4
+ module Batch
5
+ module Middleware
6
+ # Server middleware, invoked when jobs are executed
7
+ class Server
8
+ def call(worker)
9
+ Job.for(worker).execute { yield }
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cloudtasker
4
+ # Holds cloudtasker configuration. See Cloudtasker#configure
5
+ class Config
6
+ attr_accessor :redis
7
+ attr_writer :secret, :gcp_location_id, :gcp_project_id,
8
+ :gcp_queue_id, :processor_host, :processor_path
9
+
10
+ DEFAULT_LOCATION_ID = 'us-east1'
11
+ DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'
12
+
13
+ PROCESSOR_HOST_MISSING = <<~DOC
14
+ Missing host for processing.
15
+ Please specify a processor hostname in form of `https://some-public-dns.example.com`'
16
+ DOC
17
+ QUEUE_ID_MISSING_ERROR = <<~DOC
18
+ Missing GCP queue ID.
19
+ Please specify a queue ID in the form of `my-queue-id`. You can create a queue using the Google SDK via `gcloud tasks queues create my-queue-id`
20
+ DOC
21
+ PROJECT_ID_MISSING_ERROR = <<~DOC
22
+ Missing GCP project ID.
23
+ Please specify a project ID in the cloudtasker configurator.
24
+ DOC
25
+ SECRET_MISSING_ERROR = <<~DOC
26
+ Missing cloudtasker secret.
27
+ Please specify a secret in the cloudtasker initializer or add Rails secret_key_base in your credentials
28
+ DOC
29
+
30
+ #
31
+ # Return the full URL of the processor. Worker payloads will be sent
32
+ # to this URL.
33
+ #
34
+ # @return [String] The processor URL.
35
+ #
36
+ def processor_url
37
+ File.join(processor_host, processor_path)
38
+ end
39
+
40
+ #
41
+ # The hostname of the application processing the workers. The hostname must
42
+ # be reachable from Cloud Task.
43
+ #
44
+ # @return [String] The processor host.
45
+ #
46
+ def processor_host
47
+ @processor_host || raise(StandardError, PROCESSOR_HOST_MISSING)
48
+ end
49
+
50
+ #
51
+ # The path on the host when worker payloads will be sent.
52
+ # Default to `/cloudtasker/run`
53
+ #
54
+ #
55
+ # @return [String] The processor path
56
+ #
57
+ def processor_path
58
+ @processor_path || DEFAULT_PROCESSOR_PATH
59
+ end
60
+
61
+ #
62
+ # Return the ID of GCP queue where tasks will be added.
63
+ #
64
+ # @return [String] The ID of the processing queue.
65
+ #
66
+ def gcp_queue_id
67
+ @gcp_queue_id || raise(StandardError, QUEUE_ID_MISSING_ERROR)
68
+ end
69
+
70
+ #
71
+ # Return the GCP project ID.
72
+ #
73
+ # @return [String] The ID of the project for which tasks will be processed.
74
+ #
75
+ def gcp_project_id
76
+ @gcp_project_id || raise(StandardError, PROJECT_ID_MISSING_ERROR)
77
+ end
78
+
79
+ #
80
+ # Return the GCP location ID. Default to 'us-east1'
81
+ #
82
+ # @return [String] The location ID where tasks will be processed.
83
+ #
84
+ def gcp_location_id
85
+ @gcp_location_id || DEFAULT_LOCATION_ID
86
+ end
87
+
88
+ #
89
+ # Return the secret to use to sign the verification tokens
90
+ # attached to tasks.
91
+ #
92
+ # @return [String] The cloudtasker secret
93
+ #
94
+ def secret
95
+ @secret || (
96
+ defined?(Rails) && Rails.application.credentials&.secret_key_base
97
+ ) || raise(StandardError, SECRET_MISSING_ERROR)
98
+ end
99
+
100
+ #
101
+ # Return the chain of client middlewares.
102
+ #
103
+ # @return [Cloudtasker::Middleware::Chain] The chain of middlewares.
104
+ #
105
+ def client_middleware
106
+ @client_middleware ||= Middleware::Chain.new
107
+ yield @client_middleware if block_given?
108
+ @client_middleware
109
+ end
110
+
111
+ #
112
+ # Return the chain of server middlewares.
113
+ #
114
+ # @return [Cloudtasker::Middleware::Chain] The chain of middlewares.
115
+ #
116
+ def server_middleware
117
+ @server_middleware ||= Middleware::Chain.new
118
+ yield @server_middleware if block_given?
119
+ @server_middleware
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'cron/middleware'
4
+
5
+ Cloudtasker::Cron::Middleware.configure
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fugit'
4
+
5
+ module Cloudtasker
6
+ module Cron
7
+ class Config
8
+ KEY_NAMESPACE = 'cloudtasker-cron'
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,207 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fugit'
4
+
5
+ module Cloudtasker
6
+ module Cron
7
+ # TODO: handle deletion of cron jobs
8
+ #
9
+ # Manage cron jobs
10
+ class Job
11
+ attr_reader :worker
12
+
13
+ #
14
+ # Build a new instance of the class
15
+ #
16
+ # @param [Cloudtasker::Worker] worker The cloudtasker worker
17
+ #
18
+ def initialize(worker)
19
+ @worker = worker
20
+ end
21
+
22
+ #
23
+ # Return a namespaced key
24
+ #
25
+ # @param [String, Symbol] val The key to namespace
26
+ #
27
+ # @return [String] The namespaced key.
28
+ #
29
+ def key(val)
30
+ return nil if val.nil?
31
+
32
+ [Config::KEY_NAMESPACE, val.to_s].join('/')
33
+ end
34
+
35
+ #
36
+ # Add cron metadata to the worker.
37
+ #
38
+ # @param [String, Symbol] name The name of the cron task.
39
+ # @param [String] cron The cron expression.
40
+ #
41
+ # @return [Cloudtasker::Cron::Job] self.
42
+ #
43
+ def set(schedule_id:)
44
+ worker.job_meta.set(key(:schedule_id), schedule_id.to_s)
45
+ self
46
+ end
47
+
48
+ #
49
+ # Return the worker id.
50
+ #
51
+ # @return [String] The worker id.
52
+ #
53
+ def job_id
54
+ worker.job_id
55
+ end
56
+
57
+ #
58
+ # Return the namespaced worker id.
59
+ #
60
+ # @return [String] The worker namespaced id.
61
+ #
62
+ def job_gid
63
+ key(job_id)
64
+ end
65
+
66
+ #
67
+ # Return the cron schedule id.
68
+ #
69
+ # @return [String] The schedule id.
70
+ #
71
+ def schedule_id
72
+ @schedule_id ||= worker.job_meta.get(key(:schedule_id))
73
+ end
74
+
75
+ #
76
+ # Return true if the worker is tagged as a cron job.
77
+ #
78
+ # @return [Boolean] True if the worker relates to a cron schedule.
79
+ #
80
+ def cron_job?
81
+ cron_schedule
82
+ end
83
+
84
+ #
85
+ # Return true if the worker is currently processing (includes retries).
86
+ #
87
+ # @return [Boolean] True f the worker is processing.
88
+ #
89
+ def retry_instance?
90
+ cron_job? && state
91
+ end
92
+
93
+ #
94
+ # Return the job processing state.
95
+ #
96
+ # @return [String, nil] The processing state.
97
+ #
98
+ def state
99
+ redis.get(job_gid)&.to_sym
100
+ end
101
+
102
+ #
103
+ # Return the cloudtasker redis client
104
+ #
105
+ # @return [Class] The redis client.
106
+ #
107
+ def redis
108
+ RedisClient
109
+ end
110
+
111
+ #
112
+ # Return the cron schedule to use for the job.
113
+ #
114
+ # @return [Fugit::Cron] The cron schedule.
115
+ #
116
+ def cron_schedule
117
+ return nil unless schedule_id
118
+
119
+ @cron_schedule ||= Cron::Schedule.find(schedule_id)
120
+ end
121
+
122
+ #
123
+ # Return the time this cron instance is expected to run at.
124
+ #
125
+ # @return [Time] The current cron instance time.
126
+ #
127
+ def current_time
128
+ @current_time ||=
129
+ begin
130
+ Time.parse(worker.job_meta.get(key(:time_at)).to_s)
131
+ rescue ArgumentError
132
+ Time.try(:current) || Time.now
133
+ end
134
+ end
135
+
136
+ #
137
+ # Return the Time when the job should run next.
138
+ #
139
+ # @return [EtOrbi::EoTime] The time the job should run next.
140
+ #
141
+ def next_time
142
+ @next_time ||= cron_schedule&.next_time(current_time)
143
+ end
144
+
145
+ #
146
+ # Return true if the cron job is the one we are expecting. This method
147
+ # is used to ensure that jobs related to outdated cron schedules do not
148
+ # get processed.
149
+ #
150
+ # @return [Boolean] True if the cron job is expected.
151
+ #
152
+ def expected_instance?
153
+ retry_instance? || cron_schedule.job_id == job_id
154
+ end
155
+
156
+ #
157
+ # Store the cron job instance state.
158
+ #
159
+ # @param [String, Symbol] state The worker state.
160
+ #
161
+ def flag(state)
162
+ state.to_sym == :done ? redis.del(job_gid) : redis.set(job_gid, state.to_s)
163
+ end
164
+
165
+ #
166
+ # Schedule the next cron instance.
167
+ #
168
+ # The task only gets scheduled the first time a worker runs for a
169
+ # given cron instance (Typically a cron worker failing and retrying will
170
+ # not lead to a new task getting scheduled).
171
+ #
172
+ def schedule!
173
+ return false unless cron_schedule
174
+
175
+ # Configure next cron worker
176
+ next_worker = worker.new_instance.tap { |e| e.job_meta.set(key(:time_at), next_time.iso8601) }
177
+
178
+ # Schedule next worker
179
+ resp = next_worker.schedule(time_at: next_time)
180
+ cron_schedule.update(task_id: resp.name, job_id: next_worker.job_id)
181
+ end
182
+
183
+ #
184
+ # Execute the (cron) job. This method is invoked by the cron middleware.
185
+ #
186
+ def execute
187
+ # Execute the job immediately if this worker is not flagged as a cron job.
188
+ return yield unless cron_job?
189
+
190
+ # Abort and reject job if this cron instance is not expected.
191
+ return true unless expected_instance?
192
+
193
+ # Schedule the next instance of the job
194
+ schedule! unless retry_instance?
195
+
196
+ # Flag the cron instance as processing.
197
+ flag(:processing)
198
+
199
+ # Execute the cron instance
200
+ yield
201
+
202
+ # Flag the cron instance as done
203
+ flag(:done)
204
+ end
205
+ end
206
+ end
207
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cloudtasker/redis_client'
4
+
5
+ require_relative 'config'
6
+ require_relative 'schedule'
7
+ require_relative 'job'
8
+ require_relative 'middleware/server'
9
+
10
+ module Cloudtasker
11
+ module Cron
12
+ # Registration module
13
+ module Middleware
14
+ def self.configure
15
+ Cloudtasker.configure do |config|
16
+ config.server_middleware { |c| c.add(Middleware::Server) }
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cloudtasker
4
+ module Cron
5
+ module Middleware
6
+ # Server middleware, invoked when jobs are executed
7
+ class Server
8
+ def call(worker)
9
+ Job.new(worker).execute { yield }
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,227 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fugit'
4
+
5
+ module Cloudtasker
6
+ module Cron
7
+ # Manage cron schedules
8
+ class Schedule
9
+ attr_accessor :id, :cron, :worker, :task_id, :job_id
10
+
11
+ #
12
+ # Return the redis client.
13
+ #
14
+ # @return [Class] The redis client
15
+ #
16
+ def self.redis
17
+ RedisClient
18
+ end
19
+
20
+ #
21
+ # Create a new cron schedule (or update an existing one).
22
+ #
23
+ # @param [Hash] **opts Init opts. See initialize
24
+ #
25
+ # @return [Cloudtasker::Cron::Schedule] The schedule instance.
26
+ #
27
+ def self.create(**opts)
28
+ config = find(opts[:id]).to_h.merge(opts)
29
+ new(config).tap(&:save)
30
+ end
31
+
32
+ #
33
+ # Return a saved cron schedule.
34
+ #
35
+ # @param [String] id The schedule id.
36
+ #
37
+ # @return [Cloudtasker::Cron::Schedule] The schedule instance.
38
+ #
39
+ def self.find(id)
40
+ gid = [Config::KEY_NAMESPACE, id].join('/')
41
+ return nil unless (schedule_config = redis.fetch(gid))
42
+
43
+ new(schedule_config)
44
+ end
45
+
46
+ #
47
+ # Destroy a schedule by id.
48
+ #
49
+ # @param [String] id The schedule id.
50
+ #
51
+ def self.delete(id)
52
+ schedule = find(id)
53
+ return false unless schedule
54
+
55
+ # Delete task and stored schedule
56
+ Task.delete(schedule.task_id) if schedule.task_id
57
+ redis.del(schedule.gid)
58
+ end
59
+
60
+ #
61
+ # Build a new instance of the class.
62
+ #
63
+ # @param [String] id The schedule id.
64
+ # @param [String] cron The cron expression.
65
+ # @param [Class] worker The worker class to run.
66
+ # @param [String] task_id The ID of the actual backend task.
67
+ # @param [String] job_id The ID of the Cloudtasker worker.
68
+ #
69
+ def initialize(id:, cron:, worker:, task_id: nil, job_id: nil)
70
+ @id = id
71
+ @cron = cron
72
+ @worker = worker
73
+ @task_id = task_id
74
+ @job_id = job_id
75
+ end
76
+
77
+ #
78
+ # Return the redis client.
79
+ #
80
+ # @return [Class] The redis client
81
+ #
82
+ def redis
83
+ self.class.redis
84
+ end
85
+
86
+ #
87
+ # Return the namespaced schedule id.
88
+ #
89
+ # @return [String] The namespaced schedule id.
90
+ #
91
+ def gid
92
+ [Config::KEY_NAMESPACE, id].join('/')
93
+ end
94
+
95
+ #
96
+ # Return true if the schedule is valid.
97
+ #
98
+ # @return [Boolean] True if the schedule is valid.
99
+ #
100
+ def valid?
101
+ id && cron_schedule && worker
102
+ end
103
+
104
+ #
105
+ # Equality operator.
106
+ #
107
+ # @param [Any] other The object to compare.
108
+ #
109
+ # @return [Boolean] True if the object is equal.
110
+ #
111
+ def ==(other)
112
+ other.is_a?(self.class) && other.id == id
113
+ end
114
+
115
+ #
116
+ # Return true if the configuration of the schedule was
117
+ # changed (cron expression or worker).
118
+ #
119
+ # @return [Boolean] True if the schedule config was changed.
120
+ #
121
+ def config_changed?
122
+ self.class.find(id)&.to_config != to_config
123
+ end
124
+
125
+ #
126
+ # RReturn true if the instance attributes were changed compared
127
+ # to the schedule saved in Redis.
128
+ #
129
+ # @return [Boolean] True if the schedule was modified.
130
+ #
131
+ def changed?
132
+ to_h != self.class.find(id).to_h
133
+ end
134
+
135
+ #
136
+ # Return a hash describing the configuration of this schedule.
137
+ #
138
+ # @return [Hash] The config description hash.
139
+ #
140
+ def to_config
141
+ {
142
+ id: id,
143
+ cron: cron,
144
+ worker: worker
145
+ }
146
+ end
147
+
148
+ #
149
+ # Return a hash with all the schedule attributes.
150
+ #
151
+ # @return [Hash] The attributes hash.
152
+ #
153
+ def to_h
154
+ {
155
+ id: id,
156
+ cron: cron,
157
+ worker: worker,
158
+ task_id: task_id,
159
+ job_id: job_id
160
+ }
161
+ end
162
+
163
+ #
164
+ # Return the cron schedule to use for the job.
165
+ #
166
+ # @return [Fugit::Cron] The cron schedule.
167
+ #
168
+ def cron_schedule
169
+ @cron_schedule ||= Fugit::Cron.parse(cron)
170
+ end
171
+
172
+ #
173
+ # Return the next time a job should run.
174
+ #
175
+ # @param [Time] time An optional reference in time (instead of Time.now)
176
+ #
177
+ # @return [EtOrbi::EoTime] The time the schedule job should run next.
178
+ #
179
+ def next_time(*args)
180
+ cron_schedule.next_time(*args)
181
+ end
182
+
183
+ #
184
+ # Buld edit the object attributes.
185
+ #
186
+ # @param [Hash] **opts The attributes to edit.
187
+ #
188
+ def assign_attributes(**opts)
189
+ opts
190
+ .select { |k, _| instance_variables.include?("@#{k}".to_sym) }
191
+ .each { |k, v| instance_variable_set("@#{k}", v) }
192
+ end
193
+
194
+ #
195
+ # Edit the object attributes and save the object in Redis.
196
+ #
197
+ # @param [Hash] **opts The attributes to edit.
198
+ #
199
+ def update(**opts)
200
+ assign_attributes(opts)
201
+ save
202
+ end
203
+
204
+ #
205
+ # Save the object in Redis. If the configuration was changed
206
+ # then any existing cloud task is removed and a task is recreated.
207
+ #
208
+ def save(update_task: true)
209
+ return false unless valid? && changed?
210
+
211
+ # Save schedule
212
+ config_was_changed = config_changed?
213
+ redis.write(gid, to_h)
214
+
215
+ # Stop there if backend does not need update
216
+ return true unless update_task && config_was_changed
217
+
218
+ # Delete previous instance
219
+ Task.delete(task_id) if task_id
220
+
221
+ # Schedule worker
222
+ worker_instance = Object.const_get(worker).new
223
+ Job.new(worker_instance).set(schedule_id: id).schedule!
224
+ end
225
+ end
226
+ end
227
+ end