acidic_job 1.0.0.pre22 → 1.0.0.pre25

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 892c6a2598d63440a7e749484e790b3f47380954b615bd7b2c4fc4c8de541759
4
- data.tar.gz: bf7383000fa8e0fb1c545e5ccb014c7123884eb14da77178dd088bb9b12e73e1
3
+ metadata.gz: 305652928d6e3948110121767a94388b2f569a0ccc4693944cde5df2938c9a4b
4
+ data.tar.gz: 1c612e9380eaa0f73dd194441f69a988015e304c55c047793ad292f8c46e6c19
5
5
  SHA512:
6
- metadata.gz: 87610f9f920ddb6e34c727a6b4c9e8d98d9ec556bff1dea221e3f588694b4e5f77b482985ba906cc62c5a3593e398e906e8dd6aa815e9c6d3031e410b71abfeb
7
- data.tar.gz: 7ca46ec550598f545d3baaafccf09cf082b6eaad01a1d6bb4b8f5a7298d9023b0b33c824183e3c5b30b3a1fbb2a9200286761b4e671adf39569a62b6ac392740
6
+ metadata.gz: f720cc6bebf8bdc4197295a3501b750e4b8b7741d7f04d6096aba426d45b37027d650664b038f1e45a32dc0763af9cd88ccec41c6d32dbdde56d0937a5af4df2
7
+ data.tar.gz: a7639fe45ebbea79331a8355a5f8a20fc95182341d575ca8a8040a0733cfe242f5e694ec98d1b442eea5235ccabf4280bd91f20f28ff1814b0bd32ffa95a8bff
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- acidic_job (1.0.0.pre22)
4
+ acidic_job (1.0.0.pre25)
5
5
  activerecord
6
6
  activesupport
7
7
  database_cleaner
@@ -8,39 +8,59 @@ module AcidicJob
8
8
 
9
9
  private
10
10
 
11
+ def was_awaited_job?
12
+ (acidic_job_run.present? && acidic_job_run.awaited_by.present?) ||
13
+ (staged_job_run.present? && staged_job_run.awaited_by.present?)
14
+ end
15
+
16
+ def reenqueue_awaited_by_job
17
+ run = staged_job_run&.awaited_by || acidic_job_run&.awaited_by
18
+
19
+ return unless run
20
+ return if run.batched_runs.outstanding.any?
21
+
22
+ current_step = run.workflow[run.recovery_point.to_s]
23
+ step_result = run.returning_to
24
+
25
+ job = run.job_class.constantize.deserialize(run.serialized_job)
26
+ # this needs to be explicitly set so that `was_workflow_job?` appropriately returns `true`
27
+ # which is what the `after_finish :reenqueue_awaited_by_job` check needs
28
+ job.instance_variable_set(:@acidic_job_run, run)
29
+
30
+ step = Step.new(current_step, run, job, step_result)
31
+ # TODO: WRITE REGRESSION TESTS FOR PARALLEL JOB FAILING AND RETRYING THE ORIGINAL STEP
32
+ step.progress
33
+
34
+ return if run.finished?
35
+
36
+ # when a batch of jobs for a step succeeds, we begin processing the `AcidicJob::Run` record again
37
+ # process_run(run)
38
+ run.update_column(:locked_at, nil)
39
+ job.enqueue
40
+ end
41
+
11
42
  def enqueue_step_parallel_jobs(jobs_or_jobs_getter, run, step_result)
12
- # `batch` is available from Sidekiq::Pro
13
- raise SidekiqBatchRequired unless defined?(Sidekiq::Batch)
14
-
15
- jobs = case jobs_or_jobs_getter
16
- when Array
17
- jobs_or_jobs_getter
18
- when Symbol, String
19
- method(jobs_or_jobs_getter).call
20
- end
21
-
22
- step_batch = Sidekiq::Batch.new
23
- # step_batch.description = "AcidicJob::Workflow Step: #{step}"
24
- step_batch.on(
25
- :success,
26
- "#{self.class.name}#step_done",
27
- # NOTE: options are marshalled through JSON so use only basic types.
28
- {
29
- "run_id" => run.id,
30
- "step_result_yaml" => step_result.to_yaml.strip,
31
- "parent_worker" => self.class.name,
32
- "job_names" => jobs.map { |job| job_name(job) }
33
- }
34
- )
35
-
36
- # NOTE: The jobs method is atomic.
37
- # All jobs created in the block are actually pushed atomically at the end of the block.
38
- # If an error is raised, none of the jobs will go to Redis.
39
- step_batch.jobs do
40
- jobs.each do |job|
41
- worker, args, kwargs = job_args_and_kwargs(job)
42
-
43
- worker.perform_async(*args, **kwargs)
43
+ awaited_jobs = case jobs_or_jobs_getter
44
+ when Array
45
+ jobs_or_jobs_getter
46
+ when Symbol, String
47
+ method(jobs_or_jobs_getter).call
48
+ end
49
+
50
+ AcidicJob::Run.transaction do
51
+ awaited_jobs.each do |awaited_job|
52
+ worker_class, args, kwargs = job_args_and_kwargs(awaited_job)
53
+
54
+ job = worker_class.new(*args, **kwargs)
55
+
56
+ AcidicJob::Run.create!(
57
+ staged: true,
58
+ awaited_by: run,
59
+ job_class: worker_class,
60
+ serialized_job: job.serialize_job(*args, **kwargs),
61
+ idempotency_key: job.idempotency_key
62
+ )
63
+ run.update(returning_to: step_result)
44
64
  end
45
65
  end
46
66
  end
@@ -58,17 +78,6 @@ module AcidicJob
58
78
  process_run(run)
59
79
  end
60
80
 
61
- def job_name(job)
62
- case job
63
- when Class, Symbol
64
- job.to_s
65
- when String
66
- job
67
- else
68
- job.class.name
69
- end
70
- end
71
-
72
81
  def job_args_and_kwargs(job)
73
82
  case job
74
83
  when Class
@@ -13,7 +13,11 @@ module AcidicJob
13
13
  class_methods do
14
14
  # called only from `AcidicJob::Run#enqueue_staged_job`
15
15
  def deserialize(serialized_job_hash)
16
- klass = serialized_job_hash["class"].constantize
16
+ klass = if serialized_job_hash["class"].is_a?(Class)
17
+ serialized_job_hash["class"]
18
+ else
19
+ serialized_job_hash["class"].constantize
20
+ end
17
21
  worker = klass.new
18
22
  worker.jid = serialized_job_hash["jid"]
19
23
  worker.instance_variable_set(:@args, serialized_job_hash["args"])
@@ -54,7 +58,7 @@ module AcidicJob
54
58
  ::Sidekiq::Client.push(
55
59
  "class" => self.class,
56
60
  "args" => @args,
57
- "jid" => @jid
61
+ "jid" => @acidic_job_run&.staged_job_id || @jid
58
62
  )
59
63
  end
60
64
  end
@@ -20,7 +20,15 @@ module AcidicJob
20
20
  end
21
21
  end
22
22
 
23
- value || value_from_job_args(hash_or_job, *args, **kwargs)
23
+ result = value || value_from_job_args(hash_or_job, *args, **kwargs)
24
+
25
+ if result.start_with?("STG__")
26
+ # "STG__#{idempotency_key}__#{encoded_global_id}"
27
+ _prefix, idempotency_key, _encoded_global_id = result.split("__")
28
+ idempotency_key
29
+ else
30
+ result
31
+ end
24
32
  end
25
33
 
26
34
  private
@@ -12,11 +12,15 @@ module AcidicJob
12
12
 
13
13
  self.table_name = "acidic_job_runs"
14
14
 
15
+ belongs_to :awaited_by, class_name: "AcidicJob::Run", optional: true
16
+ has_many :batched_runs, class_name: "AcidicJob::Run", foreign_key: "awaited_by_id"
17
+
15
18
  after_create_commit :enqueue_staged_job, if: :staged?
16
19
 
17
20
  serialize :error_object
18
21
  serialize :serialized_job
19
22
  serialize :workflow
23
+ serialize :returning_to
20
24
  store :attr_accessors
21
25
 
22
26
  validates :staged, inclusion: { in: [true, false] } # uses database default
@@ -27,7 +31,9 @@ module AcidicJob
27
31
  scope :staged, -> { where(staged: true) }
28
32
  scope :unstaged, -> { where(staged: false) }
29
33
  scope :finished, -> { where(recovery_point: FINISHED_RECOVERY_POINT) }
30
- scope :running, -> { where.not(recovery_point: FINISHED_RECOVERY_POINT) }
34
+ scope :outstanding, lambda {
35
+ where.not(recovery_point: FINISHED_RECOVERY_POINT).or(where(recovery_point: [nil, ""]))
36
+ }
31
37
 
32
38
  with_options unless: :staged? do
33
39
  validates :last_run_at, presence: true
@@ -60,16 +66,19 @@ module AcidicJob
60
66
  error_object.present?
61
67
  end
62
68
 
63
- private
64
-
65
- def enqueue_staged_job
66
- return unless staged?
67
-
69
+ def staged_job_id
68
70
  # encode the identifier for this record in the job ID
69
71
  # base64 encoding for minimal security
70
72
  global_id = to_global_id.to_s.remove("gid://")
71
73
  encoded_global_id = Base64.encode64(global_id).strip
72
- staged_job_id = "STG_#{idempotency_key}__#{encoded_global_id}"
74
+
75
+ "STG__#{idempotency_key}__#{encoded_global_id}"
76
+ end
77
+
78
+ private
79
+
80
+ def enqueue_staged_job
81
+ return unless staged?
73
82
 
74
83
  serialized_staged_job = if serialized_job.key?("jid")
75
84
  serialized_job.merge("jid" => staged_job_id)
@@ -7,17 +7,10 @@ module AcidicJob
7
7
  module Staging
8
8
  extend ActiveSupport::Concern
9
9
 
10
- def delete_staged_job_record
11
- return unless was_staged_job?
12
-
13
- staged_job_run.delete
14
- true
15
- rescue ActiveRecord::RecordNotFound
16
- true
17
- end
10
+ private
18
11
 
19
12
  def was_staged_job?
20
- identifier.start_with? "STG_"
13
+ identifier.start_with? "STG__"
21
14
  end
22
15
 
23
16
  def staged_job_run
@@ -26,6 +19,8 @@ module AcidicJob
26
19
  staged_job_gid = "gid://#{Base64.decode64(encoded_global_id)}"
27
20
 
28
21
  GlobalID::Locator.locate(staged_job_gid)
22
+ rescue ActiveRecord::RecordNotFound
23
+ nil
29
24
  end
30
25
 
31
26
  def identifier
@@ -33,7 +28,9 @@ module AcidicJob
33
28
  return job_id if defined?(job_id) && !job_id.nil?
34
29
 
35
30
  # might be defined already in `with_acidity` method
36
- @__acidic_job_idempotency_key ||= IdempotencyKey.value_for(self, @__acidic_job_args, @__acidic_job_kwargs)
31
+ acidic_identifier = self.class.acidic_identifier
32
+ @__acidic_job_idempotency_key ||= IdempotencyKey.new(acidic_identifier)
33
+ .value_for(self, *@__acidic_job_args, **@__acidic_job_kwargs)
37
34
 
38
35
  @__acidic_job_idempotency_key
39
36
  end
@@ -42,7 +42,13 @@ module AcidicJob
42
42
  # The progression phase advances the job run state machine onto the next step
43
43
  def progress
44
44
  @run.with_lock do
45
- @step_result.call(run: @run)
45
+ if @step_result.is_a?(FinishedPoint)
46
+ @job.run_callbacks :finish do
47
+ @step_result.call(run: @run)
48
+ end
49
+ else
50
+ @step_result.call(run: @run)
51
+ end
46
52
  end
47
53
  end
48
54
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module AcidicJob
4
- VERSION = "1.0.0.pre22"
4
+ VERSION = "1.0.0.pre25"
5
5
  end
data/lib/acidic_job.rb CHANGED
@@ -48,8 +48,12 @@ module AcidicJob
48
48
  end
49
49
 
50
50
  # TODO: write test for a staged job that uses awaits
51
- klass.set_callback :perform, :after, :delete_staged_job_record, if: :was_staged_job?
51
+ klass.set_callback :perform, :after, :reenqueue_awaited_by_job,
52
+ if: -> { was_awaited_job? && !was_workflow_job? }
53
+ klass.set_callback :perform, :after, :finish_staged_job, if: -> { was_staged_job? && !was_workflow_job? }
52
54
  klass.define_callbacks :finish
55
+ klass.set_callback :finish, :after, :reenqueue_awaited_by_job,
56
+ if: -> { was_workflow_job? && was_awaited_job? }
53
57
 
54
58
  klass.instance_variable_set(:@acidic_identifier, :job_id)
55
59
  klass.define_singleton_method(:acidic_by_job_id) { @acidic_identifier = :job_id }
@@ -133,9 +137,17 @@ module AcidicJob
133
137
 
134
138
  private
135
139
 
140
+ def finish_staged_job
141
+ FinishedPoint.new.call(run: staged_job_run)
142
+ end
143
+
144
+ def was_workflow_job?
145
+ defined?(@acidic_job_run) && @acidic_job_run.present?
146
+ end
147
+
136
148
  def process_run(run)
137
149
  # if the run record is already marked as finished, immediately return its result
138
- return finish_run(run) if run.finished?
150
+ return run.succeeded? if run.finished?
139
151
 
140
152
  # otherwise, we will enter a loop to process each step of the workflow
141
153
  loop do
@@ -174,13 +186,7 @@ module AcidicJob
174
186
  end
175
187
 
176
188
  # the loop will break once the job is finished, so simply report the status
177
- finish_run(run)
178
- end
179
-
180
- def finish_run(run)
181
- run_callbacks :finish do
182
- run.succeeded?
183
- end
189
+ run.succeeded?
184
190
  end
185
191
 
186
192
  def step(method_name, awaits: [], for_each: nil)
@@ -223,9 +229,7 @@ module AcidicJob
223
229
  if run.present?
224
230
  # Programs enqueuing multiple jobs with different parameters but the
225
231
  # same idempotency key is a bug.
226
- # NOTE: WOULD THE ENQUEUED_AT OR CREATED_AT FIELD BE NECESSARILY DIFFERENT?
227
- if run.serialized_job.except("jid", "job_id",
228
- "enqueued_at") != serialized_job.except("jid", "job_id", "enqueued_at")
232
+ if run.serialized_job.slice("args", "arguments") != serialized_job.slice("args", "arguments")
229
233
  raise MismatchedIdempotencyKeyAndJobArguments
230
234
  end
231
235
 
@@ -234,7 +238,14 @@ module AcidicJob
234
238
  raise LockedIdempotencyKey if run.locked_at && run.locked_at > Time.current - IDEMPOTENCY_KEY_LOCK_TIMEOUT
235
239
 
236
240
  # Lock the run and update latest run unless the job is already finished.
237
- run.update!(last_run_at: Time.current, locked_at: Time.current, workflow: workflow) unless run.finished?
241
+ unless run.finished?
242
+ run.update!(
243
+ last_run_at: Time.current,
244
+ locked_at: Time.current,
245
+ workflow: workflow,
246
+ recovery_point: run.recovery_point || workflow.first.first
247
+ )
248
+ end
238
249
  else
239
250
  run = Run.create!(
240
251
  staged: false,
@@ -1,19 +1,19 @@
1
1
  class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
2
2
  def change
3
3
  create_table :acidic_job_runs do |t|
4
- t.boolean :staged, null: false, default: -> { false }
5
- t.string :idempotency_key, null: false
6
- t.text :serialized_job, null: false
7
- t.string :job_class, null: false
8
- t.datetime :last_run_at, null: true, default: -> { "CURRENT_TIMESTAMP" }
9
- t.datetime :locked_at, null: true
10
- t.string :recovery_point, null: true
11
- t.text :error_object, null: true
12
- t.text :attr_accessors, null: true
13
- t.text :workflow, null: true
4
+ t.boolean :staged, null: false, default: -> { false }
5
+ t.string :idempotency_key, null: false, index: { unique: true }
6
+ t.text :serialized_job, null: false
7
+ t.string :job_class, null: false
8
+ t.datetime :last_run_at, null: true, default: -> { "CURRENT_TIMESTAMP" }
9
+ t.datetime :locked_at, null: true
10
+ t.string :recovery_point, null: true
11
+ t.text :error_object, null: true
12
+ t.text :attr_accessors, null: true
13
+ t.text :workflow, null: true
14
+ t.references :awaited_by, null: true, index: true
15
+ t.text :returning_to, null: true
14
16
  t.timestamps
15
-
16
- t.index :idempotency_key, unique: true
17
17
  end
18
18
  end
19
19
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: acidic_job
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0.pre22
4
+ version: 1.0.0.pre25
5
5
  platform: ruby
6
6
  authors:
7
7
  - fractaledmind
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-05-17 00:00:00.000000000 Z
11
+ date: 2022-05-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord