pgbus 0.24.0 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/app/controllers/pgbus/deploy_tasks_controller.rb +11 -1
  4. data/app/helpers/pgbus/deploy_tasks_helper.rb +20 -0
  5. data/app/models/pgbus/deploy_task.rb +23 -1
  6. data/app/models/pgbus/deploy_task_run.rb +47 -5
  7. data/app/views/pgbus/deploy_tasks/_tasks_table.html.erb +3 -0
  8. data/app/views/pgbus/deploy_tasks/show.html.erb +25 -2
  9. data/config/i18n-tasks.yml +1 -0
  10. data/config/locales/da.yml +18 -0
  11. data/config/locales/de.yml +18 -0
  12. data/config/locales/en.yml +18 -0
  13. data/config/locales/es.yml +18 -0
  14. data/config/locales/fi.yml +18 -0
  15. data/config/locales/fr.yml +18 -0
  16. data/config/locales/it.yml +18 -0
  17. data/config/locales/ja.yml +18 -0
  18. data/config/locales/nb.yml +18 -0
  19. data/config/locales/nl.yml +18 -0
  20. data/config/locales/pt.yml +18 -0
  21. data/config/locales/sv.yml +18 -0
  22. data/lib/generators/pgbus/add_deploy_task_run_outcomes_generator.rb +48 -0
  23. data/lib/generators/pgbus/templates/add_deploy_task_run_outcomes.rb.erb +12 -0
  24. data/lib/generators/pgbus/templates/add_deploy_tasks_tables.rb.erb +5 -0
  25. data/lib/generators/pgbus/templates/deploy_tasks.yml.erb +6 -0
  26. data/lib/generators/pgbus/templates/migration.rb.erb +5 -0
  27. data/lib/pgbus/active_job/adapter.rb +13 -0
  28. data/lib/pgbus/active_job/deploy_task_run_id.rb +53 -0
  29. data/lib/pgbus/active_job/executor.rb +57 -0
  30. data/lib/pgbus/deploy/dependencies.rb +115 -0
  31. data/lib/pgbus/deploy/runner.rb +103 -18
  32. data/lib/pgbus/deploy/schema.rb +65 -0
  33. data/lib/pgbus/deploy/task.rb +23 -5
  34. data/lib/pgbus/engine.rb +1 -0
  35. data/lib/pgbus/generators/migration_detector.rb +13 -3
  36. data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -1
  37. data/lib/pgbus/version.rb +1 -1
  38. data/lib/pgbus/web/data_source/deploy_task_list.rb +39 -6
  39. data/lib/pgbus/web/deploy_task_state.rb +7 -5
  40. metadata +6 -1
@@ -44,6 +44,7 @@ module Pgbus
44
44
  source_queue: physical_queue(queue_name, source_queue))
45
45
  signal_concurrency(payload)
46
46
  signal_batch_discarded(payload)
47
+ record_deploy_task_run_outcome(payload, :failed)
47
48
  release_uniqueness_lock(Uniqueness.extract_key(payload), queue_name, message) if holds_uniqueness_row?(payload)
48
49
  record_stat(payload, queue_name, "dead_lettered", execution_start, message: message)
49
50
  instrument(
@@ -94,6 +95,7 @@ module Pgbus
94
95
  Pgbus.logger.debug { "[Pgbus::Executor] deserialized #{log_tag(message, queue_name)} job_class=#{job_class}" }
95
96
  job_succeeded = false
96
97
  retried = false
98
+ deploy_run_outcome = nil
97
99
 
98
100
  msg_id = message.msg_id.to_i
99
101
  instrument_payload = {
@@ -116,6 +118,7 @@ module Pgbus
116
118
  # job data, so hand it to the job before perform — that is what makes
117
119
  # `batch` (and `batch.enqueue` for open batches) work inside a job.
118
120
  assign_batch_id(job, payload)
121
+ mark_deploy_task_job(job, payload) if payload.key?(DeployTaskRunId::METADATA_KEY)
119
122
  Pgbus.logger.debug { "[Pgbus::Executor] running #{log_tag(message, queue_name)} job_class=#{job_class}" }
120
123
  with_visibility_heartbeat(job, queue_name, msg_id, source_queue, payload) do
121
124
  Delivery.with(msg_id: msg_id, read_ct: read_count, queue_name: queue_name,
@@ -127,6 +130,13 @@ module Pgbus
127
130
  # this attempt is done (archive it) but the job is not — the retry
128
131
  # message carries the batch tag and signals on its own outcome.
129
132
  retried = Batch.retry_reenqueued?(payload["job_id"])
133
+ # The same for a deploy task's job, batch or not: :completed when
134
+ # perform returned having done its work, nil when its retry_on
135
+ # retry is live (the retry records the outcome), :failed when the
136
+ # adapter dropped that retry or ActiveJob discarded the job
137
+ # (discard_on, a retry_on giving up through its block). Asked only
138
+ # of a marked job: an unmarked one pays one Hash lookup.
139
+ deploy_run_outcome = deploy_task_run_outcome(job) if payload.key?(DeployTaskRunId::METADATA_KEY)
130
140
  Pgbus.logger.debug { "[Pgbus::Executor] perform_returned #{log_tag(message, queue_name)} job_class=#{job_class}" }
131
141
  # Archiving is the exact-once claim on this execution.
132
142
  # :already_archived means another worker archived the message (our
@@ -200,6 +210,7 @@ module Pgbus
200
210
  # key if the first DELETE committed but the client raised afterward.
201
211
  signal_concurrency(payload)
202
212
  signal_batch_completed(payload) unless retried
213
+ record_deploy_task_run_outcome(payload, deploy_run_outcome) if deploy_run_outcome
203
214
  end
204
215
  Batch.clear_retry_reenqueued
205
216
  end
@@ -432,6 +443,7 @@ module Pgbus
432
443
  end
433
444
  signal_concurrency(payload)
434
445
  signal_batch_discarded(payload)
446
+ record_deploy_task_run_outcome(payload, :failed)
435
447
  instrument("pgbus.job_discarded", queue: queue_name, job_class: payload["job_class"], job_id: payload["job_id"],
436
448
  provider_job_id: payload["provider_job_id"], msg_id: msg_id, reason: "uniqueness")
437
449
  Pgbus.logger.info do
@@ -520,6 +532,51 @@ module Pgbus
520
532
  Pgbus.logger.warn { "[Pgbus] Batch discard signal failed: #{e.message}" }
521
533
  end
522
534
 
535
+ # The engine includes DeployTaskRunId on every job. A class without it
536
+ # (no engine) gets it on this instance, as Deploy::Runner does on the
537
+ # enqueue side, so a discard is still seen and a retry still carries the
538
+ # run id.
539
+ def mark_deploy_task_job(job, payload)
540
+ return if job.is_a?(DeployTaskRunId)
541
+
542
+ job.extend(DeployTaskRunId)
543
+ job.deploy_task_run_id = payload[DeployTaskRunId::METADATA_KEY]
544
+ end
545
+
546
+ # The outcome of a marked job's run once perform returned. nil: its
547
+ # retry is live, and the retry records it. :failed: the adapter dropped
548
+ # the retry, or a before_enqueue callback aborted it before the adapter
549
+ # saw it (ActiveJob then leaves successfully_enqueued false, where an
550
+ # untried enqueue leaves it nil); or ActiveJob discarded the job, which
551
+ # returned without doing its work (issue #662). A live retry wins over
552
+ # the discard, so a discard block that re-enqueues leaves the outcome
553
+ # to that retry. :completed otherwise.
554
+ def deploy_task_run_outcome(job)
555
+ requeue = deploy_task_run_requeue(job)
556
+ return if requeue
557
+ return :failed if requeue == false || job.try(:pgbus_discarded?)
558
+
559
+ :completed
560
+ end
561
+
562
+ def deploy_task_run_requeue(job)
563
+ flag = job.try(:deploy_task_run_enqueued)
564
+ return flag unless flag.nil?
565
+
566
+ job.successfully_enqueued? == false ? false : nil
567
+ end
568
+
569
+ # Stamps a deploy task run's outcome (issue #660). Best-effort, like
570
+ # record_stat: a failed write is reported and never fails the job.
571
+ def record_deploy_task_run_outcome(payload, outcome)
572
+ run_id = payload[DeployTaskRunId::METADATA_KEY]
573
+ return unless run_id && Deploy::Schema.outcomes?
574
+
575
+ outcome == :completed ? DeployTaskRun.mark_completed!(run_id) : DeployTaskRun.mark_failed!(run_id)
576
+ rescue StandardError => e
577
+ ErrorReporter.report(e, { action: "deploy_task_run_outcome", deploy_task_run_id: run_id, outcome: outcome })
578
+ end
579
+
523
580
  # Archiving is idempotent — archiving an already-archived message is a
524
581
  # no-op — so a connection error here is safe to retry once, unlike sends
525
582
  # where a retry could duplicate the message. Without the retry, a job
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "tsort"
4
+
5
+ module Pgbus
6
+ module Deploy
7
+ # after: (issue #660): which declared tasks can wait on which, and whether
8
+ # a task's dependencies have finished.
9
+ #
10
+ # A dependency is satisfied by a completed run (rollover or manual) at its
11
+ # current version, at any revision; an every_deploy dependency needs one on
12
+ # this revision. A run before the outcomes migration has no outcome, so it
13
+ # satisfies nothing.
14
+ class Dependencies
15
+ NOT_RUN = "not_run"
16
+ RUNNING = "running"
17
+ FAILED = "failed"
18
+
19
+ # +tasks+ (valid Deploy::Task objects) minus those whose after: can
20
+ # never be met: an unknown key, a cycle, or a dependency that is itself
21
+ # skipped. Returns [kept tasks, { skipped key => reason }].
22
+ def self.prune(tasks, declared_keys)
23
+ valid = tasks.to_h { |task| [task.key, task] }
24
+ skipped = {}
25
+ loop do
26
+ unmet = valid.values.filter_map { |task| (reason = unmet(task, valid, declared_keys)) && [task.key, reason] }
27
+ unmet = cycles(valid) if unmet.empty?
28
+ break if unmet.empty?
29
+
30
+ unmet.each do |key, reason|
31
+ skipped[key] = reason
32
+ valid.delete(key)
33
+ end
34
+ end
35
+ [valid.values, skipped]
36
+ end
37
+
38
+ def self.unmet(task, valid, declared_keys)
39
+ missing = task.dependencies.find { |dep| !valid.key?(dep) }
40
+ return unless missing
41
+
42
+ declared_keys.include?(missing) ? "after: '#{missing}' is skipped" : "after: unknown task '#{missing}'"
43
+ end
44
+
45
+ # [key, reason] for every task on a cycle (Task#valid? already rejects
46
+ # a task naming itself).
47
+ def self.cycles(valid)
48
+ graph = valid.transform_values { |task| task.dependencies.select { |dep| valid.key?(dep) }.sort }
49
+ components = TSort.strongly_connected_components(graph.method(:each_key), ->(key, &blk) { graph[key].each(&blk) })
50
+ components.select { |keys| keys.size > 1 }.flat_map do |keys|
51
+ reason = "after: cycle #{cycle_path(keys.min, graph, keys.to_set).join(" → ")}"
52
+ keys.sort.map { |key| [key, reason] }
53
+ end
54
+ end
55
+
56
+ # A path from +start+ back to itself inside its strongly connected
57
+ # component, which always has one.
58
+ def self.cycle_path(start, graph, members)
59
+ stack = [[start, [start]]]
60
+ seen = Set[start]
61
+ until stack.empty?
62
+ key, path = stack.pop
63
+ graph[key].each do |dep|
64
+ next unless members.include?(dep)
65
+ return path + [start] if dep == start
66
+ next unless seen.add?(dep)
67
+
68
+ stack << [dep, path + [dep]]
69
+ end
70
+ end
71
+ end
72
+ private_class_method :unmet, :cycles, :cycle_path
73
+
74
+ # The outcomes of +dependencies+ (anything with key, version and
75
+ # every_deploy: Deploy::Task or DeployTask), in one query.
76
+ def self.load(dependencies, revision:)
77
+ new(DeployTaskRun.outcomes_for(dependencies.map(&:key).uniq), revision: revision)
78
+ end
79
+
80
+ def initialize(outcomes, revision:)
81
+ @outcomes = outcomes.group_by(&:task_key)
82
+ @revision = revision
83
+ end
84
+
85
+ # nil when +dependency+ is satisfied, else NOT_RUN, RUNNING (the newest
86
+ # run at its version has no outcome yet) or FAILED.
87
+ def state(dependency)
88
+ current = @outcomes.fetch(dependency.key, []).select { |outcome| current?(dependency, outcome) }
89
+ return if current.any?(&:completed)
90
+ return NOT_RUN if current.empty?
91
+
92
+ current.max_by(&:latest_id).latest_failed ? FAILED : RUNNING
93
+ end
94
+
95
+ # [{ key:, state: }] for each of +keys+ (a task's after:) that is not
96
+ # satisfied. +by_key+ maps a key to its dependency; a key missing from
97
+ # it (no longer declared) reads as not run.
98
+ def waiting_on(keys, by_key)
99
+ keys.filter_map do |key|
100
+ dependency = by_key[key]
101
+ state = dependency ? state(dependency) : NOT_RUN
102
+ { key: key, state: state } if state
103
+ end
104
+ end
105
+
106
+ private
107
+
108
+ def current?(dependency, outcome)
109
+ return !@revision.nil? && outcome.revision == @revision if dependency.every_deploy
110
+
111
+ outcome.version == dependency.version.to_s
112
+ end
113
+ end
114
+ end
115
+ end
@@ -8,9 +8,12 @@ module Pgbus
8
8
  #
9
9
  # A tick: cheap pending check → the rollover gate (at most every
10
10
  # ROLLOVER_CHECK_INTERVAL seconds) → for each declared, enabled, armed
11
- # task in key order: claim the arm → perform_later → record the run. A
12
- # failed enqueue re-arms the task, logs and reports. No surrounding
13
- # transaction: the adapter defers an enqueue inside one to its commit.
11
+ # task in key order whose after: dependencies have completed (issue
12
+ # #660): claim the arm → record the run → perform_later. A task still
13
+ # waiting on a dependency keeps its arm and is not counted as a failure.
14
+ # A failed enqueue deletes its run, re-arms the task, logs and reports.
15
+ # No surrounding transaction: the adapter defers an enqueue inside one to
16
+ # its commit.
14
17
  class Runner
15
18
  ROLLOVER_CHECK_INTERVAL = 5 # seconds
16
19
  # A task whose enqueue failed is retried after 5 s, then 10, 20, … up to
@@ -35,16 +38,25 @@ module Pgbus
35
38
  @synced = nil
36
39
  @sync_failures = 0
37
40
  @sync_retry_at = nil
41
+ @logged_dependencies = {}
38
42
  end
39
43
 
40
44
  # The valid declared tasks, in key order. A class: that does not resolve
41
- # to an ActiveJob is skipped with a warning, like an invalid task.
45
+ # to an ActiveJob is skipped with a warning, like an invalid task. A
46
+ # failed schema probe (Deploy::Schema) raises out of here and leaves
47
+ # nothing memoized, so the next tick or sync retry loads them again.
42
48
  def tasks
43
49
  @tasks ||= load_tasks
44
50
  end
45
51
 
52
+ # The scheduler's heartbeat and boot log read this before the first
53
+ # tick: a failed schema probe must not stop it from booting, so it
54
+ # counts the declared entries instead.
46
55
  def task_count
47
56
  tasks.size
57
+ rescue StandardError => e
58
+ Pgbus.logger.warn { "[Pgbus] Deploy tasks not loaded yet (#{e.class}: #{e.message}); counting the declared entries" }
59
+ (@config.deploy_tasks || {}).size
48
60
  end
49
61
 
50
62
  # true once synced. A failure (a database blip at boot) is retried from
@@ -83,21 +95,28 @@ module Pgbus
83
95
 
84
96
  @logged_wait = nil
85
97
  rows = DeployTask.where(key: tasks.map(&:key)).index_by(&:key)
86
- tasks.each do |task|
98
+ due = tasks.select do |task|
87
99
  row = rows[task.key]
88
- fire(task, row, now) if row&.enabled && row.armed? && !backing_off?(task.key)
100
+ row&.enabled && row.armed? && !backing_off?(task.key)
89
101
  end
102
+ dependencies = load_dependencies(due)
103
+ due.each { |task| fire(task, rows[task.key], now) unless waiting_on_dependencies?(task, dependencies) }
90
104
  end
91
105
 
92
- # Enqueue +task+ through ActiveJob and record the run. :enqueued, or
93
- # :failed (logged and reported) when perform_later raised or did not
94
- # enqueue the job. Run now calls this directly: no gate, no claim.
106
+ # Record a run of +task+ and enqueue it through ActiveJob, carrying the
107
+ # run's id so the executor stamps its outcome. :enqueued, or :failed
108
+ # (logged and reported, the run deleted) when the run could not be
109
+ # created or perform_later raised or did not enqueue the job. Run now
110
+ # calls this directly: no gate, no claim, no dependency check.
95
111
  def enqueue(task, triggered_by:, now: Time.current)
112
+ run = DeployTaskRun.create!(task_key: task.key, version: task.version, revision: revision,
113
+ triggered_by: triggered_by, run_at: now)
96
114
  payload = { task: task.key, version: task.version, revision: revision, triggered_by: triggered_by }
97
- job = Instrumentation.instrument("pgbus.deploy_task.enqueue", payload) { submit(task) }
98
- record_run(task, job, triggered_by, now)
115
+ job = Instrumentation.instrument("pgbus.deploy_task.enqueue", payload) { submit(task, run) }
116
+ record_enqueue(task, run, job, triggered_by)
99
117
  :enqueued
100
118
  rescue StandardError => e
119
+ discard_run(task, run) if run
101
120
  Pgbus.logger.error do
102
121
  "[Pgbus] Deploy task #{task.key} (version #{task.version}, #{triggered_by}) was not enqueued: " \
103
122
  "#{e.class}: #{e.message}"
@@ -110,7 +129,55 @@ module Pgbus
110
129
  private
111
130
 
112
131
  def load_tasks
113
- (@config.deploy_tasks || {}).filter_map { |key, options| build_task(key, options) }.sort_by(&:key)
132
+ declared = @config.deploy_tasks || {}
133
+ built = declared.filter_map { |key, options| build_task(key, options) }
134
+ built = reject_dependencies_without_schema(built)
135
+ valid, skipped = Dependencies.prune(built, declared.keys.map(&:to_s))
136
+ skipped.each { |key, reason| skip_task(key, reason) }
137
+ valid.sort_by(&:key)
138
+ end
139
+
140
+ # after: needs the run outcomes, so until they are migrated a task that
141
+ # declares it is skipped (and its dependents with it, by prune).
142
+ def reject_dependencies_without_schema(tasks)
143
+ return tasks if tasks.none? { |task| task.dependencies.any? } || Schema.outcomes?
144
+
145
+ tasks.reject do |task|
146
+ next false if task.dependencies.empty?
147
+
148
+ skip_task(task.key, "after: needs the deploy task run outcomes; run `rails g pgbus:add_deploy_task_run_outcomes` " \
149
+ "(or pgbus:update) and migrate")
150
+ true
151
+ end
152
+ end
153
+
154
+ # The outcomes of the dependencies of +due+ (one query), or nil when
155
+ # none of them declares after:.
156
+ def load_dependencies(due)
157
+ keys = due.flat_map(&:dependencies).uniq
158
+ return if keys.empty?
159
+
160
+ Dependencies.load(keys.filter_map { |key| tasks_by_key[key] }, revision: revision)
161
+ end
162
+
163
+ def waiting_on_dependencies?(task, dependencies)
164
+ return false if task.dependencies.empty?
165
+
166
+ waiting = dependencies.waiting_on(task.dependencies, tasks_by_key)
167
+ log_dependencies(task, waiting)
168
+ waiting.any?
169
+ end
170
+
171
+ def tasks_by_key
172
+ @tasks_by_key ||= tasks.index_by(&:key)
173
+ end
174
+
175
+ def log_dependencies(task, waiting)
176
+ summary = waiting.map { |dep| "#{dep[:key]} (#{dep[:state].tr("_", " ")})" }.join(", ")
177
+ return if @logged_dependencies[task.key] == summary
178
+
179
+ @logged_dependencies[task.key] = summary
180
+ Pgbus.logger.info { "[Pgbus] Deploy task #{task.key} waits on #{summary}" } unless summary.empty?
114
181
  end
115
182
 
116
183
  def build_task(key, options)
@@ -218,9 +285,18 @@ module Pgbus
218
285
  # the job, and its enqueue_error, is in hand on every Rails version:
219
286
  # perform_later returns false on failure, and on Rails 7.1 a .set(...)
220
287
  # job never yields it to perform_later's block.
221
- def submit(task)
288
+ def submit(task, run)
222
289
  job = job_class_for(task).new(*job_arguments(task))
223
- return job if job.enqueue(job_options(task))
290
+ # The engine includes DeployTaskRunId on every job; without it (no
291
+ # engine loaded) this instance alone is marked.
292
+ job.extend(ActiveJob::DeployTaskRunId) unless job.is_a?(ActiveJob::DeployTaskRunId)
293
+ job.deploy_task_run_id = run.id
294
+ enqueued = job.enqueue(job_options(task))
295
+ # The pgbus adapter returns a job its uniqueness or concurrency
296
+ # on_conflict dropped without raising; nothing would run for this run.
297
+ raise EnqueueError, "the job was dropped by its uniqueness or concurrency on_conflict" if
298
+ enqueued && job.deploy_task_run_enqueued == false
299
+ return job if enqueued
224
300
 
225
301
  error = job.enqueue_error
226
302
  raise EnqueueError, error ? "#{error.class}: #{error.message}" : "the job was not enqueued (a before_enqueue callback aborted)"
@@ -244,16 +320,25 @@ module Pgbus
244
320
  task.command ? [task.command] : task.arguments
245
321
  end
246
322
 
247
- def record_run(task, job, triggered_by, now)
248
- DeployTaskRun.create!(task_key: task.key, version: task.version, revision: revision,
249
- triggered_by: triggered_by, queue_name: job.queue_name,
250
- msg_id: job.provider_job_id, run_at: now)
323
+ # queue_name marks the run as enqueued: until it is set, the run counts
324
+ # as no run (DeployTaskRun.enqueued), so a failure here leaves the
325
+ # version owed, as a missing run row did before #660.
326
+ def record_enqueue(task, run, job, triggered_by)
327
+ run.update_columns(queue_name: job.queue_name, msg_id: job.provider_job_id)
251
328
  rescue StandardError => e
252
329
  Pgbus.logger.error do
253
330
  "[Pgbus] Deploy task #{task.key} was enqueued but its run was not recorded: #{e.class}: #{e.message}"
254
331
  end
255
332
  ErrorReporter.report(e, { action: "deploy_task_record_run", task: task.key, triggered_by: triggered_by })
256
333
  end
334
+
335
+ # A run whose enqueue failed. Left behind (this delete failed too) it
336
+ # still counts as no run, having no queue_name.
337
+ def discard_run(task, run)
338
+ run.delete
339
+ rescue StandardError => e
340
+ ErrorReporter.report(e, { action: "deploy_task_discard_run", task: task.key, run_id: run.id })
341
+ end
257
342
  end
258
343
  end
259
344
  end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module Deploy
5
+ # Whether pgbus:add_deploy_task_run_outcomes has been migrated (issue
6
+ # #660): pgbus_deploy_task_runs.completed_at / failed_at and
7
+ # pgbus_deploy_tasks.dependencies. Until then after: is rejected at load,
8
+ # sync leaves dependencies alone and the executor writes no outcome.
9
+ #
10
+ # A "yes" is kept for the process; a "no" stands for RECHECK_SECONDS, so a
11
+ # process started before the migration picks the columns up without a
12
+ # restart. A probe that raises (a database blip) is not an answer: the
13
+ # error reaches the caller and nothing is kept, so a scheduler booting
14
+ # through a blip does not skip its after: tasks as unmigrated.
15
+ module Schema
16
+ RECHECK_SECONDS = 60
17
+
18
+ @mutex = Mutex.new
19
+ @answer = nil # true, nil (never probed) or the monotonic time a "no" runs out
20
+
21
+ class << self
22
+ def outcomes?
23
+ # Double-checked: the unlocked read only skips the lock while the
24
+ # answer stands; it is written under @mutex.
25
+ known = answer(@answer) # rubocop:disable ThreadSafety/ClassInstanceVariable
26
+ return known unless known.nil?
27
+
28
+ @mutex.synchronize do
29
+ known = answer(@answer)
30
+ next known unless known.nil?
31
+
32
+ supported = probe
33
+ @answer = supported || (monotonic_now + RECHECK_SECONDS)
34
+ supported
35
+ end
36
+ end
37
+
38
+ # Clears the memoized probe: specs exercise both schema shapes.
39
+ def reset!
40
+ @mutex.synchronize { @answer = nil }
41
+ end
42
+
43
+ private
44
+
45
+ def answer(state)
46
+ return true if state == true
47
+
48
+ false if state && monotonic_now < state
49
+ end
50
+
51
+ def probe
52
+ BusRecord.connection_pool.with_connection do |conn|
53
+ conn.column_exists?(:pgbus_deploy_task_runs, :completed_at) &&
54
+ conn.column_exists?(:pgbus_deploy_task_runs, :failed_at) &&
55
+ conn.column_exists?(:pgbus_deploy_tasks, :dependencies)
56
+ end
57
+ end
58
+
59
+ def monotonic_now
60
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
@@ -5,12 +5,13 @@ module Pgbus
5
5
  # One declared deploy task (issue #536): an entry of
6
6
  # config/deploy_tasks.yml. The sibling of Recurring::Task without a
7
7
  # schedule: it runs once per (key, version) after a rollover, or after
8
- # every rollover when every_deploy is set.
8
+ # every rollover when every_deploy is set. after: names the tasks it
9
+ # waits for (issue #660): see Deploy::Dependencies.
9
10
  class Task
10
11
  DEFAULT_VERSION = "1"
11
12
 
12
13
  attr_reader :key, :class_name, :command, :version, :every_deploy, :queue_name,
13
- :arguments, :priority, :description
14
+ :arguments, :priority, :description, :dependencies
14
15
 
15
16
  def self.from_configuration(key, **options)
16
17
  options = options.transform_keys(&:to_sym)
@@ -24,12 +25,14 @@ module Pgbus
24
25
  queue_name: options[:queue],
25
26
  arguments: options[:args].is_a?(Hash) ? [options[:args]] : Array(options[:args]),
26
27
  priority: options.fetch(:priority, 0).to_i,
27
- description: options[:description]
28
+ description: options[:description],
29
+ dependencies: options[:after]
28
30
  )
29
31
  end
30
32
 
31
33
  def initialize(key:, class_name: nil, command: nil, version: DEFAULT_VERSION, explicit_version: false,
32
- every_deploy: false, queue_name: nil, arguments: [], priority: 0, description: nil)
34
+ every_deploy: false, queue_name: nil, arguments: [], priority: 0, description: nil,
35
+ dependencies: nil)
33
36
  @key = key
34
37
  @class_name = class_name
35
38
  @command = command
@@ -40,6 +43,7 @@ module Pgbus
40
43
  @arguments = arguments || []
41
44
  @priority = priority || 0
42
45
  @description = description
46
+ @dependencies = dependency_list(dependencies)
43
47
  @errors = []
44
48
  @warnings = []
45
49
  end
@@ -74,7 +78,8 @@ module Pgbus
74
78
  def to_h
75
79
  {
76
80
  key: key, class_name: class_name, command: command, version: version, every_deploy: every_deploy,
77
- queue_name: queue_name, arguments: arguments, priority: priority, description: description
81
+ queue_name: queue_name, arguments: arguments, priority: priority, description: description,
82
+ dependencies: dependencies
78
83
  }
79
84
  end
80
85
 
@@ -86,11 +91,24 @@ module Pgbus
86
91
  validate_target!
87
92
  @errors << "Version must not be blank" if version.empty?
88
93
  @errors << "every_deploy must be true or false" unless [true, false].include?(every_deploy)
94
+ validate_dependencies!
89
95
  return unless every_deploy == true && @explicit_version
90
96
 
91
97
  @warnings << "every_deploy is set, so version is ignored (it runs after every rollover)"
92
98
  end
93
99
 
100
+ def validate_dependencies!
101
+ @errors << "after: must name task keys" unless dependencies.all? { |dep| dep.is_a?(String) && !dep.strip.empty? }
102
+ @errors << "after: cycle #{key} \u2192 #{key}" if dependencies.include?(key)
103
+ end
104
+
105
+ # nil, one key, or a list; a Symbol reads as its name. Anything else is
106
+ # kept as given, for validate_dependencies! to reject.
107
+ def dependency_list(value)
108
+ list = value.is_a?(Array) ? value : [value].compact
109
+ list.map { |dep| dep.is_a?(Symbol) ? dep.to_s : dep }.uniq.freeze
110
+ end
111
+
94
112
  def validate_target!
95
113
  { class: class_name, command: command }.each do |name, value|
96
114
  @errors << "#{name} must be a String" unless value.nil? || value.is_a?(String)
data/lib/pgbus/engine.rb CHANGED
@@ -92,6 +92,7 @@ module Pgbus
92
92
  include Pgbus::Concurrency
93
93
  include Pgbus::Uniqueness
94
94
  include Pgbus::ActiveJob::BatchId
95
+ include Pgbus::ActiveJob::DeployTaskRunId
95
96
  include Pgbus::ActiveJob::CurrentAttributes
96
97
  include Pgbus::VisibilityHeartbeat::JobMixin
97
98
  end
@@ -94,6 +94,7 @@ module Pgbus
94
94
  add_outbox: "pgbus:add_outbox",
95
95
  add_recurring: "pgbus:add_recurring",
96
96
  add_deploy_tasks: "pgbus:add_deploy_tasks",
97
+ add_deploy_task_run_outcomes: "pgbus:add_deploy_task_run_outcomes",
97
98
  add_workflows: "pgbus:add_workflows",
98
99
  add_workflow_signals: "pgbus:add_workflow_signals",
99
100
  add_failed_events_index: "pgbus:add_failed_events_index",
@@ -121,6 +122,7 @@ module Pgbus
121
122
  add_outbox: "outbox entries table (transactional outbox)",
122
123
  add_recurring: "recurring tasks + executions tables",
123
124
  add_deploy_tasks: "deploy tasks + runs tables (one-time jobs run after a rollover)",
125
+ add_deploy_task_run_outcomes: "deploy task run outcomes (completed_at, failed_at) and after: dependencies",
124
126
  add_workflows: "workflow tables (durable multi-step workflows: runs, step attempts, audit trail)",
125
127
  add_workflow_signals: "waiting workflow steps (fan_out, wait_for: signals, wait timeouts)",
126
128
  add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
@@ -253,10 +255,18 @@ module Pgbus
253
255
 
254
256
  # Same shape as recurring_migrations: plain create_table in the
255
257
  # template, so the generator is queued only while a table is absent.
258
+ # The outcomes add-on (issue #660) follows the tables: its template
259
+ # adds each column if_not_exists, so queuing it after add_deploy_tasks
260
+ # also completes a half-present schema whose existing table lacks them.
256
261
  def deploy_tasks_migrations
257
- return [] if table_exists?("pgbus_deploy_tasks") && table_exists?("pgbus_deploy_task_runs")
258
-
259
- [:add_deploy_tasks]
262
+ return %i[add_deploy_tasks add_deploy_task_run_outcomes] unless
263
+ table_exists?("pgbus_deploy_tasks") && table_exists?("pgbus_deploy_task_runs")
264
+
265
+ runs = column_names("pgbus_deploy_task_runs")
266
+ outcomes = runs.include?("completed_at") && runs.include?("failed_at") &&
267
+ column_names("pgbus_deploy_tasks").include?("dependencies") &&
268
+ index_exists?("pgbus_deploy_task_runs", "idx_pgbus_deploy_task_runs_completed")
269
+ outcomes ? [] : [:add_deploy_task_run_outcomes]
260
270
  end
261
271
 
262
272
  # The add_workflows template uses if_not_exists throughout, so a
@@ -11,7 +11,7 @@ module Pgbus
11
11
  tool_name "pgbus_deploy_tasks"
12
12
  title "Pgbus Deploy Tasks"
13
13
  description <<~DESC
14
- Deploy tasks: one-time jobs the scheduler enqueues once every live pgbus process runs the new app revision. Lists each task's state (disabled, waiting, due, ran, never) and the rollover status the scheduler waits on.
14
+ Deploy tasks: one-time jobs the scheduler enqueues once every live pgbus process runs the new app revision. Lists each task's state (disabled, waiting, due, ran, never), the rollover status the scheduler waits on, and for a task with after: its dependencies and the ones it still waits on (waiting_on, each not_run, running or failed).
15
15
  Returns every task unless page or per_page is given; then it is
16
16
  paginated (per_page defaults to 100, capped at 100). The response
17
17
  carries page, per_page, total (capped at 10000 when paginated; the
data/lib/pgbus/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Pgbus
4
- VERSION = "0.24.0"
4
+ VERSION = "0.25.0"
5
5
  end