pgbus 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/app/controllers/pgbus/deploy_tasks_controller.rb +11 -1
- data/app/helpers/pgbus/deploy_tasks_helper.rb +20 -0
- data/app/models/pgbus/deploy_task.rb +23 -1
- data/app/models/pgbus/deploy_task_run.rb +47 -5
- data/app/views/pgbus/deploy_tasks/_tasks_table.html.erb +3 -0
- data/app/views/pgbus/deploy_tasks/show.html.erb +25 -2
- data/config/i18n-tasks.yml +1 -0
- data/config/locales/da.yml +18 -0
- data/config/locales/de.yml +18 -0
- data/config/locales/en.yml +18 -0
- data/config/locales/es.yml +18 -0
- data/config/locales/fi.yml +18 -0
- data/config/locales/fr.yml +18 -0
- data/config/locales/it.yml +18 -0
- data/config/locales/ja.yml +18 -0
- data/config/locales/nb.yml +18 -0
- data/config/locales/nl.yml +18 -0
- data/config/locales/pt.yml +18 -0
- data/config/locales/sv.yml +18 -0
- data/lib/generators/pgbus/add_deploy_task_run_outcomes_generator.rb +48 -0
- data/lib/generators/pgbus/templates/add_deploy_task_run_outcomes.rb.erb +12 -0
- data/lib/generators/pgbus/templates/add_deploy_tasks_tables.rb.erb +5 -0
- data/lib/generators/pgbus/templates/deploy_tasks.yml.erb +6 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +5 -0
- data/lib/pgbus/active_job/adapter.rb +13 -0
- data/lib/pgbus/active_job/deploy_task_run_id.rb +53 -0
- data/lib/pgbus/active_job/executor.rb +57 -0
- data/lib/pgbus/deploy/dependencies.rb +115 -0
- data/lib/pgbus/deploy/runner.rb +103 -18
- data/lib/pgbus/deploy/schema.rb +65 -0
- data/lib/pgbus/deploy/task.rb +23 -5
- data/lib/pgbus/engine.rb +1 -0
- data/lib/pgbus/generators/migration_detector.rb +13 -3
- data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -1
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/deploy_task_list.rb +39 -6
- data/lib/pgbus/web/deploy_task_state.rb +7 -5
- metadata +6 -1
|
@@ -44,6 +44,7 @@ module Pgbus
|
|
|
44
44
|
source_queue: physical_queue(queue_name, source_queue))
|
|
45
45
|
signal_concurrency(payload)
|
|
46
46
|
signal_batch_discarded(payload)
|
|
47
|
+
record_deploy_task_run_outcome(payload, :failed)
|
|
47
48
|
release_uniqueness_lock(Uniqueness.extract_key(payload), queue_name, message) if holds_uniqueness_row?(payload)
|
|
48
49
|
record_stat(payload, queue_name, "dead_lettered", execution_start, message: message)
|
|
49
50
|
instrument(
|
|
@@ -94,6 +95,7 @@ module Pgbus
|
|
|
94
95
|
Pgbus.logger.debug { "[Pgbus::Executor] deserialized #{log_tag(message, queue_name)} job_class=#{job_class}" }
|
|
95
96
|
job_succeeded = false
|
|
96
97
|
retried = false
|
|
98
|
+
deploy_run_outcome = nil
|
|
97
99
|
|
|
98
100
|
msg_id = message.msg_id.to_i
|
|
99
101
|
instrument_payload = {
|
|
@@ -116,6 +118,7 @@ module Pgbus
|
|
|
116
118
|
# job data, so hand it to the job before perform — that is what makes
|
|
117
119
|
# `batch` (and `batch.enqueue` for open batches) work inside a job.
|
|
118
120
|
assign_batch_id(job, payload)
|
|
121
|
+
mark_deploy_task_job(job, payload) if payload.key?(DeployTaskRunId::METADATA_KEY)
|
|
119
122
|
Pgbus.logger.debug { "[Pgbus::Executor] running #{log_tag(message, queue_name)} job_class=#{job_class}" }
|
|
120
123
|
with_visibility_heartbeat(job, queue_name, msg_id, source_queue, payload) do
|
|
121
124
|
Delivery.with(msg_id: msg_id, read_ct: read_count, queue_name: queue_name,
|
|
@@ -127,6 +130,13 @@ module Pgbus
|
|
|
127
130
|
# this attempt is done (archive it) but the job is not — the retry
|
|
128
131
|
# message carries the batch tag and signals on its own outcome.
|
|
129
132
|
retried = Batch.retry_reenqueued?(payload["job_id"])
|
|
133
|
+
# The same for a deploy task's job, batch or not: :completed when
|
|
134
|
+
# perform returned having done its work, nil when its retry_on
|
|
135
|
+
# retry is live (the retry records the outcome), :failed when the
|
|
136
|
+
# adapter dropped that retry or ActiveJob discarded the job
|
|
137
|
+
# (discard_on, a retry_on giving up through its block). Asked only
|
|
138
|
+
# of a marked job: an unmarked one pays one Hash lookup.
|
|
139
|
+
deploy_run_outcome = deploy_task_run_outcome(job) if payload.key?(DeployTaskRunId::METADATA_KEY)
|
|
130
140
|
Pgbus.logger.debug { "[Pgbus::Executor] perform_returned #{log_tag(message, queue_name)} job_class=#{job_class}" }
|
|
131
141
|
# Archiving is the exact-once claim on this execution.
|
|
132
142
|
# :already_archived means another worker archived the message (our
|
|
@@ -200,6 +210,7 @@ module Pgbus
|
|
|
200
210
|
# key if the first DELETE committed but the client raised afterward.
|
|
201
211
|
signal_concurrency(payload)
|
|
202
212
|
signal_batch_completed(payload) unless retried
|
|
213
|
+
record_deploy_task_run_outcome(payload, deploy_run_outcome) if deploy_run_outcome
|
|
203
214
|
end
|
|
204
215
|
Batch.clear_retry_reenqueued
|
|
205
216
|
end
|
|
@@ -432,6 +443,7 @@ module Pgbus
|
|
|
432
443
|
end
|
|
433
444
|
signal_concurrency(payload)
|
|
434
445
|
signal_batch_discarded(payload)
|
|
446
|
+
record_deploy_task_run_outcome(payload, :failed)
|
|
435
447
|
instrument("pgbus.job_discarded", queue: queue_name, job_class: payload["job_class"], job_id: payload["job_id"],
|
|
436
448
|
provider_job_id: payload["provider_job_id"], msg_id: msg_id, reason: "uniqueness")
|
|
437
449
|
Pgbus.logger.info do
|
|
@@ -520,6 +532,51 @@ module Pgbus
|
|
|
520
532
|
Pgbus.logger.warn { "[Pgbus] Batch discard signal failed: #{e.message}" }
|
|
521
533
|
end
|
|
522
534
|
|
|
535
|
+
# The engine includes DeployTaskRunId on every job. A class without it
|
|
536
|
+
# (no engine) gets it on this instance, as Deploy::Runner does on the
|
|
537
|
+
# enqueue side, so a discard is still seen and a retry still carries the
|
|
538
|
+
# run id.
|
|
539
|
+
def mark_deploy_task_job(job, payload)
|
|
540
|
+
return if job.is_a?(DeployTaskRunId)
|
|
541
|
+
|
|
542
|
+
job.extend(DeployTaskRunId)
|
|
543
|
+
job.deploy_task_run_id = payload[DeployTaskRunId::METADATA_KEY]
|
|
544
|
+
end
|
|
545
|
+
|
|
546
|
+
# The outcome of a marked job's run once perform returned. nil: its
|
|
547
|
+
# retry is live, and the retry records it. :failed: the adapter dropped
|
|
548
|
+
# the retry, or a before_enqueue callback aborted it before the adapter
|
|
549
|
+
# saw it (ActiveJob then leaves successfully_enqueued false, where an
|
|
550
|
+
# untried enqueue leaves it nil); or ActiveJob discarded the job, which
|
|
551
|
+
# returned without doing its work (issue #662). A live retry wins over
|
|
552
|
+
# the discard, so a discard block that re-enqueues leaves the outcome
|
|
553
|
+
# to that retry. :completed otherwise.
|
|
554
|
+
def deploy_task_run_outcome(job)
|
|
555
|
+
requeue = deploy_task_run_requeue(job)
|
|
556
|
+
return if requeue
|
|
557
|
+
return :failed if requeue == false || job.try(:pgbus_discarded?)
|
|
558
|
+
|
|
559
|
+
:completed
|
|
560
|
+
end
|
|
561
|
+
|
|
562
|
+
def deploy_task_run_requeue(job)
|
|
563
|
+
flag = job.try(:deploy_task_run_enqueued)
|
|
564
|
+
return flag unless flag.nil?
|
|
565
|
+
|
|
566
|
+
job.successfully_enqueued? == false ? false : nil
|
|
567
|
+
end
|
|
568
|
+
|
|
569
|
+
# Stamps a deploy task run's outcome (issue #660). Best-effort, like
|
|
570
|
+
# record_stat: a failed write is reported and never fails the job.
|
|
571
|
+
def record_deploy_task_run_outcome(payload, outcome)
|
|
572
|
+
run_id = payload[DeployTaskRunId::METADATA_KEY]
|
|
573
|
+
return unless run_id && Deploy::Schema.outcomes?
|
|
574
|
+
|
|
575
|
+
outcome == :completed ? DeployTaskRun.mark_completed!(run_id) : DeployTaskRun.mark_failed!(run_id)
|
|
576
|
+
rescue StandardError => e
|
|
577
|
+
ErrorReporter.report(e, { action: "deploy_task_run_outcome", deploy_task_run_id: run_id, outcome: outcome })
|
|
578
|
+
end
|
|
579
|
+
|
|
523
580
|
# Archiving is idempotent — archiving an already-archived message is a
|
|
524
581
|
# no-op — so a connection error here is safe to retry once, unlike sends
|
|
525
582
|
# where a retry could duplicate the message. Without the retry, a job
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tsort"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Deploy
|
|
7
|
+
# after: (issue #660): which declared tasks can wait on which, and whether
|
|
8
|
+
# a task's dependencies have finished.
|
|
9
|
+
#
|
|
10
|
+
# A dependency is satisfied by a completed run (rollover or manual) at its
|
|
11
|
+
# current version, at any revision; an every_deploy dependency needs one on
|
|
12
|
+
# this revision. A run before the outcomes migration has no outcome, so it
|
|
13
|
+
# satisfies nothing.
|
|
14
|
+
class Dependencies
|
|
15
|
+
NOT_RUN = "not_run"
|
|
16
|
+
RUNNING = "running"
|
|
17
|
+
FAILED = "failed"
|
|
18
|
+
|
|
19
|
+
# +tasks+ (valid Deploy::Task objects) minus those whose after: can
|
|
20
|
+
# never be met: an unknown key, a cycle, or a dependency that is itself
|
|
21
|
+
# skipped. Returns [kept tasks, { skipped key => reason }].
|
|
22
|
+
def self.prune(tasks, declared_keys)
|
|
23
|
+
valid = tasks.to_h { |task| [task.key, task] }
|
|
24
|
+
skipped = {}
|
|
25
|
+
loop do
|
|
26
|
+
unmet = valid.values.filter_map { |task| (reason = unmet(task, valid, declared_keys)) && [task.key, reason] }
|
|
27
|
+
unmet = cycles(valid) if unmet.empty?
|
|
28
|
+
break if unmet.empty?
|
|
29
|
+
|
|
30
|
+
unmet.each do |key, reason|
|
|
31
|
+
skipped[key] = reason
|
|
32
|
+
valid.delete(key)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
[valid.values, skipped]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def self.unmet(task, valid, declared_keys)
|
|
39
|
+
missing = task.dependencies.find { |dep| !valid.key?(dep) }
|
|
40
|
+
return unless missing
|
|
41
|
+
|
|
42
|
+
declared_keys.include?(missing) ? "after: '#{missing}' is skipped" : "after: unknown task '#{missing}'"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# [key, reason] for every task on a cycle (Task#valid? already rejects
|
|
46
|
+
# a task naming itself).
|
|
47
|
+
def self.cycles(valid)
|
|
48
|
+
graph = valid.transform_values { |task| task.dependencies.select { |dep| valid.key?(dep) }.sort }
|
|
49
|
+
components = TSort.strongly_connected_components(graph.method(:each_key), ->(key, &blk) { graph[key].each(&blk) })
|
|
50
|
+
components.select { |keys| keys.size > 1 }.flat_map do |keys|
|
|
51
|
+
reason = "after: cycle #{cycle_path(keys.min, graph, keys.to_set).join(" → ")}"
|
|
52
|
+
keys.sort.map { |key| [key, reason] }
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# A path from +start+ back to itself inside its strongly connected
|
|
57
|
+
# component, which always has one.
|
|
58
|
+
def self.cycle_path(start, graph, members)
|
|
59
|
+
stack = [[start, [start]]]
|
|
60
|
+
seen = Set[start]
|
|
61
|
+
until stack.empty?
|
|
62
|
+
key, path = stack.pop
|
|
63
|
+
graph[key].each do |dep|
|
|
64
|
+
next unless members.include?(dep)
|
|
65
|
+
return path + [start] if dep == start
|
|
66
|
+
next unless seen.add?(dep)
|
|
67
|
+
|
|
68
|
+
stack << [dep, path + [dep]]
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
private_class_method :unmet, :cycles, :cycle_path
|
|
73
|
+
|
|
74
|
+
# The outcomes of +dependencies+ (anything with key, version and
|
|
75
|
+
# every_deploy: Deploy::Task or DeployTask), in one query.
|
|
76
|
+
def self.load(dependencies, revision:)
|
|
77
|
+
new(DeployTaskRun.outcomes_for(dependencies.map(&:key).uniq), revision: revision)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def initialize(outcomes, revision:)
|
|
81
|
+
@outcomes = outcomes.group_by(&:task_key)
|
|
82
|
+
@revision = revision
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# nil when +dependency+ is satisfied, else NOT_RUN, RUNNING (the newest
|
|
86
|
+
# run at its version has no outcome yet) or FAILED.
|
|
87
|
+
def state(dependency)
|
|
88
|
+
current = @outcomes.fetch(dependency.key, []).select { |outcome| current?(dependency, outcome) }
|
|
89
|
+
return if current.any?(&:completed)
|
|
90
|
+
return NOT_RUN if current.empty?
|
|
91
|
+
|
|
92
|
+
current.max_by(&:latest_id).latest_failed ? FAILED : RUNNING
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# [{ key:, state: }] for each of +keys+ (a task's after:) that is not
|
|
96
|
+
# satisfied. +by_key+ maps a key to its dependency; a key missing from
|
|
97
|
+
# it (no longer declared) reads as not run.
|
|
98
|
+
def waiting_on(keys, by_key)
|
|
99
|
+
keys.filter_map do |key|
|
|
100
|
+
dependency = by_key[key]
|
|
101
|
+
state = dependency ? state(dependency) : NOT_RUN
|
|
102
|
+
{ key: key, state: state } if state
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
private
|
|
107
|
+
|
|
108
|
+
def current?(dependency, outcome)
|
|
109
|
+
return !@revision.nil? && outcome.revision == @revision if dependency.every_deploy
|
|
110
|
+
|
|
111
|
+
outcome.version == dependency.version.to_s
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
data/lib/pgbus/deploy/runner.rb
CHANGED
|
@@ -8,9 +8,12 @@ module Pgbus
|
|
|
8
8
|
#
|
|
9
9
|
# A tick: cheap pending check → the rollover gate (at most every
|
|
10
10
|
# ROLLOVER_CHECK_INTERVAL seconds) → for each declared, enabled, armed
|
|
11
|
-
# task in key order
|
|
12
|
-
#
|
|
13
|
-
#
|
|
11
|
+
# task in key order whose after: dependencies have completed (issue
|
|
12
|
+
# #660): claim the arm → record the run → perform_later. A task still
|
|
13
|
+
# waiting on a dependency keeps its arm and is not counted as a failure.
|
|
14
|
+
# A failed enqueue deletes its run, re-arms the task, logs and reports.
|
|
15
|
+
# No surrounding transaction: the adapter defers an enqueue inside one to
|
|
16
|
+
# its commit.
|
|
14
17
|
class Runner
|
|
15
18
|
ROLLOVER_CHECK_INTERVAL = 5 # seconds
|
|
16
19
|
# A task whose enqueue failed is retried after 5 s, then 10, 20, … up to
|
|
@@ -35,16 +38,25 @@ module Pgbus
|
|
|
35
38
|
@synced = nil
|
|
36
39
|
@sync_failures = 0
|
|
37
40
|
@sync_retry_at = nil
|
|
41
|
+
@logged_dependencies = {}
|
|
38
42
|
end
|
|
39
43
|
|
|
40
44
|
# The valid declared tasks, in key order. A class: that does not resolve
|
|
41
|
-
# to an ActiveJob is skipped with a warning, like an invalid task.
|
|
45
|
+
# to an ActiveJob is skipped with a warning, like an invalid task. A
|
|
46
|
+
# failed schema probe (Deploy::Schema) raises out of here and leaves
|
|
47
|
+
# nothing memoized, so the next tick or sync retry loads them again.
|
|
42
48
|
def tasks
|
|
43
49
|
@tasks ||= load_tasks
|
|
44
50
|
end
|
|
45
51
|
|
|
52
|
+
# The scheduler's heartbeat and boot log read this before the first
|
|
53
|
+
# tick: a failed schema probe must not stop it from booting, so it
|
|
54
|
+
# counts the declared entries instead.
|
|
46
55
|
def task_count
|
|
47
56
|
tasks.size
|
|
57
|
+
rescue StandardError => e
|
|
58
|
+
Pgbus.logger.warn { "[Pgbus] Deploy tasks not loaded yet (#{e.class}: #{e.message}); counting the declared entries" }
|
|
59
|
+
(@config.deploy_tasks || {}).size
|
|
48
60
|
end
|
|
49
61
|
|
|
50
62
|
# true once synced. A failure (a database blip at boot) is retried from
|
|
@@ -83,21 +95,28 @@ module Pgbus
|
|
|
83
95
|
|
|
84
96
|
@logged_wait = nil
|
|
85
97
|
rows = DeployTask.where(key: tasks.map(&:key)).index_by(&:key)
|
|
86
|
-
tasks.
|
|
98
|
+
due = tasks.select do |task|
|
|
87
99
|
row = rows[task.key]
|
|
88
|
-
|
|
100
|
+
row&.enabled && row.armed? && !backing_off?(task.key)
|
|
89
101
|
end
|
|
102
|
+
dependencies = load_dependencies(due)
|
|
103
|
+
due.each { |task| fire(task, rows[task.key], now) unless waiting_on_dependencies?(task, dependencies) }
|
|
90
104
|
end
|
|
91
105
|
|
|
92
|
-
#
|
|
93
|
-
#
|
|
94
|
-
#
|
|
106
|
+
# Record a run of +task+ and enqueue it through ActiveJob, carrying the
|
|
107
|
+
# run's id so the executor stamps its outcome. :enqueued, or :failed
|
|
108
|
+
# (logged and reported, the run deleted) when the run could not be
|
|
109
|
+
# created or perform_later raised or did not enqueue the job. Run now
|
|
110
|
+
# calls this directly: no gate, no claim, no dependency check.
|
|
95
111
|
def enqueue(task, triggered_by:, now: Time.current)
|
|
112
|
+
run = DeployTaskRun.create!(task_key: task.key, version: task.version, revision: revision,
|
|
113
|
+
triggered_by: triggered_by, run_at: now)
|
|
96
114
|
payload = { task: task.key, version: task.version, revision: revision, triggered_by: triggered_by }
|
|
97
|
-
job = Instrumentation.instrument("pgbus.deploy_task.enqueue", payload) { submit(task) }
|
|
98
|
-
|
|
115
|
+
job = Instrumentation.instrument("pgbus.deploy_task.enqueue", payload) { submit(task, run) }
|
|
116
|
+
record_enqueue(task, run, job, triggered_by)
|
|
99
117
|
:enqueued
|
|
100
118
|
rescue StandardError => e
|
|
119
|
+
discard_run(task, run) if run
|
|
101
120
|
Pgbus.logger.error do
|
|
102
121
|
"[Pgbus] Deploy task #{task.key} (version #{task.version}, #{triggered_by}) was not enqueued: " \
|
|
103
122
|
"#{e.class}: #{e.message}"
|
|
@@ -110,7 +129,55 @@ module Pgbus
|
|
|
110
129
|
private
|
|
111
130
|
|
|
112
131
|
def load_tasks
|
|
113
|
-
|
|
132
|
+
declared = @config.deploy_tasks || {}
|
|
133
|
+
built = declared.filter_map { |key, options| build_task(key, options) }
|
|
134
|
+
built = reject_dependencies_without_schema(built)
|
|
135
|
+
valid, skipped = Dependencies.prune(built, declared.keys.map(&:to_s))
|
|
136
|
+
skipped.each { |key, reason| skip_task(key, reason) }
|
|
137
|
+
valid.sort_by(&:key)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# after: needs the run outcomes, so until they are migrated a task that
|
|
141
|
+
# declares it is skipped (and its dependents with it, by prune).
|
|
142
|
+
def reject_dependencies_without_schema(tasks)
|
|
143
|
+
return tasks if tasks.none? { |task| task.dependencies.any? } || Schema.outcomes?
|
|
144
|
+
|
|
145
|
+
tasks.reject do |task|
|
|
146
|
+
next false if task.dependencies.empty?
|
|
147
|
+
|
|
148
|
+
skip_task(task.key, "after: needs the deploy task run outcomes; run `rails g pgbus:add_deploy_task_run_outcomes` " \
|
|
149
|
+
"(or pgbus:update) and migrate")
|
|
150
|
+
true
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# The outcomes of the dependencies of +due+ (one query), or nil when
|
|
155
|
+
# none of them declares after:.
|
|
156
|
+
def load_dependencies(due)
|
|
157
|
+
keys = due.flat_map(&:dependencies).uniq
|
|
158
|
+
return if keys.empty?
|
|
159
|
+
|
|
160
|
+
Dependencies.load(keys.filter_map { |key| tasks_by_key[key] }, revision: revision)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def waiting_on_dependencies?(task, dependencies)
|
|
164
|
+
return false if task.dependencies.empty?
|
|
165
|
+
|
|
166
|
+
waiting = dependencies.waiting_on(task.dependencies, tasks_by_key)
|
|
167
|
+
log_dependencies(task, waiting)
|
|
168
|
+
waiting.any?
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def tasks_by_key
|
|
172
|
+
@tasks_by_key ||= tasks.index_by(&:key)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def log_dependencies(task, waiting)
|
|
176
|
+
summary = waiting.map { |dep| "#{dep[:key]} (#{dep[:state].tr("_", " ")})" }.join(", ")
|
|
177
|
+
return if @logged_dependencies[task.key] == summary
|
|
178
|
+
|
|
179
|
+
@logged_dependencies[task.key] = summary
|
|
180
|
+
Pgbus.logger.info { "[Pgbus] Deploy task #{task.key} waits on #{summary}" } unless summary.empty?
|
|
114
181
|
end
|
|
115
182
|
|
|
116
183
|
def build_task(key, options)
|
|
@@ -218,9 +285,18 @@ module Pgbus
|
|
|
218
285
|
# the job, and its enqueue_error, is in hand on every Rails version:
|
|
219
286
|
# perform_later returns false on failure, and on Rails 7.1 a .set(...)
|
|
220
287
|
# job never yields it to perform_later's block.
|
|
221
|
-
def submit(task)
|
|
288
|
+
def submit(task, run)
|
|
222
289
|
job = job_class_for(task).new(*job_arguments(task))
|
|
223
|
-
|
|
290
|
+
# The engine includes DeployTaskRunId on every job; without it (no
|
|
291
|
+
# engine loaded) this instance alone is marked.
|
|
292
|
+
job.extend(ActiveJob::DeployTaskRunId) unless job.is_a?(ActiveJob::DeployTaskRunId)
|
|
293
|
+
job.deploy_task_run_id = run.id
|
|
294
|
+
enqueued = job.enqueue(job_options(task))
|
|
295
|
+
# The pgbus adapter returns a job its uniqueness or concurrency
|
|
296
|
+
# on_conflict dropped without raising; nothing would run for this run.
|
|
297
|
+
raise EnqueueError, "the job was dropped by its uniqueness or concurrency on_conflict" if
|
|
298
|
+
enqueued && job.deploy_task_run_enqueued == false
|
|
299
|
+
return job if enqueued
|
|
224
300
|
|
|
225
301
|
error = job.enqueue_error
|
|
226
302
|
raise EnqueueError, error ? "#{error.class}: #{error.message}" : "the job was not enqueued (a before_enqueue callback aborted)"
|
|
@@ -244,16 +320,25 @@ module Pgbus
|
|
|
244
320
|
task.command ? [task.command] : task.arguments
|
|
245
321
|
end
|
|
246
322
|
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
323
|
+
# queue_name marks the run as enqueued: until it is set, the run counts
|
|
324
|
+
# as no run (DeployTaskRun.enqueued), so a failure here leaves the
|
|
325
|
+
# version owed, as a missing run row did before #660.
|
|
326
|
+
def record_enqueue(task, run, job, triggered_by)
|
|
327
|
+
run.update_columns(queue_name: job.queue_name, msg_id: job.provider_job_id)
|
|
251
328
|
rescue StandardError => e
|
|
252
329
|
Pgbus.logger.error do
|
|
253
330
|
"[Pgbus] Deploy task #{task.key} was enqueued but its run was not recorded: #{e.class}: #{e.message}"
|
|
254
331
|
end
|
|
255
332
|
ErrorReporter.report(e, { action: "deploy_task_record_run", task: task.key, triggered_by: triggered_by })
|
|
256
333
|
end
|
|
334
|
+
|
|
335
|
+
# A run whose enqueue failed. Left behind (this delete failed too) it
|
|
336
|
+
# still counts as no run, having no queue_name.
|
|
337
|
+
def discard_run(task, run)
|
|
338
|
+
run.delete
|
|
339
|
+
rescue StandardError => e
|
|
340
|
+
ErrorReporter.report(e, { action: "deploy_task_discard_run", task: task.key, run_id: run.id })
|
|
341
|
+
end
|
|
257
342
|
end
|
|
258
343
|
end
|
|
259
344
|
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Deploy
|
|
5
|
+
# Whether pgbus:add_deploy_task_run_outcomes has been migrated (issue
|
|
6
|
+
# #660): pgbus_deploy_task_runs.completed_at / failed_at and
|
|
7
|
+
# pgbus_deploy_tasks.dependencies. Until then after: is rejected at load,
|
|
8
|
+
# sync leaves dependencies alone and the executor writes no outcome.
|
|
9
|
+
#
|
|
10
|
+
# A "yes" is kept for the process; a "no" stands for RECHECK_SECONDS, so a
|
|
11
|
+
# process started before the migration picks the columns up without a
|
|
12
|
+
# restart. A probe that raises (a database blip) is not an answer: the
|
|
13
|
+
# error reaches the caller and nothing is kept, so a scheduler booting
|
|
14
|
+
# through a blip does not skip its after: tasks as unmigrated.
|
|
15
|
+
module Schema
|
|
16
|
+
RECHECK_SECONDS = 60
|
|
17
|
+
|
|
18
|
+
@mutex = Mutex.new
|
|
19
|
+
@answer = nil # true, nil (never probed) or the monotonic time a "no" runs out
|
|
20
|
+
|
|
21
|
+
class << self
|
|
22
|
+
def outcomes?
|
|
23
|
+
# Double-checked: the unlocked read only skips the lock while the
|
|
24
|
+
# answer stands; it is written under @mutex.
|
|
25
|
+
known = answer(@answer) # rubocop:disable ThreadSafety/ClassInstanceVariable
|
|
26
|
+
return known unless known.nil?
|
|
27
|
+
|
|
28
|
+
@mutex.synchronize do
|
|
29
|
+
known = answer(@answer)
|
|
30
|
+
next known unless known.nil?
|
|
31
|
+
|
|
32
|
+
supported = probe
|
|
33
|
+
@answer = supported || (monotonic_now + RECHECK_SECONDS)
|
|
34
|
+
supported
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Clears the memoized probe: specs exercise both schema shapes.
|
|
39
|
+
def reset!
|
|
40
|
+
@mutex.synchronize { @answer = nil }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def answer(state)
|
|
46
|
+
return true if state == true
|
|
47
|
+
|
|
48
|
+
false if state && monotonic_now < state
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def probe
|
|
52
|
+
BusRecord.connection_pool.with_connection do |conn|
|
|
53
|
+
conn.column_exists?(:pgbus_deploy_task_runs, :completed_at) &&
|
|
54
|
+
conn.column_exists?(:pgbus_deploy_task_runs, :failed_at) &&
|
|
55
|
+
conn.column_exists?(:pgbus_deploy_tasks, :dependencies)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def monotonic_now
|
|
60
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
data/lib/pgbus/deploy/task.rb
CHANGED
|
@@ -5,12 +5,13 @@ module Pgbus
|
|
|
5
5
|
# One declared deploy task (issue #536): an entry of
|
|
6
6
|
# config/deploy_tasks.yml. The sibling of Recurring::Task without a
|
|
7
7
|
# schedule: it runs once per (key, version) after a rollover, or after
|
|
8
|
-
# every rollover when every_deploy is set.
|
|
8
|
+
# every rollover when every_deploy is set. after: names the tasks it
|
|
9
|
+
# waits for (issue #660): see Deploy::Dependencies.
|
|
9
10
|
class Task
|
|
10
11
|
DEFAULT_VERSION = "1"
|
|
11
12
|
|
|
12
13
|
attr_reader :key, :class_name, :command, :version, :every_deploy, :queue_name,
|
|
13
|
-
:arguments, :priority, :description
|
|
14
|
+
:arguments, :priority, :description, :dependencies
|
|
14
15
|
|
|
15
16
|
def self.from_configuration(key, **options)
|
|
16
17
|
options = options.transform_keys(&:to_sym)
|
|
@@ -24,12 +25,14 @@ module Pgbus
|
|
|
24
25
|
queue_name: options[:queue],
|
|
25
26
|
arguments: options[:args].is_a?(Hash) ? [options[:args]] : Array(options[:args]),
|
|
26
27
|
priority: options.fetch(:priority, 0).to_i,
|
|
27
|
-
description: options[:description]
|
|
28
|
+
description: options[:description],
|
|
29
|
+
dependencies: options[:after]
|
|
28
30
|
)
|
|
29
31
|
end
|
|
30
32
|
|
|
31
33
|
def initialize(key:, class_name: nil, command: nil, version: DEFAULT_VERSION, explicit_version: false,
|
|
32
|
-
every_deploy: false, queue_name: nil, arguments: [], priority: 0, description: nil
|
|
34
|
+
every_deploy: false, queue_name: nil, arguments: [], priority: 0, description: nil,
|
|
35
|
+
dependencies: nil)
|
|
33
36
|
@key = key
|
|
34
37
|
@class_name = class_name
|
|
35
38
|
@command = command
|
|
@@ -40,6 +43,7 @@ module Pgbus
|
|
|
40
43
|
@arguments = arguments || []
|
|
41
44
|
@priority = priority || 0
|
|
42
45
|
@description = description
|
|
46
|
+
@dependencies = dependency_list(dependencies)
|
|
43
47
|
@errors = []
|
|
44
48
|
@warnings = []
|
|
45
49
|
end
|
|
@@ -74,7 +78,8 @@ module Pgbus
|
|
|
74
78
|
def to_h
|
|
75
79
|
{
|
|
76
80
|
key: key, class_name: class_name, command: command, version: version, every_deploy: every_deploy,
|
|
77
|
-
queue_name: queue_name, arguments: arguments, priority: priority, description: description
|
|
81
|
+
queue_name: queue_name, arguments: arguments, priority: priority, description: description,
|
|
82
|
+
dependencies: dependencies
|
|
78
83
|
}
|
|
79
84
|
end
|
|
80
85
|
|
|
@@ -86,11 +91,24 @@ module Pgbus
|
|
|
86
91
|
validate_target!
|
|
87
92
|
@errors << "Version must not be blank" if version.empty?
|
|
88
93
|
@errors << "every_deploy must be true or false" unless [true, false].include?(every_deploy)
|
|
94
|
+
validate_dependencies!
|
|
89
95
|
return unless every_deploy == true && @explicit_version
|
|
90
96
|
|
|
91
97
|
@warnings << "every_deploy is set, so version is ignored (it runs after every rollover)"
|
|
92
98
|
end
|
|
93
99
|
|
|
100
|
+
def validate_dependencies!
|
|
101
|
+
@errors << "after: must name task keys" unless dependencies.all? { |dep| dep.is_a?(String) && !dep.strip.empty? }
|
|
102
|
+
@errors << "after: cycle #{key} \u2192 #{key}" if dependencies.include?(key)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# nil, one key, or a list; a Symbol reads as its name. Anything else is
|
|
106
|
+
# kept as given, for validate_dependencies! to reject.
|
|
107
|
+
def dependency_list(value)
|
|
108
|
+
list = value.is_a?(Array) ? value : [value].compact
|
|
109
|
+
list.map { |dep| dep.is_a?(Symbol) ? dep.to_s : dep }.uniq.freeze
|
|
110
|
+
end
|
|
111
|
+
|
|
94
112
|
def validate_target!
|
|
95
113
|
{ class: class_name, command: command }.each do |name, value|
|
|
96
114
|
@errors << "#{name} must be a String" unless value.nil? || value.is_a?(String)
|
data/lib/pgbus/engine.rb
CHANGED
|
@@ -94,6 +94,7 @@ module Pgbus
|
|
|
94
94
|
add_outbox: "pgbus:add_outbox",
|
|
95
95
|
add_recurring: "pgbus:add_recurring",
|
|
96
96
|
add_deploy_tasks: "pgbus:add_deploy_tasks",
|
|
97
|
+
add_deploy_task_run_outcomes: "pgbus:add_deploy_task_run_outcomes",
|
|
97
98
|
add_workflows: "pgbus:add_workflows",
|
|
98
99
|
add_workflow_signals: "pgbus:add_workflow_signals",
|
|
99
100
|
add_failed_events_index: "pgbus:add_failed_events_index",
|
|
@@ -121,6 +122,7 @@ module Pgbus
|
|
|
121
122
|
add_outbox: "outbox entries table (transactional outbox)",
|
|
122
123
|
add_recurring: "recurring tasks + executions tables",
|
|
123
124
|
add_deploy_tasks: "deploy tasks + runs tables (one-time jobs run after a rollover)",
|
|
125
|
+
add_deploy_task_run_outcomes: "deploy task run outcomes (completed_at, failed_at) and after: dependencies",
|
|
124
126
|
add_workflows: "workflow tables (durable multi-step workflows: runs, step attempts, audit trail)",
|
|
125
127
|
add_workflow_signals: "waiting workflow steps (fan_out, wait_for: signals, wait timeouts)",
|
|
126
128
|
add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)",
|
|
@@ -253,10 +255,18 @@ module Pgbus
|
|
|
253
255
|
|
|
254
256
|
# Same shape as recurring_migrations: plain create_table in the
|
|
255
257
|
# template, so the generator is queued only while a table is absent.
|
|
258
|
+
# The outcomes add-on (issue #660) follows the tables: its template
|
|
259
|
+
# adds each column if_not_exists, so queuing it after add_deploy_tasks
|
|
260
|
+
# also completes a half-present schema whose existing table lacks them.
|
|
256
261
|
def deploy_tasks_migrations
|
|
257
|
-
return []
|
|
258
|
-
|
|
259
|
-
|
|
262
|
+
return %i[add_deploy_tasks add_deploy_task_run_outcomes] unless
|
|
263
|
+
table_exists?("pgbus_deploy_tasks") && table_exists?("pgbus_deploy_task_runs")
|
|
264
|
+
|
|
265
|
+
runs = column_names("pgbus_deploy_task_runs")
|
|
266
|
+
outcomes = runs.include?("completed_at") && runs.include?("failed_at") &&
|
|
267
|
+
column_names("pgbus_deploy_tasks").include?("dependencies") &&
|
|
268
|
+
index_exists?("pgbus_deploy_task_runs", "idx_pgbus_deploy_task_runs_completed")
|
|
269
|
+
outcomes ? [] : [:add_deploy_task_run_outcomes]
|
|
260
270
|
end
|
|
261
271
|
|
|
262
272
|
# The add_workflows template uses if_not_exists throughout, so a
|
|
@@ -11,7 +11,7 @@ module Pgbus
|
|
|
11
11
|
tool_name "pgbus_deploy_tasks"
|
|
12
12
|
title "Pgbus Deploy Tasks"
|
|
13
13
|
description <<~DESC
|
|
14
|
-
Deploy tasks: one-time jobs the scheduler enqueues once every live pgbus process runs the new app revision. Lists each task's state (disabled, waiting, due, ran, never)
|
|
14
|
+
Deploy tasks: one-time jobs the scheduler enqueues once every live pgbus process runs the new app revision. Lists each task's state (disabled, waiting, due, ran, never), the rollover status the scheduler waits on, and for a task with after: its dependencies and the ones it still waits on (waiting_on, each not_run, running or failed).
|
|
15
15
|
Returns every task unless page or per_page is given; then it is
|
|
16
16
|
paginated (per_page defaults to 100, capped at 100). The response
|
|
17
17
|
carries page, per_page, total (capped at 10000 when paginated; the
|
data/lib/pgbus/version.rb
CHANGED