pact_broker 2.118.0 → 2.120.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +49 -0
- data/Gemfile +2 -0
- data/README.md +1 -1
- data/db/migrations/000015_create_pact_version_content.rb +1 -1
- data/db/migrations/000023_create_pact_versions_table.rb +1 -1
- data/db/migrations/000033_create_config_table.rb +1 -1
- data/db/migrations/000036_create_webhook_execution.rb +1 -1
- data/db/migrations/20180108_create_certificates_table.rb +1 -1
- data/db/migrations/20251106_add_branch_versions_version_id_branch_name_index.rb +21 -0
- data/db/migrations/20260119_add_index_on_consumer_id_provider_id_to_pact_versions.rb +20 -0
- data/db/migrations/20260123_drop_branch_versions_branch_name_index.rb +17 -0
- data/db/migrations/20260210_add_index_on_pact_version_id_to_latest_pact_publication.rb +22 -0
- data/db/migrations/20260217_add_wip_query_optimization_indexes.rb +31 -0
- data/db/migrations/migration_helper.rb +5 -1
- data/lib/pact_broker/api/decorators/triggered_webhook_logs_decorator.rb +4 -0
- data/lib/pact_broker/api/decorators/version_decorator.rb +24 -4
- data/lib/pact_broker/api/resources/branch_versions.rb +5 -1
- data/lib/pact_broker/api/resources/can_i_deploy_pacticipant_version_by_branch_to_environment.rb +4 -1
- data/lib/pact_broker/api/resources/can_i_deploy_pacticipant_version_by_branch_to_environment_badge.rb +1 -0
- data/lib/pact_broker/api/resources/clean.rb +9 -1
- data/lib/pact_broker/api/resources/pacticipants_for_label.rb +29 -1
- data/lib/pact_broker/api/resources/provider_pacts_for_verification.rb +1 -1
- data/lib/pact_broker/api/resources/tag_versions.rb +5 -1
- data/lib/pact_broker/api/resources/version.rb +5 -1
- data/lib/pact_broker/api/resources/versions.rb +5 -1
- data/lib/pact_broker/configuration.rb +26 -0
- data/lib/pact_broker/db/clean/branch_selector.rb +40 -0
- data/lib/pact_broker/db/clean.rb +45 -0
- data/lib/pact_broker/db/clean_incremental.rb +77 -9
- data/lib/pact_broker/deployments/deployed_version_service.rb +1 -0
- data/lib/pact_broker/deployments/released_version_service.rb +14 -0
- data/lib/pact_broker/domain/pacticipant.rb +12 -1
- data/lib/pact_broker/logging.rb +10 -0
- data/lib/pact_broker/matrix/parse_can_i_deploy_query.rb +5 -0
- data/lib/pact_broker/pacticipants/repository.rb +1 -1
- data/lib/pact_broker/pacts/pact_publication_dataset_module.rb +5 -1
- data/lib/pact_broker/pacts/pact_publication_wip_dataset_module.rb +13 -4
- data/lib/pact_broker/pacts/pacts_for_verification_repository.rb +90 -2
- data/lib/pact_broker/pacts/repository.rb +2 -2
- data/lib/pact_broker/pacts/service.rb +1 -1
- data/lib/pact_broker/tasks/clean_task.rb +23 -1
- data/lib/pact_broker/version.rb +1 -1
- data/lib/pact_broker/versions/branch_repository.rb +1 -1
- data/lib/pact_broker/versions/branch_service.rb +1 -1
- data/lib/pact_broker/versions/branch_version_repository.rb +2 -1
- data/lib/pact_broker/webhooks/event_listener.rb +1 -1
- data/lib/pact_broker/webhooks/execution_configuration.rb +4 -0
- data/lib/pact_broker/webhooks/redact_logs.rb +4 -4
- data/lib/sequel/plugins/upsert.rb +7 -0
- data/pact_broker.gemspec +6 -5
- metadata +30 -28
data/lib/pact_broker/db/clean.rb
CHANGED
|
@@ -2,6 +2,7 @@ require "sequel"
|
|
|
2
2
|
require "pact_broker/project_root"
|
|
3
3
|
require "pact_broker/logging"
|
|
4
4
|
require "pact_broker/db/clean/selector"
|
|
5
|
+
require "pact_broker/db/clean/branch_selector"
|
|
5
6
|
|
|
6
7
|
module PactBroker
|
|
7
8
|
module DB
|
|
@@ -130,6 +131,9 @@ module PactBroker
|
|
|
130
131
|
delete_orphan_tags
|
|
131
132
|
delete_orphan_versions
|
|
132
133
|
|
|
134
|
+
deleted_counts[:stale_branches] = delete_stale_branches
|
|
135
|
+
kept_counts[:stale_branches] = db[:branches].count
|
|
136
|
+
|
|
133
137
|
{ kept: kept_counts, deleted: deleted_counts }
|
|
134
138
|
end
|
|
135
139
|
|
|
@@ -180,6 +184,47 @@ module PactBroker
|
|
|
180
184
|
db[:versions].where(id: referenced_version_ids).invert.delete
|
|
181
185
|
end
|
|
182
186
|
|
|
187
|
+
def delete_stale_branches
|
|
188
|
+
return 0 unless keep_branches && !keep_branches.empty?
|
|
189
|
+
|
|
190
|
+
db[:branches].where(id: stale_branch_ids_to_delete.from_self.select(:id)).delete
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def stale_branch_ids_to_delete
|
|
194
|
+
db[:branches]
|
|
195
|
+
.select(Sequel[:branches][:id])
|
|
196
|
+
.left_outer_join(stale_branch_ids_to_keep, { Sequel[:branches][:id] => Sequel[:keep_branches][:id] }, table_alias: :keep_branches)
|
|
197
|
+
.where(Sequel[:keep_branches][:id] => nil)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def stale_branch_ids_to_keep
|
|
201
|
+
queries = [main_branch_ids_to_keep]
|
|
202
|
+
|
|
203
|
+
keep_branches.each do | selector |
|
|
204
|
+
if selector.max_age
|
|
205
|
+
cutoff = Date.today - selector.max_age
|
|
206
|
+
queries << db[:branches].where { updated_at >= cutoff }.select(:id)
|
|
207
|
+
end
|
|
208
|
+
if selector.branch
|
|
209
|
+
queries << db[:branches].where(name: Array(selector.branch)).select(:id)
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
queries.reduce(:union)
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def main_branch_ids_to_keep
|
|
217
|
+
db[:branches]
|
|
218
|
+
.join(:pacticipants, id: Sequel[:branches][:pacticipant_id])
|
|
219
|
+
.exclude(Sequel[:pacticipants][:main_branch] => nil)
|
|
220
|
+
.where(Sequel[:branches][:name] => Sequel[:pacticipants][:main_branch])
|
|
221
|
+
.select(Sequel[:branches][:id])
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def keep_branches
|
|
225
|
+
@keep_branches ||= options[:keep_branches]&.collect { | selector_hash | BranchSelector.from_hash(selector_hash.to_hash) }
|
|
226
|
+
end
|
|
227
|
+
|
|
183
228
|
def delete_overwritten_verifications
|
|
184
229
|
verification_ids = db[:verifications].select(:id).where(id: db[:latest_verification_id_for_pact_version_and_provider_version].select(:verification_id)).invert
|
|
185
230
|
deleted_counts = { verification_results: verification_ids.count }
|
|
@@ -2,6 +2,7 @@ require "pact_broker/logging"
|
|
|
2
2
|
require "pact_broker/matrix/unresolved_selector"
|
|
3
3
|
require "pact_broker/date_helper"
|
|
4
4
|
require "pact_broker/db/clean/selector"
|
|
5
|
+
require "pact_broker/db/clean/branch_selector"
|
|
5
6
|
|
|
6
7
|
module PactBroker
|
|
7
8
|
module DB
|
|
@@ -14,7 +15,10 @@ module PactBroker
|
|
|
14
15
|
PactBroker::DB::Clean::Selector.new(released: true),
|
|
15
16
|
PactBroker::DB::Clean::Selector.new(max_age: 90)
|
|
16
17
|
]
|
|
17
|
-
|
|
18
|
+
DEFAULT_KEEP_BRANCH_SELECTORS = [
|
|
19
|
+
PactBroker::DB::Clean::BranchSelector.new(max_age: 90)
|
|
20
|
+
]
|
|
21
|
+
TABLES = [:versions, :pact_publications, :pact_versions, :verifications, :triggered_webhooks, :webhook_executions, :branches]
|
|
18
22
|
|
|
19
23
|
def self.call database_connection, options = {}
|
|
20
24
|
new(database_connection, options).call
|
|
@@ -44,6 +48,7 @@ module PactBroker
|
|
|
44
48
|
before_counts = current_counts
|
|
45
49
|
PactBroker::Domain::Version.where(id: versions_to_delete.from_self.select_map(:id)).delete
|
|
46
50
|
delete_orphan_pact_versions
|
|
51
|
+
delete_stale_branches
|
|
47
52
|
after_counts = current_counts
|
|
48
53
|
|
|
49
54
|
TABLES.each_with_object({}) do | table_name, comparison_counts |
|
|
@@ -68,10 +73,22 @@ module PactBroker
|
|
|
68
73
|
end
|
|
69
74
|
end
|
|
70
75
|
|
|
76
|
+
def keep_branches
|
|
77
|
+
@keep_branches ||= if options.key?(:keep_branches)
|
|
78
|
+
options[:keep_branches]&.collect { | selector_hash | PactBroker::DB::Clean::BranchSelector.from_hash(selector_hash.to_hash) }
|
|
79
|
+
else
|
|
80
|
+
DEFAULT_KEEP_BRANCH_SELECTORS
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
71
84
|
def limit
|
|
72
85
|
options[:limit] || 1000
|
|
73
86
|
end
|
|
74
87
|
|
|
88
|
+
def branch_limit
|
|
89
|
+
options[:branch_limit] || limit
|
|
90
|
+
end
|
|
91
|
+
|
|
75
92
|
def versions_to_delete(columns = [:id])
|
|
76
93
|
fully_qualified_columns = columns.collect { |col| Sequel[:versions][col] }
|
|
77
94
|
PactBroker::Domain::Version
|
|
@@ -96,6 +113,47 @@ module PactBroker
|
|
|
96
113
|
options[:dry_run]
|
|
97
114
|
end
|
|
98
115
|
|
|
116
|
+
def delete_stale_branches
|
|
117
|
+
return unless keep_branches && !keep_branches.empty?
|
|
118
|
+
|
|
119
|
+
# A direct subquery here fails in MySQL, so wrap the dataset in a
|
|
120
|
+
# derived table with `from_self` before selecting the ids for deletion.
|
|
121
|
+
PactBroker::Versions::Branch.where(id: stale_branch_ids_to_delete.from_self.select(:id)).delete
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def stale_branch_ids_to_delete
|
|
125
|
+
PactBroker::Versions::Branch
|
|
126
|
+
.select(Sequel[:branches][:id])
|
|
127
|
+
.left_outer_join(stale_branch_ids_to_keep, { Sequel[:branches][:id] => Sequel[:keep_branches][:id] }, table_alias: :keep_branches)
|
|
128
|
+
.where(Sequel[:keep_branches][:id] => nil)
|
|
129
|
+
.limit(branch_limit)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def stale_branch_ids_to_keep
|
|
133
|
+
# Always keep main branches for each pacticipant
|
|
134
|
+
queries = [main_branch_ids_to_keep]
|
|
135
|
+
|
|
136
|
+
keep_branches.each do | selector |
|
|
137
|
+
if selector.max_age
|
|
138
|
+
cutoff = Date.today - selector.max_age
|
|
139
|
+
queries << PactBroker::Versions::Branch.where { updated_at >= cutoff }.select(Sequel[:branches][:id])
|
|
140
|
+
end
|
|
141
|
+
if selector.branch
|
|
142
|
+
queries << PactBroker::Versions::Branch.where(name: Array(selector.branch)).select(Sequel[:branches][:id])
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
queries.reduce(&:union)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def main_branch_ids_to_keep
|
|
150
|
+
PactBroker::Versions::Branch
|
|
151
|
+
.join(:pacticipants, id: Sequel[:branches][:pacticipant_id])
|
|
152
|
+
.exclude(Sequel[:pacticipants][:main_branch] => nil)
|
|
153
|
+
.where(Sequel[:branches][:name] => Sequel[:pacticipants][:main_branch])
|
|
154
|
+
.select(Sequel[:branches][:id])
|
|
155
|
+
end
|
|
156
|
+
|
|
99
157
|
def delete_orphan_pact_versions
|
|
100
158
|
db[:pact_versions].where(id: orphan_pact_versions).delete
|
|
101
159
|
rescue Sequel::DatabaseError => e
|
|
@@ -142,19 +200,29 @@ module PactBroker
|
|
|
142
200
|
}
|
|
143
201
|
end
|
|
144
202
|
|
|
203
|
+
{
|
|
204
|
+
"counts" => dry_run_counts(kept_per_selector),
|
|
205
|
+
"versionSummary" => pacticipant_results
|
|
206
|
+
}
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def dry_run_counts(kept_per_selector)
|
|
145
210
|
total_versions_count = PactBroker::Domain::Version.count
|
|
146
211
|
versions_to_keep_count = version_ids_to_keep.count
|
|
147
212
|
versions_to_delete_count = versions_to_delete.count
|
|
148
213
|
|
|
214
|
+
total_branches_count = PactBroker::Versions::Branch.count
|
|
215
|
+
branches_to_delete_count = keep_branches && !keep_branches.empty? ? stale_branch_ids_to_delete.count : 0
|
|
216
|
+
|
|
149
217
|
{
|
|
150
|
-
"
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
"
|
|
218
|
+
"totalVersions" => total_versions_count,
|
|
219
|
+
"versionsToDelete" => versions_to_delete_count,
|
|
220
|
+
"versionsNotToKeep" => total_versions_count - versions_to_keep_count,
|
|
221
|
+
"versionsToKeep" => versions_to_keep_count,
|
|
222
|
+
"versionsToKeepBySelector" => kept_per_selector,
|
|
223
|
+
"totalBranches" => total_branches_count,
|
|
224
|
+
"branchesToDelete" => branches_to_delete_count,
|
|
225
|
+
"branchesToKeep" => total_branches_count - branches_to_delete_count
|
|
158
226
|
}
|
|
159
227
|
end
|
|
160
228
|
|
|
@@ -46,6 +46,20 @@ module PactBroker
|
|
|
46
46
|
.single_record
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
+
def self.find_released_versions_for_version(version)
|
|
50
|
+
scope_for(ReleasedVersion)
|
|
51
|
+
.where(version_id: version.id)
|
|
52
|
+
.eager(:environment)
|
|
53
|
+
.all
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def self.find_released_versions_for_versions(versions_array)
|
|
57
|
+
scope_for(ReleasedVersion)
|
|
58
|
+
.where(version_id: versions_array.map(&:id))
|
|
59
|
+
.eager(:environment)
|
|
60
|
+
.to_hash_groups(:version_id)
|
|
61
|
+
end
|
|
62
|
+
|
|
49
63
|
def self.record_version_support_ended(released_version)
|
|
50
64
|
released_version.record_support_ended
|
|
51
65
|
end
|
|
@@ -55,11 +55,22 @@ module PactBroker
|
|
|
55
55
|
PactBroker::Pacts::PactPublication.where(provider: self).delete
|
|
56
56
|
PactBroker::Domain::Verification.where(consumer: self).or(provider: self).delete
|
|
57
57
|
PactBroker::Domain::Version.where(pacticipant: self).delete
|
|
58
|
-
|
|
58
|
+
delete_pact_versions_in_batches
|
|
59
59
|
PactBroker::Domain::Label.where(pacticipant: self).destroy
|
|
60
60
|
super
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
+
BATCH_DELETE_SIZE = 500
|
|
64
|
+
def delete_pact_versions_in_batches
|
|
65
|
+
dataset = PactBroker::Pacts::PactVersion.where(consumer: self).or(provider: self).order(:id)
|
|
66
|
+
loop do
|
|
67
|
+
deleted = PactBroker::Pacts::PactVersion
|
|
68
|
+
.where(id: dataset.limit(BATCH_DELETE_SIZE).select(:id))
|
|
69
|
+
.delete
|
|
70
|
+
break if deleted.zero?
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
63
74
|
def before_save
|
|
64
75
|
super
|
|
65
76
|
self.display_name = generate_display_name(name) if display_name.blank?
|
data/lib/pact_broker/logging.rb
CHANGED
|
@@ -52,6 +52,16 @@ module PactBroker
|
|
|
52
52
|
end
|
|
53
53
|
end
|
|
54
54
|
|
|
55
|
+
def measure_debug(message, payload: {})
|
|
56
|
+
if logger.respond_to?(:measure_debug)
|
|
57
|
+
logger.measure_debug(message, payload: payload) do
|
|
58
|
+
yield
|
|
59
|
+
end
|
|
60
|
+
else
|
|
61
|
+
yield
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
55
65
|
def log_error e, description = nil
|
|
56
66
|
if logger.instance_of?(SemanticLogger::Logger)
|
|
57
67
|
if description
|
|
@@ -26,6 +26,11 @@ module PactBroker
|
|
|
26
26
|
|
|
27
27
|
if params[:environment].is_a?(String)
|
|
28
28
|
options[:environment_name] = params[:environment]
|
|
29
|
+
# Multiple versions of an integrated application can be simultaneously released to an
|
|
30
|
+
# environment (record-release), or deployed to it on different targets (record-deployment).
|
|
31
|
+
# "cvp" collapses these down to the latest provider version, hiding incompatible versions
|
|
32
|
+
# that are still live. "cvpv" evaluates every co-resident version independently. See issue #903.
|
|
33
|
+
options[:latestby] = "cvpv"
|
|
29
34
|
end
|
|
30
35
|
|
|
31
36
|
if params[:ignore].is_a?(Array)
|
|
@@ -41,7 +41,7 @@ module PactBroker
|
|
|
41
41
|
return [] if query.empty?
|
|
42
42
|
|
|
43
43
|
query = query.select_all_qualified
|
|
44
|
-
query = query.filter(:name, options[:query_string]) if options[:query_string]
|
|
44
|
+
query = query.filter(Sequel[:pacticipants][:name], options[:query_string]) if options[:query_string]
|
|
45
45
|
query = query.label(options[:label_name]) if options[:label_name]
|
|
46
46
|
query.order_ignore_case(Sequel[:pacticipants][:name]).eager(*eager_load_associations).all_with_pagination_options(pagination_options)
|
|
47
47
|
end
|
|
@@ -287,9 +287,13 @@ module PactBroker
|
|
|
287
287
|
# NEW LOGIC
|
|
288
288
|
# @return [Sequel::Dataset<PactBroker::Pacts::PactPublication>]
|
|
289
289
|
def for_all_tag_heads
|
|
290
|
+
# Optimization: Only aggregate tags for consumers that have pacts for this provider.
|
|
291
|
+
relevant_consumer_ids = self.select(:consumer_id).distinct
|
|
292
|
+
|
|
290
293
|
head_tags = PactBroker::Domain::Tag
|
|
291
294
|
.select_group(:pacticipant_id, :name)
|
|
292
|
-
.select_append{ max(version_order).as(:latest_version_order) }
|
|
295
|
+
.select_append{ max(:version_order).as(:latest_version_order) }
|
|
296
|
+
.where(pacticipant_id: relevant_consumer_ids)
|
|
293
297
|
|
|
294
298
|
head_tags_join = {
|
|
295
299
|
Sequel[:pact_publications][:consumer_id] => Sequel[:head_tags][:pacticipant_id],
|
|
@@ -19,13 +19,19 @@ module PactBroker
|
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
def join_branch_versions_excluding_branch(provider_id, branch_name)
|
|
22
|
+
# Only check branches that have been active in the last N days
|
|
23
|
+
# to avoid old stale branches causing cartesian explosion.
|
|
24
|
+
# Filter by verification execution_date rather than version created_at
|
|
25
|
+
# to avoid incorrectly excluding old versions with recent verifications.
|
|
26
|
+
recent_branch_cutoff = DateTime.now - PactBroker.configuration.verified_by_other_branch_before_this_branch_look_back
|
|
27
|
+
|
|
22
28
|
branch_versions_join = {
|
|
23
29
|
Sequel[:verifications][:provider_version_id] => Sequel[:branch_versions][:version_id],
|
|
24
30
|
Sequel[:branch_versions][:pacticipant_id] => provider_id
|
|
25
31
|
}
|
|
26
|
-
join(:branch_versions, branch_versions_join)
|
|
27
|
-
|
|
28
|
-
|
|
32
|
+
join(:branch_versions, branch_versions_join)
|
|
33
|
+
.where { Sequel[:verifications][:execution_date] >= recent_branch_cutoff }
|
|
34
|
+
.where { Sequel.lit("branch_versions.branch_name != ?", branch_name) }
|
|
29
35
|
end
|
|
30
36
|
|
|
31
37
|
def join_provider_versions_for_provider_id_and_branch(provider_id, provider_version_branch)
|
|
@@ -45,10 +51,13 @@ module PactBroker
|
|
|
45
51
|
end
|
|
46
52
|
|
|
47
53
|
def successfully_verified_by_provider_branch_when_not_wip(provider_id, provider_version_branch)
|
|
54
|
+
# Only check verifications from last N days (old verifications unlikely relevant for WIP)
|
|
55
|
+
recent_cutoff = DateTime.now - PactBroker.configuration.verified_by_other_branch_before_this_branch_look_back
|
|
48
56
|
successful_verifications = VerificationForWipCalculations
|
|
49
57
|
.select(:pact_version_id)
|
|
50
58
|
.distinct
|
|
51
59
|
.successful_non_wip_by_provider(provider_id)
|
|
60
|
+
.where { Sequel[:verifications][:execution_date] >= recent_cutoff }
|
|
52
61
|
.join_provider_versions_for_provider_id_and_branch(provider_id, provider_version_branch)
|
|
53
62
|
|
|
54
63
|
|
|
@@ -65,8 +74,8 @@ module PactBroker
|
|
|
65
74
|
.select(:pact_version_id)
|
|
66
75
|
.distinct
|
|
67
76
|
.successful_non_wip_by_provider(provider_id)
|
|
68
|
-
.join_branch_versions_excluding_branch(provider_id, provider_version_branch)
|
|
69
77
|
.verified_before_creation_date_of(first_version_for_branch)
|
|
78
|
+
.join_branch_versions_excluding_branch(provider_id, provider_version_branch)
|
|
70
79
|
|
|
71
80
|
from_self(alias: :pp)
|
|
72
81
|
.select(Sequel[:pp].*)
|
|
@@ -57,7 +57,7 @@ module PactBroker
|
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
provider = pacticipant_repository.find_by_name(provider_name)
|
|
60
|
-
wip_start_date = options
|
|
60
|
+
wip_start_date = determine_wip_start_date(provider, options)
|
|
61
61
|
|
|
62
62
|
wip_by_consumer_tags = find_wip_pact_versions_for_provider_by_provider_tags(
|
|
63
63
|
provider,
|
|
@@ -80,6 +80,94 @@ module PactBroker
|
|
|
80
80
|
|
|
81
81
|
private
|
|
82
82
|
|
|
83
|
+
# Determine the WIP start date based on configuration and options
|
|
84
|
+
def determine_wip_start_date(provider, options)
|
|
85
|
+
dynamic_enabled = PactBroker.configuration.dynamic_wip_window_enabled?
|
|
86
|
+
logger.debug("Dynamic WIP window: #{dynamic_enabled ? 'enabled' : 'disabled'}") if logger.debug?
|
|
87
|
+
|
|
88
|
+
if dynamic_enabled
|
|
89
|
+
calculate_wip_window(provider)
|
|
90
|
+
else
|
|
91
|
+
wip_start_date = options.fetch(:include_wip_pacts_since)
|
|
92
|
+
logger.debug("Using user-specified WIP window: #{wip_start_date}") if logger.debug?
|
|
93
|
+
wip_start_date
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Calculate optimal WIP window using P80 (80th percentile) of unverified pact ages
|
|
98
|
+
# Uses percentile instead of MAX to focus on active work and ignore abandoned pacts
|
|
99
|
+
# Returns: Date from N days ago (min 7 for weekly coverage, max 14 to align with query scope)
|
|
100
|
+
def calculate_wip_window(provider)
|
|
101
|
+
# Step 1: Get list of unverified pact ages in days (e.g., [2, 5, 7, 10, 12, 14])
|
|
102
|
+
unverified_ages = get_unverified_pact_ages_in_days(provider)
|
|
103
|
+
|
|
104
|
+
# Step 2: If no unverified pacts, use default configured window
|
|
105
|
+
if unverified_ages.empty?
|
|
106
|
+
default_window = PactBroker.configuration.default_wip_window_days
|
|
107
|
+
logger.debug("No unverified pacts found for #{provider.name}, using default #{default_window}-day window") if logger.debug?
|
|
108
|
+
return Date.today - default_window
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Step 3: Calculate 80th percentile (ignore oldest 20% as abandoned work)
|
|
112
|
+
p80_age = percentile_80(unverified_ages)
|
|
113
|
+
|
|
114
|
+
# Step 4: Ensure window is between configured min-max days, then return date
|
|
115
|
+
min_window = PactBroker.configuration.min_wip_window_days
|
|
116
|
+
max_window = PactBroker.configuration.max_wip_lookback_days
|
|
117
|
+
window_days = clamp_between(p80_age, min_window, max_window)
|
|
118
|
+
wip_start_date = Date.today - window_days
|
|
119
|
+
|
|
120
|
+
if logger.debug?
|
|
121
|
+
logger.debug("Dynamic WIP window for #{provider.name}: found #{unverified_ages.size} unverified pacts, P80=#{p80_age} days, window=#{window_days} days (#{wip_start_date})")
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
wip_start_date
|
|
125
|
+
rescue StandardError => e
|
|
126
|
+
default_window = PactBroker.configuration.default_wip_window_days
|
|
127
|
+
logger.error("Failed to calculate dynamic WIP window for provider #{provider.name}", error: e.class.name, message: e.message)
|
|
128
|
+
Date.today - default_window
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Get ages (in days) of pacts that haven't been successfully verified yet
|
|
132
|
+
# Only looks back the configured max_wip_lookback_days to keep query fast
|
|
133
|
+
def get_unverified_pact_ages_in_days(provider)
|
|
134
|
+
max_days = PactBroker.configuration.max_wip_lookback_days
|
|
135
|
+
cutoff_date = DateTime.now - max_days
|
|
136
|
+
|
|
137
|
+
# Use database-specific date arithmetic that works on both PostgreSQL and SQLite
|
|
138
|
+
# PostgreSQL: CURRENT_DATE - created_at::date returns integer days
|
|
139
|
+
# SQLite: julianday(date('now')) - julianday(date(created_at)) returns float days
|
|
140
|
+
db = PactPublication.db
|
|
141
|
+
age_expr = if db.database_type == :postgres
|
|
142
|
+
Sequel.lit("CURRENT_DATE - ?::date", Sequel[:pact_publications][:created_at])
|
|
143
|
+
else
|
|
144
|
+
# SQLite
|
|
145
|
+
Sequel.function(:julianday, Sequel.function(:date, "now")) -
|
|
146
|
+
Sequel.function(:julianday, Sequel.function(:date, Sequel[:pact_publications][:created_at]))
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
PactPublication
|
|
150
|
+
.where(Sequel[:pact_publications][:provider_id] => provider.id)
|
|
151
|
+
.where { Sequel[:pact_publications][:created_at] >= cutoff_date }
|
|
152
|
+
.left_join(:verifications, { pact_version_id: :pact_version_id, success: true })
|
|
153
|
+
.where(Sequel[:verifications][:id] => nil)
|
|
154
|
+
.select_map { Sequel.cast(age_expr, :integer).as(:age_days) }
|
|
155
|
+
.compact
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Calculate 80th percentile: find value where 80% of data points are less than or equal
|
|
159
|
+
# Example: [2,3,5,7,10,12,14,20] → 80% index = 6 → value = 14
|
|
160
|
+
def percentile_80(values)
|
|
161
|
+
sorted = values.sort
|
|
162
|
+
index = [(sorted.length * 0.80).ceil - 1, 0].max
|
|
163
|
+
sorted[index]
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Constrain value between min and max bounds
|
|
167
|
+
def clamp_between(value, min, max)
|
|
168
|
+
[[value, max].min, min].max
|
|
169
|
+
end
|
|
170
|
+
|
|
83
171
|
# Note: created_at is coming back as a string for sqlite
|
|
84
172
|
# Can't work out how to to tell Sequel that this should be a date
|
|
85
173
|
def to_datetime string_or_datetime
|
|
@@ -227,7 +315,7 @@ module PactBroker
|
|
|
227
315
|
|
|
228
316
|
def find_wip_pact_versions_for_provider_by_provider_branch(provider_name, provider_version_branch, explicitly_specified_verifiable_pacts, options)
|
|
229
317
|
provider = pacticipant_repository.find_by_name(provider_name)
|
|
230
|
-
wip_start_date = options
|
|
318
|
+
wip_start_date = determine_wip_start_date(provider, options)
|
|
231
319
|
|
|
232
320
|
potential_wip_by_consumer_branch = PactPublication.for_provider(provider).created_after(wip_start_date).for_all_branch_heads
|
|
233
321
|
potential_wip_by_consumer_tag = PactPublication.for_provider(provider).created_after(wip_start_date).for_all_tag_heads
|
|
@@ -149,7 +149,7 @@ module PactBroker
|
|
|
149
149
|
query = query.overall_latest
|
|
150
150
|
end
|
|
151
151
|
|
|
152
|
-
query.all.
|
|
152
|
+
query.all.sort.collect(&:to_head_pact)
|
|
153
153
|
end
|
|
154
154
|
|
|
155
155
|
def find_pacts_by_consumer_branch(provider_name, options = {})
|
|
@@ -179,7 +179,7 @@ module PactBroker
|
|
|
179
179
|
query = query.for_branch_name(branch)
|
|
180
180
|
end
|
|
181
181
|
end
|
|
182
|
-
query.all.
|
|
182
|
+
query.all.sort.collect(&:to_head_pact)
|
|
183
183
|
end
|
|
184
184
|
|
|
185
185
|
def find_for_verification(provider_name, consumer_version_selectors)
|
|
@@ -210,7 +210,7 @@ module PactBroker
|
|
|
210
210
|
|
|
211
211
|
# When no publication for the given consumer/provider/consumer version number exists
|
|
212
212
|
def create_pact(params, version, provider)
|
|
213
|
-
logger.
|
|
213
|
+
logger.debug("Creating new pact publication", params.without(:json_content))
|
|
214
214
|
logger.debug("Content #{params[:json_content]}")
|
|
215
215
|
json_content = add_interaction_ids(params[:json_content])
|
|
216
216
|
pact = pact_repository.create(
|
|
@@ -10,7 +10,9 @@ module PactBroker
|
|
|
10
10
|
|
|
11
11
|
attr_accessor :database_connection
|
|
12
12
|
attr_reader :keep_version_selectors
|
|
13
|
+
attr_reader :keep_branch_selectors
|
|
13
14
|
attr_accessor :version_deletion_limit
|
|
15
|
+
attr_accessor :branch_deletion_limit
|
|
14
16
|
attr_accessor :logger
|
|
15
17
|
attr_accessor :dry_run
|
|
16
18
|
attr_accessor :use_lock # allow disabling of postgres lock if it is causing problems
|
|
@@ -21,6 +23,7 @@ module PactBroker
|
|
|
21
23
|
@dry_run = false
|
|
22
24
|
@use_lock = true
|
|
23
25
|
@keep_version_selectors = PactBroker::DB::CleanIncremental::DEFAULT_KEEP_SELECTORS
|
|
26
|
+
@keep_branch_selectors = PactBroker::DB::CleanIncremental::DEFAULT_KEEP_BRANCH_SELECTORS
|
|
24
27
|
rake_task(&block)
|
|
25
28
|
end
|
|
26
29
|
|
|
@@ -31,10 +34,17 @@ module PactBroker
|
|
|
31
34
|
end
|
|
32
35
|
end
|
|
33
36
|
|
|
37
|
+
def keep_branch_selectors=(keep_branch_selectors)
|
|
38
|
+
require "pact_broker/db/clean/branch_selector"
|
|
39
|
+
@keep_branch_selectors = keep_branch_selectors.nil? ? nil : [*keep_branch_selectors].collect do | hash |
|
|
40
|
+
PactBroker::DB::Clean::BranchSelector.from_hash(hash)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
34
44
|
def rake_task &block
|
|
35
45
|
namespace :pact_broker do
|
|
36
46
|
namespace :db do
|
|
37
|
-
desc "Clean unnecessary pacts and
|
|
47
|
+
desc "Clean unnecessary pacts, verifications and stale branches from database"
|
|
38
48
|
task :clean do | _t, _args |
|
|
39
49
|
instance_eval(&block)
|
|
40
50
|
|
|
@@ -61,10 +71,14 @@ module PactBroker
|
|
|
61
71
|
output "Deleting oldest #{version_deletion_limit} versions, keeping versions that match the configured selectors", keep_version_selectors.collect(&:to_hash)
|
|
62
72
|
end
|
|
63
73
|
|
|
74
|
+
output_keep_branch_selectors
|
|
75
|
+
|
|
64
76
|
start_time = Time.now
|
|
65
77
|
results = PactBroker::DB::CleanIncremental.call(database_connection,
|
|
66
78
|
keep: keep_version_selectors,
|
|
79
|
+
keep_branches: keep_branch_selectors,
|
|
67
80
|
limit: version_deletion_limit,
|
|
81
|
+
branch_limit: branch_deletion_limit,
|
|
68
82
|
logger: logger,
|
|
69
83
|
dry_run: dry_run
|
|
70
84
|
)
|
|
@@ -109,6 +123,14 @@ module PactBroker
|
|
|
109
123
|
logger ? logger.info("#{prefix}#{string}", payload) : puts("#{prefix}#{string} #{payload.to_json}")
|
|
110
124
|
end
|
|
111
125
|
|
|
126
|
+
def output_keep_branch_selectors
|
|
127
|
+
if keep_branch_selectors.nil? || keep_branch_selectors.empty?
|
|
128
|
+
output "Stale branch cleanup is disabled (no keep_branch_selectors configured)"
|
|
129
|
+
else
|
|
130
|
+
output "Deleting stale branches that do not match the configured selectors (main branches are always kept)", keep_branch_selectors.collect(&:to_hash)
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
112
134
|
def add_defaults_to_keep_selectors
|
|
113
135
|
if keep_version_selectors.none?(&:currently_deployed?)
|
|
114
136
|
selector = PactBroker::DB::Clean::Selector.new(deployed: true)
|
data/lib/pact_broker/version.rb
CHANGED
|
@@ -13,7 +13,7 @@ module PactBroker
|
|
|
13
13
|
query = scope_for(Branch).where(pacticipant_id: pacticipant.id).select_all_qualified
|
|
14
14
|
query = query.filter(:name, filter_options[:query_string]) if filter_options[:query_string]
|
|
15
15
|
query
|
|
16
|
-
.order(Sequel.desc(:
|
|
16
|
+
.order(Sequel.desc(:updated_at), Sequel.desc(:id))
|
|
17
17
|
.eager(*eager_load_associations)
|
|
18
18
|
.all_with_pagination_options(pagination_options)
|
|
19
19
|
end
|
|
@@ -21,7 +21,7 @@ module PactBroker
|
|
|
21
21
|
# @return [Array<PactBroker::Contracts::Notice>]
|
|
22
22
|
def branch_deletion_notices(pacticipant, exclude:)
|
|
23
23
|
count = branch_repository.count_branches_to_delete(pacticipant, exclude: exclude)
|
|
24
|
-
remaining = branch_repository.remaining_branches_after_future_deletion(pacticipant, exclude: exclude).sort_by(&:
|
|
24
|
+
remaining = branch_repository.remaining_branches_after_future_deletion(pacticipant, exclude: exclude).sort_by(&:updated_at).collect(&:name).join(", ")
|
|
25
25
|
[PactBroker::Contracts::Notice.success(message("messages.branch.bulk_delete", count: count, pacticipant_name: pacticipant.name, remaining: remaining))]
|
|
26
26
|
end
|
|
27
27
|
end
|
|
@@ -22,7 +22,7 @@ module PactBroker
|
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
def add_branch(version, branch_name, auto_created: false)
|
|
25
|
-
PactBroker.logger.
|
|
25
|
+
PactBroker.logger.debug("BranchVersionRepository#add_branch method called with version #{version.inspect} and branch_name '#{branch_name}'")
|
|
26
26
|
Sequel::Model.db.transaction do
|
|
27
27
|
branch = find_or_create_branch(version.pacticipant, branch_name)
|
|
28
28
|
branch_version = version.branch_version_for_branch(branch)
|
|
@@ -34,6 +34,7 @@ module PactBroker
|
|
|
34
34
|
branch_version = PactBroker::Versions::BranchVersion.new(version: version, branch: branch, auto_created: auto_created).insert_ignore
|
|
35
35
|
PactBroker::Versions::BranchHead.new(branch: branch, branch_version: branch_version).upsert
|
|
36
36
|
end
|
|
37
|
+
branch.update(updated_at: Sequel.datetime_class.now)
|
|
37
38
|
pacticipant_service.maybe_set_main_branch(version.pacticipant, branch_name)
|
|
38
39
|
branch_version
|
|
39
40
|
end
|
|
@@ -56,7 +56,7 @@ module PactBroker
|
|
|
56
56
|
logger.debug("Event detected", event_name: event.name, event_comment: event.comment)
|
|
57
57
|
if event.triggered_webhooks&.any?
|
|
58
58
|
triggered_webhook_descriptions = event.triggered_webhooks.collect{ |tw| { event_name: event.name, webhook_uuid: tw.webhook_uuid, triggered_webhook_uuid: tw.uuid, webhook_description: tw.webhook.description } }
|
|
59
|
-
logger.
|
|
59
|
+
logger.debug("Triggered webhooks for #{event.name}", triggered_webhooks: triggered_webhook_descriptions)
|
|
60
60
|
else
|
|
61
61
|
logger.debug "No enabled webhooks found for event #{event.name}"
|
|
62
62
|
end
|
|
@@ -7,12 +7,12 @@ module PactBroker
|
|
|
7
7
|
|
|
8
8
|
using PactBroker::StringRefinements
|
|
9
9
|
|
|
10
|
-
def redact_logs(logs, values)
|
|
11
|
-
RedactLogs.call(logs, values)
|
|
10
|
+
def redact_logs(logs, values, pattern_substitutions = [])
|
|
11
|
+
RedactLogs.call(logs, values, pattern_substitutions)
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
def self.call logs, values
|
|
15
|
-
substitutions = HEADER_SUBSTITUTIONS + value_substitutions(values)
|
|
14
|
+
def self.call logs, values, pattern_substitutions = []
|
|
15
|
+
substitutions = HEADER_SUBSTITUTIONS + pattern_substitutions + value_substitutions(values)
|
|
16
16
|
|
|
17
17
|
substitutions.reduce(logs) do | agg_logs, (find, replace) |
|
|
18
18
|
agg_logs.gsub(find, replace)
|