pact_broker 2.109.1 → 2.111.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -0
- data/docs/developer/design_pattern_for_eager_loading_collections.md +23 -0
- data/docs/developer/matrix.md +95 -0
- data/docs/developer/rack.md +11 -0
- data/lib/pact_broker/api/contracts/consumer_version_selector_contract.rb +1 -1
- data/lib/pact_broker/api/decorators/base_decorator.rb +10 -1
- data/lib/pact_broker/api/decorators/label_decorator.rb +19 -13
- data/lib/pact_broker/api/decorators/labels_decorator.rb +23 -0
- data/lib/pact_broker/api/decorators/pacticipant_decorator.rb +1 -1
- data/lib/pact_broker/api/decorators/version_decorator.rb +14 -0
- data/lib/pact_broker/api/paths.rb +2 -1
- data/lib/pact_broker/api/resources/branch_versions.rb +1 -1
- data/lib/pact_broker/api/resources/can_i_merge_badge.rb +36 -0
- data/lib/pact_broker/api/resources/labels.rb +37 -0
- data/lib/pact_broker/api/resources/provider_pacts_for_verification.rb +5 -0
- data/lib/pact_broker/api/resources/versions.rb +1 -1
- data/lib/pact_broker/api.rb +4 -0
- data/lib/pact_broker/app.rb +24 -9
- data/lib/pact_broker/badges/service.rb +18 -0
- data/lib/pact_broker/contracts/service.rb +6 -2
- data/lib/pact_broker/db/advisory_lock.rb +58 -0
- data/lib/pact_broker/domain/version.rb +3 -2
- data/lib/pact_broker/integrations/integration.rb +11 -1
- data/lib/pact_broker/integrations/repository.rb +46 -7
- data/lib/pact_broker/integrations/service.rb +2 -0
- data/lib/pact_broker/labels/repository.rb +5 -0
- data/lib/pact_broker/labels/service.rb +4 -0
- data/lib/pact_broker/matrix/service.rb +26 -0
- data/lib/pact_broker/pacticipants/repository.rb +10 -1
- data/lib/pact_broker/pacts/pact_publication_dataset_module.rb +56 -3
- data/lib/pact_broker/pacts/pact_publication_selector_dataset_module.rb +1 -0
- data/lib/pact_broker/pacts/pacts_for_verification_repository.rb +4 -4
- data/lib/pact_broker/pacts/repository.rb +2 -2
- data/lib/pact_broker/pacts/selector.rb +8 -2
- data/lib/pact_broker/tasks/clean_task.rb +68 -26
- data/lib/pact_broker/test/test_data_builder.rb +17 -5
- data/lib/pact_broker/version.rb +1 -1
- data/lib/pact_broker/versions/repository.rb +2 -20
- data/lib/pact_broker/versions/service.rb +2 -6
- data/lib/pact_broker/webhooks/execution.rb +8 -8
- data/lib/sequel/extensions/pg_advisory_lock.rb +101 -0
- data/pact_broker.gemspec +1 -1
- metadata +13 -5
@@ -7,7 +7,17 @@ require "pact_broker/verifications/latest_verification_for_consumer_and_provider
|
|
7
7
|
|
8
8
|
module PactBroker
|
9
9
|
module Integrations
|
10
|
-
|
10
|
+
# The columns are explicitly specified for the Integration object so that the consumer_name and provider_name columns aren't included
|
11
|
+
# in the model.
|
12
|
+
# Those columns exist in the integrations table because the integrations table used to be an integrations view based on the
|
13
|
+
# pact_publications table, and those columns existed in the view.
|
14
|
+
# When the view was migrated to be a table (in db/migrations/20211102_create_table_temp_integrations.rb and the following migrations)
|
15
|
+
# the columns had to be maintained for backwards compatiblity.
|
16
|
+
# They are not used by the current code, however.
|
17
|
+
class Integration < Sequel::Model
|
18
|
+
INTEGRATION_COLUMNS = Sequel::Model.db.schema(:integrations).collect(&:first) - [:consumer_name, :provider_name]
|
19
|
+
set_dataset(Sequel::Model.db[:integrations].select(*INTEGRATION_COLUMNS))
|
20
|
+
|
11
21
|
set_primary_key :id
|
12
22
|
plugin :insert_ignore, identifying_columns: [:consumer_id, :provider_id]
|
13
23
|
associate(:many_to_one, :consumer, :class => "PactBroker::Domain::Pacticipant", :key => :consumer_id, :primary_key => :id)
|
@@ -28,6 +28,20 @@ module PactBroker
|
|
28
28
|
nil
|
29
29
|
end
|
30
30
|
|
31
|
+
# Ensure an Integration exists for each consumer/provider pair.
|
32
|
+
# Using SELECT ... INSERT IGNORE rather than just INSERT IGNORE so that we do not
|
33
|
+
# need to lock the table at all when the integrations already exist, which will
|
34
|
+
# be the most common use case. New integrations get created incredibly rarely.
|
35
|
+
# The INSERT IGNORE is used rather than just INSERT to handle race conditions
|
36
|
+
# when requests come in parallel.
|
37
|
+
# @param [Array<Object>] where each object has a consumer and a provider
|
38
|
+
def create_for_pacts(objects_with_consumer_and_provider)
|
39
|
+
published_integrations = objects_with_consumer_and_provider.collect{ |i| { consumer_id: i.consumer.id, provider_id: i.provider.id } }
|
40
|
+
existing_integrations = Sequel::Model.db[:integrations].select(:consumer_id, :provider_id).where(Sequel.|(*published_integrations) ).all
|
41
|
+
new_integrations = (published_integrations - existing_integrations).collect{ |i| i.merge(created_at: Sequel.datetime_class.now, contract_data_updated_at: Sequel.datetime_class.now) }
|
42
|
+
Integration.dataset.insert_ignore.multi_insert(new_integrations)
|
43
|
+
end
|
44
|
+
|
31
45
|
def delete(consumer_id, provider_id)
|
32
46
|
Integration.where(consumer_id: consumer_id, provider_id: provider_id).delete
|
33
47
|
end
|
@@ -36,18 +50,43 @@ module PactBroker
|
|
36
50
|
# @param [PactBroker::Domain::Pacticipant, nil] consumer the consumer for the integration, or nil if for a provider-only event (eg. Pactflow provider contract published)
|
37
51
|
# @param [PactBroker::Domain::Pacticipant] provider the provider for the integration
|
38
52
|
def set_contract_data_updated_at(consumer, provider)
|
39
|
-
|
40
|
-
.where({ consumer_id: consumer&.id, provider_id: provider.id }.compact )
|
41
|
-
.update(contract_data_updated_at: Sequel.datetime_class.now)
|
53
|
+
set_contract_data_updated_at_for_multiple_integrations([OpenStruct.new(consumer: consumer, provider: provider)])
|
42
54
|
end
|
43
55
|
|
44
56
|
|
45
|
-
# Sets the contract_data_updated_at for the integrations as specified by an array of objects which each have a consumer and provider
|
46
|
-
#
|
57
|
+
# Sets the contract_data_updated_at for the integrations as specified by an array of objects which each have a consumer and provider.
|
58
|
+
#
|
59
|
+
# The contract_data_updated_at attribute is only ever used for ordering the list of integrations on the index page of the *Pact Broker* UI,
|
60
|
+
# so that the most recently updated integrations (the ones you're likely working on) are showed at the top of the first page.
|
61
|
+
# There is often contention around updating it however, which can cause deadlocks, and slow down API responses.
|
62
|
+
# Because it's not a critical field (eg. it won't change any can-i-deploy results), the easiest way to reduce this contention
|
63
|
+
# is to just not update it if the row is locked, because if it is locked, the value of contract_data_updated_at is already
|
64
|
+
# going to be a date from a few seconds ago, which is perfectly fine for the purposes for which we are using the value.
|
65
|
+
#
|
66
|
+
# Notes on SKIP LOCKED:
|
67
|
+
# SKIP LOCKED is only supported by Postgres.
|
68
|
+
# When executing SELECT ... FOR UPDATE SKIP LOCKED, the SELECT will run immediately, not waiting for any other transactions,
|
69
|
+
# and only return rows that are not already locked by another transaction.
|
70
|
+
# The FOR UPDATE is required to make it work this way - SKIP LOCKED on its own does not work.
|
71
|
+
#
|
72
|
+
# @param [Array<Object>] where each object MAY have a consumer and does have a provider (for Pactflow provider contract published there is no consumer)
|
47
73
|
def set_contract_data_updated_at_for_multiple_integrations(objects_with_consumer_and_provider)
|
48
|
-
consumer_and_provider_ids = objects_with_consumer_and_provider.collect{ | object |
|
74
|
+
consumer_and_provider_ids = objects_with_consumer_and_provider.collect{ | object | { consumer_id: object.consumer&.id, provider_id: object.provider.id }.compact }.uniq
|
75
|
+
|
76
|
+
# MySQL doesn't support an UPDATE with a subquery. FFS. Really need to do a major version release and delete the support code.
|
77
|
+
criteria = if Integration.dataset.supports_skip_locked?
|
78
|
+
integration_ids_to_update = Integration
|
79
|
+
.select(:id)
|
80
|
+
.where(Sequel.|(*consumer_and_provider_ids))
|
81
|
+
.for_update
|
82
|
+
.skip_locked
|
83
|
+
{ id: integration_ids_to_update }
|
84
|
+
else
|
85
|
+
Sequel.|(*consumer_and_provider_ids)
|
86
|
+
end
|
87
|
+
|
49
88
|
Integration
|
50
|
-
.where(
|
89
|
+
.where(criteria)
|
51
90
|
.update(contract_data_updated_at: Sequel.datetime_class.now)
|
52
91
|
end
|
53
92
|
end
|
@@ -21,6 +21,7 @@ module PactBroker
|
|
21
21
|
# @param [PactBroker::Domain::Pacticipant] consumer or nil
|
22
22
|
# @param [PactBroker::Domain::Pacticipant] provider
|
23
23
|
def self.handle_contract_data_published(consumer, provider)
|
24
|
+
integration_repository.create_for_pact(consumer.id, provider.id)
|
24
25
|
integration_repository.set_contract_data_updated_at(consumer, provider)
|
25
26
|
end
|
26
27
|
|
@@ -28,6 +29,7 @@ module PactBroker
|
|
28
29
|
# Callback to invoke when a batch of contract data is published (eg. the publish contracts endpoint)
|
29
30
|
# @param [Array<Object>] where each object has a consumer and a provider
|
30
31
|
def self.handle_bulk_contract_data_published(objects_with_consumer_and_provider)
|
32
|
+
integration_repository.create_for_pacts(objects_with_consumer_and_provider)
|
31
33
|
integration_repository.set_contract_data_updated_at_for_multiple_integrations(objects_with_consumer_and_provider)
|
32
34
|
end
|
33
35
|
|
@@ -3,6 +3,11 @@ require "pact_broker/domain/label"
|
|
3
3
|
module PactBroker
|
4
4
|
module Labels
|
5
5
|
class Repository
|
6
|
+
|
7
|
+
def get_all_unique_labels pagination_options = {}
|
8
|
+
PactBroker::Domain::Label.distinct.select(:name).all_with_pagination_options(pagination_options)
|
9
|
+
end
|
10
|
+
|
6
11
|
def create args
|
7
12
|
Domain::Label.new(name: args.fetch(:name), pacticipant: args.fetch(:pacticipant)).save
|
8
13
|
end
|
@@ -9,6 +9,10 @@ module PactBroker
|
|
9
9
|
|
10
10
|
extend PactBroker::Repositories
|
11
11
|
|
12
|
+
def get_all_unique_labels pagination_options = {}
|
13
|
+
label_repository.get_all_unique_labels(pagination_options)
|
14
|
+
end
|
15
|
+
|
12
16
|
def create args
|
13
17
|
pacticipant = pacticipant_repository.find_by_name_or_create args.fetch(:pacticipant_name)
|
14
18
|
label_repository.create pacticipant: pacticipant, name: args.fetch(:label_name)
|
@@ -22,6 +22,32 @@ module PactBroker
|
|
22
22
|
QueryResultsWithDeploymentStatusSummary.new(query_results, DeploymentStatusSummary.new(query_results))
|
23
23
|
end
|
24
24
|
|
25
|
+
def can_i_merge(pacticipant_name: nil, pacticipant: nil, latest_main_branch_version: nil)
|
26
|
+
# first we find the pacticipant by name (or use the one passed in) if pacticipant is nil
|
27
|
+
if pacticipant.nil?
|
28
|
+
pacticipant = pacticipant_service.find_pacticipant_by_name(pacticipant_name)
|
29
|
+
raise PactBroker::Error.new("No pacticipant found with name '#{pacticipant_name}'") unless pacticipant
|
30
|
+
else
|
31
|
+
pacticipant_name = pacticipant.name
|
32
|
+
end
|
33
|
+
|
34
|
+
# then we find the latest version from the main branch if not passed in
|
35
|
+
if latest_main_branch_version.nil?
|
36
|
+
latest_main_branch_version = version_service.find_latest_version_from_main_branch(pacticipant)
|
37
|
+
raise PactBroker::Error.new("No main branch version found for pacticipant '#{pacticipant_name}'") unless latest_main_branch_version
|
38
|
+
end
|
39
|
+
|
40
|
+
selectors = PactBroker::Matrix::UnresolvedSelector.from_hash(
|
41
|
+
pacticipant_name: pacticipant_name,
|
42
|
+
pacticipant_version_number: latest_main_branch_version.number
|
43
|
+
)
|
44
|
+
|
45
|
+
options = { main_branch: true, latest: true, latestby: "cvp" }
|
46
|
+
query_results = can_i_deploy([selectors], options)
|
47
|
+
|
48
|
+
query_results.deployable?
|
49
|
+
end
|
50
|
+
|
25
51
|
def find selectors, options = {}
|
26
52
|
logger.info "Querying matrix", selectors: selectors, options: options
|
27
53
|
matrix_repository.find(selectors, options)
|
@@ -102,7 +102,16 @@ module PactBroker
|
|
102
102
|
|
103
103
|
def search_by_name(pacticipant_name)
|
104
104
|
terms = pacticipant_name.split.map { |v| v.gsub("_", "\\_") }
|
105
|
-
|
105
|
+
columns = [:name, :display_name]
|
106
|
+
string_match_query = Sequel.|(
|
107
|
+
*terms.map do |term|
|
108
|
+
Sequel.|(
|
109
|
+
*columns.map do |column|
|
110
|
+
Sequel.ilike(Sequel[:pacticipants][column], "%#{term}%")
|
111
|
+
end
|
112
|
+
)
|
113
|
+
end
|
114
|
+
)
|
106
115
|
scope_for(PactBroker::Domain::Pacticipant).where(string_match_query)
|
107
116
|
end
|
108
117
|
|
@@ -57,7 +57,13 @@ module PactBroker
|
|
57
57
|
end
|
58
58
|
end
|
59
59
|
|
60
|
-
#
|
60
|
+
# Returns the latest pact for each branch, returning a pact for every branch, even if
|
61
|
+
# the most recent version of that branch does not have a pact.
|
62
|
+
# This is different from for_all_branch_heads, which will find the branch head versions,
|
63
|
+
# and return the pacts associated with those versions.
|
64
|
+
# This method should not be used for 'pacts for verification', because it will return
|
65
|
+
# a pact for branches where that integration should no longer exist.
|
66
|
+
# @return [Dataset<PactBroker::Pacts::PactPublication>]
|
61
67
|
def latest_by_consumer_branch
|
62
68
|
branch_versions_join = {
|
63
69
|
Sequel[:pact_publications][:consumer_version_id] => Sequel[:branch_versions][:version_id]
|
@@ -112,10 +118,29 @@ module PactBroker
|
|
112
118
|
.limit(1)
|
113
119
|
end
|
114
120
|
|
115
|
-
#
|
121
|
+
# Returns the pacts (if they exist) for all the branch heads.
|
122
|
+
# If the version for the branch head does not have a pact, then no pact is returned,
|
123
|
+
# (unlike latest_by_consumer_branch)
|
124
|
+
# This is much more performant than latest_by_consumer_branch and should be used
|
125
|
+
# for the 'pacts for verification' response
|
126
|
+
# @return [Dataset<PactBroker::Pacts::PactPublication>]
|
127
|
+
def for_all_branch_heads
|
128
|
+
base_query = self
|
129
|
+
base_query = base_query.join(:branch_heads, { Sequel[:bh][:version_id] => Sequel[:pact_publications][:consumer_version_id] }, { table_alias: :bh })
|
130
|
+
|
131
|
+
if no_columns_selected?
|
132
|
+
base_query = base_query.select_all_qualified.select_append(Sequel[:bh][:branch_name].as(:branch_name))
|
133
|
+
end
|
134
|
+
|
135
|
+
base_query.remove_overridden_revisions
|
136
|
+
end
|
137
|
+
|
138
|
+
# Return the pacts (if they exist) for the branch heads of the given branch names
|
116
139
|
# This uses the new logic of finding the branch head and returning any associated pacts,
|
117
140
|
# rather than the old logic of returning the pact for the latest version
|
118
141
|
# on the branch that had a pact.
|
142
|
+
# @param [String] branch_name
|
143
|
+
# @return [Sequel::Dataset<PactBroker::Pacts::PactPublication>]
|
119
144
|
def for_branch_heads(branch_name)
|
120
145
|
branch_head_join = {
|
121
146
|
Sequel[:pact_publications][:consumer_version_id] => Sequel[:branch_heads][:version_id],
|
@@ -174,7 +199,10 @@ module PactBroker
|
|
174
199
|
# The latest pact publication for each tag
|
175
200
|
# This uses the old logic of "the latest pact for a version that has a tag" (which always returns a pact)
|
176
201
|
# rather than "the pact for the latest version with a tag"
|
177
|
-
#
|
202
|
+
#
|
203
|
+
# For 'pacts for verification' this has been replaced by for_all_tag_heads
|
204
|
+
# This should only be used for the UI
|
205
|
+
# @return [Sequel::Dataset<PactBroker::Pacts::PactPublication>]
|
178
206
|
def latest_by_consumer_tag
|
179
207
|
tags_join = {
|
180
208
|
Sequel[:pact_publications][:consumer_version_id] => Sequel[:tags][:version_id],
|
@@ -203,6 +231,7 @@ module PactBroker
|
|
203
231
|
# This uses the old logic of "the latest pact for a version that has a tag" (which always returns a pact)
|
204
232
|
# rather than "the pact for the latest version with a tag"
|
205
233
|
# Need to see about updating this.
|
234
|
+
# @return [Sequel::Dataset<PactBroker::Pacts::PactPublication>]
|
206
235
|
def latest_for_consumer_tag(tag_name)
|
207
236
|
tags_join = {
|
208
237
|
Sequel[:pact_publications][:consumer_version_id] => Sequel[:tags][:version_id],
|
@@ -253,6 +282,30 @@ module PactBroker
|
|
253
282
|
.remove_overridden_revisions_from_complete_query
|
254
283
|
end
|
255
284
|
|
285
|
+
# The pacts for the latest versions for each tag.
|
286
|
+
# Will not return a pact if the pact is no longer published for a particular tag
|
287
|
+
# NEW LOGIC
|
288
|
+
# @return [Sequel::Dataset<PactBroker::Pacts::PactPublication>]
|
289
|
+
def for_all_tag_heads
|
290
|
+
head_tags = PactBroker::Domain::Tag
|
291
|
+
.select_group(:pacticipant_id, :name)
|
292
|
+
.select_append{ max(version_order).as(:latest_version_order) }
|
293
|
+
|
294
|
+
head_tags_join = {
|
295
|
+
Sequel[:pact_publications][:consumer_id] => Sequel[:head_tags][:pacticipant_id],
|
296
|
+
Sequel[:pact_publications][:consumer_version_order] => Sequel[:head_tags][:latest_version_order]
|
297
|
+
}
|
298
|
+
|
299
|
+
base_query = self
|
300
|
+
if no_columns_selected?
|
301
|
+
base_query = base_query.select_all_qualified.select_append(Sequel[:head_tags][:name].as(:tag_name))
|
302
|
+
end
|
303
|
+
|
304
|
+
base_query
|
305
|
+
.join(head_tags, head_tags_join, { table_alias: :head_tags })
|
306
|
+
.remove_overridden_revisions_from_complete_query
|
307
|
+
end
|
308
|
+
|
256
309
|
def in_environments
|
257
310
|
currently_deployed_join = {
|
258
311
|
Sequel[:pact_publications][:consumer_version_id] => Sequel[:currently_deployed_version_ids][:version_id]
|
@@ -15,6 +15,7 @@ module PactBroker
|
|
15
15
|
|
16
16
|
# Do the "latest" logic last so that the provider/consumer criteria get included in the "latest" query before the join, rather than after
|
17
17
|
query = query.latest_for_main_branches if selector.latest_for_main_branch?
|
18
|
+
query = query.for_all_branch_heads if selector.latest_for_each_branch?
|
18
19
|
query = query.latest_for_consumer_branch(selector.branch) if selector.latest_for_branch?
|
19
20
|
query = query.for_latest_consumer_versions_with_tag(selector.tag) if selector.latest_for_tag?
|
20
21
|
query = query.overall_latest if selector.overall_latest?
|
@@ -64,7 +64,7 @@ module PactBroker
|
|
64
64
|
provider_tags_names,
|
65
65
|
wip_start_date,
|
66
66
|
explicitly_specified_verifiable_pacts,
|
67
|
-
:
|
67
|
+
:for_all_tag_heads
|
68
68
|
)
|
69
69
|
|
70
70
|
wip_by_consumer_branches = find_wip_pact_versions_for_provider_by_provider_tags(
|
@@ -72,7 +72,7 @@ module PactBroker
|
|
72
72
|
provider_tags_names,
|
73
73
|
wip_start_date,
|
74
74
|
explicitly_specified_verifiable_pacts,
|
75
|
-
:
|
75
|
+
:for_all_branch_heads
|
76
76
|
)
|
77
77
|
|
78
78
|
deduplicate_verifiable_pacts(wip_by_consumer_tags + wip_by_consumer_branches).sort
|
@@ -229,8 +229,8 @@ module PactBroker
|
|
229
229
|
provider = pacticipant_repository.find_by_name(provider_name)
|
230
230
|
wip_start_date = options.fetch(:include_wip_pacts_since)
|
231
231
|
|
232
|
-
potential_wip_by_consumer_branch = PactPublication.for_provider(provider).created_after(wip_start_date).
|
233
|
-
potential_wip_by_consumer_tag = PactPublication.for_provider(provider).created_after(wip_start_date).
|
232
|
+
potential_wip_by_consumer_branch = PactPublication.for_provider(provider).created_after(wip_start_date).for_all_branch_heads
|
233
|
+
potential_wip_by_consumer_tag = PactPublication.for_provider(provider).created_after(wip_start_date).for_all_tag_heads
|
234
234
|
|
235
235
|
log_debug_for_wip do
|
236
236
|
log_pact_publications_from_query("Potential WIP pacts for provider branch #{provider_version_branch} created after #{wip_start_date} by consumer branch", potential_wip_by_consumer_branch)
|
@@ -33,8 +33,8 @@ module PactBroker
|
|
33
33
|
scope
|
34
34
|
end
|
35
35
|
|
36
|
-
|
37
|
-
|
36
|
+
# @return [PactBroker::Domain::Pact]
|
37
|
+
def create(params)
|
38
38
|
pact_version = find_or_create_pact_version(
|
39
39
|
params.fetch(:consumer_id),
|
40
40
|
params.fetch(:provider_id),
|
@@ -31,6 +31,8 @@ module PactBroker
|
|
31
31
|
def type
|
32
32
|
if latest_for_main_branch?
|
33
33
|
:latest_for_main_branch
|
34
|
+
elsif latest_for_each_branch?
|
35
|
+
:latest_for_each_branch
|
34
36
|
elsif latest_for_branch?
|
35
37
|
:latest_for_branch
|
36
38
|
elsif matching_branch?
|
@@ -265,12 +267,16 @@ module PactBroker
|
|
265
267
|
# Not sure if the fallback_tag logic is needed
|
266
268
|
def latest_for_branch? potential_branch = nil
|
267
269
|
if potential_branch
|
268
|
-
|
270
|
+
latest == true && branch == potential_branch
|
269
271
|
else
|
270
|
-
|
272
|
+
latest == true && branch.is_a?(String)
|
271
273
|
end
|
272
274
|
end
|
273
275
|
|
276
|
+
def latest_for_each_branch?
|
277
|
+
latest == true && branch == true
|
278
|
+
end
|
279
|
+
|
274
280
|
def all_for_tag_and_consumer?
|
275
281
|
!!(tag && !latest? && consumer)
|
276
282
|
end
|
@@ -1,3 +1,8 @@
|
|
1
|
+
# This task is used to clean up old data in a Pact Broker database
|
2
|
+
# to stop performance issues from slowing down responses when there is
|
3
|
+
# too much data.
|
4
|
+
# See https://docs.pact.io/pact_broker/administration/maintenance
|
5
|
+
|
1
6
|
module PactBroker
|
2
7
|
module DB
|
3
8
|
class CleanTask < ::Rake::TaskLib
|
@@ -7,11 +12,13 @@ module PactBroker
|
|
7
12
|
attr_accessor :version_deletion_limit
|
8
13
|
attr_accessor :logger
|
9
14
|
attr_accessor :dry_run
|
15
|
+
attr_accessor :use_lock # allow disabling of postgres lock if it is causing problems
|
10
16
|
|
11
17
|
def initialize &block
|
12
18
|
require "pact_broker/db/clean_incremental"
|
13
19
|
@version_deletion_limit = 1000
|
14
20
|
@dry_run = false
|
21
|
+
@use_lock = true
|
15
22
|
@keep_version_selectors = PactBroker::DB::CleanIncremental::DEFAULT_KEEP_SELECTORS
|
16
23
|
rake_task(&block)
|
17
24
|
end
|
@@ -28,42 +35,77 @@ module PactBroker
|
|
28
35
|
namespace :db do
|
29
36
|
desc "Clean unnecessary pacts and verifications from database"
|
30
37
|
task :clean do | _t, _args |
|
31
|
-
|
32
38
|
instance_eval(&block)
|
33
39
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
40
|
+
with_lock do
|
41
|
+
perform_clean
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def perform_clean
|
49
|
+
require "pact_broker/db/clean_incremental"
|
50
|
+
require "pact_broker/error"
|
51
|
+
require "yaml"
|
52
|
+
require "benchmark"
|
38
53
|
|
39
|
-
|
54
|
+
raise PactBroker::Error.new("You must specify the version_deletion_limit") unless version_deletion_limit
|
40
55
|
|
41
|
-
|
56
|
+
if keep_version_selectors.nil? || keep_version_selectors.empty?
|
57
|
+
raise PactBroker::Error.new("You must specify which versions to keep")
|
58
|
+
else
|
59
|
+
add_defaults_to_keep_selectors
|
60
|
+
output "Deleting oldest #{version_deletion_limit} versions, keeping versions that match the configured selectors", keep_version_selectors.collect(&:to_hash)
|
61
|
+
end
|
42
62
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
63
|
+
start_time = Time.now
|
64
|
+
results = PactBroker::DB::CleanIncremental.call(database_connection,
|
65
|
+
keep: keep_version_selectors,
|
66
|
+
limit: version_deletion_limit,
|
67
|
+
logger: logger,
|
68
|
+
dry_run: dry_run
|
69
|
+
)
|
70
|
+
end_time = Time.now
|
71
|
+
elapsed_seconds = (end_time - start_time).to_i
|
72
|
+
output "Results (#{elapsed_seconds} seconds)", results
|
73
|
+
end
|
49
74
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
75
|
+
# Use a Postgres advisory lock to ensure that only one clean can run at a time.
|
76
|
+
# This allows a cron schedule to be used on the Pact Broker Docker image when deployed
|
77
|
+
# on a multi-instance architecture, without all the instances stepping on each other's toes.
|
78
|
+
#
|
79
|
+
# Any tasks that attempt to run while a clean job is running will skip the clean
|
80
|
+
# and exit with a message and a success code.
|
81
|
+
#
|
82
|
+
# To test that the lock works, run:
|
83
|
+
# script/docker/db-start.sh
|
84
|
+
# script/docker/db-migrate.sh
|
85
|
+
# for i in {0..3}; do PACT_BROKER_TEST_DATABASE_URL=postgres://postgres:postgres@localhost/postgres bundle exec rake pact_broker:db:clean &; done;
|
86
|
+
#
|
87
|
+
# There will be 3 messages saying "Clean was not performed" and output from one thread showing the clean is being done.
|
88
|
+
def with_lock
|
89
|
+
if use_lock
|
90
|
+
require "pact_broker/db/advisory_lock"
|
91
|
+
|
92
|
+
lock = PactBroker::DB::AdvisoryLock.new(database_connection, :clean, :pg_try_advisory_lock)
|
93
|
+
results = lock.with_lock do
|
94
|
+
yield
|
95
|
+
end
|
96
|
+
|
97
|
+
if !lock.lock_obtained?
|
98
|
+
output("Clean was not performed as a clean is already in progress. Exiting.")
|
61
99
|
end
|
100
|
+
results
|
101
|
+
else
|
102
|
+
yield
|
62
103
|
end
|
63
104
|
end
|
64
105
|
|
65
|
-
def output
|
66
|
-
|
106
|
+
def output(string, payload = {})
|
107
|
+
prefix = dry_run ? "[DRY RUN] " : ""
|
108
|
+
logger ? logger.info("#{prefix}#{string}") : puts("#{prefix}#{string} #{payload.to_json}")
|
67
109
|
end
|
68
110
|
|
69
111
|
def add_defaults_to_keep_selectors
|
@@ -43,7 +43,6 @@ module PactBroker
|
|
43
43
|
include PactBroker::Services
|
44
44
|
using PactBroker::StringRefinements
|
45
45
|
|
46
|
-
|
47
46
|
attr_reader :pacticipant
|
48
47
|
attr_reader :consumer
|
49
48
|
attr_reader :provider
|
@@ -166,9 +165,16 @@ module PactBroker
|
|
166
165
|
|
167
166
|
def create_pacticipant pacticipant_name, params = {}
|
168
167
|
params.delete(:comment)
|
168
|
+
version_to_create = params.delete(:version)
|
169
|
+
|
169
170
|
repository_url = "https://github.com/#{params[:repository_namespace] || "example-organization"}/#{params[:repository_name] || pacticipant_name}"
|
170
171
|
merged_params = { name: pacticipant_name, repository_url: repository_url }.merge(params)
|
171
172
|
@pacticipant = PactBroker::Domain::Pacticipant.create(merged_params)
|
173
|
+
|
174
|
+
version = create_pacticipant_version(version_to_create, @pacticipant) if version_to_create
|
175
|
+
main_branch = params[:main_branch]
|
176
|
+
PactBroker::Versions::BranchVersionRepository.new.add_branch(version, main_branch) if version && main_branch
|
177
|
+
|
172
178
|
self
|
173
179
|
end
|
174
180
|
|
@@ -192,8 +198,11 @@ module PactBroker
|
|
192
198
|
self
|
193
199
|
end
|
194
200
|
|
201
|
+
# Create an Integration object for the current consumer and provider
|
202
|
+
# @return [PactBroker::Test::TestDataBuilder]
|
195
203
|
def create_integration
|
196
|
-
PactBroker::Integrations::Repository.new.create_for_pact(consumer.id, provider.id)
|
204
|
+
@integration = PactBroker::Integrations::Repository.new.create_for_pact(consumer.id, provider.id)
|
205
|
+
set_created_at_if_set(@now, :integrations, { consumer_id: consumer.id, provider_id: provider.id })
|
197
206
|
self
|
198
207
|
end
|
199
208
|
|
@@ -280,7 +289,9 @@ module PactBroker
|
|
280
289
|
self
|
281
290
|
end
|
282
291
|
|
283
|
-
|
292
|
+
# Creates a pact (and integration if one does not already exist) from the given params
|
293
|
+
# @return [PactBroker::Test::TestDataBuilder]
|
294
|
+
def create_pact(params = {})
|
284
295
|
params.delete(:comment)
|
285
296
|
json_content = params[:json_content] || default_json_content
|
286
297
|
pact_version_sha = params[:pact_version_sha] || generate_pact_version_sha(json_content)
|
@@ -293,6 +304,7 @@ module PactBroker
|
|
293
304
|
json_content: prepare_json_content(json_content),
|
294
305
|
version: @consumer_version
|
295
306
|
)
|
307
|
+
integration_service.handle_bulk_contract_data_published([@pact])
|
296
308
|
pact_versions_count_after = PactBroker::Pacts::PactVersion.count
|
297
309
|
set_created_at_if_set(params[:created_at], :pact_publications, id: @pact.id)
|
298
310
|
set_created_at_if_set(params[:created_at], :pact_versions, sha: @pact.pact_version_sha) if pact_versions_count_after > pact_versions_count_before
|
@@ -634,8 +646,6 @@ module PactBroker
|
|
634
646
|
}.to_json
|
635
647
|
end
|
636
648
|
|
637
|
-
private
|
638
|
-
|
639
649
|
def create_pacticipant_version(version_number, pacticipant, params = {})
|
640
650
|
params.delete(:comment)
|
641
651
|
tag_names = [params.delete(:tag_names), params.delete(:tag_name)].flatten.compact
|
@@ -660,6 +670,8 @@ module PactBroker
|
|
660
670
|
version
|
661
671
|
end
|
662
672
|
|
673
|
+
private
|
674
|
+
|
663
675
|
def create_deployed_version(uuid: , currently_deployed: , version:, environment_name: , target: nil, created_at: nil)
|
664
676
|
env = find_environment(environment_name)
|
665
677
|
@deployed_version = PactBroker::Deployments::DeployedVersionService.find_or_create(uuid, version, env, target)
|
data/lib/pact_broker/version.rb
CHANGED
@@ -57,29 +57,11 @@ module PactBroker
|
|
57
57
|
.single_record
|
58
58
|
end
|
59
59
|
|
60
|
-
|
61
|
-
# Newer "find all" implementations for other models pass the relations to eager load in
|
62
|
-
# from the decorator via the resource.
|
63
|
-
def find_all_pacticipant_versions_in_reverse_order name, pagination_options = {}
|
64
|
-
pacticipant = pacticipant_repository.find_by_name!(name)
|
65
|
-
query = PactBroker::Domain::Version
|
66
|
-
.where(pacticipant: pacticipant)
|
67
|
-
.eager(:pacticipant)
|
68
|
-
.eager(branch_versions: [:version, :branch_head, { branch: :pacticipant }])
|
69
|
-
.eager(tags: :head_tag)
|
70
|
-
.eager(:pact_publications)
|
71
|
-
.reverse_order(:order)
|
72
|
-
query.all_with_pagination_options(pagination_options)
|
73
|
-
end
|
74
|
-
|
75
|
-
def find_pacticipant_versions_in_reverse_order(pacticipant_name, options = {}, pagination_options = {})
|
60
|
+
def find_pacticipant_versions_in_reverse_order(pacticipant_name, options = {}, pagination_options = {}, eager_load_associations = [])
|
76
61
|
pacticipant = pacticipant_repository.find_by_name!(pacticipant_name)
|
77
62
|
query = PactBroker::Domain::Version
|
78
63
|
.where(pacticipant: pacticipant)
|
79
|
-
.eager(
|
80
|
-
.eager(branch_versions: [:version, :branch_head, { branch: :pacticipant }])
|
81
|
-
.eager(tags: :head_tag)
|
82
|
-
.eager(:pact_publications)
|
64
|
+
.eager(*eager_load_associations)
|
83
65
|
.reverse_order(:order)
|
84
66
|
|
85
67
|
if options[:branch_name]
|
@@ -26,12 +26,8 @@ module PactBroker
|
|
26
26
|
version_repository.find_latest_by_pacticipant_name_and_branch_name(pacticipant_name, branch_name)
|
27
27
|
end
|
28
28
|
|
29
|
-
def self.
|
30
|
-
version_repository.
|
31
|
-
end
|
32
|
-
|
33
|
-
def self.find_pacticipant_versions_in_reverse_order(pacticipant_name, options, pagination_options = {})
|
34
|
-
version_repository.find_pacticipant_versions_in_reverse_order(pacticipant_name, options, pagination_options)
|
29
|
+
def self.find_pacticipant_versions_in_reverse_order(pacticipant_name, options, pagination_options = {}, eager_load_associations = [])
|
30
|
+
version_repository.find_pacticipant_versions_in_reverse_order(pacticipant_name, options, pagination_options, eager_load_associations)
|
35
31
|
end
|
36
32
|
|
37
33
|
def self.create_or_overwrite(pacticipant_name, version_number, version)
|
@@ -2,14 +2,14 @@ require "pact_broker/dataset"
|
|
2
2
|
|
3
3
|
module PactBroker
|
4
4
|
module Webhooks
|
5
|
-
class Execution < Sequel::Model(
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
5
|
+
class Execution < Sequel::Model(:webhook_executions)
|
6
|
+
# Ignore the columns that were used before the TriggeredWebhook class existed.
|
7
|
+
# It used to go Webhook -> Execution, and now it goes Webhook -> TriggeredWebhook -> Execution
|
8
|
+
# If we ever release a major version where we drop unused columns, those columns could be deleted.
|
9
|
+
EXECUTION_COLUMNS = Sequel::Model.db.schema(:webhook_executions).collect(&:first) - [:webhook_id, :pact_publication_id, :consumer_id, :provider_id]
|
10
|
+
|
11
|
+
set_dataset(Sequel::Model.db[:webhook_executions].select(*EXECUTION_COLUMNS))
|
12
|
+
|
13
13
|
set_primary_key :id
|
14
14
|
plugin :timestamps
|
15
15
|
|