pact_broker 2.109.1 → 2.111.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -0
- data/docs/developer/design_pattern_for_eager_loading_collections.md +23 -0
- data/docs/developer/matrix.md +95 -0
- data/docs/developer/rack.md +11 -0
- data/lib/pact_broker/api/contracts/consumer_version_selector_contract.rb +1 -1
- data/lib/pact_broker/api/decorators/base_decorator.rb +10 -1
- data/lib/pact_broker/api/decorators/label_decorator.rb +19 -13
- data/lib/pact_broker/api/decorators/labels_decorator.rb +23 -0
- data/lib/pact_broker/api/decorators/pacticipant_decorator.rb +1 -1
- data/lib/pact_broker/api/decorators/version_decorator.rb +14 -0
- data/lib/pact_broker/api/paths.rb +2 -1
- data/lib/pact_broker/api/resources/branch_versions.rb +1 -1
- data/lib/pact_broker/api/resources/can_i_merge_badge.rb +36 -0
- data/lib/pact_broker/api/resources/labels.rb +37 -0
- data/lib/pact_broker/api/resources/provider_pacts_for_verification.rb +5 -0
- data/lib/pact_broker/api/resources/versions.rb +1 -1
- data/lib/pact_broker/api.rb +4 -0
- data/lib/pact_broker/app.rb +24 -9
- data/lib/pact_broker/badges/service.rb +18 -0
- data/lib/pact_broker/contracts/service.rb +6 -2
- data/lib/pact_broker/db/advisory_lock.rb +58 -0
- data/lib/pact_broker/domain/version.rb +3 -2
- data/lib/pact_broker/integrations/integration.rb +11 -1
- data/lib/pact_broker/integrations/repository.rb +46 -7
- data/lib/pact_broker/integrations/service.rb +2 -0
- data/lib/pact_broker/labels/repository.rb +5 -0
- data/lib/pact_broker/labels/service.rb +4 -0
- data/lib/pact_broker/matrix/service.rb +26 -0
- data/lib/pact_broker/pacticipants/repository.rb +10 -1
- data/lib/pact_broker/pacts/pact_publication_dataset_module.rb +56 -3
- data/lib/pact_broker/pacts/pact_publication_selector_dataset_module.rb +1 -0
- data/lib/pact_broker/pacts/pacts_for_verification_repository.rb +4 -4
- data/lib/pact_broker/pacts/repository.rb +2 -2
- data/lib/pact_broker/pacts/selector.rb +8 -2
- data/lib/pact_broker/tasks/clean_task.rb +68 -26
- data/lib/pact_broker/test/test_data_builder.rb +17 -5
- data/lib/pact_broker/version.rb +1 -1
- data/lib/pact_broker/versions/repository.rb +2 -20
- data/lib/pact_broker/versions/service.rb +2 -6
- data/lib/pact_broker/webhooks/execution.rb +8 -8
- data/lib/sequel/extensions/pg_advisory_lock.rb +101 -0
- data/pact_broker.gemspec +1 -1
- metadata +13 -5
@@ -7,7 +7,17 @@ require "pact_broker/verifications/latest_verification_for_consumer_and_provider
|
|
7
7
|
|
8
8
|
module PactBroker
|
9
9
|
module Integrations
|
10
|
-
|
10
|
+
# The columns are explicitly specified for the Integration object so that the consumer_name and provider_name columns aren't included
|
11
|
+
# in the model.
|
12
|
+
# Those columns exist in the integrations table because the integrations table used to be an integrations view based on the
|
13
|
+
# pact_publications table, and those columns existed in the view.
|
14
|
+
# When the view was migrated to be a table (in db/migrations/20211102_create_table_temp_integrations.rb and the following migrations)
|
15
|
+
# the columns had to be maintained for backwards compatiblity.
|
16
|
+
# They are not used by the current code, however.
|
17
|
+
class Integration < Sequel::Model
|
18
|
+
INTEGRATION_COLUMNS = Sequel::Model.db.schema(:integrations).collect(&:first) - [:consumer_name, :provider_name]
|
19
|
+
set_dataset(Sequel::Model.db[:integrations].select(*INTEGRATION_COLUMNS))
|
20
|
+
|
11
21
|
set_primary_key :id
|
12
22
|
plugin :insert_ignore, identifying_columns: [:consumer_id, :provider_id]
|
13
23
|
associate(:many_to_one, :consumer, :class => "PactBroker::Domain::Pacticipant", :key => :consumer_id, :primary_key => :id)
|
@@ -28,6 +28,20 @@ module PactBroker
|
|
28
28
|
nil
|
29
29
|
end
|
30
30
|
|
31
|
+
# Ensure an Integration exists for each consumer/provider pair.
|
32
|
+
# Using SELECT ... INSERT IGNORE rather than just INSERT IGNORE so that we do not
|
33
|
+
# need to lock the table at all when the integrations already exist, which will
|
34
|
+
# be the most common use case. New integrations get created incredibly rarely.
|
35
|
+
# The INSERT IGNORE is used rather than just INSERT to handle race conditions
|
36
|
+
# when requests come in parallel.
|
37
|
+
# @param [Array<Object>] where each object has a consumer and a provider
|
38
|
+
def create_for_pacts(objects_with_consumer_and_provider)
|
39
|
+
published_integrations = objects_with_consumer_and_provider.collect{ |i| { consumer_id: i.consumer.id, provider_id: i.provider.id } }
|
40
|
+
existing_integrations = Sequel::Model.db[:integrations].select(:consumer_id, :provider_id).where(Sequel.|(*published_integrations) ).all
|
41
|
+
new_integrations = (published_integrations - existing_integrations).collect{ |i| i.merge(created_at: Sequel.datetime_class.now, contract_data_updated_at: Sequel.datetime_class.now) }
|
42
|
+
Integration.dataset.insert_ignore.multi_insert(new_integrations)
|
43
|
+
end
|
44
|
+
|
31
45
|
def delete(consumer_id, provider_id)
|
32
46
|
Integration.where(consumer_id: consumer_id, provider_id: provider_id).delete
|
33
47
|
end
|
@@ -36,18 +50,43 @@ module PactBroker
|
|
36
50
|
# @param [PactBroker::Domain::Pacticipant, nil] consumer the consumer for the integration, or nil if for a provider-only event (eg. Pactflow provider contract published)
|
37
51
|
# @param [PactBroker::Domain::Pacticipant] provider the provider for the integration
|
38
52
|
def set_contract_data_updated_at(consumer, provider)
|
39
|
-
|
40
|
-
.where({ consumer_id: consumer&.id, provider_id: provider.id }.compact )
|
41
|
-
.update(contract_data_updated_at: Sequel.datetime_class.now)
|
53
|
+
set_contract_data_updated_at_for_multiple_integrations([OpenStruct.new(consumer: consumer, provider: provider)])
|
42
54
|
end
|
43
55
|
|
44
56
|
|
45
|
-
# Sets the contract_data_updated_at for the integrations as specified by an array of objects which each have a consumer and provider
|
46
|
-
#
|
57
|
+
# Sets the contract_data_updated_at for the integrations as specified by an array of objects which each have a consumer and provider.
|
58
|
+
#
|
59
|
+
# The contract_data_updated_at attribute is only ever used for ordering the list of integrations on the index page of the *Pact Broker* UI,
|
60
|
+
# so that the most recently updated integrations (the ones you're likely working on) are showed at the top of the first page.
|
61
|
+
# There is often contention around updating it however, which can cause deadlocks, and slow down API responses.
|
62
|
+
# Because it's not a critical field (eg. it won't change any can-i-deploy results), the easiest way to reduce this contention
|
63
|
+
# is to just not update it if the row is locked, because if it is locked, the value of contract_data_updated_at is already
|
64
|
+
# going to be a date from a few seconds ago, which is perfectly fine for the purposes for which we are using the value.
|
65
|
+
#
|
66
|
+
# Notes on SKIP LOCKED:
|
67
|
+
# SKIP LOCKED is only supported by Postgres.
|
68
|
+
# When executing SELECT ... FOR UPDATE SKIP LOCKED, the SELECT will run immediately, not waiting for any other transactions,
|
69
|
+
# and only return rows that are not already locked by another transaction.
|
70
|
+
# The FOR UPDATE is required to make it work this way - SKIP LOCKED on its own does not work.
|
71
|
+
#
|
72
|
+
# @param [Array<Object>] where each object MAY have a consumer and does have a provider (for Pactflow provider contract published there is no consumer)
|
47
73
|
def set_contract_data_updated_at_for_multiple_integrations(objects_with_consumer_and_provider)
|
48
|
-
consumer_and_provider_ids = objects_with_consumer_and_provider.collect{ | object |
|
74
|
+
consumer_and_provider_ids = objects_with_consumer_and_provider.collect{ | object | { consumer_id: object.consumer&.id, provider_id: object.provider.id }.compact }.uniq
|
75
|
+
|
76
|
+
# MySQL doesn't support an UPDATE with a subquery. FFS. Really need to do a major version release and delete the support code.
|
77
|
+
criteria = if Integration.dataset.supports_skip_locked?
|
78
|
+
integration_ids_to_update = Integration
|
79
|
+
.select(:id)
|
80
|
+
.where(Sequel.|(*consumer_and_provider_ids))
|
81
|
+
.for_update
|
82
|
+
.skip_locked
|
83
|
+
{ id: integration_ids_to_update }
|
84
|
+
else
|
85
|
+
Sequel.|(*consumer_and_provider_ids)
|
86
|
+
end
|
87
|
+
|
49
88
|
Integration
|
50
|
-
.where(
|
89
|
+
.where(criteria)
|
51
90
|
.update(contract_data_updated_at: Sequel.datetime_class.now)
|
52
91
|
end
|
53
92
|
end
|
@@ -21,6 +21,7 @@ module PactBroker
|
|
21
21
|
# @param [PactBroker::Domain::Pacticipant] consumer or nil
|
22
22
|
# @param [PactBroker::Domain::Pacticipant] provider
|
23
23
|
def self.handle_contract_data_published(consumer, provider)
|
24
|
+
integration_repository.create_for_pact(consumer.id, provider.id)
|
24
25
|
integration_repository.set_contract_data_updated_at(consumer, provider)
|
25
26
|
end
|
26
27
|
|
@@ -28,6 +29,7 @@ module PactBroker
|
|
28
29
|
# Callback to invoke when a batch of contract data is published (eg. the publish contracts endpoint)
|
29
30
|
# @param [Array<Object>] where each object has a consumer and a provider
|
30
31
|
def self.handle_bulk_contract_data_published(objects_with_consumer_and_provider)
|
32
|
+
integration_repository.create_for_pacts(objects_with_consumer_and_provider)
|
31
33
|
integration_repository.set_contract_data_updated_at_for_multiple_integrations(objects_with_consumer_and_provider)
|
32
34
|
end
|
33
35
|
|
@@ -3,6 +3,11 @@ require "pact_broker/domain/label"
|
|
3
3
|
module PactBroker
|
4
4
|
module Labels
|
5
5
|
class Repository
|
6
|
+
|
7
|
+
def get_all_unique_labels pagination_options = {}
|
8
|
+
PactBroker::Domain::Label.distinct.select(:name).all_with_pagination_options(pagination_options)
|
9
|
+
end
|
10
|
+
|
6
11
|
def create args
|
7
12
|
Domain::Label.new(name: args.fetch(:name), pacticipant: args.fetch(:pacticipant)).save
|
8
13
|
end
|
@@ -9,6 +9,10 @@ module PactBroker
|
|
9
9
|
|
10
10
|
extend PactBroker::Repositories
|
11
11
|
|
12
|
+
def get_all_unique_labels pagination_options = {}
|
13
|
+
label_repository.get_all_unique_labels(pagination_options)
|
14
|
+
end
|
15
|
+
|
12
16
|
def create args
|
13
17
|
pacticipant = pacticipant_repository.find_by_name_or_create args.fetch(:pacticipant_name)
|
14
18
|
label_repository.create pacticipant: pacticipant, name: args.fetch(:label_name)
|
@@ -22,6 +22,32 @@ module PactBroker
|
|
22
22
|
QueryResultsWithDeploymentStatusSummary.new(query_results, DeploymentStatusSummary.new(query_results))
|
23
23
|
end
|
24
24
|
|
25
|
+
def can_i_merge(pacticipant_name: nil, pacticipant: nil, latest_main_branch_version: nil)
|
26
|
+
# first we find the pacticipant by name (or use the one passed in) if pacticipant is nil
|
27
|
+
if pacticipant.nil?
|
28
|
+
pacticipant = pacticipant_service.find_pacticipant_by_name(pacticipant_name)
|
29
|
+
raise PactBroker::Error.new("No pacticipant found with name '#{pacticipant_name}'") unless pacticipant
|
30
|
+
else
|
31
|
+
pacticipant_name = pacticipant.name
|
32
|
+
end
|
33
|
+
|
34
|
+
# then we find the latest version from the main branch if not passed in
|
35
|
+
if latest_main_branch_version.nil?
|
36
|
+
latest_main_branch_version = version_service.find_latest_version_from_main_branch(pacticipant)
|
37
|
+
raise PactBroker::Error.new("No main branch version found for pacticipant '#{pacticipant_name}'") unless latest_main_branch_version
|
38
|
+
end
|
39
|
+
|
40
|
+
selectors = PactBroker::Matrix::UnresolvedSelector.from_hash(
|
41
|
+
pacticipant_name: pacticipant_name,
|
42
|
+
pacticipant_version_number: latest_main_branch_version.number
|
43
|
+
)
|
44
|
+
|
45
|
+
options = { main_branch: true, latest: true, latestby: "cvp" }
|
46
|
+
query_results = can_i_deploy([selectors], options)
|
47
|
+
|
48
|
+
query_results.deployable?
|
49
|
+
end
|
50
|
+
|
25
51
|
def find selectors, options = {}
|
26
52
|
logger.info "Querying matrix", selectors: selectors, options: options
|
27
53
|
matrix_repository.find(selectors, options)
|
@@ -102,7 +102,16 @@ module PactBroker
|
|
102
102
|
|
103
103
|
def search_by_name(pacticipant_name)
|
104
104
|
terms = pacticipant_name.split.map { |v| v.gsub("_", "\\_") }
|
105
|
-
|
105
|
+
columns = [:name, :display_name]
|
106
|
+
string_match_query = Sequel.|(
|
107
|
+
*terms.map do |term|
|
108
|
+
Sequel.|(
|
109
|
+
*columns.map do |column|
|
110
|
+
Sequel.ilike(Sequel[:pacticipants][column], "%#{term}%")
|
111
|
+
end
|
112
|
+
)
|
113
|
+
end
|
114
|
+
)
|
106
115
|
scope_for(PactBroker::Domain::Pacticipant).where(string_match_query)
|
107
116
|
end
|
108
117
|
|
@@ -57,7 +57,13 @@ module PactBroker
|
|
57
57
|
end
|
58
58
|
end
|
59
59
|
|
60
|
-
#
|
60
|
+
# Returns the latest pact for each branch, returning a pact for every branch, even if
|
61
|
+
# the most recent version of that branch does not have a pact.
|
62
|
+
# This is different from for_all_branch_heads, which will find the branch head versions,
|
63
|
+
# and return the pacts associated with those versions.
|
64
|
+
# This method should not be used for 'pacts for verification', because it will return
|
65
|
+
# a pact for branches where that integration should no longer exist.
|
66
|
+
# @return [Dataset<PactBroker::Pacts::PactPublication>]
|
61
67
|
def latest_by_consumer_branch
|
62
68
|
branch_versions_join = {
|
63
69
|
Sequel[:pact_publications][:consumer_version_id] => Sequel[:branch_versions][:version_id]
|
@@ -112,10 +118,29 @@ module PactBroker
|
|
112
118
|
.limit(1)
|
113
119
|
end
|
114
120
|
|
115
|
-
#
|
121
|
+
# Returns the pacts (if they exist) for all the branch heads.
|
122
|
+
# If the version for the branch head does not have a pact, then no pact is returned,
|
123
|
+
# (unlike latest_by_consumer_branch)
|
124
|
+
# This is much more performant than latest_by_consumer_branch and should be used
|
125
|
+
# for the 'pacts for verification' response
|
126
|
+
# @return [Dataset<PactBroker::Pacts::PactPublication>]
|
127
|
+
def for_all_branch_heads
|
128
|
+
base_query = self
|
129
|
+
base_query = base_query.join(:branch_heads, { Sequel[:bh][:version_id] => Sequel[:pact_publications][:consumer_version_id] }, { table_alias: :bh })
|
130
|
+
|
131
|
+
if no_columns_selected?
|
132
|
+
base_query = base_query.select_all_qualified.select_append(Sequel[:bh][:branch_name].as(:branch_name))
|
133
|
+
end
|
134
|
+
|
135
|
+
base_query.remove_overridden_revisions
|
136
|
+
end
|
137
|
+
|
138
|
+
# Return the pacts (if they exist) for the branch heads of the given branch names
|
116
139
|
# This uses the new logic of finding the branch head and returning any associated pacts,
|
117
140
|
# rather than the old logic of returning the pact for the latest version
|
118
141
|
# on the branch that had a pact.
|
142
|
+
# @param [String] branch_name
|
143
|
+
# @return [Sequel::Dataset<PactBroker::Pacts::PactPublication>]
|
119
144
|
def for_branch_heads(branch_name)
|
120
145
|
branch_head_join = {
|
121
146
|
Sequel[:pact_publications][:consumer_version_id] => Sequel[:branch_heads][:version_id],
|
@@ -174,7 +199,10 @@ module PactBroker
|
|
174
199
|
# The latest pact publication for each tag
|
175
200
|
# This uses the old logic of "the latest pact for a version that has a tag" (which always returns a pact)
|
176
201
|
# rather than "the pact for the latest version with a tag"
|
177
|
-
#
|
202
|
+
#
|
203
|
+
# For 'pacts for verification' this has been replaced by for_all_tag_heads
|
204
|
+
# This should only be used for the UI
|
205
|
+
# @return [Sequel::Dataset<PactBroker::Pacts::PactPublication>]
|
178
206
|
def latest_by_consumer_tag
|
179
207
|
tags_join = {
|
180
208
|
Sequel[:pact_publications][:consumer_version_id] => Sequel[:tags][:version_id],
|
@@ -203,6 +231,7 @@ module PactBroker
|
|
203
231
|
# This uses the old logic of "the latest pact for a version that has a tag" (which always returns a pact)
|
204
232
|
# rather than "the pact for the latest version with a tag"
|
205
233
|
# Need to see about updating this.
|
234
|
+
# @return [Sequel::Dataset<PactBroker::Pacts::PactPublication>]
|
206
235
|
def latest_for_consumer_tag(tag_name)
|
207
236
|
tags_join = {
|
208
237
|
Sequel[:pact_publications][:consumer_version_id] => Sequel[:tags][:version_id],
|
@@ -253,6 +282,30 @@ module PactBroker
|
|
253
282
|
.remove_overridden_revisions_from_complete_query
|
254
283
|
end
|
255
284
|
|
285
|
+
# The pacts for the latest versions for each tag.
|
286
|
+
# Will not return a pact if the pact is no longer published for a particular tag
|
287
|
+
# NEW LOGIC
|
288
|
+
# @return [Sequel::Dataset<PactBroker::Pacts::PactPublication>]
|
289
|
+
def for_all_tag_heads
|
290
|
+
head_tags = PactBroker::Domain::Tag
|
291
|
+
.select_group(:pacticipant_id, :name)
|
292
|
+
.select_append{ max(version_order).as(:latest_version_order) }
|
293
|
+
|
294
|
+
head_tags_join = {
|
295
|
+
Sequel[:pact_publications][:consumer_id] => Sequel[:head_tags][:pacticipant_id],
|
296
|
+
Sequel[:pact_publications][:consumer_version_order] => Sequel[:head_tags][:latest_version_order]
|
297
|
+
}
|
298
|
+
|
299
|
+
base_query = self
|
300
|
+
if no_columns_selected?
|
301
|
+
base_query = base_query.select_all_qualified.select_append(Sequel[:head_tags][:name].as(:tag_name))
|
302
|
+
end
|
303
|
+
|
304
|
+
base_query
|
305
|
+
.join(head_tags, head_tags_join, { table_alias: :head_tags })
|
306
|
+
.remove_overridden_revisions_from_complete_query
|
307
|
+
end
|
308
|
+
|
256
309
|
def in_environments
|
257
310
|
currently_deployed_join = {
|
258
311
|
Sequel[:pact_publications][:consumer_version_id] => Sequel[:currently_deployed_version_ids][:version_id]
|
@@ -15,6 +15,7 @@ module PactBroker
|
|
15
15
|
|
16
16
|
# Do the "latest" logic last so that the provider/consumer criteria get included in the "latest" query before the join, rather than after
|
17
17
|
query = query.latest_for_main_branches if selector.latest_for_main_branch?
|
18
|
+
query = query.for_all_branch_heads if selector.latest_for_each_branch?
|
18
19
|
query = query.latest_for_consumer_branch(selector.branch) if selector.latest_for_branch?
|
19
20
|
query = query.for_latest_consumer_versions_with_tag(selector.tag) if selector.latest_for_tag?
|
20
21
|
query = query.overall_latest if selector.overall_latest?
|
@@ -64,7 +64,7 @@ module PactBroker
|
|
64
64
|
provider_tags_names,
|
65
65
|
wip_start_date,
|
66
66
|
explicitly_specified_verifiable_pacts,
|
67
|
-
:
|
67
|
+
:for_all_tag_heads
|
68
68
|
)
|
69
69
|
|
70
70
|
wip_by_consumer_branches = find_wip_pact_versions_for_provider_by_provider_tags(
|
@@ -72,7 +72,7 @@ module PactBroker
|
|
72
72
|
provider_tags_names,
|
73
73
|
wip_start_date,
|
74
74
|
explicitly_specified_verifiable_pacts,
|
75
|
-
:
|
75
|
+
:for_all_branch_heads
|
76
76
|
)
|
77
77
|
|
78
78
|
deduplicate_verifiable_pacts(wip_by_consumer_tags + wip_by_consumer_branches).sort
|
@@ -229,8 +229,8 @@ module PactBroker
|
|
229
229
|
provider = pacticipant_repository.find_by_name(provider_name)
|
230
230
|
wip_start_date = options.fetch(:include_wip_pacts_since)
|
231
231
|
|
232
|
-
potential_wip_by_consumer_branch = PactPublication.for_provider(provider).created_after(wip_start_date).
|
233
|
-
potential_wip_by_consumer_tag = PactPublication.for_provider(provider).created_after(wip_start_date).
|
232
|
+
potential_wip_by_consumer_branch = PactPublication.for_provider(provider).created_after(wip_start_date).for_all_branch_heads
|
233
|
+
potential_wip_by_consumer_tag = PactPublication.for_provider(provider).created_after(wip_start_date).for_all_tag_heads
|
234
234
|
|
235
235
|
log_debug_for_wip do
|
236
236
|
log_pact_publications_from_query("Potential WIP pacts for provider branch #{provider_version_branch} created after #{wip_start_date} by consumer branch", potential_wip_by_consumer_branch)
|
@@ -33,8 +33,8 @@ module PactBroker
|
|
33
33
|
scope
|
34
34
|
end
|
35
35
|
|
36
|
-
|
37
|
-
|
36
|
+
# @return [PactBroker::Domain::Pact]
|
37
|
+
def create(params)
|
38
38
|
pact_version = find_or_create_pact_version(
|
39
39
|
params.fetch(:consumer_id),
|
40
40
|
params.fetch(:provider_id),
|
@@ -31,6 +31,8 @@ module PactBroker
|
|
31
31
|
def type
|
32
32
|
if latest_for_main_branch?
|
33
33
|
:latest_for_main_branch
|
34
|
+
elsif latest_for_each_branch?
|
35
|
+
:latest_for_each_branch
|
34
36
|
elsif latest_for_branch?
|
35
37
|
:latest_for_branch
|
36
38
|
elsif matching_branch?
|
@@ -265,12 +267,16 @@ module PactBroker
|
|
265
267
|
# Not sure if the fallback_tag logic is needed
|
266
268
|
def latest_for_branch? potential_branch = nil
|
267
269
|
if potential_branch
|
268
|
-
|
270
|
+
latest == true && branch == potential_branch
|
269
271
|
else
|
270
|
-
|
272
|
+
latest == true && branch.is_a?(String)
|
271
273
|
end
|
272
274
|
end
|
273
275
|
|
276
|
+
def latest_for_each_branch?
|
277
|
+
latest == true && branch == true
|
278
|
+
end
|
279
|
+
|
274
280
|
def all_for_tag_and_consumer?
|
275
281
|
!!(tag && !latest? && consumer)
|
276
282
|
end
|
@@ -1,3 +1,8 @@
|
|
1
|
+
# This task is used to clean up old data in a Pact Broker database
|
2
|
+
# to stop performance issues from slowing down responses when there is
|
3
|
+
# too much data.
|
4
|
+
# See https://docs.pact.io/pact_broker/administration/maintenance
|
5
|
+
|
1
6
|
module PactBroker
|
2
7
|
module DB
|
3
8
|
class CleanTask < ::Rake::TaskLib
|
@@ -7,11 +12,13 @@ module PactBroker
|
|
7
12
|
attr_accessor :version_deletion_limit
|
8
13
|
attr_accessor :logger
|
9
14
|
attr_accessor :dry_run
|
15
|
+
attr_accessor :use_lock # allow disabling of postgres lock if it is causing problems
|
10
16
|
|
11
17
|
def initialize &block
|
12
18
|
require "pact_broker/db/clean_incremental"
|
13
19
|
@version_deletion_limit = 1000
|
14
20
|
@dry_run = false
|
21
|
+
@use_lock = true
|
15
22
|
@keep_version_selectors = PactBroker::DB::CleanIncremental::DEFAULT_KEEP_SELECTORS
|
16
23
|
rake_task(&block)
|
17
24
|
end
|
@@ -28,42 +35,77 @@ module PactBroker
|
|
28
35
|
namespace :db do
|
29
36
|
desc "Clean unnecessary pacts and verifications from database"
|
30
37
|
task :clean do | _t, _args |
|
31
|
-
|
32
38
|
instance_eval(&block)
|
33
39
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
40
|
+
with_lock do
|
41
|
+
perform_clean
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def perform_clean
|
49
|
+
require "pact_broker/db/clean_incremental"
|
50
|
+
require "pact_broker/error"
|
51
|
+
require "yaml"
|
52
|
+
require "benchmark"
|
38
53
|
|
39
|
-
|
54
|
+
raise PactBroker::Error.new("You must specify the version_deletion_limit") unless version_deletion_limit
|
40
55
|
|
41
|
-
|
56
|
+
if keep_version_selectors.nil? || keep_version_selectors.empty?
|
57
|
+
raise PactBroker::Error.new("You must specify which versions to keep")
|
58
|
+
else
|
59
|
+
add_defaults_to_keep_selectors
|
60
|
+
output "Deleting oldest #{version_deletion_limit} versions, keeping versions that match the configured selectors", keep_version_selectors.collect(&:to_hash)
|
61
|
+
end
|
42
62
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
63
|
+
start_time = Time.now
|
64
|
+
results = PactBroker::DB::CleanIncremental.call(database_connection,
|
65
|
+
keep: keep_version_selectors,
|
66
|
+
limit: version_deletion_limit,
|
67
|
+
logger: logger,
|
68
|
+
dry_run: dry_run
|
69
|
+
)
|
70
|
+
end_time = Time.now
|
71
|
+
elapsed_seconds = (end_time - start_time).to_i
|
72
|
+
output "Results (#{elapsed_seconds} seconds)", results
|
73
|
+
end
|
49
74
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
75
|
+
# Use a Postgres advisory lock to ensure that only one clean can run at a time.
|
76
|
+
# This allows a cron schedule to be used on the Pact Broker Docker image when deployed
|
77
|
+
# on a multi-instance architecture, without all the instances stepping on each other's toes.
|
78
|
+
#
|
79
|
+
# Any tasks that attempt to run while a clean job is running will skip the clean
|
80
|
+
# and exit with a message and a success code.
|
81
|
+
#
|
82
|
+
# To test that the lock works, run:
|
83
|
+
# script/docker/db-start.sh
|
84
|
+
# script/docker/db-migrate.sh
|
85
|
+
# for i in {0..3}; do PACT_BROKER_TEST_DATABASE_URL=postgres://postgres:postgres@localhost/postgres bundle exec rake pact_broker:db:clean &; done;
|
86
|
+
#
|
87
|
+
# There will be 3 messages saying "Clean was not performed" and output from one thread showing the clean is being done.
|
88
|
+
def with_lock
|
89
|
+
if use_lock
|
90
|
+
require "pact_broker/db/advisory_lock"
|
91
|
+
|
92
|
+
lock = PactBroker::DB::AdvisoryLock.new(database_connection, :clean, :pg_try_advisory_lock)
|
93
|
+
results = lock.with_lock do
|
94
|
+
yield
|
95
|
+
end
|
96
|
+
|
97
|
+
if !lock.lock_obtained?
|
98
|
+
output("Clean was not performed as a clean is already in progress. Exiting.")
|
61
99
|
end
|
100
|
+
results
|
101
|
+
else
|
102
|
+
yield
|
62
103
|
end
|
63
104
|
end
|
64
105
|
|
65
|
-
def output
|
66
|
-
|
106
|
+
def output(string, payload = {})
|
107
|
+
prefix = dry_run ? "[DRY RUN] " : ""
|
108
|
+
logger ? logger.info("#{prefix}#{string}") : puts("#{prefix}#{string} #{payload.to_json}")
|
67
109
|
end
|
68
110
|
|
69
111
|
def add_defaults_to_keep_selectors
|
@@ -43,7 +43,6 @@ module PactBroker
|
|
43
43
|
include PactBroker::Services
|
44
44
|
using PactBroker::StringRefinements
|
45
45
|
|
46
|
-
|
47
46
|
attr_reader :pacticipant
|
48
47
|
attr_reader :consumer
|
49
48
|
attr_reader :provider
|
@@ -166,9 +165,16 @@ module PactBroker
|
|
166
165
|
|
167
166
|
def create_pacticipant pacticipant_name, params = {}
|
168
167
|
params.delete(:comment)
|
168
|
+
version_to_create = params.delete(:version)
|
169
|
+
|
169
170
|
repository_url = "https://github.com/#{params[:repository_namespace] || "example-organization"}/#{params[:repository_name] || pacticipant_name}"
|
170
171
|
merged_params = { name: pacticipant_name, repository_url: repository_url }.merge(params)
|
171
172
|
@pacticipant = PactBroker::Domain::Pacticipant.create(merged_params)
|
173
|
+
|
174
|
+
version = create_pacticipant_version(version_to_create, @pacticipant) if version_to_create
|
175
|
+
main_branch = params[:main_branch]
|
176
|
+
PactBroker::Versions::BranchVersionRepository.new.add_branch(version, main_branch) if version && main_branch
|
177
|
+
|
172
178
|
self
|
173
179
|
end
|
174
180
|
|
@@ -192,8 +198,11 @@ module PactBroker
|
|
192
198
|
self
|
193
199
|
end
|
194
200
|
|
201
|
+
# Create an Integration object for the current consumer and provider
|
202
|
+
# @return [PactBroker::Test::TestDataBuilder]
|
195
203
|
def create_integration
|
196
|
-
PactBroker::Integrations::Repository.new.create_for_pact(consumer.id, provider.id)
|
204
|
+
@integration = PactBroker::Integrations::Repository.new.create_for_pact(consumer.id, provider.id)
|
205
|
+
set_created_at_if_set(@now, :integrations, { consumer_id: consumer.id, provider_id: provider.id })
|
197
206
|
self
|
198
207
|
end
|
199
208
|
|
@@ -280,7 +289,9 @@ module PactBroker
|
|
280
289
|
self
|
281
290
|
end
|
282
291
|
|
283
|
-
|
292
|
+
# Creates a pact (and integration if one does not already exist) from the given params
|
293
|
+
# @return [PactBroker::Test::TestDataBuilder]
|
294
|
+
def create_pact(params = {})
|
284
295
|
params.delete(:comment)
|
285
296
|
json_content = params[:json_content] || default_json_content
|
286
297
|
pact_version_sha = params[:pact_version_sha] || generate_pact_version_sha(json_content)
|
@@ -293,6 +304,7 @@ module PactBroker
|
|
293
304
|
json_content: prepare_json_content(json_content),
|
294
305
|
version: @consumer_version
|
295
306
|
)
|
307
|
+
integration_service.handle_bulk_contract_data_published([@pact])
|
296
308
|
pact_versions_count_after = PactBroker::Pacts::PactVersion.count
|
297
309
|
set_created_at_if_set(params[:created_at], :pact_publications, id: @pact.id)
|
298
310
|
set_created_at_if_set(params[:created_at], :pact_versions, sha: @pact.pact_version_sha) if pact_versions_count_after > pact_versions_count_before
|
@@ -634,8 +646,6 @@ module PactBroker
|
|
634
646
|
}.to_json
|
635
647
|
end
|
636
648
|
|
637
|
-
private
|
638
|
-
|
639
649
|
def create_pacticipant_version(version_number, pacticipant, params = {})
|
640
650
|
params.delete(:comment)
|
641
651
|
tag_names = [params.delete(:tag_names), params.delete(:tag_name)].flatten.compact
|
@@ -660,6 +670,8 @@ module PactBroker
|
|
660
670
|
version
|
661
671
|
end
|
662
672
|
|
673
|
+
private
|
674
|
+
|
663
675
|
def create_deployed_version(uuid: , currently_deployed: , version:, environment_name: , target: nil, created_at: nil)
|
664
676
|
env = find_environment(environment_name)
|
665
677
|
@deployed_version = PactBroker::Deployments::DeployedVersionService.find_or_create(uuid, version, env, target)
|
data/lib/pact_broker/version.rb
CHANGED
@@ -57,29 +57,11 @@ module PactBroker
|
|
57
57
|
.single_record
|
58
58
|
end
|
59
59
|
|
60
|
-
|
61
|
-
# Newer "find all" implementations for other models pass the relations to eager load in
|
62
|
-
# from the decorator via the resource.
|
63
|
-
def find_all_pacticipant_versions_in_reverse_order name, pagination_options = {}
|
64
|
-
pacticipant = pacticipant_repository.find_by_name!(name)
|
65
|
-
query = PactBroker::Domain::Version
|
66
|
-
.where(pacticipant: pacticipant)
|
67
|
-
.eager(:pacticipant)
|
68
|
-
.eager(branch_versions: [:version, :branch_head, { branch: :pacticipant }])
|
69
|
-
.eager(tags: :head_tag)
|
70
|
-
.eager(:pact_publications)
|
71
|
-
.reverse_order(:order)
|
72
|
-
query.all_with_pagination_options(pagination_options)
|
73
|
-
end
|
74
|
-
|
75
|
-
def find_pacticipant_versions_in_reverse_order(pacticipant_name, options = {}, pagination_options = {})
|
60
|
+
def find_pacticipant_versions_in_reverse_order(pacticipant_name, options = {}, pagination_options = {}, eager_load_associations = [])
|
76
61
|
pacticipant = pacticipant_repository.find_by_name!(pacticipant_name)
|
77
62
|
query = PactBroker::Domain::Version
|
78
63
|
.where(pacticipant: pacticipant)
|
79
|
-
.eager(
|
80
|
-
.eager(branch_versions: [:version, :branch_head, { branch: :pacticipant }])
|
81
|
-
.eager(tags: :head_tag)
|
82
|
-
.eager(:pact_publications)
|
64
|
+
.eager(*eager_load_associations)
|
83
65
|
.reverse_order(:order)
|
84
66
|
|
85
67
|
if options[:branch_name]
|
@@ -26,12 +26,8 @@ module PactBroker
|
|
26
26
|
version_repository.find_latest_by_pacticipant_name_and_branch_name(pacticipant_name, branch_name)
|
27
27
|
end
|
28
28
|
|
29
|
-
def self.
|
30
|
-
version_repository.
|
31
|
-
end
|
32
|
-
|
33
|
-
def self.find_pacticipant_versions_in_reverse_order(pacticipant_name, options, pagination_options = {})
|
34
|
-
version_repository.find_pacticipant_versions_in_reverse_order(pacticipant_name, options, pagination_options)
|
29
|
+
def self.find_pacticipant_versions_in_reverse_order(pacticipant_name, options, pagination_options = {}, eager_load_associations = [])
|
30
|
+
version_repository.find_pacticipant_versions_in_reverse_order(pacticipant_name, options, pagination_options, eager_load_associations)
|
35
31
|
end
|
36
32
|
|
37
33
|
def self.create_or_overwrite(pacticipant_name, version_number, version)
|
@@ -2,14 +2,14 @@ require "pact_broker/dataset"
|
|
2
2
|
|
3
3
|
module PactBroker
|
4
4
|
module Webhooks
|
5
|
-
class Execution < Sequel::Model(
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
5
|
+
class Execution < Sequel::Model(:webhook_executions)
|
6
|
+
# Ignore the columns that were used before the TriggeredWebhook class existed.
|
7
|
+
# It used to go Webhook -> Execution, and now it goes Webhook -> TriggeredWebhook -> Execution
|
8
|
+
# If we ever release a major version where we drop unused columns, those columns could be deleted.
|
9
|
+
EXECUTION_COLUMNS = Sequel::Model.db.schema(:webhook_executions).collect(&:first) - [:webhook_id, :pact_publication_id, :consumer_id, :provider_id]
|
10
|
+
|
11
|
+
set_dataset(Sequel::Model.db[:webhook_executions].select(*EXECUTION_COLUMNS))
|
12
|
+
|
13
13
|
set_primary_key :id
|
14
14
|
plugin :timestamps
|
15
15
|
|