pact_broker 2.71.0 → 2.72.0

data/lib/pact_broker/db/delete_overwritten_data.rb

@@ -11,65 +11,93 @@ module PactBroker
       def initialize database_connection, options = {}
         @db = database_connection
         @options = options
-        @before = options[:before] || DateTime.now
+        @cut_off_date = options[:max_age] ? (DateTime.now - options[:max_age]) : DateTime.now
+        @limit = options[:limit] || 1000
       end
 
       def call
+        require 'pact_broker/pacts/pact_publication'
+        require 'pact_broker/domain/verification'
+
         deleted_counts = {}
         kept_counts = {}
 
-
         deleted_counts.merge!(delete_overwritten_pact_publications)
         deleted_counts.merge!(delete_overwritten_verifications)
         deleted_counts.merge!(delete_orphan_pact_versions)
+        deleted_counts.merge!(delete_webhook_data)
 
         kept_counts[:pact_publications] = db[:pact_publications].count
         kept_counts[:verification_results] = db[:verifications].count
         kept_counts[:pact_versions] = db[:pact_versions].count
+        kept_counts[:triggered_webhooks] = db[:triggered_webhooks].count
 
-
-        { deleted: deleted_counts, kept: kept_counts }
+        if dry_run?
+          to_keep = deleted_counts.keys.each_with_object({}) do | table_name, new_counts |
+            new_counts[table_name] = kept_counts[table_name] - deleted_counts[table_name]
+          end
+          { toDelete: deleted_counts, toKeep: to_keep }
+        else
+          { deleted: deleted_counts, kept: kept_counts }
+        end
       end
 
       private
 
-      attr_reader :db, :options, :before
+      attr_reader :db, :options, :cut_off_date, :limit
+
+      def dry_run?
+        options[:dry_run]
+      end
+
+      def delete_webhook_data
+        ids_to_keep = db[:latest_triggered_webhooks].select(:id)
+        resolved_ids_to_delete = db[:triggered_webhooks]
+          .where(id: ids_to_keep)
+          .invert
+          .where(Sequel.lit('created_at < ?', cut_off_date))
+          .limit(limit)
+          .collect{ |row| row[:id] }
 
-      def delete_webhook_data(triggered_webhook_ids)
-        db[:webhook_executions].where(triggered_webhook_id: triggered_webhook_ids).delete
-        db[:triggered_webhooks].where(id: triggered_webhook_ids).delete
+        PactBroker::Webhooks::TriggeredWebhook.where(id: resolved_ids_to_delete).delete unless dry_run?
+        { triggered_webhooks: resolved_ids_to_delete.count }
       end
 
       def delete_orphan_pact_versions
         referenced_pact_version_ids = db[:pact_publications].select(:pact_version_id).union(db[:verifications].select(:pact_version_id))
-        pact_version_ids_to_delete = db[:pact_versions].where(id: referenced_pact_version_ids).invert
-        deleted_counts = { pact_versions: pact_version_ids_to_delete.count }
-        pact_version_ids_to_delete.delete
-        deleted_counts
+        pact_version_ids_to_delete = db[:pact_versions].where(id: referenced_pact_version_ids).invert.order(:id).limit(limit).collect{ |row| row[:id] }
+        db[:pact_versions].where(id: pact_version_ids_to_delete).delete unless dry_run?
+        { pact_versions: pact_version_ids_to_delete.count }
       end
 
       def delete_overwritten_pact_publications
-        pact_publication_ids_to_delete = db[:pact_publications]
-          .select(:id)
-          .where(id: db[:latest_pact_publication_ids_for_consumer_versions].select(:pact_publication_id))
+        ids_to_keep = db[:latest_pact_publication_ids_for_consumer_versions].select(:pact_publication_id)
+
+        resolved_ids_to_delete = db[:pact_publications]
+          .where(id: ids_to_keep)
           .invert
-          .where(Sequel.lit('created_at < ?', before))
+          .where(Sequel.lit('created_at < ?', cut_off_date))
+          .order(:id)
+          .limit(limit)
+          .collect{ |row| row[:id] }
 
-        deleted_counts = { pact_publications: pact_publication_ids_to_delete.count }
-        delete_webhook_data(db[:triggered_webhooks].where(pact_publication_id: pact_publication_ids_to_delete).select(:id))
-        pact_publication_ids_to_delete.delete
-        deleted_counts
+        PactBroker::Pacts::PactPublication.where(id: resolved_ids_to_delete).delete unless dry_run?
+
+        { pact_publications: resolved_ids_to_delete.count }
       end
 
       def delete_overwritten_verifications
-        verification_ids = db[:verifications].select(:id)
-          .where(id: db[:latest_verification_id_for_pact_version_and_provider_version].select(:verification_id))
+        ids_to_keep = db[:latest_verification_id_for_pact_version_and_provider_version].select(:verification_id)
+        resolved_ids_to_delete = db[:verifications]
+          .where(id: ids_to_keep)
           .invert
-          .where(Sequel.lit('created_at < ?', before))
-        deleted_counts = { verification_results: verification_ids.count }
-        delete_webhook_data(db[:triggered_webhooks].where(verification_id: verification_ids).select(:id))
-        verification_ids.delete
-        deleted_counts
+          .where(Sequel.lit('created_at < ?', cut_off_date))
+          .order(:id)
+          .limit(limit)
+          .collect{ |row| row[:id] }
+
+        PactBroker::Domain::Verification.where(id: resolved_ids_to_delete).delete unless dry_run?
+        { verification_results: resolved_ids_to_delete.count }
       end
     end
   end

data/lib/pact_broker/domain/tag.rb

@@ -13,6 +13,48 @@ module PactBroker
       dataset_module do
         include PactBroker::Repositories::Helpers
 
+        def latest_tags
+          tags_versions_join = {
+            Sequel[:tags][:version_id] => Sequel[:versions][:id],
+          }
+
+          latest_tags_versions_join = {
+            Sequel[:latest_tags][:name] => Sequel[:tags][:name],
+            Sequel[:latest_tags][:latest_order] => Sequel[:versions][:order],
+            Sequel[:latest_tags][:pacticipant_id] => Sequel[:versions][:pacticipant_id],
+          }
+
+          latest_tags = PactBroker::Domain::Tag
+            .select_group(Sequel[:tags][:name], Sequel[:versions][:pacticipant_id])
+            .select_append{ max(order).as(latest_order) }
+            .join(:versions, tags_versions_join)
+
+          PactBroker::Domain::Tag
+            .select_all_qualified
+            .join(:versions,
+              { Sequel[:tags][:version_id] => Sequel[:versions][:id] }
+            )
+            .join(latest_tags, latest_tags_versions_join, { table_alias: :latest_tags })
+        end
+
+        # Ron's fancy join
+        # performs every so slightly better
+        def latest_tags_2
+          tag_versions = PactBroker::Domain::Tag
+            .select_all_qualified
+            .select_append(Sequel[:versions][:pacticipant_id])
+            .select_append(Sequel[:versions][:order])
+            .join(:versions,
+              { Sequel[:tags][:version_id] => Sequel[:versions][:id] }
+            )
+
+          tag_versions
+            .left_join(tag_versions, { Sequel[:tags][:name] =>  Sequel[:tags_2][:name], Sequel[:versions][:pacticipant_id] => Sequel[:tags_2][:pacticipant_id] }, { table_alias: :tags_2 }) do | table, joined_table, js |
+              Sequel.qualify(table, :order) > Sequel.qualify(joined_table, :order)
+            end
+            .where(Sequel[:tags_2][:name] => nil)
+        end
+
         # Does NOT care about whether or not there is a pact publication
         # for the version
         def latest_tags_for_pacticipant_ids(pacticipant_ids)

data/lib/pact_broker/domain/verification.rb

@@ -3,11 +3,13 @@ require 'sequel'
 require 'pact_broker/repositories/helpers'
 require 'pact_broker/tags/tag_with_latest_flag'
 require 'pact_broker/pacts/content'
+require 'sequel/extensions/symbol_aref_refinement'
 
 
 module PactBroker
   module Domain
     class Verification < Sequel::Model
+      using Sequel::SymbolAref
 
       set_primary_key :id
       associate(:many_to_one, :pact_version, class: "PactBroker::Pacts::PactVersion", key: :pact_version_id, primary_key: :id)
@@ -25,6 +27,82 @@ module PactBroker
       dataset_module do
         include PactBroker::Repositories::Helpers
 
+        def latest_verification_ids_for_all_consumer_version_tags
+          verif_pact_join = { Sequel[:verifications][:pact_version_id] => Sequel[:lpp][:pact_version_id] }
+          tag_join = { Sequel[:lpp][:consumer_version_id] => Sequel[:cvt][:version_id] }
+          verisons_join = { Sequel[:v][:provider_version_id] => Sequel[:pv][:id] }
+
+          db[:verifications]
+            .select_group(
+              Sequel[:pv][:pacticipant_id].as(:provider_id),
+              Sequel[:lpp][:consumer_id],
+              Sequel[:cvt][:name].as(:consumer_version_tag_name)
+            )
+            .select_append{ max(verifications[id]).as(latest_verification_id) }
+            .join(:latest_pact_publication_ids_for_consumer_versions, verif_pact_join, { table_alias: :lpp } )
+            .join(:tags, tag_join, { table_alias: :cvt })
+            .join(:versions, verisons_join, { table_alias: :pv })
+        end
+
+        # Do not use this query. It performs worse than the view.
+        # Keeping for posterity
+        def latest_verifications_for_all_consumer_version_tags
+          verif_pact_join = { Sequel[:v][:pact_version_id] => Sequel[:lpp][:pact_version_id] }
+          tag_join = { Sequel[:lpp][:consumer_version_id] => Sequel[:cvt][:version_id] }
+          verisons_join = { Sequel[:v][:provider_version_id] => Sequel[:pv][:id] }
+
+          base_query = db[Sequel.as(:latest_verification_id_for_pact_version_and_provider_version, :v)]
+            .select(:v[:verification_id], :pv[:pacticipant_id].as(:provider_id), :lpp[:consumer_id], :cvt[:name].as(:consumer_version_tag_name))
+            .join(:latest_pact_publication_ids_for_consumer_versions, verif_pact_join, { table_alias: :lpp } )
+            .join(:tags, tag_join, { table_alias: :cvt })
+            .join(:versions, verisons_join, { table_alias: :pv })
+
+
+          base_join = {
+            :pv[:pacticipant_id] => :v2[:provider_id],
+            :lpp[:consumer_id] => :v2[:consumer_id],
+            :cvt[:name] => :v2[:consumer_version_tag_name]
+          }
+
+          thing = base_query
+            .left_join(base_query, base_join, { table_alias: :v2 }) do | table, joined_table, something |
+              :v2[:verification_id] > :v[:verification_id]
+            end.where(:v2[:verification_id] => nil)
+
+          where(id: thing.from_self.select(:verification_id))
+        end
+
+        def latest_verification_ids_for_consumer_version_tags(consumer_ids, consumer_version_tag_names)
+          pact_join = { :verifications[:pact_version_id] => :lpp[:pact_version_id], :lpp[:consumer_id] => consumer_ids }
+          tag_join = { :lpp[:consumer_version_id] => :cvt[:version_id], :cvt[:name] => consumer_version_tag_names }
+          provider_versions_join = { :verifications[:provider_version_id] => :pv[:id] }
+
+          db[Sequel.as(:latest_verification_id_for_pact_version_and_provider_version, :verifications)]
+            .select_group(
+              :pv[:pacticipant_id].as(:provider_id),
+              :lpp[:consumer_id],
+              :cvt[:name].as(:consumer_version_tag_name)
+            )
+            .select_append{ max(verifications[verification_id]).as(latest_verification_id) }
+            .join(:latest_pact_publication_ids_for_consumer_versions, pact_join, { table_alias: :lpp } )
+            .join(:tags, tag_join, { table_alias: :cvt })
+            .join(:versions, provider_versions_join, { table_alias: :pv })
+            .where(:verifications[:consumer_id] => consumer_ids)
+        end
+
+        # Do not use this query. It performs worse than the view.
+        # Keeping for posterity
+        def latest_verifications_for_consumer_version_tags(consumer_ids, consumer_version_tag_names)
+          latest_ids_for_cv_tags = latest_verification_ids_for_consumer_version_tags(consumer_ids, consumer_version_tag_names)
+          join_cols = {
+            Sequel[:verifications][:id] => Sequel[:t2][:latest_verification_id]
+          }
+          select_all_qualified
+            .select_append(Sequel[:t2][:consumer_version_tag_name])
+            .where(Sequel[:verifications][:consumer_id] => consumer_ids)
+            .join(latest_ids_for_cv_tags, join_cols, { table_alias: :t2 })
+        end
+
         # Expects to be joined with AllPactPublications or subclass
         # Beware that when columns with the same name exist in both datasets
         # you may get the wrong column back in your model.
@@ -102,6 +180,15 @@ module PactBroker
       def pact_content_with_test_results
         @pact_content_with_test_results = PactBroker::Pacts::Content.from_json(pact_version.content).with_test_results(test_results)
       end
+
+      # So consumer_version_tag_name can be accessed by method name
+      def method_missing(m, *args, &block)
+        if values.key?(m) && args.size == 0
+          values[m]
+        else
+          super
+        end
+      end
     end
 
     Verification.plugin :timestamps

data/lib/pact_broker/metrics/service.rb

@@ -54,6 +54,11 @@ module PactBroker
           webhooks: {
             count: PactBroker::Webhooks::Webhook.count
           },
+          tags: {
+            count: PactBroker::Domain::Tag.count,
+            distinctCount: PactBroker::Domain::Tag.select(:name).distinct.count,
+            distinctWithPacticipantCount: PactBroker::Domain::Tag.join(:versions, { id: :version_id }).select_group(:name, :pacticipant_id).count
+          },
           triggeredWebhooks: {
             count: PactBroker::Webhooks::TriggeredWebhook.count
           },
@@ -62,9 +67,6 @@ module PactBroker
           },
           matrix: {
             count: PactBroker::Matrix::Row.count
-          },
-          headMatrix: {
-            count: PactBroker::Matrix::HeadRow.count
           }
         }
       end

data/lib/pact_broker/pacts/all_pact_publications.rb

@@ -54,6 +54,14 @@ module PactBroker
           where(name_like(:consumer_version_number, number))
         end
 
+        def maybe_consumer_version_number number
+          if number
+            where(name_like(:consumer_version_number, number))
+          else
+            self
+          end
+        end
+
         def revision_number number
           where(revision_number: number)
         end

data/lib/pact_broker/pacts/repository.rb

@@ -3,6 +3,7 @@ require 'ostruct'
 require 'pact_broker/logging'
 require 'pact_broker/pacts/generate_sha'
 require 'pact_broker/pacts/pact_publication'
+require 'pact_broker/pacts/pact_version'
 require 'pact_broker/pacts/all_pact_publications'
 require 'pact_broker/pacts/latest_pact_publications_by_consumer_version'
 require 'pact_broker/pacts/latest_pact_publications'
@@ -295,20 +296,43 @@ module PactBroker
       end
 
       def find_pact consumer_name, consumer_version, provider_name, pact_version_sha = nil
-        query = if pact_version_sha
-          scope_for(AllPactPublications)
+        pact_publication_by_consumer_version = scope_for(LatestPactPublicationsByConsumerVersion)
+            .consumer(consumer_name)
+            .provider(provider_name)
+            .maybe_consumer_version_number(consumer_version)
+            .limit(1)
+
+        latest_pact_publication_by_sha = scope_for(AllPactPublications)
+            .consumer(consumer_name)
+            .provider(provider_name)
             .pact_version_sha(pact_version_sha)
-            .reverse_order(:consumer_version_order)
+            .reverse_order(:consumer_version_order, :revision_number)
             .limit(1)
+
+        query = if consumer_version && !pact_version_sha
+          pact_publication_by_consumer_version
+            .eager(:tags)
+            .collect(&:to_domain_with_content).first
+        elsif pact_version_sha && !consumer_version
+          latest_pact_publication_by_sha
+            .eager(:tags)
+            .collect(&:to_domain_with_content).first
+        elsif consumer_version && pact_version_sha
+          pact_publication = pact_publication_by_consumer_version.all.first
+          if pact_publication && pact_publication.pact_version.sha == pact_version_sha
+            pact_publication.tags
+            pact_publication.to_domain_with_content
+          else
+            latest_pact_publication_by_sha
+              .eager(:tags)
+              .collect(&:to_domain_with_content).first
+          end
         else
-          scope_for(LatestPactPublicationsByConsumerVersion)
+          pact_publication_by_consumer_version
+            .eager(:tags)
+            .reverse_order(:consumer_version_order, :revision_number)
+            .collect(&:to_domain_with_content).first
         end
-        query = query
-          .eager(:tags)
-          .consumer(consumer_name)
-          .provider(provider_name)
-        query = query.consumer_version_number(consumer_version) if consumer_version
-        query.collect(&:to_domain_with_content)[0]
       end
 
       def find_all_revisions consumer_name, consumer_version, provider_name
@@ -502,7 +526,7 @@ module PactBroker
       end
 
       def create_pact_version consumer_id, provider_id, sha, json_content
-        PactVersion.new(
+        PactBroker::Pacts::PactVersion.new(
           consumer_id: consumer_id,
           provider_id: provider_id,
           sha: sha,

data/lib/pact_broker/tasks/clean_task.rb

@@ -38,15 +38,21 @@ module PactBroker
 
               raise PactBroker::Error.new("You must specify the version_deletion_limit") unless version_deletion_limit
 
+              prefix = dry_run ? "[DRY RUN] " : ""
+
               if keep_version_selectors.nil? || keep_version_selectors.empty?
                 raise PactBroker::Error.new("You must specify which versions to keep")
               else
-                output "Deleting oldest #{version_deletion_limit} versions, keeping versions that match the configured selectors", keep_version_selectors
+                output "#{prefix}Deleting oldest #{version_deletion_limit} versions, keeping versions that match the configured selectors", keep_version_selectors
               end
 
               start_time = Time.now
-              results = PactBroker::DB::CleanIncremental.call(
-                database_connection, keep: keep_version_selectors, limit: version_deletion_limit, logger: logger, dry_run: dry_run)
+              results = PactBroker::DB::CleanIncremental.call(database_connection,
+                keep: keep_version_selectors,
+                limit: version_deletion_limit,
+                logger: logger,
+                dry_run: dry_run
+              )
               end_time = Time.now
               elapsed_seconds = (end_time - start_time).to_i
               output "Results (#{elapsed_seconds} seconds)", results

data/lib/pact_broker/tasks/delete_overwritten_data_task.rb

@@ -2,9 +2,13 @@ module PactBroker
   module DB
     class DeleteOverwrittenDataTask < ::Rake::TaskLib
       attr_accessor :database_connection
-      attr_accessor :age_in_days
+      attr_accessor :max_age
+      attr_accessor :logger
+      attr_accessor :deletion_limit
+      attr_accessor :dry_run
 
       def initialize &block
+        @max_age = 7
         rake_task &block
       end
 
@@ -19,15 +23,27 @@ module PactBroker
               instance_eval(&block)
               options = {}
 
-              if age_in_days
-                options[:before] = (Date.today - age_in_days.to_i).to_datetime
-                $stdout.puts "Deleting overwritten pact publications and verifications older than #{age_in_days} days"
+              prefix = dry_run ? "[DRY RUN] " : ""
+
+              if max_age
+                options[:max_age] = max_age
+                output "#{prefix}Deleting overwritten pact publications and verifications older than #{max_age} days"
               else
-                $stdout.puts "Deleting overwritten pact publications and verifications"
+                output "#{prefix}Deleting overwritten pact publications and verifications"
               end
 
-              report = PactBroker::DB::DeleteOverwrittenData.call(database_connection, options)
-              $stdout.puts report.to_yaml
+              options[:limit] = deletion_limit if deletion_limit
+              options[:dry_run] = dry_run
+
+              start_time = Time.now
+              results = PactBroker::DB::DeleteOverwrittenData.call(database_connection, options)
+              end_time = Time.now
+              elapsed_seconds = (end_time - start_time).to_i
+              output "Results (#{elapsed_seconds} seconds)", results
+            end
+
+            def output string, payload = {}
+              logger ? logger.info(string, payload: payload) : puts("#{string} #{payload.to_json}")
             end
           end
         end