pact_broker 2.15.0 → 2.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 404111dc7e6ebc927394295e7d75df78fdd1a1e3
-  data.tar.gz: 52f7cbf51020f206ae07a5a876c4d04d4adecec8
+  metadata.gz: '08afc57ea5d32509b4533f1667506bf2225d1d68'
+  data.tar.gz: e1a451ae25b6e50d1f5839aa4ff25990f3d9e112
 SHA512:
-  metadata.gz: b0e7b2db30b4e2f4045cfb391c23b88df8384676d314e867bdc7ef72597eadce0f0abf6c13bd6186269bb01573e71b912cfa34a307ddef2cf8076a2977d72020
-  data.tar.gz: afd391d5952f47ac4278c676e2846521b94b6d58fdc7c6658697dfbbcb0d8f9b2e4e77b160c605a6abf8353854bdfc4fa958669ba72ef4d75bf48c1835bdcd9d
+  metadata.gz: 02b6ebf75e3840706d78a435a241da778d4155fe469431c9f670c55780ca7c338c0ec7c937c353d65359aa477ebeb589690d300cb3f1c235055584c80b7abe05
+  data.tar.gz: 7810448e6235d088de9c4646b74b895de50f23d8b87ad8841e7e22e098d4745d006ed01d85c6e574ccaa0d496e6ea2fa6bb83c7f38643285c3e419524d512c09

data/.codeclimate.yml

@@ -4,12 +4,12 @@ engines:
     enabled: true
   duplication:
     enabled: true
+    exclude_patterns:
+    - spec/
     config:
       languages:
       - ruby
       - javascript
-      exclude_paths:
-      - spec/
   eslint:
     enabled: true
   fixme:
@@ -26,8 +26,12 @@ ratings:
   - "**.php"
   - "**.py"
   - "**.rb"
-exclude_paths:
+exclude_patterns:
 - script/
 - vendor/
 - public/Network Graph_files/
 - public/js/bootstrap*
+- public/javascripts/d3.v3.js.pagespeed.ce.dFNRrGTALe.js
+- public/javascripts/highlight.pack.js
+- public/javascripts/jquery*.js
+- public/js/bootstrap*.js

data/CHANGELOG.md

@@ -1,3 +1,56 @@
+<a name="v2.16.0"></a>
+### v2.16.0 (2018-03-05)
+
+
+#### Features
+
+* upgrade haml to 5.0	 ([4415686](/../../commit/4415686))
+* upgrade sequel to 5.6	 ([a37c59a](/../../commit/a37c59a))
+* abbreviate git shas in index and matrix UIs	 ([9a21e10](/../../commit/9a21e10))
+* add latest provider version tag names to dashboard UI	 ([de179d7](/../../commit/de179d7))
+* created dashboard API	 ([044bab7](/../../commit/044bab7))
+* allow database to have missing migration files so that we can deploy branches with different migrations to the same database	 ([c9fe988](/../../commit/c9fe988))
+* add dius logo to new index view with tags	 ([8a4e727](/../../commit/8a4e727))
+
+* **matrix**
+  * make index refresh happen synchronously to prevent data corruption	 ([acd64e6](/../../commit/acd64e6))
+  * speed up query to refresh index	 ([011d7c9](/../../commit/011d7c9))
+  * refresh head matrix asynchronously to speed up pact publishing and tagging	 ([989e6e3](/../../commit/989e6e3))
+  * speed up matrix query for latestby=cvpv and latestby=cvp	 ([6ae039d](/../../commit/6ae039d))
+
+* **index**
+  * sort verifications by execution date	 ([2523982](/../../commit/2523982))
+  * show tag names as slugs	 ([117c89b](/../../commit/117c89b))
+
+* **matrix ui**
+  * rename "rows" to "results"	 ([2023265](/../../commit/2023265))
+  * show row count	 ([b50bd9a](/../../commit/b50bd9a))
+  * remove logic for hiding links until I can get it right	 ([107fca2](/../../commit/107fca2))
+  * correct logic for hiding pact links	 ([825af6d](/../../commit/825af6d))
+  * don't show potentially confusing links for overwritten pact revisions	 ([ed7498a](/../../commit/ed7498a))
+  * add links to all resources referenced in matrix	 ([2958ae8](/../../commit/2958ae8))
+  * add tags	 ([869d49d](/../../commit/869d49d))
+
+* **dashboard api**
+  * correct latest verification results	 ([d02ea24](/../../commit/d02ea24))
+  * fix query for displaying dashboard with tags	 ([924aaae](/../../commit/924aaae))
+  * optimise dashboard query by creating manual materialized views for the matrix	 ([494f553](/../../commit/494f553))
+  * add plain text content type for easy debugging	 ([317a64d](/../../commit/317a64d))
+  * speed up loading of verification tags	 ([a3aea48](/../../commit/a3aea48))
+  * include verification tags	 ([e8d0fae](/../../commit/e8d0fae))
+  * add pact tags	 ([0c79440](/../../commit/0c79440))
+
+
+#### Bug Fixes
+
+* **matrix**
+  * correct logic for selecting matrix rows on MySQL	 ([06f6dd4](/../../commit/06f6dd4))
+  * ensure matrix for latest consumer version/provider version shows correct results when a pact is published, published again for the same version with different content, then published again for the same version with the original content.	 ([679eec1](/../../commit/679eec1))
+
+* **matrix ui**
+  * fix provider version sorting	 ([611ab7e](/../../commit/611ab7e))
+
+
 <a name="v2.15.0"></a>
 ### v2.15.0 (2018-02-26)
 

data/DEVELOPER_DOCUMENTATION.md

@@ -58,4 +58,31 @@ Domain classes are found in `lib/pact_broker/domain`. Many of these classes are
 
 * `matrix` - The matrix of every pact publication and verification. Includes every pact revision (eg. publishing to the same consumer version twice, or using PATCH) and every verification (including 'overwritten' ones. eg. when the same provider build runs twice.)
 
-* `latest_matrix` - This view is a subset of, and has the same columns as, the `matrix`. It removes 'overwritten' pacts and verifications from the matrix (ie. only show latest pact revision for each consumer version and latest verification for each provider version)
+* `latest_matrix_for_consumer_version_and_provider_version` - This view is a subset of, and has the same columns as, the `matrix`. It removes 'overwritten' pacts and verifications from the matrix (ie. only show latest pact revision for each consumer version and latest verification for each provider version)
+
+### Materialized Views
+
+We can't use proper materialized views because we have to support MySQL :|
+
+So as a hacky solution, there are two tables which act as materialized views to speed up the performance of the matrix and index queries. These tables are updated after any resource is published with a `consumer_name`, `provider_name` or `pacticipant_name` in the URL (see lib/pact_broker/api/resources/base_resource.rb#finish_request).
+
+* `materialized_matrix` table - is populated from the `matrix` view.
+
+* `materialized_head_matrix` table - is populated from `head_matrix` view, and is based on `materialized_matrix`.
+
+### Dependencies
+
+materialized_head_matrix table (is populated from...)
+  = head_matrix view
+    -> latest_matrix_for_consumer_version_and_provider_version view
+      -> materialized_matrix table (is populated from...)
+        = matrix view
+          -> verifications table
+          -> versions table
+          -> all_pact_publications view
+            -> pact_versions table
+            -> pact_publications table
+            -> pacticipants table
+            -> versions table
+      -> latest_verification_id_for_consumer_version_and_provider_version view
+      -> latest_pact_publication_revision_numbers view

data/DEVELOPER_SETUP.md

@@ -1,6 +1,6 @@
 # Developer setup
 
-* You will need to install Ruby 2.4, and preferably a ruby version manager. I recommend using [chruby](chruby) and [ruby-install](ruby-install).
+* You will need to install Ruby 2.4, and preferably a ruby version manager. I recommend using [chruby][chruby] and [ruby-install][ruby-install].
 * Install bundler (the Ruby gem dependency manager) `gem install bundler`
 * Check out the pact_broker repository.
 * Run `bundle exec pact_broker:dev:setup`. This will create an example application that you can run locally, that uses the local source code.

data/README.md

@@ -6,8 +6,11 @@
  [![Issue Count](https://codeclimate.com/github/pact-foundation/pact_broker/badges/issue_count.svg)](https://codeclimate.com/github/pact-foundation/pact_broker)
  [![Dependency Status](https://gemnasium.com/badges/github.com/pact-foundation/pact_broker.svg)](https://gemnasium.com/github.com/pact-foundation/pact_broker)
 
-The Pact Broker is an application for sharing for consumer driven contracts. It is optimised for use with "pacts" (contracts created by the [Pact][pact-docs] framework), but can be used for any type of contract that can be serialized to JSON.
+The Pact Broker is an application for sharing for consumer driven contracts and verification results. It is optimised for use with "pacts" (contracts created by the [Pact][pact-docs] framework), but can be used for any type of contract that can be serialized to JSON.
 
+**Why do I need one?**
+
+Contract testing is an alternative approach to traditional integration testing that gives you tests that are quicker to execute, and that are more maintainable at scale. One down side of the approach is that the important information that would be available all in one place at the end of an integration test suite execution (that is, the version numbers of all the applications that were tested together, and whether the tests passed or failed) is now spread out across many different builds. The Pact Broker is a tool that brings all this information back together again, and allows you to deploy with safety.
 
 It:
 
@@ -29,7 +32,7 @@ Features:
 * Displays provider verificaton results so you know if you can deploy safely.
 * Provides a "matrix" of compatible consumer and provider versions, so you know which versions can be safely deployed together.
 * Provides badges to display pact verification statuses in your READMEs.
-* Enables an application version to be tagged (ie. "prod", "feat/customer-preferences") to allow repository-like workflows. 
+* Enables an application version to be tagged (ie. "prod", "feat/customer-preferences") to allow repository-like workflows.
 * Provides webhooks to trigger actions when pacts change eg. run provider build, notify a Slack channel.
 * View diffs between Pact versions so you can tell what expectations have changed.
 * [Docker Pact Broker][docker]
@@ -40,7 +43,7 @@ Features:
 #### Step 1. Consumer CI build
 1. The consumer project runs its tests using the [Pact][pact] library to provide a mock service.
 2. While the tests run, the mock service writes the requests and the expected responses to a JSON "pact" file - this is the consumer contract.
-3. The generated pact is then published to the Pact Broker. Most Pact libries will make a task available for you to do this easily, however, at its simplest, it is a `PUT` to a resource that specifies the consumer name and application version, and the provider name. eg `http://my-pact-broker/pacts/provider/Animal%20Service/consumer/Zoo%20App/version/1.0.0` 
+3. The generated pact is then published to the Pact Broker. Most Pact libries will make a task available for you to do this easily, however, at its simplest, it is a `PUT` to a resource that specifies the consumer name and application version, and the provider name. eg `http://my-pact-broker/pacts/provider/Animal%20Service/consumer/Zoo%20App/version/1.0.0`
 (Note that you are specifying the _consumer application version_ in the URL, not the pact version. The broker will take care of versioning the pact behind the scenes when its content changes. It is expected that the consumer application version will increment with every CI build.)
 4. When a pact is published, a webhook in the Pact Broker kicks off a build of the provider project if the pact content has changed since the previous version.
 
@@ -63,7 +66,7 @@ Read more about how to use the Pact Broker in the [overview][overview] on the wi
 
 ## Documentation
 
-See the [wiki][wiki] for documentation on the Pact Broker.
+See the [wiki][wiki] for documentation on the Pact Broker. Please read the [overview][overview] page first to get an understanding of the HTTP resources in the broker, and how they relate to each other.
 
 ## Support
 
@@ -139,7 +142,7 @@ You can use the [Pact Broker Docker image][docker] or [Terraform on AWS][terrafo
  * Check the [travis.yml][travisyml] file to make sure you're using the version of the database that we're currently running our tests against.
 * If you're using PostgreSQL (did we mention this was _recommended!_) you'll find the database creation script in the [example/config.ru](https://github.com/pact-foundation/pact_broker/blob/master/example/config.ru).
 * Install ruby 2.2.0 or later and bundler >= 1.12.0 (if you've come this far, I'm assuming you know how to do both of these. Did I mention there was a [Docker][docker] image?)
-* Copy the [pact\_broker](https://github.com/DiUS/pact_broker-docker/tree/master/pact_broker) directory from the Pact Broker Docker project. This will have the recommended settings for database connections, logging, basic auth etc. Note that the Docker image uses Phusion Passenger as the web application server in front of the Pact Broker Ruby application, which is the recommended set up. 
+* Copy the [pact\_broker](https://github.com/DiUS/pact_broker-docker/tree/master/pact_broker) directory from the Pact Broker Docker project. This will have the recommended settings for database connections, logging, basic auth etc. Note that the Docker image uses Phusion Passenger as the web application server in front of the Pact Broker Ruby application, which is the recommended set up.
 * Modify the config.ru and Gemfile as desired (eg. choose database driver gem, set your database credentials. Use the "pg" gem if using Postgres and the "mysql2" gem if using MySQL)
 * Please ensure you use `encoding: 'utf8'` in your Sequel options to avoid encoding issues.
 * For production usage, use a web application server like [Phusion Passenger](https://www.phusionpassenger.com) or [Nginx](http://nginx.org/) to serve the Pact Broker application. You'll need to read up on the documentation for these yourself as it is beyond the scope of this documentation. See the [wiki][reverse-proxy-docs] for instructions on using a reverse proxy with SSL.

data/db/migrations/000004_create_tags_table.rb

@@ -3,7 +3,7 @@ Sequel.migration do
     create_table(:tags, charset: 'utf8') do
       String :name
       foreign_key :version_id, :versions
-      primary_key [:version_id, :name], :name=>:tags_pk
+      primary_key [:version_id, :name], name: :tags_pk
     end
   end
 end

data/db/migrations/000050_create_latest_matrix.rb

@@ -5,6 +5,7 @@ Sequel.migration do
     # latest verification for each provider version)
     # Must include lines where verification_id is null so that we don't
     # lose the unverified pacts.
+    # In this view there will be one row for each consumer version/provider version
     create_view(:latest_matrix,
       "SELECT matrix.* FROM matrix
       INNER JOIN latest_verification_id_for_consumer_version_and_provider_version AS lv

data/db/migrations/20180122_create_head_pact_publications.rb

@@ -0,0 +1,21 @@
+Sequel.migration do
+  change do
+
+    # This view includes all the latest pacts, as well as the latest pacts
+    # for each tag.
+    # If a pact publication is the latest AND the latest tagged version
+    # there will be two rows in this view for it - one for the top
+    # query,and one for the bottom.
+    create_view(:head_pact_publications,
+      "select lp.*, null as tag_name, 1 as latest
+      from latest_pact_publications lp
+
+      UNION
+
+      select ltp.*, null as latest
+      from latest_tagged_pact_publications ltp
+      "
+    )
+
+  end
+end

data/db/migrations/20180123_create_tags_with_latest_flag.rb

@@ -0,0 +1,28 @@
+require_relative 'migration_helper'
+
+Sequel.migration do
+  change do
+    create_view(:latest_tagged_version_orders,
+      PactBroker::MigrationHelper.sqlite_safe("
+        select v.pacticipant_id, t.name as tag_name, max(v.order) as latest_version_order, 1 as latest
+        from tags t
+        inner join versions v
+        on v.id = t.version_id
+        group by v.pacticipant_id, t.name
+      ")
+    )
+
+    create_view(:tags_with_latest_flag,
+      PactBroker::MigrationHelper.sqlite_safe("
+        select t.*, ltvo.latest
+        from tags t
+        inner join versions v
+        on v.id = t.version_id
+        left outer join latest_tagged_version_orders ltvo
+        on t.name = ltvo.tag_name
+        and v.pacticipant_id = ltvo.pacticipant_id
+        and v.order = ltvo.latest_version_order
+      ")
+    )
+  end
+end

data/db/migrations/20180129_create_latest_matrix_for_cv_and_pv.rb

@@ -0,0 +1,91 @@
+Sequel.migration do
+  up do
+    # Removes 'overwritten' pacts and verifications from the matrix
+    # (ie. only show latest pact revision for each consumer version and
+    # latest verification for each provider version)
+    # Must include lines where verification_id is null so that we don't
+    # lose the unverified pacts.
+    # In this view there will be one row for each consumer version/provider version
+    # This view used to be (stupidly) called latest_matrix
+    create_view(:latest_matrix_for_consumer_version_and_provider_version,
+      "SELECT matrix.* FROM matrix
+      INNER JOIN latest_verification_id_for_consumer_version_and_provider_version AS lv
+      ON ((matrix.consumer_version_id = lv.consumer_version_id)
+      AND (matrix.provider_version_id = lv.provider_version_id)
+      AND ((matrix.verification_id = lv.latest_verification_id)))
+
+      UNION
+
+      select matrix.* from matrix
+      inner join latest_pact_publication_revision_numbers lr
+      on matrix.consumer_id = lr.consumer_id
+      and matrix.provider_id = lr.provider_id
+      and matrix.consumer_version_order = lr.consumer_version_order
+      and matrix.pact_revision_number = lr.latest_revision_number
+      where verification_id is null
+      "
+    )
+
+    # it's a bit dodgey to be using the max(id) of the verification to determine the latest,
+    # but otherwise we'd need to add an extra step and find the latest_provider_version_order
+    # and then find the latest verification within that provider_version_order, which we'd
+    # probably do by using the ID anyway. And I'm feeling lazy.
+    create_view(:latest_verification_id_for_consumer_version_and_provider,
+      "select matrix.consumer_version_id, matrix.provider_id, max(verification_id) as latest_verification_id
+      from latest_matrix_for_consumer_version_and_provider_version matrix
+      where matrix.verification_id is not null
+      group by matrix.consumer_version_id, matrix.provider_id
+      "
+    )
+
+    # update the definition of latest_matrix to actually be the latest_matrix
+    # in the same way that latest_pact_publications is.
+    # It contains the latest verification results for the latest pacts.
+    create_or_replace_view(:latest_matrix,
+      "SELECT matrix.* FROM latest_matrix_for_consumer_version_and_provider_version matrix
+      INNER JOIN latest_pact_consumer_version_orders lpcvo
+      ON matrix.consumer_id = lpcvo.consumer_id
+      AND matrix.provider_id = lpcvo.provider_id
+      AND matrix.consumer_version_order = lpcvo.latest_consumer_version_order
+      INNER JOIN latest_verification_id_for_consumer_version_and_provider AS lv
+      ON ((matrix.consumer_version_id = lv.consumer_version_id)
+      AND (matrix.provider_id = lv.provider_id)
+      AND ((matrix.verification_id = lv.latest_verification_id)))
+
+      UNION
+
+      SELECT matrix.* FROM latest_matrix_for_consumer_version_and_provider_version matrix
+      INNER JOIN latest_pact_consumer_version_orders lpcvo
+      ON matrix.consumer_id = lpcvo.consumer_id
+      AND matrix.provider_id = lpcvo.provider_id
+      AND matrix.consumer_version_order = lpcvo.latest_consumer_version_order
+      where verification_id is null
+      "
+    )
+  end
+
+  down do
+    # revert to previous crappy definition
+    create_or_replace_view(:latest_matrix,
+      "SELECT matrix.* FROM matrix
+      INNER JOIN latest_verification_id_for_consumer_version_and_provider_version AS lv
+      ON ((matrix.consumer_version_id = lv.consumer_version_id)
+      AND (matrix.provider_version_id = lv.provider_version_id)
+      AND ((matrix.verification_id = lv.latest_verification_id)))
+
+      UNION
+
+      select matrix.* from matrix
+      inner join latest_pact_publication_revision_numbers lr
+      on matrix.consumer_id = lr.consumer_id
+      and matrix.provider_id = lr.provider_id
+      and matrix.consumer_version_order = lr.consumer_version_order
+      and matrix.pact_revision_number = lr.latest_revision_number
+      where verification_id is null
+      "
+    )
+
+    drop_view(:latest_verification_id_for_consumer_version_and_provider)
+    drop_view(:latest_matrix_for_consumer_version_and_provider_version)
+  end
+end

data/db/migrations/20180130_create_materialized_matrix.rb

@@ -0,0 +1,35 @@
+Sequel.migration do
+  up do
+    create_table(:materialized_matrix, charset: 'utf8') do
+      Integer :consumer_id, null: false
+      String :consumer_name, null: false
+      Integer :consumer_version_id, null: false
+      String :consumer_version_number, null: false
+      Integer :consumer_version_order, null: false
+      Integer :pact_publication_id, null: false
+      Integer :pact_version_id, null: false
+      String :pact_version_sha, null: false
+      Integer :pact_revision_number, null: false
+      DateTime :pact_created_at, null: false
+      Integer :provider_id, null: false
+      String :provider_name, null: false
+      Integer :provider_version_id
+      String :provider_version_number
+      Integer :provider_version_order
+      Integer :verification_id
+      Boolean :success
+      Integer :verification_number
+      DateTime :verification_executed_at
+      String :verification_build_url
+      index [:consumer_id], name: 'ndx_mm_consumer_id'
+      index [:provider_id], name: 'ndx_mm_provider_id'
+      index [:consumer_version_order], name: 'ndx_mm_cv_ord'
+    end
+
+    from(:materialized_matrix).insert(from(:matrix).select_all)
+  end
+
+  down do
+    drop_table(:materialized_matrix)
+  end
+end

data/db/migrations/20180131_create_materialized_latest_matrix.rb

@@ -0,0 +1,35 @@
+Sequel.migration do
+  up do
+    create_table(:materialized_latest_matrix, charset: 'utf8') do
+      Integer :consumer_id, null: false
+      String :consumer_name, null: false
+      Integer :consumer_version_id, null: false
+      String :consumer_version_number, null: false
+      Integer :consumer_version_order, null: false
+      Integer :pact_publication_id, null: false
+      Integer :pact_version_id, null: false
+      String :pact_version_sha, null: false
+      Integer :pact_revision_number, null: false
+      DateTime :pact_created_at, null: false
+      Integer :provider_id, null: false
+      String :provider_name, null: false
+      Integer :provider_version_id
+      String :provider_version_number
+      Integer :provider_version_order
+      Integer :verification_id
+      Boolean :success
+      Integer :verification_number
+      DateTime :verification_executed_at
+      String :verification_build_url
+      index [:consumer_id], name: 'ndx_mlm_consumer_id'
+      index [:provider_id], name: 'ndx_mlm_provider_id'
+      index [:consumer_version_order], name: 'ndx_mlm_cv_ord'
+    end
+
+    from(:materialized_latest_matrix).insert(from(:latest_matrix).select_all)
+  end
+
+  down do
+    drop_table(:materialized_latest_matrix)
+  end
+end

data/db/migrations/20180201_create_head_matrix.rb

@@ -0,0 +1,33 @@
+Sequel.migration do
+  up do
+    # a row for each of the latest pact publications,
+    # and a row for each of the latest tagged pact publications
+    create_view(:head_matrix,
+      "SELECT matrix.*, hpp.tag_name as consumer_tag_name
+      FROM latest_matrix_for_consumer_version_and_provider_version matrix
+      INNER JOIN head_pact_publications hpp
+      ON matrix.consumer_id = hpp.consumer_id
+      AND matrix.provider_id = hpp.provider_id
+      AND matrix.consumer_version_order = hpp.consumer_version_order
+      INNER JOIN latest_verification_id_for_consumer_version_and_provider AS lv
+      ON matrix.consumer_version_id = lv.consumer_version_id
+      AND matrix.provider_id = lv.provider_id
+      AND matrix.verification_id = lv.latest_verification_id
+
+      UNION
+
+      SELECT matrix.*, hpp.tag_name as consumer_tag_name
+      FROM latest_matrix_for_consumer_version_and_provider_version matrix
+      INNER JOIN head_pact_publications hpp
+      ON matrix.consumer_id = hpp.consumer_id
+      AND matrix.provider_id = hpp.provider_id
+      AND matrix.consumer_version_order = hpp.consumer_version_order
+      where verification_id is null
+      "
+    )
+  end
+
+  down do
+    drop_view(:head_matrix)
+  end
+end