govuk-diff-pages 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d7461d5624d6438b64b89ee14db3a51a49c6789d
+  data.tar.gz: 5575a5c4d1a74062a4d50c72dd6d29b2e2e05293
+SHA512:
+  metadata.gz: 9363e7d7ef669768b36d6acea6c8a2301c3dbe0d32dfc51d83f872a803d0a66003e28ceeb13759a4ac07dba798dcc58dece2a0c394ed933a8d7c1e59e6267e41
+  data.tar.gz: abfc0131220eb9ad866edfd83f2b36e7ea958f86cbc4608be01483a9d7c2471ec46a4971c5a8e6d9057c3cdd40b725f2887c056688aad565a55c2e1434227fc3

data/.gitignore

@@ -0,0 +1,33 @@
+# See https://help.github.com/articles/ignoring-files for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile '~/.gitignore_global'
+
+# Ignore bundler config.
+/.bundle
+
+# Ignore the default SQLite database.
+/db/*.sqlite3
+/db/*.sqlite3-journal
+
+# Ignore all logfiles and tempfiles.
+/log/*.log
+/tmp
+
+/coverage
+/spec/reports
+
+
+# OS-generated files
+.DS_Store
+
+# shots generated
+/shots
+
+# generated config files
+config/govuk_pages.yml
+config/wraith.yaml
+
+# This is a gem
+Gemfile.lock

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+2.2.3

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'wraith', git: 'git@github.com:alphagov/wraith.git', branch: 'psr'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Ben Lovell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,75 @@
+# govuk-diff-pages
+
+This app provides a rake task to produce visual diffs as screenshots, HTML
+diffs and textual diffs of the production GOVUK website as compared with
+staging. Viewable as browser pages or directly in the terminal. It looks a the
+10 most popular pages (this is configurable) of each document format.
+
+## Screenshots
+
+![Example output](docs/screenshots/gallery.png?raw=true "Example gallery of
+differing pages")
+
+
+## Technical documentation for visual and HTML diffs
+
+It uses a fork of the BBC's wraith gem (The fork is at
+https://github.com/alphagov/wraith, the main gem at
+https://github.com/BBC-News/wraith).  The fork adds two extra configuration
+variables, allowing the user to specify the number of threads to use, and the
+maximum timeout when loading pages.  The output is written to an html file
+which can be viewed in a browser.
+
+When `bundle exec rake diff` is run, a list of all the document formats on
+govuk is obtained using the search api, and then the top n pages for each
+format (n being a configuration variable).  Diffs are produced for each of
+these pages.
+
+
+### Dependencies
+
+- [ImageMagick] (http://www.imagemagick.org/script/index.php)
+- [phantomjs] (http://phantomjs.org/) - preferbaly 1.9 rather than 2.0
+
+
+## How to run
+
+### Running the application locally
+
+  bundle exec rake diff
+
+### Checking plain-text diffs
+
+  bundle exec rake diff:text pages.yml
+
+Where `pages.yml` is a YAML array of paths to compare. For example:
+
+  - government/organisations/prime-ministers-office-10-downing-street
+  - government/topical-events/budget-2016
+  - topic/competition/regulatory-appeals-references
+
+Text diffs can also specify the domains to compare using the `LEFT` and `RIGHT`
+environment variables. Defaulting to our `www-origin.staging` and
+`www-origin.publishing` domains respectively.
+
+Plain-text diffing can be parallelised by starting multiple processes with
+individual page files.
+
+### Using the gem from an existing project
+
+  # Gemfile
+  gem 'govuk-diff-pages'
+
+  # Rakefile
+  load 'govuk/diff/pages/tasks/rakefile.rake'
+
+  # Shell
+  bundle exec rake -T
+
+### Running the test suite
+
+  bundle exec rake
+
+## Licence
+
+[MIT License](LICENCE)

data/Rakefile

@@ -0,0 +1,14 @@
+require "rspec/core/rake_task"
+require "gem_publisher"
+
+load File.dirname(__FILE__) + "/lib/govuk/diff/pages/tasks/rakefile.rake"
+
+RSpec::Core::RakeTask.new(:spec)
+task default: :spec
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :publish_gem do
+  gem = GemPublisher.publish_if_updated("govuk-diff-pages.gemspec", :rubygems)
+  puts "Published #{gem}" if gem
+end

data/config/settings.yml

@@ -0,0 +1,53 @@
+
+# base url for production and staging domains - WITHOUT TRAILING slash
+domains:
+  production: https://www-origin.publishing.service.gov.uk
+  staging: https://www-origin.staging.publishing.service.gov.uk
+
+# Enter a list of pages that you want to diff, but are not in the list of pages returned by the 
+# page indexer
+hard_coded_pages:
+
+verbose: true
+
+html_diff:
+  directory: "shots/html"
+
+page_indexer:
+  # The maximum number of formats to search for.
+  # There are less than 100, so a value of 100 will ensure we capture them all
+  # max_formats: 100 # use this value in production
+  max_formats: 6 # use this value for a quicker test
+
+  # The maximum number of pages for each format - we will capture the <n> most popular pages
+  # max_pages_per_format: 10 # use this value in production
+  max_pages_per_format: 3 # use this value for a quicker test
+
+# Configuration options for wraith
+wraith:
+  # Headless browser option​
+  browser:
+    phantomjs: phantomjs
+
+  # name of the directory where the screenshots will be stored
+  directory: "shots/visual"
+
+  # Screen widths
+  screen_widths:
+  - 1024
+
+  # Choose how results are displayed, by default alphanumeric. Different screen widths are always grouped.
+  # alphanumeric - all paths (with, and without, a difference) are shown, sorted by path​
+  # diffs_first - all paths (with, and without, a difference) are shown, sorted by difference size (largest first)​
+  # diffs_only - only paths with a difference are shown, sorted by difference size (largest first)​
+  mode: diffs_only
+
+  # Amount of fuzz ImageMagick will use
+  fuzz: "5%"
+  threshold: 5
+
+  # number of threads to use (reduce this number if there are timeouts getting page captures)
+  num_threads: 4
+
+  # timeout in milliseconds for browser to wait for pages to load
+  timeout_ms: 2000

data/govuk-diff-pages.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'govuk/diff/pages/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "govuk-diff-pages"
+  spec.version       = Govuk::Diff::Pages::VERSION
+  spec.authors       = ["Ben Lovell"]
+  spec.email         = ["benjamin.lovell@gmail.com"]
+
+  spec.summary       = %q{Visual and textual page diffing.}
+  spec.description   = %q{Diffs web pages both visually and textually.}
+  spec.homepage      = "https://github.com/alphagov/govuk-diff-pages"
+  spec.license       = "MIT"
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "diffy", "~> 3.1"
+  spec.add_dependency "rest-client", "~> 1.8"
+  spec.add_dependency "nokogiri"
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "pry-byebug"
+  spec.add_development_dependency "gem_publisher", "~> 1.1"
+end

data/lib/govuk/diff/pages.rb

@@ -0,0 +1,30 @@
+require "govuk/diff/pages/version"
+
+module Govuk
+  module Diff
+    module Pages
+      autoload :AppConfig, 'govuk/diff/pages/app_config'
+      autoload :FormatSearcher, 'govuk/diff/pages/format_searcher'
+      autoload :LinkChecker, 'govuk/diff/pages/link_checker'
+      autoload :PageIndexer, 'govuk/diff/pages/page_indexer'
+      autoload :PageSearcher, 'govuk/diff/pages/page_searcher'
+      autoload :UrlChecker, 'govuk/diff/pages/url_checker'
+      autoload :WraithConfigGenerator, 'govuk/diff/pages/wraith_config_generator'
+
+      autoload :HtmlDiff, 'govuk/diff/pages/html_diff'
+      autoload :TextDiff, 'govuk/diff/pages/text_diff'
+
+      def self.root_dir
+        File.dirname __dir__
+      end
+
+      def self.govuk_pages_file
+        File.expand_path(root_dir + '/../../config/govuk_pages.yml')
+      end
+
+      def self.wraith_config_file
+        File.expand_path(root_dir + '/../../config/wraith.yaml')
+      end
+    end
+  end
+end

data/lib/govuk/diff/pages/app_config.rb

@@ -0,0 +1,61 @@
+require 'yaml'
+require 'ostruct'
+
+module Govuk
+  module Diff
+    module Pages
+      class AppConfig
+        class MissingKeyError < RuntimeError; end
+
+        # param can be one of the following:
+        # * a file path - the config will be loaded from the specified yaml file
+        # * a Hash - the config will be loaded directly from the hash,
+        # * nil - the config will be loaded from the default yaml file (config/settings.yml)
+        #
+        def initialize(path_or_hash = nil)
+          if path_or_hash.is_a?(Hash)
+            @config = populate_config(path_or_hash)
+          else
+            path_or_hash ||= "#{Govuk::Diff::Pages.root_dir}/config/settings.yml"
+            hash = YAML.load_file(path_or_hash)
+            @config = populate_config(hash)
+          end
+        end
+
+        def method_missing(method, *_params)
+          result = @config.public_send(method)
+          raise MissingKeyError.new "No such config key '#{method}'" if result.nil?
+          result = nil if result == :nil_value
+          result
+        end
+
+        def to_h
+          result = {}
+          @config.to_h.each do |key, value|
+            value = nil if value == :nil_value
+            if value.is_a?(AppConfig)
+              result[key.to_s] = value.to_h
+            else
+              result[key.to_s] = value
+            end
+          end
+          result
+        end
+
+      private
+        def populate_config(hash)
+          config = OpenStruct.new
+          hash.each do |key, value|
+            if value.is_a?(Hash)
+              config[key] = AppConfig.new(value)
+            else
+              value = :nil_value if value.nil?
+              config[key] = value
+            end
+          end
+          config
+        end
+      end
+    end
+  end
+end

data/lib/govuk/diff/pages/format_searcher.rb

@@ -0,0 +1,37 @@
+require 'rest-client'
+require 'json'
+
+module Govuk
+  module Diff
+    module Pages
+      class FormatSearcher
+        def initialize(config)
+          @config = config
+        end
+
+        def run
+          puts "Getting list of formats in GOVUK" if verbose?
+          response = get_facets
+          extract_formats(response)
+        end
+
+        def verbose?
+          @config.verbose
+        end
+
+      private
+
+        def extract_formats(response)
+          parsed_response = JSON.parse(response)
+          options = parsed_response.fetch('facets').fetch('format').fetch('options')
+          options.map { |o| o['value']['slug'] }
+        end
+
+        def get_facets
+          url = "#{@config.domains.production}/api/search.json?facet_format=#{@config.page_indexer.max_formats}"
+          RestClient.get(url)
+        end
+      end
+    end
+  end
+end

data/lib/govuk/diff/pages/html_diff.rb

@@ -0,0 +1,10 @@
+module Govuk
+  module Diff
+    module Pages
+      module HtmlDiff
+        autoload :Differ, 'govuk/diff/pages/html_diff/differ'
+        autoload :Runner, 'govuk/diff/pages/html_diff/runner'
+      end
+    end
+  end
+end

data/lib/govuk/diff/pages/html_diff/assets/gallery_template.erb

@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Gallery of HTML Diffs</title>
+
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css">
+    <link rel="stylesheet" href="../../lib/html_diff/assets/html_diff_styles.css">
+  </head>
+  <body>
+    <div class="container">
+
+      <h1>List of HTML Diffs</h1>
+      <h2><%= @result_hash.size %> pages out of <%= @govuk_pages.size %> compared have differences</h2>
+
+      <div class="row">
+        <div class="col-md-12">
+        <table>
+          <% @result_hash.each do |base_path, filename| %>
+            <tr>
+              <td><%= base_path %></td>
+              <td><a href= 'file://<%= filename %>' target='_blank'>HTML Diff</a></td>
+            </tr>
+          <% end %>
+        </table>
+    </div>
+  </body>
+</html>

data/lib/govuk/diff/pages/html_diff/assets/html_diff_styles.css

@@ -0,0 +1,59 @@
+.diff {
+  overflow: auto;
+}
+
+.diff ul {
+  background: #fff;
+  display: table;
+  font-size: 13px;
+  list-style: none;
+  margin: 0;
+  overflow: auto;
+  padding: 0;
+  width: 100%;
+}
+
+.diff del, .diff ins {
+  display: block;
+  text-decoration: none;
+}
+
+.diff li {
+  display: table-row;
+  height: 1em;
+  margin: 0;
+  padding: 0;
+}
+
+.example {
+  padding: 10px;
+}
+
+.left {
+  background: #ddefff;
+}
+
+.right {
+  background: #af8ed0;
+}
+
+.diff li.ins {
+  background: #d8b6fa;
+}
+
+.diff li.del {
+  background: #ddefff;
+}
+
+.diff del, .diff ins, .diff span {
+  font-family: courier;
+  white-space: pre-wrap;
+}
+
+.diff li.diff-comment {
+  display: none;
+}
+
+.diff li.diff-block-info {
+  background: none repeat scroll 0 0 gray;
+}