govspeak 6.2.1 → 6.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 39b6f369a4fda004a0d426e27bd298465354769215c0e5147caa0191456879c1
-  data.tar.gz: ad5e45eb16183f9498ba686ff3694c7ec645c0d2959f271b9cae39746e090934
+  metadata.gz: bc558071169d3b288446dddcb1657bef2dfac22add83004e13f8acfbdc6f8b90
+  data.tar.gz: 4147b7545a3fb2aa1fe09a1d494d02f14065cf3020a65ac46d985e7fc8059f35
 SHA512:
-  metadata.gz: 162a60aad8e18cb213a6df073c9d3f3fae8dc41d490cea9ed871804b3e9d66d1829e2f8a58a20e07d09d80af595450110a4dbc4c7b424dc8d0b990fec0d17da7
-  data.tar.gz: f5ba3ef15abed2fd01750ce68f6aff8482c7e7407d6f4ac689e75f23aa7f6fa507171e2eae1815424c4593686234351fc88d12d09c63dde9b560409f9c3a9fd0
+  metadata.gz: 412b37d58e002eb577de464235f5bc1ec59f7267835320f4d6e509ea16c99d45943922f2697443bdad125cb8dbb68c903062b78185c3ae1bc15f98bb80ae7192
+  data.tar.gz: '0869244c4588b27a308cee8c54e3aed025a8b7b4f921ebe4989a9e490eb90a6f21e69af31f71bb7b00db750627c4d686ff6a96b55b1c2d5b1f8c28af6d034481'

data/CHANGELOG.md

@@ -1,8 +1,13 @@
+## 6.3.0
+
+* Unicode characters forbidden in HTML are stripped from input
+* Validation is now more lenient for HTML input
+
 ## 6.2.1
 
 * Update warning callout label text from 'Help' to 'Warning'
 
-# 6.2.0
+## 6.2.0
 
 * Remove experimental status on `AttachementLink:attachment-id` and `Attachement:attachment-id`
 * Deprecate `embed:attachments:inline:content-id`
@@ -54,25 +59,32 @@
 * Update sanitize version to 4.6.x [#127](https://github.com/alphagov/govspeak/issues/127)
 
 ## 5.5.0
+
 * Ignore links with blank or missing `href`s when extracting links from a document with `Govspeak::Document#extracted_links` [#124](https://github.com/alphagov/govspeak/pull/124)
 
 ## 5.4.0
+
 * Add an optional `website_root` argument to `Govspeak::Document#extracted_links` in order to get all links as fully qualified URLs [#122](https://github.com/alphagov/govspeak/pull/122)
 
 ## 5.3.0
+
 * Add a link extraction class for finding links in documents [#120](https://github.com/alphagov/govspeak/pull/120)
 
 ## 5.2.2
+
 * Fix rendering buttons with inconsistent linebreaks seen in publishing [#118](https://github.com/alphagov/govspeak/pull/118)
 
 ## 5.2.1
+
 * Fix validation to make sure buttons are considered valid
 * Only allow buttons to be used on new lines, not when indented or inline within text (useful for guides) [#116](https://github.com/alphagov/govspeak/pull/116)
 
 ## 5.2.0
+
 * Add button component for govspeak [#114](https://github.com/alphagov/govspeak/pull/114) see README for usage
 
 ## 5.1.0
+
 * Update Kramdown version to 1.15.0
 
 ## 5.0.3
@@ -80,14 +92,17 @@
 * Fix matching links/attachments/contacts by regex to use equality [#105](https://github.com/alphagov/govspeak/pull/105)
 
 ## 5.0.2
+
 * Loosen ActionView dependency to allow use with Rails
 5 [#99](https://github.com/alphagov/govspeak/pull/99)
 
 ## 5.0.1
+
 * Move presenters into the Govspeak namespace [#93](https://github.com/alphagov/govspeak/pull/93)
 * Embedded links now will automatically be marked with `rel="external"` [#96](https://github.com/alphagov/govspeak/pull/96)
 
 ## 5.0.0
+
 * Update Kramdown version to 1.12.0
 * Add pry-byebug to development dependencies
 * Ability to run Govspeak as a binary from command line [#87](https://github.com/alphagov/govspeak/pull/87)

data/lib/govspeak.rb

@@ -1,11 +1,14 @@
 require 'active_support/core_ext/hash'
 require 'active_support/core_ext/array'
 require 'erb'
+require 'govuk_publishing_components'
 require 'htmlentities'
 require 'kramdown'
 require 'kramdown/parser/govuk'
+require 'nokogiri'
+require 'nokogumbo'
 require 'rinku'
-require 'govuk_publishing_components'
+require 'sanitize'
 require 'govspeak/header_extractor'
 require 'govspeak/structured_header_extractor'
 require 'govspeak/html_validator'
@@ -103,6 +106,7 @@ module Govspeak
 
     def preprocess(source)
       source = Govspeak::BlockquoteExtraQuoteRemover.remove(source)
+      source = remove_forbidden_characters(source)
       self.class.extensions.each do |_, regexp, block|
         source.gsub!(regexp) {
           instance_exec(*Regexp.last_match.captures, &block)
@@ -111,6 +115,12 @@ module Govspeak
       source
     end
 
+    def remove_forbidden_characters(source)
+      # These are characters that are not deemed not suitable for
+      # markup: https://www.w3.org/TR/unicode-xml/#Charlist
+      source.gsub(Sanitize::REGEX_UNSUITABLE_CHARS, '')
+    end
+
     def self.extension(title, regexp = nil, &block)
       regexp ||= %r${::#{title}}(.*?){:/#{title}}$m
       @extensions << [title, regexp, block]

data/lib/govspeak/html_sanitizer.rb

@@ -1,5 +1,4 @@
 require 'addressable/uri'
-require 'sanitize'
 
 class Govspeak::HtmlSanitizer
   class ImageSourceWhitelister
@@ -50,10 +49,10 @@ class Govspeak::HtmlSanitizer
   end
 
   def button_sanitize_config
-    [
-      "data-module",
-      "data-tracking-code",
-      "data-tracking-name"
+    %w[
+      data-module
+      data-tracking-code
+      data-tracking-name
     ]
   end
 
@@ -62,7 +61,7 @@ class Govspeak::HtmlSanitizer
       Sanitize::Config::RELAXED,
       elements: Sanitize::Config::RELAXED[:elements] + %w[govspeak-embed-attachment govspeak-embed-attachment-link],
       attributes: {
-        :all => Sanitize::Config::RELAXED[:attributes][:all] + ["role", "aria-label"],
+        :all => Sanitize::Config::RELAXED[:attributes][:all] + %w[role aria-label],
         "a"  => Sanitize::Config::RELAXED[:attributes]["a"] + button_sanitize_config,
         "th"  => Sanitize::Config::RELAXED[:attributes]["th"] + %w[style],
         "td"  => Sanitize::Config::RELAXED[:attributes]["td"] + %w[style],

data/lib/govspeak/html_validator.rb

@@ -18,7 +18,7 @@ class Govspeak::HtmlValidator
 
   # Make whitespace in html tags consistent
   def normalise_html(html)
-    Nokogiri::HTML.parse(html).to_s
+    Nokogiri::HTML5.fragment(html).to_s
   end
 
   def govspeak_to_html

data/lib/govspeak/post_processor.rb

@@ -1,5 +1,3 @@
-require 'nokogiri'
-
 module Govspeak
   class PostProcessor
     @extensions = []

data/lib/govspeak/version.rb

@@ -1,3 +1,3 @@
 module Govspeak
-  VERSION = "6.2.1".freeze
+  VERSION = "6.3.0".freeze
 end

data/test/govspeak_test.rb

@@ -18,6 +18,13 @@ class GovspeakTest < Minitest::Test
     assert_equal "<p><em>this is markdown</em></p>\n", rendered
   end
 
+  test "strips forbidden unicode characters" do
+    rendered = Govspeak::Document.new(
+      "this is text with forbidden characters \ufffc\u2028\ufeff\u202c\u202a"
+    ).to_html
+    assert_equal "<p>this is text with forbidden characters</p>\n", rendered
+  end
+
   test "highlight-answer block extension" do
     rendered = Govspeak::Document.new("this \n{::highlight-answer}Lead in to *BIG TEXT*\n{:/highlight-answer}").to_html
     assert_equal %{<p>this</p>\n\n<div class="highlight-answer">\n<p>Lead in to <em>BIG TEXT</em></p>\n</div>\n}, rendered

data/test/html_validator_test.rb

@@ -101,4 +101,11 @@ class HtmlValidatorTest < Minitest::Test
     assert Govspeak::HtmlValidator.new("{button start}[Start now](https://gov.uk){/button}").valid?
     assert Govspeak::HtmlValidator.new("{button start cross-domain-tracking:UA-XXXXXX-Y}[Start now](https://gov.uk){/button}").valid?
   end
+
+  test "allow HTML tables with and without tbody elements" do
+    # An upgrade of govspeak broke HTML table entries as tbody elements were inserted.
+    # An example of one of these is: https://www.gov.uk/government/publications/what-works-network-membership-requirements/what-works-network
+    assert Govspeak::HtmlValidator.new("<table><tr><td>Hello</td></tr></table>").valid?, "No <tbody> is valid"
+    assert Govspeak::HtmlValidator.new("<table><tbody><tr><td>Hello</td></tr></tbody></table>").valid?, "<tbody> is valid"
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: govspeak
 version: !ruby/object:Gem::Version
-  version: 6.2.1
+  version: 6.3.0
 platform: ruby
 authors:
 - GOV.UK Dev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-04 00:00:00.000000000 Z
+date: 2019-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionview
@@ -114,6 +114,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: nokogumbo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: rinku
   requirement: !ruby/object:Gem::Requirement
@@ -146,16 +160,16 @@ dependencies:
   name: govuk-lint
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.11.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.11.5
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -344,22 +358,22 @@ signing_key:
 specification_version: 4
 summary: Markup language for single domain
 test_files:
+- test/blockquote_extra_quote_remover_test.rb
+- test/govspeak_test_helper.rb
+- test/govspeak_structured_headers_test.rb
 - test/govspeak_attachment_link_test.rb
+- test/govspeak_attachments_image_test.rb
 - test/test_helper.rb
+- test/govspeak_attachments_inline_test.rb
+- test/html_sanitizer_test.rb
 - test/govspeak_button_test.rb
-- test/govspeak_test.rb
+- test/govspeak_images_bang_test.rb
+- test/govspeak_images_test.rb
 - test/html_validator_test.rb
 - test/govspeak_attachment_test.rb
-- test/govspeak_contacts_test.rb
-- test/govspeak_test_helper.rb
-- test/blockquote_extra_quote_remover_test.rb
-- test/govspeak_attachments_image_test.rb
-- test/html_sanitizer_test.rb
-- test/govspeak_link_test.rb
 - test/govspeak_extract_contact_content_ids_test.rb
-- test/govspeak_structured_headers_test.rb
-- test/govspeak_images_test.rb
-- test/presenters/h_card_presenter_test.rb
-- test/govspeak_images_bang_test.rb
+- test/govspeak_test.rb
 - test/govspeak_link_extractor_test.rb
-- test/govspeak_attachments_inline_test.rb
+- test/govspeak_link_test.rb
+- test/govspeak_contacts_test.rb
+- test/presenters/h_card_presenter_test.rb