govspeak 5.9.1 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a66dce46f2af9208dfc78c813b601fe8a447a0c5a68c085516dcf37d9fd5567d
-  data.tar.gz: 62323ec0ee81856973bea87d1575a5aac0d1c4f957ad2261d3081ec6206a45f9
+  metadata.gz: 93b2022644e3af2c929faf57f33c9fe70680977e599bde24d59c71ee20402713
+  data.tar.gz: 7dec8938eedb2183514c54665dd9609f130ac535fbb336739f0f8b6e3d528bc3
 SHA512:
-  metadata.gz: 4b14f7a1707a14e6342b709425556c5d69f64c12ba6523fc0c30ad5a8a98580e05115842cb7c70aa949c219683901e050c415df7b63249c910c59c73a75e011c
-  data.tar.gz: '02386b5204b077af3a5fa03512e0ed451b77ec3a65e38b17015bab5267431b25382b0103338128c02583e6a3b075eb6ff7f41210a3bdcb8a236d139d182cd6c6'
+  metadata.gz: 7b96f54ac8cfdf3e0595cb2de3bc205df86c7efc9869c4e9d06f22623067f4e114d878e9f31c72817d76657b9d759e0b3344ae87724db8394a5a79a727765407
+  data.tar.gz: 9fc03735199e9fe6016ffe1196e57eac617a744c7aaa3421fc1a57a4ba53190c3dc85c5dda7f76b202aaf29c7c5da51f47ed0fcf71ff43a5b504097bcf8b6ebb

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 6.0.0
+
+* BREAKING CHANGE: Input is sanitized by default, to use unsafe HTML initialize with a sanitize option of false
+* Allow sanitize option on remove invalid HTML from source input
+* BREAKING CHANGE: Remove `to_sanitized_html` method in favour of `sanitize` option on initialize
+* BREAKING CHANGE: Remove `to_sanitized_html_without_images` as no apps use this anymore
+* BREAKING CHANGE: Remove CLI usage
+
 ## 5.9.1
 
 * Don't render `[Image: {file-name}]` within a paragraph to avoid invalid HTML

data/README.md

@@ -8,7 +8,7 @@ Install the gem
 
 or add it to your Gemfile
 
-    gem "govspeak", "~> 3.4.0"
+    gem "govspeak"
 
 then create a new document
 
@@ -18,18 +18,6 @@ then create a new document
     doc = Govspeak::Document.new "^Test^"
     puts doc.to_html
 
-or alternatively, run it from the command line
-
-    $ govspeak "render-me"
-    $ govspeak --file render-me.md
-    $ echo "render-me" | govspeak
-
-options can be passed in through `--options` as a string of JSON or a file
-of JSON can be passed in as `--options-file options.json`.
-
-if installed via bundler prefix commands with bundle exec eg `$ bundle exec govspeak "render-me"`
-
-
 # Extensions
 
 In addition to the [standard Markdown syntax](http://daringfireball.net/projects/markdown/syntax "Markdown syntax"), we have added our own extensions.

data/lib/govspeak.rb

@@ -13,6 +13,7 @@ require 'govspeak/kramdown_overrides'
 require 'govspeak/blockquote_extra_quote_remover'
 require 'govspeak/post_processor'
 require 'govspeak/link_extractor'
+require 'govspeak/template_renderer'
 require 'govspeak/presenters/attachment_presenter'
 require 'govspeak/presenters/contact_presenter'
 require 'govspeak/presenters/h_card_presenter'
@@ -47,41 +48,32 @@ module Govspeak
     def initialize(source, options = {})
       options = options.dup.deep_symbolize_keys
       @source = source ? source.dup : ""
+
       @images = options.delete(:images) || []
       @attachments = Array.wrap(options.delete(:attachments))
       @links = Array.wrap(options.delete(:links))
       @contacts = Array.wrap(options.delete(:contacts))
       @locale = options.fetch(:locale, "en")
-      @options = { input: PARSER_CLASS_NAME }.merge(options)
+      @options = { input: PARSER_CLASS_NAME, sanitize: true }.merge(options)
       @options[:entity_output] = :symbolic
     end
 
     def to_html
-      @to_html ||= Govspeak::PostProcessor.process(kramdown_doc.to_html)
+      @to_html ||= begin
+                     html = if @options[:sanitize]
+                              HtmlSanitizer.new(kramdown_doc.to_html).sanitize
+                            else
+                              kramdown_doc.to_html
+                            end
+
+                     Govspeak::PostProcessor.process(html, self)
+                   end
     end
 
     def to_liquid
       to_html
     end
 
-    def t(*args)
-      options = args.last.is_a?(Hash) ? args.last.dup : {}
-      key = args.shift
-      I18n.t!(key, options.merge(locale: locale))
-    end
-
-    def format_with_html_line_breaks(string)
-      ERB::Util.html_escape(string || "").strip.gsub(/(?:\r?\n)/, "<br/>").html_safe
-    end
-
-    def to_sanitized_html
-      HtmlSanitizer.new(to_html).sanitize
-    end
-
-    def to_sanitized_html_without_images
-      HtmlSanitizer.new(to_html).sanitize_without_images
-    end
-
     def to_text
       HTMLEntities.new.decode(to_html.gsub(/(?:<[^>]+>|\s)+/, " ").strip)
     end
@@ -225,13 +217,11 @@ module Govspeak
       render_image(ImagePresenter.new(image))
     end
 
-    extension('attachment', /\[embed:attachments:(?!inline:|image:)\s*(.*?)\s*\]/) do |content_id, body|
-      attachment = attachments.detect { |a| a[:content_id] == content_id }
-      next "" unless attachment
-
-      attachment = AttachmentPresenter.new(attachment)
-      content = File.read(__dir__ + '/templates/attachment.html.erb')
-      ERB.new(content).result(binding)
+    extension('attachment', /\[embed:attachments:(?!inline:|image:)\s*(.*?)\s*\]/) do |content_id|
+      # not treating this as a self closing tag seems to avoid some oddities
+      # such as an extra new line being inserted when explicitly closed or
+      # swallowing subsequent elements when not closed
+      %{<govspeak-embed-attachment content-id="#{content_id}"></govspeak-embed-attachment>}
     end
 
     extension('attachment inline', /\[embed:attachments:inline:\s*(.*?)\s*\]/) do |content_id|
@@ -353,9 +343,8 @@ module Govspeak
       contact = contacts.detect { |c| c[:content_id] == content_id }
       next "" unless contact
 
-      contact = ContactPresenter.new(contact)
-      @renderer ||= ERB.new(File.read(__dir__ + '/templates/contact.html.erb'))
-      @renderer.result(binding)
+      renderer = TemplateRenderer.new('contact.html.erb', locale)
+      renderer.render(contact: ContactPresenter.new(contact))
     end
 
     extension('Image', /#{NEW_PARAGRAPH_LOOKBEHIND}\[Image:\s*(.*?)\s*\]/) do |image_id|
@@ -374,10 +363,6 @@ module Govspeak
     def encode(text)
       HTMLEntities.new.encode(text)
     end
-
-    def render_hcard_address(contact_address)
-      HCardPresenter.new(contact_address).render
-    end
   end
 end
 

data/lib/govspeak/html_sanitizer.rb

@@ -49,11 +49,6 @@ class Govspeak::HtmlSanitizer
     Sanitize.clean(@dirty_html, Sanitize::Config.merge(sanitize_config, transformers: transformers))
   end
 
-  def sanitize_without_images
-    config = sanitize_config
-    Sanitize.clean(@dirty_html, Sanitize::Config.merge(config, elements: config[:elements] - %w[img]))
-  end
-
   def button_sanitize_config
     [
       "data-module",
@@ -65,11 +60,13 @@ class Govspeak::HtmlSanitizer
   def sanitize_config
     Sanitize::Config.merge(
       Sanitize::Config::RELAXED,
+      elements: Sanitize::Config::RELAXED[:elements] + %w[govspeak-embed-attachment],
       attributes: {
         :all => Sanitize::Config::RELAXED[:attributes][:all] + ["role", "aria-label"],
         "a"  => Sanitize::Config::RELAXED[:attributes]["a"] + button_sanitize_config,
         "th"  => Sanitize::Config::RELAXED[:attributes]["th"] + %w[style],
         "td"  => Sanitize::Config::RELAXED[:attributes]["td"] + %w[style],
+        "govspeak-embed-attachment" => %w[content-id],
       }
     )
   end

data/lib/govspeak/html_validator.rb

@@ -22,6 +22,6 @@ class Govspeak::HtmlValidator
   end
 
   def govspeak_to_html
-    Govspeak::Document.new(govspeak_string).to_html
+    Govspeak::Document.new(govspeak_string, sanitize: false).to_html
   end
 end

data/lib/govspeak/post_processor.rb

@@ -8,8 +8,8 @@ module Govspeak
       @extensions
     end
 
-    def self.process(html)
-      new(html).output
+    def self.process(html, govspeak_document)
+      new(html, govspeak_document).output
     end
 
     def self.extension(title, &block)
@@ -49,10 +49,25 @@ module Govspeak
       end
     end
 
-    attr_reader :input
+    extension("embed attachment HTML") do |document|
+      document.css("govspeak-embed-attachment").map do |el|
+        attachment = govspeak_document.attachments.detect { |a| a[:content_id] == el["content-id"] }
+        unless attachment
+          el.remove
+          next
+        end
 
-    def initialize(html)
+        renderer = TemplateRenderer.new('attachment.html.erb', govspeak_document.locale)
+        attachment_html = renderer.render(attachment: AttachmentPresenter.new(attachment))
+        el.swap(attachment_html)
+      end
+    end
+
+    attr_reader :input, :govspeak_document
+
+    def initialize(html, govspeak_document)
       @input = html
+      @govspeak_document = govspeak_document
     end
 
     def output
@@ -63,7 +78,6 @@ module Govspeak
       document.to_html
     end
 
-
   private
 
     def nokogiri_document

data/lib/govspeak/template_renderer.rb

@@ -0,0 +1,27 @@
+module Govspeak
+  class TemplateRenderer
+    attr_reader :template, :locale
+
+    def initialize(template, locale)
+      @template = template
+      @locale = locale
+    end
+
+    def render(locals)
+      template_binding = binding
+      locals.each { |k, v| template_binding.local_variable_set(k, v) }
+      erb = ERB.new(File.read(__dir__ + "/../templates/#{template}"))
+      erb.result(template_binding)
+    end
+
+    def t(*args)
+      options = args.last.is_a?(Hash) ? args.last.dup : {}
+      key = args.shift
+      I18n.t!(key, options.merge(locale: locale))
+    end
+
+    def format_with_html_line_breaks(string)
+      ERB::Util.html_escape(string || "").strip.gsub(/(?:\r?\n)/, "<br/>").html_safe
+    end
+  end
+end

data/lib/govspeak/version.rb

@@ -1,3 +1,3 @@
 module Govspeak
-  VERSION = "5.9.1".freeze
+  VERSION = "6.0.0".freeze
 end

data/lib/templates/contact.html.erb

@@ -7,7 +7,7 @@
     <h3><%= contact.title %></h3>
     <div class="vcard contact-inner">
     <% contact.post_addresses.each do |address| %>
-      <%= render_hcard_address(address) %>
+      <%= Govspeak::HCardPresenter.new(address).render %>
     <% end %>
     <% if contact.email_addresses.any? || contact.phone_numbers.any? || contact.contact_form_links.any? %>
       <div class="email-url-number">

data/test/govspeak_test.rb

@@ -632,14 +632,14 @@ Teston
       }
   end
 
-  test "can sanitize a document" do
+  test 'sanitize source input by default' do
     document = Govspeak::Document.new("<script>doBadThings();</script>")
-    assert_equal "doBadThings();", document.to_sanitized_html.strip
+    assert_equal "", document.to_html.strip
   end
 
-  test "can sanitize a document without image" do
-    document = Govspeak::Document.new("<script>doBadThings();</script><img src='https://example.com/image.jpg'>")
-    assert_equal "doBadThings();<p></p>", document.to_sanitized_html_without_images.gsub(/\s/, "")
+  test 'it can have sanitizing disabled' do
+    document = Govspeak::Document.new("<script>doGoodThings();</script>", sanitize: false)
+    assert_equal "<script>doGoodThings();</script>", document.to_html.strip
   end
 
   test "identifies a Govspeak document containing malicious HTML as invalid" do

data/test/html_sanitizer_test.rb

@@ -3,7 +3,7 @@ require "test_helper"
 class HtmlSanitizerTest < Minitest::Test
   test "disallow a script tag" do
     html = "<script>alert('XSS')</script>"
-    assert_equal "alert('XSS')", Govspeak::HtmlSanitizer.new(html).sanitize
+    assert_equal "", Govspeak::HtmlSanitizer.new(html).sanitize
   end
 
   test "disallow a javascript protocol in an attribute" do
@@ -46,11 +46,6 @@ class HtmlSanitizerTest < Minitest::Test
     assert_equal "", Govspeak::HtmlSanitizer.new(html, allowed_image_hosts: ['allowed.com']).sanitize
   end
 
-  test "can strip images" do
-    html = "<img src='http://example.com/image.jgp'>"
-    assert_equal "", Govspeak::HtmlSanitizer.new(html).sanitize_without_images
-  end
-
   test "allows table cells and table headings without a style attribute" do
     html = "<table><thead><tr><th>thing</th></tr></thead><tbody><tr><td>thing</td></tr></tbody></table>"
     assert_equal html, Govspeak::HtmlSanitizer.new(html).sanitize

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: govspeak
 version: !ruby/object:Gem::Version
-  version: 5.9.1
+  version: 6.0.0
 platform: ruby
 authors:
 - GOV.UK Dev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-07 00:00:00.000000000 Z
+date: 2019-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionview
@@ -44,20 +44,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
-- !ruby/object:Gem::Dependency
-  name: commander
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.4'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.4'
 - !ruby/object:Gem::Dependency
   name: htmlentities
   requirement: !ruby/object:Gem::Requirement
@@ -148,14 +134,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.6'
+        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.6'
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: govuk-lint
   requirement: !ruby/object:Gem::Requirement
@@ -245,8 +231,7 @@ description: |-
   library for use in the UK Government Single Domain project
 email:
 - govuk-dev@digital.cabinet-office.gov.uk
-executables:
-- govspeak
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
@@ -258,11 +243,9 @@ files:
 - assets/example.png
 - assets/information_callout.png
 - assets/warning_callout.png
-- bin/govspeak
 - config/address_formats.yml
 - lib/govspeak.rb
 - lib/govspeak/blockquote_extra_quote_remover.rb
-- lib/govspeak/cli.rb
 - lib/govspeak/header_extractor.rb
 - lib/govspeak/html_sanitizer.rb
 - lib/govspeak/html_validator.rb
@@ -275,11 +258,11 @@ files:
 - lib/govspeak/presenters/h_card_presenter.rb
 - lib/govspeak/presenters/image_presenter.rb
 - lib/govspeak/structured_header_extractor.rb
+- lib/govspeak/template_renderer.rb
 - lib/govspeak/version.rb
 - lib/kramdown/parser/kramdown_with_automatic_external_links.rb
 - lib/templates/attachment.html.erb
 - lib/templates/contact.html.erb
-- lib/templates/inline_attachment.html.erb
 - locales/ar.yml
 - locales/be.yml
 - locales/bg.yml
@@ -364,21 +347,21 @@ signing_key:
 specification_version: 4
 summary: Markup language for single domain
 test_files:
-- test/govspeak_link_extractor_test.rb
-- test/govspeak_structured_headers_test.rb
-- test/govspeak_images_bang_test.rb
-- test/govspeak_button_test.rb
-- test/govspeak_extract_contact_content_ids_test.rb
 - test/blockquote_extra_quote_remover_test.rb
 - test/govspeak_test_helper.rb
-- test/govspeak_link_test.rb
-- test/govspeak_images_test.rb
-- test/govspeak_contacts_test.rb
+- test/govspeak_structured_headers_test.rb
+- test/govspeak_attachments_image_test.rb
+- test/govspeak_attachments_test.rb
 - test/test_helper.rb
-- test/html_validator_test.rb
-- test/html_sanitizer_test.rb
 - test/govspeak_attachments_inline_test.rb
+- test/html_sanitizer_test.rb
+- test/govspeak_button_test.rb
+- test/govspeak_images_bang_test.rb
+- test/govspeak_images_test.rb
+- test/html_validator_test.rb
+- test/govspeak_extract_contact_content_ids_test.rb
 - test/govspeak_test.rb
-- test/govspeak_attachments_test.rb
-- test/govspeak_attachments_image_test.rb
+- test/govspeak_link_extractor_test.rb
+- test/govspeak_link_test.rb
+- test/govspeak_contacts_test.rb
 - test/presenters/h_card_presenter_test.rb

data/bin/govspeak

@@ -1,8 +0,0 @@
-#!/usr/bin/env ruby
-
-lib = File.expand_path('../lib', __dir__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-require "govspeak/cli"
-
-Govspeak::CLI.new.run

data/lib/govspeak/cli.rb

@@ -1,53 +0,0 @@
-require 'govspeak/version'
-require 'govspeak'
-require 'commander'
-
-module Govspeak
-  class CLI
-    include Commander::Methods
-
-    def run
-      program(:name, 'Govspeak')
-      program(:version, Govspeak::VERSION)
-      program(:description, "A tool for rendering the GOV.UK dialect of markdown into HTML")
-      default_command(:render)
-      command(:render) do |command|
-        command.syntax = "govspeak render [options] <input>"
-        command.description = "Render Govspeak into HTML, can be sourced from stdin, as an argument or from a file"
-        command.option("--file FILENAME", String, "File to render")
-        command.option("--options JSON", String, "JSON to use as options")
-        command.option("--options-file FILENAME", String, "A file of JSON options")
-        command.action do |args, options|
-          input = get_input($stdin, args, options)
-          raise "Nothing to render. Use --help for assistance" unless input
-
-          puts Govspeak::Document.new(input, govspeak_options(options)).to_html
-        end
-      end
-      run!
-    end
-
-  private
-
-    def get_input(stdin, args, options)
-      return stdin.read unless stdin.tty?
-      return read_file(options.file) if options.file
-
-      args.empty? ? nil : args.join(" ")
-    end
-
-    def read_file(file_path)
-      path = Pathname.new(file_path).realpath
-      File.read(path)
-    end
-
-    def govspeak_options(command_options)
-      string = if command_options.options_file
-                 read_file(command_options.options_file)
-               else
-                 command_options.options
-               end
-      string ? JSON.parse(string) : {}
-    end
-  end
-end

data/lib/templates/inline_attachment.html.erb

@@ -1,6 +0,0 @@
-<span <% if attachment.id %>id="attachment_<%= attachment.id %>" <% end %>class="attachment-inline">
-  <%= attachment.link attachment.title, attachment.url %>
-  <% unless attachment.attachment_attributes.empty? %>
-    (<%= attachment.attachment_attributes %>)
-  <% end %>
-</span>