html-pipeline 2.14.3 → 3.0.1

data/UPGRADING.md

@@ -0,0 +1,34 @@
+# Upgrade Guide
+
+## From v2 to v3
+
+HTMLPipeline v3 is a massive improvement over this still much loved (and woefully under-maintained) project. This section will attempt to list all of the breaking changes between the two versions and provide suggestions on how to upgrade.
+
+### Changed namespace
+
+This project is now under a module called `HTMLPipeline`, not `HTML::Pipeline`.
+
+### Removed filters
+
+The following filters were removed:
+
+- `AutolinkFilter`: this is handled by [Commonmarker](https://www.github.com/gjtorikian/commonmarker) and can be disabled/enabled through the `MarkdownFilter`'s `context` hash
+- `SanitizationFilter`: this is handled by [Selma](https://www.github.com/gjtorikian/selma); configuration can be done through the `sanitization_config` hash
+
+- `EmailReplyFilter`
+- `CamoFilter`
+- `TextFilter`
+
+### Changed API
+
+The new way to call this project is as follows:
+
+```ruby
+HTMLPipeline.new(
+    text_filters: [], # array of instantiated (`.new`ed) `HTMLPipeline::TextFilter`
+    convert_filter:, # a filter that runs to turn text into HTML
+    sanitization_config: {}, # an allowlist of elements/attributes/protocols to keep
+    node_filters: []) # array of instantiated (`.new`ed) `HTMLPipeline::NodeFilter`
+```
+
+Please refer to the README for more information on constructing filters. In most cases, the underlying filter needs only a few changes, primarily to make use of [Selma](https://www.github.com/gjtorikian/selma) rather than Nokogiri.

data/html-pipeline.gemspec

@@ -1,29 +1,39 @@
 # frozen_string_literal: true
 
-require File.expand_path('../lib/html/pipeline/version', __FILE__)
+$LOAD_PATH.push(File.expand_path("lib", __dir__))
+require "html_pipeline/version"
 
 Gem::Specification.new do |gem|
-  gem.name          = 'html-pipeline'
-  gem.version       = HTML::Pipeline::VERSION
-  gem.license       = 'MIT'
-  gem.authors       = ['Ryan Tomayko', 'Jerry Cheung', 'Garen J. Torikian']
-  gem.email         = ['ryan@github.com', 'jerry@github.com', 'gjtorikian@gmail.com']
-  gem.description   = 'GitHub HTML processing filters and utilities'
-  gem.summary       = 'Helpers for processing content through a chain of filters'
-  gem.homepage      = 'https://github.com/jch/html-pipeline'
+  gem.name          = "html-pipeline"
+  gem.version       = HTMLPipeline::VERSION
+  gem.license       = "MIT"
+  gem.authors       = ["Garen J. Torikian"]
+  gem.email         = ["gjtorikian@gmail.com"]
+  gem.description   = "HTML processing filters and utilities"
+  gem.summary       = "Helpers for processing content through a chain of filters"
+  gem.homepage      = "https://github.com/gjtorikian/html-pipeline"
 
-  gem.files         = `git ls-files -z`.split("\x0").reject { |f| f =~ %r{^(test|gemfiles|script)/} }
-  gem.require_paths = ['lib']
+  gem.files         = %x(git ls-files -z).split("\x0").reject { |f| f =~ %r{^(test|gemfiles|script)/} }
+  gem.require_paths = ["lib"]
 
-  gem.add_dependency 'activesupport', '>= 2'
-  gem.add_dependency 'nokogiri', '>= 1.4'
+  gem.required_ruby_version = "~> 3.1"
+  # https://github.com/rubygems/rubygems/pull/5852#issuecomment-1231118509
+  gem.required_rubygems_version = ">= 3.3.22"
 
-  gem.post_install_message = <<msg
--------------------------------------------------
-Thank you for installing html-pipeline!
-You must bundle Filter gem dependencies.
-See html-pipeline README.md for more details.
-https://github.com/jch/html-pipeline#dependencies
--------------------------------------------------
-msg
+  gem.metadata = {
+    "funding_uri" => "https://github.com/sponsors/gjtorikian/",
+    "rubygems_mfa_required" => "true",
+  }
+
+  gem.add_dependency("selma", "~> 0.1")
+  gem.add_dependency("zeitwerk", "~> 2.5")
+
+  gem.post_install_message = <<~MSG
+    -------------------------------------------------
+    Thank you for installing html-pipeline!
+    You must bundle filter gem dependencies.
+    See the html-pipeline README.md for more details:
+    https://github.com/gjtorikian/html-pipeline#dependencies
+    -------------------------------------------------
+  MSG
 end

data/lib/html-pipeline.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "html_pipeline"

data/lib/html_pipeline/convert_filter/markdown_filter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+HTMLPipeline.require_dependency("commonmarker", "MarkdownFilter")
+
+class HTMLPipeline
+  class ConvertFilter
+    # HTML Filter that converts Markdown text into HTML.
+    #
+    # Context options:
+    #   :markdown[:parse] => Commonmarker parse options
+    #   :markdown[:render] => Commonmarker render options
+    #   :markdown[:extensions] => Commonmarker extensions options
+    class MarkdownFilter < ConvertFilter
+      def initialize(context: {}, result: {})
+        super(context: context, result: result)
+      end
+
+      # Convert Commonmark to HTML using the best available implementation.
+      def call(text)
+        options = @context.fetch(:markdown, {})
+        plugins = options.fetch(:plugins, {})
+        Commonmarker.to_html(text, options: options, plugins: plugins).rstrip!
+      end
+    end
+  end
+end

data/lib/html_pipeline/convert_filter.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class HTMLPipeline
+  class ConvertFilter < Filter
+    attr_reader :text, :html
+
+    def initialize(context: {}, result: {})
+      super(context: context, result: result)
+    end
+
+    class << self
+      def call(text, context: {}, result: {})
+        new(context: context, result: result).call(text)
+      end
+    end
+  end
+end

data/lib/html_pipeline/filter.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+class HTMLPipeline
+  # Base class for user content HTML filters. Each filter takes an
+  # HTML string, performs modifications on it, and/or writes information to a result hash.
+  # Filters must return a String with HTML markup.
+  #
+  # The `context` Hash passes options to filters and should not be changed in
+  # place. A `result` Hash allows filters to make extracted information
+  # available to the caller, and is mutable.
+  #
+  # Common context options:
+  #   :base_url   - The site's base URL
+  #   :repository - A Repository providing context for the HTML being processed
+  #
+  # Each filter may define additional options and output values. See the class
+  # docs for more info.
+  class Filter
+    class InvalidDocumentException < StandardError; end
+
+    def initialize(context: {}, result: {})
+      @context = context
+      @result = result
+      validate
+    end
+
+    # Public: Returns a simple Hash used to pass extra information into filters
+    # and also to allow filters to make extracted information available to the
+    # caller.
+    attr_reader :context
+
+    # Public: Returns a Hash used to allow filters to pass back information
+    # to callers of the various Pipelines.  This can be used for
+    # #mentioned_users, for example.
+    attr_reader :result
+
+    # The main filter entry point. The doc attribute is guaranteed to be a
+    # string when invoked. Subclasses should modify
+    # this text in place or extract information and add it to the context
+    # hash.
+    def call
+      raise NoMethodError
+    end
+
+    class << self
+      # Perform a filter on doc with the given context.
+      #
+      # Returns a String comprised of HTML markup.
+      def call(input, context: {})
+        raise NoMethodError
+      end
+    end
+    # Make sure the context has everything we need. Noop: Subclasses can override.
+    def validate; end
+
+    # The site's base URL provided in the context hash, or '/' when no
+    # base URL was specified.
+    def base_url
+      context[:base_url] || "/"
+    end
+
+    # Helper method for filter subclasses used to determine if any of a node's
+    # ancestors have one of the tag names specified.
+    #
+    # node - The Node object to check.
+    # tags - An array of tag name strings to check. These should be downcase.
+    #
+    # Returns true when the node has a matching ancestor.
+    def has_ancestor?(element, ancestor)
+      ancestors = element.ancestors
+      ancestors.include?(ancestor)
+    end
+
+    # Validator for required context. This will check that anything passed in
+    # contexts exists in @contexts
+    #
+    # If any errors are found an ArgumentError will be raised with a
+    # message listing all the missing contexts and the filters that
+    # require them.
+    def needs(*keys)
+      missing = keys.reject { |key| context.include?(key) }
+
+      return if missing.none?
+
+      raise ArgumentError,
+        "Missing context keys for #{self.class.name}: #{missing.map(&:inspect).join(", ")}"
+    end
+  end
+end

data/lib/html_pipeline/node_filter/absolute_source_filter.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "uri"
+
+class HTMLPipeline
+  class NodeFilter
+    # HTML Filter for replacing relative and root relative image URLs with
+    # fully qualified URLs
+    #
+    # This is useful if an image is root relative but should really be going
+    # through a cdn, or if the content for the page assumes the host is known
+    # i.e. scraped webpages and some RSS feeds.
+    #
+    # Context options:
+    #   :image_base_url - Base URL for image host for root relative src.
+    #   :image_subpage_url - For relative src.
+    #
+    # This filter does not write additional information to the context.
+    # Note: This filter would need to be run before AssetProxyFilter.
+    class AbsoluteSourceFilter < NodeFilter
+      SELECTOR = Selma::Selector.new(match_element: "img")
+
+      def selector
+        SELECTOR
+      end
+
+      def handle_element(element)
+        src = element["src"]
+        return if src.nil? || src.empty?
+
+        src = src.strip
+        return if src.start_with?("http")
+
+        base = if src.start_with?("/")
+          image_base_url
+        else
+          image_subpage_url
+        end
+
+        element["src"] = URI.join(base, src).to_s
+      end
+
+      # Private: the base url you want to use
+      def image_base_url
+        context[:image_base_url] || raise("Missing context :image_base_url for #{self.class.name}")
+      end
+
+      # Private: the relative url you want to use
+      def image_subpage_url
+        context[:image_subpage_url] || raise("Missing context :image_subpage_url for #{self.class.name}")
+      end
+    end
+  end
+end

data/lib/html_pipeline/node_filter/asset_proxy_filter.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "openssl"
+
+class HTMLPipeline
+  class NodeFilter
+    # Proxy images/assets to another server, such as
+    # [cactus/go-camo](https://github.com/cactus/go-camo#).
+    # Reduces mixed content warnings as well as hiding the customer's
+    # IP address when requesting images.
+    # Copies the original img `src` to `data-canonical-src` then replaces the
+    # `src` with a new url to the proxy server.
+    #
+    # Based on https://github.com/gjtorikian/html-pipeline/blob/v2.14.3/lib/html/pipeline/camo_filter.rb
+    class AssetProxyFilter < NodeFilter
+      SELECTOR = Selma::Selector.new(match_element: "img")
+
+      def selector
+        SELECTOR
+      end
+
+      def handle_element(element)
+        original_src = element["src"]
+        return unless original_src
+
+        begin
+          uri = URI.parse(original_src)
+        rescue StandardError
+          return
+        end
+
+        return if uri.host.nil? && !original_src.start_with?("///")
+        return if asset_host_allowed?(uri.host)
+
+        element["src"] = asset_proxy_url(original_src)
+        element["data-canonical-src"] = original_src
+      end
+
+      def validate
+        needs(:asset_proxy, :asset_proxy_secret_key)
+      end
+
+      def asset_host_allowed?(host)
+        context[:asset_proxy_domain_regexp] ? context[:asset_proxy_domain_regexp].match?(host) : false
+      end
+
+      class << self
+        # This helps setup the context. It's not needed if you're always providing
+        # all the necessary keys in the context. One example would be to override
+        # this and pull the settings from a set of global application settings.
+        def transform_context(context, proxy_settings = {})
+          context[:asset_proxy] = proxy_settings[:url] if proxy_settings[:url]
+          context[:asset_proxy_secret_key] = proxy_settings[:secret_key] if proxy_settings[:secret_key]
+
+          allowlist = determine_allowlist(proxy_settings)
+          context[:asset_proxy_domain_regexp] ||= compile_allowlist(allowlist)
+
+          context
+        end
+
+        def compile_allowlist(domain_list)
+          return if domain_list.empty?
+
+          escaped = domain_list.map { |domain| Regexp.escape(domain).gsub("\\*", ".*?") }
+          Regexp.new("^(#{escaped.join("|")})$", Regexp::IGNORECASE)
+        end
+
+        def determine_allowlist(proxy_settings)
+          proxy_settings[:allowlist] || []
+        end
+      end
+
+      private def asset_proxy_url(url)
+        "#{context[:asset_proxy]}/#{asset_url_hash(url)}/#{hexencode(url)}"
+      end
+
+      private def asset_url_hash(url)
+        OpenSSL::HMAC.hexdigest("sha1", context[:asset_proxy_secret_key], url)
+      end
+
+      private def hexencode(str)
+        str.unpack1("H*")
+      end
+    end
+  end
+end

data/lib/{html/pipeline → html_pipeline/node_filter}/emoji_filter.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require 'cgi'
-HTML::Pipeline.require_dependency('gemoji', 'EmojiFilter')
+require "cgi"
+HTMLPipeline.require_dependencies(["gemoji", "gemojione"], "EmojiFilter")
 
-module HTML
-  class Pipeline
+class HTMLPipeline
+  class NodeFilter
     # HTML filter that replaces :emoji: with images.
     #
     # Context:
@@ -12,25 +12,28 @@ module HTML
     #   :asset_path (optional) - url path to link to emoji sprite. :file_name can be used as a placeholder for the sprite file name. If no asset_path is set "emoji/:file_name" is used.
     #   :ignored_ancestor_tags (optional) - Tags to stop the emojification. Node has matched ancestor HTML tags will not be emojified. Default to pre, code, and tt tags. Extra tags please pass in the form of array, e.g., %w(blockquote summary).
     #   :img_attrs (optional) - Attributes for generated img tag. E.g. Pass { "draggble" => true, "height" => nil } to set draggable attribute to "true" and clear height attribute of generated img tag.
-    class EmojiFilter < Filter
-      DEFAULT_IGNORED_ANCESTOR_TAGS = %w[pre code tt].freeze
-
-      def call
-        doc.search('.//text()').each do |node|
-          content = node.text
-          next unless content.include?(':')
-          next if has_ancestor?(node, ignored_ancestor_tags)
-          html = emoji_image_filter(content)
-          next if html == content
-          node.replace(html)
-        end
-        doc
+    class EmojiFilter < NodeFilter
+      DEFAULT_IGNORED_ANCESTOR_TAGS = ["pre", "code", "tt"].freeze
+
+      # Build a regexp that matches all valid :emoji: names.
+      def after_initialize
+        @emoji_pattern ||= /:(#{emoji_names.map { |name| Regexp.escape(name) }.join("|")}):/
+      end
+
+      def selector
+        Selma::Selector.new(match_text_within: "*", ignore_text_within: ignored_ancestor_tags)
+      end
+
+      def handle_text_chunk(text)
+        return unless text.to_s.include?(":")
+
+        text.replace(emoji_image_filter(text.to_s), as: :html)
       end
 
       # Implementation of validate hook.
       # Errors should raise exceptions or use an existing validator.
       def validate
-        needs :asset_root
+        needs(:asset_root)
       end
 
       # Replace :emoji: with corresponding images.
@@ -39,7 +42,7 @@ module HTML
       #
       # Returns a String with :emoji: replaced with images.
       def emoji_image_filter(text)
-        text.gsub(emoji_pattern) do |_match|
+        text.gsub(@emoji_pattern) do
           emoji_image_tag(Regexp.last_match(1))
         end
       end
@@ -58,64 +61,65 @@ module HTML
       # Returns the context's asset_path or the default path if no context asset_path is given.
       def asset_path(name)
         if context[:asset_path]
-          context[:asset_path].gsub(':file_name', emoji_filename(name))
+          context[:asset_path].gsub(":file_name", emoji_filename(name))
         else
-          File.join('emoji', emoji_filename(name))
+          File.join("emoji", emoji_filename(name))
         end
       end
 
-      private
-
       # Build an emoji image tag
-      def emoji_image_tag(name)
-        require 'active_support/core_ext/hash/indifferent_access'
+      private def emoji_image_tag(name)
         html_attrs =
-          default_img_attrs(name)
-          .merge!((context[:img_attrs] || {}).with_indifferent_access)
-          .map { |attr, value| !value.nil? && %(#{attr}="#{value.respond_to?(:call) && value.call(name) || value}") }
-          .reject(&:blank?).join(' '.freeze)
+          default_img_attrs(name).transform_keys(&:to_sym)
+            .merge!(context[:img_attrs] || {}).transform_keys(&:to_sym)
+            .each_with_object([]) do |(attr, value), arr|
+            next if value.nil?
+
+            value = value.respond_to?(:call) && value.call(name) || value
+            arr << %(#{attr}="#{value}")
+          end.compact.join(" ")
 
         "<img #{html_attrs}>"
       end
 
+      def emoji_names
+        if self.class.gemoji_loaded?
+          Emoji.all.map(&:aliases)
+        else
+          Gemojione::Index.new.all.map { |i| i[1]["name"] }
+        end.flatten.sort
+      end
+
       # Default attributes for img tag
-      def default_img_attrs(name)
+      private def default_img_attrs(name)
         {
-          'class' => 'emoji'.freeze,
-          'title' => ":#{name}:",
-          'alt' => ":#{name}:",
-          'src' => emoji_url(name).to_s,
-          'height' => '20'.freeze,
-          'width' => '20'.freeze,
-          'align' => 'absmiddle'.freeze
+          "class" => "emoji",
+          "title" => ":#{name}:",
+          "alt" => ":#{name}:",
+          "src" => emoji_url(name).to_s,
+          "height" => "20",
+          "width" => "20",
+          "align" => "absmiddle",
         }
       end
 
-      def emoji_url(name)
+      private def emoji_url(name)
         File.join(asset_root, asset_path(name))
       end
 
-      # Build a regexp that matches all valid :emoji: names.
-      def self.emoji_pattern
-        @emoji_pattern ||= /:(#{emoji_names.map { |name| Regexp.escape(name) }.join('|')}):/
-      end
-
-      def emoji_pattern
-        self.class.emoji_pattern
-      end
-
-      def self.emoji_names
-        Emoji.all.map(&:aliases).flatten.sort
-      end
-
-      def emoji_filename(name)
-        Emoji.find_by_alias(name).image_filename
+      private def emoji_filename(name)
+        if self.class.gemoji_loaded?
+          Emoji.find_by_alias(name).image_filename
+        else
+          # replace their asset_host with ours
+          Gemojione.image_url_for_name(name).sub(Gemojione.asset_host, "")
+        end
       end
 
       # Return ancestor tags to stop the emojification.
       #
       # @return [Array<String>] Ancestor tags.
-      def ignored_ancestor_tags
+      private def ignored_ancestor_tags
         if context[:ignored_ancestor_tags]
           DEFAULT_IGNORED_ANCESTOR_TAGS | context[:ignored_ancestor_tags]
         else

data/lib/html_pipeline/node_filter/https_filter.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class HTMLPipeline
+  class NodeFilter
+    # HTML Filter for replacing http references to :http_url with https versions.
+    # Subdomain references are not rewritten.
+    #
+    # Context options:
+    #   :http_url - The HTTP url to force HTTPS. Falls back to :base_url
+    class HttpsFilter < NodeFilter
+      SELECTOR = Selma::Selector.new(match_element: %(a[href^="http:"]))
+
+      def selector
+        SELECTOR
+      end
+
+      def handle_element(element)
+        element["href"] = element["href"].sub(/^http:/, "https:")
+      end
+    end
+  end
+end

data/lib/html_pipeline/node_filter/image_max_width_filter.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class HTMLPipeline
+  class NodeFilter
+    # This filter rewrites image tags with a max-width inline style and also wraps
+    # the image in an <a> tag that causes the full size image to be opened in a
+    # new tab.
+    #
+    # The max-width inline styles are especially useful in HTML email which
+    # don't use a global stylesheets.
+    class ImageMaxWidthFilter < NodeFilter
+      SELECTOR = Selma::Selector.new(match_element: "img")
+
+      def selector
+        SELECTOR
+      end
+
+      def handle_element(element)
+        # Skip if there's already a style attribute. Not sure how this
+        # would happen but we can reconsider it in the future.
+        return if element["style"]
+
+        # Bail out if src doesn't look like a valid http url. trying to avoid weird
+        # js injection via javascript: urls.
+        return if /\Ajavascript/i.match?(element["src"].to_s.strip)
+
+        element["style"] = "max-width:100%;"
+
+        link_image(element) unless has_ancestor?(element, "a")
+      end
+
+      def link_image(element)
+        link_start = %(<a target="_blank" href="#{element["src"]}">)
+        element.before(link_start, as: :html)
+        link_end = "</a>"
+        element.after(link_end, as: :html)
+      end
+    end
+  end
+end