html-pipeline-plus 2.10.1

data/lib/html/pipeline-plus/sanitization_filter.rb

@@ -0,0 +1,137 @@
+HTML::Pipeline.require_dependency('sanitize', 'SanitizationFilter')
+
+module HTML
+  class Pipeline
+    # HTML filter with sanization routines and whitelists. This module defines
+    # what HTML is allowed in user provided content and fixes up issues with
+    # unbalanced tags and whatnot.
+    #
+    # See the Sanitize docs for more information on the underlying library:
+    #
+    # https://github.com/rgrove/sanitize/#readme
+    #
+    # Context options:
+    #   :whitelist      - The sanitizer whitelist configuration to use. This
+    #                     can be one of the options constants defined in this
+    #                     class or a custom sanitize options hash.
+    #   :anchor_schemes - The URL schemes to allow in <a href> attributes. The
+    #                     default set is provided in the ANCHOR_SCHEMES
+    #                     constant in this class. If passed, this overrides any
+    #                     schemes specified in the whitelist configuration.
+    #
+    # This filter does not write additional information to the context.
+    class SanitizationFilter < Filter
+      LISTS     = Set.new(%w[ul ol].freeze)
+      LIST_ITEM = 'li'.freeze
+
+      # List of table child elements. These must be contained by a <table> element
+      # or they are not allowed through. Otherwise they can be used to break out
+      # of places we're using tables to contain formatted user content (like pull
+      # request review comments).
+      TABLE_ITEMS = Set.new(%w[tr td th].freeze)
+      TABLE = 'table'.freeze
+      TABLE_SECTIONS = Set.new(%w[thead tbody tfoot].freeze)
+
+      # These schemes are the only ones allowed in <a href> attributes by default.
+      ANCHOR_SCHEMES = ['http', 'https', 'mailto', 'xmpp', :relative, 'github-windows', 'github-mac', 'irc', 'ircs'].freeze
+
+      # The main sanitization whitelist. Only these elements and attributes are
+      # allowed through by default.
+      WHITELIST = {
+        elements: %w[
+          h1 h2 h3 h4 h5 h6 h7 h8 br b i strong em a pre code img tt
+          div ins del sup sub p ol ul table thead tbody tfoot blockquote
+          dl dt dd kbd q samp var hr ruby rt rp li tr td th s strike summary
+          details caption figure figcaption
+          abbr bdo cite dfn mark small span time wbr
+        ].freeze,
+        remove_contents: ['script'].freeze,
+        attributes: {
+          'a'          => ['href'].freeze,
+          'img'        => %w[src longdesc].freeze,
+          'div'        => %w[itemscope itemtype].freeze,
+          'blockquote' => ['cite'].freeze,
+          'del'        => ['cite'].freeze,
+          'ins'        => ['cite'].freeze,
+          'q'          => ['cite'].freeze,
+          all: %w[abbr accept accept-charset
+                  accesskey action align alt
+                  aria-describedby aria-hidden aria-label aria-labelledby
+                  axis border cellpadding cellspacing char
+                  charoff charset checked
+                  clear cols colspan color
+                  compact coords datetime dir
+                  disabled enctype for frame
+                  headers height hreflang
+                  hspace ismap label lang
+                  maxlength media method
+                  multiple name nohref noshade
+                  nowrap open prompt readonly rel rev
+                  rows rowspan rules scope
+                  selected shape size span
+                  start summary tabindex target
+                  title type usemap valign value
+                  vspace width itemprop].freeze
+        }.freeze,
+        protocols: {
+          'a'          => { 'href' => ANCHOR_SCHEMES }.freeze,
+          'blockquote' => { 'cite' => ['http', 'https', :relative].freeze },
+          'del'        => { 'cite' => ['http', 'https', :relative].freeze },
+          'ins'        => { 'cite' => ['http', 'https', :relative].freeze },
+          'q'          => { 'cite' => ['http', 'https', :relative].freeze },
+          'img'        => {
+            'src'      => ['http', 'https', :relative].freeze,
+            'longdesc' => ['http', 'https', :relative].freeze
+          }.freeze
+        },
+        transformers: [
+          # Top-level <li> elements are removed because they can break out of
+          # containing markup.
+          lambda { |env|
+            name = env[:node_name]
+            node = env[:node]
+            if name == LIST_ITEM && node.ancestors.none? { |n| LISTS.include?(n.name) }
+              node.replace(node.children)
+            end
+          },
+
+          # Table child elements that are not contained by a <table> are removed.
+          lambda { |env|
+            name = env[:node_name]
+            node = env[:node]
+            if (TABLE_SECTIONS.include?(name) || TABLE_ITEMS.include?(name)) && node.ancestors.none? { |n| n.name == TABLE }
+              node.replace(node.children)
+            end
+          }
+        ].freeze
+      }.freeze
+
+      # A more limited sanitization whitelist. This includes all attributes,
+      # protocols, and transformers from WHITELIST but with a more locked down
+      # set of allowed elements.
+      LIMITED = WHITELIST.merge(
+        elements: %w[b i strong em a pre code img ins del sup sub mark abbr p ol ul li]
+      )
+
+      # Strip all HTML tags from the document.
+      FULL = { elements: [] }.freeze
+
+      # Sanitize markup using the Sanitize library.
+      def call
+        Sanitize.clean_node!(doc, whitelist)
+      end
+
+      # The whitelist to use when sanitizing. This can be passed in the context
+      # hash to the filter but defaults to WHITELIST constant value above.
+      def whitelist
+        whitelist = context[:whitelist] || WHITELIST
+        anchor_schemes = context[:anchor_schemes]
+        return whitelist unless anchor_schemes
+        whitelist = whitelist.dup
+        whitelist[:protocols] = (whitelist[:protocols] || {}).dup
+        whitelist[:protocols]['a'] = (whitelist[:protocols]['a'] || {}).merge('href' => anchor_schemes)
+        whitelist
+      end
+    end
+  end
+end

data/lib/html/pipeline-plus/syntax_highlight_filter.rb

@@ -0,0 +1,44 @@
+HTML::Pipeline.require_dependency('rouge', 'SyntaxHighlightFilter')
+
+module HTML
+  class Pipeline
+    # HTML Filter that syntax highlights code blocks wrapped
+    # in <pre lang="...">.
+    class SyntaxHighlightFilter < Filter
+      def initialize(*args)
+        super(*args)
+        @formatter = Rouge::Formatters::HTML.new
+      end
+
+      def call
+        doc.search('pre').each do |node|
+          default = context[:highlight] && context[:highlight].to_s
+          next unless lang = node['lang'] || default
+          next unless lexer = lexer_for(lang)
+          text = node.inner_text
+
+          html = highlight_with_timeout_handling(text, lang)
+          next if html.nil?
+
+          node.inner_html = html
+          klass = node['class']
+          scope = context[:scope] || "highlight-#{lang}"
+          klass = [klass, scope].compact.join ' '
+
+          node['class'] = klass
+        end
+        doc
+      end
+
+      def highlight_with_timeout_handling(text, lang)
+        Rouge.highlight(text, lang, @formatter)
+      rescue Timeout::Error => _
+        nil
+      end
+
+      def lexer_for(lang)
+        Rouge::Lexer.find(lang)
+      end
+    end
+  end
+end

data/lib/html/pipeline-plus/text_filter.rb

@@ -0,0 +1,14 @@
+module HTML
+  class Pipeline
+    class TextFilter < Filter
+      attr_reader :text
+
+      def initialize(text, context = nil, result = nil)
+        raise TypeError, 'text cannot be HTML' if text.is_a?(DocumentFragment)
+        # Ensure that this is always a string
+        @text = text.respond_to?(:to_str) ? text.to_str : text.to_s
+        super nil, context, result
+      end
+    end
+  end
+end

data/lib/html/pipeline-plus/textile_filter.rb

@@ -0,0 +1,23 @@
+HTML::Pipeline.require_dependency('redcloth', 'RedCloth')
+
+module HTML
+  class Pipeline
+    # HTML Filter that converts Textile text into HTML and converts into a
+    # DocumentFragment. This is different from most filters in that it can take a
+    # non-HTML as input. It must be used as the first filter in a pipeline.
+    #
+    # Context options:
+    #   :autolink => false    Disable autolinking URLs
+    #
+    # This filter does not write any additional information to the context hash.
+    #
+    # NOTE This filter is provided for really old comments only. It probably
+    # shouldn't be used for anything new.
+    class TextileFilter < TextFilter
+      # Convert Textile to HTML and convert into a DocumentFragment.
+      def call
+        RedCloth.new(@text).to_html
+      end
+    end
+  end
+end

data/lib/html/pipeline-plus/toc_filter.rb

@@ -0,0 +1,67 @@
+HTML::Pipeline.require_dependency('escape_utils', 'TableOfContentsFilter')
+
+module HTML
+  class Pipeline
+    # HTML filter that adds an 'id' attribute to all headers
+    # in a document, so they can be accessed from a table of contents.
+    #
+    # Generates the Table of Contents, with links to each header.
+    #
+    # Examples
+    #
+    #  TocPipeline =
+    #    HTML::Pipeline.new [
+    #      HTML::Pipeline::TableOfContentsFilter
+    #    ]
+    #  # => #<HTML::Pipeline:0x007fc13c4528d8...>
+    #  orig = %(<h1>Ice cube</h1><p>is not for the pop chart</p>)
+    #  # => "<h1>Ice cube</h1><p>is not for the pop chart</p>"
+    #  result = {}
+    #  # => {}
+    #  TocPipeline.call(orig, {}, result)
+    #  # => {:toc=> ...}
+    #  result[:toc]
+    #  # => "<ul class=\"section-nav\">\n<li><a href=\"#ice-cube\">...</li><ul>"
+    #  result[:output].to_s
+    #  # => "<h1>\n<a id=\"ice-cube\" class=\"anchor\" href=\"#ice-cube\">..."
+    class TableOfContentsFilter < Filter
+      PUNCTUATION_REGEXP = RUBY_VERSION > '1.9' ? /[^\p{Word}\- ]/u : /[^\w\- ]/
+
+      # The icon that will be placed next to an anchored rendered markdown header
+      def anchor_icon
+        context[:anchor_icon] || '<span aria-hidden="true" class="octicon octicon-link"></span>'
+      end
+
+      def call
+        result[:toc] = ''
+
+        headers = Hash.new(0)
+        doc.css('h1, h2, h3, h4, h5, h6').each do |node|
+          text = node.text
+          id = ascii_downcase(text)
+          id.gsub!(PUNCTUATION_REGEXP, '') # remove punctuation
+          id.tr!(' ', '-') # replace spaces with dash
+
+          uniq = headers[id] > 0 ? "-#{headers[id]}" : ''
+          headers[id] += 1
+          if header_content = node.children.first
+            result[:toc] << %(<li><a href="##{id}#{uniq}">#{EscapeUtils.escape_html(text)}</a></li>\n)
+            header_content.add_previous_sibling(%(<a id="#{id}#{uniq}" class="anchor" href="##{id}#{uniq}" aria-hidden="true">#{anchor_icon}</a>))
+          end
+        end
+        result[:toc] = %(<ul class="section-nav">\n#{result[:toc]}</ul>) unless result[:toc].empty?
+        doc
+      end
+
+      if RUBY_VERSION >= '2.4'
+        def ascii_downcase(str)
+          str.downcase(:ascii)
+        end
+      else
+        def ascii_downcase(str)
+          str.downcase
+        end
+      end
+    end
+  end
+end

data/lib/html/pipeline-plus/version.rb

@@ -0,0 +1,5 @@
+module HTML
+  class Pipeline
+    VERSION = '2.10.1'.freeze
+  end
+end

data/lib/html/pipeline-plus.rb

@@ -0,0 +1,207 @@
+require 'nokogiri'
+require 'active_support/xml_mini/nokogiri' # convert Documents to hashes
+
+module HTML
+  # GitHub HTML processing filters and utilities. This module includes a small
+  # framework for defining DOM based content filters and applying them to user
+  # provided content.
+  #
+  # See HTML::Pipeline::Filter for information on building filters.
+  #
+  # Construct a Pipeline for running multiple HTML filters.  A pipeline is created once
+  # with one to many filters, and it then can be `call`ed many times over the course
+  # of its lifetime with input.
+  #
+  # filters         - Array of Filter objects. Each must respond to call(doc,
+  #                   context) and return the modified DocumentFragment or a
+  #                   String containing HTML markup. Filters are performed in the
+  #                   order provided.
+  # default_context - The default context hash. Values specified here will be merged
+  #                   into values from the each individual pipeline run.  Can NOT be
+  #                   nil.  Default: empty Hash.
+  # result_class    - The default Class of the result object for individual
+  #                   calls.  Default: Hash.  Protip:  Pass in a Struct to get
+  #                   some semblance of type safety.
+  class Pipeline
+    autoload :VERSION,               'html/pipeline/version'
+    autoload :Filter,                'html/pipeline/filter'
+    autoload :AbsoluteSourceFilter,  'html/pipeline/absolute_source_filter'
+    autoload :BodyContent,           'html/pipeline/body_content'
+    autoload :AutolinkFilter,        'html/pipeline/autolink_filter'
+    autoload :CamoFilter,            'html/pipeline/camo_filter'
+    autoload :EmailReplyFilter,      'html/pipeline/email_reply_filter'
+    autoload :EmojiFilter,           'html/pipeline/emoji_filter'
+    autoload :HttpsFilter,           'html/pipeline/https_filter'
+    autoload :ImageFilter,           'html/pipeline/image_filter'
+    autoload :ImageMaxWidthFilter,   'html/pipeline/image_max_width_filter'
+    autoload :MarkdownFilter,        'html/pipeline/markdown_filter'
+    autoload :MentionFilter,         'html/pipeline/@mention_filter'
+    autoload :PlainTextInputFilter,  'html/pipeline/plain_text_input_filter'
+    autoload :SanitizationFilter,    'html/pipeline/sanitization_filter'
+    autoload :SyntaxHighlightFilter, 'html/pipeline/syntax_highlight_filter'
+    autoload :TextileFilter,         'html/pipeline/textile_filter'
+    autoload :TableOfContentsFilter, 'html/pipeline/toc_filter'
+    autoload :TextFilter,            'html/pipeline/text_filter'
+
+    class MissingDependencyError < RuntimeError; end
+    def self.require_dependency(name, requirer)
+      require name
+    rescue LoadError => e
+      raise MissingDependencyError,
+            "Missing dependency '#{name}' for #{requirer}. See README.md for details.\n#{e.class.name}: #{e}"
+    end
+
+    # Our DOM implementation.
+    DocumentFragment = Nokogiri::HTML::DocumentFragment
+
+    # Parse a String into a DocumentFragment object. When a DocumentFragment is
+    # provided, return it verbatim.
+    def self.parse(document_or_html)
+      document_or_html ||= ''
+      if document_or_html.is_a?(String)
+        DocumentFragment.parse(document_or_html)
+      else
+        document_or_html
+      end
+    end
+
+    # Public: Returns an Array of Filter objects for this Pipeline.
+    attr_reader :filters
+
+    # Public: Instrumentation service for the pipeline.
+    # Set an ActiveSupport::Notifications compatible object to enable.
+    attr_accessor :instrumentation_service
+
+    # Public: String name for this Pipeline. Defaults to Class name.
+    attr_writer :instrumentation_name
+    def instrumentation_name
+      return @instrumentation_name if defined?(@instrumentation_name)
+      @instrumentation_name = self.class.name
+    end
+
+    class << self
+      # Public: Default instrumentation service for new pipeline objects.
+      attr_accessor :default_instrumentation_service
+    end
+
+    def initialize(filters, default_context = {}, result_class = nil)
+      raise ArgumentError, 'default_context cannot be nil' if default_context.nil?
+      @filters = filters.flatten.freeze
+      @default_context = default_context.freeze
+      @result_class = result_class || Hash
+      @instrumentation_service = self.class.default_instrumentation_service
+    end
+
+    # Apply all filters in the pipeline to the given HTML.
+    #
+    # html    - A String containing HTML or a DocumentFragment object.
+    # context - The context hash passed to each filter. See the Filter docs
+    #           for more info on possible values. This object MUST NOT be modified
+    #           in place by filters.  Use the Result for passing state back.
+    # result  - The result Hash passed to each filter for modification.  This
+    #           is where Filters store extracted information from the content.
+    #
+    # Returns the result Hash after being filtered by this Pipeline.  Contains an
+    # :output key with the DocumentFragment or String HTML markup based on the
+    # output of the last filter in the pipeline.
+    def call(html, context = {}, result = nil)
+      context = @default_context.merge(context)
+      context = context.freeze
+      result ||= @result_class.new
+      payload = default_payload filters: @filters.map(&:name),
+                                context: context, result: result
+      instrument 'call_pipeline.html_pipeline', payload do
+        result[:output] =
+          @filters.inject(html) do |doc, filter|
+            perform_filter(filter, doc, context, result)
+          end
+      end
+      result
+    end
+
+    # Internal: Applies a specific filter to the supplied doc.
+    #
+    # The filter is instrumented.
+    #
+    # Returns the result of the filter.
+    def perform_filter(filter, doc, context, result)
+      payload = default_payload filter: filter.name,
+                                context: context, result: result
+      instrument 'call_filter.html_pipeline', payload do
+        filter.call(doc, context, result)
+      end
+    end
+
+    # Like call but guarantee the value returned is a DocumentFragment.
+    # Pipelines may return a DocumentFragment or a String. Callers that need a
+    # DocumentFragment should use this method.
+    def to_document(input, context = {}, result = nil)
+      result = call(input, context, result)
+      HTML::Pipeline.parse(result[:output])
+    end
+
+    # Like call but guarantee the value returned is a string of HTML markup.
+    def to_html(input, context = {}, result = nil)
+      result = call(input, context, result = nil)
+      output = result[:output]
+      if output.respond_to?(:to_html)
+        output.to_html
+      else
+        output.to_s
+      end
+    end
+
+    # Public: setup instrumentation for this pipeline.
+    #
+    # Returns nothing.
+    def setup_instrumentation(name = nil, service = nil)
+      self.instrumentation_name = name
+      self.instrumentation_service =
+        service || self.class.default_instrumentation_service
+    end
+
+    # Internal: if the `instrumentation_service` object is set, instruments the
+    # block, otherwise the block is ran without instrumentation.
+    #
+    # Returns the result of the provided block.
+    def instrument(event, payload = nil)
+      payload ||= default_payload
+      return yield(payload) unless instrumentation_service
+      instrumentation_service.instrument event, payload do |payload|
+        yield payload
+      end
+    end
+
+    # Internal: Default payload for instrumentation.
+    #
+    # Accepts a Hash of additional payload data to be merged.
+    #
+    # Returns a Hash.
+    def default_payload(payload = {})
+      { pipeline: instrumentation_name }.merge(payload)
+    end
+  end
+end
+
+# XXX nokogiri monkey patches for 1.8
+unless ''.respond_to?(:force_encoding)
+  class Nokogiri::XML::Node
+    # Work around an issue with utf-8 encoded data being erroneously converted to
+    # ... some other shit when replacing text nodes. See 'utf-8 output 2' in
+    # user_content_test.rb for details.
+    def replace_with_encoding_fix(replacement)
+      if replacement.respond_to?(:to_str)
+        replacement = document.fragment("<div>#{replacement}</div>").children.first.children
+      end
+      replace_without_encoding_fix(replacement)
+    end
+
+    alias replace_without_encoding_fix replace
+    alias replace replace_with_encoding_fix
+
+    def swap(replacement)
+      replace(replacement)
+      self
+    end
+  end
+end

data/test.txt

@@ -0,0 +1,13 @@
+
+install depend library:
+
+gem install appraisal
+gem install email_reply_parser
+gem install RedCloth
+gem install RedCloth --platform x64_mingw32
+
+bundle install
+
+test:
+
+ruby -Ilib:test ./test/test_helper.rb

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: html-pipeline-plus
+version: !ruby/object:Gem::Version
+  version: 2.10.1
+platform: ruby
+authors:
+- Ryan Tomayko
+- Jerry Cheung
+- Garen J. Torikian
+- shines77
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-02-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.4'
+description: GitHub HTML processing filters and utilities
+email:
+- ryan@github.com
+- jerry@github.com
+- gjtorikian@gmail.com
+- gz_shines@msn.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Appraisals
+- CHANGELOG.md
+- CONTRIBUTING.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/html-pipeline-plus
+- html-pipeline-plus.gemspec
+- lib/html/pipeline-plus.rb
+- lib/html/pipeline-plus/@mention_filter.rb
+- lib/html/pipeline-plus/absolute_source_filter.rb
+- lib/html/pipeline-plus/autolink_filter.rb
+- lib/html/pipeline-plus/body_content.rb
+- lib/html/pipeline-plus/camo_filter.rb
+- lib/html/pipeline-plus/email_reply_filter.rb
+- lib/html/pipeline-plus/emoji_filter.rb
+- lib/html/pipeline-plus/filter.rb
+- lib/html/pipeline-plus/https_filter.rb
+- lib/html/pipeline-plus/image_filter.rb
+- lib/html/pipeline-plus/image_max_width_filter.rb
+- lib/html/pipeline-plus/markdown_filter.rb
+- lib/html/pipeline-plus/plain_text_input_filter.rb
+- lib/html/pipeline-plus/sanitization_filter.rb
+- lib/html/pipeline-plus/syntax_highlight_filter.rb
+- lib/html/pipeline-plus/text_filter.rb
+- lib/html/pipeline-plus/textile_filter.rb
+- lib/html/pipeline-plus/toc_filter.rb
+- lib/html/pipeline-plus/version.rb
+- test.txt
+homepage: https://github.com/shines77/html-pipeline-plus/
+licenses:
+- MIT
+metadata: {}
+post_install_message: |
+  -------------------------------------------------
+  Thank you for installing html-pipeline-plus!
+  You must bundle Filter gem dependencies.
+  See html-pipeline-plus README.md for more details.
+  https://github.com/shines77/html-pipeline-plus#dependencies
+  -------------------------------------------------
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.2
+signing_key: 
+specification_version: 4
+summary: Helpers for processing content through a chain of filters
+test_files: []