geothird-html-pipeline 0.0.12

data/geothird-html-pipeline.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path("../lib/html/pipeline/version", __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "geothird-html-pipeline"
+  gem.version       = HTML::Pipeline::VERSION
+  gem.license       = "MIT"
+  gem.authors       = ["Ryan Tomayko", "Jerry Cheung"]
+  gem.email         = ["ryan@github.com", "jerry@github.com"]
+  gem.description   = %q{GitHub HTML processing filters and utilities}
+  gem.summary       = %q{Helpers for processing content through a chain of filters}
+  gem.homepage      = "https://github.com/jch/html-pipeline"
+
+  gem.files         = `git ls-files`.split $/
+  gem.test_files    = gem.files.grep(%r{^test})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "gemoji",          "~> 1.0"
+  gem.add_dependency "nokogiri",        "~> 1.4"
+  gem.add_dependency "github-markdown", "~> 0.5"
+  gem.add_dependency "sanitize",        "~> 2.0"
+  gem.add_dependency "rinku",           "~> 1.7"
+  gem.add_dependency "escape_utils",    "~> 0.3"
+  gem.add_dependency "activesupport",   ">= 2"
+
+  gem.add_development_dependency "geothird-linguist", "~> 2.6.8"
+end

data/lib/html/pipeline.rb

@@ -0,0 +1,198 @@
+require "nokogiri"
+require "active_support/xml_mini/nokogiri" # convert Documents to hashes
+require "escape_utils"
+
+module HTML
+  # GitHub HTML processing filters and utilities. This module includes a small
+  # framework for defining DOM based content filters and applying them to user
+  # provided content.
+  #
+  # See HTML::Pipeline::Filter for information on building filters.
+  #
+  # Construct a Pipeline for running multiple HTML filters.  A pipeline is created once
+  # with one to many filters, and it then can be `call`ed many times over the course
+  # of its lifetime with input.
+  #
+  # filters         - Array of Filter objects. Each must respond to call(doc,
+  #                   context) and return the modified DocumentFragment or a
+  #                   String containing HTML markup. Filters are performed in the
+  #                   order provided.
+  # default_context - The default context hash. Values specified here will be merged
+  #                   into values from the each individual pipeline run.  Can NOT be
+  #                   nil.  Default: empty Hash.
+  # result_class    - The default Class of the result object for individual
+  #                   calls.  Default: Hash.  Protip:  Pass in a Struct to get
+  #                   some semblance of type safety.
+  class Pipeline
+    autoload :VERSION,               'html/pipeline/version'
+    autoload :Filter,                'html/pipeline/filter'
+    autoload :AbsoluteSourceFilter,  'html/pipeline/absolute_source_filter'
+    autoload :BodyContent,           'html/pipeline/body_content'
+    autoload :AutolinkFilter,        'html/pipeline/autolink_filter'
+    autoload :CamoFilter,            'html/pipeline/camo_filter'
+    autoload :EmailReplyFilter,      'html/pipeline/email_reply_filter'
+    autoload :EmojiFilter,           'html/pipeline/emoji_filter'
+    autoload :HttpsFilter,           'html/pipeline/https_filter'
+    autoload :ImageMaxWidthFilter,   'html/pipeline/image_max_width_filter'
+    autoload :MarkdownFilter,        'html/pipeline/markdown_filter'
+    autoload :MentionFilter,         'html/pipeline/@mention_filter'
+    autoload :PlainTextInputFilter,  'html/pipeline/plain_text_input_filter'
+    autoload :SanitizationFilter,    'html/pipeline/sanitization_filter'
+    autoload :SyntaxHighlightFilter, 'html/pipeline/syntax_highlight_filter'
+    autoload :TextileFilter,         'html/pipeline/textile_filter'
+    autoload :TableOfContentsFilter, 'html/pipeline/toc_filter'
+    autoload :TextFilter,            'html/pipeline/text_filter'
+
+    # Our DOM implementation.
+    DocumentFragment = Nokogiri::HTML::DocumentFragment
+
+    # Parse a String into a DocumentFragment object. When a DocumentFragment is
+    # provided, return it verbatim.
+    def self.parse(document_or_html)
+      document_or_html ||= ''
+      if document_or_html.is_a?(String)
+        DocumentFragment.parse(document_or_html)
+      else
+        document_or_html
+      end
+    end
+
+    # Public: Returns an Array of Filter objects for this Pipeline.
+    attr_reader :filters
+
+    # Public: Instrumentation service for the pipeline.
+    # Set an ActiveSupport::Notifications compatible object to enable.
+    attr_accessor :instrumentation_service
+
+    # Public: String name for this Pipeline. Defaults to Class name.
+    attr_writer :instrumentation_name
+    def instrumentation_name
+      @instrumentation_name || self.class.name
+    end
+
+    class << self
+      # Public: Default instrumentation service for new pipeline objects.
+      attr_accessor :default_instrumentation_service
+    end
+
+    def initialize(filters, default_context = {}, result_class = nil)
+      raise ArgumentError, "default_context cannot be nil" if default_context.nil?
+      @filters = filters.flatten.freeze
+      @default_context = default_context.freeze
+      @result_class = result_class || Hash
+      @instrumentation_service = self.class.default_instrumentation_service
+    end
+
+    # Apply all filters in the pipeline to the given HTML.
+    #
+    # html    - A String containing HTML or a DocumentFragment object.
+    # context - The context hash passed to each filter. See the Filter docs
+    #           for more info on possible values. This object MUST NOT be modified
+    #           in place by filters.  Use the Result for passing state back.
+    # result  - The result Hash passed to each filter for modification.  This
+    #           is where Filters store extracted information from the content.
+    #
+    # Returns the result Hash after being filtered by this Pipeline.  Contains an
+    # :output key with the DocumentFragment or String HTML markup based on the
+    # output of the last filter in the pipeline.
+    def call(html, context = {}, result = nil)
+      context = @default_context.merge(context)
+      context = context.freeze
+      result ||= @result_class.new
+      payload = default_payload :filters => @filters.map(&:name),
+        :context => context, :result => result
+      instrument "call_pipeline.html_pipeline", payload do
+        result[:output] =
+          @filters.inject(html) do |doc, filter|
+            perform_filter(filter, doc, context, result)
+          end
+      end
+      result
+    end
+
+    # Internal: Applies a specific filter to the supplied doc.
+    #
+    # The filter is instrumented.
+    #
+    # Returns the result of the filter.
+    def perform_filter(filter, doc, context, result)
+      payload = default_payload :filter => filter.name,
+        :context => context, :result => result
+      instrument "call_filter.html_pipeline", payload do
+        filter.call(doc, context, result)
+      end
+    end
+
+    # Like call but guarantee the value returned is a DocumentFragment.
+    # Pipelines may return a DocumentFragment or a String. Callers that need a
+    # DocumentFragment should use this method.
+    def to_document(input, context = {}, result = nil)
+      result = call(input, context, result)
+      HTML::Pipeline.parse(result[:output])
+    end
+
+    # Like call but guarantee the value returned is a string of HTML markup.
+    def to_html(input, context = {}, result = nil)
+      result = call(input, context, result = nil)
+      output = result[:output]
+      if output.respond_to?(:to_html)
+        output.to_html
+      else
+        output.to_s
+      end
+    end
+
+    # Public: setup instrumentation for this pipeline.
+    #
+    # Returns nothing.
+    def setup_instrumentation(name = nil, service = nil)
+      self.instrumentation_name = name
+      self.instrumentation_service =
+        service || self.class.default_instrumentation_service
+    end
+
+    # Internal: if the `instrumentation_service` object is set, instruments the
+    # block, otherwise the block is ran without instrumentation.
+    #
+    # Returns the result of the provided block.
+    def instrument(event, payload = nil)
+      payload ||= default_payload
+      return yield(payload) unless instrumentation_service
+      instrumentation_service.instrument event, payload do |payload|
+        yield payload
+      end
+    end
+
+    # Internal: Default payload for instrumentation.
+    #
+    # Accepts a Hash of additional payload data to be merged.
+    #
+    # Returns a Hash.
+    def default_payload(payload = {})
+      {:pipeline => instrumentation_name}.merge(payload)
+    end
+  end
+end
+
+# XXX nokogiri monkey patches for 1.8
+if not ''.respond_to?(:force_encoding)
+  class Nokogiri::XML::Node
+    # Work around an issue with utf-8 encoded data being erroneously converted to
+    # ... some other shit when replacing text nodes. See 'utf-8 output 2' in
+    # user_content_test.rb for details.
+    def replace_with_encoding_fix(replacement)
+      if replacement.respond_to?(:to_str)
+        replacement = document.fragment("<div>#{replacement}</div>").children.first.children
+      end
+      replace_without_encoding_fix(replacement)
+    end
+
+    alias_method :replace_without_encoding_fix, :replace
+    alias_method :replace, :replace_with_encoding_fix
+
+    def swap(replacement)
+      replace(replacement)
+      self
+    end
+  end
+end

data/lib/html/pipeline/@mention_filter.rb

@@ -0,0 +1,121 @@
+require 'set'
+
+module HTML
+  class Pipeline
+    # HTML filter that replaces @user mentions with links. Mentions within <pre>,
+    # <code>, and <a> elements are ignored. Mentions that reference users that do
+    # not exist are ignored.
+    #
+    # Context options:
+    #   :base_url - Used to construct links to user profile pages for each
+    #               mention.
+    #   :info_url - Used to link to "more info" when someone mentions @mention
+    #               or @mentioned.
+    #
+    class MentionFilter < Filter
+      # Public: Find user @mentions in text.  See
+      # MentionFilter#mention_link_filter.
+      #
+      #   MentionFilter.mentioned_logins_in(text) do |match, login, is_mentioned|
+      #     "<a href=...>#{login}</a>"
+      #   end
+      #
+      # text - String text to search.
+      #
+      # Yields the String match, the String login name, and a Boolean determining
+      # if the match = "@mention[ed]".  The yield's return replaces the match in
+      # the original text.
+      #
+      # Returns a String replaced with the return of the block.
+      def self.mentioned_logins_in(text)
+        text.gsub MentionPattern do |match|
+          login = $1
+          yield match, login, MentionLogins.include?(login.downcase)
+        end
+      end
+
+      # Pattern used to extract @mentions from text.
+      MentionPattern = /
+        (?:^|\W)                   # beginning of string or non-word char
+        @((?>[a-z0-9][a-z0-9-]*))  # @username
+        (?!\/)                     # without a trailing slash
+        (?=
+          \.+[ \t\W]|              # dots followed by space or non-word character
+          \.+$|                    # dots at end of line
+          [^0-9a-zA-Z_.]|          # non-word character except dot
+          $                        # end of line
+        )
+      /ix
+
+      # List of username logins that, when mentioned, link to the blog post
+      # about @mentions instead of triggering a real mention.
+      MentionLogins = %w(
+        mention
+        mentions
+        mentioned
+        mentioning
+      )
+
+      # Don't look for mentions in text nodes that are children of these elements
+      IGNORE_PARENTS = %w(pre code a).to_set
+
+      def call
+        result[:mentioned_usernames] ||= []
+
+        doc.search('text()').each do |node|
+          content = node.to_html
+          next if !content.include?('@')
+          next if has_ancestor?(node, IGNORE_PARENTS)
+          html = mention_link_filter(content, base_url, info_url)
+          next if html == content
+          node.replace(html)
+        end
+        doc
+      end
+
+      # The URL to provide when someone @mentions a "mention" name, such as
+      # @mention or @mentioned, that will give them more info on mentions.
+      def info_url
+        context[:info_url] || nil
+      end
+
+      # Replace user @mentions in text with links to the mentioned user's
+      # profile page.
+      #
+      # text      - String text to replace @mention usernames in.
+      # base_url  - The base URL used to construct user profile URLs.
+      # info_url  - The "more info" URL used to link to more info on @mentions.
+      #             If nil we don't link @mention or @mentioned.
+      #
+      # Returns a string with @mentions replaced with links. All links have a
+      # 'user-mention' class name attached for styling.
+      def mention_link_filter(text, base_url='/', info_url=nil)
+        self.class.mentioned_logins_in(text) do |match, login, is_mentioned|
+          link =
+            if is_mentioned
+              link_to_mention_info(login, info_url)
+            else
+              link_to_mentioned_user(login)
+            end
+
+          link ? match.sub("@#{login}", link) : match
+        end
+      end
+
+      def link_to_mention_info(text, info_url=nil)
+        return "@#{text}" if info_url.nil?
+        "<a href='#{info_url}' class='user-mention'>" +
+        "@#{text}" +
+        "</a>"
+      end
+
+      def link_to_mentioned_user(login)
+        result[:mentioned_usernames] |= [login]
+        url = File.join(base_url, login)
+        "<a href='#{url}' class='user-mention'>" +
+        "@#{login}" +
+        "</a>"
+      end
+    end
+  end
+end

data/lib/html/pipeline/absolute_source_filter.rb

@@ -0,0 +1,48 @@
+require 'uri'
+
+module HTML
+  class Pipeline
+
+    class AbsoluteSourceFilter < Filter
+      # HTML Filter for replacing relative and root relative image URLs with
+      # fully qualified URLs
+      #
+      # This is useful if an image is root relative but should really be going
+      # through a cdn, or if the content for the page assumes the host is known
+      # i.e. scraped webpages and some RSS feeds.
+      #
+      # Context options:
+      #   :image_base_url - Base URL for image host for root relative src.
+      #   :image_subpage_url - For relative src.
+      #
+      # This filter does not write additional information to the context.
+      # This filter would need to be run before CamoFilter.
+      def call
+        doc.search("img").each do |element| 
+          next if element['src'].nil? || element['src'].empty?
+          src = element['src'].strip
+          unless src.start_with? 'http'
+            if src.start_with? '/'
+              base = image_base_url
+            else
+              base = image_subpage_url
+            end
+            element["src"] = URI.join(base, src).to_s
+          end
+        end
+        doc
+      end
+      
+      # Private: the base url you want to use
+      def image_base_url
+        context[:image_base_url] or raise "Missing context :image_base_url for #{self.class.name}"
+      end
+
+      # Private: the relative url you want to use
+      def image_subpage_url
+        context[:image_subpage_url] or raise "Missing context :image_subpage_url for #{self.class.name}"
+      end
+    
+    end
+  end
+end

data/lib/html/pipeline/autolink_filter.rb

@@ -0,0 +1,22 @@
+require 'rinku'
+
+module HTML
+  class Pipeline
+    # HTML Filter for auto_linking urls in HTML.
+    #
+    # Context options:
+    #   :autolink - boolean whether to autolink urls
+    #   :flags    - additional Rinku flags. See https://github.com/vmg/rinku
+    #
+    # This filter does not write additional information to the context.
+    class AutolinkFilter < Filter
+      def call
+        return html if context[:autolink] == false
+        flags = 0
+        flags |= context[:flags] if context[:flags]
+
+        Rinku.auto_link(html, :urls, nil, %w[a script kbd pre code], flags)
+      end
+    end
+  end
+end

data/lib/html/pipeline/body_content.rb

@@ -0,0 +1,42 @@
+module HTML
+  class Pipeline
+    # Public: Runs a String of content through an HTML processing pipeline,
+    # providing easy access to a generated DocumentFragment.
+    class BodyContent
+      attr_reader :result
+
+      # Public: Initialize a BodyContent.
+      #
+      # body     - A String body.
+      # context  - A Hash of context options for the filters.
+      # pipeline - A HTML::Pipeline object with one or more Filters.
+      def initialize(body, context, pipeline)
+        @body = body
+        @context = context
+        @pipeline = pipeline
+      end
+
+      # Public: Gets the memoized result of the body content as it passed through
+      # the Pipeline.
+      #
+      # Returns a Hash, or something similar as defined by @pipeline.result_class.
+      def result
+        @result ||= @pipeline.call @body, @context
+      end
+
+      # Public: Gets the updated body from the Pipeline result.
+      #
+      # Returns a String or DocumentFragment.
+      def output
+        @output ||= result[:output]
+      end
+
+      # Public: Parses the output into a DocumentFragment.
+      #
+      # Returns a DocumentFragment.
+      def document
+        @document ||= HTML::Pipeline.parse output
+      end
+    end
+  end
+end