html-pipeline 0.0.4

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+bin/*
+vendor/gems

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -qq libicu-dev
+
+script: "bundle exec rake"
+
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - ree

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in html-pipeline.gemspec
+gemspec
+
+group :development do
+  gem 'bundler'
+  gem 'rake'
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 GitHub Inc. and Jerry Cheung
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,128 @@
+# HTML::Pipeline [![Build Status](https://secure.travis-ci.org/jch/html-pipeline.png)](http://travis-ci.org/jch/html-pipeline)
+
+GitHub HTML processing filters and utilities. This module includes a small
+framework for defining DOM based content filters and applying them to user
+provided content.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'html-pipeline'
+```
+
+And then execute:
+
+```sh
+$ bundle
+```
+
+Or install it yourself as:
+
+```sh
+$ gem install html-pipeline
+```
+
+## Usage
+
+This library provides a handful of chainable HTML filters to transform user
+content into markup. A filter takes an HTML string or
+`Nokogiri::HTML::DocumentFragment`, optionally manipulates it, and then
+outputs the result.
+
+For example, to transform Markdown source into Markdown HTML:
+
+```ruby
+require 'html/pipeline'
+
+filter = HTML::Pipeline::MarkdownFilter.new("Hi **world**!")
+filter.call
+```
+
+Filters can be combined into a pipeline which causes each filter to hand its
+output to the next filter's input. So if you wanted to have content be
+filtered through Markdown and be syntax highlighted, you can create the
+following pipeline:
+
+```ruby
+pipeline = HTML::Pipeline.new [
+  HTML::Pipeline::MarkdownFilter,
+  HTML::Pipeline::SyntaxHighlightFilter
+]
+result = pipeline.call <<CODE
+This is *great*:
+
+    some_code(:first)
+
+CODE
+result[:output].to_s
+```
+
+Prints:
+
+```html
+<p>This is <em>great</em>:</p>
+
+<div class="highlight">
+<pre><span class="n">some_code</span><span class="p">(</span><span class="ss">:first</span><span class="p">)</span>
+</pre>
+</div>
+```
+
+Some filters take an optional **context** and/or **result** hash. These are
+used to pass around arguments and metadata between filters in a pipeline. For
+example, if you want don't want to use GitHub formatted Markdown, you can
+pass an option in the context hash:
+
+```ruby
+filter = HTML::Pipeline::MarkdownFilter.new("Hi **world**!", :gfm => false)
+filter.call
+```
+
+## Filters
+
+* `MentionFilter` - replace `@user` mentions with links
+* `AutoLinkFilter` - auto_linking urls in HTML
+* `CamoFilter` - replace http image urls with [camo-fied](https://github.com/github/camo) https versions
+* `EmailReplyFilter` - util filter for working with emails
+* `EmojiFilter` - everyone loves [emoji](http://www.emoji-cheat-sheet.com/)!
+* `ImageMaxWidthFilter` - link to full size image for large images
+* `MarkdownFilter` - convert markdown to html
+* `PlainTextInputFilter` - html escape text and wrap the result in a div
+* `SanitizationFilter` - whitelist santize user markup
+* `SyntaxHighlightFilter` - code syntax highlighter with [linguist](https://github.com/github/linguist)
+* `TextileFilter` - convert textile to html
+* `TableOfContentsFilter` - anchor headings with name attributes
+
+## Development Setup
+
+```sh
+bundle
+rake test
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+
+## TODO
+
+* test whether emoji filter works on heroku
+* test whether nokogiri monkey patch is still necessary
+
+## Contributors
+
+* [Aman Gupta](mailto:aman@tmm1.net)
+* [Jake Boxer](mailto:jake@github.com)
+* [Joshua Peek](mailto:josh@joshpeek.com)
+* [Kyle Neath](mailto:kneath@gmail.com)
+* [Rob Sanheim](mailto:rsanheim@gmail.com)
+* [Simon Rozet](mailto:simon@rozet.name)
+* [Vicent Martí](mailto:tanoku@gmail.com)
+* [Risk :danger: Olson](mailto:technoweenie@gmail.com)

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = true
+end
+
+task :default => :test

data/html-pipeline.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/html/pipeline/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "html-pipeline"
+  gem.version       = HTML::Pipeline::VERSION
+  gem.authors       = ["Ryan Tomayko", "Jerry Cheung"]
+  gem.email         = ["ryan@github.com", "jerry@github.com"]
+  gem.description   = %q{GitHub HTML processing filters and utilities}
+  gem.summary       = %q{Helpers for processing content through a chain of filters}
+  gem.homepage      = "https://github.com/jch/html-pipeline"
+
+  gem.files         = `git ls-files`.split $/
+  gem.test_files    = gem.files.grep(%r{^test})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'gemoji',          '~> 1.1.1'
+  gem.add_dependency 'nokogiri',        '~> 1.4'
+  gem.add_dependency 'github-markdown', '~> 0.5'
+  gem.add_dependency 'sanitize',        '~> 2.0'
+  gem.add_dependency 'github-linguist', '~> 2.1'
+  gem.add_dependency 'rinku',           '~> 1.7'
+  gem.add_dependency 'escape_utils',    '~> 0.2'
+  gem.add_dependency 'activesupport',   '>= 2'
+end

data/lib/html/pipeline.rb

@@ -0,0 +1,130 @@
+require "nokogiri"
+require "active_support/xml_mini/nokogiri" # convert Documents to hashes
+require "escape_utils"
+
+module HTML
+  # GitHub HTML processing filters and utilities. This module includes a small
+  # framework for defining DOM based content filters and applying them to user
+  # provided content.
+  #
+  # See HTML::Pipeline::Filter for information on building filters.
+  #
+  # Contruct a Pipeline for running multiple HTML filters.  A pipeline is created once
+  # with one to many filters, and is then can be `call`ed many times over the course
+  # of its lifetime with input.
+  #
+  # filters         - Array of Filter objects. Each must respond to call(doc,
+  #                   context) and return the modified DocumentFragment or a
+  #                   String containing HTML markup. Filters are performed in the
+  #                   order provided.
+  # default_context - The default context hash. Values specified here will be merged
+  #                   into values from the each individual pipeline run.  Can NOT be
+  #                   nil.  Default: empty Hash.
+  # result_class    - The default Class of the result object for individual
+  #                   calls.  Default: Hash.  Protip:  Pass in a Struct to get
+  #                   some semblence of type safety.
+  class Pipeline
+    autoload :VERSION,               'html/pipeline/version'
+    autoload :Pipeline,              'html/pipeline/pipeline'
+    autoload :Filter,                'html/pipeline/filter'
+    autoload :BodyContent,           'html/pipeline/body_content'
+    autoload :AutolinkFilter,        'html/pipeline/autolink_filter'
+    autoload :CamoFilter,            'html/pipeline/camo_filter'
+    autoload :EmailReplyFilter,      'html/pipeline/email_reply_filter'
+    autoload :EmojiFilter,           'html/pipeline/emoji_filter'
+    autoload :HttpsFilter,           'html/pipeline/https_filter'
+    autoload :ImageMaxWidthFilter,   'html/pipeline/image_max_width_filter'
+    autoload :MarkdownFilter,        'html/pipeline/markdown_filter'
+    autoload :MentionFilter,         'html/pipeline/@mention_filter'
+    autoload :PlainTextInputFilter,  'html/pipeline/plain_text_input_filter'
+    autoload :SanitizationFilter,    'html/pipeline/sanitization_filter'
+    autoload :SyntaxHighlightFilter, 'html/pipeline/syntax_highlight_filter'
+    autoload :TextileFilter,         'html/pipeline/textile_filter'
+    autoload :TableOfContentsFilter, 'html/pipeline/toc_filter'
+    autoload :TextFilter,            'html/pipeline/text_filter'
+
+    # Our DOM implementation.
+    DocumentFragment = Nokogiri::HTML::DocumentFragment
+
+    # Parse a String into a DocumentFragment object. When a DocumentFragment is
+    # provided, return it verbatim.
+    def self.parse(document_or_html)
+      document_or_html ||= ''
+      if document_or_html.is_a?(String)
+        DocumentFragment.parse(document_or_html)
+      else
+        document_or_html
+      end
+    end
+
+    # Public: Returns an Array of Filter objects for this Pipeline.
+    attr_reader :filters
+
+    def initialize(filters, default_context = {}, result_class = nil)
+      raise ArgumentError, "default_context cannot be nil" if default_context.nil?
+      @filters = filters.flatten.freeze
+      @default_context = default_context.freeze
+      @result_class = result_class || Hash
+    end
+
+    # Apply all filters in the pipeline to the given HTML.
+    #
+    # html    - A String containing HTML or a DocumentFragment object.
+    # context - The context hash passed to each filter. See the Filter docs
+    #           for more info on possible values. This object MUST NOT be modified
+    #           in place by filters.  Use the Result for passing state back.
+    # result  - The result Hash passed to each filter for modification.  This
+    #           is where Filters store extracted information from the content.
+    #
+    # Returns the result Hash after being filtered by this Pipeline.  Contains an
+    # :output key with the DocumentFragment or String HTML markup based on the
+    # output of the last filter in the pipeline.
+    def call(html, context = {}, result = nil)
+      context = @default_context.merge(context)
+      context = context.freeze
+      result ||= @result_class.new
+      result[:output] = @filters.inject(html) { |doc, filter| filter.call(doc, context, result) }
+      result
+    end
+
+    # Like call but guarantee the value returned is a DocumentFragment.
+    # Pipelines may return a DocumentFragment or a String. Callers that need a
+    # DocumentFragment should use this method.
+    def to_document(input, context = {}, result = nil)
+      result = call(input, context, result)
+      HTML::Pipeline.parse(result[:output])
+    end
+
+    # Like call but guarantee the value returned is a string of HTML markup.
+    def to_html(input, context = {}, result = nil)
+      result = call(input, context, result = nil)
+      output = result[:output]
+      if output.respond_to?(:to_html)
+        output.to_html
+      else
+        output.to_s
+      end
+    end
+  end
+end
+
+# XXX nokogiri monkey patches
+class Nokogiri::XML::Node
+  # Work around an issue with utf-8 encoded data being erroneously converted to
+  # ... some other shit when replacing text nodes. See 'utf-8 output 2' in
+  # user_content_test.rb for details.
+  def replace_with_encoding_fix(replacement)
+    if replacement.respond_to?(:to_str)
+      replacement = document.fragment("<div>#{replacement}</div>").children.first.children
+    end
+    replace_without_encoding_fix(replacement)
+  end
+
+  alias_method :replace_without_encoding_fix, :replace
+  alias_method :replace, :replace_with_encoding_fix
+
+  def swap(replacement)
+    replace(replacement)
+    self
+  end
+end

data/lib/html/pipeline/@mention_filter.rb

@@ -0,0 +1,118 @@
+require 'set'
+
+module HTML
+  class Pipeline
+    # HTML filter that replaces @user mentions with links. Mentions within <pre>,
+    # <code>, and <a> elements are ignored. Mentions that reference users that do
+    # not exist are ignored.
+    #
+    # Context options:
+    #   :base_url - Used to construct links to user profile pages for each
+    #               mention.
+    #   :info_url - Used to link to "more info" when someone mentions @mention
+    #               or @mentioned.
+    #
+    class MentionFilter < Filter
+      # Public: Find user @mentions in text.  See
+      # MentionFilter#mention_link_filter.
+      #
+      #   MentionFilter.mentioned_logins_in(text) do |match, login, is_mentioned|
+      #     "<a href=...>#{login}</a>"
+      #   end
+      #
+      # text - String text to search.
+      #
+      # Yields the String match, the String login name, and a Boolean determining
+      # if the match = "@mention[ed]".  The yield's return replaces the match in
+      # the original text.
+      #
+      # Returns a String replaced with the return of the block.
+      def self.mentioned_logins_in(text)
+        text.gsub MentionPattern do |match|
+          login = $1
+          yield match, login, MentionLogins.include?(login.downcase)
+        end
+      end
+
+      # Pattern used to extract @mentions from text.
+      MentionPattern = /
+        (?:^|\W)                   # beginning of string or non-word char
+        @((?>[a-z0-9][a-z0-9-]*))  # @username
+        (?!\/)                     # without a trailing slash
+        (?=
+          \.+[ \t\W]|              # dots followed by space or non-word character
+          \.+$|                    # dots at end of line
+          [^0-9a-zA-Z_.]|          # non-word character except dot
+          $                        # end of line
+        )
+      /ix
+
+      # List of username logins that, when mentioned, link to the blog post
+      # about @mentions instead of triggering a real mention.
+      MentionLogins = %w(
+        mention
+        mentions
+        mentioned
+        mentioning
+      )
+
+      # Don't look for mentions in text nodes that are children of these elements
+      IGNORE_PARENTS = %w(pre code a).to_set
+
+      def call
+        doc.search('text()').each do |node|
+          content = node.to_html
+          next if !content.include?('@')
+          next if has_ancestor?(node, IGNORE_PARENTS)
+          html = mention_link_filter(content, base_url, info_url)
+          next if html == content
+          node.replace(html)
+        end
+        doc
+      end
+
+      # The URL to provide when someone @mentions a "mention" name, such as
+      # @mention or @mentioned, that will give them more info on mentions.
+      def info_url
+        context[:info_url] || nil
+      end
+
+      # Replace user @mentions in text with links to the mentioned user's
+      # profile page.
+      #
+      # text      - String text to replace @mention usernames in.
+      # base_url  - The base URL used to construct user profile URLs.
+      # info_url  - The "more info" URL used to link to more info on @mentions.
+      #             If nil we don't link @mention or @mentioned.
+      #
+      # Returns a string with @mentions replaced with links. All links have a
+      # 'user-mention' class name attached for styling.
+      def mention_link_filter(text, base_url='/', info_url=nil)
+        self.class.mentioned_logins_in(text) do |match, login, is_mentioned|
+          link =
+            if is_mentioned
+              link_to_mention_info(login, info_url)
+            else
+              link_to_mentioned_user(login)
+            end
+
+          link ? match.sub("@#{login}", link) : match
+        end
+      end
+
+      def link_to_mention_info(text, info_url=nil)
+        return "@#{text}" if info_url.nil?
+        "<a href='#{info_url}' class='user-mention'>" +
+        "@#{text}" +
+        "</a>"
+      end
+
+      def link_to_mentioned_user(login)
+        url = File.join(base_url, login)
+        "<a href='#{url}' class='user-mention'>" +
+        "@#{login}" +
+        "</a>"
+      end
+    end
+  end
+end