incollege-text 1.12.0

data/lib/incollege-text/rewriter.rb

@@ -0,0 +1,63 @@
+module Incollege
+  # A module provides base methods to rewrite usernames, lists, hashtags and URLs.
+  module Rewriter extend self
+    def rewrite_entities(text, entities)
+      chars = text.to_s.to_char_a
+
+      # sort by start index
+      entities = entities.sort_by do |entity|
+        indices = entity.respond_to?(:indices) ? entity.indices : entity[:indices]
+        indices.first
+      end
+
+      result = []
+      last_index = entities.inject(0) do |index, entity|
+        indices = entity.respond_to?(:indices) ? entity.indices : entity[:indices]
+        result << chars[index...indices.first]
+        result << yield(entity, chars)
+        indices.last
+      end
+      result << chars[last_index..-1]
+
+      result.flatten.join
+    end
+
+    # These methods are deprecated, will be removed in future.
+    extend Deprecation
+
+    def rewrite(text, options = {})
+      [:hashtags, :urls, :usernames_or_lists].inject(text) do |key|
+        options[key] ? send(:"rewrite_#{key}", text, &options[key]) : text
+      end
+    end
+    deprecate :rewrite, :rewrite_entities
+
+    def rewrite_usernames_or_lists(text)
+      entities = Extractor.extract_mentions_or_lists_with_indices(text)
+      rewrite_entities(text, entities) do |entity, chars|
+        at = chars[entity[:indices].first]
+        list_slug = entity[:list_slug]
+        list_slug = nil if list_slug.empty?
+        yield(at, entity[:screen_name], list_slug)
+      end
+    end
+    deprecate :rewrite_usernames_or_lists, :rewrite_entities
+
+    def rewrite_hashtags(text)
+      entities = Extractor.extract_hashtags_with_indices(text)
+      rewrite_entities(text, entities) do |entity, chars|
+        hash = chars[entity[:indices].first]
+        yield(hash, entity[:hashtag])
+      end
+    end
+    deprecate :rewrite_hashtags, :rewrite_entities
+
+    def rewrite_urls(text)
+      entities = Extractor.extract_urls_with_indices(text, :extract_url_without_protocol => false)
+      rewrite_entities(text, entities) do |entity, chars|
+        yield(entity[:url])
+      end
+    end
+    deprecate :rewrite_urls, :rewrite_entities
+  end
+end

data/lib/incollege-text/unicode.rb

@@ -0,0 +1,26 @@
+module Incollege
+  # This module lazily defines constants of the form Uxxxx for all Unicode
+  # codepoints from U0000 to U10FFFF. The value of each constant is the
+  # UTF-8 string for the codepoint.
+  # Examples:
+  #   copyright = Unicode::U00A9
+  #   euro = Unicode::U20AC
+  #   infinity = Unicode::U221E
+  #
+  module Unicode
+    CODEPOINT_REGEX = /^U_?([0-9a-fA-F]{4,5}|10[0-9a-fA-F]{4})$/
+
+    def self.const_missing(name)
+      # Check that the constant name is of the right form: U0000 to U10FFFF
+      if name.to_s =~ CODEPOINT_REGEX
+        # Convert the codepoint to an immutable UTF-8 string,
+        # define a real constant for that value and return the value
+        #p name, name.class
+        const_set(name, [$1.to_i(16)].pack("U").freeze)
+      else  # Raise an error for constants that are not Unicode.
+        raise NameError, "Uninitialized constant: Unicode::#{name}"
+      end
+    end
+  end
+
+end

data/lib/incollege-text/validation.rb

@@ -0,0 +1,113 @@
+require 'unf'
+
+module Incollege
+  module Validation extend self
+    MAX_LENGTH = 140
+
+    DEFAULT_TCO_URL_LENGTHS = {
+      :short_url_length => 23,
+      :short_url_length_https => 23,
+      :characters_reserved_per_media => 23
+    }.freeze
+
+    # Returns the length of the string as it would be displayed. This is equivilent to the length of the Unicode NFC
+    # (See: http://www.unicode.org/reports/tr15). This is needed in order to consistently calculate the length of a
+    # string no matter which actual form was transmitted. For example:
+    #
+    #     U+0065  Latin Small Letter E
+    # +   U+0301  Combining Acute Accent
+    # ----------
+    # =   2 bytes, 2 characters, displayed as é (1 visual glyph)
+    #     … The NFC of {U+0065, U+0301} is {U+00E9}, which is a single chracter and a +display_length+ of 1
+    #
+    # The string could also contain U+00E9 already, in which case the canonicalization will not change the value.
+    #
+    def tweet_length(text, options = {})
+      options = DEFAULT_TCO_URL_LENGTHS.merge(options)
+
+      length = text.to_nfc.unpack("U*").length
+
+      Incollege::Extractor.extract_urls_with_indices(text) do |url, start_position, end_position|
+        length += start_position - end_position
+        length += url.downcase =~ /^https:\/\// ? options[:short_url_length_https] : options[:short_url_length]
+      end
+
+      length
+    end
+
+    # Check the <tt>text</tt> for any reason that it may not be valid as a Tweet. This is meant as a pre-validation
+    # before posting to api.twitter.com. There are several server-side reasons for Tweets to fail but this pre-validation
+    # will allow quicker feedback.
+    #
+    # Returns <tt>false</tt> if this <tt>text</tt> is valid. Otherwise one of the following Symbols will be returned:
+    #
+    #   <tt>:too_long</tt>:: if the <tt>text</tt> is too long
+    #   <tt>:empty</tt>:: if the <tt>text</tt> is nil or empty
+    #   <tt>:invalid_characters</tt>:: if the <tt>text</tt> contains non-Unicode or any of the disallowed Unicode characters
+    def tweet_invalid?(text)
+      return :empty if !text || text.empty?
+      begin
+        return :too_long if tweet_length(text) > MAX_LENGTH
+        return :invalid_characters if Incollege::Regex::INVALID_CHARACTERS.any?{|invalid_char| text.include?(invalid_char) }
+      rescue ArgumentError
+        # non-Unicode value.
+        return :invalid_characters
+      end
+
+      return false
+    end
+
+    def valid_tweet_text?(text)
+      !tweet_invalid?(text)
+    end
+
+    def valid_username?(username)
+      return false if !username || username.empty?
+
+      extracted = Incollege::Extractor.extract_mentioned_screen_names(username)
+      # Should extract the username minus the @ sign, hence the [1..-1]
+      extracted.size == 1 && extracted.first == username[1..-1]
+    end
+
+    VALID_LIST_RE = /\A#{Incollege::Regex[:valid_mention_or_list]}\z/o
+    def valid_list?(username_list)
+      match = username_list.match(VALID_LIST_RE)
+      # Must have matched and had nothing before or after
+      !!(match && match[1] == "" && match[4] && !match[4].empty?)
+    end
+
+    def valid_hashtag?(hashtag)
+      return false if !hashtag || hashtag.empty?
+
+      extracted = Incollege::Extractor.extract_hashtags(hashtag)
+      # Should extract the hashtag minus the # sign, hence the [1..-1]
+      extracted.size == 1 && extracted.first == hashtag[1..-1]
+    end
+
+    def valid_url?(url, unicode_domains=true, require_protocol=true)
+      return false if !url || url.empty?
+
+      url_parts = url.match(Incollege::Regex[:validate_url_unencoded])
+      return false unless (url_parts && url_parts.to_s == url)
+
+      scheme, authority, path, query, fragment = url_parts.captures
+
+      return false unless ((!require_protocol ||
+                           (valid_match?(scheme, Incollege::Regex[:validate_url_scheme]) && scheme.match(/\Ahttps?\Z/i))) &&
+                           valid_match?(path, Incollege::Regex[:validate_url_path]) &&
+                           valid_match?(query, Incollege::Regex[:validate_url_query], true) &&
+                           valid_match?(fragment, Incollege::Regex[:validate_url_fragment], true))
+
+      return (unicode_domains && valid_match?(authority, Incollege::Regex[:validate_url_unicode_authority])) ||
+             (!unicode_domains && valid_match?(authority, Incollege::Regex[:validate_url_authority]))
+    end
+
+    private
+
+    def valid_match?(string, regex, optional=false)
+      return (string && string.match(regex) && $~.to_s == string) unless optional
+
+      !(string && (!string.match(regex) || $~.to_s != string))
+    end
+  end
+end

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:newgem_simple, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:newgem_simple, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/spec/autolinking_spec.rb

@@ -0,0 +1,844 @@
+# encoding: utf-8
+require File.dirname(__FILE__) + '/spec_helper'
+
+class TestAutolink
+  include Incollege::Autolink
+end
+
+describe Incollege::Autolink do
+  def original_text; end
+  def url; end
+
+  describe "auto_link_custom" do
+    before do
+      @autolinked_text = TestAutolink.new.auto_link(original_text) if original_text
+    end
+
+    describe "username autolinking" do
+      context "username preceded by a space" do
+        def original_text; "hello @jacob"; end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_screen_name('jacob')
+        end
+      end
+
+      context "username in camelCase" do
+        def original_text() "@jaCob iS cOoL" end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_screen_name('jaCob')
+        end
+      end
+
+      context "username at beginning of line" do
+        def original_text; "@jacob you're cool"; end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_screen_name('jacob')
+        end
+      end
+
+      context "username preceded by word character" do
+        def original_text; "meet@the beach"; end
+
+        it "should not be linked" do
+          Nokogiri::HTML(@autolinked_text).search('a').should be_empty
+        end
+      end
+
+      context "username preceded by non-word character" do
+        def original_text; "great.@jacob"; end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_screen_name('jacob')
+        end
+      end
+
+      context "username containing non-word characters" do
+        def original_text; "@zach&^$%^"; end
+
+        it "should not be linked" do
+          @autolinked_text.should link_to_screen_name('zach')
+        end
+      end
+
+      context "username over twenty characters" do
+        def original_text
+          @twenty_character_username = "zach" * 5
+          "@" + @twenty_character_username + "1"
+        end
+
+        it "should not be linked" do
+          @autolinked_text.should link_to_screen_name(@twenty_character_username)
+        end
+      end
+
+      context "username followed by japanese" do
+        def original_text; "@jacobの"; end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_screen_name('jacob')
+        end
+      end
+
+      context "username preceded by japanese" do
+        def original_text; "あ@matz"; end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_screen_name('matz')
+        end
+      end
+
+      context "username surrounded by japanese" do
+        def original_text; "あ@yoshimiの"; end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_screen_name('yoshimi')
+        end
+      end
+
+      context "username using full-width at-sign" do
+        def original_text
+          "#{[0xFF20].pack('U')}jacob"
+        end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_screen_name('jacob')
+        end
+      end
+    end
+
+    describe "list path autolinking" do
+
+      context "when List is not available" do
+        it "should not be linked" do
+          @autolinked_text = TestAutolink.new.auto_link_usernames_or_lists("hello @jacob/my-list", :suppress_lists => true)
+          @autolinked_text.should_not link_to_list_path('jacob/my-list')
+          @autolinked_text.should include('my-list')
+        end
+      end
+
+      context "slug preceded by a space" do
+        def original_text; "hello @jacob/my-list"; end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_list_path('jacob/my-list')
+        end
+      end
+
+      context "username followed by a slash but no list" do
+        def original_text; "hello @jacob/ my-list"; end
+
+        it "should NOT be linked" do
+          @autolinked_text.should_not link_to_list_path('jacob/my-list')
+          @autolinked_text.should link_to_screen_name('jacob')
+        end
+      end
+
+      context "empty username followed by a list" do
+        def original_text; "hello @/my-list"; end
+
+        it "should NOT be linked" do
+          Nokogiri::HTML(@autolinked_text).search('a').should be_empty
+        end
+      end
+
+      context "list slug at beginning of line" do
+        def original_text; "@jacob/my-list"; end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_list_path('jacob/my-list')
+        end
+      end
+
+      context "username preceded by alpha-numeric character" do
+        def original_text; "meet@the/beach"; end
+
+        it "should not be linked" do
+          Nokogiri::HTML(@autolinked_text).search('a').should be_empty
+        end
+      end
+
+      context "username preceded by non-word character" do
+        def original_text; "great.@jacob/my-list"; end
+
+        it "should be linked" do
+          @autolinked_text = TestAutolink.new.auto_link("great.@jacob/my-list")
+          @autolinked_text.should link_to_list_path('jacob/my-list')
+        end
+      end
+
+      context "username containing non-word characters" do
+        def original_text; "@zach/test&^$%^"; end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_list_path('zach/test')
+        end
+      end
+
+      context "username over twenty characters" do
+        def original_text
+          @twentyfive_character_list = "jack/" + ("a" * 25)
+          "@#{@twentyfive_character_list}12345"
+        end
+
+        it "should be linked" do
+          @autolinked_text.should link_to_list_path(@twentyfive_character_list)
+        end
+      end
+    end
+
+    describe "hashtag autolinking" do
+      context "with an all numeric hashtag" do
+        def original_text; "#123"; end
+
+        it "should not be linked" do
+          @autolinked_text.should_not have_autolinked_hashtag('#123')
+        end
+      end
+
+      context "with a hashtag with alphanumeric characters" do
+        def original_text; "#ab1d"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_hashtag('#ab1d')
+        end
+      end
+
+      context "with a hashtag with underscores" do
+        def original_text; "#a_b_c_d"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_hashtag(original_text)
+        end
+      end
+
+      context "with a hashtag that is preceded by a word character" do
+        def original_text; "ab#cd"; end
+
+        it "should not be linked" do
+          @autolinked_text.should_not have_autolinked_hashtag(original_text)
+        end
+      end
+
+      context "with a page anchor in a url" do
+        def original_text; "Here's my url: http://foobar.com/#home"; end
+
+        it "should not link the hashtag" do
+          @autolinked_text.should_not have_autolinked_hashtag('#home')
+        end
+
+        it "should link the url" do
+          @autolinked_text.should have_autolinked_url('http://foobar.com/#home')
+        end
+      end
+
+      context "with a hashtag that starts with a number but has word characters" do
+        def original_text; "#2ab"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_hashtag(original_text)
+        end
+      end
+
+      context "with multiple valid hashtags" do
+        def original_text; "I'm frickin' awesome #ab #cd #ef"; end
+
+        it "links each hashtag" do
+          @autolinked_text.should have_autolinked_hashtag('#ab')
+          @autolinked_text.should have_autolinked_hashtag('#cd')
+          @autolinked_text.should have_autolinked_hashtag('#ef')
+        end
+      end
+
+      context "with a hashtag preceded by a ." do
+        def original_text; "ok, great.#abc"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_hashtag('#abc')
+        end
+      end
+
+      context "with a hashtag preceded by a &" do
+        def original_text; "&#nbsp;"; end
+
+        it "should not be linked" do
+          @autolinked_text.should_not have_autolinked_hashtag('#nbsp;')
+        end
+      end
+
+      context "with a hashtag that ends in an !" do
+        def original_text; "#great!"; end
+
+        it "should be linked, but should not include the !" do
+          @autolinked_text.should have_autolinked_hashtag('#great')
+        end
+      end
+
+      context "with a hashtag followed by Japanese" do
+         def original_text; "#twj_devの"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_hashtag('#twj_devの')
+        end
+      end
+
+      context "with a hashtag preceded by a full-width space" do
+        def original_text; "#{[0x3000].pack('U')}#twj_dev"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_hashtag('#twj_dev')
+        end
+      end
+
+      context "with a hashtag followed by a full-width space" do
+        def original_text; "#twj_dev#{[0x3000].pack('U')}"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_hashtag('#twj_dev')
+        end
+      end
+
+      context "with a hashtag using full-width hash" do
+        def original_text; "#{[0xFF03].pack('U')}twj_dev"; end
+
+        it "should be linked" do
+          link = Nokogiri::HTML(@autolinked_text).search('a')
+          (link.inner_text.respond_to?(:force_encoding) ? link.inner_text.force_encoding("utf-8") : link.inner_text).should == "#{[0xFF03].pack('U')}twj_dev"
+          link.first['href'].should == 'https://twitter.com/#!/search?q=%23twj_dev'
+        end
+      end
+
+      context "with a hashtag containing an accented latin character" do
+        def original_text
+          # the hashtag is #éhashtag
+          "##{[0x00e9].pack('U')}hashtag"
+        end
+
+        it "should be linked" do
+          @autolinked_text.should == "<a class=\"tweet-url hashtag\" href=\"https://twitter.com/#!/search?q=%23éhashtag\" rel=\"nofollow\" title=\"#éhashtag\">#éhashtag</a>"
+        end
+      end
+
+    end
+
+    describe "URL autolinking" do
+      def url; "http://www.google.com"; end
+
+      context "when embedded in plain text" do
+        def original_text; "On my search engine #{url} I found good links."; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_url(url)
+        end
+      end
+
+      context "when surrounded by Japanese;" do
+        def original_text; "いまなにしてる#{url}いまなにしてる"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_url(url)
+        end
+      end
+
+      context "with a path surrounded by parentheses;" do
+        def original_text; "I found a neatness (#{url})"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_url(url)
+        end
+
+        context "when the URL ends with a slash;" do
+          def url; "http://www.google.com/"; end
+
+          it "should be linked" do
+            @autolinked_text.should have_autolinked_url(url)
+          end
+        end
+
+        context "when the URL has a path;" do
+          def url; "http://www.google.com/fsdfasdf"; end
+
+          it "should be linked" do
+            @autolinked_text.should have_autolinked_url(url)
+          end
+        end
+      end
+
+      context "when path contains parens" do
+        def original_text; "I found a neatness (#{url})"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_url(url)
+        end
+
+        context "wikipedia" do
+          def url; "http://en.wikipedia.org/wiki/Madonna_(artist)"; end
+
+          it "should be linked" do
+            @autolinked_text.should have_autolinked_url(url)
+          end
+        end
+
+        context "IIS session" do
+          def url; "http://msdn.com/S(deadbeef)/page.htm"; end
+
+          it "should be linked" do
+            @autolinked_text.should have_autolinked_url(url)
+          end
+        end
+
+        context "unbalanced parens" do
+          def url; "http://example.com/i_has_a_("; end
+
+          it "should be linked" do
+            @autolinked_text.should have_autolinked_url("http://example.com/i_has_a_")
+          end
+        end
+
+        context "balanced parens with a double quote inside" do
+          def url; "http://foo.com/foo_(\")_bar" end
+
+          it "should be linked" do
+            @autolinked_text.should have_autolinked_url("http://foo.com/foo_")
+          end
+        end
+
+        context "balanced parens hiding XSS" do
+          def url; 'http://x.xx.com/("style="color:red"onmouseover="alert(1)' end
+
+          it "should be linked" do
+            @autolinked_text.should have_autolinked_url("http://x.xx.com/")
+          end
+        end
+      end
+
+      context "when preceded by a :" do
+        def original_text; "Check this out @hoverbird:#{url}"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_url(url)
+        end
+      end
+
+      context "with a URL ending in allowed punctuation" do
+        it "does not consume ending punctuation" do
+          matcher = TestAutolink.new
+          %w| ? ! , . : ; ] ) } = \ ' |.each do |char|
+            matcher.auto_link("#{url}#{char}").should have_autolinked_url(url)
+          end
+        end
+      end
+
+      context "with a URL preceded in forbidden characters" do
+        it "should be linked" do
+          matcher = TestAutolink.new
+          %w| \ ' / ! = |.each do |char|
+            matcher.auto_link("#{char}#{url}").should have_autolinked_url(url)
+          end
+        end
+      end
+
+      context "when embedded in a link tag" do
+        def original_text; "<link rel='true'>#{url}</link>"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_url(url)
+        end
+      end
+
+      context "with multiple URLs" do
+        def original_text; "http://www.links.org link at start of page, link at end http://www.foo.org"; end
+
+        it "should autolink each one" do
+          @autolinked_text.should have_autolinked_url('http://www.links.org')
+          @autolinked_text.should have_autolinked_url('http://www.foo.org')
+        end
+      end
+
+      context "with multiple URLs in different formats" do
+        def original_text; "http://foo.com https://bar.com http://mail.foobar.org"; end
+
+        it "should autolink each one, in the proper order" do
+          @autolinked_text.should have_autolinked_url('http://foo.com')
+          @autolinked_text.should have_autolinked_url('https://bar.com')
+          @autolinked_text.should have_autolinked_url('http://mail.foobar.org')
+        end
+      end
+
+      context "with a URL having a long TLD" do
+        def original_text; "Yahoo integriert Facebook http://golem.mobi/0912/71607.html"; end
+
+        it "should autolink it" do
+          @autolinked_text.should have_autolinked_url('http://golem.mobi/0912/71607.html')
+        end
+      end
+
+      context "with a url lacking the protocol" do
+        def original_text; "I like www.foobar.com dudes"; end
+
+        it "does not link at all" do
+          link = Nokogiri::HTML(@autolinked_text).search('a')
+          link.should be_empty
+        end
+      end
+
+      context "with a @ in a URL" do
+        context "with XSS attack" do
+          def original_text; 'http://x.xx.com/@"style="color:pink"onmouseover=alert(1)//'; end
+
+          it "should not allow XSS follwing @" do
+            @autolinked_text.should have_autolinked_url('http://x.xx.com/')
+          end
+        end
+
+        context "with a username not followed by a /" do
+          def original_text; 'http://example.com/@foobar'; end
+
+          it "should link url" do
+            @autolinked_text.should have_autolinked_url('http://example.com/@foobar')
+          end
+        end
+
+        context "with a username followed by a /" do
+          def original_text; 'http://example.com/@foobar/'; end
+
+          it "should not link the username but link full url" do
+            @autolinked_text.should have_autolinked_url('http://example.com/@foobar/')
+            @autolinked_text.should_not link_to_screen_name('foobar')
+          end
+        end
+      end
+
+      context "regex engine quirks" do
+        context "does not spiral out of control on repeated periods" do
+          def original_text; "Test a ton of periods http://example.com/path.........................................."; end
+
+          it "should autolink" do
+            @autolinked_text.should have_autolinked_url('http://example.com/path')
+          end
+        end
+
+        context "does not spiral out of control on repeated dashes" do
+          def original_text; "Single char file ext http://www.bestbuy.com/site/Currie+Technologies+-+Ezip+400+Scooter/9885188.p?id=1218189013070&skuId=9885188"; end
+
+          it "should autolink" do
+            @autolinked_text.should have_autolinked_url('http://www.bestbuy.com/site/Currie+Technologies+-+Ezip+400+Scooter/9885188.p?id=1218189013070&skuId=9885188')
+          end
+        end
+      end
+
+    end
+
+    describe "Autolink all" do
+      before do
+        @linker = TestAutolink.new
+      end
+
+      it "should allow url/hashtag overlap" do
+        auto_linked = @linker.auto_link("https://twitter.com/#search")
+        auto_linked.should have_autolinked_url('https://twitter.com/#search')
+      end
+
+      it "should not add invalid option in HTML tags" do
+        auto_linked = @linker.auto_link("https://twitter.com/ is a URL, not a hashtag", :hashtag_class => 'hashtag_classname')
+        auto_linked.should have_autolinked_url('https://twitter.com/')
+        auto_linked.should_not include('hashtag_class')
+        auto_linked.should_not include('hashtag_classname')
+      end
+
+      it "should autolink url/hashtag/mention in text with Unicode supplementary characters" do
+        auto_linked = @linker.auto_link("#{[0x10400].pack('U')} #hashtag #{[0x10400].pack('U')} @mention #{[0x10400].pack('U')} http://twitter.com/")
+        auto_linked.should have_autolinked_hashtag('#hashtag')
+        auto_linked.should link_to_screen_name('mention')
+        auto_linked.should have_autolinked_url('http://twitter.com/')
+      end
+    end
+
+  end
+
+  describe "autolinking options" do
+    before do
+      @linker = TestAutolink.new
+    end
+
+    it "should show display_url when :url_entities provided" do
+      linked = @linker.auto_link("http://t.co/0JG5Mcq", :url_entities => [{
+        "url" => "http://t.co/0JG5Mcq",
+        "display_url" => "blog.twitter.com/2011/05/twitte…",
+        "expanded_url" => "http://blog.twitter.com/2011/05/twitter-for-mac-update.html",
+        "indices" => [
+          84,
+          103
+        ]
+      }])
+      html = Nokogiri::HTML(linked)
+      html.search('a').should_not be_empty
+      html.search('a[@href="http://t.co/0JG5Mcq"]').should_not be_empty
+      html.search('span[@class=js-display-url]').inner_text.should == "blog.twitter.com/2011/05/twitte"
+      html.inner_text.should == " http://blog.twitter.com/2011/05/twitter-for-mac-update.html …"
+      html.search('span[@style="position:absolute;left:-9999px;"]').size.should == 4
+    end
+
+    it "should accept invisible_tag_attrs option" do
+      linked = @linker.auto_link("http://t.co/0JG5Mcq",
+        {
+          :url_entities => [{
+            "url" => "http://t.co/0JG5Mcq",
+            "display_url" => "blog.twitter.com/2011/05/twitte…",
+            "expanded_url" => "http://blog.twitter.com/2011/05/twitter-for-mac-update.html",
+            "indices" => [
+              0,
+              19
+            ]
+          }],
+          :invisible_tag_attrs => "style='dummy;'"
+      })
+      html = Nokogiri::HTML(linked)
+      html.search('span[@style="dummy;"]').size.should == 4
+    end
+
+    it "should show display_url if available in entity" do
+      linked = @linker.auto_link_entities("http://t.co/0JG5Mcq",
+        [{
+          :url => "http://t.co/0JG5Mcq",
+          :display_url => "blog.twitter.com/2011/05/twitte…",
+          :expanded_url => "http://blog.twitter.com/2011/05/twitter-for-mac-update.html",
+          :indices => [0, 19]
+        }]
+      )
+      html = Nokogiri::HTML(linked)
+      html.search('a').should_not be_empty
+      html.search('a[@href="http://t.co/0JG5Mcq"]').should_not be_empty
+      html.search('span[@class=js-display-url]').inner_text.should == "blog.twitter.com/2011/05/twitte"
+      html.inner_text.should == " http://blog.twitter.com/2011/05/twitter-for-mac-update.html …"
+    end
+
+    it "should apply :class as a CSS class" do
+      linked = @linker.auto_link("http://example.com/", :class => 'myclass')
+      linked.should have_autolinked_url('http://example.com/')
+      linked.should match(/myclass/)
+    end
+
+    it "should apply :url_class only on URL" do
+      linked = @linker.auto_link("http://twitter.com")
+      linked.should have_autolinked_url('http://twitter.com')
+      linked.should_not match(/class/)
+
+      linked = @linker.auto_link("http://twitter.com", :url_class => 'testClass')
+      linked.should have_autolinked_url('http://twitter.com')
+      linked.should match(/class=\"testClass\"/)
+
+      linked = @linker.auto_link("#hash @tw", :url_class => 'testClass')
+      linked.should match(/class=\"tweet-url hashtag\"/)
+      linked.should match(/class=\"tweet-url username\"/)
+      linked.should_not match(/class=\"testClass\"/)
+    end
+
+    it "should add rel=nofollow by default" do
+      linked = @linker.auto_link("http://example.com/")
+      linked.should have_autolinked_url('http://example.com/')
+      linked.should match(/nofollow/)
+    end
+
+    it "should include the '@' symbol in a username when passed :username_include_symbol" do
+      linked = @linker.auto_link("@user", :username_include_symbol => true)
+      linked.should link_to_screen_name('user', '@user')
+    end
+
+    it "should include the '@' symbol in a list when passed :username_include_symbol" do
+      linked = @linker.auto_link("@user/list", :username_include_symbol => true)
+      linked.should link_to_list_path('user/list', '@user/list')
+    end
+
+    it "should not add rel=nofollow when passed :suppress_no_follow" do
+      linked = @linker.auto_link("http://example.com/", :suppress_no_follow => true)
+      linked.should have_autolinked_url('http://example.com/')
+      linked.should_not match(/nofollow/)
+    end
+
+    it "should not add a target attribute by default" do
+      linked = @linker.auto_link("http://example.com/")
+      linked.should have_autolinked_url('http://example.com/')
+      linked.should_not match(/target=/)
+    end
+
+    it "should respect the :target option" do
+      linked = @linker.auto_link("http://example.com/", :target => 'mywindow')
+      linked.should have_autolinked_url('http://example.com/')
+      linked.should match(/target="mywindow"/)
+    end
+
+    it "should customize href by username_url_block option" do
+      linked = @linker.auto_link("@test", :username_url_block => lambda{|a| "dummy"})
+      linked.should have_autolinked_url('dummy', 'test')
+    end
+
+    it "should customize href by list_url_block option" do
+      linked = @linker.auto_link("@test/list", :list_url_block => lambda{|a| "dummy"})
+      linked.should have_autolinked_url('dummy', 'test/list')
+    end
+
+    it "should customize href by hashtag_url_block option" do
+      linked = @linker.auto_link("#hashtag", :hashtag_url_block => lambda{|a| "dummy"})
+      linked.should have_autolinked_url('dummy', '#hashtag')
+    end
+
+    it "should customize href by cashtag_url_block option" do
+      linked = @linker.auto_link("$CASH", :cashtag_url_block => lambda{|a| "dummy"})
+      linked.should have_autolinked_url('dummy', '$CASH')
+    end
+
+    it "should customize href by link_url_block option" do
+      linked = @linker.auto_link("http://example.com/", :link_url_block => lambda{|a| "dummy"})
+      linked.should have_autolinked_url('dummy', 'http://example.com/')
+    end
+
+    it "should modify link attributes by link_attribute_block" do
+      linked = @linker.auto_link("#hash @mention",
+        :link_attribute_block => lambda{|entity, attributes|
+          attributes[:"dummy-hash-attr"] = "test" if entity[:hashtag]
+        }
+      )
+      linked.should match(/<a[^>]+hashtag[^>]+dummy-hash-attr=\"test\"[^>]+>/)
+      linked.should_not match(/<a[^>]+username[^>]+dummy-hash-attr=\"test\"[^>]+>/)
+      linked.should_not match(/link_attribute_block/i)
+
+      linked = @linker.auto_link("@mention http://twitter.com/",
+        :link_attribute_block => lambda{|entity, attributes|
+          attributes["dummy-url-attr"] = entity[:url] if entity[:url]
+        }
+      )
+      linked.should_not match(/<a[^>]+username[^>]+dummy-url-attr=\"http:\/\/twitter.com\/\"[^>]*>/)
+      linked.should match(/<a[^>]+dummy-url-attr=\"http:\/\/twitter.com\/\"/)
+    end
+
+    it "should modify link text by link_text_block" do
+      linked = @linker.auto_link("#hash @mention",
+        :link_text_block => lambda{|entity, text|
+          entity[:hashtag] ? "#replaced" : "pre_#{text}_post"
+        }
+      )
+      linked.should match(/<a[^>]+>#replaced<\/a>/)
+      linked.should match(/<a[^>]+>pre_mention_post<\/a>/)
+
+      linked = @linker.auto_link("#hash @mention", {
+        :link_text_block => lambda{|entity, text|
+          "pre_#{text}_post"
+        },
+        :symbol_tag => "s", :text_with_symbol_tag => "b", :username_include_symbol => true
+      })
+      linked.should match(/<a[^>]+>pre_<s>#<\/s><b>hash<\/b>_post<\/a>/)
+      linked.should match(/<a[^>]+>pre_<s>@<\/s><b>mention<\/b>_post<\/a>/)
+    end
+
+    it "should apply :url_target only to auto-linked URLs" do
+      auto_linked = @linker.auto_link("#hashtag @mention http://test.com/", {:url_target => '_blank'})
+      auto_linked.should have_autolinked_hashtag('#hashtag')
+      auto_linked.should link_to_screen_name('mention')
+      auto_linked.should have_autolinked_url('http://test.com/')
+      auto_linked.should_not match(/<a[^>]+hashtag[^>]+target[^>]+>/)
+      auto_linked.should_not match(/<a[^>]+username[^>]+target[^>]+>/)
+      auto_linked.should match(/<a[^>]+test.com[^>]+target=\"_blank\"[^>]*>/)
+    end
+
+    it "should apply target='_blank' only to auto-linked URLs when :target_blank is set to true" do
+      auto_linked = @linker.auto_link("#hashtag @mention http://test.com/", {:target_blank => true})
+      auto_linked.should have_autolinked_hashtag('#hashtag')
+      auto_linked.should link_to_screen_name('mention')
+      auto_linked.should have_autolinked_url('http://test.com/')
+      auto_linked.should match(/<a[^>]+hashtag[^>]+target=\"_blank\"[^>]*>/)
+      auto_linked.should match(/<a[^>]+username[^>]+target=\"_blank\"[^>]*>/)
+      auto_linked.should match(/<a[^>]+test.com[^>]+target=\"_blank\"[^>]*>/)
+    end
+  end
+
+  describe "link_url_with_entity" do
+    before do
+      @linker = TestAutolink.new
+    end
+
+    it "should use display_url and expanded_url" do
+      @linker.send(:link_url_with_entity,
+        {
+          :url => "http://t.co/abcde",
+          :display_url => "twitter.com",
+          :expanded_url => "http://twitter.com/"},
+        {:invisible_tag_attrs => "class='invisible'"}).gsub('"', "'").should == "<span class='tco-ellipsis'><span class='invisible'>&nbsp;</span></span><span class='invisible'>http://</span><span class='js-display-url'>twitter.com</span><span class='invisible'>/</span><span class='tco-ellipsis'><span class='invisible'>&nbsp;</span></span>";
+    end
+
+    it "should correctly handle display_url ending with '…'" do
+      @linker.send(:link_url_with_entity,
+        {
+          :url => "http://t.co/abcde",
+          :display_url => "twitter.com…",
+          :expanded_url => "http://twitter.com/abcdefg"},
+        {:invisible_tag_attrs => "class='invisible'"}).gsub('"', "'").should == "<span class='tco-ellipsis'><span class='invisible'>&nbsp;</span></span><span class='invisible'>http://</span><span class='js-display-url'>twitter.com</span><span class='invisible'>/abcdefg</span><span class='tco-ellipsis'><span class='invisible'>&nbsp;</span>…</span>";
+    end
+
+    it "should correctly handle display_url starting with '…'" do
+      @linker.send(:link_url_with_entity,
+        {
+          :url => "http://t.co/abcde",
+          :display_url => "…tter.com/abcdefg",
+          :expanded_url => "http://twitter.com/abcdefg"},
+        {:invisible_tag_attrs => "class='invisible'"}).gsub('"', "'").should == "<span class='tco-ellipsis'>…<span class='invisible'>&nbsp;</span></span><span class='invisible'>http://twi</span><span class='js-display-url'>tter.com/abcdefg</span><span class='invisible'></span><span class='tco-ellipsis'><span class='invisible'>&nbsp;</span></span>";
+    end
+
+    it "should not create spans if display_url and expanded_url are on different domains" do
+      @linker.send(:link_url_with_entity,
+        {
+          :url => "http://t.co/abcde",
+          :display_url => "pic.twitter.com/xyz",
+          :expanded_url => "http://twitter.com/foo/statuses/123/photo/1"},
+        {:invisible_tag_attrs => "class='invisible'"}).gsub('"', "'").should == "pic.twitter.com/xyz"
+    end
+  end
+
+  describe "symbol_tag" do
+    before do
+      @linker = TestAutolink.new
+    end
+    it "should put :symbol_tag around symbol" do
+      @linker.auto_link("@mention", {:symbol_tag => 's', :username_include_symbol=>true}).should match(/<s>@<\/s>mention/)
+      @linker.auto_link("#hash", {:symbol_tag => 's'}).should match(/<s>#<\/s>hash/)
+      result = @linker.auto_link("@mention #hash $CASH", {:symbol_tag => 'b', :username_include_symbol=>true})
+      result.should match(/<b>@<\/b>mention/)
+      result.should match(/<b>#<\/b>hash/)
+      result.should match(/<b>\$<\/b>CASH/)
+    end
+    it "should put :text_with_symbol_tag around text" do
+      result = @linker.auto_link("@mention #hash $CASH", {:text_with_symbol_tag => 'b'})
+      result.should match(/<b>mention<\/b>/)
+      result.should match(/<b>hash<\/b>/)
+      result.should match(/<b>CASH<\/b>/)
+    end
+    it "should put :symbol_tag around symbol and :text_with_symbol_tag around text" do
+      result = @linker.auto_link("@mention #hash $CASH", {:symbol_tag => 's', :text_with_symbol_tag => 'b', :username_include_symbol=>true})
+      result.should match(/<s>@<\/s><b>mention<\/b>/)
+      result.should match(/<s>#<\/s><b>hash<\/b>/)
+      result.should match(/<s>\$<\/s><b>CASH<\/b>/)
+    end
+  end
+
+  describe "html_escape" do
+    before do
+      @linker = TestAutolink.new
+    end
+    it "should escape html entities properly" do
+      @linker.html_escape("&").should == "&amp;"
+      @linker.html_escape(">").should == "&gt;"
+      @linker.html_escape("<").should == "&lt;"
+      @linker.html_escape("\"").should == "&quot;"
+      @linker.html_escape("'").should == "&#39;"
+      @linker.html_escape("&<>\"").should == "&amp;&lt;&gt;&quot;"
+      @linker.html_escape("<div>").should == "&lt;div&gt;"
+      @linker.html_escape("a&b").should == "a&amp;b"
+      @linker.html_escape("<a href=\"https://twitter.com\" target=\"_blank\">twitter & friends</a>").should == "&lt;a href=&quot;https://twitter.com&quot; target=&quot;_blank&quot;&gt;twitter &amp; friends&lt;/a&gt;"
+      @linker.html_escape("&amp;").should == "&amp;amp;"
+      @linker.html_escape(nil).should == nil
+    end
+  end
+
+end