twitter-text 1.4.8 → 1.4.9

data/lib/twitter-text.rb

@@ -1,15 +1,18 @@
 major, minor, patch = RUBY_VERSION.split('.')
 
-if major.to_i == 1 && minor.to_i < 9
+$RUBY_1_9 = if major.to_i == 1 && minor.to_i < 9
   # Ruby 1.8 KCODE check. Not needed on 1.9 and later.
   raise("twitter-text requires the $KCODE variable be set to 'UTF8' or 'u'") unless $KCODE[0].chr =~ /u/i
+  false
+else
+  true
 end
 
-# External libraries required. (for gems, use: ruby -rubygems ...)
-require 'action_pack'
-require 'action_view'
+require 'active_support'
+require 'active_support/core_ext/string/multibyte.rb'
 
 require File.join(File.dirname(__FILE__), 'regex')
+require File.join(File.dirname(__FILE__), 'rewriter')
 require File.join(File.dirname(__FILE__), 'autolink')
 require File.join(File.dirname(__FILE__), 'extractor')
 require File.join(File.dirname(__FILE__), 'unicode')

data/lib/validation.rb

@@ -35,7 +35,7 @@ module Twitter
     #   <tt>:empty</tt>:: if the <tt>text</tt> is nil or empty
     #   <tt>:invalid_characters</tt>:: if the <tt>text</tt> contains non-Unicode or any of the disallowed Unicode characters
     def tweet_invalid?(text)
-      return :empty if text.blank?
+      return :empty if !text || text.empty?
       begin
         return :too_long if tweet_length(text) > MAX_LENGTH
         return :invalid_characters if INVALID_CHARACTERS.any?{|invalid_char| text.include?(invalid_char) }
@@ -52,7 +52,7 @@ module Twitter
     end
 
     def valid_username?(username)
-      return false if username.blank?
+      return false if !username || username.empty?
 
       extracted = Twitter::Extractor.extract_mentioned_screen_names(username)
       # Should extract the username minus the @ sign, hence the [1..-1]
@@ -63,11 +63,11 @@ module Twitter
     def valid_list?(username_list)
       match = username_list.match(VALID_LIST_RE)
       # Must have matched and had nothing before or after
-      !!(match && match[1] == "" && !match[4].blank?)
+      !!(match && match[1] == "" && match[4] && !match[4].empty?)
     end
 
     def valid_hashtag?(hashtag)
-      return false if hashtag.blank?
+      return false if !hashtag || hashtag.empty?
 
       extracted = Twitter::Extractor.extract_hashtags(hashtag)
       # Should extract the hashtag minus the # sign, hence the [1..-1]
@@ -75,7 +75,7 @@ module Twitter
     end
 
     def valid_url?(url, unicode_domains=true)
-      return false if url.blank?
+      return false if !url || url.empty?
 
       url_parts = url.match(Twitter::Regex[:validate_url_unencoded])
       return false unless (url_parts && url_parts.to_s == url)

data/spec/autolinking_spec.rb

@@ -36,7 +36,7 @@ describe Twitter::Autolink do
         def original_text; "meet@the beach"; end
 
         it "should not be linked" do
-          Nokogiri::HTML(@autolinked_text).search('a').should be_blank
+          Nokogiri::HTML(@autolinked_text).search('a').should be_empty
         end
       end
 
@@ -133,7 +133,7 @@ describe Twitter::Autolink do
         def original_text; "hello @/my-list"; end
 
         it "should NOT be linked" do
-          Nokogiri::HTML(@autolinked_text).search('a').should be_blank
+          Nokogiri::HTML(@autolinked_text).search('a').should be_empty
         end
       end
 
@@ -149,7 +149,7 @@ describe Twitter::Autolink do
         def original_text; "meet@the/beach"; end
 
         it "should not be linked" do
-          Nokogiri::HTML(@autolinked_text).search('a').should be_blank
+          Nokogiri::HTML(@autolinked_text).search('a').should be_empty
         end
       end
 
@@ -505,6 +505,24 @@ describe Twitter::Autolink do
         end
       end
 
+      context "regex engine quirks" do
+        context "does not spiral out of control on repeated periods" do
+          def original_text; "Test a ton of periods http://example.com/path.........................................."; end
+
+          it "should autolink" do
+            @autolinked_text.should have_autolinked_url('http://example.com/path')
+          end
+        end
+
+        context "does not spiral out of control on repeated dashes" do
+          def original_text; "Single char file ext http://www.bestbuy.com/site/Currie+Technologies+-+Ezip+400+Scooter/9885188.p?id=1218189013070&skuId=9885188"; end
+
+          it "should autolink" do
+            @autolinked_text.should have_autolinked_url('http://www.bestbuy.com/site/Currie+Technologies+-+Ezip+400+Scooter/9885188.p?id=1218189013070&skuId=9885188')
+          end
+        end
+      end
+
     end
 
     describe "Autolink all" do
@@ -558,6 +576,26 @@ describe Twitter::Autolink do
       linked.should have_autolinked_url('http://example.com/')
       linked.should match(/target="mywindow"/)
     end
+
+    it "should customize href by username_url_block option" do
+      linked = TestAutolink.new.auto_link("@test", :username_url_block => lambda{|a| "dummy"})
+      linked.should have_autolinked_url('dummy', 'test')
+    end
+
+    it "should customize href by list_url_block option" do
+      linked = TestAutolink.new.auto_link("@test/list", :list_url_block => lambda{|a| "dummy"})
+      linked.should have_autolinked_url('dummy', 'test/list')
+    end
+
+    it "should customize href by hashtag_url_block option" do
+      linked = TestAutolink.new.auto_link("#hashtag", :hashtag_url_block => lambda{|a| "dummy"})
+      linked.should have_autolinked_url('dummy', '#hashtag')
+    end
+
+    it "should customize href by link_url_block option" do
+      linked = TestAutolink.new.auto_link("http://example.com/", :link_url_block => lambda{|a| "dummy"})
+      linked.should have_autolinked_url('dummy', 'http://example.com/')
+    end
   end
 
   describe "html_escape" do

data/spec/extractor_spec.rb

@@ -228,11 +228,11 @@ describe Twitter::Extractor do
         end
 
         it "should not allow the multiplication character" do
-          @extractor.extract_hashtags("#pre#{[0xd7].pack('U')}post").should == []
+          @extractor.extract_hashtags("#pre#{Twitter::Unicode::U00D7}post").should == []
         end
 
         it "should not allow the division character" do
-          @extractor.extract_hashtags("#pre#{[0xf7].pack('U')}post").should == []
+          @extractor.extract_hashtags("#pre#{Twitter::Unicode::U00F7}post").should == []
         end
       end
 

data/spec/rewriter_spec.rb

@@ -0,0 +1,558 @@
+# encoding: UTF-8
+
+require 'spec_helper'
+
+describe Twitter::Rewriter do
+  def original_text; end
+  def url; end
+
+  def block(*args)
+    if Array === @block_args
+      unless Array === @block_args.first
+        @block_args = [@block_args]
+      end
+      @block_args << args
+    else
+      @block_args = args
+    end
+    "[rewritten]"
+  end
+
+  describe "rewrite usernames" do #{{{
+    before do
+      @rewritten_text = Twitter::Rewriter.rewrite_usernames_or_lists(original_text, &method(:block))
+    end
+
+    context "username preceded by a space" do
+      def original_text; "hello @jacob"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", nil]
+        @rewritten_text.should == "hello [rewritten]"
+      end
+    end
+
+    context "username at beginning of line" do
+      def original_text; "@jacob you're cool"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", nil]
+        @rewritten_text.should == "[rewritten] you're cool"
+      end
+    end
+
+    context "username preceded by word character" do
+      def original_text; "meet@the beach"; end
+
+      it "should not be rewritten" do
+        @block_args.should be_nil
+        @rewritten_text.should == "meet@the beach"
+      end
+    end
+
+    context "username preceded by non-word character" do
+      def original_text; "great.@jacob"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", nil]
+        @rewritten_text.should == "great.[rewritten]"
+      end
+    end
+
+    context "username containing non-word characters" do
+      def original_text; "@jacob&^$%^"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", nil]
+        @rewritten_text.should == "[rewritten]&^$%^"
+      end
+    end
+
+    context "username over twenty characters" do
+      def original_text
+        @twenty_character_username = "zach" * 5
+        "@" + @twenty_character_username + "1"
+      end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", @twenty_character_username, nil]
+        @rewritten_text.should == "[rewritten]1"
+      end
+    end
+
+    context "username followed by japanese" do
+      def original_text; "@jacobの"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", nil]
+        @rewritten_text.should == "[rewritten]の"
+      end
+    end
+
+    context "username preceded by japanese" do
+      def original_text; "あ@jacob"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", nil]
+        @rewritten_text.should == "あ[rewritten]"
+      end
+    end
+
+    context "username surrounded by japanese" do
+      def original_text; "あ@jacobの"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", nil]
+        @rewritten_text.should == "あ[rewritten]の"
+      end
+    end
+
+    context "username using full-width at-sign" do
+      def original_text
+        "#{[0xFF20].pack('U')}jacob"
+      end
+
+      it "should be rewritten" do
+        @block_args.should == ["＠", "jacob", nil]
+        @rewritten_text.should == "[rewritten]"
+      end
+    end
+  end #}}}
+
+  describe "rewrite lists" do #{{{
+    before do
+      @rewritten_text = Twitter::Rewriter.rewrite_usernames_or_lists(original_text, &method(:block))
+    end
+
+    context "slug preceded by a space" do
+      def original_text; "hello @jacob/my-list"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", "/my-list"]
+        @rewritten_text.should == "hello [rewritten]"
+      end
+    end
+
+    context "username followed by a slash but no list" do
+      def original_text; "hello @jacob/ my-list"; end
+
+      it "should not be rewritten" do
+        @block_args.should == ["@", "jacob", nil]
+        @rewritten_text.should == "hello [rewritten]/ my-list"
+      end
+    end
+
+    context "empty username followed by a list" do
+      def original_text; "hello @/my-list"; end
+
+      it "should not be rewritten" do
+        @block_args.should be_nil
+        @rewritten_text.should == "hello @/my-list"
+      end
+    end
+
+    context "list slug at beginning of line" do
+      def original_text; "@jacob/my-list"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", "/my-list"]
+        @rewritten_text.should == "[rewritten]"
+      end
+    end
+
+    context "username preceded by alpha-numeric character" do
+      def original_text; "meet@jacob/my-list"; end
+
+      it "should not be rewritten" do
+        @block_args.should be_nil
+        @rewritten_text.should == "meet@jacob/my-list"
+      end
+    end
+
+    context "username preceded by non-word character" do
+      def original_text; "great.@jacob/my-list"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", "/my-list"]
+        @rewritten_text.should == "great.[rewritten]"
+      end
+    end
+
+    context "username containing non-word characters" do
+      def original_text; "@jacob/my-list&^$%^"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", "/my-list"]
+        @rewritten_text.should == "[rewritten]&^$%^"
+      end
+    end
+
+    context "username over twenty characters" do
+      def original_text
+        @twentyfive_character_list = "a" * 25
+        "@jacob/#{@twentyfive_character_list}12345"
+      end
+
+      it "should be rewritten" do
+        @block_args.should == ["@", "jacob", "/#{@twentyfive_character_list}"]
+        @rewritten_text.should == "[rewritten]12345"
+      end
+    end
+  end #}}}
+
+  describe "rewrite hashtags" do #{{{
+    before do
+      @rewritten_text = Twitter::Rewriter.rewrite_hashtags(original_text, &method(:block))
+    end
+
+    context "with an all numeric hashtag" do
+      def original_text; "#123"; end
+
+      it "should not be rewritten" do
+        @block_args.should be_nil
+        @rewritten_text.should == "#123"
+      end
+    end
+
+    context "with a hashtag with alphanumeric characters" do
+      def original_text; "#ab1d"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["#", "ab1d"]
+        @rewritten_text.should == "[rewritten]"
+      end
+    end
+
+    context "with a hashtag with underscores" do
+      def original_text; "#a_b_c_d"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["#", "a_b_c_d"]
+        @rewritten_text.should == "[rewritten]"
+      end
+    end
+
+    context "with a hashtag that is preceded by a word character" do
+      def original_text; "ab#cd"; end
+
+      it "should not be rewritten" do
+        @block_args.should be_nil
+        @rewritten_text.should == "ab#cd"
+      end
+    end
+
+    context "with a page anchor in a url" do
+      def original_text; "Here's my url: http://foobar.com/#home"; end
+
+      it "should not link the hashtag" do
+        @block_args.should be_nil
+        @rewritten_text.should == "Here's my url: http://foobar.com/#home"
+      end
+    end
+
+    context "with a hashtag that starts with a number but has word characters" do
+      def original_text; "#2ab"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["#", "2ab"]
+        @rewritten_text.should == "[rewritten]"
+      end
+    end
+
+    context "with multiple valid hashtags" do
+      def original_text; "I'm frickin' awesome #ab #cd #ef"; end
+
+      it "rewrites each hashtag" do
+        @block_args.should == [["#", "ab"], ["#", "cd"], ["#", "ef"]]
+        @rewritten_text.should == "I'm frickin' awesome [rewritten] [rewritten] [rewritten]"
+      end
+    end
+
+    context "with a hashtag preceded by a ." do
+      def original_text; "ok, great.#abc"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["#", "abc"]
+        @rewritten_text.should == "ok, great.[rewritten]"
+      end
+    end
+
+    context "with a hashtag preceded by a &" do
+      def original_text; "&#nbsp;"; end
+
+      it "should not be rewritten" do
+        @block_args.should be_nil
+        @rewritten_text.should == "&#nbsp;"
+      end
+    end
+
+    context "with a hashtag that ends in an !" do
+      def original_text; "#great!"; end
+
+      it "should be rewritten, but should not include the !" do
+        @block_args.should == ["#", "great"];
+        @rewritten_text.should == "[rewritten]!"
+      end
+    end
+
+    context "with a hashtag followed by Japanese" do
+      def original_text; "#twj_devの"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["#", "twj_devの"];
+        @rewritten_text.should == "[rewritten]"
+      end
+    end
+
+    context "with a hashtag preceded by a full-width space" do
+      def original_text; "#{[0x3000].pack('U')}#twj_dev"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["#", "twj_dev"];
+        @rewritten_text.should == "　[rewritten]"
+      end
+    end
+
+    context "with a hashtag followed by a full-width space" do
+      def original_text; "#twj_dev#{[0x3000].pack('U')}"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["#", "twj_dev"];
+        @rewritten_text.should == "[rewritten]　"
+      end
+    end
+
+    context "with a hashtag using full-width hash" do
+      def original_text; "#{[0xFF03].pack('U')}twj_dev"; end
+
+      it "should be rewritten" do
+        @block_args.should == ["＃", "twj_dev"];
+        @rewritten_text.should == "[rewritten]"
+      end
+    end
+
+    context "with a hashtag containing an accented latin character" do
+      def original_text
+        # the hashtag is #éhashtag
+        "##{[0x00e9].pack('U')}hashtag"
+      end
+
+      it "should be rewritten" do
+        @block_args.should == ["#", "éhashtag"];
+        @rewritten_text.should == "[rewritten]"
+      end
+    end
+  end #}}}
+
+  describe "rewrite urls" do #{{{
+    def url; "http://www.google.com"; end
+
+    before do
+      @rewritten_text = Twitter::Rewriter.rewrite_urls(original_text, &method(:block))
+    end
+
+    context "when embedded in plain text" do
+      def original_text; "On my search engine #{url} I found good links."; end
+
+      it "should be rewritten" do
+        @block_args.should == [url];
+        @rewritten_text.should == "On my search engine [rewritten] I found good links."
+      end
+    end
+
+    context "when surrounded by Japanese;" do
+      def original_text; "いまなにしてる#{url}いまなにしてる"; end
+
+      it "should be rewritten" do
+        @block_args.should == [url];
+        @rewritten_text.should == "いまなにしてる[rewritten]いまなにしてる"
+      end
+    end
+
+    context "with a path surrounded by parentheses;" do
+      def original_text; "I found a neatness (#{url})"; end
+
+      it "should be rewritten" do
+        @block_args.should == [url];
+        @rewritten_text.should == "I found a neatness ([rewritten])"
+      end
+
+      context "when the URL ends with a slash;" do
+        def url; "http://www.google.com/"; end
+
+        it "should be rewritten" do
+          @block_args.should == [url];
+          @rewritten_text.should == "I found a neatness ([rewritten])"
+        end
+      end
+
+      context "when the URL has a path;" do
+        def url; "http://www.google.com/fsdfasdf"; end
+
+        it "should be rewritten" do
+          @block_args.should == [url];
+          @rewritten_text.should == "I found a neatness ([rewritten])"
+        end
+      end
+    end
+
+    context "when path contains parens" do
+      def original_text; "I found a neatness (#{url})"; end
+
+      it "should be rewritten" do
+        @block_args.should == [url];
+        @rewritten_text.should == "I found a neatness ([rewritten])"
+      end
+
+      context "wikipedia" do
+        def url; "http://en.wikipedia.org/wiki/Madonna_(artist)"; end
+
+        it "should be rewritten" do
+          @block_args.should == [url];
+          @rewritten_text.should == "I found a neatness ([rewritten])"
+        end
+      end
+
+      context "IIS session" do
+        def url; "http://msdn.com/S(deadbeef)/page.htm"; end
+
+        it "should be rewritten" do
+          @block_args.should == [url];
+          @rewritten_text.should == "I found a neatness ([rewritten])"
+        end
+      end
+
+      context "unbalanced parens" do
+        def url; "http://example.com/i_has_a_("; end
+
+        it "should be rewritten" do
+          @block_args.should == ["http://example.com/i_has_a_"];
+          @rewritten_text.should == "I found a neatness ([rewritten]()"
+        end
+      end
+
+      context "balanced parens with a double quote inside" do
+        def url; "http://foo.bar/foo_(\")_bar" end
+
+        it "should be rewritten" do
+          @block_args.should == ["http://foo.bar/foo_"];
+          @rewritten_text.should == "I found a neatness ([rewritten](\")_bar)"
+        end
+      end
+
+      context "balanced parens hiding XSS" do
+        def url; 'http://x.xx/("style="color:red"onmouseover="alert(1)' end
+
+        it "should be rewritten" do
+          @block_args.should == ["http://x.xx/"];
+          @rewritten_text.should == 'I found a neatness ([rewritten]("style="color:red"onmouseover="alert(1))'
+        end
+      end
+    end
+
+    context "when preceded by a :" do
+      def original_text; "Check this out @hoverbird:#{url}"; end
+
+      it "should be rewritten" do
+        @block_args.should == [url];
+        @rewritten_text.should == "Check this out @hoverbird:[rewritten]"
+      end
+    end
+
+    context "with a URL ending in allowed punctuation" do
+      it "does not consume ending punctuation" do
+        %w| ? ! , . : ; ] ) } = \ ' |.each do |char|
+          Twitter::Rewriter.rewrite_urls("#{url}#{char}") do |url|
+            url.should == url; "[rewritten]"
+          end.should == "[rewritten]#{char}"
+        end
+      end
+    end
+
+    context "with a URL preceded in forbidden characters" do
+      it "should not be rewritten" do
+        %w| \ ' / ! = |.each do |char|
+          Twitter::Rewriter.rewrite_urls("#{char}#{url}") do |url|
+            "[rewritten]" # should not be called here.
+          end.should == "#{char}#{url}"
+        end
+      end
+    end
+
+    context "when embedded in a link tag" do
+      def original_text; "<link rel='true'>#{url}</link>"; end
+
+      it "should be rewritten" do
+        @block_args.should == [url];
+        @rewritten_text.should == "<link rel='true'>[rewritten]</link>"
+      end
+    end
+
+    context "with multiple URLs" do
+      def original_text; "http://www.links.org link at start of page, link at end http://www.foo.org"; end
+
+      it "should autolink each one" do
+        @block_args.should == [["http://www.links.org"], ["http://www.foo.org"]];
+        @rewritten_text.should == "[rewritten] link at start of page, link at end [rewritten]"
+      end
+    end
+
+    context "with multiple URLs in different formats" do
+      def original_text; "http://foo.com https://bar.com http://mail.foobar.org"; end
+
+      it "should autolink each one, in the proper order" do
+        @block_args.should == [["http://foo.com"], ["https://bar.com"], ["http://mail.foobar.org"]];
+        @rewritten_text.should == "[rewritten] [rewritten] [rewritten]"
+      end
+    end
+
+    context "with a URL having a long TLD" do
+      def original_text; "Yahoo integriert Facebook http://golem.mobi/0912/71607.html"; end
+
+      it "should autolink it" do
+        @block_args.should == ["http://golem.mobi/0912/71607.html"]
+        @rewritten_text.should == "Yahoo integriert Facebook [rewritten]"
+      end
+    end
+
+    context "with a url lacking the protocol" do
+      def original_text; "I like www.foobar.com dudes"; end
+
+      it "does not link at all" do
+        @block_args.should be_nil
+        @rewritten_text.should == "I like www.foobar.com dudes"
+      end
+    end
+
+    context "with a @ in a URL" do
+      context "with XSS attack" do
+        def original_text; 'http://x.xx/@"style="color:pink"onmouseover=alert(1)//'; end
+
+        it "should not allow XSS follwing @" do
+          @block_args.should == ["http://x.xx/"]
+          @rewritten_text.should == '[rewritten]@"style="color:pink"onmouseover=alert(1)//'
+        end
+      end
+
+      context "with a username not followed by a /" do
+        def original_text; "http://example.com/@foobar"; end
+
+        it "should link small url and username" do
+          @block_args.should == ["http://example.com/"]
+          @rewritten_text.should == "[rewritten]@foobar"
+        end
+      end
+
+      context "with a username followed by a /" do
+        def original_text; "http://example.com/@foobar/"; end
+
+        it "should not link the username but link full url" do
+          @block_args.should == ["http://example.com/@foobar/"]
+          @rewritten_text.should == "[rewritten]"
+        end
+      end
+    end
+  end #}}}
+end
+
+# vim: foldmethod=marker