twitter-text 1.0

data/spec/extractor_spec.rb

@@ -0,0 +1,195 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+class TestExtractor
+  include Twitter::Extractor
+end
+
+describe Twitter::Extractor do
+  before do
+    @extractor = TestExtractor.new
+  end
+
+  describe "mentions" do
+    context "single screen name alone " do
+      it "should be linked" do
+        @extractor.extract_mentioned_screen_names("@alice").should == ["alice"]
+      end
+
+      it "should be linked with _" do
+        @extractor.extract_mentioned_screen_names("@alice_adams").should == ["alice_adams"]
+      end
+
+      it "should be linked if numeric" do
+        @extractor.extract_mentioned_screen_names("@1234").should == ["1234"]
+      end
+    end
+
+    context "multiple screen names" do
+      it "should both be linked" do
+        @extractor.extract_mentioned_screen_names("@alice @bob").should == ["alice", "bob"]
+      end
+    end
+
+    context "screen names embedded in text" do
+      it "should be linked in Latin text" do
+        @extractor.extract_mentioned_screen_names("waiting for @alice to arrive").should == ["alice"]
+      end
+
+      it "should be linked in Japanese text" do
+        @extractor.extract_mentioned_screen_names("の@aliceに到着を待っている").should == ["alice"]
+      end
+    end
+
+    it "should accept a block arugment and call it in order" do
+      needed = ["alice", "bob"]
+      @extractor.extract_mentioned_screen_names("@alice @bob") do |sn|
+        sn.should == needed.shift
+      end
+      needed.should == []
+    end
+  end
+
+  describe "replies" do
+    context "should be extracted from" do
+      it "should extract from lone name" do
+        @extractor.extract_reply_screen_name("@alice").should == "alice"
+      end
+
+      it "should extract from the start" do
+        @extractor.extract_reply_screen_name("@alice reply text").should == "alice"
+      end
+
+      it "should extract preceeded by a space" do
+        @extractor.extract_reply_screen_name(" @alice reply text").should == "alice"
+      end
+
+      it "should extract preceeded by a full-width space" do
+        @extractor.extract_reply_screen_name("#{[0x3000].pack('U')}@alice reply text").should == "alice"
+      end
+    end
+
+    context "should not be extracted from" do
+      it "should not be extracted when preceeded by text" do
+        @extractor.extract_reply_screen_name("reply @alice text").should == nil
+      end
+
+      it "should not be extracted when preceeded by puctuation" do
+        %w(. / _ - + # ! @).each do |punct|
+          @extractor.extract_reply_screen_name("#{punct}@alice text").should == nil
+        end
+      end
+    end
+
+    context "should accept a block arugment" do
+      it "should call the block on match" do
+        @extractor.extract_reply_screen_name("@alice") do |sn|
+          sn.should == "alice"
+        end
+      end
+
+      it "should not call the block on no match" do
+        calls = 0
+        @extractor.extract_reply_screen_name("not a reply") do |sn|
+          calls += 1
+        end
+        calls.should == 0
+      end
+    end
+  end
+
+  describe "urls" do
+    describe "matching URLS" do
+      @urls = [
+        "http://google.com",
+        "http://foobar.com/#",
+        "http://google.com/#foo",
+        "http://google.com/#search?q=iphone%20-filter%3Alinks",
+        "http://twitter.com/#search?q=iphone%20-filter%3Alinks",
+        "http://www.boingboing.net/2007/02/14/katamari_damacy_phon.html",
+        "http://somehost.com:3000",
+        "http://x.com/~matthew+%-x",
+        "http://x.com/~matthew+%-,.;@:x",
+        "http://x.com/,.;@:x",
+        "http://en.wikipedia.org/wiki/Primer_(film)",
+        "http://www.ams.org/bookstore-getitem/item=mbk-59",
+        "http://chilp.it/?77e8fd",
+      ]
+
+      @urls.each do |url|
+        it "should extract the URL #{url}" do
+          @extractor.extract_urls(url).should == [url]
+        end
+
+        it "should match the URL #{url} when it's embedded in other text" do
+          text = "Sweet url: #{url} I found. #awesome"
+          @extractor.extract_urls(text).should == [url]
+        end
+      end
+    end
+
+    describe "invalid URLS" do
+     it "does not link urls with invalid_domains" do
+        [ "http://doman-dash_2314352345_dfasd.foo-cow_4352.com",
+          "http://no-tld",
+          "http://tld-too-short.x",
+          "http://doman-dash_2314352345_dfasd.foo-cow_4352.com",
+        ].each {|url| @extractor.extract_urls(url).should == [] }
+      end
+    end
+  end
+
+  describe "hashtags" do
+    context "extracts latin/numeric hashtags" do
+      %w(text text123 123text).each do |hashtag|
+        it "should extract ##{hashtag}" do
+          @extractor.extract_hashtags("##{hashtag}").should == [hashtag]
+        end
+
+        it "should extract ##{hashtag} within text" do
+          @extractor.extract_hashtags("pre-text ##{hashtag} post-text").should == [hashtag]
+        end
+      end
+    end
+
+    context "international hashtags" do
+
+      context "should allow accents" do
+        %w(mañana café münchen).each do |hashtag|
+          it "should extract ##{hashtag}" do
+            @extractor.extract_hashtags("##{hashtag}").should == [hashtag]
+          end
+
+          it "should extract ##{hashtag} within text" do
+            @extractor.extract_hashtags("pre-text ##{hashtag} post-text").should == [hashtag]
+          end
+        end
+
+        it "should not allow the multiplication character" do
+          @extractor.extract_hashtags("#pre#{[0xd7].pack('U')}post").should == ["pre"]
+        end
+
+        it "should not allow the division character" do
+          @extractor.extract_hashtags("#pre#{[0xf7].pack('U')}post").should == ["pre"]
+        end
+      end
+
+      context "should NOT allow Japanese" do
+        %w(会議中 ハッシュ).each do |hashtag|
+          it "should NOT extract ##{hashtag}" do
+            @extractor.extract_hashtags("##{hashtag}").should == []
+          end
+
+          it "should NOT extract ##{hashtag} within text" do
+            @extractor.extract_hashtags("pre-text ##{hashtag} post-text").should == []
+          end
+        end
+      end
+
+    end
+
+    it "should not extract numeric hashtags" do
+      @extractor.extract_hashtags("#1234").should == []
+    end
+  end
+
+end

data/spec/regex_spec.rb

@@ -0,0 +1,44 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe "Twitter::Regex regular expressions" do
+  describe "matching URLS" do
+    @urls = [
+      "http://google.com",
+      "http://foobar.com/#",
+      "http://google.com/#foo",
+      "http://google.com/#search?q=iphone%20-filter%3Alinks",
+      "http://twitter.com/#search?q=iphone%20-filter%3Alinks",
+      "http://www.boingboing.net/2007/02/14/katamari_damacy_phon.html",
+      "http://somehost.com:3000",
+      "http://x.com/~matthew+%-x",
+      "http://x.com/~matthew+%-,.;@:x",
+      "http://x.com/,.;@:x",
+      "http://en.wikipedia.org/wiki/Primer_(film)",
+      "http://www.ams.org/bookstore-getitem/item=mbk-59",
+      "http://chilp.it/?77e8fd",
+    ]
+
+    @urls.each do |url|
+      it "should match the URL #{url}" do
+        url.should match_autolink_expression
+      end
+
+      it "should match the URL #{url} when it's embedded in other text" do
+        text = "Sweet url: #{url} I found. #awesome"
+        url.should match_autolink_expression_in(text)
+      end
+    end
+  end
+
+  describe "invalid URLS" do
+   it "does not link urls with invalid_domains" do
+      [ "http://doman-dash_2314352345_dfasd.foo-cow_4352.com",
+        "http://no-tld",
+        "http://tld-too-short.x",
+        "http://x.com/,,,/.../@@@/;;;/:::/---/%%%x",
+        "http://doman-dash_2314352345_dfasd.foo-cow_4352.com",
+      ].each {|url| url.should_not have_autolinked_url(url)}
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,86 @@
+$TESTING=true
+$:.push File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'twitter-text'
+require 'hpricot'
+
+Spec::Matchers.define :match_autolink_expression do
+  match do |string|
+    Twitter::Regex[:valid_url].match(string)
+  end
+end
+
+Spec::Matchers.define :match_autolink_expression_in do |text|
+  match do |url|
+    @match_data = Twitter::Regex[:valid_url].match(text)
+    @match_data && @match_data.to_s.strip == url
+  end
+
+  failure_message_for_should do |url|
+    "Expected to find url '#{url}' in text '#{text}', but the match was #{@match_data.captures}'"
+  end
+end
+
+Spec::Matchers.define :have_autolinked_url do |url|
+  match do |text|
+    @link = Hpricot(text).at("a[@href='#{url}']")
+    @link &&
+    @link.inner_text &&
+    @link.inner_text == url
+  end
+
+  failure_message_for_should do |text|
+    "Expected url '#{url}' to be autolinked in '#{text}'"
+  end
+end
+
+Spec::Matchers.define :link_to_screen_name do |screen_name|
+  match do |text|
+    @link = Hpricot(text).at("a.username")
+    @link && @link.inner_text == screen_name && "http://twitter.com/#{screen_name}".downcase.should == @link['href']
+  end
+
+  failure_message_for_should do |text|
+    "expected link #{@link.inner_text} with href #{@link['href']} to match screen_name #{@screen_name}, but it does not"
+  end
+
+  failure_message_for_should_not do |text|
+    "expected link #{@link.inner_text} with href #{@link['href']} not to match screen_name #{@screen_name}, but it does"
+  end
+
+  description do
+    "contain a link with the name and href pointing to the expected screen_name"
+  end
+end
+
+Spec::Matchers.define :link_to_list_path do |list_path|
+  match do |text|
+    @link = Hpricot(text).at("a.list-slug")
+    !@link.nil? && @link.inner_text == list_path && "http://twitter.com/#{list_path}".downcase.should == @link['href']
+  end
+
+  failure_message_for_should do |text|
+    "expected link #{@link.inner_text} with href #{@link['href']} to match the list path #{list_path}, but it does not"
+  end
+
+  failure_message_for_should_not do |text|
+    "expected link #{@link.inner_text} with href #{@link['href']} not to match the list path #{@list_path}, but it does"
+  end
+
+  description do
+    "contain a link with the list title and an href pointing to the list path"
+  end
+end
+
+Spec::Matchers.define :have_autolinked_hashtag do |hashtag|
+  match do |text|
+    @link = Hpricot(text).at("a[@href='http://twitter.com/search?q=#{CGI.escape hashtag}']")
+    @link &&
+    @link.inner_text &&
+    @link.inner_text == hashtag
+  end
+
+  failure_message_for_should do |text|
+    "Expected hashtag #{hashtag} to be autolinked in '#{text}'"
+  end
+end

data/spec/unicode_spec.rb

@@ -0,0 +1,30 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Twitter::Unicode do
+
+  it "should lazy-init constants" do
+    Twitter::Unicode.const_defined?(:UFEB6).should == false
+    Twitter::Unicode::UFEB6.should_not be_nil
+    Twitter::Unicode::UFEB6.should be_kind_of(String)
+    Twitter::Unicode.const_defined?(:UFEB6).should == true
+  end
+
+  it "should return corresponding character" do
+    Twitter::Unicode::UFEB6.should == [0xfeb6].pack('U')
+  end
+
+  it "should allow lowercase notation" do
+    Twitter::Unicode::Ufeb6.should == Twitter::Unicode::UFEB6
+    Twitter::Unicode::Ufeb6.should === Twitter::Unicode::UFEB6
+  end
+
+  it "should allow underscore notation" do
+    Twitter::Unicode::U_FEB6.should == Twitter::Unicode::UFEB6
+    Twitter::Unicode::U_FEB6.should === Twitter::Unicode::UFEB6
+  end
+
+  it "should raise on invalid codepoints" do
+    lambda { Twitter::Unicode::FFFFFF }.should raise_error(NameError)
+  end
+
+end

data/spec/validation_spec.rb

@@ -0,0 +1,42 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+class TestValidation
+  include Twitter::Validation
+end
+
+describe Twitter::Validation do
+
+  it "should disallow invalid BOM character" do
+    TestValidation.new.tweet_invalid?("Bom:#{Twitter::Unicode::UFFFE}").should == :invalid_characters
+    TestValidation.new.tweet_invalid?("Bom:#{Twitter::Unicode::UFEFF}").should == :invalid_characters
+  end
+
+  it "should disallow invalid U+FFFF character" do
+    TestValidation.new.tweet_invalid?("Bom:#{Twitter::Unicode::UFFFF}").should == :invalid_characters
+  end
+
+  it "should disallow direction change characters" do
+    [0x202A, 0x202B, 0x202C, 0x202D, 0x202E].map{|cp| [cp].pack('U') }.each do |char|
+      TestValidation.new.tweet_invalid?("Invalid:#{char}").should == :invalid_characters
+    end
+  end
+
+  it "should disallow non-Unicode" do
+    TestValidation.new.tweet_invalid?("not-Unicode:\xfff0").should == :invalid_characters
+  end
+
+  it "should allow <= 140 combined accent characters" do
+    char = [0x65, 0x0301].pack('U')
+    TestValidation.new.tweet_invalid?(char * 139).should == false
+    TestValidation.new.tweet_invalid?(char * 140).should == false
+    TestValidation.new.tweet_invalid?(char * 141).should == :too_long
+  end
+
+  it "should allow <= 140 multi-byte characters" do
+    char = [ 0x1d106 ].pack('U')
+    TestValidation.new.tweet_invalid?(char * 139).should == false
+    TestValidation.new.tweet_invalid?(char * 140).should == false
+    TestValidation.new.tweet_invalid?(char * 141).should == :too_long
+  end
+
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification 
+name: twitter-text
+version: !ruby/object:Gem::Version 
+  version: "1.0"
+platform: ruby
+authors: 
+- Matt Sanford
+autorequire: ""
+bindir: bin
+cert_chain: []
+
+date: 2010-01-27 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: action_view
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: A gem that provides text handling for Twitter
+email: matt@twitter.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- LICENSE
+- README.rdoc
+- Rakefile
+- TODO
+- lib/autolink.rb
+- lib/extractor.rb
+- lib/regex.rb
+- lib/twitter-text.rb
+- lib/unicode.rb
+- lib/validation.rb
+- spec/autolinking_spec.rb
+- spec/extractor_spec.rb
+- spec/regex_spec.rb
+- spec/spec_helper.rb
+- spec/unicode_spec.rb
+- spec/validation_spec.rb
+has_rdoc: true
+homepage: http://twitter.com
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Twitter text handling library
+test_files: []
+