twitter-text 1.0 → 1.0.1

data/Rakefile

@@ -5,11 +5,12 @@ require 'rubygems/specification'
 require 'date'
 require 'spec/rake/spectask'
 require 'spec/rake/verify_rcov'
+require 'digest'
 
 
 spec = Gem::Specification.new do |s|
   s.name = "twitter-text"
-  s.version = "1.0"
+  s.version = "1.0.1"
   s.author = "Matt Sanford"
   s.email = "matt@twitter.com"
   s.homepage = "http://twitter.com"
@@ -19,7 +20,7 @@ spec = Gem::Specification.new do |s|
   s.has_rdoc = true
   s.summary = "Twitter text handling library"
 
-  s.add_dependency "action_view"
+  s.add_dependency "actionpack"
 
   s.require_path = 'lib'
   s.autorequire = ''
@@ -41,16 +42,39 @@ Spec::Rake::SpecTask.new('spec:rcov') do |t|
   t.rcov_opts = ['--exclude', 'spec']
 end
 
+
+def conformance_version(dir)
+  Dir[File.join(dir, '*')].inject(Digest::SHA1.new){|digest, file| digest.update(Digest::SHA1.file(file).hexdigest) }
+end
+
 namespace :test do
   namespace :conformance do
+
+
     desc "Update conformance testing data"
     task :update do
-      dir = File.join(File.dirname(__FILE__), "test", "twitter-text-conformance")
       puts "Updating conformance data ... "
-      system("cd #{dir} && git pull origin master") || exit(1)
+      system("git submodule init") || raise("Failed to init submodule")
+      system("git submodule update") || raise("Failed to update submodule")
       puts "Updating conformance data ... DONE"
     end
 
+    desc "Change conformance test data to the lastest version"
+    task :latest => ['conformance:update'] do
+      current_dir = File.dirname(__FILE__)
+      submodule_dir = File.join(File.dirname(__FILE__), "test", "twitter-text-conformance")
+      version_before = conformance_version(submodule_dir)
+      system("cd #{submodule_dir} && git pull origin master") || raise("Failed to pull submodule version")
+      system("cd #{current_dir}")
+      if conformance_version(submodule_dir) != version_before
+        system("cd #{current_dir} && git add #{submodule_dir}") || raise("Failed to add upgrade files")
+        system("git commit -m \"Upgraded to the latest conformance suite\" #{submodule_dir}") || raise("Failed to commit upgraded conformacne data")
+        puts "Upgraded conformance suite."
+      else
+        puts "No conformance suite changes."
+      end
+    end
+
     desc "Run conformance test suite"
     task :run do
       ruby "test/conformance_test.rb"
@@ -58,7 +82,7 @@ namespace :test do
   end
 
   desc "Run conformance test suite"
-  task :conformance => ['conformance:update', 'conformance:run'] do
+  task :conformance => ['conformance:latest', 'conformance:run'] do
   end
 end
 

data/lib/autolink.rb

@@ -5,6 +5,8 @@ module Twitter
   module Autolink
     include ActionView::Helpers::TagHelper #tag_options needed by auto_link
 
+    WWW_REGEX = /www\./i #:nodoc:
+
     # Default CSS class for auto-linked URLs
     DEFAULT_URL_CLASS = "tweet-url"
     # Default CSS class for auto-linked lists (along with the url class)
@@ -13,6 +15,8 @@ module Twitter
     DEFAULT_USERNAME_CLASS = "username"
     # Default CSS class for auto-linked hashtags (along with the url class)
     DEFAULT_HASHTAG_CLASS = "hashtag"
+    # HTML attribute for robot nofollow behavior (default)
+    HTML_ATTR_NO_FOLLOW = " rel=\"nofollow\""
 
     # Add <tt><a></a></tt> tags around the usernames, lists, hashtags and URLs in the provided <tt>text</tt>. The
     # <tt><a></tt> tags can be controlled with the following entries in the <tt>options</tt>
@@ -25,9 +29,14 @@ module Twitter
     # <tt>:username_url_base</tt>::      the value for <tt>href</tt> attribute on username links. The <tt>@username</tt> (minus the <tt>@</tt>) will be appended at the end of this.
     # <tt>:list_url_base</tt>::      the value for <tt>href</tt> attribute on list links. The <tt>@username/list</tt> (minus the <tt>@</tt>) will be appended at the end of this.
     # <tt>:hashtag_url_base</tt>::      the value for <tt>href</tt> attribute on hashtag links. The <tt>#hashtag</tt> (minus the <tt>#</tt>) will be appended at the end of this.
-    def auto_link(text, options = {}, &block) # :yields: hashtag_or_list_or_username
-      options = options.dup
-      auto_link_usernames_or_lists(auto_link_urls_custom(auto_link_hashtags(text), options, &block), &block)
+    # <tt>:suppress_lists</tt>::    disable auto-linking to lists
+    # <tt>:suppress_no_follow</tt>::   Do not add <tt>rel="nofollow"</tt> to auto-linked items
+    def auto_link(text, options = {})
+      auto_link_usernames_or_lists(
+        auto_link_urls_custom(
+          auto_link_hashtags(text, options),
+        options),
+      options)
     end
 
     # Add <tt><a></a></tt> tags around the usernames and lists in the provided <tt>text</tt>. The
@@ -39,6 +48,8 @@ module Twitter
     # <tt>:username_class</tt>::    class to add to username <tt><a></tt> tags
     # <tt>:username_url_base</tt>::      the value for <tt>href</tt> attribute on username links. The <tt>@username</tt> (minus the <tt>@</tt>) will be appended at the end of this.
     # <tt>:list_url_base</tt>::      the value for <tt>href</tt> attribute on list links. The <tt>@username/list</tt> (minus the <tt>@</tt>) will be appended at the end of this.
+    # <tt>:suppress_lists</tt>::    disable auto-linking to lists
+    # <tt>:suppress_no_follow</tt>::   Do not add <tt>rel="nofollow"</tt> to auto-linked items
     def auto_link_usernames_or_lists(text, options = {}) # :yields: list_or_username
       options = options.dup
       options[:url_class] ||= DEFAULT_URL_CLASS
@@ -46,18 +57,19 @@ module Twitter
       options[:username_class] ||= DEFAULT_USERNAME_CLASS
       options[:username_url_base] ||= "http://twitter.com/"
       options[:list_url_base] ||= "http://twitter.com/"
+      extra_html = HTML_ATTR_NO_FOLLOW unless options[:suppress_no_follow]
 
       text.gsub(Twitter::Regex[:auto_link_usernames_or_lists]) do
         if $4 && !options[:suppress_lists]
           # the link is a list
           text = list = "#{$3}#{$4}"
           text = yield(list) if block_given?
-          "#{$1}#{$2}<a class=\"#{options[:url_class]} #{options[:list_class]}\" href=\"#{options[:list_url_base]}#{list.downcase}\">#{text}</a>"
+          "#{$1}#{$2}<a class=\"#{options[:url_class]} #{options[:list_class]}\" href=\"#{options[:list_url_base]}#{list.downcase}\"#{extra_html}>#{text}</a>"
         else
           # this is a screen name
           text = $3
           text = yield(text) if block_given?
-          "#{$1}#{$2}<a class=\"#{options[:url_class]} #{options[:username_class]}\" href=\"#{options[:username_url_base]}#{$3}\">#{text}</a>"
+          "#{$1}#{$2}<a class=\"#{options[:url_class]} #{options[:username_class]}\" href=\"#{options[:username_url_base]}#{text}\"#{extra_html}>#{text}</a>"
         end
       end
     end
@@ -69,31 +81,36 @@ module Twitter
     # <tt>:url_class</tt>::     class to add to all <tt><a></tt> tags
     # <tt>:hashtag_class</tt>:: class to add to hashtag <tt><a></tt> tags
     # <tt>:hashtag_url_base</tt>::      the value for <tt>href</tt> attribute. The hashtag text (minus the <tt>#</tt>) will be appended at the end of this.
-    #
+    # <tt>:suppress_no_follow</tt>::   Do not add <tt>rel="nofollow"</tt> to auto-linked items
     def auto_link_hashtags(text, options = {})  # :yields: hashtag_text
       options = options.dup
       options[:url_class] ||= DEFAULT_URL_CLASS
       options[:hashtag_class] ||= DEFAULT_HASHTAG_CLASS
       options[:hashtag_url_base] ||= "http://twitter.com/search?q=%23"
+      extra_html = HTML_ATTR_NO_FOLLOW unless options[:suppress_no_follow]
 
       text.gsub(Twitter::Regex[:auto_link_hashtags]) do
         before = $1
         hash = $2
         text = $3
         text = yield(text) if block_given?
-        "#{before}<a href=\"#{options[:hashtag_url_base]}#{text}\" title=\"##{text}\" class=\"#{options[:url_class]} #{options[:hashtag_class]}\">#{hash}#{text}</a>"
+        "#{before}<a href=\"#{options[:hashtag_url_base]}#{text}\" title=\"##{text}\" class=\"#{options[:url_class]} #{options[:hashtag_class]}\"#{extra_html}>#{hash}#{text}</a>"
       end
     end
 
     # Add <tt><a></a></tt> tags around the URLs in the provided <tt>text</tt>. Any
     # elements in the <tt>href_options</tt> hash will be converted to HTML attributes
-    # and place in the <tt><a></tt> tag.
+    # and place in the <tt><a></tt> tag. Unless <tt>href_options</tt> contains <tt>:suppress_no_follow</tt>
+    # the <tt>rel="nofollow"</tt> attribute will be added.
     def auto_link_urls_custom(text, href_options = {})
+      options = href_options.dup
+      options[:rel] = "nofollow" unless options.delete(:suppress_no_follow)
+
       text.gsub(Twitter::Regex[:valid_url]) do
         all, before, url, protocol = $1, $2, $3, $4
-        options = tag_options(href_options.stringify_keys) || ""
-        full_url = (protocol == "www." ? "http://#{url}" : url)
-        "#{before}<a href=\"#{full_url}\"#{options}>#{url}</a>"
+        html_attrs = tag_options(options.stringify_keys) || ""
+        full_url = (protocol =~ WWW_REGEX ? "http://#{url}" : url)
+        "#{before}<a href=\"#{full_url}\"#{html_attrs}>#{url}</a>"
       end
     end
 

data/lib/regex.rb

@@ -26,7 +26,7 @@ module Twitter
         ].flatten.freeze
     REGEXEN[:spaces] = Regexp.new(UNICODE_SPACES.collect{ |e| [e].pack 'U*' }.join('|'))
 
-    REGEXEN[:extract_mentions] = /(^|[^a-zA-Z0-9_])[@＠]([a-zA-Z0-9_]{1,20})/
+    REGEXEN[:extract_mentions] = /(^|[^a-zA-Z0-9_])[@＠]([a-zA-Z0-9_]{1,20})(?!@)/
     REGEXEN[:extract_reply] = /^(?:#{REGEXEN[:spaces]})*[@＠]([a-zA-Z0-9_]{1,20})/o
 
     REGEXEN[:list_name] = /^[a-zA-Z\x80-\xff].{0,79}$/
@@ -42,14 +42,14 @@ module Twitter
     REGEXEN[:auto_link_emoticon] = /(8\-\#|8\-E|\+\-\(|\`\@|\`O|\<\|:~\(|\}:o\{|:\-\[|\>o\<|X\-\/|\[:-\]\-I\-|\/\/\/\/Ö\\\\\\\\|\(\|:\|\/\)|∑:\*\)|\( \| \))/
 
     # URL related hash regex collection
-    REGEXEN[:valid_preceeding_chars] = /(?:[^\/"':!=]|^)/
-    REGEXEN[:valid_domain] = /[a-z0-9\.-]+\.[a-z]{2,}(?::[0-9]+)?/i
-    REGEXEN[:valid_url_path_chars] = /[a-z0-9!\*'\(\);:@&=\+\$\/%#\[\]\-_\.,~]/i
+    REGEXEN[:valid_preceeding_chars] = /(?:[^\/"':!=]|^|\:)/
+    REGEXEN[:valid_domain] = /(?:[\.-]|[^[:punct:]])+\.[a-z]{2,}(?::[0-9]+)?/i
+    REGEXEN[:valid_url_path_chars] = /[a-z0-9!\*'\(\);:&=\+\$\/%#\[\]\-_\.,~]/i
     # Valid end-of-path chracters (so /foo. does not gobble the period).
     #   1. Allow ) for Wikipedia URLs.
     #   2. Allow =&# for empty URL parameters and other URL-join artifacts
     REGEXEN[:valid_url_path_ending_chars] = /[a-z0-9\)=#\/]/i
-    REGEXEN[:valid_url_query_chars] = /[a-z0-9!\*'\(\);:@&=\+\$\/%#\[\]\-_\.,~]/i
+    REGEXEN[:valid_url_query_chars] = /[a-z0-9!\*'\(\);:&=\+\$\/%#\[\]\-_\.,~]/i
     REGEXEN[:valid_url_query_ending_chars] = /[a-z0-9_&=#]/i
     REGEXEN[:valid_url] = %r{
       (                                                                                     #   $1 total match
@@ -71,4 +71,4 @@ module Twitter
       REGEXEN[key]
     end
   end
-end
+end

data/spec/autolinking_spec.rb

@@ -346,6 +346,14 @@ describe Twitter::Autolink do
         end
       end
 
+      context "when preceded by a :" do
+        def original_text; "Check this out @hoverbird:#{url}"; end
+
+        it "should be linked" do
+          @autolinked_text.should have_autolinked_url(url)
+        end
+      end
+
       context "with a URL ending in allowed punctuation" do
         it "does not consume ending punctuation" do
           matcher = TestAutolink.new
@@ -358,7 +366,7 @@ describe Twitter::Autolink do
       context "with a URL preceded in forbidden characters" do
         it "should not be linked" do
           matcher = TestAutolink.new
-          %w| \ ' / : ! = |.each do |char|
+          %w| \ ' / ! = |.each do |char|
             matcher.auto_link("#{char}#{url}").should_not have_autolinked_url(url)
           end
         end
@@ -424,4 +432,4 @@ describe Twitter::Autolink do
     end
   end
 
-end
+end

data/spec/extractor_spec.rb

@@ -108,8 +108,6 @@ describe Twitter::Extractor do
         "http://www.boingboing.net/2007/02/14/katamari_damacy_phon.html",
         "http://somehost.com:3000",
         "http://x.com/~matthew+%-x",
-        "http://x.com/~matthew+%-,.;@:x",
-        "http://x.com/,.;@:x",
         "http://en.wikipedia.org/wiki/Primer_(film)",
         "http://www.ams.org/bookstore-getitem/item=mbk-59",
         "http://chilp.it/?77e8fd",

data/spec/regex_spec.rb

@@ -11,11 +11,15 @@ describe "Twitter::Regex regular expressions" do
       "http://www.boingboing.net/2007/02/14/katamari_damacy_phon.html",
       "http://somehost.com:3000",
       "http://x.com/~matthew+%-x",
-      "http://x.com/~matthew+%-,.;@:x",
-      "http://x.com/,.;@:x",
       "http://en.wikipedia.org/wiki/Primer_(film)",
       "http://www.ams.org/bookstore-getitem/item=mbk-59",
       "http://chilp.it/?77e8fd",
+      "www.foobar.com",
+      "WWW.FOOBAR.COM",
+      "http://tell.me/why",
+      "http://longtlds.mobi",
+      "http://✪df.ws/ejp",
+      "http://日本.com"
     ]
 
     @urls.each do |url|
@@ -31,14 +35,20 @@ describe "Twitter::Regex regular expressions" do
   end
 
   describe "invalid URLS" do
-   it "does not link urls with invalid_domains" do
+    it "does not link urls with invalid characters" do
       [ "http://doman-dash_2314352345_dfasd.foo-cow_4352.com",
         "http://no-tld",
         "http://tld-too-short.x",
         "http://x.com/,,,/.../@@@/;;;/:::/---/%%%x",
-        "http://doman-dash_2314352345_dfasd.foo-cow_4352.com",
+        "http://doman_dash_2314352345_dfasd.foo-cow_4352.com",
       ].each {|url| url.should_not have_autolinked_url(url)}
     end
+    
+    it "does not link domains beginning with a hypen" do
+      pending
+      "http://-doman_dash_2314352345_dfasd.com".should_not match_autolink_expression
+    end
+    
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: twitter-text
 version: !ruby/object:Gem::Version 
-  version: "1.0"
+  version: 1.0.1
 platform: ruby
 authors: 
 - Matt Sanford
@@ -9,11 +9,11 @@ autorequire: ""
 bindir: bin
 cert_chain: []
 
-date: 2010-01-27 00:00:00 -08:00
+date: 2010-02-10 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: action_view
+  name: actionpack
   type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement