twitter-text 1.2.4 → 1.2.5

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+*.rbc
+.bundle
+pkg/*
+coverage
+doc

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/twitter-text-conformance"]
+	path = test/twitter-text-conformance
+	url = git://github.com/mzsanford/twitter-text-conformance.git

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format=nested

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify the gem's dependencies in twitter-text.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    twitter-text (1.2.5)
+      actionpack
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    abstract (1.0.0)
+    actionpack (3.0.1)
+      activemodel (= 3.0.1)
+      activesupport (= 3.0.1)
+      builder (~> 2.1.2)
+      erubis (~> 2.6.6)
+      i18n (~> 0.4.1)
+      rack (~> 1.2.1)
+      rack-mount (~> 0.6.12)
+      rack-test (~> 0.5.4)
+      tzinfo (~> 0.3.23)
+    activemodel (3.0.1)
+      activesupport (= 3.0.1)
+      builder (~> 2.1.2)
+      i18n (~> 0.4.1)
+    activesupport (3.0.1)
+    builder (2.1.2)
+    diff-lcs (1.1.2)
+    erubis (2.6.6)
+      abstract (>= 1.0.0)
+    i18n (0.4.2)
+    nokogiri (1.4.3.1)
+    nokogiri (1.4.3.1-java)
+      weakling (>= 0.0.3)
+    rack (1.2.1)
+    rack-mount (0.6.13)
+      rack (>= 1.0.0)
+    rack-test (0.5.6)
+      rack (>= 1.0)
+    rake (0.8.7)
+    rspec (2.1.0)
+      rspec-core (~> 2.1.0)
+      rspec-expectations (~> 2.1.0)
+      rspec-mocks (~> 2.1.0)
+    rspec-core (2.1.0)
+    rspec-expectations (2.1.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.1.0)
+    simplecov (0.3.7)
+      simplecov-html (>= 0.3.7)
+    simplecov-html (0.3.9)
+    tzinfo (0.3.23)
+    weakling (0.0.4-java)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  actionpack
+  nokogiri
+  rake
+  rspec
+  simplecov
+  twitter-text!

data/Rakefile

@@ -1,50 +1,13 @@
-require 'rubygems' unless ENV['NO_RUBYGEMS']
-require 'rake/gempackagetask'
-require 'rake/rdoctask'
-require 'rubygems/specification'
-require 'date'
-
-gem 'rspec'
-require 'spec/rake/spectask'
-require 'spec/rake/verify_rcov'
-require 'digest'
-
-spec = Gem::Specification.new do |s|
-  s.name = "twitter-text"
-  s.version = "1.2.4"
-  s.authors = ["Matt Sanford", "Patrick Ewing", "Ben Cherry", "Britt Selvitelle", "Raffi Krikorian"]
-  s.email = ["matt@twitter.com", "patrick.henry.ewing@gmail.com", "bcherry@gmail.com", "bs@brittspace.com", "raffi@twitter.com"]
-  s.homepage = "http://twitter.com"
-  s.description = s.summary = "A gem that provides text handling for Twitter"
-
-  s.platform = Gem::Platform::RUBY
-  s.has_rdoc = true
-  s.summary = "Twitter text handling library"
-
-  s.add_dependency "actionpack"
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
-  s.require_path = 'lib'
-  s.autorequire = ''
-  s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,spec}/**/*")
-end
-
-task :default => :spec
-
-desc "Run specs"
-Spec::Rake::SpecTask.new do |t|
-  t.spec_files = FileList['spec/**/*_spec.rb']
-  t.spec_opts = %w(-fs --color)
-  t.libs << ["spec", '.']
-end
+task :default => ["spec", "test:conformance"]
 
-desc "Run all examples with RCov"
-Spec::Rake::SpecTask.new('spec:rcov') do |t|
-  t.spec_files = FileList['spec/**/*.rb']
-  t.rcov = true
-  t.rcov_opts = ['--exclude', 'spec']
-end
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
 
 def conformance_version(dir)
+  require 'digest'
   Dir[File.join(dir, '*')].inject(Digest::SHA1.new){|digest, file| digest.update(Digest::SHA1.file(file).hexdigest) }
 end
 
@@ -85,6 +48,7 @@ namespace :test do
   end
 end
 
+require 'rake/rdoctask'
 namespace :doc do
   Rake::RDocTask.new do |rd|
     rd.main = "README.rdoc"
@@ -93,22 +57,6 @@ namespace :doc do
   end
 end
 
-Rake::GemPackageTask.new(spec) do |pkg|
-  pkg.gem_spec = spec
-end
-
-desc "install the gem locally"
-task :install => [:package] do
-  sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
-end
-
-desc "create a gemspec file"
-task :make_spec do
-  File.open("#{GEM}.gemspec", "w") do |file|
-    file.puts spec.to_ruby
-  end
-end
-
-desc "runs cruise control build"
+desc "Run cruise control build"
 task :cruise => [:spec, 'test:conformance'] do
 end

data/lib/autolink.rb

@@ -1,4 +1,3 @@
-
 module Twitter
   # A module for including Tweet auto-linking in a class. The primary use of this is for helpers/views so they can auto-link
   # usernames, lists, hashtags and URLs.
@@ -140,10 +139,17 @@ module Twitter
 
       text.gsub(Twitter::Regex[:valid_url]) do
         all, before, url, protocol, domain, path, query_string = $1, $2, $3, $4, $5, $6, $7
-        if !protocol.blank? || domain =~ Twitter::Regex[:probable_tld]
+        if !protocol.blank? # || domain =~ Twitter::Regex[:probable_tld_domain]
           html_attrs = tag_options(options.stringify_keys) || ""
           full_url = ((protocol =~ Twitter::Regex[:www] || protocol.blank?) ? "http://#{url}" : url)
           "#{before}<a href=\"#{html_escape(full_url)}\"#{html_attrs}>#{html_escape(url)}</a>"
+        elsif all =~ Twitter::Regex[:probable_tld_domain]
+          before_tld, tld_domain = $1, $2
+
+          html_attrs = tag_options(options.stringify_keys) || ""
+          full_url = "http://#{tld_domain}"
+          prefix = (before_tld == before ? before : "#{before}#{before_tld}")
+          "#{prefix}<a href=\"#{html_escape(full_url)}\"#{html_attrs}>#{html_escape(tld_domain)}</a>"
         else
           all
         end
@@ -151,4 +157,4 @@ module Twitter
     end
 
   end
-end
+end

data/lib/extractor.rb

@@ -20,18 +20,23 @@ class String
       char_array
     end
   end
+end
 
-  # Helper function to find the index of the <tt>sub_string</tt> in
-  # <tt>str</tt>.  This is needed because with unicode strings, the return
-  # of index may be incorrect.
-  def sub_string_search(sub_str, position = 0)
-    if respond_to? :codepoints
-      index(sub_str, position)
+# Helper functions to return character offsets instead of byte offsets.
+class MatchData
+  def char_begin(n)
+    if string.respond_to? :codepoints
+      self.begin(n)
     else
-      index = to_char_a[position..-1].each_with_index.find do |e|
-        to_char_a.slice(e.last + position, sub_str.char_length).map{|ci| ci.first }.join == sub_str
-      end
-      index.nil? ? -1 : index.last + position
+      string[0, self.begin(n)].char_length
+    end
+  end
+
+  def char_end(n)
+    if string.respond_to? :codepoints
+      self.end(n)
+    else
+      string[0, self.end(n)].char_length
     end
   end
 end
@@ -63,14 +68,14 @@ module Twitter
       return [] unless text
 
       possible_screen_names = []
-      position = 0
       text.to_s.scan(Twitter::Regex[:extract_mentions]) do |before, sn, after|
+        extract_mentions_match_data = $~
         unless after =~ Twitter::Regex[:end_screen_name_match]
-          start_position = text.to_s.sub_string_search(sn, position) - 1
-          position = start_position + sn.char_length + 1
+          start_position = extract_mentions_match_data.char_begin(2) - 1
+          end_position = extract_mentions_match_data.char_end(2)
           possible_screen_names << {
             :screen_name => sn,
-            :indices => [start_position, position]
+            :indices => [start_position, end_position]
           }
         end
       end
@@ -117,14 +122,22 @@ module Twitter
       urls = []
       position = 0
       text.to_s.scan(Twitter::Regex[:valid_url]) do |all, before, url, protocol, domain, path, query|
-        if !protocol.blank? || domain =~ Twitter::Regex[:probable_tld]
-          start_position = text.to_s.sub_string_search(url, position)
-          end_position = start_position + url.char_length
-          position = end_position
+        valid_url_match_data = $~
+        if !protocol.blank?
+          start_position = valid_url_match_data.char_begin(3)
+          end_position = valid_url_match_data.char_end(3)
           urls << {
             :url => ((protocol =~ Twitter::Regex[:www] || protocol.blank?) ? "http://#{url}" : url),
             :indices => [start_position, end_position]
           }
+        elsif all =~ Twitter::Regex[:probable_tld_domain]
+          tld_domain = $2
+          start_position = valid_url_match_data.char_begin(1) + $~.char_begin(2)
+          end_position = valid_url_match_data.char_begin(1) + $~.char_end(2)
+          urls << {
+            :url => "http://#{tld_domain}",
+            :indices => [start_position, end_position]
+          }
         end
       end
       urls.each{|url| yield url[:url], url[:indices].first, url[:indices].last } if block_given?
@@ -153,13 +166,12 @@ module Twitter
       return [] unless text
 
       tags = []
-      position = 0
       text.scan(Twitter::Regex[:auto_link_hashtags]) do |before, hash, hash_text|
-        start_position = text.to_s.sub_string_search(hash + hash_text, position)
-        position = start_position + hash_text.char_length + 1
+        start_position = $~.char_begin(2)
+        end_position = $~.char_end(3)
         tags << {
           :hashtag => hash_text,
-          :indices => [start_position, position]
+          :indices => [start_position, end_position]
         }
       end
       tags.each{|tag| yield tag[:hashtag], tag[:indices].first, tag[:indices].last } if block_given?

data/lib/hithighlighter.rb

@@ -1,4 +1,3 @@
-
 module Twitter
   # Module for doing "hit highlighting" on tweets that have been auto-linked already.
   # Useful with the results returned from the Search API.

data/lib/regex.rb

@@ -1,5 +1,4 @@
 # encoding: utf-8
-
 module Twitter
   # A collection of regular expressions for parsing Tweet text. The regular expression
   # list is frozen at load time to ensure immutability. These reular expressions are
@@ -31,15 +30,17 @@ module Twitter
     REGEXEN[:extract_mentions] = /(^|[^a-zA-Z0-9_])#{REGEXEN[:at_signs]}([a-zA-Z0-9_]{1,20})(?=(.|$))/o
     REGEXEN[:extract_reply] = /^(?:#{REGEXEN[:spaces]})*#{REGEXEN[:at_signs]}([a-zA-Z0-9_]{1,20})/o
 
-    major, minor, patch = RUBY_VERSION.split(/\./)
-    if major.to_i >= 1 && minor.to_i >= 9
+    major, minor, patch = RUBY_VERSION.split('.')
+    if major.to_i >= 2 || major.to_i == 1 && minor.to_i >= 9 || (defined?(RUBY_ENGINE) && ["jruby", "rbx"].include?(RUBY_ENGINE))
       REGEXEN[:list_name] = /[a-zA-Z][a-zA-Z0-9_\-\u0080-\u00ff]{0,24}/
     else
-      # This line barfs at compile time in Ruby 1.9.
+      # This line barfs at compile time in Ruby 1.9, JRuby, or Rubinius.
       REGEXEN[:list_name] = eval("/[a-zA-Z][a-zA-Z0-9_\\-\x80-\xff]{0,24}/")
     end
 
-    # Latin accented characters (subtracted 0xD7 from the range, it's a confusable multiplication sign. Looks like "x")
+    # Latin accented characters
+    # Excludes 0xd7 from the range (the multiplication sign, confusable with "x").
+    # Also excludes 0xf7, the division sign
     LATIN_ACCENTS = [(0xc0..0xd6).to_a, (0xd8..0xf6).to_a, (0xf8..0xff).to_a].flatten.pack('U*').freeze
     REGEXEN[:latin_accents] = /[#{LATIN_ACCENTS}]+/o
 
@@ -47,7 +48,7 @@ module Twitter
 
     # Characters considered valid in a hashtag but not at the beginning, where only a-z and 0-9 are valid.
     HASHTAG_CHARACTERS = /[a-z0-9_#{LATIN_ACCENTS}]/io
-    REGEXEN[:auto_link_hashtags] = /(^|[^0-9A-Z&\/\?]+)(#|＃)([0-9A-Z_]*[A-Z_]+#{HASHTAG_CHARACTERS}*)/io
+    REGEXEN[:auto_link_hashtags] = /(^|[^0-9A-Z&\/\?]+)(#|＃)([0-9a-z_]*[a-z_]+#{HASHTAG_CHARACTERS}*)/io
     REGEXEN[:auto_link_usernames_or_lists] = /([^a-zA-Z0-9_]|^|RT:?)([@＠]+)([a-zA-Z0-9_]{1,20})(\/[a-zA-Z][a-zA-Z0-9_\-]{0,24})?/o
     REGEXEN[:auto_link_emoticon] = /(8\-\#|8\-E|\+\-\(|\`\@|\`O|\<\|:~\(|\}:o\{|:\-\[|\>o\<|X\-\/|\[:-\]\-I\-|\/\/\/\/Ö\\\\\\\\|\(\|:\|\/\)|∑:\*\)|\( \| \))/
 
@@ -56,7 +57,7 @@ module Twitter
     REGEXEN[:valid_domain] = /(?:[^[:punct:]\s][\.-](?=[^[:punct:]\s])|[^[:punct:]\s]){1,}\.[a-z]{2,}(?::[0-9]+)?/i
 
     # For protocol-less URLs, we'll accept them if they end in one of a handful of likely TLDs
-    REGEXEN[:probable_tld] = /\.(?:com|net|org|gov|edu)$/i
+    REGEXEN[:probable_tld_domain] = /^(.*?)((?:[a-z0-9_\.\-]+)\.(?:com|net|org|gov|edu))$/i
 
     REGEXEN[:www] = /www\./i
 

data/lib/twitter-text.rb

@@ -1,15 +1,12 @@
-
 major, minor, patch = RUBY_VERSION.split('.')
 
-if major == 1 && minor < 9
+if major.to_i == 1 && minor.to_i < 9
   # Ruby 1.8 KCODE check. Not needed on 1.9 and later.
-  raise("twitter-text requires the $KCODE variable be set to 'UTF8' or 'u'") unless ['u','UTF8'].include?($KCODE)
+  raise("twitter-text requires the $KCODE variable be set to 'UTF8' or 'u'") unless $KCODE[0].chr =~ /u/i
 end
 
-require 'rubygems'
-
-# Needed for auto-linking
-gem 'actionpack'
+# External libraries required. (for gems, use: ruby -rubygems ...)
+require 'action_pack'
 require 'action_view'
 
 require File.join(File.dirname(__FILE__), 'regex')

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:newgem_simple, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:newgem_simple, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/spec/autolinking_spec.rb

@@ -36,7 +36,7 @@ describe Twitter::Autolink do
         def original_text; "meet@the beach"; end
 
         it "should not be linked" do
-          Hpricot(@autolinked_text).search('a').should be_blank
+          Nokogiri::HTML(@autolinked_text).search('a').should be_blank
         end
       end
 
@@ -132,7 +132,7 @@ describe Twitter::Autolink do
         def original_text; "hello @/my-list"; end
 
         it "should NOT be linked" do
-          Hpricot(@autolinked_text).search('a').should be_blank
+          Nokogiri::HTML(@autolinked_text).search('a').should be_blank
         end
       end
 
@@ -148,7 +148,7 @@ describe Twitter::Autolink do
         def original_text; "meet@the/beach"; end
 
         it "should not be linked" do
-          Hpricot(@autolinked_text).search('a').should be_blank
+          Nokogiri::HTML(@autolinked_text).search('a').should be_blank
         end
       end
 
@@ -304,9 +304,9 @@ describe Twitter::Autolink do
         def original_text; "#{[0xFF03].pack('U')}twj_dev"; end
 
         it "should be linked" do
-          link = Hpricot(@autolinked_text).at('a')
+          link = Nokogiri::HTML(@autolinked_text).search('a')
           (link.inner_text.respond_to?(:force_encoding) ? link.inner_text.force_encoding("utf-8") : link.inner_text).should == "#{[0xFF03].pack('U')}twj_dev"
-          link['href'].should == 'http://twitter.com/search?q=%23twj_dev'
+          link.first['href'].should == 'http://twitter.com/search?q=%23twj_dev'
         end
       end
 
@@ -468,9 +468,9 @@ describe Twitter::Autolink do
         def original_text; "I like www.foobar.com dudes"; end
 
         it "links to the original text with the full href" do
-          link = Hpricot(@autolinked_text).at('a')
+          link = Nokogiri::HTML(@autolinked_text).search('a')
           link.inner_text.should == 'www.foobar.com'
-          link['href'].should == 'http://www.foobar.com'
+          link.first['href'].should == 'http://www.foobar.com'
         end
       end
 

data/spec/spec_helper.rb

@@ -1,21 +1,27 @@
 $TESTING=true
+$KCODE='u'
 $:.push File.join(File.dirname(__FILE__), '..', 'lib')
 
-require 'twitter-text'
-require 'hpricot'
-require 'spec/test_urls'
+require 'nokogiri'
+require 'simplecov'
+SimpleCov.start do
+  add_group 'Libraries', 'lib'
+end
+
+require File.expand_path('../../lib/twitter-text', __FILE__)
+require File.expand_path('../test_urls', __FILE__)
 
-Spec::Runner.configure do |config|
+RSpec.configure do |config|
   config.include TestUrls
 end
 
-Spec::Matchers.define :match_autolink_expression do
+RSpec::Matchers.define :match_autolink_expression do
   match do |string|
     Twitter::Regex[:valid_url].match(string)
   end
 end
 
-Spec::Matchers.define :match_autolink_expression_in do |text|
+RSpec::Matchers.define :match_autolink_expression_in do |text|
   match do |url|
     @match_data = Twitter::Regex[:valid_url].match(text)
     @match_data && @match_data.to_s.strip == url
@@ -26,9 +32,9 @@ Spec::Matchers.define :match_autolink_expression_in do |text|
   end
 end
 
-Spec::Matchers.define :have_autolinked_url do |url|
+RSpec::Matchers.define :have_autolinked_url do |url|
   match do |text|
-    @link = Hpricot(text).at("a[@href='#{url}']")
+    @link = Nokogiri::HTML(text).search("a[@href='#{url}']")
     @link &&
     @link.inner_text &&
     @link.inner_text == url
@@ -39,10 +45,10 @@ Spec::Matchers.define :have_autolinked_url do |url|
   end
 end
 
-Spec::Matchers.define :link_to_screen_name do |screen_name|
+RSpec::Matchers.define :link_to_screen_name do |screen_name|
   match do |text|
-    @link = Hpricot(text).at("a.username")
-    @link && @link.inner_text == screen_name && "http://twitter.com/#{screen_name}".downcase.should == @link['href']
+    @link = Nokogiri::HTML(text).search("a.username")
+    @link && @link.inner_text == screen_name && "http://twitter.com/#{screen_name}".downcase.should == @link.first['href']
   end
 
   failure_message_for_should do |text|
@@ -58,10 +64,10 @@ Spec::Matchers.define :link_to_screen_name do |screen_name|
   end
 end
 
-Spec::Matchers.define :link_to_list_path do |list_path|
+RSpec::Matchers.define :link_to_list_path do |list_path|
   match do |text|
-    @link = Hpricot(text).at("a.list-slug")
-    !@link.nil? && @link.inner_text == list_path && "http://twitter.com/#{list_path}".downcase.should == @link['href']
+    @link = Nokogiri::HTML(text).search("a.list-slug")
+    !@link.nil? && @link.inner_text == list_path && "http://twitter.com/#{list_path}".downcase.should == @link.first['href']
   end
 
   failure_message_for_should do |text|
@@ -77,9 +83,9 @@ Spec::Matchers.define :link_to_list_path do |list_path|
   end
 end
 
-Spec::Matchers.define :have_autolinked_hashtag do |hashtag|
+RSpec::Matchers.define :have_autolinked_hashtag do |hashtag|
   match do |text|
-    @link = Hpricot(text).at("a[@href='http://twitter.com/search?q=#{CGI.escape hashtag}']")
+    @link = Nokogiri::HTML(text).search("a[@href='http://twitter.com/search?q=#{CGI.escape hashtag}']")
     @link &&
     @link.inner_text &&
     @link.inner_text == hashtag

data/spec/test_urls.rb

@@ -23,12 +23,12 @@ module TestUrls
     "http://x.com/has/one/char/domain",
     "http://t.co/nwcLTFF",
     # "t.co/nwcLTFF"
-  ]
+  ] unless defined?(TestUrls::VALID)
 
   INVALID = [
     "http://no-tld",
     "http://tld-too-short.x",
     "http://-doman_dash.com"
-  ]
+  ] unless defined?(TestUrls::INVALID)
 
 end

data/spec/twitter_text_spec.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+major, minor, patch = RUBY_VERSION.split('.')
+if major.to_i == 1 && minor.to_i < 9
+  describe "base" do
+    before do
+      $KCODE = 'NONE'
+    end
+
+    after do
+      $KCODE = 'u'
+    end
+
+    it "should raise with invalid KCODE on Ruby < 1.9" do
+      lambda do
+        require 'twitter-text'
+      end.should raise_error
+    end
+  end
+end

data/test/conformance_test.rb

@@ -0,0 +1,126 @@
+require 'test/unit'
+require 'yaml'
+$KCODE = 'UTF8'
+require File.dirname(__FILE__) + '/../lib/twitter-text'
+
+class ConformanceTest < Test::Unit::TestCase
+  include Twitter::Extractor
+  include Twitter::Autolink
+  include Twitter::HitHighlighter
+
+  def setup
+    @conformance_dir = ENV['CONFORMANCE_DIR'] || File.join(File.dirname(__FILE__), 'twitter-text-conformance')
+  end
+
+  module ExtractorConformance
+    def test_replies_extractor_conformance
+      run_conformance_test(File.join(@conformance_dir, 'extract.yml'), :replies) do |description, expected, input|
+        assert_equal expected, extract_reply_screen_name(input), description
+      end
+    end
+
+    def test_mentions_extractor_conformance
+      run_conformance_test(File.join(@conformance_dir, 'extract.yml'), :mentions) do |description, expected, input|
+        assert_equal expected, extract_mentioned_screen_names(input), description
+      end
+    end
+
+    def test_mentions_with_indices_extractor_conformance
+      run_conformance_test(File.join(@conformance_dir, 'extract.yml'), :mentions_with_indices) do |description, expected, input|
+        expected = expected.map{|elem| elem.symbolize_keys }
+        assert_equal expected, extract_mentioned_screen_names_with_indices(input), description
+      end
+    end
+
+    def test_url_extractor_conformance
+      run_conformance_test(File.join(@conformance_dir, 'extract.yml'), :urls) do |description, expected, input|
+        assert_equal expected, extract_urls(input), description
+      end
+    end
+
+    def test_urls_with_indices_extractor_conformance
+      run_conformance_test(File.join(@conformance_dir, 'extract.yml'), :urls_with_indices) do |description, expected, input|
+        expected = expected.map{|elem| elem.symbolize_keys }
+        assert_equal expected, extract_urls_with_indices(input), description
+      end
+    end
+
+    def test_hashtag_extractor_conformance
+      run_conformance_test(File.join(@conformance_dir, 'extract.yml'), :hashtags) do |description, expected, input|
+        assert_equal expected, extract_hashtags(input), description
+      end
+    end
+
+    def test_hashtags_with_indices_extractor_conformance
+      run_conformance_test(File.join(@conformance_dir, 'extract.yml'), :hashtags_with_indices) do |description, expected, input|
+        expected = expected.map{|elem| elem.symbolize_keys }
+        assert_equal expected, extract_hashtags_with_indices(input), description
+      end
+    end
+  end
+  include ExtractorConformance
+
+  module AutolinkConformance
+    def test_users_autolink_conformance
+      run_conformance_test(File.join(@conformance_dir, 'autolink.yml'), :usernames) do |description, expected, input|
+        assert_equal expected, auto_link_usernames_or_lists(input, :suppress_no_follow => true), description
+      end
+    end
+
+    def test_lists_autolink_conformance
+      run_conformance_test(File.join(@conformance_dir, 'autolink.yml'), :lists) do |description, expected, input|
+        assert_equal expected, auto_link_usernames_or_lists(input, :suppress_no_follow => true), description
+      end
+    end
+
+    def test_urls_autolink_conformance
+      run_conformance_test(File.join(@conformance_dir, 'autolink.yml'), :urls) do |description, expected, input|
+        assert_equal expected, auto_link_urls_custom(input, :suppress_no_follow => true), description
+      end
+    end
+
+    def test_hashtags_autolink_conformance
+      run_conformance_test(File.join(@conformance_dir, 'autolink.yml'), :hashtags) do |description, expected, input|
+        assert_equal expected, auto_link_hashtags(input, :suppress_no_follow => true), description
+      end
+    end
+
+    def test_all_autolink_conformance
+      run_conformance_test(File.join(@conformance_dir, 'autolink.yml'), :all) do |description, expected, input|
+        assert_equal expected, auto_link(input, :suppress_no_follow => true), description
+      end
+    end
+  end
+  include AutolinkConformance
+
+  module HitHighlighterConformance
+
+    def test_plain_text_conformance
+      run_conformance_test(File.join(@conformance_dir, 'hit_highlighting.yml'), :plain_text, true) do |config|
+        assert_equal config['expected'], hit_highlight(config['text'], config['hits']), config['description']
+      end
+    end
+
+    def test_with_links_conformance
+      run_conformance_test(File.join(@conformance_dir, 'hit_highlighting.yml'), :with_links, true) do |config|
+        assert_equal config['expected'], hit_highlight(config['text'], config['hits']), config['description']
+      end
+    end
+  end
+  include HitHighlighterConformance
+
+  private
+
+  def run_conformance_test(file, test_type, hash_config = false, &block)
+    yaml = YAML.load_file(file)
+    assert yaml["tests"][test_type.to_s], "No such test suite: #{test_type.to_s}"
+
+    yaml["tests"][test_type.to_s].each do |test_info|
+      if hash_config
+        yield test_info
+      else
+        yield test_info['description'], test_info['expected'], test_info['text']
+      end
+    end
+  end
+end

data/twitter-text.gemspec

@@ -0,0 +1,23 @@
+spec = Gem::Specification.new do |s|
+  s.name = "twitter-text"
+  s.version = "1.2.5"
+  s.authors = ["Matt Sanford", "Patrick Ewing", "Ben Cherry", "Britt Selvitelle", "Raffi Krikorian"]
+  s.email = ["matt@twitter.com", "patrick.henry.ewing@gmail.com", "bcherry@gmail.com", "bs@brittspace.com", "raffi@twitter.com"]
+  s.homepage = "http://twitter.com"
+  s.description = s.summary = "A gem that provides text handling for Twitter"
+
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.summary = "Twitter text handling library"
+
+  s.add_development_dependency "nokogiri"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "simplecov"
+  s.add_runtime_dependency "actionpack"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: twitter-text
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 21
   prerelease: false
   segments: 
   - 1
   - 2
-  - 4
-  version: 1.2.4
+  - 5
+  version: 1.2.5
 platform: ruby
 authors: 
 - Matt Sanford
@@ -15,15 +15,15 @@ authors:
 - Ben Cherry
 - Britt Selvitelle
 - Raffi Krikorian
-autorequire: ""
+autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2010-11-03 00:00:00 -07:00
+date: 2010-11-18 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: actionpack
+  name: nokogiri
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
@@ -34,8 +34,64 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :runtime
+  type: :development
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: simplecov
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: actionpack
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
 description: A gem that provides text handling for Twitter
 email: 
 - matt@twitter.com
@@ -50,6 +106,11 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- .gitignore
+- .gitmodules
+- .rspec
+- Gemfile
+- Gemfile.lock
 - LICENSE
 - README.rdoc
 - Rakefile
@@ -61,14 +122,19 @@ files:
 - lib/twitter-text.rb
 - lib/unicode.rb
 - lib/validation.rb
+- script/destroy
+- script/generate
 - spec/autolinking_spec.rb
 - spec/extractor_spec.rb
 - spec/hithighlighter_spec.rb
 - spec/regex_spec.rb
 - spec/spec_helper.rb
 - spec/test_urls.rb
+- spec/twitter_text_spec.rb
 - spec/unicode_spec.rb
 - spec/validation_spec.rb
+- test/conformance_test.rb
+- twitter-text.gemspec
 has_rdoc: true
 homepage: http://twitter.com
 licenses: []
@@ -103,5 +169,14 @@ rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Twitter text handling library
-test_files: []
-
+test_files: 
+- spec/autolinking_spec.rb
+- spec/extractor_spec.rb
+- spec/hithighlighter_spec.rb
+- spec/regex_spec.rb
+- spec/spec_helper.rb
+- spec/test_urls.rb
+- spec/twitter_text_spec.rb
+- spec/unicode_spec.rb
+- spec/validation_spec.rb
+- test/conformance_test.rb