incollege-text 1.12.0

data/spec/spec_helper.rb

@@ -0,0 +1,128 @@
+$TESTING=true
+
+# Ruby 1.8 encoding check
+major, minor, patch = RUBY_VERSION.split('.')
+if major.to_i == 1 && minor.to_i < 9
+  $KCODE='u'
+end
+
+$:.push File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'nokogiri'
+require 'json'
+require 'simplecov'
+SimpleCov.start do
+  add_group 'Libraries', 'lib'
+end
+
+require File.expand_path('../../lib/incollege-text', __FILE__)
+require File.expand_path('../test_urls', __FILE__)
+
+RSpec.configure do |config|
+  config.include TestUrls
+end
+
+RSpec::Matchers.define :match_autolink_expression do
+  match do |string|
+    !Twitter::Extractor.extract_urls(string).empty?
+  end
+end
+
+RSpec::Matchers.define :match_autolink_expression_in do |text|
+  match do |url|
+    @match_data = Twitter::Regex[:valid_url].match(text)
+    @match_data && @match_data.to_s.strip == url
+  end
+
+  failure_message_for_should do |url|
+    "Expected to find url '#{url}' in text '#{text}', but the match was #{@match_data.captures}'"
+  end
+end
+
+RSpec::Matchers.define :have_autolinked_url do |url, inner_text|
+  match do |text|
+    @link = Nokogiri::HTML(text).search("a[@href='#{url}']")
+    @link &&
+    @link.inner_text &&
+    (inner_text && @link.inner_text == inner_text) || (!inner_text && @link.inner_text == url)
+  end
+
+  failure_message_for_should do |text|
+    "Expected url '#{url}'#{", inner_text '#{inner_text}'" if inner_text} to be autolinked in '#{text}'"
+  end
+end
+
+RSpec::Matchers.define :link_to_screen_name do |screen_name, inner_text|
+  expected = inner_text ? inner_text : screen_name
+
+  match do |text|
+    @link = Nokogiri::HTML(text).search("a.username")
+    @link &&
+    @link.inner_text == expected &&
+    "https://incollege.com/#{screen_name}".should == @link.first['href']
+  end
+
+  failure_message_for_should do |text|
+    if @link.first
+      "Expected link '#{@link.inner_text}' with href '#{@link.first['href']}' to match screen_name '#{expected}', but it does not."
+    else
+      "Expected screen name '#{screen_name}' to be autolinked in '#{text}', but no link was found."
+    end
+  end
+
+  failure_message_for_should_not do |text|
+    "Expected link '#{@link.inner_text}' with href '#{@link.first['href']}' not to match screen_name '#{expected}', but it does."
+  end
+
+  description do
+    "contain a link with the name and href pointing to the expected screen_name"
+  end
+end
+
+RSpec::Matchers.define :link_to_list_path do |list_path, inner_text|
+  expected = inner_text ? inner_text : list_path
+
+  match do |text|
+    @link = Nokogiri::HTML(text).search("a.list-slug")
+    @link &&
+    @link.inner_text == expected &&
+    "https://incollege.com/#{list_path}".downcase.should == @link.first['href']
+  end
+
+  failure_message_for_should do |text|
+    if @link.first
+      "Expected link '#{@link.inner_text}' with href '#{@link.first['href']}' to match the list path '#{expected}', but it does not."
+    else
+      "Expected list path '#{list_path}' to be autolinked in '#{text}', but no link was found."
+    end
+  end
+
+  failure_message_for_should_not do |text|
+    "Expected link '#{@link.inner_text}' with href '#{@link.first['href']}' not to match the list path '#{expected}', but it does."
+  end
+
+  description do
+    "contain a link with the list title and an href pointing to the list path"
+  end
+end
+
+RSpec::Matchers.define :have_autolinked_hashtag do |hashtag|
+  match do |text|
+    @link = Nokogiri::HTML(text).search("a[@href='https://incollege.com/#!/search?q=#{hashtag.sub(/^#/, '%23')}']")
+    @link &&
+    @link.inner_text &&
+    @link.inner_text == hashtag
+  end
+
+  failure_message_for_should do |text|
+    if @link.first
+      "Expected link text to be [#{hashtag}], but it was [#{@link.inner_text}] in #{text}"
+    else
+      "Expected hashtag #{hashtag} to be autolinked in '#{text}', but no link was found."
+    end
+  end
+
+  failure_message_for_should_not do |text|
+    "Expected link '#{@link.inner_text}' with href '#{@link.first['href']}' not to match the hashtag '#{hashtag}', but it does."
+  end
+end

data/spec/test_urls.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+
+module TestUrls
+  VALID = [
+    "http://google.com",
+    "http://foobar.com/#",
+    "http://google.com/#foo",
+    "http://google.com/#search?q=iphone%20-filter%3Alinks",
+    "http://twitter.com/#search?q=iphone%20-filter%3Alinks",
+    "http://somedomain.com/index.php?path=/abc/def/",
+    "http://www.boingboing.net/2007/02/14/katamari_damacy_phon.html",
+    "http://somehost.com:3000",
+    "http://xo.com/~matthew+%-x",
+    "http://en.wikipedia.org/wiki/Primer_(film)",
+    "http://www.ams.org/bookstore-getitem/item=mbk-59",
+    "http://chilp.it/?77e8fd",
+    "http://tell.me/why",
+    "http://longtlds.info",
+    "http://✪df.ws/ejp",
+    "http://日本.com",
+    "http://search.twitter.com/search?q=avro&lang=en",
+    "http://mrs.domain-dash.biz",
+    "http://x.com/has/one/char/domain",
+    "http://t.co/nwcLTFF",
+    "http://sub_domain-dash.twitter.com",
+    "http://a.b.cd",
+    "http://a_b.c-d.com",
+    "http://a-b.b.com",
+    "http://twitter-dash.com",
+    "http://msdn.microsoft.com/ja-jp/library/system.net.httpwebrequest(v=VS.100).aspx",
+    "www.foobar.com",
+    "WWW.FOOBAR.COM",
+    "www.foobar.co.jp",
+    "http://t.co",
+    "t.co/nwcLTFF",
+    "http://foobar.みんな",
+    "http://foobar.中国",
+    "http://foobar.پاکستان",
+    "https://www.youtube.com/playlist?list=PL0ZPu8XSRTB7wZzn0mLHMvyzVFeRxbWn-"
+  ] unless defined?(TestUrls::VALID)
+
+  INVALID = [
+    "http://no-tld",
+    "http://tld-too-short.x",
+    "http://-doman_dash.com",
+    "http://_leadingunderscore.twitter.com",
+    "http://trailingunderscore_.twitter.com",
+    "http://-leadingdash.twitter.com",
+    "http://trailingdash-.twitter.com",
+    "http://-leadingdash.com",
+    "http://trailingdash-.com",
+    "http://no_underscores.com",
+    "http://test.c_o_m",
+    "http://test.c-o-m",
+    "http://twitt#{[0x202A].pack('U')}er.com",
+    "http://twitt#{[0x202B].pack('U')}er.com",
+    "http://twitt#{[0x202C].pack('U')}er.com",
+    "http://twitt#{[0x202D].pack('U')}er.com",
+    "http://twitt#{[0x202E].pack('U')}er.com"
+  ] unless defined?(TestUrls::INVALID)
+
+  TCO = [
+    "http://t.co/P53cv5yO!",
+    "http://t.co/fQJmiPGg***",
+    "http://t.co/pbY2NfTZ's",
+    "http://t.co/2vYHpAc5;",
+    "http://t.co/ulYGBYSo:",
+    "http://t.co/GeT4bSiw=win",
+    "http://t.co/8MkmHU0k+fun",
+    "http://t.co/TKLp64dY.yes,",
+    "http://t.co/8vuO27cI$$",
+    "http://t.co/rPYTvdA8/",
+    "http://t.co/WvtMw5ku%",
+    "http://t.co/8t7G3ddS#",
+    "http://t.co/nfHNJDV2/#!",
+    "http://t.co/gK6NOXHs[good]",
+    "http://t.co/dMrT0o1Y]bad",
+    "http://t.co/FNkPfmii-",
+    "http://t.co/sMgS3pjI_oh",
+    "http://t.co/F8Dq3Plb~",
+    "http://t.co/ivvH58vC&help",
+    "http://t.co/iUBL15zD|NZ5KYLQ8"
+  ] unless defined?(TestUrls::TCO)
+end

data/spec/unicode_spec.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Incollege::Unicode do
+
+  it "should lazy-init constants" do
+    Incollege::Unicode.const_defined?(:UFEB6).should == false
+    Incollege::Unicode::UFEB6.should_not be_nil
+    Incollege::Unicode::UFEB6.should be_kind_of(String)
+    Incollege::Unicode.const_defined?(:UFEB6).should == true
+  end
+
+  it "should return corresponding character" do
+    Incollege::Unicode::UFEB6.should == [0xfeb6].pack('U')
+  end
+
+  it "should allow lowercase notation" do
+    Incollege::Unicode::Ufeb6.should == Incollege::Unicode::UFEB6
+    Incollege::Unicode::Ufeb6.should === Incollege::Unicode::UFEB6
+  end
+
+  it "should allow underscore notation" do
+    Incollege::Unicode::U_FEB6.should == Incollege::Unicode::UFEB6
+    Incollege::Unicode::U_FEB6.should === Incollege::Unicode::UFEB6
+  end
+
+  it "should raise on invalid codepoints" do
+    lambda { Incollege::Unicode::FFFFFF }.should raise_error(NameError)
+  end
+
+end

data/spec/validation_spec.rb

@@ -0,0 +1,43 @@
+# encoding: binary
+require File.dirname(__FILE__) + '/spec_helper'
+
+class TestValidation
+  include Incollege::Validation
+end
+
+describe Incollege::Validation do
+
+  it "should disallow invalid BOM character" do
+    TestValidation.new.tweet_invalid?("Bom:#{Incollege::Unicode::UFFFE}").should == :invalid_characters
+    TestValidation.new.tweet_invalid?("Bom:#{Incollege::Unicode::UFEFF}").should == :invalid_characters
+  end
+
+  it "should disallow invalid U+FFFF character" do
+    TestValidation.new.tweet_invalid?("Bom:#{Incollege::Unicode::UFFFF}").should == :invalid_characters
+  end
+
+  it "should disallow direction change characters" do
+    [0x202A, 0x202B, 0x202C, 0x202D, 0x202E].map{|cp| [cp].pack('U') }.each do |char|
+      TestValidation.new.tweet_invalid?("Invalid:#{char}").should == :invalid_characters
+    end
+  end
+
+  it "should disallow non-Unicode" do
+    TestValidation.new.tweet_invalid?("not-Unicode:\xfff0").should == :invalid_characters
+  end
+
+  it "should allow <= 140 combined accent characters" do
+    char = [0x65, 0x0301].pack('U')
+    TestValidation.new.tweet_invalid?(char * 139).should == false
+    TestValidation.new.tweet_invalid?(char * 140).should == false
+    TestValidation.new.tweet_invalid?(char * 141).should == :too_long
+  end
+
+  it "should allow <= 140 multi-byte characters" do
+    char = [ 0x1d106 ].pack('U')
+    TestValidation.new.tweet_invalid?(char * 139).should == false
+    TestValidation.new.tweet_invalid?(char * 140).should == false
+    TestValidation.new.tweet_invalid?(char * 141).should == :too_long
+  end
+
+end

data/test/conformance_test.rb

@@ -0,0 +1,207 @@
+require 'multi_json'
+require 'nokogiri'
+require 'test/unit'
+require 'yaml'
+
+# Detect Ruby 1.8 and older to apply necessary encoding fixes
+major, minor, patch = RUBY_VERSION.split('.')
+OLD_RUBY = major.to_i == 1 && minor.to_i < 9
+
+if OLD_RUBY
+  $KCODE='u'
+end
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require 'incollege-text'
+
+class ConformanceTest < Test::Unit::TestCase
+  include Incollege::Extractor
+  include Incollege::Autolink
+  include Incollege::HitHighlighter
+  include Incollege::Validation
+
+  private
+
+  %w(description expected json hits).each do |key|
+    define_method key.to_sym do
+      @test_info[key]
+    end
+  end
+
+  if OLD_RUBY
+    def text
+      @test_info['text'].gsub(/\\u([0-9a-f]{8})/i) do
+        [$1.to_i(16)].pack('U*')
+      end
+    end
+  else
+    def text
+      @test_info['text']
+    end
+  end
+
+  def assert_equal_without_attribute_order(expected, actual, failure_message = nil)
+    assert_block(build_message(failure_message, "<?> expected but was\n<?>", expected, actual)) do
+      equal_nodes?(Nokogiri::HTML(expected).root, Nokogiri::HTML(actual).root)
+    end
+  end
+
+  def equal_nodes?(expected, actual)
+    return false unless expected.name == actual.name
+    return false unless ordered_attributes(expected) == ordered_attributes(actual)
+    return false if expected.text? && actual.text? && expected.content != actual.content
+
+    expected.children.each_with_index do |child, index|
+      return false unless equal_nodes?(child, actual.children[index])
+    end
+
+    true
+  end
+
+  def ordered_attributes(element)
+    element.attribute_nodes.map{|attr| [attr.name, attr.value]}.sort
+  end
+
+  CONFORMANCE_DIR = ENV['CONFORMANCE_DIR'] || File.expand_path("../../../conformance", __FILE__)
+
+  def self.def_conformance_test(file, test_type, &block)
+    yaml = YAML.load_file(File.join(CONFORMANCE_DIR, file))
+    raise  "No such test suite: #{test_type.to_s}" unless yaml["tests"][test_type.to_s]
+
+    file_name = file.split('.').first
+
+    yaml["tests"][test_type.to_s].each do |test_info|
+      name = :"test_#{file_name}_#{test_type} #{test_info['description']}"
+      define_method name do
+        @test_info = test_info
+        instance_eval(&block)
+      end
+    end
+  end
+
+  public
+
+  # Extractor Conformance
+  def_conformance_test("extract.yml", :replies) do
+    assert_equal expected, extract_reply_screen_name(text), description
+  end
+
+  def_conformance_test("extract.yml", :mentions) do
+    assert_equal expected, extract_mentioned_screen_names(text), description
+  end
+
+  def_conformance_test("extract.yml", :mentions_with_indices) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_mentioned_screen_names_with_indices(text), description
+  end
+
+  def_conformance_test("extract.yml", :mentions_or_lists_with_indices) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_mentions_or_lists_with_indices(text), description
+  end
+
+  def_conformance_test("extract.yml", :urls) do
+    assert_equal expected, extract_urls(text), description
+    expected.each do |expected_url|
+      assert_equal true, valid_url?(expected_url, true, false), "expected url [#{expected_url}] not valid"
+    end
+  end
+
+  def_conformance_test("tlds.yml", :generic) do
+    assert_equal expected, extract_urls(text), description
+  end
+
+  def_conformance_test("tlds.yml", :country) do
+    assert_equal expected, extract_urls(text), description
+  end
+
+  def_conformance_test("extract.yml", :urls_with_indices) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_urls_with_indices(text), description
+  end
+
+  def_conformance_test("extract.yml", :hashtags) do
+    assert_equal expected, extract_hashtags(text), description
+  end
+
+  def_conformance_test("extract.yml", :hashtags_with_indices) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_hashtags_with_indices(text), description
+  end
+
+  def_conformance_test("extract.yml", :cashtags) do
+    assert_equal expected, extract_cashtags(text), description
+  end
+
+  def_conformance_test("extract.yml", :cashtags_with_indices) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_cashtags_with_indices(text), description
+  end
+
+  # Autolink Conformance
+  def_conformance_test("autolink.yml", :usernames) do
+    assert_equal_without_attribute_order expected, auto_link_usernames_or_lists(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :lists) do
+    assert_equal_without_attribute_order expected, auto_link_usernames_or_lists(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :urls) do
+    assert_equal_without_attribute_order expected, auto_link_urls(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :hashtags) do
+    assert_equal_without_attribute_order expected, auto_link_hashtags(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :cashtags) do
+    assert_equal_without_attribute_order expected, auto_link_cashtags(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :all) do
+    assert_equal_without_attribute_order expected, auto_link(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :json) do
+    assert_equal_without_attribute_order expected, auto_link_with_json(text, MultiJson.load(json), :suppress_no_follow => true), description
+  end
+
+  # HitHighlighter Conformance
+  def_conformance_test("hit_highlighting.yml", :plain_text) do
+    assert_equal expected, hit_highlight(text, hits), description
+  end
+
+  def_conformance_test("hit_highlighting.yml", :with_links) do
+    assert_equal expected, hit_highlight(text, hits), description
+  end
+
+  # Validation Conformance
+  def_conformance_test("validate.yml", :tweets) do
+    assert_equal expected, valid_tweet_text?(text), description
+  end
+
+  def_conformance_test("validate.yml", :usernames) do
+    assert_equal expected, valid_username?(text), description
+  end
+
+  def_conformance_test("validate.yml", :lists) do
+    assert_equal expected, valid_list?(text), description
+  end
+
+  def_conformance_test("validate.yml", :urls) do
+    assert_equal expected, valid_url?(text), description
+  end
+
+  def_conformance_test("validate.yml", :urls_without_protocol) do
+    assert_equal expected, valid_url?(text, true, false), description
+  end
+
+  def_conformance_test("validate.yml", :hashtags) do
+    assert_equal expected, valid_hashtag?(text), description
+  end
+
+  def_conformance_test("validate.yml", :lengths) do
+    assert_equal expected, tweet_length(text), description
+  end
+end