twitter-text-kow 1.3.1.0

data/spec/spec_helper.rb

@@ -0,0 +1,139 @@
+# Copyright 2018 Twitter, Inc.
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+$TESTING=true
+
+# Ruby 1.8 encoding check
+major, minor, patch = RUBY_VERSION.split('.')
+if major.to_i == 1 && minor.to_i < 9
+  $KCODE='u'
+end
+
+$:.push File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'nokogiri'
+require 'json'
+require 'simplecov'
+SimpleCov.start do
+  add_group 'Libraries', 'lib'
+end
+
+require File.expand_path('../../lib/twitter-text', __FILE__)
+require File.expand_path('../test_urls', __FILE__)
+
+RSpec.configure do |config|
+  config.include TestUrls
+
+  config.filter_run_excluding :ruby => lambda { |version|
+    case version.to_s
+    when /^> (.*)/
+      !(RUBY_VERSION.to_s > $1)
+    else
+      !(RUBY_VERSION.to_s =~ /^#{version.to_s}/)
+    end
+  }
+end
+
+RSpec::Matchers.define :match_autolink_expression do
+  match do |string|
+    !Twitter::TwitterText::Extractor.extract_urls(string).empty?
+  end
+end
+
+RSpec::Matchers.define :match_autolink_expression_in do |text|
+  match do |url|
+    @match_data = Twitter::TwitterText::Regex[:valid_url].match(text)
+    @match_data && @match_data.to_s.strip == url
+  end
+
+  failure_message do |url|
+    "Expected to find url '#{url}' in text '#{text}', but the match was #{@match_data.captures}'"
+  end
+end
+
+RSpec::Matchers.define :have_autolinked_url do |url, inner_text|
+  match do |text|
+    @link = Nokogiri::HTML(text).search("a[@href='#{url}']")
+    @link &&
+    @link.inner_text &&
+    (inner_text && @link.inner_text == inner_text) || (!inner_text && @link.inner_text == url)
+  end
+
+  failure_message do |text|
+    "Expected url '#{url}'#{", inner_text '#{inner_text}'" if inner_text} to be autolinked in '#{text}'"
+  end
+end
+
+RSpec::Matchers.define :link_to_screen_name do |screen_name, inner_text|
+  expected = inner_text ? inner_text : screen_name
+
+  match do |text|
+    @link = Nokogiri::HTML(text).search("a.username")
+    return false unless @link && @link.inner_text == expected
+    expect("https://twitter.com/#{screen_name}").to eq(@link.first['href'])
+  end
+
+  failure_message do |text|
+    if @link.first
+      "Expected link '#{@link.inner_text}' with href '#{@link.first['href']}' to match screen_name '#{expected}', but it does not."
+    else
+      "Expected screen name '#{screen_name}' to be autolinked in '#{text}', but no link was found."
+    end
+  end
+
+  failure_message_when_negated do |text|
+    "Expected link '#{@link.inner_text}' with href '#{@link.first['href']}' not to match screen_name '#{expected}', but it does."
+  end
+
+  description do
+    "contain a link with the name and href pointing to the expected screen_name"
+  end
+end
+
+RSpec::Matchers.define :link_to_list_path do |list_path, inner_text|
+  expected = inner_text ? inner_text : list_path
+
+  match do |text|
+    @link = Nokogiri::HTML(text).search("a.list-slug")
+    return false unless @link && @link.inner_text == expected
+    expect("https://twitter.com/#{list_path}".downcase).to eq(@link.first['href'])
+  end
+
+  failure_message do |text|
+    if @link.first
+      "Expected link '#{@link.inner_text}' with href '#{@link.first['href']}' to match the list path '#{expected}', but it does not."
+    else
+      "Expected list path '#{list_path}' to be autolinked in '#{text}', but no link was found."
+    end
+  end
+
+  failure_message_when_negated do |text|
+    "Expected link '#{@link.inner_text}' with href '#{@link.first['href']}' not to match the list path '#{expected}', but it does."
+  end
+
+  description do
+    "contain a link with the list title and an href pointing to the list path"
+  end
+end
+
+RSpec::Matchers.define :have_autolinked_hashtag do |hashtag|
+  match do |text|
+    @link = Nokogiri::HTML(text).search("a[@href='https://twitter.com/search?q=#{hashtag.sub(/^#/, '%23')}']")
+    @link &&
+    @link.inner_text &&
+    @link.inner_text == hashtag
+  end
+
+  failure_message do |text|
+    if @link.first
+      "Expected link text to be [#{hashtag}], but it was [#{@link.inner_text}] in #{text}"
+    else
+      "Expected hashtag #{hashtag} to be autolinked in '#{text}', but no link was found."
+    end
+  end
+
+  failure_message_when_negated do |text|
+    "Expected link '#{@link.inner_text}' with href '#{@link.first['href']}' not to match the hashtag '#{hashtag}', but it does."
+  end
+end

data/spec/test_urls.rb

@@ -0,0 +1,90 @@
+# Copyright 2018 Twitter, Inc.
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# encoding: utf-8
+
+module TestUrls
+  VALID = [
+    "http://google.com",
+    "http://foobar.com/#",
+    "http://google.com/#foo",
+    "http://google.com/#search?q=iphone%20-filter%3Alinks",
+    "http://twitter.com/#search?q=iphone%20-filter%3Alinks",
+    "http://somedomain.com/index.php?path=/abc/def/",
+    "http://www.boingboing.net/2007/02/14/katamari_damacy_phon.html",
+    "http://somehost.com:3000",
+    "http://a_b.c-d.com",
+    "http://sub_domain-dash.twitter.com",
+    "http://xo.com/~matthew+%-x",
+    "http://en.wikipedia.org/wiki/Primer_(film)",
+    "http://www.ams.org/bookstore-getitem/item=mbk-59",
+    "http://chilp.it/?77e8fd",
+    "http://tell.me/why",
+    "http://longtlds.info",
+    "http://✪df.ws/ejp",
+    "http://日本.com",
+    "http://search.twitter.com/search?q=avro&lang=en",
+    "http://mrs.domain-dash.biz",
+    "http://x.com/has/one/char/domain",
+    "http://t.co/nwcLTFF",
+    "http://a.b.cd",
+    "http://a-b.b.com",
+    "http://twitter-dash.com",
+    "http://msdn.microsoft.com/ja-jp/library/system.net.httpwebrequest(v=VS.100).aspx",
+    "www.foobar.com",
+    "WWW.FOOBAR.COM",
+    "www.foobar.co.jp",
+    "http://t.co",
+    "t.co/nwcLTFF",
+    "http://foobar.みんな",
+    "http://foobar.中国",
+    "http://foobar.پاکستان",
+    "https://www.youtube.com/playlist?list=PL0ZPu8XSRTB7wZzn0mLHMvyzVFeRxbWn-",
+    "http://ああ.com"
+  ] unless defined?(TestUrls::VALID)
+
+  INVALID = [
+    "http://no-tld",
+    "http://tld-too-short.x",
+    "http://-doman_dash.com",
+    "http://_leadingunderscore.twitter.com",
+    "http://trailingunderscore_.twitter.com",
+    "http://-leadingdash.twitter.com",
+    "http://trailingdash-.twitter.com",
+    "http://-leadingdash.com",
+    "http://trailingdash-.com",
+    "http://no_underscores.com",
+    "http://test.c_o_m",
+    "http://test.c-o-m",
+    "http://twitt#{[0x202A].pack('U')}.com",
+    "http://twitt#{[0x202B].pack('U')}.com",
+    "http://twitt#{[0x202C].pack('U')}.com",
+    "http://twitt#{[0x202D].pack('U')}.com",
+    "http://twitt#{[0x202E].pack('U')}.com",
+    "https://somesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurl.com/foo https://somesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurl.com/foo https://somesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurl.com/foo"
+  ] unless defined?(TestUrls::INVALID)
+
+  TCO = [
+    "http://t.co/P53cv5yO!",
+    "http://t.co/fQJmiPGg***",
+    "http://t.co/pbY2NfTZ's",
+    "http://t.co/2vYHpAc5;",
+    "http://t.co/ulYGBYSo:",
+    "http://t.co/GeT4bSiw=win",
+    "http://t.co/8MkmHU0k+fun",
+    "http://t.co/TKLp64dY.yes,",
+    "http://t.co/8vuO27cI$$",
+    "http://t.co/rPYTvdA8/",
+    "http://t.co/WvtMw5ku%",
+    "http://t.co/8t7G3ddS#",
+    "http://t.co/nfHNJDV2/#!",
+    "http://t.co/gK6NOXHs[good]",
+    "http://t.co/dMrT0o1Y]bad",
+    "http://t.co/FNkPfmii-",
+    "http://t.co/sMgS3pjI_oh",
+    "http://t.co/F8Dq3Plb~",
+    "http://t.co/ivvH58vC&help",
+    "http://t.co/iUBL15zD|NZ5KYLQ8"
+  ] unless defined?(TestUrls::TCO)
+end

data/spec/twitter_text_spec.rb

@@ -0,0 +1,25 @@
+# Copyright 2018 Twitter, Inc.
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# encoding: utf-8
+require File.dirname(__FILE__) + '/spec_helper'
+
+major, minor, patch = RUBY_VERSION.split('.')
+if major.to_i == 1 && minor.to_i < 9
+  describe "base" do
+    before do
+      $KCODE = 'NONE'
+    end
+
+    after do
+      $KCODE = 'u'
+    end
+
+    it "should raise with invalid KCODE on Ruby < 1.9" do
+      expect(lambda do
+        require 'twitter-text'
+      end).to raise_error
+    end
+  end
+end

data/spec/unicode_spec.rb

@@ -0,0 +1,35 @@
+# Copyright 2018 Twitter, Inc.
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# encoding: utf-8
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Twitter::TwitterText::Unicode do
+
+  it "should lazy-init constants" do
+    expect(Twitter::TwitterText::Unicode.const_defined?(:UFEB6)).to eq(false)
+    expect(Twitter::TwitterText::Unicode::UFEB6).to_not be_nil
+    expect(Twitter::TwitterText::Unicode::UFEB6).to be_kind_of(String)
+    expect(Twitter::TwitterText::Unicode.const_defined?(:UFEB6)).to eq(true)
+  end
+
+  it "should return corresponding character" do
+    expect(Twitter::TwitterText::Unicode::UFEB6).to be == [0xfeb6].pack('U')
+  end
+
+  it "should allow lowercase notation" do
+    expect(Twitter::TwitterText::Unicode::Ufeb6).to be == Twitter::TwitterText::Unicode::UFEB6
+    expect(Twitter::TwitterText::Unicode::Ufeb6).to be === Twitter::TwitterText::Unicode::UFEB6
+  end
+
+  it "should allow underscore notation" do
+    expect(Twitter::TwitterText::Unicode::U_FEB6).to be == Twitter::TwitterText::Unicode::UFEB6
+    expect(Twitter::TwitterText::Unicode::U_FEB6).to be === Twitter::TwitterText::Unicode::UFEB6
+  end
+
+  it "should raise on invalid codepoints" do
+    expect(lambda { Twitter::TwitterText::Unicode::FFFFFF }).to raise_error(NameError)
+  end
+
+end

data/spec/validation_spec.rb

@@ -0,0 +1,87 @@
+# Copyright 2018 Twitter, Inc.
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# encoding: utf-8
+require File.dirname(__FILE__) + '/spec_helper'
+
+class TestValidation
+  include Twitter::TwitterText::Validation
+end
+
+describe Twitter::TwitterText::Validation do
+
+  it "should disallow invalid BOM character" do
+    expect(TestValidation.new.tweet_invalid?("Bom:#{Twitter::TwitterText::Unicode::UFFFE}")).to be == :invalid_characters
+    expect(TestValidation.new.tweet_invalid?("Bom:#{Twitter::TwitterText::Unicode::UFEFF}")).to be == :invalid_characters
+  end
+
+  it "should disallow invalid U+FFFF character" do
+    expect(TestValidation.new.tweet_invalid?("Bom:#{Twitter::TwitterText::Unicode::UFFFF}")).to be == :invalid_characters
+  end
+
+  it "should allow direction change characters" do
+    [0x202A, 0x202B, 0x202C, 0x202D, 0x202E].map{|cp| [cp].pack('U') }.each do |char|
+      expect(TestValidation.new.tweet_invalid?("Invalid:#{char}")).to be false
+    end
+  end
+
+  it "should disallow non-Unicode" do
+    expect(TestValidation.new.tweet_invalid?("not-Unicode:\xfff0")).to be == :invalid_characters
+  end
+
+  it "should allow <= 140 combined accent characters" do
+    char = [0x65, 0x0301].pack('U')
+    expect(TestValidation.new.tweet_invalid?(char * 139)).to be false
+    expect(TestValidation.new.tweet_invalid?(char * 140)).to be false
+    expect(TestValidation.new.tweet_invalid?(char * 141)).to eq(:too_long)
+  end
+
+  it "should allow <= 140 multi-byte characters" do
+    char = [ 0x1d106 ].pack('U')
+    expect(TestValidation.new.tweet_invalid?(char * 139)).to be false
+    expect(TestValidation.new.tweet_invalid?(char * 140)).to be false
+    expect(TestValidation.new.tweet_invalid?(char * 141)).to eq(:too_long)
+  end
+
+  context "when returning results" do
+    it "should properly create new fully-populated results from arguments" do
+      results = Twitter::TwitterText::Validation::ParseResults.new(weighted_length: 26, permillage: 92, valid: true, display_range_start: 0, display_range_end: 16, valid_range_start: 0, valid_range_end:16)
+      expect(results).to_not be nil
+      expect(results[:weighted_length]).to eq(26)
+      expect(results[:permillage]).to eq(92)
+      expect(results[:valid]).to be true
+      expect(results[:display_range_start]).to eq(0)
+      expect(results[:display_range_end]).to eq(16)
+      expect(results[:valid_range_start]).to eq(0)
+      expect(results[:valid_range_end]).to eq(16)
+    end
+
+    it "should properly create empty results" do
+      results = Twitter::TwitterText::Validation::ParseResults.empty()
+      expect(results[:weighted_length]).to eq(0)
+      expect(results[:permillage]).to eq(0)
+      expect(results[:valid]).to be true
+      expect(results[:display_range_start]).to eq(0)
+      expect(results[:display_range_end]).to eq(0)
+      expect(results[:valid_range_start]).to eq(0)
+      expect(results[:valid_range_end]).to eq(0)
+    end
+  end
+
+  context "when parsing tweet text" do
+    it "should properly parse ZWJ and ZWNJ when grapheme clusters are enabled", ruby: ">= 2.5.0" do
+      # Grapheme clustering of devenghali script differs based on platform implementation
+      text = "ZWJ: क्ष -> क्\u200Dष; ZWNJ: क्ष -> क्\u200Cष"
+      config = Twitter::TwitterText::Configuration::configuration_from_file(Twitter::TwitterText::Configuration::CONFIG_V3)
+      results = Twitter::TwitterText::Validation::parse_tweet(text, config: config)
+      expect(results[:weighted_length]).to eq(29)
+      expect(results[:permillage]).to eq(103)
+      expect(results[:valid]).to be true
+      expect(results[:display_range_start]).to eq(0)
+      expect(results[:display_range_end]).to eq(34)
+      expect(results[:valid_range_start]).to eq(0)
+      expect(results[:valid_range_end]).to eq(34)
+    end
+  end
+end

data/test/conformance_test.rb

@@ -0,0 +1,242 @@
+# Copyright 2018 Twitter, Inc.
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'multi_json'
+require 'nokogiri'
+require 'test/unit'
+require 'yaml'
+
+# Detect Ruby 1.8 and older to apply necessary encoding fixes
+major, minor, patch = RUBY_VERSION.split('.')
+OLD_RUBY = major.to_i == 1 && minor.to_i < 9
+
+if OLD_RUBY
+  $KCODE='u'
+end
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require 'twitter-text'
+
+class ConformanceTest < Test::Unit::TestCase
+  include Twitter::TwitterText::Extractor
+  include Twitter::TwitterText::Autolink
+  include Twitter::TwitterText::HitHighlighter
+  include Twitter::TwitterText::Validation
+
+  private
+
+  %w(description expected json hits).each do |key|
+    define_method key.to_sym do
+      @test_info[key]
+    end
+  end
+
+  if OLD_RUBY
+    def text
+      @test_info['text'].gsub(/\\u([0-9a-f]{8})/i) do
+        [$1.to_i(16)].pack('U*')
+      end
+    end
+  else
+    def text
+      @test_info['text']
+    end
+  end
+
+  def assert_equal_without_attribute_order(expected, actual, failure_message = nil)
+    assert_block(build_message(failure_message, "<?> expected but was\n<?>", expected, actual)) do
+      equal_nodes?(Nokogiri::HTML(expected).root, Nokogiri::HTML(actual).root)
+    end
+  end
+
+  def equal_nodes?(expected, actual)
+    return false unless expected.name == actual.name
+    return false unless ordered_attributes(expected) == ordered_attributes(actual)
+    return false if expected.text? && actual.text? && expected.content != actual.content
+
+    expected.children.each_with_index do |child, index|
+      return false unless equal_nodes?(child, actual.children[index])
+    end
+
+    true
+  end
+
+  def ordered_attributes(element)
+    element.attribute_nodes.map{|attr| [attr.name, attr.value]}.sort
+  end
+
+  def assert_equal_parse_results(expected, actual, failure_message = nil)
+    e = {}
+    # Note that we don't assert display and valid ranges because of differences
+    # in how ruby counts characters (wrt surrogate pairs) vs. other platforms
+    range_keys = [:display_range_start, :display_range_end, :valid_range_start, :valid_range_end]
+    expected.keys.each { |k| e[k.gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase.to_sym] = expected[k] }
+    [e, actual].each { |a| range_keys.each { |k| a.delete(k) } }
+    assert_equal e, actual, failure_message
+  end
+
+  CONFORMANCE_DIR = ENV['CONFORMANCE_DIR'] || File.expand_path("../../../conformance", __FILE__)
+
+  def self.def_conformance_test(file, test_type, &block)
+    yaml = YAML.load_file(File.join(CONFORMANCE_DIR, file))
+    raise  "No such test suite: #{test_type.to_s}" unless yaml["tests"][test_type.to_s]
+
+    file_name = file.split('.').first
+
+    yaml["tests"][test_type.to_s].each do |test_info|
+      name = :"test_#{file_name}_#{test_type} #{test_info['description']}"
+      define_method name do
+        @test_info = test_info
+        instance_eval(&block)
+      end
+    end
+  end
+
+  public
+
+  # Extractor Conformance
+  def_conformance_test("extract.yml", :replies) do
+    assert_equal expected, extract_reply_screen_name(text), description
+  end
+
+  def_conformance_test("extract.yml", :mentions) do
+    assert_equal expected, extract_mentioned_screen_names(text), description
+  end
+
+  def_conformance_test("extract.yml", :mentions_with_indices) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_mentioned_screen_names_with_indices(text), description
+  end
+
+  def_conformance_test("extract.yml", :mentions_or_lists_with_indices) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_mentions_or_lists_with_indices(text), description
+  end
+
+  def_conformance_test("extract.yml", :urls) do
+    assert_equal expected, extract_urls(text), description
+    expected.each do |expected_url|
+      assert_equal true, valid_url?(expected_url, true, false), "expected url [#{expected_url}] not valid"
+    end
+  end
+
+  def_conformance_test("tlds.yml", :generic) do
+    assert_equal expected, extract_urls(text), description
+  end
+
+  def_conformance_test("tlds.yml", :country) do
+    assert_equal expected, extract_urls(text), description
+  end
+
+  def_conformance_test("extract.yml", :urls_with_indices) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_urls_with_indices(text), description
+  end
+
+  def_conformance_test("extract.yml", :urls_with_directional_markers) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_urls_with_indices(text), description
+  end
+
+  def_conformance_test("extract.yml", :hashtags) do
+    assert_equal expected, extract_hashtags(text), description
+  end
+
+  def_conformance_test("extract.yml", :hashtags_from_astral) do
+    assert_equal expected, extract_hashtags(text), description
+  end
+
+  def_conformance_test("extract.yml", :hashtags_with_indices) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_hashtags_with_indices(text), description
+  end
+
+  def_conformance_test("extract.yml", :cashtags) do
+    assert_equal expected, extract_cashtags(text), description
+  end
+
+  def_conformance_test("extract.yml", :cashtags_with_indices) do
+    e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
+    assert_equal e, extract_cashtags_with_indices(text), description
+  end
+
+  # Autolink Conformance
+  def_conformance_test("autolink.yml", :usernames) do
+    assert_equal_without_attribute_order expected, auto_link_usernames_or_lists(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :lists) do
+    assert_equal_without_attribute_order expected, auto_link_usernames_or_lists(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :urls) do
+    assert_equal_without_attribute_order expected, auto_link_urls(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :hashtags) do
+    assert_equal_without_attribute_order expected, auto_link_hashtags(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :cashtags) do
+    assert_equal_without_attribute_order expected, auto_link_cashtags(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :all) do
+    assert_equal_without_attribute_order expected, auto_link(text, :suppress_no_follow => true), description
+  end
+
+  def_conformance_test("autolink.yml", :json) do
+    assert_equal_without_attribute_order expected, auto_link_with_json(text, MultiJson.load(json), :suppress_no_follow => true), description
+  end
+
+  # HitHighlighter Conformance
+  def_conformance_test("hit_highlighting.yml", :plain_text) do
+    assert_equal expected, hit_highlight(text, hits), description
+  end
+
+  def_conformance_test("hit_highlighting.yml", :with_links) do
+    assert_equal expected, hit_highlight(text, hits), description
+  end
+
+  # Validation Conformance
+  def_conformance_test("validate.yml", :tweets) do
+    assert_equal expected, parse_tweet(text)[:valid], description
+  end
+
+  def_conformance_test("validate.yml", :usernames) do
+    assert_equal expected, valid_username?(text), description
+  end
+
+  def_conformance_test("validate.yml", :lists) do
+    assert_equal expected, valid_list?(text), description
+  end
+
+  def_conformance_test("validate.yml", :urls) do
+    assert_equal expected, valid_url?(text), description
+  end
+
+  def_conformance_test("validate.yml", :urls_without_protocol) do
+    assert_equal expected, valid_url?(text, true, false), description
+  end
+
+  def_conformance_test("validate.yml", :hashtags) do
+    assert_equal expected, valid_hashtag?(text), description
+  end
+
+  def_conformance_test("validate.yml", :WeightedTweetsCounterTest) do
+    # Force v2 configuration, basic weighted code point support
+    config = Twitter::TwitterText::Configuration::configuration_from_file(Twitter::TwitterText::Configuration::CONFIG_V2)
+    assert_equal_parse_results expected, parse_tweet(text, config: config), description
+  end
+
+  def_conformance_test("validate.yml", :WeightedTweetsWithDiscountedEmojiCounterTest) do
+    # Force v3 configuration, which supports discounting grapheme clusters that are emoji
+    config = Twitter::TwitterText::Configuration::configuration_from_file(Twitter::TwitterText::Configuration::CONFIG_V3)
+    assert_equal_parse_results expected, parse_tweet(text, config: config), description
+  end
+
+  def_conformance_test("validate.yml", :UnicodeDirectionalMarkerCounterTest) do
+    assert_equal_parse_results expected, parse_tweet(text), description
+  end
+end

data/twitter-text.gemspec

@@ -0,0 +1,35 @@
+# Copyright 2018 Twitter, Inc.
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# encoding: utf-8
+
+Gem::Specification.new do |s|
+  s.name = "twitter-text-kow"
+  s.version = "1.3.1.0"
+  s.authors = ["David LaMacchia", "Sudheer Guntupalli", "Kaushik Lakshmikanth", "Jose Antonio Marquez Russo", "Lee Adams",
+               "Yoshimasa Niwa"]
+  s.email = ["opensource@twitter.com"]
+  s.homepage = "http://twitter.com"
+  s.description = s.summary = "A KOW fixed gem that provides text handling for Twitter"
+  s.license = "Apache 2.0"
+
+  s.platform = Gem::Platform::RUBY
+  s.summary = "Twitter text handling library"
+
+  s.add_development_dependency "test-unit"
+  s.add_development_dependency "multi_json", "~> 1.3"
+  s.add_development_dependency "nokogiri", "~> 1.17.2"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rdoc"
+  s.add_development_dependency "rspec", "~> 3.0"
+  s.add_development_dependency "simplecov"
+  s.add_runtime_dependency     "unf", "~> 0.1.0"
+  # Use of idn-ruby requires libidn to be installed separately
+  s.add_runtime_dependency     "idn-ruby"
+
+  s.files         = `git ls-files`.split("\n") + ['lib/assets/tld_lib.yml'] + Dir['config/*']
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end