twitter-text 1.10.0 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9bf8f0458f83224b2f818d0167deece43aa85875
-  data.tar.gz: dadc8f874379ed383cf7b66172b5eb65ec130906
+  metadata.gz: f435b95a47697def8e995b0d2dbdbbbde2b7e568
+  data.tar.gz: 10d1c8186c09520f2dfcbb9de8a3d336fdac7b04
 SHA512:
-  metadata.gz: efadacd1f014d0b422dc513c548e3c7cf22df17a21c42c382160a54883045530355b4766d21bfd038d9dd20ef4fcf9c33767874e2e708a357d9c63ab8dfbd46b
-  data.tar.gz: 43d0537e6b2e0411a3929b89ce5977c91ec786ed7e409fa7b0b0ed16beb8fff425b7c012e1f2073bc279af3c1b9814b0d51b6bf10c5f05b59dbefeec2696e2ec
+  metadata.gz: 0d4584b362d0bf89d5173d44949f4fa1187c8d82dd69c61fc4f6038b8fd0f7badbbb78551f04b63c63aa5074de3539784f7291891231a7e120592bd1dd0edfca
+  data.tar.gz: 065adc1b81bf6452b162a94f23cc91748b5cf1938ef6dc54211825a3c08139304deb3911f1c2484d81980fe3125a6190bef83c9f7f5a61f4ed271b911ee3829a

data/README.rdoc

@@ -1,4 +1,6 @@
-== twitter-text {<img src="https://secure.travis-ci.org/twitter/twitter-text-rb.png?branch=master" />}[http://travis-ci.org/twitter/twitter-text-rb] {<img src="https://codeclimate.com/badge.png" />}[https://codeclimate.com/github/twitter/twitter-text-rb]
+{rdoc-image:https://img.shields.io/gem/v/twitter-text.svg}[https://rubygems.org/gems/twitter-text]
+
+== twitter-text
 
 A gem that provides text processing routines for Twitter Tweets. The major
 reason for this is to unify the various auto-linking and extraction of
@@ -77,14 +79,6 @@ words should work equally well.
 Use to provide emphasis around the "hits" returned from the Search API, built
 to work against text that has been auto-linked already.
 
-=== Conformance
-
-To run the Conformance suite, you'll need to add that project as a git submodule.  From the root twitter-text-rb directory, run:
-
-  git submodule add git@github.com:twitter/twitter-text-conformance.git test/twitter-text-conformance/
-  git submodule init
-  git submodule update
-
 === Thanks
 
 Thanks to everybody who has filed issues, provided feedback or contributed patches. Patches courtesy of:

data/Rakefile

@@ -5,40 +5,12 @@ Bundler::GemHelper.install_tasks
 task :default => ['spec', 'test:conformance']
 task :test => :spec
 
+require 'rubygems'
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec)
 
-def conformance_version(dir)
-  require 'digest'
-  Dir[File.join(dir, '*')].inject(Digest::SHA1.new){|digest, file| digest.update(Digest::SHA1.file(file).hexdigest) }
-end
-
 namespace :test do
   namespace :conformance do
-    desc "Update conformance testing data"
-    task :update do
-      puts "Updating conformance data ... "
-      system("git submodule init") || raise("Failed to init submodule")
-      system("git submodule update") || raise("Failed to update submodule")
-      puts "Updating conformance data ... DONE"
-    end
-
-    desc "Change conformance test data to the lastest version"
-    task :latest => ['conformance:update'] do
-      current_dir = File.dirname(__FILE__)
-      submodule_dir = File.join(File.dirname(__FILE__), "test", "twitter-text-conformance")
-      version_before = conformance_version(submodule_dir)
-      system("cd #{submodule_dir} && git pull origin master") || raise("Failed to pull submodule version")
-      system("cd #{current_dir}")
-      if conformance_version(submodule_dir) != version_before
-        system("cd #{current_dir} && git add #{submodule_dir}") || raise("Failed to add upgrade files")
-        system("git commit -m \"Upgraded to the latest conformance suite\" #{submodule_dir}") || raise("Failed to commit upgraded conformacne data")
-        puts "Upgraded conformance suite."
-      else
-        puts "No conformance suite changes."
-      end
-    end
-
     desc "Run conformance test suite"
     task :run do
       ruby '-rubygems', "test/conformance_test.rb"
@@ -46,7 +18,7 @@ namespace :test do
   end
 
   desc "Run conformance test suite"
-  task :conformance => ['conformance:update', 'conformance:run'] do
+  task :conformance => ['conformance:run'] do
   end
 end
 

data/{test/twitter-text-conformance → lib/assets}/tld_lib.yml

@@ -262,6 +262,7 @@ country:
 - "срб"
 - "укр"
 - "қаз"
+- "հայ"
 - "الاردن"
 - "الجزائر"
 - "السعودية"
@@ -304,11 +305,14 @@ generic:
 - accountants
 - active
 - actor
+- adult
 - aero
 - agency
 - airforce
 - allfinanz
 - alsace
+- android
+- aquarelle
 - archi
 - army
 - arpa
@@ -332,6 +336,7 @@ generic:
 - biz
 - black
 - blackfriday
+- bloomberg
 - blue
 - bmw
 - bnpparibas
@@ -356,6 +361,7 @@ generic:
 - care
 - career
 - careers
+- cartier
 - casa
 - cash
 - cat
@@ -376,6 +382,7 @@ generic:
 - clinic
 - clothing
 - club
+- coach
 - codes
 - coffee
 - college
@@ -394,6 +401,7 @@ generic:
 - country
 - credit
 - creditcard
+- cricket
 - crs
 - cruises
 - cuisinella
@@ -404,6 +412,7 @@ generic:
 - day
 - deals
 - degree
+- delivery
 - democrat
 - dental
 - dentist
@@ -423,22 +432,27 @@ generic:
 - education
 - email
 - emerck
+- energy
 - engineer
 - engineering
 - enterprises
 - equipment
 - esq
 - estate
+- eurovision
 - eus
 - events
+- everbank
 - exchange
 - expert
 - exposed
 - fail
 - farm
+- fashion
 - feedback
 - finance
 - financial
+- firmdale
 - fish
 - fishing
 - fitness
@@ -505,6 +519,7 @@ generic:
 - int
 - international
 - investments
+- irish
 - jetzt
 - jobs
 - joburg
@@ -518,8 +533,11 @@ generic:
 - kred
 - lacaixa
 - land
+- latrobe
 - lawyer
+- lds
 - lease
+- legal
 - lgbt
 - life
 - lighting
@@ -532,6 +550,7 @@ generic:
 - ltda
 - luxe
 - luxury
+- madrid
 - maison
 - management
 - mango
@@ -541,6 +560,7 @@ generic:
 - meet
 - melbourne
 - meme
+- memorial
 - menu
 - miami
 - mil
@@ -549,6 +569,8 @@ generic:
 - moda
 - moe
 - monash
+- money
+- mormon
 - mortgage
 - moscow
 - motorcycles
@@ -579,6 +601,7 @@ generic:
 - paris
 - partners
 - parts
+- party
 - pharmacy
 - photo
 - photography
@@ -592,6 +615,7 @@ generic:
 - plumbing
 - pohl
 - poker
+- porn
 - post
 - praxi
 - press
@@ -610,6 +634,7 @@ generic:
 - rehab
 - reise
 - reisen
+- reit
 - ren
 - rentals
 - repair
@@ -627,11 +652,13 @@ generic:
 - ruhr
 - ryukyu
 - saarland
+- samsung
 - sarl
 - sca
 - scb
 - schmidt
 - schule
+- science
 - scot
 - services
 - sexy
@@ -652,6 +679,7 @@ generic:
 - surf
 - surgery
 - suzuki
+- sydney
 - systems
 - taipei
 - tatar
@@ -671,6 +699,7 @@ generic:
 - trade
 - training
 - travel
+- trust
 - tui
 - university
 - uno
@@ -728,14 +757,17 @@ generic:
 - "موقع"
 - "संगठन"
 - "みんな"
+- "グーグル"
 - "世界"
 - "中信"
 - "中文网"
 - "企业"
 - "佛山"
+- "八卦"
 - "公司"
 - "公益"
 - "商城"
+- "商店"
 - "商标"
 - "在线"
 - "广东"
@@ -747,6 +779,9 @@ generic:
 - "移动"
 - "组织机构"
 - "网址"
+- "网店"
 - "网络"
+- "谷歌"
 - "集团"
 - "삼성"
+- onion

data/lib/twitter-text/regex.rb

@@ -29,7 +29,7 @@ module Twitter
     TLDS = YAML.load_file(
       File.join(
         File.expand_path('../../..', __FILE__), # project root
-        'test', 'twitter-text-conformance', 'tld_lib.yml'
+        'lib', 'assets', 'tld_lib.yml'
       )
     )
 
@@ -90,6 +90,7 @@ module Twitter
           regex_range(0x0300, 0x036f),
           regex_range(0x1e00, 0x1eff)
     ].join('').freeze
+    REGEXEN[:latin_accents] = /[#{LATIN_ACCENTS}]+/o
 
     RTL_CHARACTERS = [
       regex_range(0x0600,0x06FF),
@@ -98,78 +99,14 @@ module Twitter
       regex_range(0xFE70,0xFEFF)
     ].join('').freeze
 
-
-    NON_LATIN_HASHTAG_CHARS = [
-      # Cyrillic (Russian, Ukrainian, etc.)
-      regex_range(0x0400, 0x04ff), # Cyrillic
-      regex_range(0x0500, 0x0527), # Cyrillic Supplement
-      regex_range(0x2de0, 0x2dff), # Cyrillic Extended A
-      regex_range(0xa640, 0xa69f), # Cyrillic Extended B
-      regex_range(0x0591, 0x05bf), # Hebrew
-      regex_range(0x05c1, 0x05c2),
-      regex_range(0x05c4, 0x05c5),
-      regex_range(0x05c7),
-      regex_range(0x05d0, 0x05ea),
-      regex_range(0x05f0, 0x05f4),
-      regex_range(0xfb12, 0xfb28), # Hebrew Presentation Forms
-      regex_range(0xfb2a, 0xfb36),
-      regex_range(0xfb38, 0xfb3c),
-      regex_range(0xfb3e),
-      regex_range(0xfb40, 0xfb41),
-      regex_range(0xfb43, 0xfb44),
-      regex_range(0xfb46, 0xfb4f),
-      regex_range(0x0610, 0x061a), # Arabic
-      regex_range(0x0620, 0x065f),
-      regex_range(0x066e, 0x06d3),
-      regex_range(0x06d5, 0x06dc),
-      regex_range(0x06de, 0x06e8),
-      regex_range(0x06ea, 0x06ef),
-      regex_range(0x06fa, 0x06fc),
-      regex_range(0x06ff),
-      regex_range(0x0750, 0x077f), # Arabic Supplement
-      regex_range(0x08a0),         # Arabic Extended A
-      regex_range(0x08a2, 0x08ac),
-      regex_range(0x08e4, 0x08fe),
-      regex_range(0xfb50, 0xfbb1), # Arabic Pres. Forms A
-      regex_range(0xfbd3, 0xfd3d),
-      regex_range(0xfd50, 0xfd8f),
-      regex_range(0xfd92, 0xfdc7),
-      regex_range(0xfdf0, 0xfdfb),
-      regex_range(0xfe70, 0xfe74), # Arabic Pres. Forms B
-      regex_range(0xfe76, 0xfefc),
-      regex_range(0x200c, 0x200c), # Zero-Width Non-Joiner
-      regex_range(0x0e01, 0x0e3a), # Thai
-      regex_range(0x0e40, 0x0e4e), # Hangul (Korean)
-      regex_range(0x1100, 0x11ff), # Hangul Jamo
-      regex_range(0x3130, 0x3185), # Hangul Compatibility Jamo
-      regex_range(0xA960, 0xA97F), # Hangul Jamo Extended-A
-      regex_range(0xAC00, 0xD7AF), # Hangul Syllables
-      regex_range(0xD7B0, 0xD7FF), # Hangul Jamo Extended-B
-      regex_range(0xFFA1, 0xFFDC)  # Half-width Hangul
-    ].join('').freeze
-    REGEXEN[:latin_accents] = /[#{LATIN_ACCENTS}]+/o
-
-    CJ_HASHTAG_CHARACTERS = [
-      regex_range(0x30A1, 0x30FA), regex_range(0x30FC, 0x30FE), # Katakana (full-width)
-      regex_range(0xFF66, 0xFF9F), # Katakana (half-width)
-      regex_range(0xFF10, 0xFF19), regex_range(0xFF21, 0xFF3A), regex_range(0xFF41, 0xFF5A), # Latin (full-width)
-      regex_range(0x3041, 0x3096), regex_range(0x3099, 0x309E), # Hiragana
-      regex_range(0x3400, 0x4DBF), # Kanji (CJK Extension A)
-      regex_range(0x4E00, 0x9FFF), # Kanji (Unified)
-      regex_range(0x20000, 0x2A6DF), # Kanji (CJK Extension B)
-      regex_range(0x2A700, 0x2B73F), # Kanji (CJK Extension C)
-      regex_range(0x2B740, 0x2B81F), # Kanji (CJK Extension D)
-      regex_range(0x2F800, 0x2FA1F), regex_range(0x3003), regex_range(0x3005), regex_range(0x303B) # Kanji (CJK supplement)
-    ].join('').freeze
-
     PUNCTUATION_CHARS = '!"#$%&\'()*+,-./:;<=>?@\[\]^_\`{|}~'
     SPACE_CHARS = " \t\n\x0B\f\r"
     CTRL_CHARS = "\x00-\x1F\x7F"
 
-    # A hashtag must contain latin characters, numbers and underscores, but not all numbers.
-    HASHTAG_ALPHA = /[a-z_#{LATIN_ACCENTS}#{NON_LATIN_HASHTAG_CHARS}#{CJ_HASHTAG_CHARACTERS}]/io
-    HASHTAG_ALPHANUMERIC = /[a-z0-9_#{LATIN_ACCENTS}#{NON_LATIN_HASHTAG_CHARS}#{CJ_HASHTAG_CHARACTERS}]/io
-    HASHTAG_BOUNDARY = /\A|\z|[^&a-z0-9_#{LATIN_ACCENTS}#{NON_LATIN_HASHTAG_CHARS}#{CJ_HASHTAG_CHARACTERS}]/o
+    # A hashtag must contain at least one unicode letter or mark, as well as numbers, underscores, and select special characters.
+    HASHTAG_ALPHA = /[\p{L}\p{M}]/
+    HASHTAG_ALPHANUMERIC = /[\p{L}\p{M}\p{Nd}_\u200c\u0482\ua673\ua67e\u05be\u05f3\u05f4\u309b\u309c\u30a0\u30fb\u3003\u0f0b\u0f0c\u0f0d]/
+    HASHTAG_BOUNDARY = /\A|\z|[^&\p{L}\p{M}\p{Nd}_\u200c\u0482\ua673\ua67e\u05be\u05f3\u05f4\u309b\u309c\u30a0\u30fb\u3003\u0f0b\u0f0c\u0f0d]/
 
     HASHTAG = /(#{HASHTAG_BOUNDARY})(#|＃)(#{HASHTAG_ALPHANUMERIC}*#{HASHTAG_ALPHA}#{HASHTAG_ALPHANUMERIC}*)/io
 

data/test/conformance_test.rb

@@ -62,7 +62,7 @@ class ConformanceTest < Test::Unit::TestCase
     element.attribute_nodes.map{|attr| [attr.name, attr.value]}.sort
   end
 
-  CONFORMANCE_DIR = ENV['CONFORMANCE_DIR'] || File.expand_path("../twitter-text-conformance", __FILE__)
+  CONFORMANCE_DIR = ENV['CONFORMANCE_DIR'] || File.expand_path("../../../conformance", __FILE__)
 
   def self.def_conformance_test(file, test_type, &block)
     yaml = YAML.load_file(File.join(CONFORMANCE_DIR, file))

data/twitter-text.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = "twitter-text"
-  s.version = "1.10.0"
+  s.version = "1.11.0"
   s.authors = ["Matt Sanford", "Patrick Ewing", "Ben Cherry", "Britt Selvitelle",
                "Raffi Krikorian", "J.P. Cummins", "Yoshimasa Niwa", "Keita Fujii", "James Koval"]
   s.email = ["matt@twitter.com", "patrick.henry.ewing@gmail.com", "bcherry@gmail.com", "bs@brittspace.com",
@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "simplecov", "~> 0.8.0"
   s.add_runtime_dependency     "unf", "~> 0.1.0"
 
-  s.files         = `git ls-files`.split("\n") + ['test/twitter-text-conformance/tld_lib.yml']
+  s.files         = `git ls-files`.split("\n") + ['lib/assets/tld_lib.yml']
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twitter-text
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.11.0
 platform: ruby
 authors:
 - Matt Sanford
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-29 00:00:00.000000000 Z
+date: 2015-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -135,11 +135,11 @@ files:
 - ".gitignore"
 - ".gitmodules"
 - ".rspec"
-- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.rdoc
 - Rakefile
+- lib/assets/tld_lib.yml
 - lib/twitter-text.rb
 - lib/twitter-text/autolink.rb
 - lib/twitter-text/deprecation.rb
@@ -163,7 +163,6 @@ files:
 - spec/unicode_spec.rb
 - spec/validation_spec.rb
 - test/conformance_test.rb
-- test/twitter-text-conformance/tld_lib.yml
 - twitter-text.gemspec
 homepage: http://twitter.com
 licenses:
@@ -185,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.4
 signing_key: 
 specification_version: 4
 summary: Twitter text handling library

data/.travis.yml

@@ -1,9 +0,0 @@
-language: ruby
-rvm:
-  - 1.8.7
-  - 1.9.3
-  - 2.0.0
-  - 2.1.0
-before_install:
-  - gem update --system
-  - gem --version