twitter-text 1.13.4 → 1.14.0

data/lib/twitter-text/regex.rb

@@ -103,12 +103,46 @@ module Twitter
     SPACE_CHARS = " \t\n\x0B\f\r"
     CTRL_CHARS = "\x00-\x1F\x7F"
 
-    # A hashtag must contain at least one unicode letter or mark, as well as numbers, underscores, and select special characters.
-    HASHTAG_ALPHA = /[\p{L}\p{M}]/
-    HASHTAG_ALPHANUMERIC = /[\p{L}\p{M}\p{Nd}_\u200c\u200d\u0482\ua673\ua67e\u05be\u05f3\u05f4\uff5e\u301c\u309b\u309c\u30a0\u30fb\u3003\u0f0b\u0f0c\u00b7]/
-    HASHTAG_BOUNDARY = /\A|\z|[^&\p{L}\p{M}\p{Nd}_\u200c\u200d\u0482\ua673\ua67e\u05be\u05f3\u05f4\u309b\u309c\u30a0\u30fb\u3003\u0f0b\u0f0c\u00b7]/
-
-    HASHTAG = /(#{HASHTAG_BOUNDARY})(#|＃)(?!\ufe0f|\u20e3)(#{HASHTAG_ALPHANUMERIC}*#{HASHTAG_ALPHA}#{HASHTAG_ALPHANUMERIC}*)/io
+    # Generated from unicode_regex/unicode_regex_groups.scala, more inclusive than Ruby's \p{L}\p{M}
+    HASHTAG_LETTERS_AND_MARKS = "\\p{L}\\p{M}" +
+      "\u037f\u0528-\u052f\u08a0-\u08b2\u08e4-\u08ff\u0978\u0980\u0c00\u0c34\u0c81\u0d01\u0ede\u0edf" +
+      "\u10c7\u10cd\u10fd-\u10ff\u16f1-\u16f8\u17b4\u17b5\u191d\u191e\u1ab0-\u1abe\u1bab-\u1bad\u1bba-" +
+      "\u1bbf\u1cf3-\u1cf6\u1cf8\u1cf9\u1de7-\u1df5\u2cf2\u2cf3\u2d27\u2d2d\u2d66\u2d67\u9fcc\ua674-" +
+      "\ua67b\ua698-\ua69d\ua69f\ua792-\ua79f\ua7aa-\ua7ad\ua7b0\ua7b1\ua7f7-\ua7f9\ua9e0-\ua9ef\ua9fa-" +
+      "\ua9fe\uaa7c-\uaa7f\uaae0-\uaaef\uaaf2-\uaaf6\uab30-\uab5a\uab5c-\uab5f\uab64\uab65\uf870-\uf87f" +
+      "\uf882\uf884-\uf89f\uf8b8\uf8c1-\uf8d6\ufa2e\ufa2f\ufe27-\ufe2d\u{102e0}\u{1031f}\u{10350}-\u{1037a}" +
+      "\u{10500}-\u{10527}\u{10530}-\u{10563}\u{10600}-\u{10736}\u{10740}-\u{10755}\u{10760}-\u{10767}" +
+      "\u{10860}-\u{10876}\u{10880}-\u{1089e}\u{10980}-\u{109b7}\u{109be}\u{109bf}\u{10a80}-\u{10a9c}" +
+      "\u{10ac0}-\u{10ac7}\u{10ac9}-\u{10ae6}\u{10b80}-\u{10b91}\u{1107f}\u{110d0}-\u{110e8}\u{11100}-" +
+      "\u{11134}\u{11150}-\u{11173}\u{11176}\u{11180}-\u{111c4}\u{111da}\u{11200}-\u{11211}\u{11213}-" +
+      "\u{11237}\u{112b0}-\u{112ea}\u{11301}-\u{11303}\u{11305}-\u{1130c}\u{1130f}\u{11310}\u{11313}-" +
+      "\u{11328}\u{1132a}-\u{11330}\u{11332}\u{11333}\u{11335}-\u{11339}\u{1133c}-\u{11344}\u{11347}" +
+      "\u{11348}\u{1134b}-\u{1134d}\u{11357}\u{1135d}-\u{11363}\u{11366}-\u{1136c}\u{11370}-\u{11374}" +
+      "\u{11480}-\u{114c5}\u{114c7}\u{11580}-\u{115b5}\u{115b8}-\u{115c0}\u{11600}-\u{11640}\u{11644}" +
+      "\u{11680}-\u{116b7}\u{118a0}-\u{118df}\u{118ff}\u{11ac0}-\u{11af8}\u{1236f}-\u{12398}\u{16a40}-" +
+      "\u{16a5e}\u{16ad0}-\u{16aed}\u{16af0}-\u{16af4}\u{16b00}-\u{16b36}\u{16b40}-\u{16b43}\u{16b63}-" +
+      "\u{16b77}\u{16b7d}-\u{16b8f}\u{16f00}-\u{16f44}\u{16f50}-\u{16f7e}\u{16f8f}-\u{16f9f}\u{1bc00}-" +
+      "\u{1bc6a}\u{1bc70}-\u{1bc7c}\u{1bc80}-\u{1bc88}\u{1bc90}-\u{1bc99}\u{1bc9d}\u{1bc9e}\u{1e800}-" +
+      "\u{1e8c4}\u{1e8d0}-\u{1e8d6}\u{1ee00}-\u{1ee03}\u{1ee05}-\u{1ee1f}\u{1ee21}\u{1ee22}\u{1ee24}" +
+      "\u{1ee27}\u{1ee29}-\u{1ee32}\u{1ee34}-\u{1ee37}\u{1ee39}\u{1ee3b}\u{1ee42}\u{1ee47}\u{1ee49}" +
+      "\u{1ee4b}\u{1ee4d}-\u{1ee4f}\u{1ee51}\u{1ee52}\u{1ee54}\u{1ee57}\u{1ee59}\u{1ee5b}\u{1ee5d}\u{1ee5f}" +
+      "\u{1ee61}\u{1ee62}\u{1ee64}\u{1ee67}-\u{1ee6a}\u{1ee6c}-\u{1ee72}\u{1ee74}-\u{1ee77}\u{1ee79}-" +
+      "\u{1ee7c}\u{1ee7e}\u{1ee80}-\u{1ee89}\u{1ee8b}-\u{1ee9b}\u{1eea1}-\u{1eea3}\u{1eea5}-\u{1eea9}" +
+      "\u{1eeab}-\u{1eebb}"
+
+    # Generated from unicode_regex/unicode_regex_groups.scala, more inclusive than Ruby's \p{Nd}
+    HASHTAG_NUMERALS = "\\p{Nd}" +
+      "\u0de6-\u0def\ua9f0-\ua9f9\u{110f0}-\u{110f9}\u{11136}-\u{1113f}\u{111d0}-\u{111d9}\u{112f0}-" +
+      "\u{112f9}\u{114d0}-\u{114d9}\u{11650}-\u{11659}\u{116c0}-\u{116c9}\u{118e0}-\u{118e9}\u{16a60}-" +
+      "\u{16a69}\u{16b50}-\u{16b59}"
+
+    HASHTAG_SPECIAL_CHARS = "_\u200c\u200d\ua67e\u05be\u05f3\u05f4\uff5e\u301c\u309b\u309c\u30a0\u30fb\u3003\u0f0b\u0f0c\u00b7"
+
+    HASHTAG_LETTERS_NUMERALS = "#{HASHTAG_LETTERS_AND_MARKS}#{HASHTAG_NUMERALS}#{HASHTAG_SPECIAL_CHARS}"
+    HASHTAG_LETTERS_NUMERALS_SET = "[#{HASHTAG_LETTERS_NUMERALS}]"
+    HASHTAG_LETTERS_SET = "[#{HASHTAG_LETTERS_AND_MARKS}]"
+
+    HASHTAG = /(\A|[^&#{HASHTAG_LETTERS_NUMERALS}])(#|＃)(?!\ufe0f|\u20e3)(#{HASHTAG_LETTERS_NUMERALS_SET}*#{HASHTAG_LETTERS_SET}#{HASHTAG_LETTERS_NUMERALS_SET}*)/io
 
     REGEXEN[:valid_hashtag] = /#{HASHTAG}/io
     # Used in Extractor for final filtering

data/test/conformance_test.rb

@@ -124,6 +124,10 @@ class ConformanceTest < Test::Unit::TestCase
     assert_equal expected, extract_hashtags(text), description
   end
 
+  def_conformance_test("extract.yml", :hashtags_from_astral) do
+    assert_equal expected, extract_hashtags(text), description
+  end
+
   def_conformance_test("extract.yml", :hashtags_with_indices) do
     e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
     assert_equal e, extract_hashtags_with_indices(text), description

data/twitter-text.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = "twitter-text"
-  s.version = "1.13.4"
+  s.version = "1.14.0"
   s.authors = ["Matt Sanford", "Patrick Ewing", "Ben Cherry", "Britt Selvitelle",
                "Raffi Krikorian", "J.P. Cummins", "Yoshimasa Niwa", "Keita Fujii", "James Koval"]
   s.email = ["matt@twitter.com", "patrick.henry.ewing@gmail.com", "bcherry@gmail.com", "bs@brittspace.com",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: twitter-text
 version: !ruby/object:Gem::Version
-  version: 1.13.4
+  version: 1.14.0
 platform: ruby
 authors:
 - Matt Sanford
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-03 00:00:00.000000000 Z
+date: 2016-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -137,7 +137,7 @@ files:
 - ".rspec"
 - Gemfile
 - LICENSE
-- README.rdoc
+- README.md
 - Rakefile
 - lib/assets/tld_lib.yml
 - lib/twitter-text.rb
@@ -184,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Twitter text handling library

data/README.rdoc

@@ -1,106 +0,0 @@
-{rdoc-image:https://img.shields.io/gem/v/twitter-text.svg}[https://rubygems.org/gems/twitter-text]
-
-== twitter-text
-
-A gem that provides text processing routines for Twitter Tweets. The major
-reason for this is to unify the various auto-linking and extraction of
-usernames, lists, hashtags and URLs.
-
-== Extraction Examples
-
-  # Extraction
-  class MyClass
-    include Twitter::Extractor
-    usernames = extract_mentioned_screen_names("Mentioning @twitter and @jack")
-    # usernames = ["twitter", "jack"]
-  end
-
-  # Extraction with a block argument
-  class MyClass
-    include Twitter::Extractor
-    extract_reply_screen_name("@twitter are you hiring?").do |username|
-      # username = "twitter"
-    end
-  end
-
-== Auto-linking Examples
-
-  # Auto-link
-  class MyClass
-    include Twitter::Autolink
-
-    html = auto_link("link @user, please #request")
-  end
-
-  # For Ruby on Rails you want to add this to app/helpers/application_helper.rb
-  module ApplicationHelper
-    include Twitter::Autolink
-  end
-
-  # Now the auto_link function is available in every view. So in index.html.erb:
-  <%= auto_link("link @user, please #request") %>
-
-=== Usernames
-
-Username extraction and linking matches all valid Twitter usernames but does
-not verify that the username is a valid Twitter account.
-
-=== Lists
-
-Auto-link and extract list names when they are written in @user/list-name
-format.
-
-=== Hashtags
-
-Auto-link and extract hashtags, where a hashtag can contain most letters or
-numbers but cannot be solely numbers and cannot contain punctuation.
-
-=== URLs
-
-Asian languages like Chinese, Japanese or Korean may not use a delimiter such as
-a space to separate normal text from URLs making it difficult to identify where
-the URL ends and the text starts.
-
-For this reason twitter-text currently does not support extracting or auto-linking
-of URLs immediately followed by non-Latin characters.
-
-Example: "http://twitter.com/は素晴らしい" .
-The normal text is "は素晴らしい" and is not part of the URL even though
-it isn't space separated.
-
-=== International
-
-Special care has been taken to be sure that auto-linking and extraction work
-in Tweets of all languages. This means that languages without spaces between
-words should work equally well.
-
-=== Hit Highlighting
-
-Use to provide emphasis around the "hits" returned from the Search API, built
-to work against text that has been auto-linked already.
-
-=== Thanks
-
-Thanks to everybody who has filed issues, provided feedback or contributed patches. Patches courtesy of:
-
-* At Twitter …
-  * Matt Sanford - http://github.com/mzsanford
-  * Raffi Krikorian - http://github.com/r
-  * Ben Cherry - http://github.com/bcherry
-  * Patrick Ewing - http://github.com/hoverbird
-  * Jeff Smick - http://github.com/sprsquish
-  * Kenneth Kufluk - https://github.com/kennethkufluk
-  * Keita Fujii - https://github.com/keitaf
-  * Yoshimasa Niwa - https://github.com/niw
-
-* Patches from the community …
-  * Jean-Philippe Bougie - http://github.com/jpbougie
-  * Erik Michaels-Ober - https://github.com/sferik
-
-* Anyone who has filed an issue. It helps. Really.
-
-=== Copyright and License
-
-Copyright 2011 Twitter, Inc.
-
-Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0