twitter-text 1.4.2 → 1.4.5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitter-text (1.3.2)
+    twitter-text (1.4.5)
       actionpack
 
 GEM

data/LICENSE

@@ -1,13 +1,188 @@
-Copyright 2010 Twitter, Inc.
+Copyright 2011 Twitter, Inc.
 
-Licensed under the Apache License, Version 2.0 (the "License"); you may not
-use this file except in compliance with the License. You may obtain a copy of
-the License at
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this work except in compliance with the License.
+You may obtain a copy of the License below, or at:
 
-http://www.apache.org/licenses/LICENSE-2.0
+   http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations under
-the License.
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+                             Apache License
+                       Version 2.0, January 2004
+                    http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+  "License" shall mean the terms and conditions for use, reproduction,
+  and distribution as defined by Sections 1 through 9 of this document.
+
+  "Licensor" shall mean the copyright owner or entity authorized by
+  the copyright owner that is granting the License.
+
+  "Legal Entity" shall mean the union of the acting entity and all
+  other entities that control, are controlled by, or are under common
+  control with that entity. For the purposes of this definition,
+  "control" means (i) the power, direct or indirect, to cause the
+  direction or management of such entity, whether by contract or
+  otherwise, or (ii) ownership of fifty percent (50%) or more of the
+  outstanding shares, or (iii) beneficial ownership of such entity.
+
+  "You" (or "Your") shall mean an individual or Legal Entity
+  exercising permissions granted by this License.
+
+  "Source" form shall mean the preferred form for making modifications,
+  including but not limited to software source code, documentation
+  source, and configuration files.
+
+  "Object" form shall mean any form resulting from mechanical
+  transformation or translation of a Source form, including but
+  not limited to compiled object code, generated documentation,
+  and conversions to other media types.
+
+  "Work" shall mean the work of authorship, whether in Source or
+  Object form, made available under the License, as indicated by a
+  copyright notice that is included in or attached to the work
+  (an example is provided in the Appendix below).
+
+  "Derivative Works" shall mean any work, whether in Source or Object
+  form, that is based on (or derived from) the Work and for which the
+  editorial revisions, annotations, elaborations, or other modifications
+  represent, as a whole, an original work of authorship. For the purposes
+  of this License, Derivative Works shall not include works that remain
+  separable from, or merely link (or bind by name) to the interfaces of,
+  the Work and Derivative Works thereof.
+
+  "Contribution" shall mean any work of authorship, including
+  the original version of the Work and any modifications or additions
+  to that Work or Derivative Works thereof, that is intentionally
+  submitted to Licensor for inclusion in the Work by the copyright owner
+  or by an individual or Legal Entity authorized to submit on behalf of
+  the copyright owner. For the purposes of this definition, "submitted"
+  means any form of electronic, verbal, or written communication sent
+  to the Licensor or its representatives, including but not limited to
+  communication on electronic mailing lists, source code control systems,
+  and issue tracking systems that are managed by, or on behalf of, the
+  Licensor for the purpose of discussing and improving the Work, but
+  excluding communication that is conspicuously marked or otherwise
+  designated in writing by the copyright owner as "Not a Contribution."
+
+  "Contributor" shall mean Licensor and any individual or Legal Entity
+  on behalf of whom a Contribution has been received by Licensor and
+  subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+  this License, each Contributor hereby grants to You a perpetual,
+  worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+  copyright license to reproduce, prepare Derivative Works of,
+  publicly display, publicly perform, sublicense, and distribute the
+  Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+  this License, each Contributor hereby grants to You a perpetual,
+  worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+  (except as stated in this section) patent license to make, have made,
+  use, offer to sell, sell, import, and otherwise transfer the Work,
+  where such license applies only to those patent claims licensable
+  by such Contributor that are necessarily infringed by their
+  Contribution(s) alone or by combination of their Contribution(s)
+  with the Work to which such Contribution(s) was submitted. If You
+  institute patent litigation against any entity (including a
+  cross-claim or counterclaim in a lawsuit) alleging that the Work
+  or a Contribution incorporated within the Work constitutes direct
+  or contributory patent infringement, then any patent licenses
+  granted to You under this License for that Work shall terminate
+  as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+  Work or Derivative Works thereof in any medium, with or without
+  modifications, and in Source or Object form, provided that You
+  meet the following conditions:
+
+  (a) You must give any other recipients of the Work or
+      Derivative Works a copy of this License; and
+
+  (b) You must cause any modified files to carry prominent notices
+      stating that You changed the files; and
+
+  (c) You must retain, in the Source form of any Derivative Works
+      that You distribute, all copyright, patent, trademark, and
+      attribution notices from the Source form of the Work,
+      excluding those notices that do not pertain to any part of
+      the Derivative Works; and
+
+  (d) If the Work includes a "NOTICE" text file as part of its
+      distribution, then any Derivative Works that You distribute must
+      include a readable copy of the attribution notices contained
+      within such NOTICE file, excluding those notices that do not
+      pertain to any part of the Derivative Works, in at least one
+      of the following places: within a NOTICE text file distributed
+      as part of the Derivative Works; within the Source form or
+      documentation, if provided along with the Derivative Works; or,
+      within a display generated by the Derivative Works, if and
+      wherever such third-party notices normally appear. The contents
+      of the NOTICE file are for informational purposes only and
+      do not modify the License. You may add Your own attribution
+      notices within Derivative Works that You distribute, alongside
+      or as an addendum to the NOTICE text from the Work, provided
+      that such additional attribution notices cannot be construed
+      as modifying the License.
+
+  You may add Your own copyright statement to Your modifications and
+  may provide additional or different license terms and conditions
+  for use, reproduction, or distribution of Your modifications, or
+  for any such Derivative Works as a whole, provided Your use,
+  reproduction, and distribution of the Work otherwise complies with
+  the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+  any Contribution intentionally submitted for inclusion in the Work
+  by You to the Licensor shall be under the terms and conditions of
+  this License, without any additional terms or conditions.
+  Notwithstanding the above, nothing herein shall supersede or modify
+  the terms of any separate license agreement you may have executed
+  with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+  names, trademarks, service marks, or product names of the Licensor,
+  except as required for reasonable and customary use in describing the
+  origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+  agreed to in writing, Licensor provides the Work (and each
+  Contributor provides its Contributions) on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+  implied, including, without limitation, any warranties or conditions
+  of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+  PARTICULAR PURPOSE. You are solely responsible for determining the
+  appropriateness of using or redistributing the Work and assume any
+  risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+  whether in tort (including negligence), contract, or otherwise,
+  unless required by applicable law (such as deliberate and grossly
+  negligent acts) or agreed to in writing, shall any Contributor be
+  liable to You for damages, including any direct, indirect, special,
+  incidental, or consequential damages of any character arising as a
+  result of this License or out of the use or inability to use the
+  Work (including but not limited to damages for loss of goodwill,
+  work stoppage, computer failure or malfunction, or any and all
+  other commercial damages or losses), even if such Contributor
+  has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+  the Work or Derivative Works thereof, You may choose to offer,
+  and charge a fee for, acceptance of support, warranty, indemnity,
+  or other liability obligations and/or rights consistent with this
+  License. However, in accepting such obligations, You may act only
+  on Your own behalf and on Your sole responsibility, not on behalf
+  of any other Contributor, and only if You agree to indemnify,
+  defend, and hold each Contributor harmless for any liability
+  incurred by, or claims asserted against, such Contributor by reason
+  of your accepting any such warranty or additional liability.

data/README.rdoc

@@ -29,15 +29,15 @@ usernames, lists, hashtags and URLs.
 
     html = auto_link("link @user, please #request")
   end
-  
+
   # For Ruby on Rails you want to add this to app/helpers/application_helper.rb
   module ApplicationHelper
     include Twitter::Autolink
   end
-  
+
   # Now the auto_link function is available in every view. So in index.html.erb:
   <%= auto_link("link @user, please #request") %>
-  
+
 === Usernames
 
 Username extraction and linking matches all valid Twitter usernames but does
@@ -50,8 +50,8 @@ format.
 
 === Hashtags
 
-Auto-link and extract hashtags, where a hashtag contains any latin letter or
-number but cannot be solely numbers.
+Auto-link and extract hashtags, where a hashtag can contain most letters or
+numbers but cannot be solely numbers and cannot contain punctuation.
 
 === URLs
 
@@ -74,7 +74,7 @@ to work against text that has been auto-linked already.
 
 To run the Conformance suite, you'll need to add that project as a git submodule.  From the root twitter-text-rb directory, run:
 
-  git submodule add git@github.com:mzsanford/twitter-text-conformance.git test/twitter-text-conformance/
+  git submodule add git@github.com:twitter/twitter-text-conformance.git test/twitter-text-conformance/
   git submodule init
   git submodule update
 
@@ -83,6 +83,7 @@ To run the Conformance suite, you'll need to add that project as a git submodule
 Thanks to everybody who has filed issues, provided feedback or contributed patches. Patches courtesy of:
 
 * At Twitter …
+  * Matt Sanford - http://github.com/mzsanford
   * Raffi Krikorian - http://github.com/r
   * Ben Cherry - http://github.com/bcherry
   * Patrick Ewing - http://github.com/hoverbird
@@ -93,3 +94,19 @@ Thanks to everybody who has filed issues, provided feedback or contributed patch
   * Jean-Philippe Bougie - http://github.com/jpbougie
 
 * Anyone who has filed an issue. It helps. Really.
+
+=== Copyright and License
+
+    Copyright 2011 Twitter, Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this work except in compliance with the License.
+    You may obtain a copy of the License in the LICENSE file, or at:
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.

data/lib/autolink.rb

@@ -137,6 +137,12 @@ module Twitter
         text = $3
         text = yield(text) if block_given?
         "#{before}<a href=\"#{options[:hashtag_url_base]}#{html_escape(text)}\" title=\"##{html_escape(text)}\" #{target_tag(options)}class=\"#{options[:url_class]} #{options[:hashtag_class]}\"#{extra_html}>#{html_escape(hash)}#{html_escape(text)}</a>"
+      end.gsub(Twitter::Regex[:auto_link_cj_hashtags]) do
+          before = $1
+          hash = $2
+          text = $3
+          text = yield(text) if block_given?
+          "#{before}<a href=\"#{options[:hashtag_url_base]}#{html_escape(text)}\" title=\"##{html_escape(text)}\" #{target_tag(options)}class=\"#{options[:url_class]} #{options[:hashtag_class]}\"#{extra_html}>#{html_escape(hash)}#{html_escape(text)}</a>"
       end
     end
 

data/lib/extractor.rb

@@ -165,6 +165,13 @@ module Twitter
           :hashtag => hash_text,
           :indices => [start_position, end_position]
         }
+      end.scan(Twitter::Regex[:auto_link_cj_hashtags]) do |before, hash, hash_text|
+        start_position = $~.char_begin(2)
+        end_position = $~.char_end(3)
+        tags << {
+          :hashtag => hash_text,
+          :indices => [start_position, end_position]
+        }
       end
       tags.each{|tag| yield tag[:hashtag], tag[:indices].first, tag[:indices].last } if block_given?
       tags

data/lib/regex.rb

@@ -24,6 +24,7 @@ module Twitter
           0x205F,          # White_Space # Zs       MEDIUM MATHEMATICAL SPACE
           0x3000,          # White_Space # Zs       IDEOGRAPHIC SPACE
         ].flatten.freeze
+    SPACE_CHAR_CLASS_VALUE = Regexp.new(UNICODE_SPACES.collect{ |e| [e].pack 'U*' }.join(''))
     REGEXEN[:spaces] = Regexp.new(UNICODE_SPACES.collect{ |e| [e].pack 'U*' }.join('|'))
 
     REGEXEN[:at_signs] = /[@＠]/
@@ -42,25 +43,58 @@ module Twitter
     # Excludes 0xd7 from the range (the multiplication sign, confusable with "x").
     # Also excludes 0xf7, the division sign
     LATIN_ACCENTS = [(0xc0..0xd6).to_a, (0xd8..0xf6).to_a, (0xf8..0xff).to_a].flatten.pack('U*').freeze
+    NON_LATIN_HASHTAG_CHARS = [
+      # Cyrillic (Russian, Ukrainian, etc.)
+      (0x0400..0x04ff).to_a, # Cyrillic
+      (0x0500..0x0527).to_a, # Cyrillic Supplement
+      # Hangul (Korean)
+      (0x1100..0x11ff).to_a, # Hangul Jamo
+      (0x3130..0x3185).to_a, # Hangul Compatibility Jamo
+      (0xA960..0xA97F).to_a, # Hangul Jamo Extended-A
+      (0xAC00..0xD7AF).to_a, # Hangul Syllables
+      (0xD7B0..0xD7FF).to_a # Hangul Jamo Extended-B
+    ].flatten.pack('U*').freeze
     REGEXEN[:latin_accents] = /[#{LATIN_ACCENTS}]+/o
 
     REGEXEN[:end_screen_name_match] = /^(?:#{REGEXEN[:at_signs]}|#{REGEXEN[:latin_accents]}|:\/\/)/o
 
+    CJ_HASHTAG_CHARACTERS = [
+      (0x30A1..0x30FA).to_a, # Katakana (full-width)
+      (0xFF66..0xFF9D).to_a, # Katakana (half-width)
+      (0xFF10..0xFF19).to_a, (0xFF21..0xFF3A).to_a, (0xFF41..0xFF5A).to_a, # Latin (full-width)
+      (0x3041..0x3096).to_a, # Hiragana
+      (0x3400..0x4DBF).to_a, # Kanji (CJK Extension A)
+      (0x4E00..0x9FFF).to_a, # Kanji (Unified)
+      (0x20000..0x2A6DF).to_a, # Kanji (CJK Extension B)
+      (0x2A700..0x2B73F).to_a, # Kanji (CJK Extension C)
+      (0x2B740..0x2B81F).to_a, # Kanji (CJK Extension D)
+      (0x2F800..0x2FA1F).to_a # Kanji (CJK supplement)
+    ].flatten.pack('U*').freeze
+
+    CJ_BOUNDARY = /(?:\A|\z|#{REGEXEN[:spaces]}|「|」|。|\.)/
+
     # A hashtag must contain latin characters, numbers and underscores, but not all numbers.
-    HASHTAG_ALPHA = /[a-z_#{LATIN_ACCENTS}]/io
-    HASHTAG_ALPHANUMERIC = /[a-z0-9_#{LATIN_ACCENTS}]/io
-    REGEXEN[:auto_link_hashtags] = /(^|[^0-9A-Z&\/\?]+)(#|＃)(#{HASHTAG_ALPHANUMERIC}*#{HASHTAG_ALPHA}#{HASHTAG_ALPHANUMERIC}*)/io
+    HASHTAG_ALPHA = /[a-z_#{LATIN_ACCENTS}#{NON_LATIN_HASHTAG_CHARS}]/io
+    HASHTAG_ALPHANUMERIC = /[a-z0-9_#{LATIN_ACCENTS}#{NON_LATIN_HASHTAG_CHARS}]/io
+
+    CJ_HASHTAG = /(#{CJ_BOUNDARY})(#|＃)([#{CJ_HASHTAG_CHARACTERS}]+)(?=#{CJ_BOUNDARY})/o
+    NON_CJ_HASHTAG = /(^|[^0-9A-Z&\/\?]+)(#|＃)(#{HASHTAG_ALPHANUMERIC}*#{HASHTAG_ALPHA}#{HASHTAG_ALPHANUMERIC}*)/io
+
+    REGEXEN[:auto_link_hashtags] = /#{NON_CJ_HASHTAG}/io
+    REGEXEN[:auto_link_cj_hashtags] = /#{CJ_HASHTAG}/io
+
     REGEXEN[:auto_link_usernames_or_lists] = /([^a-zA-Z0-9_]|^|RT:?)([@＠]+)([a-zA-Z0-9_]{1,20})(\/[a-zA-Z][a-zA-Z0-9_\-]{0,24})?/o
     REGEXEN[:auto_link_emoticon] = /(8\-\#|8\-E|\+\-\(|\`\@|\`O|\<\|:~\(|\}:o\{|:\-\[|\>o\<|X\-\/|\[:-\]\-I\-|\/\/\/\/Ö\\\\\\\\|\(\|:\|\/\)|∑:\*\)|\( \| \))/
 
     # URL related hash regex collection
     REGEXEN[:valid_preceding_chars] = /(?:[^-\/"':!=A-Z0-9_@＠]|^|\:)/i
 
-    REGEXEN[:valid_subdomain] = /([^[:punct:]\s]([_-]|[^[:punct:]\s])*)?[^[:punct:]\s]\./
-    REGEXEN[:valid_domain_name] = /([^[:punct:]\s]([-]|[^[:punct:]\s])*)?[^[:punct:]\s]/
-    REGEXEN[:valid_domain] = /#{REGEXEN[:valid_subdomain]}*#{REGEXEN[:valid_domain_name]}\.[a-z]{2,}(?::[0-9]+)?/i
+    DOMAIN_EXCLUDE_PART = "[:punct:][:space:][:blank:]#{[0x00A0].pack('U')}"
+    REGEXEN[:valid_subdomain] = /(?:[^#{DOMAIN_EXCLUDE_PART}](?:[_-]|[^#{DOMAIN_EXCLUDE_PART}])*)?[^#{DOMAIN_EXCLUDE_PART}]\./
+    REGEXEN[:valid_domain_name] = /(?:[^#{DOMAIN_EXCLUDE_PART}](?:[-]|[^#{DOMAIN_EXCLUDE_PART}])*)?[^#{DOMAIN_EXCLUDE_PART}]/
+    REGEXEN[:valid_domain] = /#{REGEXEN[:valid_subdomain]}*#{REGEXEN[:valid_domain_name]}\.(?:xn--[a-z0-9]{2,}|[a-z]{2,})(?::[0-9]+)?/i
 
-    REGEXEN[:valid_general_url_path_chars] = /[a-z0-9!\*';:=\+\,\$\/%#\[\]\-_~|]/i
+    REGEXEN[:valid_general_url_path_chars] = /[a-z0-9!\*';:=\+\,\$\/%#\[\]\-_~|\.]/i
     # Allow URL paths to contain balanced parens
     #  1. Used in Wikipedia URLs like /Primer_(film)
     #  2. Used in IIS sessions like /S(dfd346)/

data/spec/extractor_spec.rb

@@ -236,17 +236,6 @@ describe Twitter::Extractor do
         end
       end
 
-      context "should NOT allow Japanese" do
-        %w(会議中 ハッシュ).each do |hashtag|
-          it "should NOT extract ##{hashtag}" do
-            @extractor.extract_hashtags("##{hashtag}").should == []
-          end
-
-          it "should NOT extract ##{hashtag} within text" do
-            @extractor.extract_hashtags("pre-text ##{hashtag} post-text").should == []
-          end
-        end
-      end
     end
 
     it "should not extract numeric hashtags" do
@@ -301,18 +290,6 @@ describe Twitter::Extractor do
           match_hashtag_in_text('pre', "#pre#{[0xf7].pack('U')}post")
         end
       end
-
-      context "should NOT allow Japanese" do
-        %w(会議中 ハッシュ).each do |hashtag|
-          it "should NOT extract ##{hashtag}" do
-            no_match_hashtag_in_text("##{hashtag}")
-          end
-
-          it "should NOT extract ##{hashtag} within text" do
-            no_match_hashtag_in_text("pre-text ##{hashtag} post-text")
-          end
-        end
-      end
     end
 
     it "should not extract numeric hashtags" do

data/twitter-text.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s|
   s.name = "twitter-text"
-  s.version = "1.4.2"
+  s.version = "1.4.5"
   s.authors = ["Matt Sanford", "Patrick Ewing", "Ben Cherry", "Britt Selvitelle", "Raffi Krikorian", "J.P. Cummins", "Yoshimasa Niwa"]
   s.email = ["matt@twitter.com", "patrick.henry.ewing@gmail.com", "bcherry@gmail.com", "bs@brittspace.com", "raffi@twitter.com", "jcummins@twitter.com", "niw@niw.at"]
   s.homepage = "http://twitter.com"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: twitter-text
 version: !ruby/object:Gem::Version 
-  hash: 3
-  prerelease: 
+  hash: 13
+  prerelease: false
   segments: 
   - 1
   - 4
-  - 2
-  version: 1.4.2
+  - 5
+  version: 1.4.5
 platform: ruby
 authors: 
 - Matt Sanford
@@ -21,7 +21,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-19 00:00:00 -07:00
+date: 2011-06-22 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -169,7 +169,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.4.1
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Twitter text handling library