twitter-text-relative 1.6.2.pre.3

data/lib/twitter-text-relative/regex.rb

@@ -0,0 +1,362 @@
+# encoding: UTF-8
+
+module Twitter
+  # A collection of regular expressions for parsing Tweet text. The regular expression
+  # list is frozen at load time to ensure immutability. These regular expressions are
+  # used throughout the <tt>Twitter</tt> classes. Special care has been taken to make
+  # sure these reular expressions work with Tweets in all languages.
+  class Regex
+    REGEXEN = {} # :nodoc:
+
+    def self.regex_range(from, to = nil) # :nodoc:
+      if $RUBY_1_9
+        if to
+          "\\u{#{from.to_s(16).rjust(4, '0')}}-\\u{#{to.to_s(16).rjust(4, '0')}}"
+        else
+          "\\u{#{from.to_s(16).rjust(4, '0')}}"
+        end
+      else
+        if to
+          [from].pack('U') + '-' + [to].pack('U')
+        else
+          [from].pack('U')
+        end
+      end
+   end
+
+    # Space is more than %20, U+3000 for example is the full-width space used with Kanji. Provide a short-hand
+    # to access both the list of characters and a pattern suitible for use with String#split
+    #  Taken from: ActiveSupport::Multibyte::Handlers::UTF8Handler::UNICODE_WHITESPACE
+    UNICODE_SPACES = [
+          (0x0009..0x000D).to_a,  # White_Space # Cc   [5] <control-0009>..<control-000D>
+          0x0020,          # White_Space # Zs       SPACE
+          0x0085,          # White_Space # Cc       <control-0085>
+          0x00A0,          # White_Space # Zs       NO-BREAK SPACE
+          0x1680,          # White_Space # Zs       OGHAM SPACE MARK
+          0x180E,          # White_Space # Zs       MONGOLIAN VOWEL SEPARATOR
+          (0x2000..0x200A).to_a, # White_Space # Zs  [11] EN QUAD..HAIR SPACE
+          0x2028,          # White_Space # Zl       LINE SEPARATOR
+          0x2029,          # White_Space # Zp       PARAGRAPH SEPARATOR
+          0x202F,          # White_Space # Zs       NARROW NO-BREAK SPACE
+          0x205F,          # White_Space # Zs       MEDIUM MATHEMATICAL SPACE
+          0x3000,          # White_Space # Zs       IDEOGRAPHIC SPACE
+    ].flatten.map{|c| [c].pack('U*')}.freeze
+    REGEXEN[:spaces] = /[#{UNICODE_SPACES.join('')}]/o
+
+    # Character not allowed in Tweets
+    INVALID_CHARACTERS = [
+      0xFFFE, 0xFEFF, # BOM
+      0xFFFF,         # Special
+      0x202A, 0x202B, 0x202C, 0x202D, 0x202E # Directional change
+    ].map{|cp| [cp].pack('U') }.freeze
+    REGEXEN[:invalid_control_characters] = /[#{INVALID_CHARACTERS.join('')}]/o
+
+    major, minor, patch = RUBY_VERSION.split('.')
+    if major.to_i >= 2 || major.to_i == 1 && minor.to_i >= 9 || (defined?(RUBY_ENGINE) && ["jruby", "rbx"].include?(RUBY_ENGINE))
+      REGEXEN[:list_name] = /[a-zA-Z][a-zA-Z0-9_\-\u0080-\u00ff]{0,24}/
+    else
+      # This line barfs at compile time in Ruby 1.9, JRuby, or Rubinius.
+      REGEXEN[:list_name] = eval("/[a-zA-Z][a-zA-Z0-9_\\-\x80-\xff]{0,24}/")
+    end
+
+    # Latin accented characters
+    # Excludes 0xd7 from the range (the multiplication sign, confusable with "x").
+    # Also excludes 0xf7, the division sign
+    LATIN_ACCENTS = [
+          regex_range(0xc0, 0xd6),
+          regex_range(0xd8, 0xf6),
+          regex_range(0xf8, 0xff),
+          regex_range(0x0100, 0x024f),
+          regex_range(0x0253, 0x0254),
+          regex_range(0x0256, 0x0257),
+          regex_range(0x0259),
+          regex_range(0x025b),
+          regex_range(0x0263),
+          regex_range(0x0268),
+          regex_range(0x026f),
+          regex_range(0x0272),
+          regex_range(0x0289),
+          regex_range(0x028b),
+          regex_range(0x02bb),
+          regex_range(0x0300, 0x036f),
+          regex_range(0x1e00, 0x1eff)
+    ].join('').freeze
+
+    RTL_CHARACTERS = [
+      regex_range(0x0600,0x06FF),
+      regex_range(0x0750,0x077F),
+      regex_range(0x0590,0x05FF),
+      regex_range(0xFE70,0xFEFF)
+    ].join('').freeze
+
+
+    NON_LATIN_HASHTAG_CHARS = [
+      # Cyrillic (Russian, Ukrainian, etc.)
+      regex_range(0x0400, 0x04ff), # Cyrillic
+      regex_range(0x0500, 0x0527), # Cyrillic Supplement
+      regex_range(0x2de0, 0x2dff), # Cyrillic Extended A
+      regex_range(0xa640, 0xa69f), # Cyrillic Extended B
+      regex_range(0x0591, 0x05bf), # Hebrew
+      regex_range(0x05c1, 0x05c2),
+      regex_range(0x05c4, 0x05c5),
+      regex_range(0x05c7),
+      regex_range(0x05d0, 0x05ea),
+      regex_range(0x05f0, 0x05f4),
+      regex_range(0xfb12, 0xfb28), # Hebrew Presentation Forms
+      regex_range(0xfb2a, 0xfb36),
+      regex_range(0xfb38, 0xfb3c),
+      regex_range(0xfb3e),
+      regex_range(0xfb40, 0xfb41),
+      regex_range(0xfb43, 0xfb44),
+      regex_range(0xfb46, 0xfb4f),
+      regex_range(0x0610, 0x061a), # Arabic
+      regex_range(0x0620, 0x065f),
+      regex_range(0x066e, 0x06d3),
+      regex_range(0x06d5, 0x06dc),
+      regex_range(0x06de, 0x06e8),
+      regex_range(0x06ea, 0x06ef),
+      regex_range(0x06fa, 0x06fc),
+      regex_range(0x06ff),
+      regex_range(0x0750, 0x077f), # Arabic Supplement
+      regex_range(0x08a0),         # Arabic Extended A
+      regex_range(0x08a2, 0x08ac),
+      regex_range(0x08e4, 0x08fe),
+      regex_range(0xfb50, 0xfbb1), # Arabic Pres. Forms A
+      regex_range(0xfbd3, 0xfd3d),
+      regex_range(0xfd50, 0xfd8f),
+      regex_range(0xfd92, 0xfdc7),
+      regex_range(0xfdf0, 0xfdfb),
+      regex_range(0xfe70, 0xfe74), # Arabic Pres. Forms B
+      regex_range(0xfe76, 0xfefc),
+      regex_range(0x200c, 0x200c), # Zero-Width Non-Joiner
+      regex_range(0x0e01, 0x0e3a), # Thai
+      regex_range(0x0e40, 0x0e4e), # Hangul (Korean)
+      regex_range(0x1100, 0x11ff), # Hangul Jamo
+      regex_range(0x3130, 0x3185), # Hangul Compatibility Jamo
+      regex_range(0xA960, 0xA97F), # Hangul Jamo Extended-A
+      regex_range(0xAC00, 0xD7AF), # Hangul Syllables
+      regex_range(0xD7B0, 0xD7FF), # Hangul Jamo Extended-B
+      regex_range(0xFFA1, 0xFFDC)  # Half-width Hangul
+    ].join('').freeze
+    REGEXEN[:latin_accents] = /[#{LATIN_ACCENTS}]+/o
+
+    CJ_HASHTAG_CHARACTERS = [
+      regex_range(0x30A1, 0x30FA), regex_range(0x30FC, 0x30FE), # Katakana (full-width)
+      regex_range(0xFF66, 0xFF9F), # Katakana (half-width)
+      regex_range(0xFF10, 0xFF19), regex_range(0xFF21, 0xFF3A), regex_range(0xFF41, 0xFF5A), # Latin (full-width)
+      regex_range(0x3041, 0x3096), regex_range(0x3099, 0x309E), # Hiragana
+      regex_range(0x3400, 0x4DBF), # Kanji (CJK Extension A)
+      regex_range(0x4E00, 0x9FFF), # Kanji (Unified)
+      regex_range(0x20000, 0x2A6DF), # Kanji (CJK Extension B)
+      regex_range(0x2A700, 0x2B73F), # Kanji (CJK Extension C)
+      regex_range(0x2B740, 0x2B81F), # Kanji (CJK Extension D)
+      regex_range(0x2F800, 0x2FA1F), regex_range(0x3003), regex_range(0x3005), regex_range(0x303B) # Kanji (CJK supplement)
+    ].join('').freeze
+
+    PUNCTUATION_CHARS = '!"#$%&\'()*+,-./:;<=>?@\[\]^_\`{|}~'
+    SPACE_CHARS = " \t\n\x0B\f\r"
+    CTRL_CHARS = "\x00-\x1F\x7F"
+
+    # A hashtag must contain latin characters, numbers and underscores, but not all numbers.
+    HASHTAG_ALPHA = /[a-z_#{LATIN_ACCENTS}#{NON_LATIN_HASHTAG_CHARS}#{CJ_HASHTAG_CHARACTERS}]/io
+    HASHTAG_ALPHANUMERIC = /[a-z0-9_#{LATIN_ACCENTS}#{NON_LATIN_HASHTAG_CHARS}#{CJ_HASHTAG_CHARACTERS}]/io
+    HASHTAG_BOUNDARY = /\A|\z|[^&a-z0-9_#{LATIN_ACCENTS}#{NON_LATIN_HASHTAG_CHARS}#{CJ_HASHTAG_CHARACTERS}]/o
+
+    HASHTAG = /(#{HASHTAG_BOUNDARY})(#|＃)(#{HASHTAG_ALPHANUMERIC}*#{HASHTAG_ALPHA}#{HASHTAG_ALPHANUMERIC}*)/io
+
+    REGEXEN[:valid_hashtag] = /#{HASHTAG}/io
+    # Used in Extractor for final filtering
+    REGEXEN[:end_hashtag_match] = /\A(?:[#＃]|:\/\/)/o
+
+    REGEXEN[:valid_mention_preceding_chars] = /(?:[^a-zA-Z0-9_!#\$%&*@＠]|^|[rR][tT]:?)/o
+    REGEXEN[:at_signs] = /[@＠]/
+    REGEXEN[:valid_mention_or_list] = /
+      (#{REGEXEN[:valid_mention_preceding_chars]})  # $1: Preceeding character
+      (#{REGEXEN[:at_signs]})                       # $2: At mark
+      ([a-zA-Z0-9_]{1,20})                          # $3: Screen name
+      (\/[a-zA-Z][a-zA-Z0-9_\-]{0,24})?             # $4: List (optional)
+    /ox
+    REGEXEN[:valid_reply] = /^(?:#{REGEXEN[:spaces]})*#{REGEXEN[:at_signs]}([a-zA-Z0-9_]{1,20})/o
+    # Used in Extractor for final filtering
+    REGEXEN[:end_mention_match] = /\A(?:#{REGEXEN[:at_signs]}|#{REGEXEN[:latin_accents]}|:\/\/)/o
+
+    # URL related hash regex collection
+    REGEXEN[:valid_url_preceding_chars] = /(?:[^A-Z0-9@＠$#＃#{INVALID_CHARACTERS.join('')}]|^)/io
+    REGEXEN[:invalid_url_without_protocol_preceding_chars] = /[-_.\/]$/
+    DOMAIN_VALID_CHARS = "[^#{PUNCTUATION_CHARS}#{SPACE_CHARS}#{CTRL_CHARS}#{INVALID_CHARACTERS.join('')}#{UNICODE_SPACES.join('')}]"
+    REGEXEN[:valid_subdomain] = /(?:(?:#{DOMAIN_VALID_CHARS}(?:[_-]|#{DOMAIN_VALID_CHARS})*)?#{DOMAIN_VALID_CHARS}\.)/io
+    REGEXEN[:valid_domain_name] = /(?:(?:#{DOMAIN_VALID_CHARS}(?:[-]|#{DOMAIN_VALID_CHARS})*)?#{DOMAIN_VALID_CHARS}\.)/io
+
+    REGEXEN[:valid_gTLD] = /(?:(?:aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|xxx)(?=[^0-9a-z]|$))/i
+    REGEXEN[:valid_ccTLD] = %r{
+      (?:
+        (?:ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|
+        ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|
+        gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|
+        lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|
+        pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|
+        th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw)
+        (?=[^0-9a-z]|$)
+      )
+    }ix
+    REGEXEN[:valid_punycode] = /(?:xn--[0-9a-z]+)/i
+
+    REGEXEN[:valid_domain] = /(?:
+      #{REGEXEN[:valid_subdomain]}*#{REGEXEN[:valid_domain_name]}
+      (?:#{REGEXEN[:valid_gTLD]}|#{REGEXEN[:valid_ccTLD]}|#{REGEXEN[:valid_punycode]})
+    )/iox
+
+    # This is used in Extractor
+    REGEXEN[:valid_ascii_domain] = /
+      (?:(?:[A-Za-z0-9\-_]|#{REGEXEN[:latin_accents]})+\.)+
+      (?:#{REGEXEN[:valid_gTLD]}|#{REGEXEN[:valid_ccTLD]}|#{REGEXEN[:valid_punycode]})
+    /iox
+
+    # This is used in Extractor for stricter t.co URL extraction
+    REGEXEN[:valid_tco_url] = /^https?:\/\/t\.co\/[a-z0-9]+/i
+
+    # This is used in Extractor to filter out unwanted URLs.
+    REGEXEN[:invalid_short_domain] = /\A#{REGEXEN[:valid_domain_name]}#{REGEXEN[:valid_ccTLD]}\Z/io
+
+    REGEXEN[:valid_port_number] = /[0-9]+/
+
+    REGEXEN[:valid_general_url_path_chars] = /[a-z0-9!\*';:=\+\,\.\$\/%#\[\]\-_~&|@#{LATIN_ACCENTS}]/io
+    # Allow URL paths to contain balanced parens
+    #  1. Used in Wikipedia URLs like /Primer_(film)
+    #  2. Used in IIS sessions like /S(dfd346)/
+    REGEXEN[:valid_url_balanced_parens] = /\(#{REGEXEN[:valid_general_url_path_chars]}+\)/io
+    # Valid end-of-path chracters (so /foo. does not gobble the period).
+    #   1. Allow =&# for empty URL parameters and other URL-join artifacts
+    REGEXEN[:valid_url_path_ending_chars] = /[a-z0-9=_#\/\+\-#{LATIN_ACCENTS}]|(?:#{REGEXEN[:valid_url_balanced_parens]})/io
+    REGEXEN[:valid_url_path] = /(?:
+      (?:
+        #{REGEXEN[:valid_general_url_path_chars]}*
+        (?:#{REGEXEN[:valid_url_balanced_parens]} #{REGEXEN[:valid_general_url_path_chars]}*)*
+        #{REGEXEN[:valid_url_path_ending_chars]}
+      )|(?:#{REGEXEN[:valid_general_url_path_chars]}+\/)
+    )/iox
+
+    REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@]/i
+    REGEXEN[:valid_url_query_ending_chars] = /[a-z0-9_&=#\/]/i
+    REGEXEN[:valid_url] = %r{
+      (                                                                                     #   $1 total match
+        (#{REGEXEN[:valid_url_preceding_chars]})                                            #   $2 Preceeding chracter
+        (                                                                                   #   $3 URL
+          (https?:\/\/)?                                                                    #   $4 Protocol (optional)
+          (#{REGEXEN[:valid_domain]})                                                       #   $5 Domain(s)
+          (?::(#{REGEXEN[:valid_port_number]}))?                                            #   $6 Port number (optional)
+          (/#{REGEXEN[:valid_url_path]}*)?                                                  #   $7 URL Path and anchor
+          (\?#{REGEXEN[:valid_url_query_chars]}*#{REGEXEN[:valid_url_query_ending_chars]})? #   $8 Query String
+        )
+      )
+    }iox;
+
+    REGEXEN[:cashtag] = /[a-z]{1,6}(?:[._][a-z]{1,2})?/i
+    REGEXEN[:valid_cashtag] = /(^|#{REGEXEN[:spaces]})(\$)(#{REGEXEN[:cashtag]})(?=$|\s|[#{PUNCTUATION_CHARS}])/i
+
+    # These URL validation pattern strings are based on the ABNF from RFC 3986
+    REGEXEN[:validate_url_unreserved] = /[a-z0-9\-._~]/i
+    REGEXEN[:validate_url_pct_encoded] = /(?:%[0-9a-f]{2})/i
+    REGEXEN[:validate_url_sub_delims] = /[!$&'()*+,;=]/i
+    REGEXEN[:validate_url_pchar] = /(?:
+      #{REGEXEN[:validate_url_unreserved]}|
+      #{REGEXEN[:validate_url_pct_encoded]}|
+      #{REGEXEN[:validate_url_sub_delims]}|
+      [:\|@]
+    )/iox
+
+    REGEXEN[:validate_url_scheme] = /(?:[a-z][a-z0-9+\-.]*)/i
+    REGEXEN[:validate_url_userinfo] = /(?:
+      #{REGEXEN[:validate_url_unreserved]}|
+      #{REGEXEN[:validate_url_pct_encoded]}|
+      #{REGEXEN[:validate_url_sub_delims]}|
+      :
+    )*/iox
+
+    REGEXEN[:validate_url_dec_octet] = /(?:[0-9]|(?:[1-9][0-9])|(?:1[0-9]{2})|(?:2[0-4][0-9])|(?:25[0-5]))/i
+    REGEXEN[:validate_url_ipv4] =
+      /(?:#{REGEXEN[:validate_url_dec_octet]}(?:\.#{REGEXEN[:validate_url_dec_octet]}){3})/iox
+
+    # Punting on real IPv6 validation for now
+    REGEXEN[:validate_url_ipv6] = /(?:\[[a-f0-9:\.]+\])/i
+
+    # Also punting on IPvFuture for now
+    REGEXEN[:validate_url_ip] = /(?:
+      #{REGEXEN[:validate_url_ipv4]}|
+      #{REGEXEN[:validate_url_ipv6]}
+    )/iox
+
+    # This is more strict than the rfc specifies
+    REGEXEN[:validate_url_subdomain_segment] = /(?:[a-z0-9](?:[a-z0-9_\-]*[a-z0-9])?)/i
+    REGEXEN[:validate_url_domain_segment] = /(?:[a-z0-9](?:[a-z0-9\-]*[a-z0-9])?)/i
+    REGEXEN[:validate_url_domain_tld] = /(?:[a-z](?:[a-z0-9\-]*[a-z0-9])?)/i
+    REGEXEN[:validate_url_domain] = /(?:(?:#{REGEXEN[:validate_url_subdomain_segment]}\.)*
+                                     (?:#{REGEXEN[:validate_url_domain_segment]}\.)
+                                     #{REGEXEN[:validate_url_domain_tld]})/iox
+
+    REGEXEN[:validate_url_host] = /(?:
+      #{REGEXEN[:validate_url_ip]}|
+      #{REGEXEN[:validate_url_domain]}
+    )/iox
+
+    # Unencoded internationalized domains - this doesn't check for invalid UTF-8 sequences
+    REGEXEN[:validate_url_unicode_subdomain_segment] =
+      /(?:(?:[a-z0-9]|[^\x00-\x7f])(?:(?:[a-z0-9_\-]|[^\x00-\x7f])*(?:[a-z0-9]|[^\x00-\x7f]))?)/ix
+    REGEXEN[:validate_url_unicode_domain_segment] =
+      /(?:(?:[a-z0-9]|[^\x00-\x7f])(?:(?:[a-z0-9\-]|[^\x00-\x7f])*(?:[a-z0-9]|[^\x00-\x7f]))?)/ix
+    REGEXEN[:validate_url_unicode_domain_tld] =
+      /(?:(?:[a-z]|[^\x00-\x7f])(?:(?:[a-z0-9\-]|[^\x00-\x7f])*(?:[a-z0-9]|[^\x00-\x7f]))?)/ix
+    REGEXEN[:validate_url_unicode_domain] = /(?:(?:#{REGEXEN[:validate_url_unicode_subdomain_segment]}\.)*
+                                             (?:#{REGEXEN[:validate_url_unicode_domain_segment]}\.)
+                                             #{REGEXEN[:validate_url_unicode_domain_tld]})/iox
+
+    REGEXEN[:validate_url_unicode_host] = /(?:
+      #{REGEXEN[:validate_url_ip]}|
+      #{REGEXEN[:validate_url_unicode_domain]}
+    )/iox
+
+    REGEXEN[:validate_url_port] = /[0-9]{1,5}/
+
+    REGEXEN[:validate_url_unicode_authority] = %r{
+      (?:(#{REGEXEN[:validate_url_userinfo]})@)?     #  $1 userinfo
+      (#{REGEXEN[:validate_url_unicode_host]})       #  $2 host
+      (?::(#{REGEXEN[:validate_url_port]}))?         #  $3 port
+    }iox
+
+    REGEXEN[:validate_url_authority] = %r{
+      (?:(#{REGEXEN[:validate_url_userinfo]})@)?     #  $1 userinfo
+      (#{REGEXEN[:validate_url_host]})               #  $2 host
+      (?::(#{REGEXEN[:validate_url_port]}))?         #  $3 port
+    }iox
+
+    REGEXEN[:validate_url_path] = %r{(/#{REGEXEN[:validate_url_pchar]}*)*}i
+    REGEXEN[:validate_url_query] = %r{(#{REGEXEN[:validate_url_pchar]}|/|\?)*}i
+    REGEXEN[:validate_url_fragment] = %r{(#{REGEXEN[:validate_url_pchar]}|/|\?)*}i
+
+    # Modified version of RFC 3986 Appendix B
+    REGEXEN[:validate_url_unencoded] = %r{
+      \A                                #  Full URL
+      (?:
+        ([^:/?#]+)://                  #  $1 Scheme
+      )?
+      ([^/?#]*)                        #  $2 Authority
+      ([^?#]*)                         #  $3 Path
+      (?:
+        \?([^#]*)                      #  $4 Query
+      )?
+      (?:
+        \#(.*)                         #  $5 Fragment
+      )?\Z
+    }ix
+
+    REGEXEN[:rtl_chars] = /[#{RTL_CHARACTERS}]/io
+
+    REGEXEN.each_pair{|k,v| v.freeze }
+
+    # Return the regular expression for a given <tt>key</tt>. If the <tt>key</tt>
+    # is not a known symbol a <tt>nil</tt> will be returned.
+    def self.[](key)
+      REGEXEN[key]
+    end
+  end
+end

data/lib/twitter-text-relative/rewriter.rb

@@ -0,0 +1,59 @@
+module Twitter
+  # A module provides base methods to rewrite usernames, lists, hashtags and URLs.
+  module Rewriter extend self
+    def rewrite_entities(text, entities)
+      chars = text.to_s.to_char_a
+
+      # sort by start index
+      entities = entities.sort_by{|entity| entity[:indices].first}
+
+      result = []
+      last_index = entities.inject(0) do |last_index, entity|
+        result << chars[last_index...entity[:indices].first]
+        result << yield(entity, chars)
+        entity[:indices].last
+      end
+      result << chars[last_index..-1]
+
+      result.flatten.join
+    end
+
+    # These methods are deprecated, will be removed in future.
+    extend Deprecation
+
+    def rewrite(text, options = {})
+      [:hashtags, :urls, :usernames_or_lists].inject(text) do |key|
+        options[key] ? send(:"rewrite_#{key}", text, &options[key]) : text
+      end
+    end
+    deprecate :rewrite, :rewrite_entities
+
+    def rewrite_usernames_or_lists(text)
+      entities = Extractor.extract_mentions_or_lists_with_indices(text)
+      rewrite_entities(text, entities) do |entity, chars|
+        at = chars[entity[:indices].first]
+        list_slug = entity[:list_slug]
+        list_slug = nil if list_slug.empty?
+        yield(at, entity[:screen_name], list_slug)
+      end
+    end
+    deprecate :rewrite_usernames_or_lists, :rewrite_entities
+
+    def rewrite_hashtags(text)
+      entities = Extractor.extract_hashtags_with_indices(text)
+      rewrite_entities(text, entities) do |entity, chars|
+        hash = chars[entity[:indices].first]
+        yield(hash, entity[:hashtag])
+      end
+    end
+    deprecate :rewrite_hashtags, :rewrite_entities
+
+    def rewrite_urls(text)
+      entities = Extractor.extract_urls_with_indices(text, :extract_url_without_protocol => false)
+      rewrite_entities(text, entities) do |entity, chars|
+        yield(entity[:url])
+      end
+    end
+    deprecate :rewrite_urls, :rewrite_entities
+  end
+end

data/lib/twitter-text-relative/unicode.rb

@@ -0,0 +1,26 @@
+module Twitter
+  # This module lazily defines constants of the form Uxxxx for all Unicode
+  # codepoints from U0000 to U10FFFF. The value of each constant is the
+  # UTF-8 string for the codepoint.
+  # Examples:
+  #   copyright = Unicode::U00A9
+  #   euro = Unicode::U20AC
+  #   infinity = Unicode::U221E
+  #
+  module Unicode
+    CODEPOINT_REGEX = /^U_?([0-9a-fA-F]{4,5}|10[0-9a-fA-F]{4})$/
+
+    def self.const_missing(name)
+      # Check that the constant name is of the right form: U0000 to U10FFFF
+      if name.to_s =~ CODEPOINT_REGEX
+        # Convert the codepoint to an immutable UTF-8 string,
+        # define a real constant for that value and return the value
+        #p name, name.class
+        const_set(name, [$1.to_i(16)].pack("U").freeze)
+      else  # Raise an error for constants that are not Unicode.
+        raise NameError, "Uninitialized constant: Unicode::#{name}"
+      end
+    end
+  end
+
+end

data/lib/twitter-text-relative/validation.rb

@@ -0,0 +1,113 @@
+require 'unf'
+
+module Twitter
+  module Validation extend self
+    MAX_LENGTH = 140
+
+    DEFAULT_TCO_URL_LENGTHS = {
+      :short_url_length => 22,
+      :short_url_length_https => 23,
+      :characters_reserved_per_media => 22
+    }.freeze
+
+    # Returns the length of the string as it would be displayed. This is equivilent to the length of the Unicode NFC
+    # (See: http://www.unicode.org/reports/tr15). This is needed in order to consistently calculate the length of a
+    # string no matter which actual form was transmitted. For example:
+    #
+    #     U+0065  Latin Small Letter E
+    # +   U+0301  Combining Acute Accent
+    # ----------
+    # =   2 bytes, 2 characters, displayed as é (1 visual glyph)
+    #     … The NFC of {U+0065, U+0301} is {U+00E9}, which is a single chracter and a +display_length+ of 1
+    #
+    # The string could also contain U+00E9 already, in which case the canonicalization will not change the value.
+    #
+    def tweet_length(text, options = {})
+      options = DEFAULT_TCO_URL_LENGTHS.merge(options)
+
+      length = text.to_nfc.unpack("U*").length
+
+      Twitter::Extractor.extract_urls_with_indices(text) do |url, start_position, end_position|
+        length += start_position - end_position
+        length += url.downcase =~ /^https:\/\// ? options[:short_url_length_https] : options[:short_url_length]
+      end
+
+      length
+    end
+
+    # Check the <tt>text</tt> for any reason that it may not be valid as a Tweet. This is meant as a pre-validation
+    # before posting to api.twitter.com. There are several server-side reasons for Tweets to fail but this pre-validation
+    # will allow quicker feedback.
+    #
+    # Returns <tt>false</tt> if this <tt>text</tt> is valid. Otherwise one of the following Symbols will be returned:
+    #
+    #   <tt>:too_long</tt>:: if the <tt>text</tt> is too long
+    #   <tt>:empty</tt>:: if the <tt>text</tt> is nil or empty
+    #   <tt>:invalid_characters</tt>:: if the <tt>text</tt> contains non-Unicode or any of the disallowed Unicode characters
+    def tweet_invalid?(text)
+      return :empty if !text || text.empty?
+      begin
+        return :too_long if tweet_length(text) > MAX_LENGTH
+        return :invalid_characters if Twitter::Regex::INVALID_CHARACTERS.any?{|invalid_char| text.include?(invalid_char) }
+      rescue ArgumentError => e
+        # non-Unicode value.
+        return :invalid_characters
+      end
+
+      return false
+    end
+
+    def valid_tweet_text?(text)
+      !tweet_invalid?(text)
+    end
+
+    def valid_username?(username)
+      return false if !username || username.empty?
+
+      extracted = Twitter::Extractor.extract_mentioned_screen_names(username)
+      # Should extract the username minus the @ sign, hence the [1..-1]
+      extracted.size == 1 && extracted.first == username[1..-1]
+    end
+
+    VALID_LIST_RE = /\A#{Twitter::Regex[:valid_mention_or_list]}\z/o
+    def valid_list?(username_list)
+      match = username_list.match(VALID_LIST_RE)
+      # Must have matched and had nothing before or after
+      !!(match && match[1] == "" && match[4] && !match[4].empty?)
+    end
+
+    def valid_hashtag?(hashtag)
+      return false if !hashtag || hashtag.empty?
+
+      extracted = Twitter::Extractor.extract_hashtags(hashtag)
+      # Should extract the hashtag minus the # sign, hence the [1..-1]
+      extracted.size == 1 && extracted.first == hashtag[1..-1]
+    end
+
+    def valid_url?(url, unicode_domains=true, require_protocol=true)
+      return false if !url || url.empty?
+
+      url_parts = url.match(Twitter::Regex[:validate_url_unencoded])
+      return false unless (url_parts && url_parts.to_s == url)
+
+      scheme, authority, path, query, fragment = url_parts.captures
+
+      return false unless ((!require_protocol ||
+                           (valid_match?(scheme, Twitter::Regex[:validate_url_scheme]) && scheme.match(/\Ahttps?\Z/i))) &&
+                           valid_match?(path, Twitter::Regex[:validate_url_path]) &&
+                           valid_match?(query, Twitter::Regex[:validate_url_query], true) &&
+                           valid_match?(fragment, Twitter::Regex[:validate_url_fragment], true))
+
+      return (unicode_domains && valid_match?(authority, Twitter::Regex[:validate_url_unicode_authority])) ||
+             (!unicode_domains && valid_match?(authority, Twitter::Regex[:validate_url_authority]))
+    end
+
+    private
+
+    def valid_match?(string, regex, optional=false)
+      return (string && string.match(regex) && $~.to_s == string) unless optional
+
+      !(string && (!string.match(regex) || $~.to_s != string))
+    end
+  end
+end