twitter-text-simpleidn 3.0.0.0

data/lib/twitter-text/configuration.rb

@@ -0,0 +1,68 @@
+# Copyright 2018 Twitter, Inc.
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# encoding: UTF-8
+
+module Twitter
+  module TwitterText
+    class Configuration
+      require 'json'
+
+      PARSER_VERSION_CLASSIC = "v1"
+      PARSER_VERSION_WEIGHTED = "v2"
+      PARSER_VERSION_EMOJI_PARSING = "v3"
+
+      PARSER_VERSION_DEFAULT = PARSER_VERSION_WEIGHTED
+
+      class << self
+        attr_accessor :default_configuration
+      end
+
+      attr_reader :version, :max_weighted_tweet_length, :scale
+      attr_reader :default_weight, :transformed_url_length, :ranges
+      attr_reader :emoji_parsing_enabled
+
+      CONFIG_V1 = File.join(
+        File.expand_path('../../../config', __FILE__), # project root
+        "#{PARSER_VERSION_CLASSIC}.json"
+      )
+
+      CONFIG_V2 = File.join(
+        File.expand_path('../../../config', __FILE__), # project root
+        "#{PARSER_VERSION_WEIGHTED}.json"
+      )
+
+      CONFIG_V3 = File.join(
+        File.expand_path('../../../config', __FILE__), # project root
+        "#{PARSER_VERSION_EMOJI_PARSING}.json"
+      )
+
+      def self.parse_string(string, options = {})
+        JSON.parse(string, options.merge(symbolize_names: true))
+      end
+
+      def self.parse_file(filename)
+        string = File.open(filename, 'rb') { |f| f.read }
+        parse_string(string)
+      end
+
+      def self.configuration_from_file(filename)
+        config = parse_file(filename)
+        config ? self.new(config) : nil
+      end
+
+      def initialize(config = {})
+        @version = config[:version]
+        @max_weighted_tweet_length = config[:maxWeightedTweetLength]
+        @scale = config[:scale]
+        @default_weight = config[:defaultWeight]
+        @transformed_url_length = config[:transformedURLLength]
+        @emoji_parsing_enabled = config[:emojiParsingEnabled]
+        @ranges = config[:ranges].map { |range| Twitter::TwitterText::WeightedRange.new(range) } if config.key?(:ranges) && config[:ranges].is_a?(Array)
+      end
+
+      self.default_configuration = self.configuration_from_file(CONFIG_V3)
+    end
+  end
+end

data/lib/twitter-text/deprecation.rb

@@ -0,0 +1,21 @@
+# Copyright 2018 Twitter, Inc.
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module Twitter
+  module TwitterText
+    module Deprecation
+      def deprecate(method, new_method = nil)
+        deprecated_method = :"deprecated_#{method}"
+        message = "Deprecation: `#{method}` is deprecated."
+        message << " Please use `#{new_method}` instead." if new_method
+
+        alias_method(deprecated_method, method)
+        define_method method do |*args, &block|
+          warn message unless $TESTING
+          send(deprecated_method, *args, &block)
+        end
+      end
+    end
+  end
+end

data/lib/twitter-text/emoji_regex.rb

@@ -0,0 +1,27 @@
+#
+#  emoji_regex.rb
+#
+#  Copyright © 2018 Twitter. All rights reserved.
+#
+# DO NOT MODIFY THIS FILE -- it is generated for twitter-text automatically
+
+# encoding: utf-8
+
+module Twitter
+  module TwitterText
+    class Regex
+      class Emoji
+        REGEXEN = {} # :nodoc:
+
+        # This regex pattern matches a single emoji
+        REGEXEN[:valid_emoji] = %r{
+        [\u{01f468}\u{01f469}][\u{01f3fb}-\u{01f3ff}]?\u200d(?:\u2695\ufe0f|\u2696\ufe0f|\u2708\ufe0f|[\u{01f33e}\u{01f373}\u{01f393}\u{01f3a4}\u{01f3a8}\u{01f3eb}\u{01f3ed}\u{01f4bb}\u{01f4bc}\u{01f527}\u{01f52c}\u{01f680}\u{01f692}\u{01f9b0}-\u{01f9b3}])|[\u26f9\u{01f3cb}\u{01f3cc}\u{01f574}\u{01f575}](?:[\ufe0f\u{01f3fb}-\u{01f3ff}]\u200d[\u2640\u2642]\ufe0f)|[\u{01f3c3}\u{01f3c4}\u{01f3ca}\u{01f46e}\u{01f471}\u{01f473}\u{01f477}\u{01f481}\u{01f482}\u{01f486}\u{01f487}\u{01f645}-\u{01f647}\u{01f64b}\u{01f64d}\u{01f64e}\u{01f6a3}\u{01f6b4}-\u{01f6b6}\u{01f926}\u{01f935}\u{01f937}-\u{01f939}\u{01f93d}\u{01f93e}\u{01f9b8}\u{01f9b9}\u{01f9d6}-\u{01f9dd}][\u{01f3fb}-\u{01f3ff}]?\u200d[\u2640\u2642]\ufe0f|(?:\u{01f468}\u200d\u2764\ufe0f\u200d\u{01f48b}\u200d\u{01f468}|\u{01f469}\u200d\u2764\ufe0f\u200d\u{01f48b}\u200d[\u{01f468}\u{01f469}]|\u{01f468}\u200d\u{01f468}\u200d\u{01f466}\u200d\u{01f466}|\u{01f468}\u200d\u{01f468}\u200d\u{01f467}\u200d[\u{01f466}\u{01f467}]|\u{01f468}\u200d\u{01f469}\u200d\u{01f466}\u200d\u{01f466}|\u{01f468}\u200d\u{01f469}\u200d\u{01f467}\u200d[\u{01f466}\u{01f467}]|\u{01f469}\u200d\u{01f469}\u200d\u{01f466}\u200d\u{01f466}|\u{01f469}\u200d\u{01f469}\u200d\u{01f467}\u200d[\u{01f466}\u{01f467}]|\u{01f468}\u200d\u2764\ufe0f\u200d\u{01f468}|\u{01f469}\u200d\u2764\ufe0f\u200d[\u{01f468}\u{01f469}]|\u{01f468}\u200d\u{01f466}\u200d\u{01f466}|\u{01f468}\u200d\u{01f467}\u200d[\u{01f466}\u{01f467}]|\u{01f468}\u200d\u{01f468}\u200d[\u{01f466}\u{01f467}]|\u{01f468}\u200d\u{01f469}\u200d[\u{01f466}\u{01f467}]|\u{01f469}\u200d\u{01f466}\u200d\u{01f466}|\u{01f469}\u200d\u{01f467}\u200d[\u{01f466}\u{01f467}]|\u{01f469}\u200d\u{01f469}\u200d[\u{01f466}\u{01f467}]|\u{01f3f3}\ufe0f\u200d\u{01f308}|\u{01f3f4}\u200d\u2620\ufe0f|\u{01f46f}\u200d\u2640\ufe0f|\u{01f46f}\u200d\u2642\ufe0f|\u{01f93c}\u200d\u2640\ufe0f|\u{01f93c}\u200d\u2642\ufe0f|\u{01f9de}\u200d\u2640\ufe0f|\u{01f9de}\u200d\u2642\ufe0f|\u{01f9df}\u200d\u2640\ufe0f|\u{01f9df}\u200d\u2642\ufe0f|\u{01f441}\u200d\u{01f5e8}|\u{01f468}\u200d[\u{01f466}\u{01f467}]|\u{01f469}\u200d[\u{01f466}\u{01f467}])|[#*0-9]\ufe0f?\u20e3|(?:[©®\u2122\u265f]\ufe0f)|[\u203c\u2049\u2139\u2194-\u2199\u21a9\u21aa\u231a\u231b\u2328\u23cf\u23ed-\u23ef\u23f1\u23f2\u23f8-\u23fa\u24c2\u25aa\u25ab\u25b6\u25c0\u25fb-\u25fe\u2600-\u2604\u260e\u2611\u2614\u2615\u2618\u2620\u2622\u2623\u2626\u262a\u262e\u262f\u2638-\u263a\u2640\u2642\u2648-\u2653\u2660\u2663\u2665\u2666\u2668\u267b\u267f\u2692-\u2697\u2699\u269b\u269c\u26a0\u26a1\u26aa\u26ab\u26b0\u26b1\u26bd\u26be\u26c4\u26c5\u26c8\u26cf\u26d1\u26d3\u26d4\u26e9\u26ea\u26f0-\u26f5\u26f8\u26fa\u26fd\u2702\u2708\u2709\u270f\u2712\u2714\u2716\u271d\u2721\u2733\u2734\u2744\u2747\u2757\u2763\u2764\u27a1\u2934\u2935\u2b05-\u2b07\u2b1b\u2b1c\u2b50\u2b55\u3030\u303d\u3297\u3299\u{01f004}\u{01f170}\u{01f171}\u{01f17e}\u{01f17f}\u{01f202}\u{01f21a}\u{01f22f}\u{01f237}\u{01f321}\u{01f324}-\u{01f32c}\u{01f336}\u{01f37d}\u{01f396}\u{01f397}\u{01f399}-\u{01f39b}\u{01f39e}\u{01f39f}\u{01f3cd}\u{01f3ce}\u{01f3d4}-\u{01f3df}\u{01f3f3}\u{01f3f5}\u{01f3f7}\u{01f43f}\u{01f441}\u{01f4fd}\u{01f549}\u{01f54a}\u{01f56f}\u{01f570}\u{01f573}\u{01f576}-\u{01f579}\u{01f587}\u{01f58a}-\u{01f58d}\u{01f5a5}\u{01f5a8}\u{01f5b1}\u{01f5b2}\u{01f5bc}\u{01f5c2}-\u{01f5c4}\u{01f5d1}-\u{01f5d3}\u{01f5dc}-\u{01f5de}\u{01f5e1}\u{01f5e3}\u{01f5e8}\u{01f5ef}\u{01f5f3}\u{01f5fa}\u{01f6cb}\u{01f6cd}-\u{01f6cf}\u{01f6e0}-\u{01f6e5}\u{01f6e9}\u{01f6f0}\u{01f6f3}](?:\ufe0f|(?!\ufe0e))|(?:[\u261d\u26f7\u26f9\u270c\u270d\u{01f3cb}\u{01f3cc}\u{01f574}\u{01f575}\u{01f590}](?:\ufe0f|(?!\ufe0e))|[\u270a\u270b\u{01f385}\u{01f3c2}-\u{01f3c4}\u{01f3c7}\u{01f3ca}\u{01f442}\u{01f443}\u{01f446}-\u{01f450}\u{01f466}-\u{01f469}\u{01f46e}\u{01f470}-\u{01f478}\u{01f47c}\u{01f481}-\u{01f483}\u{01f485}-\u{01f487}\u{01f4aa}\u{01f57a}\u{01f595}\u{01f596}\u{01f645}-\u{01f647}\u{01f64b}-\u{01f64f}\u{01f6a3}\u{01f6b4}-\u{01f6b6}\u{01f6c0}\u{01f6cc}\u{01f918}-\u{01f91c}\u{01f91e}\u{01f91f}\u{01f926}\u{01f930}-\u{01f939}\u{01f93d}\u{01f93e}\u{01f9b5}\u{01f9b6}\u{01f9b8}\u{01f9b9}\u{01f9d1}-\u{01f9dd}])[\u{01f3fb}-\u{01f3ff}]?|(?:\u{01f3f4}\u{0e0067}\u{0e0062}\u{0e0065}\u{0e006e}\u{0e0067}\u{0e007f}|\u{01f3f4}\u{0e0067}\u{0e0062}\u{0e0073}\u{0e0063}\u{0e0074}\u{0e007f}|\u{01f3f4}\u{0e0067}\u{0e0062}\u{0e0077}\u{0e006c}\u{0e0073}\u{0e007f}|\u{01f1e6}[\u{01f1e8}-\u{01f1ec}\u{01f1ee}\u{01f1f1}\u{01f1f2}\u{01f1f4}\u{01f1f6}-\u{01f1fa}\u{01f1fc}\u{01f1fd}\u{01f1ff}]|\u{01f1e7}[\u{01f1e6}\u{01f1e7}\u{01f1e9}-\u{01f1ef}\u{01f1f1}-\u{01f1f4}\u{01f1f6}-\u{01f1f9}\u{01f1fb}\u{01f1fc}\u{01f1fe}\u{01f1ff}]|\u{01f1e8}[\u{01f1e6}\u{01f1e8}\u{01f1e9}\u{01f1eb}-\u{01f1ee}\u{01f1f0}-\u{01f1f5}\u{01f1f7}\u{01f1fa}-\u{01f1ff}]|\u{01f1e9}[\u{01f1ea}\u{01f1ec}\u{01f1ef}\u{01f1f0}\u{01f1f2}\u{01f1f4}\u{01f1ff}]|\u{01f1ea}[\u{01f1e6}\u{01f1e8}\u{01f1ea}\u{01f1ec}\u{01f1ed}\u{01f1f7}-\u{01f1fa}]|\u{01f1eb}[\u{01f1ee}-\u{01f1f0}\u{01f1f2}\u{01f1f4}\u{01f1f7}]|\u{01f1ec}[\u{01f1e6}\u{01f1e7}\u{01f1e9}-\u{01f1ee}\u{01f1f1}-\u{01f1f3}\u{01f1f5}-\u{01f1fa}\u{01f1fc}\u{01f1fe}]|\u{01f1ed}[\u{01f1f0}\u{01f1f2}\u{01f1f3}\u{01f1f7}\u{01f1f9}\u{01f1fa}]|\u{01f1ee}[\u{01f1e8}-\u{01f1ea}\u{01f1f1}-\u{01f1f4}\u{01f1f6}-\u{01f1f9}]|\u{01f1ef}[\u{01f1ea}\u{01f1f2}\u{01f1f4}\u{01f1f5}]|\u{01f1f0}[\u{01f1ea}\u{01f1ec}-\u{01f1ee}\u{01f1f2}\u{01f1f3}\u{01f1f5}\u{01f1f7}\u{01f1fc}\u{01f1fe}\u{01f1ff}]|\u{01f1f1}[\u{01f1e6}-\u{01f1e8}\u{01f1ee}\u{01f1f0}\u{01f1f7}-\u{01f1fb}\u{01f1fe}]|\u{01f1f2}[\u{01f1e6}\u{01f1e8}-\u{01f1ed}\u{01f1f0}-\u{01f1ff}]|\u{01f1f3}[\u{01f1e6}\u{01f1e8}\u{01f1ea}-\u{01f1ec}\u{01f1ee}\u{01f1f1}\u{01f1f4}\u{01f1f5}\u{01f1f7}\u{01f1fa}\u{01f1ff}]|\u{01f1f4}\u{01f1f2}|\u{01f1f5}[\u{01f1e6}\u{01f1ea}-\u{01f1ed}\u{01f1f0}-\u{01f1f3}\u{01f1f7}-\u{01f1f9}\u{01f1fc}\u{01f1fe}]|\u{01f1f6}\u{01f1e6}|\u{01f1f7}[\u{01f1ea}\u{01f1f4}\u{01f1f8}\u{01f1fa}\u{01f1fc}]|\u{01f1f8}[\u{01f1e6}-\u{01f1ea}\u{01f1ec}-\u{01f1f4}\u{01f1f7}-\u{01f1f9}\u{01f1fb}\u{01f1fd}-\u{01f1ff}]|\u{01f1f9}[\u{01f1e6}\u{01f1e8}\u{01f1e9}\u{01f1eb}-\u{01f1ed}\u{01f1ef}-\u{01f1f4}\u{01f1f7}\u{01f1f9}\u{01f1fb}\u{01f1fc}\u{01f1ff}]|\u{01f1fa}[\u{01f1e6}\u{01f1ec}\u{01f1f2}\u{01f1f3}\u{01f1f8}\u{01f1fe}\u{01f1ff}]|\u{01f1fb}[\u{01f1e6}\u{01f1e8}\u{01f1ea}\u{01f1ec}\u{01f1ee}\u{01f1f3}\u{01f1fa}]|\u{01f1fc}[\u{01f1eb}\u{01f1f8}]|\u{01f1fd}\u{01f1f0}|\u{01f1fe}[\u{01f1ea}\u{01f1f9}]|\u{01f1ff}[\u{01f1e6}\u{01f1f2}\u{01f1fc}]|[\u23e9-\u23ec\u23f0\u23f3\u267e\u26ce\u2705\u2728\u274c\u274e\u2753-\u2755\u2795-\u2797\u27b0\u27bf\ue50a\u{01f0cf}\u{01f18e}\u{01f191}-\u{01f19a}\u{01f1e6}-\u{01f1ff}\u{01f201}\u{01f232}-\u{01f236}\u{01f238}-\u{01f23a}\u{01f250}\u{01f251}\u{01f300}-\u{01f320}\u{01f32d}-\u{01f335}\u{01f337}-\u{01f37c}\u{01f37e}-\u{01f384}\u{01f386}-\u{01f393}\u{01f3a0}-\u{01f3c1}\u{01f3c5}\u{01f3c6}\u{01f3c8}\u{01f3c9}\u{01f3cf}-\u{01f3d3}\u{01f3e0}-\u{01f3f0}\u{01f3f4}\u{01f3f8}-\u{01f43e}\u{01f440}\u{01f444}\u{01f445}\u{01f451}-\u{01f465}\u{01f46a}-\u{01f46d}\u{01f46f}\u{01f479}-\u{01f47b}\u{01f47d}-\u{01f480}\u{01f484}\u{01f488}-\u{01f4a9}\u{01f4ab}-\u{01f4fc}\u{01f4ff}-\u{01f53d}\u{01f54b}-\u{01f54e}\u{01f550}-\u{01f567}\u{01f5a4}\u{01f5fb}-\u{01f644}\u{01f648}-\u{01f64a}\u{01f680}-\u{01f6a2}\u{01f6a4}-\u{01f6b3}\u{01f6b7}-\u{01f6bf}\u{01f6c1}-\u{01f6c5}\u{01f6d0}-\u{01f6d2}\u{01f6eb}\u{01f6ec}\u{01f6f4}-\u{01f6f9}\u{01f910}-\u{01f917}\u{01f91d}\u{01f920}-\u{01f925}\u{01f927}-\u{01f92f}\u{01f93a}\u{01f93c}\u{01f940}-\u{01f945}\u{01f947}-\u{01f970}\u{01f973}-\u{01f976}\u{01f97a}\u{01f97c}-\u{01f9a2}\u{01f9b4}\u{01f9b7}\u{01f9c0}-\u{01f9c2}\u{01f9d0}\u{01f9de}-\u{01f9ff}])|\ufe0f
+  }iox
+
+        def self.[](key)
+          REGEXEN[key]
+        end
+      end
+    end
+  end
+end

data/lib/twitter-text/extractor.rb

@@ -0,0 +1,388 @@
+# Copyright 2018 Twitter, Inc.
+# Licensed under the Apache License, Version 2.0
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# encoding: utf-8
+require 'simpleidn'
+
+class String
+  # Helper function to count the character length by first converting to an
+  # array.  This is needed because with unicode strings, the return value
+  # of length may be incorrect
+  def codepoint_length
+    if respond_to? :codepoints
+      length
+    else
+      chars.kind_of?(Enumerable) ? chars.to_a.size : chars.size
+    end
+  end
+
+  # Helper function to convert this string into an array of unicode code points.
+  def to_codepoint_a
+    @to_codepoint_a ||= if chars.kind_of?(Enumerable)
+      chars.to_a
+    else
+      codepoint_array = []
+      0.upto(codepoint_length - 1) { |i| codepoint_array << [chars.slice(i)].pack('U') }
+      codepoint_array
+    end
+  end
+end
+
+# Helper functions to return code point offsets instead of byte offsets.
+class MatchData
+  def char_begin(n)
+    if string.respond_to? :codepoints
+      self.begin(n)
+    else
+      string[0, self.begin(n)].codepoint_length
+    end
+  end
+
+  def char_end(n)
+    if string.respond_to? :codepoints
+      self.end(n)
+    else
+      string[0, self.end(n)].codepoint_length
+    end
+  end
+end
+
+module Twitter
+  module TwitterText
+    # A module for including Tweet parsing in a class. This module provides function for the extraction and processing
+    # of usernames, lists, URLs and hashtags.
+    module Extractor extend self
+
+      # Maximum URL length as defined by Twitter's backend.
+      MAX_URL_LENGTH = 4096
+
+      # The maximum t.co path length that the Twitter backend supports.
+      MAX_TCO_SLUG_LENGTH = 40
+
+      URL_PROTOCOL_LENGTH = "https://".length
+
+      # Remove overlapping entities.
+      # This returns a new array with no overlapping entities.
+      def remove_overlapping_entities(entities)
+        # sort by start index
+        entities = entities.sort_by{|entity| entity[:indices].first}
+
+        # remove duplicates
+        prev = nil
+        entities.reject!{|entity| (prev && prev[:indices].last > entity[:indices].first) || (prev = entity) && false}
+        entities
+      end
+
+      # Extracts all usernames, lists, hashtags and URLs  in the Tweet <tt>text</tt>
+      # along with the indices for where the entity ocurred
+      # If the <tt>text</tt> is <tt>nil</tt> or contains no entity an empty array
+      # will be returned.
+      #
+      # If a block is given then it will be called for each entity.
+      def extract_entities_with_indices(text, options = {}, &block)
+        config = options[:config] || Twitter::TwitterText::Configuration.default_configuration
+
+        # extract all entities
+        entities = extract_urls_with_indices(text, options) +
+                   extract_hashtags_with_indices(text, :check_url_overlap => false) +
+                   extract_mentions_or_lists_with_indices(text) +
+                   extract_cashtags_with_indices(text)
+        entities += extract_emoji_with_indices(text) if config.emoji_parsing_enabled
+
+        return [] if entities.empty?
+
+        entities = remove_overlapping_entities(entities)
+
+        entities.each(&block) if block_given?
+        entities
+      end
+
+      # Extracts a list of all usernames mentioned in the Tweet <tt>text</tt>. If the
+      # <tt>text</tt> is <tt>nil</tt> or contains no username mentions an empty array
+      # will be returned.
+      #
+      # If a block is given then it will be called for each username.
+      def extract_mentioned_screen_names(text, &block) # :yields: username
+        screen_names = extract_mentioned_screen_names_with_indices(text).map{|m| m[:screen_name]}
+        screen_names.each(&block) if block_given?
+        screen_names
+      end
+
+      # Extracts a list of all usernames mentioned in the Tweet <tt>text</tt>
+      # along with the indices for where the mention ocurred.  If the
+      # <tt>text</tt> is nil or contains no username mentions, an empty array
+      # will be returned.
+      #
+      # If a block is given, then it will be called with each username, the start
+      # index, and the end index in the <tt>text</tt>.
+      def extract_mentioned_screen_names_with_indices(text) # :yields: username, start, end
+        return [] unless text
+
+        possible_screen_names = []
+        extract_mentions_or_lists_with_indices(text) do |screen_name, list_slug, start_position, end_position|
+          next unless list_slug.empty?
+          possible_screen_names << {
+            :screen_name => screen_name,
+            :indices => [start_position, end_position]
+          }
+        end
+
+        if block_given?
+          possible_screen_names.each do |mention|
+            yield mention[:screen_name], mention[:indices].first, mention[:indices].last
+          end
+        end
+
+        possible_screen_names
+      end
+
+      # Extracts a list of all usernames or lists mentioned in the Tweet <tt>text</tt>
+      # along with the indices for where the mention ocurred.  If the
+      # <tt>text</tt> is nil or contains no username or list mentions, an empty array
+      # will be returned.
+      #
+      # If a block is given, then it will be called with each username, list slug, the start
+      # index, and the end index in the <tt>text</tt>. The list_slug will be an empty stirng
+      # if this is a username mention.
+      def extract_mentions_or_lists_with_indices(text) # :yields: username, list_slug, start, end
+        return [] unless text =~ /[@＠]/
+
+        possible_entries = []
+        text.to_s.scan(Twitter::TwitterText::Regex[:valid_mention_or_list]) do |before, at, screen_name, list_slug|
+          match_data = $~
+                        after = $'
+          unless after =~ Twitter::TwitterText::Regex[:end_mention_match]
+            start_position = match_data.char_begin(3) - 1
+            end_position = match_data.char_end(list_slug.nil? ? 3 : 4)
+            possible_entries << {
+              :screen_name => screen_name,
+              :list_slug => list_slug || "",
+              :indices => [start_position, end_position]
+            }
+          end
+        end
+
+        if block_given?
+          possible_entries.each do |mention|
+            yield mention[:screen_name], mention[:list_slug], mention[:indices].first, mention[:indices].last
+          end
+        end
+
+        possible_entries
+      end
+
+      # Extracts the username username replied to in the Tweet <tt>text</tt>. If the
+      # <tt>text</tt> is <tt>nil</tt> or is not a reply nil will be returned.
+      #
+      # If a block is given then it will be called with the username replied to (if any)
+      def extract_reply_screen_name(text) # :yields: username
+        return nil unless text
+
+        possible_screen_name = text.match(Twitter::TwitterText::Regex[:valid_reply])
+        return unless possible_screen_name.respond_to?(:captures)
+        return if $' =~ Twitter::TwitterText::Regex[:end_mention_match]
+        screen_name = possible_screen_name.captures.first
+        yield screen_name if block_given?
+        screen_name
+      end
+
+      # Extracts a list of all URLs included in the Tweet <tt>text</tt>. If the
+      # <tt>text</tt> is <tt>nil</tt> or contains no URLs an empty array
+      # will be returned.
+      #
+      # If a block is given then it will be called for each URL.
+      def extract_urls(text, &block) # :yields: url
+        urls = extract_urls_with_indices(text).map{|u| u[:url]}
+        urls.each(&block) if block_given?
+        urls
+      end
+
+      # Extracts a list of all URLs included in the Tweet <tt>text</tt> along
+      # with the indices. If the <tt>text</tt> is <tt>nil</tt> or contains no
+      # URLs an empty array will be returned.
+      #
+      # If a block is given then it will be called for each URL.
+      def extract_urls_with_indices(text, options = {:extract_url_without_protocol => true}) # :yields: url, start, end
+        return [] unless text && (options[:extract_url_without_protocol] ? text.index(".") : text.index(":"))
+        urls = []
+
+        text.to_s.scan(Twitter::TwitterText::Regex[:valid_url]) do |all, before, url, protocol, domain, port, path, query|
+          valid_url_match_data = $~
+
+                                  start_position = valid_url_match_data.char_begin(3)
+          end_position = valid_url_match_data.char_end(3)
+
+          # If protocol is missing and domain contains non-ASCII characters,
+          # extract ASCII-only domains.
+          if !protocol
+            next if !options[:extract_url_without_protocol] || before =~ Twitter::TwitterText::Regex[:invalid_url_without_protocol_preceding_chars]
+            last_url = nil
+            domain.scan(Twitter::TwitterText::Regex[:valid_ascii_domain]) do |ascii_domain|
+              next unless is_valid_domain(url.length, ascii_domain, protocol)
+              last_url = {
+                :url => ascii_domain,
+                :indices => [start_position + $~.char_begin(0),
+                             start_position + $~.char_end(0)]
+              }
+              urls << last_url
+            end
+
+            # no ASCII-only domain found. Skip the entire URL
+            next unless last_url
+
+            # last_url only contains domain. Need to add path and query if they exist.
+            if path
+              # last_url was not added. Add it to urls here.
+              last_url[:url] = url.sub(domain, last_url[:url])
+              last_url[:indices][1] = end_position
+            end
+          else
+            # In the case of t.co URLs, don't allow additional path characters
+            if url =~ Twitter::TwitterText::Regex[:valid_tco_url]
+              next if $1 && $1.length > MAX_TCO_SLUG_LENGTH
+              url = $&
+                    end_position = start_position + url.codepoint_length
+            end
+
+            next unless is_valid_domain(url.length, domain, protocol)
+
+            urls << {
+              :url => url,
+              :indices => [start_position, end_position]
+            }
+          end
+        end
+        urls.each{|url| yield url[:url], url[:indices].first, url[:indices].last} if block_given?
+        urls
+      end
+
+      # Extracts a list of all hashtags included in the Tweet <tt>text</tt>. If the
+      # <tt>text</tt> is <tt>nil</tt> or contains no hashtags an empty array
+      # will be returned. The array returned will not include the leading <tt>#</tt>
+      # character.
+      #
+      # If a block is given then it will be called for each hashtag.
+      def extract_hashtags(text, &block) # :yields: hashtag_text
+        hashtags = extract_hashtags_with_indices(text).map{|h| h[:hashtag]}
+        hashtags.each(&block) if block_given?
+        hashtags
+      end
+
+      # Extracts a list of all hashtags included in the Tweet <tt>text</tt>. If the
+      # <tt>text</tt> is <tt>nil</tt> or contains no hashtags an empty array
+      # will be returned. The array returned will not include the leading <tt>#</tt>
+      # character.
+      #
+      # If a block is given then it will be called for each hashtag.
+      def extract_hashtags_with_indices(text, options = {:check_url_overlap => true}) # :yields: hashtag_text, start, end
+        return [] unless text =~ /[#＃]/
+
+        tags = []
+        text.scan(Twitter::TwitterText::Regex[:valid_hashtag]) do |before, hash, hash_text|
+          match_data = $~
+                        start_position = match_data.char_begin(2)
+          end_position = match_data.char_end(3)
+          after = $'
+          unless after =~ Twitter::TwitterText::Regex[:end_hashtag_match]
+            tags << {
+              :hashtag => hash_text,
+              :indices => [start_position, end_position]
+            }
+          end
+        end
+
+        if options[:check_url_overlap]
+          # extract URLs
+          urls = extract_urls_with_indices(text)
+          unless urls.empty?
+            tags.concat(urls)
+            # remove duplicates
+            tags = remove_overlapping_entities(tags)
+            # remove URL entities
+            tags.reject!{|entity| !entity[:hashtag] }
+          end
+        end
+
+        tags.each{|tag| yield tag[:hashtag], tag[:indices].first, tag[:indices].last} if block_given?
+        tags
+      end
+
+      # Extracts a list of all cashtags included in the Tweet <tt>text</tt>. If the
+      # <tt>text</tt> is <tt>nil</tt> or contains no cashtags an empty array
+      # will be returned. The array returned will not include the leading <tt>$</tt>
+      # character.
+      #
+      # If a block is given then it will be called for each cashtag.
+      def extract_cashtags(text, &block) # :yields: cashtag_text
+        cashtags = extract_cashtags_with_indices(text).map{|h| h[:cashtag]}
+        cashtags.each(&block) if block_given?
+        cashtags
+      end
+
+      # Extracts a list of all cashtags included in the Tweet <tt>text</tt>. If the
+      # <tt>text</tt> is <tt>nil</tt> or contains no cashtags an empty array
+      # will be returned. The array returned will not include the leading <tt>$</tt>
+      # character.
+      #
+      # If a block is given then it will be called for each cashtag.
+      def extract_cashtags_with_indices(text) # :yields: cashtag_text, start, end
+        return [] unless text =~ /\$/
+
+        tags = []
+        text.scan(Twitter::TwitterText::Regex[:valid_cashtag]) do |before, dollar, cash_text|
+          match_data = $~
+                        start_position = match_data.char_begin(2)
+          end_position = match_data.char_end(3)
+          tags << {
+            :cashtag => cash_text,
+            :indices => [start_position, end_position]
+          }
+        end
+
+        tags.each{|tag| yield tag[:cashtag], tag[:indices].first, tag[:indices].last} if block_given?
+        tags
+      end
+
+      def extract_emoji_with_indices(text) # :yields: emoji, start, end
+        emoji = []
+        text.scan(Twitter::TwitterText::Regex[:valid_emoji]) do |emoji_text|
+          match_data = $~
+                        start_position = match_data.char_begin(0)
+          end_position = match_data.char_end(0)
+          emoji << {
+            :emoji => emoji_text,
+            :indices => [start_position, end_position]
+          }
+        end
+        emoji
+      end
+
+      def is_valid_emoji(text)
+        begin
+          raise ArgumentError.new("invalid empty emoji") unless text
+          entities = extract_emoji_with_indices(text)
+          entities.count == 1 && entities[0][:emoji] == text
+        rescue Exception
+          # On error don't consider this a valid domain.
+          return false
+        end
+      end
+
+      def is_valid_domain(url_length, domain, protocol)
+        begin
+          raise ArgumentError.new("invalid empty domain") unless domain
+          original_domain_length = domain.length
+          encoded_domain = SimpleIDN.to_ascii(domain)
+          updated_domain_length = encoded_domain.length
+          url_length += (updated_domain_length - original_domain_length) if (updated_domain_length > original_domain_length)
+          url_length += URL_PROTOCOL_LENGTH unless protocol
+          url_length <= MAX_URL_LENGTH
+        rescue Exception
+          # On error don't consider this a valid domain.
+          return false
+        end
+      end
+    end
+  end
+end