uts58 0.1.0

data/lib/uts58/extractor.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+require 'simpleidn'
+require 'public_suffix'
+require_relative 'constants'
+
+module Uts58
+  # Finds web links in arbitrary text per UTS #58. The public API
+  # mirrors Twitter::TwitterText::Extractor closely enough that
+  # twitter-text consumers (notably Mastodon) can swap one for the
+  # other.
+  #
+  # Instances carry only optional configuration (see #max_length=); if
+  # you don't need to set anything, the module-level
+  # Uts58.extract_urls and Uts58.extract_urls_with_indices shortcuts
+  # are simpler.
+  class Extractor
+    PATH_CLOSERS = [35, 47, 63]
+    QUERY_CLOSERS = [35] # how about &?
+    FRAGMENT_CLOSERS = []
+
+    # Maximum allowed length of the matched text, in input codepoints.
+    # Matches whose input span exceeds this are dropped from the result
+    # of #extract_urls_with_indices.
+    #
+    # "Matched text" means the substring that came out of +text+ — for
+    # example 11 for <tt>"example.com"</tt>. The returned +:url+ can
+    # be both longer and shorter, most commonly when a missing scheme
+    # is filled in ( +"https://example.com"+ is 19 codepoints). The
+    # limit is measured against the input, not against the returned
+    # URL.
+    #
+    # Default is +nil+, meaning no limit.
+    attr_accessor :max_length
+
+    def initialize
+      @max_length = nil
+    end
+
+    # Returns every URL found in +text+ as a list of hashes:
+    #
+    #   { url: String, indices: [start, end] }
+    #
+    # +url+ is the cleaned-up form: any A-labels in the hostname are
+    # decoded to U-labels, and the scheme is filled in as +https://+
+    # if the input had none. +indices+ are codepoint offsets into
+    # +text+, with +end+ exclusive, so <tt>text[start...end]</tt>
+    # gives the substring that matched.
+    #
+    # Note that the start and end may not match the length of url. One
+    # very common example is input that like "foo example.com bar",
+    # where the URL will be https://example.com, including "https://".
+    #
+    # Returns an empty array if +text+ contains no links. +options+ is
+    # accepted for twitter-text compatibility and currently ignored.
+    def extract_urls_with_indices(text, options = {})
+      result = []
+      text.to_enum(:scan,/(?<![-\p{Alnum}\p{M}.\/])(?=\p{Alnum}[-\p{L}\p{N}\p{M}\u00DF\u03C2\u06FD\u06FE\u0F0B\u3007]*[\.:。])/).map{Regexp.last_match}.each do |match|
+        # get rid of a leading protocol. We also tolerate letter/mark/number
+        # characters between the trigger and the scheme, so that input
+        # like "テストhttp://example.com" attaches the scheme correctly:
+        # the trigger fires at offset 0 (the start of "テスト") because
+        # nothing precedes it, and the actual link begins three
+        # codepoints later.
+        s = match.post_match
+        scheme_match = /^([\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}\p{Thai}\p{Lao}\p{Khmer}\p{Myanmar}]*?)(https?:\/\/)/i.match(s)
+        if scheme_match
+          scheme_offset = scheme_match[1].length
+          proto = scheme_match[2]
+          s = scheme_match.post_match
+        else
+          scheme_offset = 0
+          proto = "https://"
+        end
+        # look for the prefix that might be a hostname or an IDN.
+        # this is a somewhat sloppy match, with a few false positives.
+        prefix = /^([-\p{L}\p{N}\p{M}\u00DF\u03C2\u06FD\u06FE\u0F0B\u3007]+[\.。]){1,4}[-\p{L}\p{N}\p{M}]+(?![-\p{L}\p{N}\p{M}])/.match(s)
+        if prefix && prefix[0].length < 254
+          hn = SimpleIDN.to_unicode(prefix.match(0).gsub(/。/, "."))
+          begin
+            about = PublicSuffix.parse(hn,
+                                       ignore_private: true,
+                                       default_rule: nil)
+            if about && about.tld != "invalid" then
+              # at this point, we do have enough to mark something,
+              # the question is how much. there may be a trailing
+              # port, then a path, then a query, finally a fragment.
+              rest = prefix.post_match
+              # a port number must be 1..65535
+              port = /^:(\d+)/.match(rest)
+              if port
+                n = port[1].to_i
+                next if n < 1 || n > 65535
+                rest = port.post_match
+              end
+              # path
+              rest = skip_component(rest, PATH_CLOSERS) while rest[0] == "/"
+              # query
+              rest = skip_component(rest, QUERY_CLOSERS) if rest[0] == '?'
+              rest = skip_component(rest, FRAGMENT_CLOSERS) if rest[0] == "#"
+              rest_length = prefix.post_match.length - rest.length
+              match_length = match.post_match.length - rest.length - scheme_offset
+              next if @max_length && match_length > @max_length
+              start = match.begin(0) + scheme_offset
+              result << {
+                url: "#{proto}#{hn}#{prefix.post_match[...rest_length]}",
+                indices: [start, start + match_length]
+              }
+            end
+          rescue PublicSuffix::DomainInvalid
+            # evidently we're not looking at the start of a link
+          rescue PublicSuffix::DomainNotAllowed
+            # ditto
+          end
+        end
+      end
+      # ah! the good feeling of going home after a hard day's work
+      result
+    end
+
+    # Returns just the URLs found in +text+, as an array of strings,
+    # in the order they occur. Use #extract_urls_with_indices instead
+    # if you also need the offsets, e.g. for adding HTML markup or for
+    # pairing the found links with the form used in the text.
+    #
+    # For text such as "a example.com b", this returns ["https://example.com"].
+    def extract_urls(text, options = {})
+      extract_urls_with_indices(text, options).map { |r| r[:url] }
+    end
+
+    # Given a list of entities (hashes with an +:indices+ key of the
+    # shape <tt>[start, end]</tt>, as produced by
+    # #extract_urls_with_indices) drops every entity that overlaps an
+    # earlier one and returns the survivors.
+    #
+    # Useful when merging the output of several extractors (URLs,
+    # mentions, hashtags, …), or when #extract_urls_with_indices itself
+    # finds several partly overlapping candidate URLs and you want only
+    # the longest. The algorithm prefers entries that start earlier;
+    # ties are broken by input order.
+    #
+    # The input array is not modified.
+    def remove_overlapping_entities(entities)
+      sorted = entities.sort_by { |e| e[:indices].first }
+      prev = nil
+      sorted.reject do |e|
+        if prev && prev[:indices].last > e[:indices].first
+          true
+        else
+          prev = e
+          false
+        end
+      end
+    end
+
+    private
+
+    def followed_by_hard(codepoints, i)
+      j = i;
+      while(j < codepoints.length &&
+            Constants::TERMINATION.include?(codepoints[j]) &&
+            Constants::TERMINATION[codepoints[j]] == :soft)
+        j = j + 1
+      end
+      j >= codepoints.length ||
+        (Constants::TERMINATION.include?(codepoints[j]) &&
+         Constants::TERMINATION[codepoints[j]] == :hard)
+    end
+
+    def skip_component(string, extra_closers)
+      openers = []
+      codepoints = string.codepoints
+      codepoints.each.with_index do |cp, i|
+        if i == 0
+          # it's the lead-in character
+        elsif extra_closers.include? cp
+          return string[i..]
+        elsif Constants::TERMINATION.include?(cp)
+          case Constants::TERMINATION[cp]
+          when :hard
+            return string[i..]
+          when :soft
+            return string[i..] if followed_by_hard(codepoints, i)
+          when :close
+            if Constants::OPENERS[cp] == openers.last
+              openers.pop
+            else
+              return string[i..]
+            end
+          when :open
+            openers << cp
+          end
+        else
+          # it's a letter or something like that
+        end
+      end
+      # Input ran out before any terminator did: the whole component
+      # belongs to the URL, so there is nothing left over.
+      ""
+    end
+  end
+end

data/lib/uts58.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Ruby implementation of {UTS #58}[https://www.unicode.org/reports/tr58/],
+# the Unicode spec for finding links in running text.
+#
+# The two entry points below are module-level shortcuts around a single
+# memoised Uts58::Extractor instance. They also strip partly
+# overlapping links — if the extractor finds two candidates that share
+# any characters in the input, the wrappers keep the earlier one and
+# drop the rest. Use Uts58::Extractor directly if you want the
+# raw, possibly-overlapping list (e.g. to merge with hashtag/mention
+# extractors before resolving overlap yourself).
+#
+#   Uts58.extract_urls("see example.com here")
+#   # => ["https://example.com"]
+#
+#   Uts58.extract_urls_with_indices("see example.com here")
+#   # => [{ url: "https://example.com", indices: [4, 15] }]
+module Uts58
+  VERSION = "0.1.0"
+
+  class << self
+    # Like Uts58::Extractor#extract_urls_with_indices, but with
+    # overlapping results merged via
+    # Uts58::Extractor#remove_overlapping_entities.
+    def extract_urls_with_indices(text, options = {})
+      extractor.remove_overlapping_entities(
+        extractor.extract_urls_with_indices(text, options)
+      )
+    end
+
+    # Like Uts58::Extractor#extract_urls, but with the URLs of
+    # overlapping results merged.
+    def extract_urls(text, options = {})
+      extract_urls_with_indices(text, options).map { |r| r[:url] }
+    end
+
+    private
+
+    def extractor
+      @extractor ||= Extractor.new
+    end
+  end
+end
+
+require_relative "uts58/extractor"

data/uts58.gemspec

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |spec|
+  spec.name          = "uts58"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Arnt Gulbrandsen"]
+  spec.email         = ["arnt@gulbrandsen.priv.no"]
+
+  spec.summary       = %q{Ruby implementation of Unicode UTS58}
+  spec.description   = %q{Ruby code to detect links in text, as specified by UTS58}
+  spec.homepage      = "https://github.com/arnt/uts58"
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.1")
+  spec.licenses       = ["BSD-2-Clause"]
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z 2>/dev/null`.split("\x0").reject { |f| f.match(%r{^(bin|test|spec|features|rfcs|tools)/}) }
+  end
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "public_suffix", "~> 6.0"
+  spec.add_dependency "simpleidn", "~> 0.2"
+  spec.add_development_dependency "byebug", "~> 12.0"
+  spec.add_development_dependency "diff-lcs", '~> 1.5.1'
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: uts58
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Arnt Gulbrandsen
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: public_suffix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: simpleidn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: diff-lcs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Ruby code to detect links in text, as specified by UTS58
+email:
+- arnt@gulbrandsen.priv.no
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/ci.yml"
+- Gemfile
+- README.md
+- lib/uts58.rb
+- lib/uts58/constants.rb
+- lib/uts58/extractor.rb
+- uts58.gemspec
+homepage: https://github.com/arnt/uts58
+licenses:
+- BSD-2-Clause
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Ruby implementation of Unicode UTS58
+test_files: []