embiggen 1.8.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e7aeb6fe45886a745d9abffa7d8057d061a033515ebe56cf95968c1b4ba2cf3
-  data.tar.gz: a30690bf0741041f19cfaa7e1f8f1314814b8ce0b0f226bed829de2ad3bf11fc
+  metadata.gz: 2bf4ba25d8acb0f2f5e168445f2e857fd255503a6d666d418c530a84381282a9
+  data.tar.gz: d6d0816294987d333ae32179c6101cbd29e9591e16eb7cafb9e1bf211b92f544
 SHA512:
-  metadata.gz: c985dd04314d04d2b32d06c23585007b08ed95f3f268b0e95e3b83074d1de1ecfa5bf0049408bcc94afb69d5656afe0befdd377d579056a4de401fab01325111
-  data.tar.gz: e2db6a440f495ceba99590dee9ab4f062d322f06408563dc4bb248e32d444bec8d507af0064c7ecd450b5f778d363807f449f8094782b512670a3b8b115913ee
+  metadata.gz: 244b5a974ab6fbb38cbcce0c5e88c4f651e75b6f05a21c3e3fc67159875659dbc0d089ed7e856a85a1fe2db6b5881a5acd72af35320e91137bb4929ce7b45d1e
+  data.tar.gz: b4c0a86839b657d7f37b0d0cac6b8db4db0c2585696b5dc51643cd6e5a4b40a549ed40f9f1ca3d2616edb8dcae45d36822a1a5536909f99b085657a0fc84c984

data/README.md

@@ -2,19 +2,19 @@
 
 A Ruby library to expand shortened URLs.
 
-**Current version:** 1.8.0  
+**Current version:** 1.10.0  
 **Supported Ruby versions:** >= 2.7
 
 ## Installation
 
 ```
-gem install embiggen -v '~> 1.7'
+gem install embiggen -v '~> 1.8'
 ```
 
 Or, in your `Gemfile`:
 
 ```ruby
-gem 'embiggen', '~> 1.7'
+gem 'embiggen', '~> 1.8'
 ```
 
 ## Usage
@@ -61,6 +61,40 @@ Embiggen.configure do |config|
 end
 ```
 
+## Non-redirect shorteners
+
+Some URL shorteners (such as [LinkedIn's lnkd.in](https://lnkd.in)) do not
+issue HTTP redirects when used with an external domain.
+Instead, they serve an HTML page containing the
+destination URL in a known element. Embiggen supports these by fetching the
+page and extracting the link via a CSS selector.
+
+Embiggen ships with a default list of known non-redirect shorteners in
+[`non_redirect_shorteners.yml`](https://github.com/altmetric/embiggen/blob/master/non_redirect_shorteners.yml),
+mapping each domain to the CSS selector used to locate the destination link.
+
+```ruby
+# Expanding a LinkedIn shortened URL
+Embiggen::URI('https://lnkd.in/eB25Z2yS').expand
+#=> #<URI::HTTPS https://example.com/article>
+```
+
+You can add your own non-redirect shorteners or remove existing ones via
+`Embiggen.configure`:
+
+```ruby
+Embiggen.configure do |config|
+  # Add a new non-redirect shortener
+  config.non_redirect_shorteners['myshorten.er'] = 'article a.destination'
+
+  # Remove a specific shortener
+  config.non_redirect_shorteners.delete('lnkd.in')
+
+  # Opt out of the feature entirely
+  config.non_redirect_shorteners.clear
+end
+```
+
 ## Shorteners
 
 Embiggen ships with a default list of URL shortening service domains (c.f.
@@ -191,7 +225,9 @@ Override the following settings:
 * `redirects`: the default number of redirects to follow (can be overridden by
   passing options to `Embiggen::URI#expand`);
 * `shorteners`: the list of domains of shortening services, c.f.
-  [Shorteners](#shorteners).
+  [Shorteners](#shorteners);
+* `non_redirect_shorteners`: the mapping of non-redirect shortener domains to
+  CSS selectors, c.f. [Non-redirect shorteners](#non-redirect-shorteners).
 
 ## Acknowledgements
 

data/lib/embiggen/configuration.rb

@@ -1,10 +1,12 @@
 # encoding: utf-8
 require 'embiggen/shortener_list'
+require 'embiggen/non_redirect_shortener_list'
+require 'yaml'
 
 module Embiggen
   class Configuration
     class << self
-      attr_writer :timeout, :redirects, :shorteners
+      attr_writer :timeout, :redirects, :shorteners, :non_redirect_shorteners
     end
 
     def self.timeout
@@ -24,5 +26,14 @@ module Embiggen
       file_path = File.expand_path('../../shorteners.txt', __dir__)
       File.readlines(file_path).map(&:chomp)
     end
+
+    def self.non_redirect_shorteners
+      @non_redirect_shorteners ||= NonRedirectShortenerList.new(non_redirect_shorteners_from_file)
+    end
+
+    def self.non_redirect_shorteners_from_file
+      file_path = File.expand_path('../../non_redirect_shorteners.yml', __dir__)
+      YAML.safe_load(File.read(file_path))
+    end
   end
 end

data/lib/embiggen/html_client.rb

@@ -0,0 +1,42 @@
+require 'embiggen/error'
+require 'net/http'
+require 'nokogiri'
+
+module Embiggen
+  class HtmlClient
+    attr_reader :uri
+
+    def initialize(uri)
+      @uri = uri
+      @http = ::Net::HTTP.new(uri.host, uri.port)
+      @http.use_ssl = true if uri.scheme == 'https'
+    end
+
+    def follow(timeout, selector)
+      response = request(timeout)
+      return unless response.is_a?(::Net::HTTPOK)
+
+      document = Nokogiri::HTML(response.body)
+      element = document.at_css(selector)
+      element&.[]('href')
+    rescue ::Timeout::Error => e
+      raise NetworkError.new(
+        "Timeout::Error: could not follow #{uri}: #{e.message}", uri
+      )
+    rescue StandardError => e
+      raise NetworkError.new(
+        "StandardError: could not follow #{uri}: #{e.message}", uri
+      )
+    end
+
+    private
+
+    def request(timeout)
+      request = ::Net::HTTP::Get.new(uri.request_uri)
+      @http.open_timeout = timeout
+      @http.read_timeout = timeout
+
+      @http.request(request)
+    end
+  end
+end

data/lib/embiggen/non_redirect_shortener_list.rb

@@ -0,0 +1,41 @@
+module Embiggen
+  class NonRedirectShortenerList
+    attr_reader :domains
+
+    def initialize(domains)
+      @domains = domains.to_h.transform_keys { |domain| host_pattern(domain) }
+    end
+
+    def supported?(uri)
+      !selector_for(uri).nil?
+    end
+
+    def selector_for(uri)
+      _, selector = domains.find { |pattern, _| uri.host =~ pattern }
+      selector
+    end
+
+    def []=(domain, selector)
+      domains[host_pattern(domain)] = selector
+    end
+
+    def delete(domain)
+      domains.delete(host_pattern(domain))
+    end
+
+    def clear
+      domains.clear
+      self
+    end
+
+    def size
+      domains.size
+    end
+
+    private
+
+    def host_pattern(domain)
+      /\b#{Regexp.escape(domain)}\z/i
+    end
+  end
+end

data/lib/embiggen/shortener_list.rb

@@ -1,4 +1,5 @@
 require 'forwardable'
+require 'set'
 
 module Embiggen
   class ShortenerList
@@ -16,7 +17,15 @@ module Embiggen
     end
 
     def +(other)
-      self.class.new(domains + other)
+      other_patterns = if other.respond_to?(:domains)
+                         other.domains
+                       else
+                         Set.new(other.map { |d| host_pattern(d) })
+                       end
+
+      self.class.allocate.tap do |result|
+        result.instance_variable_set(:@domains, domains | other_patterns)
+      end
     end
 
     def <<(domain)
@@ -32,7 +41,7 @@ module Embiggen
     def_delegators :domains, :size, :empty?, :each
 
     def host_pattern(domain)
-      /\b#{domain}\z/i
+      /\b#{Regexp.escape(domain)}\z/i
     end
   end
 end

data/lib/embiggen/uri.rb

@@ -1,5 +1,6 @@
 require 'embiggen/configuration'
 require 'embiggen/error'
+require 'embiggen/html_client'
 require 'embiggen/http_client'
 require 'addressable/uri'
 require 'uri'
@@ -44,6 +45,14 @@ module Embiggen
       timeout = request_options.fetch(:timeout) { Configuration.timeout }
 
       location = http_client.follow(timeout)
+
+      if location.nil?
+        non_redirect_shorteners = Configuration.non_redirect_shorteners
+        if non_redirect_shorteners.supported?(uri)
+          location = HtmlClient.new(uri).follow(timeout, non_redirect_shorteners.selector_for(uri))
+        end
+      end
+
       unless followable?(location)
         fail BadShortenedURI.new(
           "following #{uri} did not redirect", uri

data/non_redirect_shorteners.yml

@@ -0,0 +1,2 @@
+lnkd.in: "main a"
+ebx.sh: "body a#urlToFollow"

data/shorteners.txt

@@ -4375,6 +4375,7 @@ kicking.horse
 kickson.fr
 kickz.cc
 kidd.tv
+kidney.pub
 kidneymi.org
 kidsf.it
 kidshelpline.info

data/spec/embiggen/html_client_spec.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require 'embiggen/html_client'
+
+module Embiggen
+  RSpec.describe HtmlClient do
+    describe '#follow' do
+      let(:uri) { URI('https://lnkd.in/eB25Z2yS') }
+      let(:client) { described_class.new(uri) }
+
+      it 'returns the href from the matched element' do
+        stub_request(:get, 'https://lnkd.in/eB25Z2yS')
+          .to_return(
+            status: 200,
+            body: '<html><body><main><a href="https://example.com/article">https://example.com/article</a></main></body></html>',
+            headers: { 'Content-Type' => 'text/html' }
+          )
+
+        expect(client.follow(1, 'main a')).to eq('https://example.com/article')
+      end
+
+      it 'returns nil when the response is not 200 OK' do
+        stub_request(:get, 'https://lnkd.in/eB25Z2yS').to_return(status: 302, headers: { 'Location' => 'https://example.com' })
+
+        expect(client.follow(1, 'main a')).to be_nil
+      end
+
+      it 'returns nil when no element matches the selector' do
+        stub_request(:get, 'https://lnkd.in/eB25Z2yS')
+          .to_return(status: 200, body: '<html><body><p>No link here</p></body></html>')
+
+        expect(client.follow(1, 'main a')).to be_nil
+      end
+
+      it 'raises a network error if the URI times out' do
+        stub_request(:get, 'https://lnkd.in/eB25Z2yS').to_timeout
+
+        expect { client.follow(1, 'main a') }.to raise_error(NetworkError)
+      end
+
+      it 'raises a network error if the connection resets' do
+        stub_request(:get, 'https://lnkd.in/eB25Z2yS').to_raise(::Errno::ECONNRESET)
+
+        expect { client.follow(1, 'main a') }.to raise_error(NetworkError)
+      end
+    end
+  end
+end

data/spec/embiggen/non_redirect_shortener_list_spec.rb

@@ -0,0 +1,46 @@
+require 'embiggen/non_redirect_shortener_list'
+
+RSpec.describe Embiggen::NonRedirectShortenerList do
+  describe '#supported?' do
+    it 'returns true for a URI whose host is in the list' do
+      list = described_class.new('lnkd.in' => 'main a')
+
+      expect(list.supported?(URI('https://lnkd.in/eB25Z2yS'))).to be(true)
+    end
+
+    it 'returns false for a URI whose host is not in the list' do
+      list = described_class.new('lnkd.in' => 'main a')
+
+      expect(list.supported?(URI('https://example.com/foo'))).to be(false)
+    end
+  end
+
+  describe '#selector_for' do
+    it 'returns the selector for a matching URI' do
+      list = described_class.new('lnkd.in' => 'main a')
+
+      expect(list.selector_for(URI('https://lnkd.in/eB25Z2yS'))).to eq('main a')
+    end
+
+    it 'returns nil for a non-matching URI' do
+      list = described_class.new('lnkd.in' => 'main a')
+
+      expect(list.selector_for(URI('https://example.com/foo'))).to be_nil
+    end
+
+    it 'returns nil if the URI only matches due to an unescaped dot' do
+      list = described_class.new('lnkd.in' => 'main a')
+
+      expect(list.selector_for(URI('https://lnkdXin/foo'))).to be_nil
+    end
+  end
+
+  describe '#clear' do
+    it 'removes all entries' do
+      list = described_class.new('lnkd.in' => 'main a')
+      list.clear
+
+      expect(list.size).to eq(0)
+    end
+  end
+end

data/spec/embiggen/shortener_list_spec.rb

@@ -33,6 +33,12 @@ RSpec.describe Embiggen::ShortenerList do
 
       expect(list).to include(URI('http://www.bit.ly/foo'))
     end
+
+    it 'returns false if a URL host only matches due to an unescaped dot' do
+      list = described_class.new(%w[i.ea.com])
+
+      expect(list).not_to include(URI('http://www.ikea.com/foo'))
+    end
   end
 
   describe '#<<' do
@@ -82,6 +88,20 @@ RSpec.describe Embiggen::ShortenerList do
 
       expect(list).to include(URI('http://a.com/foo'))
     end
+
+    it 'retains the original domains when combining two lists' do
+      list = described_class.new(%w[bit.ly])
+      list += described_class.new(%w[a.com])
+
+      expect(list).to include(URI('http://bit.ly/foo'))
+    end
+
+    it 'returns a list with the combined size' do
+      list = described_class.new(%w[bit.ly])
+      list += described_class.new(%w[a.com])
+
+      expect(list.size).to eq(2)
+    end
   end
 
   it 'is enumerable for 1.8 compatiblity' do

data/spec/embiggen/uri_spec.rb

@@ -209,6 +209,26 @@ module Embiggen
         expect(uri.expand).to eq(URI('http://www.altmetric.com'))
       end
 
+      it 'expands non-redirect shorteners by parsing HTML' do
+        stub_request(:get, 'https://lnkd.in/eB25Z2yS')
+          .to_return(
+            status: 200,
+            body: '<html><body><main><a href="https://example.com/article">https://example.com/article</a></main></body></html>',
+            headers: { 'Content-Type' => 'text/html' }
+          )
+        uri = described_class.new(URI('https://lnkd.in/eB25Z2yS'))
+
+        expect(uri.expand).to eq(URI('https://example.com/article'))
+      end
+
+      it 'raises an error if the non-redirect shortener HTML contains no matching element' do
+        stub_request(:get, 'https://lnkd.in/bad')
+          .to_return(status: 200, body: '<html><body><p>No link</p></body></html>')
+        uri = described_class.new(URI('https://lnkd.in/bad'))
+
+        expect { uri.expand }.to raise_error(BadShortenedURI)
+      end
+
       after do
         Configuration.redirects = 5
         Configuration.shorteners.delete('altmetric.it')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embiggen
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.10.0
 platform: ruby
 authors:
 - Paul Mucur
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -93,11 +107,16 @@ files:
 - lib/embiggen.rb
 - lib/embiggen/configuration.rb
 - lib/embiggen/error.rb
+- lib/embiggen/html_client.rb
 - lib/embiggen/http_client.rb
+- lib/embiggen/non_redirect_shortener_list.rb
 - lib/embiggen/shortener_list.rb
 - lib/embiggen/uri.rb
+- non_redirect_shorteners.yml
 - shorteners.txt
 - spec/embiggen/configuration_spec.rb
+- spec/embiggen/html_client_spec.rb
+- spec/embiggen/non_redirect_shortener_list_spec.rb
 - spec/embiggen/shortener_list_spec.rb
 - spec/embiggen/uri_spec.rb
 - spec/embiggen_spec.rb
@@ -120,11 +139,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: A library to expand shortened URLs
 test_files:
 - spec/embiggen/configuration_spec.rb
+- spec/embiggen/html_client_spec.rb
+- spec/embiggen/non_redirect_shortener_list_spec.rb
 - spec/embiggen/shortener_list_spec.rb
 - spec/embiggen/uri_spec.rb
 - spec/embiggen_spec.rb