richurls 0.5.5 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 693d986c664aa571150f5c426a7ee496125e615321c1156d2c0eaf0b3f0db6e7
-  data.tar.gz: 4699ba24e5a8f2ed3ccfd2e8aa632c4243033e72d0f6db5b54e5faa16313514d
+  metadata.gz: fae0a9f1de4986a531f74b56b352b51a004f6aedd3fc3dad551019afba58676f
+  data.tar.gz: 65c3b687abfe9de86e4101437eaa0530a46bff9203683e9b6fb69234fa9f39b7
 SHA512:
-  metadata.gz: b5d7b35375f42c0bdfecdbd05179c984c1ec0d05669c3067280bbcea38bf9f9a34e79794ce587d4f23d0b26dfd1f45a664a6575551eccea7038bd89f3a168eb1
-  data.tar.gz: 04c2167a8ee0dad5a80447522a96d188a9687771e49609bf0857aa0d223fb47659a73de66225b376ce23000271f8dbb7546f48e72cc3ace62a1aa1fd6e130b92
+  metadata.gz: 380a50c4f02c34a16d92cb10a748cd0356dc4ac9eddfa6390f2d5cb95c930a907910f076ae68f3f9af4028dfd5fbab6712cd6650e9aa1a5664eed0204dcffa7f
+  data.tar.gz: 01c47002c302abd9a667003726276e597a7ec510f522d27d31af20078df7bfbede75a5a7c39433af48927c6ac57d418a336d5016bbf1bf5ab3ed410abf403428

data/.rubocop.yml

@@ -1,5 +1,6 @@
 AllCops:
   TargetRubyVersion: 2.7
+  NewCops: disable
 
 Lint/SuppressedException:
   Enabled: false
@@ -37,3 +38,5 @@ Naming/MemoizedInstanceVariableName:
   Enabled: false
 Style/RegexpLiteral:
   Enabled: false
+Style/IfUnlessModifier:
+  Enabled: false

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    richurls (0.5.5)
+    richurls (0.6.1)
       addressable (~> 2)
       oj (~> 3)
       ox (~> 2)
@@ -12,41 +12,44 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.0)
-    diff-lcs (1.3)
-    jaro_winkler (1.5.4)
-    oj (3.10.6)
-    ox (2.13.2)
-    parallel (1.19.1)
-    parser (2.7.1.1)
-      ast (~> 2.4.0)
+    ast (2.4.2)
+    diff-lcs (1.4.4)
+    oj (3.11.3)
+    ox (2.14.2)
+    parallel (1.20.1)
+    parser (3.0.0.0)
+      ast (~> 2.4.1)
     patron (0.13.3)
-    public_suffix (4.0.4)
+    public_suffix (4.0.6)
     rainbow (3.0.0)
-    redis (4.1.4)
+    redis (4.2.5)
+    regexp_parser (2.1.1)
     rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.1)
-      rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.1)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.2)
-    rubocop (0.82.0)
-      jaro_winkler (~> 1.5.1)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (0.93.1)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8)
       rexml
+      rubocop-ast (>= 0.6.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    ruby-progressbar (1.10.1)
+    rubocop-ast (1.4.1)
+      parser (>= 2.7.1.5)
+    ruby-progressbar (1.11.0)
     unicode-display_width (1.7.0)
 
 PLATFORMS

data/README.md

@@ -1,4 +1,4 @@
-# richurls [![Build Status](https://travis-ci.com/WeTransfer/richurls.svg?branch=master)](https://travis-ci.com/WeTransfer/richurls)
+# richurls [![Build Status](https://travis-ci.com/WeTransfer/richurls.svg?branch=main)](https://travis-ci.com/WeTransfer/richurls)
 A gem which can enrich urls with speed.
 
 **Installation**
@@ -85,3 +85,20 @@ RichUrls.enrich('https://wetransfer.com', cache_time: 3600)
 This `cache_time` will be accessible through the `time` parameters in the `set`
 and `extend` methods on the `Cache::Wrapper`-instance and can be used as you
 please.
+
+**Swapping browsers**
+
+The default browser is `curl` in the form of `Patron`. However if you feel
+like swapping to a different 'browser' like `HTTParty`, `RestClient` or something
+like `Ferrum` feel free to do so. You can swap it by doing:
+
+```ruby
+class CustomBrowser < RichUrls::Browser
+  def remote_call(url)
+    # Please make sure to return the variables in the function as such:
+    [status, redirected_url, body]
+  end
+end
+
+RichUrls.browser = CustomBrowser.new
+```

data/lib/patron_browser.rb

@@ -0,0 +1,25 @@
+require 'patron'
+
+module RichUrls
+  DEFAULT_TIMEOUT = 10 # seconds
+
+  class Browser
+    def remote_call(_url)
+      raise NotImplementedError,
+            'subclasses of Browser need a remote_call method'
+    end
+  end
+
+  class PatronBrowser < Browser
+    def remote_call(url)
+      session = Patron::Session.new(
+        timeout: DEFAULT_TIMEOUT,
+        headers: RichUrls.headers
+      )
+
+      response = session.get(url)
+
+      [response.status, response.url, response.body]
+    end
+  end
+end

data/lib/richurls.rb

@@ -5,6 +5,7 @@ require 'digest'
 require_relative 'cache'
 require_relative 'url_fetcher'
 require_relative 'body_decorator'
+require_relative 'patron_browser'
 
 module RichUrls
   class MalformedURLError < StandardError; end
@@ -13,6 +14,19 @@ module RichUrls
     @cache || Cache::None.new
   end
 
+  def self.browser=(browser)
+    unless browser.is_a? Browser
+      raise ArgumentError,
+            'browser needs to be of a RichUrls::Browser type'
+    end
+
+    @browser ||= browser
+  end
+
+  def self.browser
+    @browser || PatronBrowser.new
+  end
+
   def self.cache=(wrapper)
     unless wrapper.is_a? Cache::Wrapper
       raise ArgumentError,
@@ -22,11 +36,19 @@ module RichUrls
     @cache ||= wrapper
   end
 
+  def self.headers=(headers)
+    @headers ||= headers
+  end
+
+  def self.headers
+    @headers || {}
+  end
+
   def self.enrich(url, filter: [], cache_time: nil)
     unless URI::DEFAULT_PARSER.make_regexp.match?(url)
       raise MalformedURLError, "this url is malformed: #{url}"
     end
 
-    UrlFetcher.fetch(url, filter, cache_time)
+    UrlFetcher.fetch(url, filter, browser: browser, cache_time: cache_time)
   end
 end

data/lib/url_fetcher.rb

@@ -6,15 +6,21 @@ module RichUrls
 
     class UrlFetcherError < StandardError; end
 
-    def self.fetch(url, attributes = [], cache_time = nil)
-      new(url, attributes, cache_time).fetch
+    def self.fetch(
+      url,
+      attributes = [],
+      browser: PatronBrowser.new,
+      cache_time: nil
+    )
+      new(url, attributes, browser, cache_time).fetch
     end
 
     private_class_method :new
 
-    def initialize(url, attributes, cache_time)
+    def initialize(url, attributes, browser, cache_time)
       @url = url
       @attributes = attributes
+      @browser = browser
       @cache_time = cache_time
     end
 
@@ -36,13 +42,10 @@ module RichUrls
     end
 
     def patron_call
-      session = Patron::Session.new(timeout: DEFAULT_TIMEOUT)
-      response = session.get(@url)
+      status, return_url, body = @browser.remote_call(@url)
 
-      if response.status < 400
-        decorated = BodyDecorator.decorate(
-          response.url, response.body, @attributes
-        )
+      if status < 400
+        decorated = BodyDecorator.decorate(return_url, body, @attributes)
         RichUrls.cache.set(digest, Oj.dump(decorated), @cache_time)
         decorated
       else

data/lib/url_helper.rb

@@ -9,34 +9,32 @@ class UrlHelper
 
   def initialize(domain, url)
     @domain = domain
-
-    # In some rare cases it appears to be that URL's are ending with a
-    # single whitespace character resulting in an invalid URL.
-    @url = url&.strip
+    @url = url
   end
 
   def url
     return if @url.nil?
-    return Addressable::URI.escape(@url) if valid_url?
 
-    build_url
+    parsed = Addressable::URI.parse(@url)
+    full_url = valid?(parsed) ? parsed.to_s : domain_uri
+    Addressable::URI.escape(full_url)
+  rescue Addressable::URI::InvalidURIError
   end
 
   private
 
-  def build_url
-    domain_uri = URI(@domain)
-    base = domain_uri.scheme + '://' + domain_uri.host
-    escaped_url = Addressable::URI.escape(@url)
-
-    if @url.start_with?('/')
-      base + escaped_url
-    else
-      base + domain_uri.path + '/' + escaped_url
-    end
+  def valid?(parsed)
+    parsed.host && (parsed.scheme || @url.start_with?('//'))
   end
 
-  def valid_url?
-    @url.start_with?('//') || @url =~ URI::DEFAULT_PARSER.make_regexp
+  def domain_uri
+    domain = Addressable::URI.parse(@domain)
+    domain.query = nil
+    domain.path = if @url.start_with?('/')
+                    @url
+                  else
+                    domain.path + '/' + @url
+                  end
+    domain.to_s
   end
 end

data/richurls.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name = 'richurls'
-  spec.version = '0.5.5'
+  spec.version = '0.6.1'
   spec.authors = ['grdw']
   spec.email = ['gerard@wetransfer.com']
 
@@ -11,6 +11,7 @@ Gem::Specification.new do |spec|
   spec.description = 'Service which enriches URLs fast and cheap'
   spec.homepage = 'https://github.com/wetransfer/richurls'
   spec.license = 'GPL-3.0'
+  spec.required_ruby_version = '~> 2.7'
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/wetransfer/richurls'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: richurls
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.6.1
 platform: ruby
 authors:
 - grdw
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-05-08 00:00:00.000000000 Z
+date: 2021-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -153,6 +153,7 @@ files:
 - lib/parsers/property.rb
 - lib/parsers/provider_display_parser.rb
 - lib/parsers/url.rb
+- lib/patron_browser.rb
 - lib/richurls.rb
 - lib/url_fetcher.rb
 - lib/url_helper.rb
@@ -165,15 +166,15 @@ metadata:
   homepage_uri: https://github.com/wetransfer/richurls
   source_code_uri: https://github.com/wetransfer/richurls
   changelog_uri: https://github.com/wetransfer/richurls/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - "~>"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -181,7 +182,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Service which enriches URLs
 test_files: []