directlink 0.0.4.7 → 0.0.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d38a321383a964dd0c6fcbf4fdf2671dffa0209
-  data.tar.gz: e97f410e07324784063d3799e3396f1c263832de
+  metadata.gz: b436120b4acd62ba55de62824aff4c9e04490104
+  data.tar.gz: 6db0db31513073100210447ba2f9cef30db54d96
 SHA512:
-  metadata.gz: 53353764e4a5f7443069890a38cc0dfad7453e3d05a96ffc49cebe39328090b04920e54a95446a954370a6e11ef1df9d8373b41dce52ff5ab1d9a8f7f949c5c6
-  data.tar.gz: 663560a5baec205f4c4d199c187b67d4caf213c75413049bed834783f25507fd96bbff0a775cde3b2365b2bd2361ba1f70334ebed8e3f7c6bc820e59dd822585
+  metadata.gz: 7f20abac6d10467eca10023a2e43df7d421e1c9ebd5666fac0d42413737817858cc5a3fca70ae1833bdf64a6266c225d36bddc3a09c97c00a551dfc49a2d98d1
+  data.tar.gz: b1f9d918c60bd79e16cb944cd55b6196eac4ab1bd0ef9faab3905ce781ebd93870e743fe9fd88abcba96b2b562a29006d3f8522f8def6d5f70bde9fa3edc6ead

data/.bashrc

@@ -1,3 +1,4 @@
 source api_tokens_for_travis.sh
-echo 'to test: ruby test.rb'
-echo 'to test: bundle exec ./bin/directlink --debug ...'
+echo 'to test: ruby -I../nethttputils/lib -I./lib test.rb'
+echo 'to test: ruby -I../nethttputils/lib -I./lib ./bin/directlink --debug ...'
+echo 'or: bundle exec ./bin/directlink --debug ...'

data/bin/directlink

@@ -43,7 +43,7 @@ if github
 else
   require "directlink"
 end
-NetHTTPUtils.logger.level = debug ? Logger::INFO : Logger::FATAL
+NetHTTPUtils.logger.level = debug ? Logger::INFO : Logger::FATAL unless ENV.include? "LOGLEVEL_NetHTTPUtils"
 # TODO --timeout=
 DirectLink.silent = !debug
 abort "usage: directlink [--debug] [--json] [--github] <link1> <link2> <link3> ...\n#{

data/directlink.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "directlink"
-  spec.version      = "0.0.4.7"
+  spec.version      = "0.0.4.8"
   spec.summary      = "converts any kind of image hyperlink to direct link, type of image and its resolution"
 
   spec.author       = "Victor Maslov aka Nakilon"
@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "fastimage", "~>2.1.3"
   spec.add_dependency "nokogiri"
-  spec.add_dependency "reddit_bot", "~>1.6.13"
+  spec.add_dependency "reddit_bot", "~>1.7.0"
   spec.add_dependency "kramdown"
   spec.add_dependency "addressable"
   spec.add_development_dependency "minitest"

data/lib/directlink.rb

@@ -69,7 +69,8 @@ module DirectLink
          /\Ahttps:\/\/lh3\.googleusercontent\.com\/-[a-zA-Z0-9]{11}\/W[a-zA-Z0-9]{9}I\/AAAAAAAA[a-zA-Z_]{3}\/[a-zA-Z0-9]{32}gCJoC\/s0\/[^\/]+\z/,
          /\Ahttps:\/\/lh3\.googleusercontent\.com\/-[a-zA-Z0-9]{11}\/[a-zA-Z0-9]{10}I\/AAAAAAA[a-zA-Z0-9]{4}\/[a-zA-Z0-9_-]{32}wCJoC\/s0\/[^\/]+\z/,
          /\Ahttps:\/\/lh3\.googleusercontent\.com\/-[a-zA-Z0-9]{11}\/[a-zA-Z0-9]{10}I\/AAAAAAA[A-Z]{4}\/[a-zA-Z0-9-]{32}gCJoC\/s0\/[^\/]+\z/,
-         /\Ahttps:\/\/lh3\.googleusercontent\.com\/-[a-zA-Z0-9-]{11}\/[a-zA-Z0-9-]{10}I\/AAAAAAA[a-zA-Z]{4}\/[a-zA-Z0-9_-]{32}ACJoC\/s0\/[^\/]+\z/
+         /\Ahttps:\/\/lh3\.googleusercontent\.com\/-[a-zA-Z0-9-]{11}\/[a-zA-Z0-9-]{10}I\/AAAAAAA[a-zA-Z]{4}\/[a-zA-Z0-9_-]{32}ACJoC\/s0\/[^\/]+\z/,
+         /\Ahttps:\/\/lh3\.googleusercontent\.com\/-[a-zA-Z0-9]{11}\/[a-zA-Z0-9-]{10}I\/AAAAAAA[a-zA-Z-]{4}\/[a-zA-Z0-9_]{32}gCJoC\/s0\/[^\/]+\z/
       src
     # Google Plus userpic
     when /\A(https:\/\/lh3\.googleusercontent\.com\/-[a-zA-Z0-9-]{11}\/AAAAAAAAAAI\/AAAAAAAA[a-zA-Z0-9]{3}\/[a-zA-Z0-9_-]{11}\/)s\d\d-p(?:-k)?-rw-no(\/photo\.jpg)\z/
@@ -107,7 +108,7 @@ module DirectLink
           t *= 2
           retry
         end
-        raise ErrorAssert.new "unexpected http error for #{url}"
+        raise ErrorAssert.new "unexpected http error #{e.code} for #{url}"
       end
     end
     case link
@@ -129,11 +130,12 @@ module DirectLink
         # one day single-video item should hit this but somehow it didn't yet
         raise ErrorAssert.new "unknown data format #{data.inspect} for #{link}"
       end
-    when /\Ahttps?:\/\/(?:(?:i|m|www)\.)?imgur\.com\/([a-zA-Z0-9]{7})(?:\.(?:gifv|jpg|png))?\z/,
+    when /\Ahttps?:\/\/(?:(?:i|m|www)\.)?imgur\.com\/([a-zA-Z0-9]{7,8})(?:\.(?:gifv|jpg|png))?\z/,
          /\Ahttps?:\/\/(?:(?:i|m|www)\.)?imgur\.com\/([a-zA-Z0-9]{5})\.mp4\z/,
          /\Ahttps?:\/\/imgur\.com\/([a-zA-Z0-9]{5}(?:[a-zA-Z0-9]{2})?)\z/,
          /\Ahttps?:\/\/imgur\.com\/([a-zA-Z0-9]{7})(?:\?\S+)?\z/,
-         /\Ahttps?:\/\/imgur\.com\/r\/[0-9_a-z]+\/([a-zA-Z0-9]{7})\z/
+         /\Ahttps?:\/\/imgur\.com\/r\/[0-9_a-z]+\/([a-zA-Z0-9]{7})\z/,
+         /\Ahttps?:\/\/api\.imgur\.com\/3\/image\/([a-zA-Z0-9]{7})\/0\.json\z/
       json = request_data["https://api.imgur.com/3/image/#{$1}/0.json"]
       [ JSON.load(json)["data"] ]
     else
@@ -150,8 +152,21 @@ module DirectLink
 
   def self._500px link
     raise ErrorBadLink.new link unless %r{\Ahttps://500px\.com/photo/(?<id>[^/]+)/[^/]+\z} =~ link
-    w, h = JSON.load(NetHTTPUtils.request_data "https://api.500px.com/v1/photos", form: {ids: id})["photos"].values.first.values_at("width", "height")
-    u, f = JSON.load(NetHTTPUtils.request_data "https://api.500px.com/v1/photos", form: {"image_size[]" => w, "ids" => id})["photos"].values.first["images"].first.values_at("url", "format")
+    require "nokogiri"
+    resp = NetHTTPUtils.request_data link
+    f = lambda do |form|
+      JSON.load(
+        NetHTTPUtils.request_data "https://api.500px.com/v1/photos",
+          form: form,
+          header: {
+            "Cookie" => resp.instance_variable_get(:@last_response).to_hash.fetch("set-cookie").join(?\s),
+            "X-CSRF-Token" => Nokogiri::HTML(resp).at_css("[name=csrf-token]")[:content]
+          }
+      ).fetch("photos").values.first
+    end
+    w, h = f[{"ids" => id                     }].values_at("width", "height")
+    # we need the above request to find the real resolution otherwise the "url" in the next request will be wrong
+    u, f = f[{"ids" => id, "image_size[]" => w}].fetch("images").first.values_at("url", "format")
     [w, h, u, f]
   end
 
@@ -247,6 +262,7 @@ end
 require "fastimage"
 
 def DirectLink link, max_redirect_resolving_retry_delay = nil, giveup = false
+  ArgumentError.new("link should be a <String>, not <#{link.class}>") unless link.is_a? String
   begin
     URI link
   rescue URI::InvalidURIError
@@ -273,17 +289,25 @@ def DirectLink link, max_redirect_resolving_retry_delay = nil, giveup = false
   #   > NetHTTPUtils.logger.level = Logger::DEBUG
   #   > NetHTTPUtils.request_data "http://www.aeronautica.difesa.it/organizzazione/REPARTI/divolo/PublishingImages/6%C2%B0%20Stormo/2013-decollo%20al%20tramonto%20REX%201280.jpg",
   #                               max_read_retry_delay: 5, timeout: 5
-  r = NetHTTPUtils.get_response link, header: {
-    "User-Agent" => "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"
-  }, **(max_redirect_resolving_retry_delay ? {
-    max_timeout_retry_delay: max_redirect_resolving_retry_delay,
-    max_sslerror_retry_delay: max_redirect_resolving_retry_delay,
-    max_read_retry_delay: max_redirect_resolving_retry_delay,
-    max_econnrefused_retry_delay: max_redirect_resolving_retry_delay,
-    max_socketerror_retry_delay: max_redirect_resolving_retry_delay,
-  } : {})
-  raise NetHTTPUtils::Error.new "", r.code.to_i unless "200" == r.code
-  link = r.uri.to_s
+
+  begin
+    head = NetHTTPUtils.request_data link, :head, header: {
+      "User-Agent" => "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"
+    }, **(max_redirect_resolving_retry_delay ? {timeout: max_redirect_resolving_retry_delay, max_start_http_retry_delay: max_redirect_resolving_retry_delay, max_read_retry_delay: max_redirect_resolving_retry_delay} : {})
+  rescue Net::ReadTimeout
+  rescue NetHTTPUtils::Error => e
+    raise unless e.code == 401
+  else
+    link = case head.instance_variable_get(:@last_response).code
+    when "200" ; link
+    when "302"
+      URI( head.instance_variable_get(:@last_response).to_hash.fetch("location").tap do |a|
+        raise DirectLink::ErrorAssert.new "unexpected size of locations after redirect" unless a.size == 1
+      end.first )
+    else ; raise NetHTTPUtils::Error.new "", (head.instance_variable_get(:@last_response).code || fail).to_i
+    end
+  end
+
   # why do we resolve redirects before trying the known adapters?
   #   because they can be hidden behind URL shorteners
   #   also it can resolve NetHTTPUtils::Error(404) before trying the adapter
@@ -297,10 +321,7 @@ def DirectLink link, max_redirect_resolving_retry_delay = nil, giveup = false
     # `DirectLink.imgur` return value is always an Array
     return imgur.size == 1 ? imgur.first : imgur
   rescue DirectLink::ErrorMissingEnvVar
-  end if %w{     imgur com } == URI(link).host.split(?.).last(2) ||
-         %w{   i imgur com } == URI(link).host.split(?.).last(3) ||
-         %w{   m imgur com } == URI(link).host.split(?.).last(3) ||
-         %w{ www imgur com } == URI(link).host.split(?.).last(3)
+  end if %w{ imgur com } == URI(link).host.split(?.).last(2)
 
   if %w{ 500px com } == URI(link).host.split(?.).last(2)
     w, h, u, t = DirectLink._500px(link)
@@ -331,7 +352,7 @@ def DirectLink link, max_redirect_resolving_retry_delay = nil, giveup = false
       f = ->_{ _.type == :a ? _.attr["href"] : _.children.flat_map(&f) }
       require "kramdown"
       return f[Kramdown::Document.new(u).root].map do |sublink|
-        DirectLink URI.join(link, sublink), max_redirect_resolving_retry_delay, giveup
+        DirectLink URI.join(link, sublink).to_s, max_redirect_resolving_retry_delay, giveup
       end
     end
     return struct.new *u.values_at(*%w{ fallback_url width height }), "video" if u.is_a? Hash
@@ -340,21 +361,25 @@ def DirectLink link, max_redirect_resolving_retry_delay = nil, giveup = false
   end if %w{ reddit com } == URI(link).host.split(?.).last(2) ||
          %w{ redd it    } == URI(link).host.split(?.)
 
-
   begin
-    f = FastImage.new(link, raise_on_failure: true, http_header: {"User-Agent" => "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"})
+    f = FastImage.new(link, raise_on_failure: true, timeout: 5, http_header: {"User-Agent" => "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"})
   rescue FastImage::UnknownImageType
     raise if giveup
     require "nokogiri"
-    html = Nokogiri::HTML NetHTTPUtils::request_data link, header: {"User-Agent" => "Mozilla"}
+    head = NetHTTPUtils.request_data link, :head, header: {"User-Agent" => "Mozilla"}
+    case head.instance_variable_get(:@last_response).content_type
+    when "text/html" ; nil
+    else ; raise
+    end
+    html = Nokogiri::HTML NetHTTPUtils.request_data link, header: {"User-Agent" => "Mozilla"}
     h = {}  # TODO: maybe move it outside because of possible img[:src] recursion?...
     l = lambda do |node, s = []|
       node.element_children.flat_map do |child|
         next l[child, s + [child.node_name]] unless "img" == child.node_name
         begin
-          [[s, (h[child[:src]] = h[child[:src]] || DirectLink(URI.join(link, child[:src]), nil, true))]]  # ... or wait, do we giveup?
+          [[s, (h[child[:src]] = h[child[:src]] || DirectLink(URI.join(link, child[:src]).to_s, nil, true))]]  # ... or wait, do we giveup?
         rescue => e
-          DirectLink.logger.error e
+          DirectLink.logger.error "#{e} (from no giveup)"
           []
         end
       end
@@ -367,6 +392,6 @@ def DirectLink link, max_redirect_resolving_retry_delay = nil, giveup = false
   else
     # TODO: maybe move this to right before `rescue` line
     w, h = f.size
-    struct.new link, w, h, f.type
+    struct.new f.instance_variable_get(:@parsed_uri).to_s, w, h, f.type
   end
 end

data/test.rb

@@ -132,6 +132,7 @@ describe DirectLink do
         https://lh3.googleusercontent.com/-VXUjuSl-dZY/WvK340_E9uI/AAAAAAAAVlg/HqKf1LgUcPUJNrLxHebCMuHhpDRq36_bQCJoC/s0/gplus248254159.jpg
         https://lh3.googleusercontent.com/-NlZRwcX_Lj8/WvQTijeAfJI/AAAAAAABNyo/jgoDgbZdTvsnLOGmmYlXMr2jL66ieZV4QCJoC/s0/67u8iii.png
         https://lh3.googleusercontent.com/-8baBz80pf8Y/Wu8KG5lyGhI/AAAAAAACSyU/s3hasZzObK0VlntA1EBj-WBrTRagzRnLQCJoC/s0/%25D0%2592%25D0%25B5%25D1%2581%25D0%25B5%25D0%25BD%25D0%25BD%25D0%25B8%25D0%25B5%2B%25D0%25BA%25D1%2580%25D0%25B0%25D1%2581%25D0%25B0%25D0%25B2%25D1%2586%25D1%258B.JPG
+        https://lh3.googleusercontent.com/-BBjhu17YIgg/W-gnZNaZeMI/AAAAAAABA-k/UMlSbNuE0DsSEPV8u3yf_6i2S5H9vFoBgCJoC/s0/gplus320347186.png
       }.each_with_index do |link, i|
         it "already high res image ##{i + 1}" do
           assert DirectLink.google link
@@ -226,11 +227,10 @@ describe DirectLink do
         end
       end
       it "does not throw 400 after a successfull retry" do
-        f = false
+        f = 0
         m = NetHTTPUtils.method :request_data
         NetHTTPUtils.stub :request_data, lambda{ |*args|
-          f ^= true
-          raise NetHTTPUtils::Error.new "", 400 if f
+          raise NetHTTPUtils::Error.new "", 400 if 1 == f += 1
           m.call *args
         } do
           assert_equal [["https://i.imgur.com/BLCesav.jpg", 1000, 1500, "image/jpeg"]],
@@ -432,7 +432,7 @@ describe DirectLink do
     it "Reddit correctly parses out id when no token provided" do
       t = ENV.delete "REDDIT_SECRETS"
       FastImage.stub :new, lambda{ |link, *|
-        assert_equal "https://v.redd.it/2tyovczka8m11/DASH_4_8_M", link
+        assert_equal "https://v.redd.it/2tyovczka8m11/DASH_4_8_M?source=fallback", link
         throw :_
       } do
         begin
@@ -535,7 +535,7 @@ describe DirectLink do
       ].each do |exception|
         it "raises #{exception} from the redirect resolving stage" do
           assert_raises exception do
-            NetHTTPUtils.stub :get_response, ->*{ raise exception.new } do
+            NetHTTPUtils.stub :request_data, ->*{ raise exception.new } do
               DirectLink "http://example.com/404"
             end
           end
@@ -543,7 +543,7 @@ describe DirectLink do
       end
       it "raises Net::OpenTimeout -- server side issues can happen (not related to User Agent)" do
         assert_raises Net::OpenTimeout do
-          NetHTTPUtils.stub :get_response, ->*{ raise Net::OpenTimeout.new } do
+          NetHTTPUtils.stub :request_data, ->*{ raise Net::OpenTimeout.new } do
             DirectLink "http://example.com/404"
           end
         end
@@ -569,7 +569,7 @@ describe DirectLink do
 
     describe "other domains tests" do
       [
-        ["http://www.aeronautica.difesa.it/organizzazione/REPARTI/divolo/PublishingImages/6%C2%B0%20Stormo/2013-decollo%20al%20tramonto%20REX%201280.jpg", ["http://www.aeronautica.difesa.it/organizzazione/REPARTI/divolo/PublishingImages/6%C2%B0%20Stormo/2013-decollo%20al%20tramonto%20REX%201280.jpg", 1280, 853, :jpeg]],
+        ["http://www.aeronautica.difesa.it/organizzazione/REPARTI/divolo/PublishingImages/6%C2%B0%20Stormo/2013-decollo%20al%20tramonto%20REX%201280.jpg", ["http://www.aeronautica.difesa.it/organizzazione/REPARTI/divolo/PublishingImages/6%C2%B0%20Stormo/2013-decollo%20al%20tramonto%20REX%201280.jpg", 1280, 853, :jpeg], nil, 1],
         ["http://minus.com/lkP3hgRJd9npi", SocketError, /nodename nor servname provided, or not known|No address associated with hostname/, 0],
         ["http://www.cutehalloweencostumeideas.org/wp-content/uploads/2017/10/Niagara-Falls_04.jpg", SocketError, /nodename nor servname provided, or not known|Name or service not known/, 0],
       ].each_with_index do |(input, expectation, message_string_or_regex, max_redirect_resolving_retry_delay), i|
@@ -602,7 +602,7 @@ describe DirectLink do
         ["http://redd.it/997he7",                 DirectLink::ErrorBadLink, true],
         ["http://redd.it/997he7",                 1],   # currently only links are parsed
       ].each_with_index do |(input, expectation, giveup), i|
-        it "##{i + 1} (#{URI(input).host})" do  # to match with minitest `-n` run flag
+        it "##{i + 1} (#{URI(input).host}) (giveup=#{!!giveup})" do  # to match with minitest `-n` run flag
           ti = ENV.delete "IMGUR_CLIENT_ID"
           tr = ENV.delete "REDDIT_SECRETS"
           begin
@@ -677,7 +677,7 @@ describe DirectLink do
         # TODO: a test that it appends the `exception.cause`
       ].each_with_index do |(expected_exit_code, link, expected_output, unset), i| # TODO: unset is not used anymore or I have to go sleep?
         it "##{i + 1}" do
-          string, status = Open3.capture2e "export #{File.read("api_tokens_for_travis.sh").gsub(/\n?export/, ?\s).strip}#{unset} && RUBYOPT='-rbundler/setup' ./bin/directlink #{link}"
+          string, status = Open3.capture2e "export #{File.read("api_tokens_for_travis.sh").gsub(/\n?export/, ?\s).strip}#{unset} && RUBYOPT='-rbundler/setup #{$-I.map{ |i| "-I #{i}" }.join " "}' ./bin/directlink #{link}"
           assert_equal [expected_exit_code, "#{expected_output}\n"], [status.exitstatus, string], "for #{link}"
         end
       end
@@ -688,7 +688,7 @@ describe DirectLink do
     [
       ["<= #{valid_imgur_image_url1}
         => https://i.imgur.com/QpOBvRY.png
-           image/png 460x460
+           png 460x460
         <= #{valid_imgur_image_url2}
         => https://i.imgur.com/QpOBvRY.png
            image/png 460x460
@@ -700,7 +700,7 @@ describe DirectLink do
             "url": "https://i.imgur.com/QpOBvRY.png",
             "width": 460,
             "height": 460,
-            "type": "image/png"
+            "type": "png"
           },
           [
             {
@@ -727,7 +727,7 @@ describe DirectLink do
 
     it "reddit_bot gem logger does not flood STDOUT" do
       string, status = Open3.capture2e "RUBYOPT='-rbundler/setup' ./bin/directlink http://redd.it/997he7"
-      assert_equal "<= http://redd.it/997he7\n=> https://i.imgur.com/QpOBvRY.png\n   image/png 460x460\n", string
+      assert_equal "<= http://redd.it/997he7\n=> https://i.imgur.com/QpOBvRY.png\n   png 460x460\n", string
     end
 
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: directlink
 version: !ruby/object:Gem::Version
-  version: 0.0.4.7
+  version: 0.0.4.8
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-04 00:00:00.000000000 Z
+date: 2019-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fastimage
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.13
+        version: 1.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.13
+        version: 1.7.0
 - !ruby/object:Gem::Dependency
   name: kramdown
   requirement: !ruby/object:Gem::Requirement