directlink 0.0.9.2 → 0.0.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f745bed3faf2b74dfe4532357fbb772fe32598b1
-  data.tar.gz: 203127452f8e51be364fff16a4c1eafa8a0a21df
+  metadata.gz: 1d67d884598ed66d3f4b2c09e278a7add58986e5
+  data.tar.gz: bec90a37e04dcc70c1e47c4b5906990603f3e0fb
 SHA512:
-  metadata.gz: 75bd9d5351c48c4f1df45b76f4835abc52bd057eabd01f4615a2056c16f4f9cf57546c3ff7123f479836206d1bb3164a740bebd2d81b84899b5507ee74d0e178
-  data.tar.gz: 6e20c47d7270b6bf3b0920ea964337b5143e255eb789baac70823218c9b689b13a6b16abdc729a3c134a2b5cf48fccfa074ec64bebac6ed835a32764b8f5726c
+  metadata.gz: 71f30a22295f7a04f3da474d3f1f42489e084de1b889e10ee3d10ab0cf2ab85d9e57c93067cf4225bf40751e6391f9cabe6ab1f42d77ebd1e98b88a6e16ed6c5
+  data.tar.gz: 27e321100578f58934237bd6aa0e63617b3dfa3fd618c23ee4d46e4b200b6c625c6c0a93a3839d8cdd585cb0f7eb8cfa34e160573301a0847c23ee39cb03fa70

data/bin/directlink

@@ -73,4 +73,8 @@ rescue *DirectLink::NORMAL_EXCEPTIONS => e
   cause = e.cause if e.cause if e.respond_to? :cause
   c = e.class.to_s
   abort "#{c}#{": #{e}" if c != e.to_s}#{": #{cause}" if cause && c != cause.to_s}"
+rescue
+  raise                unless $!.cause
+  raise $!.cause       unless $!.cause.cause
+  raise $!.cause.cause
 end

data/directlink.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "directlink"
-  spec.version      = "0.0.9.2"
+  spec.version      = "0.0.11.2"
   spec.summary      = "obtains from any kind of hyperlink a link to an image, its format and resolution"
 
   spec.author       = "Victor Maslov aka Nakilon"
@@ -9,20 +9,23 @@ Gem::Specification.new do |spec|
   spec.homepage     = "https://github.com/nakilon/directlink"
   spec.metadata     = {"source_code_uri" => "https://github.com/nakilon/directlink"}
 
-  spec.add_dependency "fastimage", "~>2.1.3"
-  spec.add_dependency "nokogiri"
-  spec.add_dependency "nethttputils", "~>0.4.1.0"
-  spec.add_dependency "reddit_bot", "~>1.7.8"
+  spec.required_ruby_version = ">=2.3"  # because <<~ heredocs in tests
+
+  spec.add_dependency "fastimage", "~>2.2.0"
+  spec.add_dependency "nokogiri", "<1.11"   # 1.11 requires ruby 2.5  # TODO: switch to Oga?
+  # spec.add_dependency "nethttputils", "~>0.4.2.0"
+  spec.add_dependency "reddit_bot", "~>1.10.0"
   spec.add_dependency "kramdown"
   spec.add_dependency "addressable"
-  spec.add_development_dependency "minitest"
+  spec.add_development_dependency "minitest-around"
+  spec.add_development_dependency "webmock"
 
   spec.require_path = "lib"
   spec.bindir       = "bin"
   spec.executable   = "directlink"
-  spec.test_file    = "test.rb"
+  spec.test_file    = "unit.test.rb"
   spec.files        = %w{ LICENSE directlink.gemspec lib/directlink.rb bin/directlink }
 
   spec.requirements << "you may want to create apps and provide API tokens:"
-  spec.requirements << "IMGUR_CLIENT_ID, FLICKR_API_KEY, REDDIT_SECRETS"
+  spec.requirements << "IMGUR_CLIENT_ID, FLICKR_API_KEY, REDDIT_SECRETS, VK_ACCESS_TOKEN, VK_CLIENT_SECRET"
 end

data/lib/directlink.rb

@@ -11,7 +11,7 @@ module DirectLink
     puts str unless Module.nesting.first.silent
   end
 
-  class ErrorAssert < RuntimeError
+  class ErrorAssert < RuntimeError  # gem user should not face this error
     def initialize msg
       super "#{msg} -- consider reporting this issue to GitHub"
     end
@@ -40,13 +40,15 @@ module DirectLink
     SocketError,
     Net::OpenTimeout,
     Errno::ECONNRESET,
+    Errno::ECONNREFUSED,
+    Errno::ETIMEDOUT,   # from FastImage
     NetHTTPUtils::Error,
     NetHTTPUtils::EOFError_from_rbuf_fill,
     FastImage::UnknownImageType,
     FastImage::ImageFetchFailure,
     DirectLink::ErrorNotFound,
     DirectLink::ErrorBadLink,
-  ]  # the only exceptions gem user should expect and handle
+  ]  # all known exceptions that can be raised while using Directlink but not as its fault
 
 
   def self.google src, width = 0
@@ -57,7 +59,7 @@ module DirectLink
       "#{$1}s#{width}/"
     when /\A(\/\/lh3\.googleusercontent\.com\/proxy\/[a-zA-Z0-9_-]{66,523}=)(?:w(?:[45]\d\d)-h\d\d\d-[np]|s530-p|s110-p-k)\z/
       "https:#{$1}s#{width}/"
-    when /\A(\/\/lh3\.googleusercontent\.com\/cOh2Nsv7EGo0QbuoKxoKZVZO_NcBzufuvPtzirMJfPmAzCzMtnEncfA7zGIDTJfkc1YZFX2MhgKnjA=)w530-h398-p\z/
+    when /\A(\/\/lh3\.googleusercontent\.com\/[a-zA-Z0-9]{24}_[a-zA-Z]{30}7zGIDTJfkc1YZFX2MhgKnjA=)w530-h398-p\z/
       "https:#{$1}s#{width}/"
     when /\A(\/\/lh3\.googleusercontent\.com\/-[a-zA-Z0-9-]{11}\/[VW][a-zA-Z0-9_-]{9}I\/AAAAAAA[AC][a-zA-Z0-9]{3}\/[a-zA-Z0-9_-]{32}[gwAQ]CJoC\/)w530-h[23]\d\d-p\/[^\/]+\z/,
          /\A(?:https?:)?(\/\/[1-4]\.bp\.blogspot\.com\/-[a-zA-Z0-9_-]{11}\/[UVWX][a-zA-Z0-9_-]{9}I\/AAAAAAAA[A-Z][a-zA-Z0-9_-]{2}\/[a-zA-Z0-9_-]{33}C(?:EwYBhgL|(?:Lc|Kg)BGAs(?:YHQ)?)\/)(?:s640|w\d{2,4}-h\d\d\d?-p(?:-k-no-nu)?)\/[^\/]+\z/,
@@ -78,20 +80,27 @@ module DirectLink
          /\A(https:\/\/lh[356]\.googleusercontent\.com\/-[a-zA-Z0-9]{11}\/AAAAAAAAAAI\/AAAAAAAAAAA\/[a-zA-Z0-9_]{34}\/)s(?:46|64)-c(?:-k(?:-no)?)?-mo\/photo\.jpg\z/
       "#{$1}s#{width}/"
     # Google Keep
-    when /\A(https:\/\/lh\d\.googleusercontent\.com\/[a-zA-Z0-9_-]{104,106}=s)\d\d\d\d?\Z/
+    when /\A(https:\/\/lh\d\.googleusercontent\.com\/[a-zA-Z0-9_-]{104,106}=s)\d\d\d\d?\z/
       "#{$1}#{width}"
+    # opensea
+    when /\A(https:\/\/lh3\.googleusercontent\.com\/[a-zA-Z0-9]{78}-nGx_jf_XGqqiVANe_Jr8u2g=)w1400-k\z/
+      "#{$1}s#{width}"
     # mp4
     when /\A(https:\/\/lh3\.googleusercontent\.com\/-[a-zA-Z]{11}\/W[a-zA-Z0-9]{9}I\/AAAAAAAAODw\/[a-zA-Z0-9]{32}QCJoC\/)w530-h883-n-k-no\/[^\/]+\.mp4\z/
       "#{$1}s#{width}/"
+    # something else
+    when /\A(https:\/\/lh3\.googleusercontent\.com\/-[a-zA-Z0-9_]{11}\/X-[a-zA-Z0-9]{8}I\/AAAAAAAAALE\/[a-zA-Z0-9]{23}_[a-zA-Z0-9]{19}\/)w1200-h630-p-k-no-nu\/[\d-]+\.png\z/
+      "#{$1}s#{width}/"
     else
       raise ErrorBadLink.new src
     end
   end
 
+
   require "json"
 
   # TODO make the timeout handling respect the way the Directlink method works with timeouts
-  def self.imgur link, timeout = 2000
+  def self.imgur link, timeout = 1000
     raise ErrorMissingEnvVar.new "define IMGUR_CLIENT_ID env var" unless ENV["IMGUR_CLIENT_ID"]
 
     request_data = lambda do |url|
@@ -100,7 +109,7 @@ module DirectLink
         NetHTTPUtils.request_data url, header: { Authorization: "Client-ID #{ENV["IMGUR_CLIENT_ID"]}" }
       rescue NetHTTPUtils::Error => e
         raise ErrorNotFound.new url.inspect if 404 == e.code
-        if t < timeout && [400, 500, 503].include?(e.code)
+        if t < timeout && [400, 500, 502, 503].include?(e.code)
           logger.error "retrying in #{t} seconds because of Imgur HTTP ERROR #{e.code}"
           sleep t
           t *= 2
@@ -151,7 +160,6 @@ module DirectLink
   def self._500px link
     raise ErrorBadLink.new link unless %r{\Ahttps://500px\.com/photo/(?<id>[^/]+)/[-[a-zA-Z0-9]%]+\/?\z} =~ link
     require "nokogiri"
-    resp = NetHTTPUtils.request_data link
     f = lambda do |form|
       JSON.load(NetHTTPUtils.request_data "https://api.500px.com/v1/photos", form: form).fetch("photos").values.first
     end
@@ -240,13 +248,15 @@ module DirectLink
       raise ErrorAssert.new "our knowledge about Reddit API seems to be outdated" unless data["media"].keys.sort == %w{ oembed type } && %w{ youtube.com gfycat.com imgur.com }.include?(data["media"]["type"])
       return [true, data["media"]["oembed"]["thumbnail_url"]]
     end
-    return [true, data["media_metadata"].values.map do |media|
-      next if media == {"status"=>"failed"}
-      raise ErrorAssert.new "our knowledge about Reddit API seems to be outdated" unless media["status"] == "valid"
-      [media["m"], *media["s"].values_at("x", "y"), CGI.unescapeHTML(media["s"]["u"])]
-    end.compact] if data["media_metadata"]
+    if data["media_metadata"]
+      return [true, data["media_metadata"].values.map do |media|
+        next if media == {"status"=>"failed"} || media == {"status"=>"unprocessed"}
+        raise ErrorAssert.new "our knowledge about Reddit API seems to be outdated (media == #{media.inspect})" unless media["status"] == "valid"
+        [media["m"], *media["s"].values_at("x", "y"), CGI.unescapeHTML(media["s"][media["m"]=="image/gif" ? "gif" : "u"])]
+      end.compact]
+    end
     return [true, "#{"https://www.reddit.com" if /\A\/r\/[0-9a-zA-Z_]+\/comments\/[0-9a-z]{5,6}\// =~ data["url"]}#{data["url"]}"] if data["crosspost_parent"]
-    return [true, data["url"]] unless data["is_self"]
+    return [true, CGI.unescapeHTML(data["url"])] unless data["is_self"]
     raise ErrorAssert.new "our knowledge about Reddit API seems to be outdated" if data["url"] != "https://www.reddit.com" + data["permalink"]
     return [false, data["selftext"]]
   end
@@ -255,8 +265,8 @@ module DirectLink
     id, mtd, field, f = case link
     when %r{\Ahttps://vk\.com/id(?<user_id>\d+)\?z=photo(?<id>\k<user_id>_\d+)(%2F(album\k<user_id>_0|photos\k<user_id>))?\z},
          %r{\Ahttps://vk\.com/[a-z_]+\?z=photo(?<_>)(?<id>(?<user_id>\d+)_\d+)%2Fphotos\k<user_id>\z},
-         %r{\Ahttps://vk\.com/photo(?<_>)(?<id>-?\d+_\d+)(\?(?:all|rev)=1)?\z},
-         %r{\Ahttps://vk\.com/feed\?section=likes&z=photo(?<_>)(?<id>-(?<user_id>\d+)_\d+)%2F(liked\d+|album\k<user_id>_0)\z},
+         %r{\Ahttps://vk\.com/photo(?<_>)(?<id>-?\d+_\d+)(\?(all|rev)=1)?\z},
+         %r{\Ahttps://vk\.com/feed\?(?:section=likes&)?z=photo(?<_>)(?<id>(?<user_id>-?\d+)_\d+)%2F(liked\d+|album\k<user_id>_0(0%2Frev)?)\z},
          %r{\Ahttps://vk\.com/[a-z_]+\?z=photo(?<_>)(?<id>(?<user_id>-\d+)_\d+)%2F(wall\k<user_id>_\d+|album\k<user_id>_0)\z},
          %r{\Ahttps://vk\.com/wall(?<user_id>-\d+)_\d+\?z=photo(?<id>\k<user_id>_\d+)%2F(wall\k<user_id>_\d+|album\k<user_id>_00%2Frev|\d+)\z}
       [$2, :photos, :photos, lambda do |t|
@@ -264,7 +274,7 @@ module DirectLink
         t
       end ]
     when %r{\Ahttps://vk\.com/wall(?<id>-?\d+_\d+)\z},
-         %r{\Ahttps://vk\.com/[a-z\.]+\?w=wall(?<id>\d+_\d+)\z}
+         %r{\Ahttps://vk\.com/[a-z\.]+\?w=wall(?<id>-?\d+_\d+)\z}
       [$1, :wall, :posts, lambda do |t|
         t.first.fetch("attachments").select do |item|
           case item.keys
@@ -282,14 +292,14 @@ module DirectLink
       raise ErrorBadLink.new link
     end
     raise ErrorMissingEnvVar.new "define VK_ACCESS_TOKEN and VK_CLIENT_SECRET env vars" unless ENV["VK_ACCESS_TOKEN"] && ENV["VK_CLIENT_SECRET"]
-    sleep 0.25 # "error_msg"=>"Too many requests per second"
+    sleep 0.25 unless ENV["CI"] # "error_msg"=>"Too many requests per second"
     f.call( JSON.load( NetHTTPUtils.request_data "https://api.vk.com/method/#{mtd}.getById",
       :POST, form: { field => id, :access_token => ENV["VK_ACCESS_TOKEN"], :client_secret => ENV["VK_CLIENT_SECRET"], :v => "5.101" }
     ).fetch("response") ).map do |photos|
       photos.fetch("sizes").map do |size|
         size.values_at("width", "height", "url").tap do |whu|
           w, h, u = whu
-          whu[0, 2] = FastImage.new(u, raise_on_failure: true).size if [w, h].include? 0
+          whu[0, 2] = FastImage.new(u, raise_on_failure: true).size if [w, h].include? 0 # wtf?
         end
       end.max_by{ |w, h, u| w * h }
     end
@@ -341,12 +351,13 @@ def DirectLink link, timeout = nil, proxy = nil, giveup: false, ignore_meta: fal
     head = NetHTTPUtils.request_data link, :HEAD, header: header, **(proxy ? {proxy: proxy} : {}), **(timeout ? {
       timeout: timeout,
       max_start_http_retry_delay: timeout,
-      max_read_retry_delay: timeout
+      max_read_retry_delay: timeout,
     } : {})
-  rescue Net::ReadTimeout
+  rescue Net::ReadTimeout, Errno::ETIMEDOUT
   rescue NetHTTPUtils::Error => e
     raise unless 418 == e.code
   else
+    raise DirectLink::ErrorAssert.new "last_response.uri is not set" unless head.instance_variable_get(:@last_response).uri
     link = head.instance_variable_get(:@last_response).uri.to_s
   end
 
@@ -354,10 +365,10 @@ def DirectLink link, timeout = nil, proxy = nil, giveup: false, ignore_meta: fal
   #   because they can be hidden behind URL shorteners
   #   also it can resolve NetHTTPUtils::Error(404) before trying the adapter
 
-  t = google_without_schema_crutch[] and return t
+  t = google_without_schema_crutch[] and return t   # TODO: why again?
 
   begin
-    imgur = DirectLink.imgur(link).sort_by{ |u, w, h, t| - w * h }.map do |u, w, h, t|
+    imgur = DirectLink.imgur(link, timeout).sort_by{ |u, w, h, t| - w * h }.map do |u, w, h, t|
       struct.new u, w, h, t
     end
     # `DirectLink.imgur` return value is always an Array
@@ -432,11 +443,11 @@ def DirectLink link, timeout = nil, proxy = nil, giveup: false, ignore_meta: fal
       timeout: timeout,                 # NetHTTPUtild passes this as read_timeout to Net::HTTP.start
       max_read_retry_delay: timeout     # and then compares accumulated delay to this
     # if we use :get here we will download megabytes of files just to giveup on content_type we can't process
-    case head.instance_variable_get(:@last_response).content_type
+    case head.instance_variable_get(:@last_response).content_type   # webmock should provide this
     when "text/html" ; nil
     else ; raise
     end
-    html = Nokogiri::HTML NetHTTPUtils.request_data link, header: {"User-Agent" => "Mozilla"}
+    html = Nokogiri::HTML NetHTTPUtils.request_data link, :GET, header: {"User-Agent" => "Mozilla"}
     if t = html.at_css("meta[@property='og:image']")
       begin
         return DirectLink URI.join(link, t[:content]).to_s, nil, *proxy, giveup: true