ZMediumToMarkdown 3.5.0 → 3.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 047cd0234c7a856da3d912b93d419ddcd7ad1396079ba49d3c494807ca37a8de
-  data.tar.gz: c4a96904909f9885c48bff408200a886191858be9c9f8746a01db065e66aaa8f
+  metadata.gz: 396aabb00395a5451c046ad36eca450186120dd1f836242c2bc0d40885126ee3
+  data.tar.gz: f10a39b453030fcf5286b1df32ed482b948c49c4152d6dd17e7a5871f9b5ed45
 SHA512:
-  metadata.gz: 07f6681e620d463a849ccff8e8d497c10114073d8aad3792f0d2beb6cf730145dfb36ac6e304a3ad805de56aad5f24e3f028c5e2613da4f38228657bbc9f70f7
-  data.tar.gz: 6e8056232851f1f2656a5b6340b6972bdc603f8ff4aedd18e101c9ec8eb5fe10d7b4ca4d6376086e533daddd180d169a4db7b3c2ead4bd69ba1b175cf73661d9
+  metadata.gz: 70a93c6be7b9c0e62966dc8cc1217ad95d458468833a26d2b61329eee2b0b9626d33e232490a02bab76f83f8923d5144906c41e394e4aa049b3634c37cc7382d
+  data.tar.gz: a67492a79d7857fff707c0e2aa313332db8a92a58c8e1c61941096194301910d60578051bf150f3ddbd7e04144eac9cd28c1c8a6ee2b0cdbf7e1069b967b24dd

data/lib/CLI.rb

@@ -11,7 +11,7 @@ require 'ChromeAuth'
 # All CLI-side concerns for the `ZMediumToMarkdown` executable. Pulled out
 # of bin/ so it can be exercised by unit tests without spawning processes.
 module CLI
-    COOKIE_SETUP_URL = 'https://github.com/ZhgChgLi/ZMediumToMarkdown/wiki/Setting-Up-Medium-Cookies-and-a-Cloudflare-Worker-Proxy'.freeze
+    COOKIE_SETUP_URL = 'https://github.com/ZhgChgLi/ZMediumToMarkdown/blob/main/wiki/Setting-Up-Medium-Cookies-and-a-Cloudflare-Worker-Proxy.md'.freeze
 
     DEFAULT_MEDIUM_HOST = 'https://medium.com/_/graphql'.freeze
 
@@ -158,8 +158,7 @@ module CLI
     # other than the default upstream Medium URL — i.e. user pointed it
     # at their own Cloudflare Worker (or another proxy).
     def proxyConfigured?
-        host = ENV['MEDIUM_HOST'].to_s
-        !host.empty? && host != DEFAULT_MEDIUM_HOST
+        !Request.mediumProxyOrigin.nil?
     end
 
     # Only warn when the invocation will actually hit Medium — skip for

data/lib/ImageDownloader.rb

@@ -1,21 +1,74 @@
+require 'net/http'
+require 'uri'
+
 require 'Helper'
+require 'Request'
 
 class ImageDownloader
+    USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36'.freeze
+    MAX_REDIRECTS = 5
+
+    # Downloads `url` to disk at `path`. Routes medium.com / miro.medium.com
+    # URLs through MEDIUM_HOST when configured (so requests inherit the
+    # Worker's IP reputation + auth) and attaches `X-Medium-Proxy-Secret`
+    # and the global cookie jar when the destination is the user's proxy.
+    # Other hosts (i.ytimg.com, pbs.twimg.com, etc.) are fetched directly.
     def self.download(path, url)
-        dir = path.split("/")
-        dir.pop()
-        Helper.createDirIfNotExist(dir.join("/"))
-        
-        if File.exist?(path)
-            return true
+        dir = path.split('/')
+        dir.pop
+        Helper.createDirIfNotExist(dir.join('/'))
+
+        return true if File.exist?(path)
+
+        rewritten = Request.mediumProxiedURL(url)
+        uri = URI.parse(rewritten) rescue nil
+        return false if uri.nil? || uri.host.nil?
+
+        response = fetchWithRedirects(uri, MAX_REDIRECTS)
+        return false if response.nil? || response.code.to_i != 200
+
+        body = response.body
+        return false if body.nil? || body.empty?
+
+        File.binwrite(path, body)
+        true
+    rescue StandardError
+        false
+    end
+
+    def self.fetchWithRedirects(uri, limit)
+        return nil if limit <= 0
+
+        https = Net::HTTP.new(uri.host, uri.port)
+        https.use_ssl = (uri.scheme == 'https')
+        https.open_timeout = 10
+        https.read_timeout = 60
+
+        request = Net::HTTP::Get.new(uri)
+        request['User-Agent'] = USER_AGENT
+
+        if Request.proxyURI?(uri)
+            secret = ENV['MEDIUM_HOST_SECRET'].to_s
+            request['X-Medium-Proxy-Secret'] = secret unless secret.empty?
         end
 
-        begin
-            imageResponse = URI.open(url)
-            File.write(path, imageResponse.read)
-            true
-        rescue
-            false
+        cookies = $cookies || {}
+        cookieString = cookies.reject { |_, v| v.nil? }
+                              .map { |k, v| "#{k}=#{v}" }
+                              .join('; ')
+        request['Cookie'] = cookieString unless cookieString.empty?
+
+        response = https.request(request)
+
+        case response.code.to_i
+        when 301, 302, 303, 307, 308
+            location = response['location'].to_s
+            return nil if location.empty?
+            target = URI.parse(URI.join(uri.to_s, location).to_s)
+            target = URI.parse(Request.mediumProxiedURL(target.to_s))
+            fetchWithRedirects(target, limit - 1)
+        else
+            response
         end
     end
-end
+end

data/lib/Post.rb

@@ -93,8 +93,7 @@ class Post
       "query" => queryString
     }]
 
-    host = ENV.fetch('MEDIUM_HOST', 'https://medium.com/_/graphql')
-    response = Request.body(Request.URL(host, 'POST', body))
+    response = Request.body(Request.URL(Request.mediumGraphqlEndpoint, 'POST', body))
     return nil if response.nil?
 
     JSON.parse(response)

data/lib/Request.rb

@@ -48,7 +48,7 @@ class Request
                          the MEDIUM_HOST env var. (Recommended.)
 
               Full step-by-step setup guide:
-              https://github.com/ZhgChgLi/ZMediumToMarkdown/wiki/Setting-Up-Medium-Cookies-and-a-Cloudflare-Worker-Proxy
+              https://github.com/ZhgChgLi/ZMediumToMarkdown/blob/main/wiki/Setting-Up-Medium-Cookies-and-a-Cloudflare-Worker-Proxy.md
             MSG
         end
     end
@@ -288,31 +288,47 @@ class Request
     end
 
     # If the user has configured a Cloudflare Worker proxy via MEDIUM_HOST,
-    # rewrite *any* https://medium.com/<path> URL to <worker-origin>/<path>
-    # so non-GraphQL hits (iframe metadata at /media/<id>, OG-image fallback
-    # to /<user>/<post>, etc.) also benefit from the proxy. GraphQL callers
-    # already hand us the proxy URL directly via ENV['MEDIUM_HOST'], so they
-    # short-circuit the rewrite.
+    # rewrite any https://medium.com/<path> OR https://miro.medium.com/<path>
+    # URL to <worker-origin>/<path> so non-GraphQL hits (iframe metadata at
+    # /media/<id>, OG-image fallback to /<user>/<post>, miro image downloads,
+    # etc.) all benefit from the proxy. GraphQL callers already hand us the
+    # proxy URL directly via mediumGraphqlEndpoint, so they short-circuit.
     def self.mediumProxiedURL(url)
-        return url unless url.is_a?(String) && url.start_with?('https://medium.com/')
+        return url unless url.is_a?(String)
         origin = mediumProxyOrigin
         return url if origin.nil?
-        url.sub(%r{\Ahttps://medium\.com}, origin)
+        if url.start_with?('https://medium.com/')
+            url.sub(%r{\Ahttps://medium\.com}, origin)
+        elsif url.start_with?('https://miro.medium.com/')
+            url.sub(%r{\Ahttps://miro\.medium\.com}, origin)
+        else
+            url
+        end
     end
 
     # Extract the `<scheme>://<host>[:port]` of MEDIUM_HOST, or nil if no
-    # proxy is configured (or it still points at medium.com itself).
+    # proxy is configured (or it still points at upstream medium.com).
+    # Accepts MEDIUM_HOST in any form — bare root, with /_/graphql suffix,
+    # or any other path — only the origin matters here.
     def self.mediumProxyOrigin
         host = ENV['MEDIUM_HOST'].to_s
         return nil if host.empty?
         uri = URI.parse(host)
-        return nil if uri.host.nil? || uri.host == 'medium.com'
+        return nil if uri.host.nil? || uri.host == 'medium.com' || uri.host == 'miro.medium.com'
         port = (uri.port && uri.port != uri.default_port) ? ":#{uri.port}" : ''
         "#{uri.scheme}://#{uri.host}#{port}"
     rescue URI::InvalidURIError
         nil
     end
 
+    # GraphQL endpoint the gem should POST to. When MEDIUM_HOST configures a
+    # proxy, it's <proxy-origin>/_/graphql regardless of whether the user set
+    # MEDIUM_HOST to the bare root or already with the /_/graphql suffix.
+    def self.mediumGraphqlEndpoint
+        origin = mediumProxyOrigin
+        origin.nil? ? 'https://medium.com/_/graphql' : "#{origin}/_/graphql"
+    end
+
     # Resolve the host the gem should use for miro.medium.com image fetches.
     # Single-Worker setups: the same MEDIUM_HOST proxy handles both medium.com
     # and miro.medium.com via path dispatch, so we always derive miro from
@@ -322,17 +338,16 @@ class Request
     end
 
     # True iff `uri` is hosted by the configured Worker proxy — i.e. its
-    # host matches MEDIUM_HOST and MEDIUM_HOST is set to something other
-    # than upstream medium.com. Used to gate the MEDIUM_HOST_SECRET auth
-    # header so the secret only leaves the process when heading to the
+    # host matches MEDIUM_HOST's origin. Used to gate the MEDIUM_HOST_SECRET
+    # auth header so the secret only leaves the process when heading to the
     # user's own proxy.
     def self.proxyURI?(uri)
         return false if uri.nil? || uri.host.nil?
-        envValue = ENV['MEDIUM_HOST'].to_s
-        return false if envValue.empty?
-        parsed = URI.parse(envValue) rescue nil
+        origin = mediumProxyOrigin
+        return false if origin.nil?
+        parsed = URI.parse(origin) rescue nil
         return false if parsed.nil? || parsed.host.nil?
-        parsed.host != 'medium.com' && parsed.host == uri.host
+        parsed.host == uri.host
     end
 
     # Cloudflare tags blocked responses via either the cf-mitigated header

data/lib/User.rb

@@ -22,8 +22,7 @@ class User
       }
     ]
 
-    host = ENV.fetch('MEDIUM_HOST', 'https://medium.com/_/graphql')
-    body = Request.body(Request.URL(host, "POST", query))
+    body = Request.body(Request.URL(Request.mediumGraphqlEndpoint, "POST", query))
     return nil if body.nil?
 
     json = JSON.parse(body)
@@ -44,8 +43,7 @@ class User
       }
     ]
 
-    host = ENV.fetch('MEDIUM_HOST', 'https://medium.com/_/graphql')
-    body = Request.body(Request.URL(host, "POST", query))
+    body = Request.body(Request.URL(Request.mediumGraphqlEndpoint, "POST", query))
     return { "nextID" => nil, "postURLs" => [] } if body.nil?
 
     json = JSON.parse(body)

data/lib/ZMediumFetcher.rb

@@ -419,7 +419,7 @@ class ZMediumFetcher
     # cookies belong to a Medium Member account that has access to the post.
     def paywallMessage
         if !defined?($cookies) || $cookies.nil? || ($cookies['sid'].to_s.empty? && $cookies['uid'].to_s.empty?)
-            "This post is behind Medium's paywall. Cookies (sid / uid) are REQUIRED to download the full content — without them you only get the public preview. Setup guide: https://github.com/ZhgChgLi/ZMediumToMarkdown/wiki/Setting-Up-Medium-Cookies-and-a-Cloudflare-Worker-Proxy"
+            "This post is behind Medium's paywall. Cookies (sid / uid) are REQUIRED to download the full content — without them you only get the public preview. Setup guide: https://github.com/ZhgChgLi/ZMediumToMarkdown/blob/main/wiki/Setting-Up-Medium-Cookies-and-a-Cloudflare-Worker-Proxy.md"
         else
             "This post is behind Medium's paywall and the provided cookies don't grant access. Verify your sid / uid belong to a Medium Member account that can read this post. Cookies stay valid as long as they're being used (each successful request resets a ~2-week sliding window); they only expire after ~2 weeks of inactivity."
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ZMediumToMarkdown
 version: !ruby/object:Gem::Version
-  version: 3.5.0
+  version: 3.5.2
 platform: ruby
 authors:
 - ZhgChgLi