ZMediumToMarkdown 3.3.3 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97751da25df535110c1771b77a009171f82bd761ff74a5948927a349d06358a0
-  data.tar.gz: e8063778d0980fc394ff52fb9f0e1caeb59578915e812881da385b8a4f2b0fe3
+  metadata.gz: 047cd0234c7a856da3d912b93d419ddcd7ad1396079ba49d3c494807ca37a8de
+  data.tar.gz: c4a96904909f9885c48bff408200a886191858be9c9f8746a01db065e66aaa8f
 SHA512:
-  metadata.gz: f53c5d05e53b7ed984f65e908eef65b0fdab7606e740871c8639816eb32422d47d87c75b4f2f368f8aa6e89add6b531a818f977045a1af88e720757c1a386e24
-  data.tar.gz: fc8dff70656090f4d5ce5790aadadd4fea82ac2da5989979c1c9c2fe1edb36514b5ea8027652d13dff695ad7ba080e281dab822d3b4078afd4347b457b889b25
+  metadata.gz: 07f6681e620d463a849ccff8e8d497c10114073d8aad3792f0d2beb6cf730145dfb36ac6e304a3ad805de56aad5f24e3f028c5e2613da4f38228657bbc9f70f7
+  data.tar.gz: 6e8056232851f1f2656a5b6340b6972bdc603f8ff4aedd18e101c9ec8eb5fe10d7b4ca4d6376086e533daddd180d169a4db7b3c2ead4bd69ba1b175cf73661d9

data/lib/CLI.rb

@@ -14,7 +14,6 @@ module CLI
     COOKIE_SETUP_URL = 'https://github.com/ZhgChgLi/ZMediumToMarkdown/wiki/Setting-Up-Medium-Cookies-and-a-Cloudflare-Worker-Proxy'.freeze
 
     DEFAULT_MEDIUM_HOST = 'https://medium.com/_/graphql'.freeze
-    DEFAULT_MIRO_MEDIUM_HOST = 'https://miro.medium.com'.freeze
 
     module_function
 
@@ -53,10 +52,6 @@ module CLI
                 ENV['MEDIUM_HOST'] = v
             end
 
-            opts.on('--miro_medium_host URL', 'Cloudflare Worker proxy URL for Medium image CDN (or set $MIRO_MEDIUM_HOST). Optional companion to --medium_host.') do |v|
-                ENV['MIRO_MEDIUM_HOST'] = v
-            end
-
             opts.on('-u', '--username USERNAME', 'Download all posts from a Medium username') do |v|
                 options[:username] = v
             end
@@ -167,25 +162,17 @@ module CLI
         !host.empty? && host != DEFAULT_MEDIUM_HOST
     end
 
-    def imageProxyConfigured?
-        host = ENV['MIRO_MEDIUM_HOST'].to_s
-        !host.empty? && host != DEFAULT_MIRO_MEDIUM_HOST
-    end
-
-
     # Only warn when the invocation will actually hit Medium — skip for
     # --version, --clean, --help, --new.
     def warnAboutMissingSetup(options, errput: $stderr)
         return unless willHitMedium?(options)
 
-        missingCookies     = !cookiesPresent?
-        missingProxy       = !proxyConfigured?
-        missingImageProxy  = !imageProxyConfigured?
-        return if !missingCookies && !missingProxy && !missingImageProxy
+        missingCookies = !cookiesPresent?
+        missingProxy   = !proxyConfigured?
+        return if !missingCookies && !missingProxy
 
         errput.puts buildSetupBanner(missingCookies: missingCookies,
-                                     missingProxy: missingProxy,
-                                     missingImageProxy: missingImageProxy)
+                                     missingProxy: missingProxy)
     end
 
     def willHitMedium?(options)
@@ -194,11 +181,10 @@ module CLI
 
     # One-line warning. The wiki has the actual setup steps; we just
     # nudge the user toward it instead of dumping a wall of guidance.
-    def buildSetupBanner(missingCookies:, missingProxy:, missingImageProxy:)
+    def buildSetupBanner(missingCookies:, missingProxy:)
         missing = []
         missing << 'Medium cookies (sid / uid)' if missingCookies
         missing << 'Cloudflare Worker proxy (MEDIUM_HOST)' if missingProxy
-        missing << 'Cloudflare image proxy (MIRO_MEDIUM_HOST)' if missingImageProxy
         return '' if missing.empty?
 
         "⚠  Missing #{missing.join(' / ')}. Medium / Cloudflare may block the run. Setup guide: #{COOKIE_SETUP_URL}"

data/lib/Parsers/IMGParser.rb

@@ -3,6 +3,7 @@ require 'Models/Paragraph'
 
 require 'ImageDownloader'
 require 'PathPolicy'
+require 'Request'
 
 class IMGParser < Parser
     attr_accessor :nextParser, :pathPolicy, :isForJekyll
@@ -20,7 +21,7 @@ class IMGParser < Parser
 
             fileName = paragraph.metadata.id #d*fsafwfe.jpg
 
-            miro_host = ENV.fetch('MIRO_MEDIUM_HOST', 'https://miro.medium.com')
+            miro_host = Request.miroHost
             imageURL = "#{miro_host}/#{fileName}"
 
             result = ""

data/lib/Post.rb

@@ -67,7 +67,7 @@ class Post
       imagePathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath(postID), pathPolicy.getRelativePath(postID))
       absolutePath = imagePathPolicy.getAbsolutePath(previewImageFileName)
 
-      miro_host = ENV.fetch('MIRO_MEDIUM_HOST', 'https://miro.medium.com')
+      miro_host = Request.miroHost
       imageURL = "#{miro_host}/#{previewImageFileName}"
 
       if ImageDownloader.download(absolutePath, imageURL)

data/lib/Request.rb

@@ -1,5 +1,6 @@
 require 'net/http'
 require 'nokogiri'
+require 'uri'
 require 'ChromeAuth'
 require 'CookieCache'
 
@@ -173,6 +174,7 @@ class Request
 
     def self.URL(url, method = 'GET', data = nil, retryCount = 0)
         retryCount += 1
+        url = mediumProxiedURL(url)
 
         uri = URI(url)
         https = Net::HTTP.new(uri.host, uri.port)
@@ -225,6 +227,14 @@ class Request
           request['Cookie'] = cookiesString;
         end
 
+        # When the request is going to a configured Worker proxy (and only
+        # then), attach the user's MEDIUM_HOST_SECRET as a header so the
+        # Worker can authenticate the caller. Skipped for upstream
+        # medium.com / miro.medium.com so the secret never leaks to Medium.
+        if proxyURI?(uri) && (proxySecret = ENV['MEDIUM_HOST_SECRET'].to_s) && !proxySecret.empty?
+            request['X-Medium-Proxy-Secret'] = proxySecret
+        end
+
         response = https.request(request);
 
         setCookieString = response.get_fields('set-cookie');
@@ -277,6 +287,54 @@ class Request
         response
     end
 
+    # If the user has configured a Cloudflare Worker proxy via MEDIUM_HOST,
+    # rewrite *any* https://medium.com/<path> URL to <worker-origin>/<path>
+    # so non-GraphQL hits (iframe metadata at /media/<id>, OG-image fallback
+    # to /<user>/<post>, etc.) also benefit from the proxy. GraphQL callers
+    # already hand us the proxy URL directly via ENV['MEDIUM_HOST'], so they
+    # short-circuit the rewrite.
+    def self.mediumProxiedURL(url)
+        return url unless url.is_a?(String) && url.start_with?('https://medium.com/')
+        origin = mediumProxyOrigin
+        return url if origin.nil?
+        url.sub(%r{\Ahttps://medium\.com}, origin)
+    end
+
+    # Extract the `<scheme>://<host>[:port]` of MEDIUM_HOST, or nil if no
+    # proxy is configured (or it still points at medium.com itself).
+    def self.mediumProxyOrigin
+        host = ENV['MEDIUM_HOST'].to_s
+        return nil if host.empty?
+        uri = URI.parse(host)
+        return nil if uri.host.nil? || uri.host == 'medium.com'
+        port = (uri.port && uri.port != uri.default_port) ? ":#{uri.port}" : ''
+        "#{uri.scheme}://#{uri.host}#{port}"
+    rescue URI::InvalidURIError
+        nil
+    end
+
+    # Resolve the host the gem should use for miro.medium.com image fetches.
+    # Single-Worker setups: the same MEDIUM_HOST proxy handles both medium.com
+    # and miro.medium.com via path dispatch, so we always derive miro from
+    # MEDIUM_HOST's origin. No proxy → upstream miro.medium.com.
+    def self.miroHost
+        mediumProxyOrigin || 'https://miro.medium.com'
+    end
+
+    # True iff `uri` is hosted by the configured Worker proxy — i.e. its
+    # host matches MEDIUM_HOST and MEDIUM_HOST is set to something other
+    # than upstream medium.com. Used to gate the MEDIUM_HOST_SECRET auth
+    # header so the secret only leaves the process when heading to the
+    # user's own proxy.
+    def self.proxyURI?(uri)
+        return false if uri.nil? || uri.host.nil?
+        envValue = ENV['MEDIUM_HOST'].to_s
+        return false if envValue.empty?
+        parsed = URI.parse(envValue) rescue nil
+        return false if parsed.nil? || parsed.host.nil?
+        parsed.host != 'medium.com' && parsed.host == uri.host
+    end
+
     # Cloudflare tags blocked responses via either the cf-mitigated header
     # or the standard "Just a moment..." challenge HTML. We check both
     # so we catch challenges even on Cloudflare deployments that don't

data/lib/ZMediumFetcher.rb

@@ -153,9 +153,7 @@ class ZMediumFetcher
 
         existingMeta = readExistingFrontMatter(absolutePath)
 
-        if existingMeta[:lastModifiedAt] && existingMeta[:lastModifiedAt] >= postInfo.latestPublishedAt.to_i &&
-           !isPin.nil? && isPin == existingMeta[:pin] &&
-           !isLockedPreviewOnly.nil? && isLockedPreviewOnly == existingMeta[:lockedPreviewOnly]
+        if shouldSkipExistingPost?(existingMeta, postInfo, isPin, isLockedPreviewOnly)
             # Already downloaded and nothing has changed!, Skip!
             progress.currentPostParagraphIndex = paragraphs.length
             progress.message = "Skip, Post already downloaded and nothing has changed!"
@@ -180,8 +178,8 @@ class ZMediumFetcher
 
     # Stdout fast path: render markdown directly to `stdoutIO` without
     # touching the filesystem and without downloading any images. Image
-    # references stay as remote miro.medium.com URLs (or MIRO_MEDIUM_HOST
-    # proxy if set).
+    # references stay as remote URLs on miro.medium.com (or the configured
+    # MEDIUM_HOST proxy origin when set).
     def downloadPostToStdout(postURL, isPin)
         postID = Post.getPostIDFromPostURLString(postURL)
         postPath = Post.getPostPathFromPostURLString(postURL)
@@ -374,6 +372,26 @@ class ZMediumFetcher
 
     # Reads YAML-ish front matter from a previously-generated post and
     # returns the fields we care about for skip-already-downloaded logic.
+    # Does the existing on-disk post still match the freshly-fetched
+    # metadata? All three signals must agree:
+    #   1. last_modified_at on disk is >= Medium's latestPublishedAt
+    #   2. isPin matches the file's pin flag
+    #   3. isLockedPreviewOnly matches the file's lockedPreviewOnly flag
+    #
+    # Boolean signals are normalized to true/false before comparing so that
+    # `nil` (Medium's GraphQL response can omit the field for non-paywalled
+    # / non-pinned posts) and `false` (the default written when the front-
+    # matter line is omitted by Helper.createPostInfo) are treated as
+    # equivalent — otherwise free, never-pinned posts would re-download on
+    # every run.
+    def shouldSkipExistingPost?(existingMeta, postInfo, isPin, isLockedPreviewOnly)
+        return false unless existingMeta[:lastModifiedAt]
+        return false unless existingMeta[:lastModifiedAt] >= postInfo.latestPublishedAt.to_i
+        return false unless (isPin == true) == existingMeta[:pin]
+        return false unless (isLockedPreviewOnly == true) == existingMeta[:lockedPreviewOnly]
+        true
+    end
+
     def readExistingFrontMatter(absolutePath)
         meta = { lastModifiedAt: nil, pin: false, lockedPreviewOnly: false }
         return meta unless File.file?(absolutePath)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ZMediumToMarkdown
 version: !ruby/object:Gem::Version
-  version: 3.3.3
+  version: 3.5.0
 platform: ruby
 authors:
 - ZhgChgLi