ZMediumToMarkdown 1.9.6 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: acc5589c92b8dd859f91b98fe7c2f1073a39af102ee8ebce27f1351824ffcab7
-  data.tar.gz: b14f257296741edd700bf484c37da6b2e386ddfb96dffca540b60cea8c04fecc
+  metadata.gz: 796d995f3d5f3f1edf3de599a28df4a8bea5ab9083d9bf8191d0d0535c924eb3
+  data.tar.gz: 7fa92c888507d4fea9293a3649c1656d3891a04b34d73d035d1a65676a5c8dc4
 SHA512:
-  metadata.gz: da41323a9b967724f758535d4ad8bb191cc1c015929b8af49e94851c7ff83b8a45d0ee8eb25298e2712b4acfbe1d3263cf4d327609cfc9f170e16cffadd347a9
-  data.tar.gz: c8a491d922669dd8ac947e658367fcbc093a007bda351ed9f7627213ad635c0d4246d36d09862ae597312147d4627f0314f12547df2c37d95a8bc400064c0897
+  metadata.gz: 434c7f737e281189a0feaa821d25e936632be3e72731a495ba62ea945bfef67063518637879b44ecf5f9121ce4ae8b302553bca55959a93e874904cc0fddc808
+  data.tar.gz: 298efd510208b800826cd39be456e47a7b1744291078fd7e61b954a59e9c9257e796c09ed1813b098ad0c8615c17b1b7e19217388dcbcd9f085d9b5151a35663

data/lib/Helper.rb

@@ -98,21 +98,31 @@ class Helper
         end
     end
 
-    def self.createPostInfo(postInfo)
-
-        title = postInfo.title.gsub("[","")
-        title = title.gsub("]","")
+    def self.createPostInfo(postInfo, isForJekyll)
+        if isForJekyll
+            title = postInfo.title.gsub("[","")
+            title = title.gsub("]","")
+
+            result = "---\n"
+            result += "title: #{title}\n"
+            result += "author: #{postInfo.creator}\n"
+            result += "date: #{postInfo.firstPublishedAt.strftime('%Y-%m-%dT%H:%M:%S.%LZ')}\n"
+            result += "categories: #{postInfo.collectionName}\n"
+            result += "tags: [#{postInfo.tags.join(",")}]\n"
+            result += "description: #{postInfo.description}\n"
+            if !postInfo.previewImage.nil?
+                result += "image:\r\n"
+                result += "  path: #{postInfo.previewImage}\r\n"
+            end
+            result += "render_with_liquid: false\n"
 
-        result = "---\n"
-        result += "title: #{title}\n"
-        result += "author: #{postInfo.creator}\n"
-        result += "date: #{postInfo.firstPublishedAt.strftime('%Y-%m-%dT%H:%M:%S.%LZ')}\n"
-        result += "categories: #{postInfo.collectionName}\n"
-        result += "tags: [#{postInfo.tags.join(",")}]\n"
-        result += "---\n"
-        result += "\r\n"
+            result += "---\n"
+            result += "\r\n"
 
-        result
+            result
+        else
+            nil
+        end
     end
 
     def self.printNewVersionMessageIfExists() 
@@ -183,15 +193,9 @@ class Helper
         
     def self.createWatermark(postURL)
         text = "\r\n\r\n\r\n"
-        text += "+-----------------------------------------------------------------------------------+"
-        text += "\r\n"
+        text += "_Converted [Medium Post](#{postURL}) by [ZMediumToMarkdown](https://github.com/ZhgChgLi/ZMediumToMarkdown)._"
         text += "\r\n"
-        text += "| **[View original post on Medium](#{postURL}) - Converted by [ZhgChgLi](https://zhgchg.li)/[ZMediumToMarkdown](https://github.com/ZhgChgLi/ZMediumToMarkdown)** |"
-        text += "\r\n"
-        text += "\r\n"
-        text += "+-----------------------------------------------------------------------------------+"
-        text += "\r\n"
-        
+
         text
     end
 end

data/lib/Parsers/CodeBlockParser.rb

@@ -25,15 +25,9 @@ class CodeBlockParser < Parser
     def parse(paragraph)
         if CodeBlockParser.isCodeBlock(paragraph)
             result = "```\n"
-            if isForJekyll
-                result += "{% raw %}\n"
-            end
             
-            result += paragraph.text
+            result += paragraph.text.chomp
 
-            if isForJekyll
-                result += "\n{% endraw %}"
-            end
             result += "\n```"
         else
             if !nextParser.nil?

data/lib/Parsers/IMGParser.rb

@@ -18,7 +18,7 @@ class IMGParser < Parser
 
             fileName = paragraph.metadata.id #d*fsafwfe.jpg
 
-            imageURL = "https://miro.medium.com/max/1400/#{paragraph.metadata.id}"
+            imageURL = "https://miro.medium.com/max/1400/#{fileName}"
 
             imagePathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath(nil), paragraph.postID)
             absolutePath = imagePathPolicy.getAbsolutePath(fileName)

data/lib/Parsers/IframeParser.rb

@@ -1,6 +1,7 @@
 $lib = File.expand_path('../', File.dirname(__FILE__))
 
 require 'uri'
+require 'net/http'
 
 require "Request"
 require "Parsers/Parser"
@@ -76,15 +77,9 @@ class IframeParser < Parser
                             gistRAW = Request.body(Request.URL(a['href']))
 
                             result = "```#{lang}\n"
-                            if isForJekyll
-                                result += "{% raw %}\n"
-                            end
 
-                            result += gistRAW
+                            result += gistRAW.chomp
                             
-                            if isForJekyll
-                                result += "\n{% endraw %}"
-                            end
                             result += "\n```"
                         end
                     end
@@ -96,17 +91,51 @@ class IframeParser < Parser
                             ogURL = params["url"]
                         end
                     end
-                    ogImageURL = Helper.fetchOGImage(ogURL)
 
-                    title = paragraph.iframe.title
-                    if title.nil? or title == ""
-                        title = Helper.escapeMarkdown(ogURL)
-                    end
-                    
-                    if !ogImageURL.nil?
-                        result = "\r\n\r\n[![#{title}](#{ogImageURL} \"#{title}\")](#{ogURL})\r\n\r\n"
+                    twitterID = ogURL[/^(https\:\/\/twitter\.com\/){1}.+(\/){1}(\d+)/, 3]
+
+                    if !twitterID.nil?
+                        uri = URI("https://api.twitter.com/1.1/statuses/show.json?simple_quoted_tweet=true&include_entities=true&tweet_mode=extended&include_cards=1&id=#{twitterID}")
+                        https = Net::HTTP.new(uri.host, uri.port)
+                        https.use_ssl = true
+                
+                        request = Net::HTTP::Get.new(uri)
+                        request['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.17.375.766 Safari/537.36';
+                        request['Authorization'] = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; # twitter private api
+                
+                        response = https.request(request)
+                        if response.code.to_i == 200
+                            twitterObj = JSON.parse(response.read_body)
+                            
+                            fullText = twitterObj["full_text"]
+                            twitterObj["entities"]["user_mentions"].each do |user_mention|
+                                fullText = fullText.gsub(user_mention["screen_name"],"[#{user_mention["screen_name"]}](https://twitter.com/#{user_mention["screen_name"]})")
+                            end
+                            twitterObj["entities"]["urls"].each do |url|
+                                fullText = fullText.gsub(url["url"],"[#{url["display_url"]}](#{url["expanded_url"]})")
+                            end
+
+                            createdAt = Time.parse(twitterObj["created_at"]).strftime('%Y-%m-%d %H:%M:%S')
+                            result = "\n\n"
+                            result += "■■■■■■■■■■■■■■ \n"
+                            result += "> **[#{twitterObj["user"]["name"]}](https://twitter.com/#{twitterObj["user"]["screen_name"]}) @ Twitter Says:** \n\n"
+                            result += "> > #{fullText} \n\n"
+                            result += "> **Tweeted at [#{createdAt}](#{ogURL}).** \n\n"
+                            result += "■■■■■■■■■■■■■■ \n\n"
+                        end
                     else
-                        result = "[#{title}](#{ogURL})"
+                        ogImageURL = Helper.fetchOGImage(ogURL)
+
+                        title = paragraph.iframe.title
+                        if title.nil? or title == ""
+                            title = Helper.escapeMarkdown(ogURL)
+                        end
+                        
+                        if !ogImageURL.nil?
+                            result = "\r\n\r\n[![#{title}](#{ogImageURL} \"#{title}\")](#{ogURL})\r\n\r\n"
+                        else
+                            result = "[#{title}](#{ogURL})"
+                        end
                     end
                 end
             end

data/lib/Parsers/PREParser.rb

@@ -21,17 +21,12 @@ class PREParser < Parser
     def parse(paragraph)
         if PREParser.isPRE(paragraph)
             result = "```\n"
-            if isForJekyll
-                result += "{% raw %}\n"
-            end
             
             paragraph.text.each_line do |p|
                 result += p
             end
 
-            if isForJekyll
-                result += "\n{% endraw %}"
-            end
+            result = result.chomp
             result += "\n```"
 
             result

data/lib/Post.rb

@@ -6,10 +6,12 @@ require 'nokogiri'
 require 'json'
 require 'date'
 
-class Post
+require 'ImageDownloader'
+require 'PathPolicy'
 
+class Post
   class PostInfo
-    attr_accessor :title, :tags, :creator, :firstPublishedAt, :latestPublishedAt, :collectionName
+    attr_accessor :title, :tags, :creator, :firstPublishedAt, :latestPublishedAt, :collectionName, :description, :previewImage
   end
 
   def self.getPostIDFromPostURLString(postURLString)
@@ -58,11 +60,27 @@ class Post
     end
   end
 
-  def self.parsePostInfoFromPostContent(content, postID)
+  def self.parsePostInfoFromPostContent(content, postID, pathPolicy)
     postInfo = PostInfo.new()
+    postInfo.description = content&.dig("Post:#{postID}", "previewContent", "subtitle")
     postInfo.title = content&.dig("Post:#{postID}", "title")
     postInfo.tags = content&.dig("Post:#{postID}", "tags").map{ |tag| tag["__ref"].gsub! 'Tag:', '' }
     
+    previewImage = content&.dig("Post:#{postID}", "previewImage", "__ref")
+    if !previewImage.nil?
+      previewImageFIleName = content&.dig(previewImage, "id")
+
+      imagePathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath(nil), postID)
+      absolutePath = imagePathPolicy.getAbsolutePath(previewImageFIleName)
+
+      imageURL = "https://miro.medium.com/max/1400/#{previewImageFIleName}"
+
+      if  ImageDownloader.download(absolutePath, imageURL)
+          relativePath = "#{pathPolicy.getRelativePath(nil)}/#{imagePathPolicy.getRelativePath(previewImageFIleName)}"
+          postInfo.previewImage = relativePath
+      end
+    end
+
     creatorRef = content&.dig("Post:#{postID}", "creator", "__ref")
     if !creatorRef.nil?
       postInfo.creator = content&.dig(creatorRef, "name")

data/lib/ZMediumFetcher.rb

@@ -50,9 +50,9 @@ class ZMediumFetcher
                     info += "-"
                 end
                 if !currentPostParagraphIndex.nil? && !totalPostParagraphsLength.nil?
-                    info += "[#{postPath[0..10]}...(#{currentPostParagraphIndex}/#{totalPostParagraphsLength})]"
+                    info += "[#{postPath[0..15]}...(#{currentPostParagraphIndex}/#{totalPostParagraphsLength})]"
                 else
-                    info += "[#{postPath[0..10]}...]"
+                    info += "[#{postPath[0..15]}...]"
                 end
             end
 
@@ -121,6 +121,14 @@ class ZMediumFetcher
             postPath = Post.getPostPathFromPostURLString(postURL)
         end
 
+        if isForJekyll
+            postPathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath(nil), "_posts/zmediumtomarkdown")
+            imagePathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath(nil), "assets")
+        else
+            postPathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath(nil), "zmediumtomarkdown")
+            imagePathPolicy = PathPolicy.new(postPathPolicy.getAbsolutePath(nil), "assets")
+        end
+
         progress.postPath = postPath
         progress.message = "Downloading Post..."
         progress.printLog()
@@ -132,7 +140,7 @@ class ZMediumFetcher
             raise "Error: Content is empty! PostURL: #{postURL}"
         end
         
-        postInfo = Post.parsePostInfoFromPostContent(postContent, postID)
+        postInfo = Post.parsePostInfoFromPostContent(postContent, postID, imagePathPolicy)
 
         sourceParagraphs = Post.fetchPostParagraphs(postID)
         if sourceParagraphs.nil?
@@ -207,14 +215,6 @@ class ZMediumFetcher
             paragraphs.append(paragraph)
             previousParagraph = paragraph
         end
-
-        if isForJekyll
-            postPathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath(nil), "_posts/zmediumtomarkdown")
-            imagePathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath(nil), "assets")
-        else
-            postPathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath(nil), "zmediumtomarkdown")
-            imagePathPolicy = PathPolicy.new(postPathPolicy.getAbsolutePath(nil), "assets")
-        end
         
         startParser = buildParser(imagePathPolicy)
 
@@ -230,7 +230,7 @@ class ZMediumFetcher
         absolutePath = postPathPolicy.getAbsolutePath("#{postWithDatePath}.md")
         
         # if markdown file is exists and last modification time is >= latestPublishedAt(last update post time on medium)
-        if File.file?(absolutePath) && File.mtime(absolutePath) >= postInfo.latestPublishedAt
+        if File.file?(absolutePath) && File.mtime(absolutePath).to_time.to_i >= postInfo.latestPublishedAt.to_i
             # Already downloaded and nothing has changed!, Skip!
             progress.currentPostParagraphIndex = paragraphs.length
             progress.message = "Skip, Post already downloaded and nothing has changed!"
@@ -239,7 +239,10 @@ class ZMediumFetcher
             Helper.createDirIfNotExist(postPathPolicy.getAbsolutePath(nil))
             File.open(absolutePath, "w+") do |file|
                 # write postInfo into top
-                file.puts(Helper.createPostInfo(postInfo))
+                postMetaInfo = Helper.createPostInfo(postInfo, isForJekyll)
+                if !postMetaInfo.nil?
+                    file.puts(postMetaInfo)
+                end
                 
                 index = 0
                 paragraphs.each do |paragraph|
@@ -260,7 +263,10 @@ class ZMediumFetcher
                     progress.printLog()
                 end
     
-                file.puts(Helper.createWatermark(postURL))
+                postWatermark = Helper.createWatermark(postURL)
+                if !postWatermark.nil?
+                    file.puts(postWatermark)
+                end
             end
             FileUtils.touch absolutePath, :mtime => postInfo.latestPublishedAt
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ZMediumToMarkdown
 version: !ruby/object:Gem::Version
-  version: 1.9.6
+  version: 2.0.0
 platform: ruby
 authors:
 - ZhgChgLi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-15 00:00:00.000000000 Z
+date: 2022-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri