ZMediumToMarkdown 2.0.12 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b93c95279c2e90820d9e0e9048c79f1fa5e5dd0a94dba690cf8a3c3b8679589
-  data.tar.gz: e28c0042b45bf6f29377e3c8acf4e39dff68cda0b3d1b11cba153e95c97dc5aa
+  metadata.gz: 439f4734a4dde11659f6dede1e220f9edfd2203c0a025599c8c5ca78e2e91aab
+  data.tar.gz: c83396525f04b9313ed53b23d2bdf5b7b4ca5d8a7257c5364d9988c802160bcc
 SHA512:
-  metadata.gz: 0f24fbe044fe188d121ca6486173ccd15cd8ca1de6aea5f3ef625a0929615ba33fb890a39b144d0d0914c5c78bb30d94007629b364f87577098607651bd0f4f6
-  data.tar.gz: b4d939b9288c0f57ce3dd0b78bcb6d9abf672a7375cc5b3191d3820a80c9b43558f31fbafd2d5884b1b6b31b8bc9ac40406ef9fe8414964bcda4905f0b405c91
+  metadata.gz: 5b14cc13afd0e3b16ab5195465067ba98b61827171dc38412ca75467670010c4ff0753c7ad94996d29dd3fca5d489de9f055986c0fdbc81b74e83a16bbf9a53f
+  data.tar.gz: 7f9745e8c4608bea3cdf613d262c709d317a5fe1a3e3731d27db990d639e8c1e8b162f453ecded98e6dd6c7a1b4ed54c05262b264ce8b27acecefbbcc4bbdddb

data/lib/Helper.rb

@@ -14,9 +14,9 @@ class Helper
 
     def self.fetchOGImage(url)
         html = Request.html(Request.URL(url))
-        content = html.search("meta[property='og:image']").attribute('content')
-        
-        content
+        return "" unless html
+        image = html.search("meta[property='og:image']")
+        image.attribute('content') || ""
     end
 
     def self.escapeMarkdown(text)

data/lib/ImageDownloader.rb

@@ -9,8 +9,8 @@ class ImageDownloader
         Helper.createDirIfNotExist(dir.join("/"))
         
         begin
-            imageResponse = open(url)
-            File.write(path, imageResponse.read, {mode: 'wb'})
+            imageResponse = URI.open(url)
+            File.write(path, imageResponse.read)
             true
         rescue
             false

data/lib/Models/Paragraph.rb

@@ -5,7 +5,7 @@ require 'Parsers/PParser'
 require 'securerandom'
 
 class Paragraph
-    attr_accessor :postID, :name, :orgText, :text, :type, :href, :metadata, :mixtapeMetadata, :iframe, :oliIndex, :markups, :markupLinks
+    attr_accessor :postID, :name, :orgText, :text, :type, :href, :metadata, :mixtapeMetadata, :iframe, :oliIndex, :markups, :markupLinks, :codeBlockMetadata
 
     class Iframe
         attr_accessor :id, :title, :type, :src
@@ -49,6 +49,13 @@ class Paragraph
         end
     end
 
+    class CodeBlockMetadata
+        attr_accessor :lang
+        def initialize(json)
+            @lang = json['lang']
+        end
+    end
+
     def self.makeBlankParagraph(postID)
         json = {
             "name" => "fakeBlankParagraph_#{SecureRandom.uuid}",
@@ -72,13 +79,19 @@ class Paragraph
             @metadata = MetaData.new(json['metadata'])
         end
 
+        if json['codeBlockMetadata'].nil?
+            @codeBlockMetadata = nil
+        else
+            @codeBlockMetadata = CodeBlockMetadata.new(json['codeBlockMetadata'])
+        end
+
         if json['mixtapeMetadata'].nil?
             @mixtapeMetadata = nil
         else
             @mixtapeMetadata = MixtapeMetadata.new(json['mixtapeMetadata'])
         end
 
-        if json['iframe'].nil?
+        if json['iframe'].nil? || !json['iframe'] || !json['iframe']['mediaResource']
             @iframe = nil
         else
             @iframe = Iframe.new(json['iframe']['mediaResource'])

data/lib/Parsers/IframeParser.rb

@@ -26,7 +26,7 @@ class IframeParser < Parser
         end
 
         if paragraph.type == 'IFRAME'
-            
+            return unless paragraph.iframe
             if !paragraph.iframe.src.nil? && paragraph.iframe.src != ""
                 url = paragraph.iframe.src
             else
@@ -65,9 +65,11 @@ class IframeParser < Parser
                 end
             else
                 html = Request.html(Request.URL(url))
-                src = html.search('script').first.attribute('src')
+                return "" unless html
+                src = html.search('script').first
+                srce = src.attribute('src') if src
                 result = nil
-                if !src.to_s[/^(https\:\/\/gist\.github\.com)/].nil?
+                if !srce.to_s[/^(https\:\/\/gist\.github\.com)/].nil?
                     # is gist
                     gist = Request.body(Request.URL(src)).scan(/(document\.write\('){1}(.*)(\)){1}/)[1][1]
                     gist.gsub! '\n', ''

data/lib/Parsers/PREParser.rb

@@ -20,7 +20,13 @@ class PREParser < Parser
 
     def parse(paragraph)
         if PREParser.isPRE(paragraph)
-            result = "```\n"
+            
+            lang = ""
+            if !paragraph.codeBlockMetadata.nil?
+                lang = paragraph.codeBlockMetadata.lang
+            end
+
+            result = "```#{lang}\n"
             
             paragraph.text.each_line do |p|
                 result += p

data/lib/Post.rb

@@ -23,13 +23,12 @@ class Post
 
   def self.getPostPathFromPostURLString(postURLString)
     uri = URI.parse(postURLString)
-    postPath = uri.path.split('/').last
-    
-    URI.decode(postPath)
+    uri.path.split('/').last
   end
 
   def self.parsePostContentFromHTML(html)
     json = nil
+    return "" unless html
     html.search('script').each do |script|
         match = script.to_s[/(<script>window\.__APOLLO_STATE__ \= ){1}(.*)(<\/script>){1}/,2]
         if !match.nil? && match != ""
@@ -47,7 +46,7 @@ class Post
         "variables": {
           "postId": postID
         },
-        "query": "query PostViewerEdgeContentQuery($postId: ID!, $postMeteringOptions: PostMeteringOptions) {\n  post(id: $postId) {\n    ... on Post {\n      id\n      viewerEdge {\n        id\n        fullContent(postMeteringOptions: $postMeteringOptions) {\n          isLockedPreviewOnly\n          validatedShareKey\n          bodyModel {\n            ...PostBody_bodyModel\n            __typename\n          }\n          __typename\n        }\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n}\n\nfragment PostBody_bodyModel on RichText {\n  sections {\n    name\n    startIndex\n    textLayout\n    imageLayout\n    backgroundImage {\n      id\n      originalHeight\n      originalWidth\n      __typename\n    }\n    videoLayout\n    backgroundVideo {\n      videoId\n      originalHeight\n      originalWidth\n      previewImageId\n      __typename\n    }\n    __typename\n  }\n  paragraphs {\n    id\n    ...PostBodySection_paragraph\n    __typename\n  }\n  ...normalizedBodyModel_richText\n  __typename\n}\n\nfragment PostBodySection_paragraph on Paragraph {\n  name\n  ...PostBodyParagraph_paragraph\n  __typename\n  id\n}\n\nfragment PostBodyParagraph_paragraph on Paragraph {\n  name\n  type\n  ...ImageParagraph_paragraph\n  ...TextParagraph_paragraph\n  ...IframeParagraph_paragraph\n  ...MixtapeParagraph_paragraph\n  __typename\n  id\n}\n\nfragment ImageParagraph_paragraph on Paragraph {\n  href\n  layout\n  metadata {\n    id\n    originalHeight\n    originalWidth\n    focusPercentX\n    focusPercentY\n    alt\n    __typename\n  }\n  ...Markups_paragraph\n  ...ParagraphRefsMapContext_paragraph\n  ...PostAnnotationsMarker_paragraph\n  __typename\n  id\n}\n\nfragment Markups_paragraph on Paragraph {\n  name\n  text\n  hasDropCap\n  dropCapImage {\n    ...MarkupNode_data_dropCapImage\n    __typename\n    id\n  }\n  markups {\n    type\n    start\n    end\n    href\n    anchorType\n    userId\n    linkMetadata {\n      httpStatus\n      __typename\n    }\n    __typename\n  }\n  __typename\n  id\n}\n\nfragment MarkupNode_data_dropCapImage on ImageMetadata {\n  ...DropCap_image\n  __typename\n  id\n}\n\nfragment DropCap_image on ImageMetadata {\n  id\n  originalHeight\n  originalWidth\n  __typename\n}\n\nfragment ParagraphRefsMapContext_paragraph on Paragraph {\n  id\n  name\n  text\n  __typename\n}\n\nfragment PostAnnotationsMarker_paragraph on Paragraph {\n  ...PostViewNoteCard_paragraph\n  __typename\n  id\n}\n\nfragment PostViewNoteCard_paragraph on Paragraph {\n  name\n  __typename\n  id\n}\n\nfragment TextParagraph_paragraph on Paragraph {\n  type\n  hasDropCap\n  ...Markups_paragraph\n  ...ParagraphRefsMapContext_paragraph\n  __typename\n  id\n}\n\nfragment IframeParagraph_paragraph on Paragraph {\n  iframe {\n    mediaResource {\n      id\n      iframeSrc\n      iframeHeight\n      iframeWidth\n      title\n      __typename\n    }\n    __typename\n  }\n  layout\n  ...getEmbedlyCardUrlParams_paragraph\n  ...Markups_paragraph\n  __typename\n  id\n}\n\nfragment getEmbedlyCardUrlParams_paragraph on Paragraph {\n  type\n  iframe {\n    mediaResource {\n      iframeSrc\n      __typename\n    }\n    __typename\n  }\n  __typename\n  id\n}\n\nfragment MixtapeParagraph_paragraph on Paragraph {\n  type\n  mixtapeMetadata {\n    href\n    mediaResource {\n      mediumCatalog {\n        id\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n  ...GenericMixtapeParagraph_paragraph\n  __typename\n  id\n}\n\nfragment GenericMixtapeParagraph_paragraph on Paragraph {\n  text\n  mixtapeMetadata {\n    href\n    thumbnailImageId\n    __typename\n  }\n  markups {\n    start\n    end\n    type\n    href\n    __typename\n  }\n  __typename\n  id\n}\n\nfragment normalizedBodyModel_richText on RichText {\n  paragraphs {\n    markups {\n      type\n      __typename\n    }\n    ...getParagraphHighlights_paragraph\n    ...getParagraphPrivateNotes_paragraph\n    __typename\n  }\n  sections {\n    startIndex\n    ...getSectionEndIndex_section\n    __typename\n  }\n  ...getParagraphStyles_richText\n  ...getParagraphSpaces_richText\n  __typename\n}\n\nfragment getParagraphHighlights_paragraph on Paragraph {\n  name\n  __typename\n  id\n}\n\nfragment getParagraphPrivateNotes_paragraph on Paragraph {\n  name\n  __typename\n  id\n}\n\nfragment getSectionEndIndex_section on Section {\n  startIndex\n  __typename\n}\n\nfragment getParagraphStyles_richText on RichText {\n  paragraphs {\n    text\n    type\n    __typename\n  }\n  sections {\n    ...getSectionEndIndex_section\n    __typename\n  }\n  __typename\n}\n\nfragment getParagraphSpaces_richText on RichText {\n  paragraphs {\n    layout\n    metadata {\n      originalHeight\n      originalWidth\n      __typename\n    }\n    type\n    ...paragraphExtendsImageGrid_paragraph\n    __typename\n  }\n  ...getSeriesParagraphTopSpacings_richText\n  ...getPostParagraphTopSpacings_richText\n  __typename\n}\n\nfragment paragraphExtendsImageGrid_paragraph on Paragraph {\n  layout\n  type\n  __typename\n  id\n}\n\nfragment getSeriesParagraphTopSpacings_richText on RichText {\n  paragraphs {\n    id\n    __typename\n  }\n  sections {\n    startIndex\n    __typename\n  }\n  __typename\n}\n\nfragment getPostParagraphTopSpacings_richText on RichText {\n  paragraphs {\n    layout\n    text\n    __typename\n  }\n  sections {\n    startIndex\n    __typename\n  }\n  __typename\n}\n"
+        "query": "query PostViewerEdgeContentQuery($postId: ID!, $postMeteringOptions: PostMeteringOptions) {\n  post(id: $postId) {\n    ... on Post {\n      id\n      viewerEdge {\n        id\n        fullContent(postMeteringOptions: $postMeteringOptions) {\n          isLockedPreviewOnly\n          validatedShareKey\n          bodyModel {\n            ...PostBody_bodyModel\n            __typename\n          }\n          __typename\n        }\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n}\n\nfragment PostBody_bodyModel on RichText {\n  sections {\n    name\n    startIndex\n    textLayout\n    imageLayout\n    backgroundImage {\n      id\n      originalHeight\n      originalWidth\n      __typename\n    }\n    videoLayout\n    backgroundVideo {\n      videoId\n      originalHeight\n      originalWidth\n      previewImageId\n      __typename\n    }\n    __typename\n  }\n  paragraphs {\n    id\n    ...PostBodySection_paragraph\n    __typename\n  }\n  ...normalizedBodyModel_richText\n  __typename\n}\n\nfragment PostBodySection_paragraph on Paragraph {\n  name\n  ...PostBodyParagraph_paragraph\n  __typename\n  id\n}\n\nfragment PostBodyParagraph_paragraph on Paragraph {\n  name\n  type\n  ...ImageParagraph_paragraph\n  ...TextParagraph_paragraph\n  ...IframeParagraph_paragraph\n  ...MixtapeParagraph_paragraph\n  ...CodeBlockParagraph_paragraph\n  __typename\n  id\n}\n\nfragment ImageParagraph_paragraph on Paragraph {\n  href\n  layout\n  metadata {\n    id\n    originalHeight\n    originalWidth\n    focusPercentX\n    focusPercentY\n    alt\n    __typename\n  }\n  ...Markups_paragraph\n  ...ParagraphRefsMapContext_paragraph\n  ...PostAnnotationsMarker_paragraph\n  __typename\n  id\n}\n\nfragment Markups_paragraph on Paragraph {\n  name\n  text\n  hasDropCap\n  dropCapImage {\n    ...MarkupNode_data_dropCapImage\n    __typename\n    id\n  }\n  markups {\n    type\n    start\n    end\n    href\n    anchorType\n    userId\n    linkMetadata {\n      httpStatus\n      __typename\n    }\n    __typename\n  }\n  __typename\n  id\n}\n\nfragment MarkupNode_data_dropCapImage on ImageMetadata {\n  ...DropCap_image\n  __typename\n  id\n}\n\nfragment DropCap_image on ImageMetadata {\n  id\n  originalHeight\n  originalWidth\n  __typename\n}\n\nfragment ParagraphRefsMapContext_paragraph on Paragraph {\n  id\n  name\n  text\n  __typename\n}\n\nfragment PostAnnotationsMarker_paragraph on Paragraph {\n  ...PostViewNoteCard_paragraph\n  __typename\n  id\n}\n\nfragment PostViewNoteCard_paragraph on Paragraph {\n  name\n  __typename\n  id\n}\n\nfragment TextParagraph_paragraph on Paragraph {\n  type\n  hasDropCap\n  codeBlockMetadata {\n    mode\n    lang\n    __typename\n  }\n  ...Markups_paragraph\n  ...ParagraphRefsMapContext_paragraph\n  __typename\n  id\n}\n\nfragment IframeParagraph_paragraph on Paragraph {\n  iframe {\n    mediaResource {\n      id\n      iframeSrc\n      iframeHeight\n      iframeWidth\n      title\n      __typename\n    }\n    __typename\n  }\n  layout\n  ...getEmbedlyCardUrlParams_paragraph\n  ...Markups_paragraph\n  __typename\n  id\n}\n\nfragment getEmbedlyCardUrlParams_paragraph on Paragraph {\n  type\n  iframe {\n    mediaResource {\n      iframeSrc\n      __typename\n    }\n    __typename\n  }\n  __typename\n  id\n}\n\nfragment MixtapeParagraph_paragraph on Paragraph {\n  type\n  mixtapeMetadata {\n    href\n    mediaResource {\n      mediumCatalog {\n        id\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n  ...GenericMixtapeParagraph_paragraph\n  __typename\n  id\n}\n\nfragment GenericMixtapeParagraph_paragraph on Paragraph {\n  text\n  mixtapeMetadata {\n    href\n    thumbnailImageId\n    __typename\n  }\n  markups {\n    start\n    end\n    type\n    href\n    __typename\n  }\n  __typename\n  id\n}\n\nfragment CodeBlockParagraph_paragraph on Paragraph {\n  codeBlockMetadata {\n    lang\n    mode\n    __typename\n  }\n  __typename\n  id\n}\n\nfragment normalizedBodyModel_richText on RichText {\n  paragraphs {\n    markups {\n      type\n      __typename\n    }\n    codeBlockMetadata {\n      lang\n      mode\n      __typename\n    }\n    ...getParagraphHighlights_paragraph\n    ...getParagraphPrivateNotes_paragraph\n    __typename\n  }\n  sections {\n    startIndex\n    ...getSectionEndIndex_section\n    __typename\n  }\n  ...getParagraphStyles_richText\n  ...getParagraphSpaces_richText\n  __typename\n}\n\nfragment getParagraphHighlights_paragraph on Paragraph {\n  name\n  __typename\n  id\n}\n\nfragment getParagraphPrivateNotes_paragraph on Paragraph {\n  name\n  __typename\n  id\n}\n\nfragment getSectionEndIndex_section on Section {\n  startIndex\n  __typename\n}\n\nfragment getParagraphStyles_richText on RichText {\n  paragraphs {\n    text\n    type\n    __typename\n  }\n  sections {\n    ...getSectionEndIndex_section\n    __typename\n  }\n  __typename\n}\n\nfragment getParagraphSpaces_richText on RichText {\n  paragraphs {\n    layout\n    metadata {\n      originalHeight\n      originalWidth\n      id\n      __typename\n    }\n    type\n    ...paragraphExtendsImageGrid_paragraph\n    __typename\n  }\n  ...getSeriesParagraphTopSpacings_richText\n  ...getPostParagraphTopSpacings_richText\n  __typename\n}\n\nfragment paragraphExtendsImageGrid_paragraph on Paragraph {\n  layout\n  type\n  __typename\n  id\n}\n\nfragment getSeriesParagraphTopSpacings_richText on RichText {\n  paragraphs {\n    id\n    __typename\n  }\n  sections {\n    startIndex\n    __typename\n  }\n  __typename\n}\n\nfragment getPostParagraphTopSpacings_richText on RichText {\n  paragraphs {\n    layout\n    text\n    codeBlockMetadata {\n      lang\n      mode\n      __typename\n    }\n    __typename\n  }\n  sections {\n    startIndex\n    __typename\n  }\n  __typename\n}\n"
       }
     ]
 

data/lib/Request.rb

@@ -22,34 +22,40 @@ class Request
             end
         end
 
-        response = https.request(request)
-        
-        # 3XX Redirect
-        if response.code.to_i >= 300 && response.code.to_i <= 399 && !response['location'].nil? && response['location'] != ''
-            if retryCount >= 10
-                raise "Error: Retry limit reached. path: #{url}"
-            else
-                location = response['location']
-                if !location.match? /^(http)/
-                    location = "#{uri.scheme}://#{uri.host}#{location}"
-                end
-                response = self.URL(location, method, data)
-            end
+        begin
+          response = https.request(request)
+          # 3XX Redirect
+          if response.code.to_i >= 300 && response.code.to_i <= 399 && !response['location'].nil? && response['location'] != ''
+              if retryCount >= 10
+                  raise "Error: Retry limit reached. path: #{url}"
+              else
+                  location = response['location']
+                  if !location.match? /^(http)/
+                      location = "#{uri.scheme}://#{uri.host}#{location}"
+                  end
+                  response = self.URL(location, method, data)
+              end
+          end
+        rescue
+          
         end
+
         response
     end
 
     def self.html(response)
-        if response.code.to_i != 200 
-            nil
-        end
+      if response.nil? || (response && response.code.to_i != 200)
+        nil
+      else
         Nokogiri::HTML(response.read_body)
+      end
     end
 
     def self.body(response)
-        if response.code.to_i != 200 
-            nil
-        end
+      if response.nil? || (response && response.code.to_i != 200)
+        nil
+      else
         response.read_body
+      end
     end
 end

data/lib/ZMediumFetcher.rb

@@ -26,6 +26,14 @@ require "Request"
 require "Post"
 require "User"
 require 'date'
+require "uri"
+
+module URI
+  def self.decode url
+    url ? URI.decode_www_form_component(url).gsub(" ", "%20") : ""
+  end
+end
+    
 
 class ZMediumFetcher
 
@@ -128,7 +136,7 @@ class ZMediumFetcher
             imagePathPolicy = PathPolicy.new(postPathPolicy.getAbsolutePath("assets"), "assets")
         end
 
-        progress.postPath = postPath
+        progress.postPath = URI.decode(postPath)
         progress.message = "Downloading Post..."
         progress.printLog()
 
@@ -154,6 +162,7 @@ class ZMediumFetcher
         previousParagraph = nil
         preTypeParagraphs = []
         sourceParagraphs.each do |sourcParagraph|
+          return if (!sourcParagraph || !postID)
             paragraph = Paragraph.new(sourcParagraph, postID)
             if OLIParser.isOLI(paragraph)
                 oliIndex += 1
@@ -223,8 +232,7 @@ class ZMediumFetcher
         progress.printLog()
 
         postWithDatePath = "#{postInfo.firstPublishedAt.strftime("%Y-%m-%d")}-#{postPath}"
-
-        absolutePath = postPathPolicy.getAbsolutePath("#{postWithDatePath}.md")
+        absolutePath = URI.decode(postPathPolicy.getAbsolutePath("#{postWithDatePath}")) + ".md"
         
         fileLatestPublishedAt = nil
 
@@ -287,7 +295,7 @@ class ZMediumFetcher
     
     def downloadPostsByUsername(username, pathPolicy)
         progress.username = username
-        progress.message = "Fetching infromation..."
+        progress.message = "Fetching posts..."
         progress.printLog()
 
         userID = User.convertToUserIDFromUsername(username)
@@ -295,9 +303,6 @@ class ZMediumFetcher
             raise "Medium's Username:#{username} not found!"
         end
 
-        progress.message = "Fetching posts..."
-        progress.printLog()
-
         postURLS = []
         nextID = nil
         begin
@@ -321,14 +326,19 @@ class ZMediumFetcher
             downloadPathPolicy = PathPolicy.new(pathPolicy.getAbsolutePath("users/#{username}"), pathPolicy.getRelativePath("users/#{username}"))
         end
        
-        index = 0
+        index = 1
         postURLS.each do |postURL|
-            downloadPost(postURL, downloadPathPolicy)
-
-            index += 1
-            progress.currentPostIndex = index
-            progress.message = "Downloading posts..."
-            progress.printLog()
+          begin
+            # todo: unless File.exists? Post.getPostPathFromPostURLString(postURL) +".md"
+            downloadPost(postURL, downloadPathPolicy) 
+          rescue => e
+            puts e
+          end
+
+          index += 1
+          progress.currentPostIndex = index
+          progress.message = "Downloading posts..."
+          progress.printLog()
         end
 
         progress.message = "All posts has been downloaded!, Total posts: #{postURLS.length}"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ZMediumToMarkdown
 version: !ruby/object:Gem::Version
-  version: 2.0.12
+  version: 2.1.0
 platform: ruby
 authors:
 - ZhgChgLi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-08-19 00:00:00.000000000 Z
+date: 2023-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri