ZMediumToMarkdown 1.0.3 → 1.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/ZMediumToMarkdown +59 -0
- data/lib/Helper.rb +145 -1
- data/lib/Post.rb +8 -3
- data/{bin/ZMediumFetcher → lib/ZMediumFetcher.rb} +39 -65
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d175f065345508536bf819ab29db376b591c978d3b32872496f5d8a80d2fb518
|
4
|
+
data.tar.gz: eeb5c3c303a1daef3e05bce586240d0195f3d32999208b784530a175fe58b89e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6b12ca00410c8946e1736fac5c328b892d5b4d8b08d5074101bb16ac869ad5ef92d7884201a19a0fcadf2e11a9450b407f0573382e81cfa875090768a27822ed
|
7
|
+
data.tar.gz: 396a75a07a19283d6ce215b32da304ad4f90c72131c87903b8edf04e53ac1e3cf902e61877469367dce4b749fe979c292101cfe12b167290446d3da94826a28e
|
@@ -0,0 +1,59 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
|
4
|
+
$lib = File.expand_path('../lib', File.dirname(__FILE__))
|
5
|
+
$LOAD_PATH.unshift($lib)
|
6
|
+
|
7
|
+
require "ZMediumFetcher"
|
8
|
+
require "Helper"
|
9
|
+
require "optparse"
|
10
|
+
|
11
|
+
class Main
|
12
|
+
def initialize
|
13
|
+
fetcher = ZMediumFetcher.new
|
14
|
+
ARGV << '-h' if ARGV.empty?
|
15
|
+
OptionParser.new do |opts|
|
16
|
+
opts.banner = "Usage: ZMediumFetcher [options]"
|
17
|
+
|
18
|
+
opts.on('-uUSERNAME', '--username=USERNAME', 'test') do |username|
|
19
|
+
Helper.checkNewVersion()
|
20
|
+
|
21
|
+
pathPolicy = PathPolicy.new("#{File.expand_path('../', File.dirname(__FILE__))}", "Output")
|
22
|
+
fetcher.downloadPostsByUsername(username, pathPolicy)
|
23
|
+
end
|
24
|
+
|
25
|
+
opts.on('-pPOST_URL', '--postURL=POST_URL', 'test') do |postURL|
|
26
|
+
Helper.checkNewVersion()
|
27
|
+
|
28
|
+
pathPolicy = PathPolicy.new("#{File.expand_path('../', File.dirname(__FILE__))}", "Output")
|
29
|
+
fetcher.downloadPost(postURL, pathPolicy)
|
30
|
+
end
|
31
|
+
|
32
|
+
opts.on('-n', '--new', 'Update to latest version') do |postURL|
|
33
|
+
if Helper.compareVersion(Helper.getRemoteVersionFromGithub(), Helper.getLocalVersionFromGemspec())
|
34
|
+
Helper.downloadLatestVersion()
|
35
|
+
else
|
36
|
+
puts "You're using the latest version :)"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end.parse!
|
41
|
+
|
42
|
+
Helper.logLatestRunVersion()
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
begin
|
47
|
+
puts "#https://github.com/ZhgChgLi/ZMediumToMarkdown"
|
48
|
+
puts "You have read and agree with the Disclaimer."
|
49
|
+
Main.new()
|
50
|
+
puts "Execute Successfully!!!"
|
51
|
+
puts "#https://github.com/ZhgChgLi/ZMediumToMarkdown"
|
52
|
+
puts "#Thanks for using this tool."
|
53
|
+
puts "#If this is helpful, please help to star the repo or recommend it to your friends."
|
54
|
+
rescue => e
|
55
|
+
puts "#Error: #{e.class} #{e.message}\n"
|
56
|
+
puts e.backtrace
|
57
|
+
puts "#Please feel free to open an Issue or submit a fix/contribution via Pull Request on:\n"
|
58
|
+
puts "#https://github.com/ZhgChgLi/ZMediumToMarkdown\n"
|
59
|
+
end
|
data/lib/Helper.rb
CHANGED
@@ -1,8 +1,30 @@
|
|
1
1
|
$lib = File.expand_path('../lib', File.dirname(__FILE__))
|
2
2
|
|
3
|
+
require 'fileutils'
|
4
|
+
require 'date'
|
5
|
+
require 'PathPolicy'
|
3
6
|
require 'Post'
|
7
|
+
require "Request"
|
8
|
+
require 'json'
|
9
|
+
require 'open-uri'
|
10
|
+
require 'zip'
|
4
11
|
|
5
12
|
class Helper
|
13
|
+
|
14
|
+
class Version
|
15
|
+
attr_accessor :major, :minor, :patch
|
16
|
+
|
17
|
+
def initialize(major, minor, patch)
|
18
|
+
@major = major.to_i
|
19
|
+
@minor = minor.to_i
|
20
|
+
@patch = patch.to_i
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_string()
|
24
|
+
"#{major.to_s}.#{minor.to_s}.#{patch.to_s}"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
6
28
|
def self.createDirIfNotExist(dirPath)
|
7
29
|
dirs = dirPath.split("/")
|
8
30
|
currentDir = ""
|
@@ -23,11 +45,50 @@ class Helper
|
|
23
45
|
puts "####################################################\n"
|
24
46
|
end
|
25
47
|
|
48
|
+
def self.downloadLatestVersion()
|
49
|
+
apiPath = 'https://api.github.com/repos/ZhgChgLi/ZMediumToMarkdown/releases'
|
50
|
+
versions = JSON.parse(Request.URL(apiPath).body).sort { |a,b| b["id"] <=> a["id"] }
|
51
|
+
|
52
|
+
version = nil
|
53
|
+
index = 0
|
54
|
+
while version == nil do
|
55
|
+
thisVersion = versions[index]
|
56
|
+
if thisVersion["prerelease"] == false
|
57
|
+
version = thisVersion
|
58
|
+
next
|
59
|
+
end
|
60
|
+
index += 1
|
61
|
+
end
|
62
|
+
|
63
|
+
zipFilePath = version["zipball_url"]
|
64
|
+
puts "Downloading latest version from github..."
|
65
|
+
open('latest.zip', 'wb') do |fo|
|
66
|
+
fo.print open(zipFilePath).read
|
67
|
+
end
|
68
|
+
|
69
|
+
puts "Unzip..."
|
70
|
+
Zip::File.open("latest.zip") do |zipfile|
|
71
|
+
zipfile.each do |file|
|
72
|
+
fileNames = file.name.split("/")
|
73
|
+
fileNames.shift
|
74
|
+
filePath = fileNames.join("/")
|
75
|
+
if filePath != ''
|
76
|
+
puts "Unzinp...#{filePath}"
|
77
|
+
zipfile.extract(file, filePath) { true }
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
File.delete("latest.zip")
|
82
|
+
|
83
|
+
tagName = version["tag_name"]
|
84
|
+
puts "Update to version #{tagName} successfully!"
|
85
|
+
end
|
86
|
+
|
26
87
|
def self.createPostInfo(postInfo)
|
27
88
|
result = "---\n"
|
28
89
|
result += "title: #{postInfo.title}\n"
|
29
90
|
result += "author: #{postInfo.creator}\n"
|
30
|
-
result += "date: #{postInfo.firstPublishedAt}\n"
|
91
|
+
result += "date: #{postInfo.firstPublishedAt.strftime('%Y-%m-%dT%H:%M:%S.%LZ')}\n"
|
31
92
|
result += "tags: [#{postInfo.tags.join(",")}]\n"
|
32
93
|
result += "---\n"
|
33
94
|
result += "\r\n"
|
@@ -35,6 +96,89 @@ class Helper
|
|
35
96
|
result
|
36
97
|
end
|
37
98
|
|
99
|
+
def self.checkNewVersion()
|
100
|
+
if Helper.compareVersion(Helper.getRemoteVersionFromGithub(), Helper.getLocalVersionFromGemspec())
|
101
|
+
puts "##########################################################"
|
102
|
+
puts "##### New Version Available!!! #####"
|
103
|
+
puts "##### Please type `bin/ZMediumToMarkdown -n` to update!!##"
|
104
|
+
puts "##########################################################"
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def self.getLocalVersionFromGemspec()
|
109
|
+
rootPath = File.expand_path('../', File.dirname(__FILE__))
|
110
|
+
gemspecContent = File.read("#{rootPath}/ZMediumToMarkdown.gemspec")
|
111
|
+
result = gemspecContent[/(gem\.version){1}\s+(\=)\s+(\'){1}(\d+(\.){1}\d+(\.){1}\d+){1}(\'){1}/, 4]
|
112
|
+
|
113
|
+
if !result.nil?
|
114
|
+
versions = result.split(".")
|
115
|
+
Version.new(versions[0],versions[1],versions[2])
|
116
|
+
else
|
117
|
+
nil
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def self.logLatestRunVersion()
|
122
|
+
version = Helper.getLocalVersionFromGemspec()
|
123
|
+
rootPath = File.expand_path('../', File.dirname(__FILE__))
|
124
|
+
|
125
|
+
File.open("#{rootPath}/.latestRunVersion" , 'w+') do |file|
|
126
|
+
file.puts(version.to_string())
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def self.getLocalVersionFromFile()
|
131
|
+
rootPath = File.expand_path('../', File.dirname(__FILE__))
|
132
|
+
result = File.read("#{rootPath}/.latestRunVersion")
|
133
|
+
|
134
|
+
if !result.nil?
|
135
|
+
versions = result.split(".")
|
136
|
+
Version.new(versions[0],versions[1],versions[2])
|
137
|
+
else
|
138
|
+
nil
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def self.getRemoteVersionFromGithub()
|
143
|
+
apiPath = 'https://api.github.com/repos/ZhgChgLi/ZMediumToMarkdown/releases'
|
144
|
+
versions = JSON.parse(Request.URL(apiPath).body).sort { |a,b| b["id"] <=> a["id"] }
|
145
|
+
|
146
|
+
tagName = nil
|
147
|
+
index = 0
|
148
|
+
while tagName == nil do
|
149
|
+
thisVersion = versions[index]
|
150
|
+
if thisVersion["prerelease"] == false
|
151
|
+
tagName = thisVersion["tag_name"]
|
152
|
+
next
|
153
|
+
end
|
154
|
+
index += 1
|
155
|
+
end
|
156
|
+
|
157
|
+
if !tagName.nil?
|
158
|
+
versions = (tagName.downcase.gsub! 'v','') .split(".")
|
159
|
+
Version.new(versions[0],versions[1],versions[2])
|
160
|
+
else
|
161
|
+
nil
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
def self.compareVersion(version1, version2)
|
166
|
+
if version1.major > version2.major
|
167
|
+
true
|
168
|
+
else
|
169
|
+
if version1.minor > version2.minor
|
170
|
+
true
|
171
|
+
else
|
172
|
+
if version1.patch > version2.patch
|
173
|
+
true
|
174
|
+
else
|
175
|
+
false
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
|
38
182
|
def self.createWatermark(postURL)
|
39
183
|
text = "\r\n\r\n\r\n"
|
40
184
|
text += "+-----------------------------------------------------------------------------------+"
|
data/lib/Post.rb
CHANGED
@@ -9,7 +9,7 @@ require 'date'
|
|
9
9
|
class Post
|
10
10
|
|
11
11
|
class PostInfo
|
12
|
-
attr_accessor :title, :tags, :creator, :firstPublishedAt
|
12
|
+
attr_accessor :title, :tags, :creator, :firstPublishedAt, :latestPublishedAt
|
13
13
|
end
|
14
14
|
|
15
15
|
def self.getPostIDFromPostURLString(postURLString)
|
@@ -59,9 +59,14 @@ class Post
|
|
59
59
|
|
60
60
|
firstPublishedAt = content&.dig("Post:#{postID}", "firstPublishedAt")
|
61
61
|
if !firstPublishedAt.nil?
|
62
|
-
postInfo.firstPublishedAt =
|
62
|
+
postInfo.firstPublishedAt = Time.at(0, firstPublishedAt, :millisecond)
|
63
63
|
end
|
64
|
-
|
64
|
+
|
65
|
+
latestPublishedAt = content&.dig("Post:#{postID}", "latestPublishedAt")
|
66
|
+
if !latestPublishedAt.nil?
|
67
|
+
postInfo.latestPublishedAt = Time.at(0, latestPublishedAt, :millisecond)
|
68
|
+
end
|
69
|
+
|
65
70
|
postInfo
|
66
71
|
end
|
67
72
|
end
|
@@ -1,12 +1,8 @@
|
|
1
|
-
|
2
|
-
# -*- coding: utf-8 -*-
|
1
|
+
|
3
2
|
|
4
3
|
$lib = File.expand_path('../lib', File.dirname(__FILE__))
|
5
|
-
$LOAD_PATH.unshift($lib)
|
6
4
|
|
7
|
-
require "
|
8
|
-
require 'json'
|
9
|
-
require 'optparse'
|
5
|
+
require "fileutils"
|
10
6
|
|
11
7
|
require "Parsers/H1Parser"
|
12
8
|
require "Parsers/H2Parser"
|
@@ -31,26 +27,6 @@ require "Request"
|
|
31
27
|
require "Post"
|
32
28
|
require "User"
|
33
29
|
|
34
|
-
class Main
|
35
|
-
def initialize
|
36
|
-
fetcher = ZMediumFetcher.new
|
37
|
-
ARGV << '-h' if ARGV.empty?
|
38
|
-
OptionParser.new do |opts|
|
39
|
-
opts.banner = "Usage: ZMediumFetcher [options]"
|
40
|
-
|
41
|
-
opts.on('-uUSERNAME', '--username=USERNAME', 'test') do |username|
|
42
|
-
pathPolicy = PathPolicy.new("#{File.expand_path('../', File.dirname(__FILE__))}", "Output")
|
43
|
-
fetcher.downloadPostsByUsername(username, pathPolicy)
|
44
|
-
end
|
45
|
-
|
46
|
-
opts.on('-pPOST_URL', '--postURL=POST_URL', 'test') do |postURL|
|
47
|
-
pathPolicy = PathPolicy.new("#{File.expand_path('../', File.dirname(__FILE__))}", "Output")
|
48
|
-
fetcher.downloadPost(postURL, pathPolicy)
|
49
|
-
end
|
50
|
-
end.parse!
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
30
|
class ZMediumFetcher
|
55
31
|
|
56
32
|
attr_accessor :progress, :linkParser
|
@@ -236,36 +212,48 @@ class ZMediumFetcher
|
|
236
212
|
progress.printLog()
|
237
213
|
|
238
214
|
absolutePath = postPathPolicy.getAbsolutePath("#{postPath}.md")
|
239
|
-
Helper.createDirIfNotExist(postPathPolicy.getAbsolutePath(nil))
|
240
|
-
index = 0
|
241
|
-
File.open(absolutePath, "w+") do |file|
|
242
|
-
# write postInfo into top
|
243
|
-
|
244
|
-
file.puts(Helper.createPostInfo(postInfo))
|
245
|
-
|
246
|
-
paragraphs.each do |paragraph|
|
247
|
-
markupParser = MarkupParser.new(postHtml, paragraph)
|
248
|
-
paragraph.text = markupParser.parse()
|
249
|
-
result = startParser.parse(paragraph)
|
250
215
|
|
251
|
-
|
252
|
-
|
253
|
-
|
216
|
+
# if user latest use's version is < latest local version than force download (don't skip, make sure bug fixed can be influenced)
|
217
|
+
forceDownload = Helper.compareVersion(Helper.getLocalVersionFromGemspec(), Helper.getLocalVersionFromFile())
|
218
|
+
|
219
|
+
# if markdown file is exists and last modification time is >= latestPublishedAt(last update post time on medium)
|
220
|
+
if forceDownload == false && File.file?(absolutePath) && File.mtime(absolutePath) >= postInfo.latestPublishedAt
|
221
|
+
# Already downloaded and nothing has changed!, Skip!
|
222
|
+
progress.currentPostParagraphIndex = paragraphs.length
|
223
|
+
progress.message = "Skip, Post already downloaded and nothing has changed!"
|
224
|
+
progress.printLog()
|
225
|
+
else
|
226
|
+
Helper.createDirIfNotExist(postPathPolicy.getAbsolutePath(nil))
|
227
|
+
File.open(absolutePath, "w+") do |file|
|
228
|
+
# write postInfo into top
|
229
|
+
file.puts(Helper.createPostInfo(postInfo))
|
254
230
|
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
231
|
+
index = 0
|
232
|
+
paragraphs.each do |paragraph|
|
233
|
+
markupParser = MarkupParser.new(postHtml, paragraph)
|
234
|
+
paragraph.text = markupParser.parse()
|
235
|
+
result = startParser.parse(paragraph)
|
236
|
+
|
237
|
+
if !linkParser.nil?
|
238
|
+
result = linkParser.parse(result, paragraph.markupLinks)
|
239
|
+
end
|
240
|
+
|
241
|
+
file.puts(result)
|
242
|
+
|
243
|
+
index += 1
|
244
|
+
progress.currentPostParagraphIndex = index
|
245
|
+
progress.message = "Converting Post..."
|
246
|
+
progress.printLog()
|
247
|
+
end
|
248
|
+
|
249
|
+
file.puts(Helper.createWatermark(postURL))
|
261
250
|
end
|
251
|
+
FileUtils.touch absolutePath, :mtime => postInfo.latestPublishedAt
|
262
252
|
|
263
|
-
|
253
|
+
progress.message = "Post Successfully Downloaded!"
|
254
|
+
progress.printLog()
|
264
255
|
end
|
265
|
-
|
266
|
-
progress.message = "Post Successfully Downloaded!"
|
267
|
-
progress.printLog()
|
268
|
-
|
256
|
+
|
269
257
|
progress.postPath = nil
|
270
258
|
end
|
271
259
|
|
@@ -313,18 +301,4 @@ class ZMediumFetcher
|
|
313
301
|
progress.message = "All posts has been downloaded!, Total posts: #{postURLS.length}"
|
314
302
|
progress.printLog()
|
315
303
|
end
|
316
|
-
end
|
317
|
-
|
318
|
-
begin
|
319
|
-
puts "https://github.com/ZhgChgLi/ZMediumToMarkdown"
|
320
|
-
puts "You have read and agree with the Disclaimer."
|
321
|
-
Main.new()
|
322
|
-
puts "https://github.com/ZhgChgLi/ZMediumToMarkdown"
|
323
|
-
puts "Thanks for using this tool."
|
324
|
-
puts "If this is helpful, please help to star the repo or recommend it to your friends."
|
325
|
-
rescue => e
|
326
|
-
puts "#Error: #{e.class} #{e.message}\n"
|
327
|
-
puts e.backtrace
|
328
|
-
puts "#Please feel free to open an Issue or submit a fix/contribution via Pull Request on:\n"
|
329
|
-
puts "#https://github.com/ZhgChgLi/ZMediumToMarkdown\n"
|
330
304
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ZMediumToMarkdown
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- ZhgChgLi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-05-
|
11
|
+
date: 2022-05-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nokogiri
|
@@ -56,11 +56,11 @@ description: ZMediumToMarkdown lets you download Medium post and convert it to m
|
|
56
56
|
format easily.
|
57
57
|
email:
|
58
58
|
executables:
|
59
|
-
-
|
59
|
+
- ZMediumToMarkdown
|
60
60
|
extensions: []
|
61
61
|
extra_rdoc_files: []
|
62
62
|
files:
|
63
|
-
- bin/
|
63
|
+
- bin/ZMediumToMarkdown
|
64
64
|
- lib/ErrorHandle.rb
|
65
65
|
- lib/Helper.rb
|
66
66
|
- lib/ImageDownloader.rb
|
@@ -87,6 +87,7 @@ files:
|
|
87
87
|
- lib/Post.rb
|
88
88
|
- lib/Request.rb
|
89
89
|
- lib/User.rb
|
90
|
+
- lib/ZMediumFetcher.rb
|
90
91
|
homepage: https://github.com/ZhgChgLi/ZMediumToMarkdown
|
91
92
|
licenses:
|
92
93
|
- MIT
|