mangdown 0.8.5 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/doc/help.txt +17 -8
- data/lib/mangdown/cbz.rb +1 -1
- data/lib/mangdown/chapter.rb +33 -35
- data/lib/mangdown/commands.rb +3 -9
- data/lib/mangdown/manga.rb +51 -37
- data/lib/mangdown/page.rb +13 -15
- data/lib/mangdown/popular.rb +11 -24
- data/lib/mangdown/tools.rb +4 -73
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: beac9cf983e8dae6123c6ccd3ccdbac2e2de4298
|
4
|
+
data.tar.gz: 84010f24f9890d8157815a7ed0b3430453d88bca
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4689e520bb6574dd69b2a1bbea4154092ea0a7ce0999a70ee08c58fc3a0d5cf0bf930264664b7c6d8b05c511e91f37b670a1ec65c16e27a2e25bc839f7b5642a
|
7
|
+
data.tar.gz: 510899de271d3dff43a09b82ea7d543d86b4522c68af6607f0b83a2215cec800231dbcad68d680a18225a9084a5d532e74bc57686e4dfb0dc27cfd0eaf6d24c3
|
data/doc/help.txt
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
====================================================================
|
2
2
|
Mangdown
|
3
3
|
|
4
4
|
Commands
|
@@ -11,23 +11,32 @@ Commands
|
|
11
11
|
|
12
12
|
|
13
13
|
|
14
|
-
**Use Mangdown::MDHash#
|
14
|
+
**Use Mangdown::MDHash#to_manga to get a Mangdown::Manga object**
|
15
15
|
|
16
|
-
naruto = results[0].
|
16
|
+
naruto = results[0].to_manga
|
17
17
|
|
18
18
|
|
19
19
|
|
20
|
-
|
20
|
+
Mangdown::Manga#download(int = 0, int = -1)
|
21
21
|
- Will download a manga (Mangdown::Manga object)
|
22
22
|
from the first int (index of the first chapter)
|
23
23
|
to the last int (index of the last chapter) to
|
24
|
-
a subdirectory
|
25
|
-
|
24
|
+
a subdirectory of the set DOWNLOAD_DIR which
|
25
|
+
is the Dir.home directory by default named
|
26
|
+
#{manga.name}". If no indexes are given, all
|
27
|
+
chapters are downloaded.
|
26
28
|
|
27
|
-
|
29
|
+
# naruto is the Mangdown::Manga object from the example above
|
30
|
+
naruto.download(500, 549)
|
28
31
|
|
29
|
-
|
32
|
+
Mangdown::Manga#download_to(dir, int = 0, int = -1)
|
33
|
+
- Same as download above, but you can provide
|
34
|
+
the directory when the manga subdirectory
|
35
|
+
will be downloaded
|
30
36
|
|
37
|
+
# naruto is the Mangdown::Manga object from the example above
|
38
|
+
naruto.download_to(Dir.home + '/manga', 500, 549)
|
39
|
+
|
31
40
|
M#clean_up - Will delete .manga_list.yml file from the home
|
32
41
|
directory
|
33
42
|
|
data/lib/mangdown/cbz.rb
CHANGED
@@ -50,7 +50,7 @@ module Mangdown
|
|
50
50
|
# slice the last number in the file or directory name,
|
51
51
|
# which will be either the page number or the chapter number
|
52
52
|
num = name.slice(/(\d+)(\.jpg)*\Z/, 1)
|
53
|
-
return unless num
|
53
|
+
return name unless num
|
54
54
|
|
55
55
|
zeros_to_add = (3-num.length) > 0 ? (3-num.length) : 0
|
56
56
|
num = "0" * zeros_to_add + num
|
data/lib/mangdown/chapter.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
module Mangdown
|
2
2
|
class Chapter
|
3
3
|
|
4
|
+
include Enumerable
|
4
5
|
attr_reader :name, :uri, :pages, :manga, :chapter
|
5
6
|
|
6
7
|
def initialize(name, uri)
|
@@ -14,42 +15,29 @@ module Mangdown
|
|
14
15
|
get_pages
|
15
16
|
end
|
16
17
|
|
18
|
+
# enumerates through pages
|
19
|
+
def each
|
20
|
+
@pages.each {|page| yield(page) if block_given?}
|
21
|
+
end
|
22
|
+
|
17
23
|
# explicit conversion to chapter
|
18
24
|
def to_chapter
|
19
25
|
self
|
20
26
|
end
|
21
27
|
|
22
|
-
def download
|
23
|
-
warn "#download is depreciated use #download_to"
|
24
|
-
|
25
|
-
dir = File.expand_path(@name)
|
26
|
-
Dir.mkdir(dir) unless Dir.exists?(dir)
|
27
|
-
|
28
|
-
threads = []
|
29
|
-
@pages.each do |page|
|
30
|
-
threads << Thread.new(page) do |this_page|
|
31
|
-
this_page.download_to(dir)
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
threads.each {|thread| thread.join}
|
36
|
-
return @pages.length
|
37
|
-
end
|
38
|
-
|
39
28
|
# download all pages in a chapter
|
40
29
|
def download_to(dir)
|
41
30
|
dir = File.expand_path(dir + '/' + @name)
|
42
31
|
Dir.mkdir(dir) unless Dir.exists?(dir)
|
43
32
|
|
44
33
|
threads = []
|
45
|
-
|
34
|
+
each do |page|
|
46
35
|
threads << Thread.new(page) do |this_page|
|
47
|
-
this_page.download_to(dir)
|
36
|
+
Tools.no_time_out {this_page.download_to(dir)}
|
48
37
|
end
|
49
38
|
end
|
50
39
|
|
51
40
|
threads.each {|thread| thread.join}
|
52
|
-
return @pages.length
|
53
41
|
end
|
54
42
|
|
55
43
|
private
|
@@ -57,22 +45,30 @@ module Mangdown
|
|
57
45
|
# get page objects for all pages in a chapter
|
58
46
|
def get_pages
|
59
47
|
threads = []
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
48
|
+
|
49
|
+
num_pages = get_num_pages(Tools.get_doc(uri))
|
50
|
+
1.upto(num_pages) do |num|
|
51
|
+
threads << Thread.new(num) do |this_num|
|
52
|
+
Tools.no_time_out do
|
53
|
+
tries = 0
|
54
|
+
until doc = get_page_doc(this_num) || tries > 2
|
55
|
+
tries += 1
|
56
|
+
end
|
57
|
+
return unless doc
|
58
|
+
@pages << get_page(doc)
|
59
|
+
end
|
64
60
|
end
|
65
|
-
|
61
|
+
end
|
66
62
|
|
67
63
|
threads.each {|thread| thread.join}
|
68
|
-
return @pages.length
|
69
64
|
end
|
70
65
|
|
71
|
-
|
66
|
+
# get the number of pages in a chapter
|
72
67
|
def get_num_pages(doc)
|
73
68
|
# the select is a dropdown menu of chapter pages
|
74
69
|
doc.css('select')[1].css('option').length
|
75
70
|
end
|
71
|
+
|
76
72
|
end
|
77
73
|
|
78
74
|
# mangareader chapter object
|
@@ -87,10 +83,8 @@ module Mangdown
|
|
87
83
|
page_uri = Mangdown::Uri.new(uri_str).downcase
|
88
84
|
|
89
85
|
Tools.get_doc(page_uri)
|
90
|
-
|
91
|
-
|
92
|
-
# print " uri: #{page_uri} >> "
|
93
|
-
# puts error.message
|
86
|
+
rescue SocketError => error
|
87
|
+
STDERR.puts( "#{error.message} | #{name} | #{num}" )
|
94
88
|
end
|
95
89
|
|
96
90
|
# get the page uri and name
|
@@ -100,7 +94,9 @@ module Mangdown
|
|
100
94
|
MDHash.new(
|
101
95
|
uri: image['src'],
|
102
96
|
name: (image['alt'] + ".jpg")
|
103
|
-
|
97
|
+
).to_page
|
98
|
+
rescue NoMethodError => error
|
99
|
+
puts 'doc was ' + doc.class
|
104
100
|
end
|
105
101
|
end
|
106
102
|
|
@@ -110,13 +106,15 @@ module Mangdown
|
|
110
106
|
|
111
107
|
# get the doc for a given page number
|
112
108
|
def get_page_doc(num)
|
113
|
-
|
109
|
+
Tools.get_doc(
|
114
110
|
Mangdown::Uri.new(
|
115
|
-
@properties.root
|
111
|
+
@properties.root +
|
116
112
|
"/manga/#{@manga.gsub(' ', '_')}/c#{@chapter}/" +
|
117
113
|
"#{num}.html"
|
118
114
|
).downcase
|
119
115
|
)
|
116
|
+
rescue SocketError => error
|
117
|
+
STDERR.puts( "#{error.message} | #{name} | #{num}" )
|
120
118
|
end
|
121
119
|
|
122
120
|
# get the page name and uri
|
@@ -126,7 +124,7 @@ module Mangdown
|
|
126
124
|
MDHash.new(
|
127
125
|
uri: image[:src],
|
128
126
|
name: image[:src].sub(/.+\//, '')
|
129
|
-
|
127
|
+
).to_page
|
130
128
|
end
|
131
129
|
|
132
130
|
# get the number of pages
|
data/lib/mangdown/commands.rb
CHANGED
@@ -41,13 +41,6 @@ module M
|
|
41
41
|
|
42
42
|
search_result
|
43
43
|
end
|
44
|
-
|
45
|
-
# download a manga (accepts MDHash) between the chapters given
|
46
|
-
# by default the entire manga will be downloaded
|
47
|
-
def download(manga, first = 0, last = -1)
|
48
|
-
manga = manga.to_manga
|
49
|
-
Tools::Downloader.new(manga, first, last)
|
50
|
-
end
|
51
44
|
|
52
45
|
# cbz all subdirectories in a directory
|
53
46
|
def cbz(dir)
|
@@ -56,8 +49,9 @@ module M
|
|
56
49
|
|
57
50
|
# display help file
|
58
51
|
def help
|
59
|
-
help_file = File.expand_path(
|
60
|
-
|
52
|
+
help_file = File.expand_path(
|
53
|
+
'../../doc/help.txt', File.dirname(__FILE__)
|
54
|
+
)
|
61
55
|
puts File.open(help_file, 'r').read
|
62
56
|
end
|
63
57
|
|
data/lib/mangdown/manga.rb
CHANGED
@@ -1,59 +1,73 @@
|
|
1
1
|
module Mangdown
|
2
2
|
|
3
|
+
DOWNLOAD_DIR ||= Dir.home + '/manga'
|
4
|
+
|
3
5
|
# mangdown manga object, which holds chapters
|
4
6
|
class Manga
|
5
7
|
|
6
|
-
|
8
|
+
include Enumerable
|
9
|
+
attr_reader :name, :uri, :chapters, :enum
|
7
10
|
|
8
11
|
def initialize(name, uri)
|
9
12
|
@name = name
|
10
13
|
@uri = uri
|
11
|
-
|
12
|
-
#Keeping these chapter objects in memory could be expensive
|
13
14
|
@chapters = []
|
14
|
-
@chapters_list = []
|
15
15
|
|
16
|
-
|
16
|
+
get_chapters
|
17
17
|
end
|
18
18
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
uri: (root + chapter[:href].sub(root, '')),
|
34
|
-
name: chapter.text,
|
35
|
-
)
|
19
|
+
public
|
20
|
+
# download to current directory convenience method
|
21
|
+
def download(*args)
|
22
|
+
download_to(DOWNLOAD_DIR,*args)
|
23
|
+
end
|
24
|
+
|
25
|
+
# download using enumerable
|
26
|
+
def download_to(dir, start = 0, stop = -1)
|
27
|
+
dir += "/#{name}"
|
28
|
+
reset(start, stop)
|
29
|
+
Dir.mkdir(dir) unless Dir.exist?(dir)
|
30
|
+
loop do
|
31
|
+
self.next.to_chapter.download_to(dir)
|
32
|
+
end
|
36
33
|
end
|
37
34
|
|
38
|
-
|
39
|
-
|
35
|
+
# explicit conversion to manga
|
36
|
+
def to_manga
|
37
|
+
self
|
38
|
+
end
|
40
39
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
40
|
+
# each for enumerating through chapters
|
41
|
+
def each
|
42
|
+
@chapters.each {|chapter| yield(chapter) if block_given?}
|
43
|
+
end
|
44
|
+
|
45
|
+
# go through the chapters one at a time
|
46
|
+
def next
|
47
|
+
@enum || reset
|
48
|
+
@enum.next
|
45
49
|
end
|
46
|
-
end
|
47
50
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
+
# reset enum for next
|
52
|
+
def reset(start = 0, stop = -1)
|
53
|
+
@enum = each[start..stop].lazy
|
54
|
+
end
|
55
|
+
|
56
|
+
# get push MDHashes of manga chapters to @chapters
|
57
|
+
def get_chapters
|
58
|
+
properties = Properties.new(@uri)
|
59
|
+
doc = Tools.get_doc(@uri)
|
60
|
+
root = properties.root
|
51
61
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
62
|
+
#get the link with chapter name and uri
|
63
|
+
doc.css(properties.manga_css_klass).each do |chapter|
|
64
|
+
@chapters << MDHash.new(
|
65
|
+
uri: (root + chapter[:href].sub(root, '')),
|
66
|
+
name: chapter.text,
|
67
|
+
)
|
68
|
+
end
|
69
|
+
|
70
|
+
@chapters.reverse! if properties.reverse
|
56
71
|
end
|
57
|
-
end
|
58
72
|
end
|
59
73
|
end
|
data/lib/mangdown/page.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
class Mangdown::Page
|
2
2
|
|
3
3
|
include Mangdown
|
4
|
+
include Comparable
|
4
5
|
|
5
6
|
attr_reader :name, :uri
|
6
7
|
|
@@ -9,27 +10,24 @@ class Mangdown::Page
|
|
9
10
|
@uri = Mangdown::Uri.new(uri)
|
10
11
|
end
|
11
12
|
|
13
|
+
# space ship operator for sorting
|
14
|
+
def <=>(other)
|
15
|
+
self.name <=> other.name
|
16
|
+
end
|
17
|
+
|
12
18
|
# explicit conversion to page
|
13
19
|
def to_page
|
14
20
|
self
|
15
21
|
end
|
16
22
|
|
17
|
-
# download the page
|
18
|
-
def download
|
19
|
-
warn "#download is depreciated use #download_to"
|
20
|
-
return nil if File.exist?(@name)
|
21
|
-
File.open(@name, 'wb') do |file|
|
22
|
-
file.write(open(uri).read)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
23
|
# downloads to specified directory
|
27
|
-
def download_to(dir)
|
28
|
-
path = dir + '/' +
|
24
|
+
def download_to(dir = Dir.pwd)
|
25
|
+
path = dir + '/' + name
|
29
26
|
# don't download again
|
30
|
-
return
|
31
|
-
|
32
|
-
|
33
|
-
|
27
|
+
return if File.exist?(path)
|
28
|
+
image = open(uri).read
|
29
|
+
File.open(path, 'wb') {|file| file.write(image)}
|
30
|
+
rescue SocketError => error
|
31
|
+
STDERR.puts( "#{error.message} | #{name} | #{uri}" )
|
34
32
|
end
|
35
33
|
end
|
data/lib/mangdown/popular.rb
CHANGED
@@ -1,45 +1,32 @@
|
|
1
1
|
module Mangdown
|
2
2
|
class PopularManga
|
3
3
|
|
4
|
-
|
4
|
+
include Enumerable
|
5
|
+
attr_reader :uri, :mangas, :name
|
5
6
|
|
6
|
-
def initialize(uri, num_mangas =
|
7
|
-
@uri
|
7
|
+
def initialize(uri, num_mangas = 30, name = "My Pop Manga")
|
8
|
+
@uri = uri
|
8
9
|
@num_mangas = num_mangas
|
9
|
-
@name
|
10
|
-
|
11
|
-
@mangas_list = []
|
12
|
-
|
13
|
-
#Depreciated
|
14
|
-
@mangas = []
|
10
|
+
@name = name
|
11
|
+
@mangas = []
|
15
12
|
|
16
13
|
get_mangas_list
|
17
14
|
end
|
18
15
|
|
19
|
-
#
|
20
|
-
def
|
21
|
-
|
22
|
-
manga = @mangas_list[number - 1]
|
23
|
-
|
24
|
-
unless @mangas.find {|mnga| (mnga.name == manga[:name]) or
|
25
|
-
(mnga.uri == manga[:uri])}
|
26
|
-
@mangas << manga.to_manga
|
27
|
-
else
|
28
|
-
puts "This manga has already been added.."
|
29
|
-
end
|
16
|
+
# iterate over mangas (for enumerable)
|
17
|
+
def each
|
18
|
+
@mangas.each {|manga| yield(manga) if block_given?}
|
30
19
|
end
|
31
20
|
|
32
21
|
private
|
33
22
|
def get_mangas_list
|
34
23
|
(@num_mangas / 30.0).ceil.times do |time|
|
35
|
-
get_pop_page_manga(time)
|
36
|
-
@mangas_list << manga
|
37
|
-
end
|
24
|
+
@mangas += get_pop_page_manga(time)
|
38
25
|
end
|
39
26
|
end
|
40
27
|
|
41
28
|
def get_pop_page_manga(time)
|
42
|
-
root = Tools.get_root(
|
29
|
+
root = Tools.get_root(uri)
|
43
30
|
|
44
31
|
num = 30 * (time)
|
45
32
|
page = root + '/popular/' + num.to_s
|
data/lib/mangdown/tools.rb
CHANGED
@@ -4,16 +4,7 @@ module Mangdown
|
|
4
4
|
module Tools
|
5
5
|
extend self
|
6
6
|
|
7
|
-
def
|
8
|
-
start = Dir.pwd
|
9
|
-
yield
|
10
|
-
Dir.chdir(start)
|
11
|
-
rescue Exception => error
|
12
|
-
Dir.chdir(start)
|
13
|
-
raise error
|
14
|
-
end
|
15
|
-
|
16
|
-
def get_doc(uri)
|
7
|
+
def get_doc(uri)
|
17
8
|
@doc = ::Nokogiri::HTML(open(uri))
|
18
9
|
end
|
19
10
|
|
@@ -22,69 +13,9 @@ module Mangdown
|
|
22
13
|
end
|
23
14
|
|
24
15
|
def no_time_out(tries = 3, &block)
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
rescue Timeout::Error
|
29
|
-
tries >= 0 ? no_time_out(tries, &block) : :timed_out
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
class Downloader
|
34
|
-
include Tools
|
35
|
-
|
36
|
-
def initialize(manga, first, last)
|
37
|
-
@tempfile = File.expand_path("#{manga.name}_temp.yml")
|
38
|
-
|
39
|
-
chapters = slow_get_chapters(manga, first, last)
|
40
|
-
slow_dl_chapters(chapters)
|
41
|
-
end
|
42
|
-
|
43
|
-
def slow_get_chapters(manga, bgn, nd)
|
44
|
-
# get Manga object from file if it exists
|
45
|
-
if File.exist?(@tempfile)
|
46
|
-
manga_from_file = YAML
|
47
|
-
.load(File.open(@tempfile, 'r').read)
|
48
|
-
end
|
49
|
-
|
50
|
-
# if the manga is from tempfile and the manga has chapters
|
51
|
-
# then check if it has the right chapters
|
52
|
-
if manga_from_file and (manga_from_file.chapters.length > 0)
|
53
|
-
frst = (manga.chapters_list[bgn][:name] ==
|
54
|
-
manga_from_file.chapters.first.name)
|
55
|
-
lst = (manga.chapters_list[nd][:name] ==
|
56
|
-
manga_from_file.chapters.last.name)
|
57
|
-
manga = manga_from_file if (frst and lst)
|
58
|
-
else
|
59
|
-
threads = []
|
60
|
-
manga.chapters_list[bgn..nd].each_index do |index|
|
61
|
-
threads << Thread.new(index) do |i|
|
62
|
-
no_time_out {manga.get_chapter(i + bgn)}
|
63
|
-
end
|
64
|
-
end
|
65
|
-
threads.each {|thread| thread.join}
|
66
|
-
end
|
67
|
-
|
68
|
-
File.open(@tempfile, 'w') {|f| f.write(manga.to_yaml)}
|
69
|
-
|
70
|
-
manga
|
71
|
-
end
|
72
|
-
|
73
|
-
def slow_dl_chapters(manga)
|
74
|
-
dir = File.expand_path(manga.name)
|
75
|
-
Dir.mkdir(dir) unless Dir.exist?(dir)
|
76
|
-
|
77
|
-
threads = []
|
78
|
-
manga.chapters.each do |chap|
|
79
|
-
threads << Thread.new(chap) do |this_chap|
|
80
|
-
no_time_out {this_chap.download_to(dir)}
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
threads.each {|thread| thread.join}
|
85
|
-
|
86
|
-
File.delete(@tempfile)
|
87
|
-
end
|
16
|
+
timeout(30) { yield }
|
17
|
+
rescue Timeout::Error
|
18
|
+
retry if (tries -= 1) >= 0
|
88
19
|
end
|
89
20
|
end
|
90
21
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mangdown
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- jphager2
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-08-01 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: A gem to download Manga, (pg integration in dev)
|
14
14
|
email: jphager2@gmail.com
|
@@ -73,5 +73,5 @@ rubyforge_project:
|
|
73
73
|
rubygems_version: 2.2.2
|
74
74
|
signing_key:
|
75
75
|
specification_version: 4
|
76
|
-
summary: Downloads Manga
|
76
|
+
summary: Downloads Manga, 0.9.0 has some big API changes
|
77
77
|
test_files: []
|