mangdown 0.20.8 → 0.21.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +15 -10
- data/db/migrate/001_create_manga_table.rb +12 -0
- data/lib/mangdown.rb +68 -20
- data/lib/mangdown/adapter/manga_bat.rb +148 -0
- data/lib/mangdown/adapter/mangareader.rb +119 -91
- data/lib/mangdown/chapter.rb +27 -57
- data/lib/mangdown/client.rb +30 -53
- data/lib/mangdown/db.rb +27 -0
- data/lib/mangdown/db/manga.rb +9 -0
- data/lib/mangdown/manga.rb +25 -29
- data/lib/mangdown/page.rb +17 -13
- data/lib/mangdown/support/cbz.rb +1 -2
- data/lib/mangdown/support/logging.rb +1 -0
- data/lib/mangdown/support/tools.rb +18 -10
- data/lib/mangdown/version.rb +1 -1
- data/test/lib/mangdown/chapter_test.rb +18 -74
- data/test/lib/mangdown/manga_test.rb +11 -115
- data/test/lib/mangdown/support/cbz_test.rb +21 -9
- data/test/lib/mangdown/support/equality_test.rb +9 -2
- data/test/lib/mangdown/support/logging_test.rb +5 -2
- data/test/lib/mangdown/support/tools_test.rb +19 -6
- data/test/lib/mangdown_test.rb +35 -22
- data/test/support/test_adapter.rb +63 -83
- data/test/test_helper.rb +49 -1
- metadata +66 -14
- data/lib/mangdown/adapter.rb +0 -87
- data/lib/mangdown/adapter/proxy.rb +0 -44
- data/lib/mangdown/manga_list.rb +0 -43
- data/lib/mangdown/md_hash.rb +0 -73
- data/lib/mangdown/support/properties.rb +0 -50
- data/test/lib/mangdown/adapter_test.rb +0 -117
- data/test/lib/mangdown/manga_list_test.rb +0 -50
- data/test/lib/mangdown/support/properties_test.rb +0 -59
@@ -9,6 +9,14 @@ module Mangdown
|
|
9
9
|
# Common helpers
|
10
10
|
module Tools
|
11
11
|
class << self
|
12
|
+
# rubocop:disable Metrics/LineLength
|
13
|
+
TYPHOEUS_OPTIONS = {
|
14
|
+
headers: {
|
15
|
+
'User-Agent' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36'
|
16
|
+
}
|
17
|
+
}.freeze
|
18
|
+
# rubocop:enable Metrics/LineLength
|
19
|
+
|
12
20
|
def get_doc(uri)
|
13
21
|
data = get(uri)
|
14
22
|
@doc = Nokogiri::HTML(data)
|
@@ -31,6 +39,12 @@ module Mangdown
|
|
31
39
|
Pathname.new(Dir.pwd).join(*sub_paths)
|
32
40
|
end
|
33
41
|
|
42
|
+
def file_join(safe_path, *unsafe_parts)
|
43
|
+
now_safe_parts = unsafe_parts.map { |part| part.tr('/', '') }
|
44
|
+
|
45
|
+
File.join(safe_path, *now_safe_parts)
|
46
|
+
end
|
47
|
+
|
34
48
|
def valid_path_name(name)
|
35
49
|
name.to_s.sub(/(\d+)(\.\w+)*\Z/) do
|
36
50
|
digits, ext = Regexp.last_match[1..2]
|
@@ -47,6 +61,7 @@ module Mangdown
|
|
47
61
|
mime_to_extension(mime)
|
48
62
|
end
|
49
63
|
|
64
|
+
# rubocop:disable Metrics/MethodLength
|
50
65
|
def hydra_streaming(objects, hydra_opts = {})
|
51
66
|
hydra = Typhoeus::Hydra.new(hydra_opts)
|
52
67
|
|
@@ -56,7 +71,7 @@ module Mangdown
|
|
56
71
|
request = typhoeus(obj.uri)
|
57
72
|
request.on_headers do |response|
|
58
73
|
status = response.success? ? :succeeded : :failed
|
59
|
-
yield(status, obj)
|
74
|
+
yield(status, obj, response)
|
60
75
|
end
|
61
76
|
request.on_body do |chunk|
|
62
77
|
yield(:body, obj, chunk)
|
@@ -72,17 +87,10 @@ module Mangdown
|
|
72
87
|
hydra.run
|
73
88
|
requests
|
74
89
|
end
|
90
|
+
# rubocop:enable Metrics/MethodLength
|
75
91
|
|
76
92
|
def typhoeus(uri)
|
77
|
-
Typhoeus::Request.new(uri,
|
78
|
-
end
|
79
|
-
|
80
|
-
def typhoeus_options
|
81
|
-
{
|
82
|
-
headers: {
|
83
|
-
'User-Agent' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36'
|
84
|
-
}
|
85
|
-
}
|
93
|
+
Typhoeus::Request.new(uri, TYPHOEUS_OPTIONS)
|
86
94
|
end
|
87
95
|
|
88
96
|
private
|
data/lib/mangdown/version.rb
CHANGED
@@ -1,98 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'stringio'
|
2
4
|
require 'test_helper'
|
3
5
|
|
4
6
|
module Mangdown
|
5
7
|
class ChapterTest < Minitest::Test
|
6
|
-
|
7
|
-
TestAdapter.new(:uri, :doc, :name)
|
8
|
-
end
|
9
|
-
|
8
|
+
attr_reader :adapter, :chapter
|
10
9
|
def setup
|
11
|
-
|
12
|
-
@
|
10
|
+
register_test_adapter
|
11
|
+
@adapter = TestAdapter::Chapter.new(url: 'uri', name: 'name', number: 1)
|
12
|
+
@adapter.manga = TestAdapter::Manga.new(url: 'url', name: 'manga')
|
13
|
+
@chapter = Chapter.new(adapter)
|
13
14
|
end
|
14
15
|
|
15
16
|
def test_new
|
16
|
-
|
17
|
-
|
18
|
-
assert_equal
|
19
|
-
assert_equal
|
20
|
-
assert_equal [], @chapter.pages
|
21
|
-
assert_equal 1, @chapter.chapter
|
22
|
-
assert_equal 'adapter', @chapter.adapter
|
17
|
+
assert_equal 'uri', chapter.uri
|
18
|
+
assert_equal 'name', chapter.name
|
19
|
+
assert_equal [], chapter.pages
|
20
|
+
assert_equal 1, chapter.number
|
23
21
|
end
|
24
22
|
|
25
23
|
def test_cbz
|
26
|
-
|
27
|
-
class << self
|
28
|
-
alias old_one one
|
29
|
-
def one(path)
|
30
|
-
path
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
@chapter.instance_variable_set(:@path, 'path')
|
24
|
+
chapter.instance_variable_set(:@path, 'path')
|
36
25
|
|
37
|
-
|
38
|
-
|
39
|
-
CBZ.class_eval do
|
40
|
-
class << self
|
41
|
-
alias one old_one
|
42
|
-
end
|
26
|
+
with_stubbed_cbz do
|
27
|
+
assert_equal %w[cbz_called path], chapter.cbz
|
43
28
|
end
|
44
29
|
end
|
45
30
|
|
46
|
-
def test_to_chapter
|
47
|
-
assert_equal @chapter, @chapter.to_chapter
|
48
|
-
end
|
49
|
-
|
50
31
|
def test_to_path
|
32
|
+
assert_equal(chapter.manga.name, 'manga')
|
51
33
|
assert_equal(
|
52
|
-
"#{Mangdown::DOWNLOAD_DIR}/manga/name",
|
34
|
+
"#{Mangdown::DOWNLOAD_DIR}/manga/name", chapter.to_path.to_s
|
53
35
|
)
|
54
36
|
end
|
55
37
|
|
56
38
|
def test_setup_path
|
57
|
-
|
58
|
-
assert_equal "#{Dir.pwd}/name",
|
59
|
-
end
|
60
|
-
|
61
|
-
def test_load_pages
|
62
|
-
Tools.class_eval do
|
63
|
-
class << self
|
64
|
-
alias old_hydra_streaming hydra_streaming
|
65
|
-
|
66
|
-
def hydra_streaming(objects, *other_args)
|
67
|
-
objects.map do |obj|
|
68
|
-
next unless yield(:before, obj)
|
69
|
-
yield(:succeeded, obj)
|
70
|
-
yield(:body, obj, 'data')
|
71
|
-
yield(:complete, obj)
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
76
|
-
|
77
|
-
@chapter.load_pages
|
78
|
-
|
79
|
-
refute_empty @chapter.pages
|
80
|
-
|
81
|
-
Tools.class_eval do
|
82
|
-
class << self
|
83
|
-
alias hydra_streaming old_hydra_streaming
|
84
|
-
end
|
85
|
-
end
|
86
|
-
end
|
87
|
-
|
88
|
-
def test_each
|
89
|
-
@chapter.pages.concat(Array.new(10, {}))
|
90
|
-
|
91
|
-
assert_equal @chapter.pages.length, @chapter.each.count
|
92
|
-
|
93
|
-
@chapter.each.with_index do |page, i|
|
94
|
-
assert_equal @chapter.pages[i].object_id, page.object_id
|
95
|
-
end
|
39
|
+
chapter.setup_path(Dir.pwd)
|
40
|
+
assert_equal "#{Dir.pwd}/name", chapter.to_path.to_s
|
96
41
|
end
|
97
42
|
|
98
43
|
def test_download_to
|
@@ -100,4 +45,3 @@ module Mangdown
|
|
100
45
|
end
|
101
46
|
end
|
102
47
|
end
|
103
|
-
|
@@ -1,147 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'stringio'
|
2
4
|
require 'test_helper'
|
3
5
|
|
4
6
|
module Mangdown
|
5
7
|
class MangaTest < Minitest::Test
|
6
|
-
|
7
|
-
|
8
|
+
attr_reader :adapter, :manga
|
9
|
+
|
10
|
+
def setup
|
11
|
+
register_test_adapter
|
12
|
+
@adapter = TestAdapter::Manga.new(url: 'uri', name: 'name')
|
13
|
+
@manga = Manga.new(adapter)
|
8
14
|
end
|
9
15
|
|
10
16
|
def test_new
|
11
|
-
manga = Manga.new('uri', 'name')
|
12
|
-
manga.adapter = 'adapter'
|
13
|
-
|
14
17
|
assert_equal 'uri', manga.uri
|
15
18
|
assert_equal 'name', manga.name
|
16
19
|
assert_equal [], manga.chapters
|
17
|
-
assert_equal 'adapter', manga.adapter
|
18
20
|
end
|
19
21
|
|
20
22
|
def test_cbz
|
21
|
-
CBZ.class_eval do
|
22
|
-
class << self
|
23
|
-
alias old_all all
|
24
|
-
def all(path)
|
25
|
-
path
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
manga = Manga.new('uri', 'name')
|
31
23
|
manga.instance_variable_set(:@path, 'path')
|
32
24
|
|
33
|
-
|
34
|
-
|
35
|
-
CBZ.class_eval do
|
36
|
-
class << self
|
37
|
-
alias all old_all
|
38
|
-
end
|
25
|
+
with_stubbed_cbz do
|
26
|
+
assert_equal %w[cbz_called path], manga.cbz
|
39
27
|
end
|
40
28
|
end
|
41
29
|
|
42
|
-
def test_to_manga
|
43
|
-
manga = Manga.new('uri', 'name')
|
44
|
-
|
45
|
-
assert_equal manga, manga.to_manga
|
46
|
-
end
|
47
|
-
|
48
30
|
def test_to_path
|
49
|
-
manga = Manga.new('uri', 'name')
|
50
|
-
|
51
31
|
assert_equal "#{Mangdown::DOWNLOAD_DIR}/name", manga.to_path.to_s
|
52
32
|
end
|
53
33
|
|
54
34
|
def test_setup_path
|
55
|
-
manga = Manga.new('uri', 'name')
|
56
|
-
|
57
35
|
manga.setup_path(Dir.pwd)
|
58
36
|
assert_equal "#{Dir.pwd}/name", manga.to_path.to_s
|
59
37
|
end
|
60
38
|
|
61
|
-
def test_load_chapters
|
62
|
-
manga = Manga.new('uri', 'name')
|
63
|
-
manga.adapter = test_adapter
|
64
|
-
manga.load_chapters
|
65
|
-
|
66
|
-
refute_empty manga.chapters
|
67
|
-
end
|
68
|
-
|
69
|
-
def test_each
|
70
|
-
manga = Manga.new('uri', 'name')
|
71
|
-
manga.chapters.concat(Array.new(10, {}))
|
72
|
-
|
73
|
-
assert_equal manga.chapters.length, manga.each.count
|
74
|
-
|
75
|
-
manga.each.with_index do |chapter, i|
|
76
|
-
assert_equal manga.chapters[i].object_id, chapter.object_id
|
77
|
-
end
|
78
|
-
end
|
79
|
-
|
80
|
-
def test_download_passes_args_to_download_to_with_nil_dir
|
81
|
-
Manga.class_eval do
|
82
|
-
alias old_download_to download_to
|
83
|
-
|
84
|
-
def download_to(*args)
|
85
|
-
args
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
manga = Manga.new('uri', 'name')
|
90
|
-
assert_equal [nil, 1, 2, 3], manga.download(1, 2, 3)
|
91
|
-
|
92
|
-
Manga.class_eval do
|
93
|
-
alias download_to old_download_to
|
94
|
-
end
|
95
|
-
end
|
96
|
-
|
97
39
|
def test_download_to
|
98
|
-
|
99
|
-
@@counter = 0
|
100
|
-
|
101
|
-
alias old_download_to download_to
|
102
|
-
|
103
|
-
def download_to(*args)
|
104
|
-
@@counter += 1
|
105
|
-
|
106
|
-
if @@counter % 3 == 1
|
107
|
-
{ failed: [], succeeded: [:some], skipped: [] }
|
108
|
-
elsif @@counter % 3 == 2
|
109
|
-
{ failed: [], succeeded: [], skipped: [:some] }
|
110
|
-
else
|
111
|
-
{ failed: [:some], succeeded: [], skipped: [] }
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
alias old_fetch_each_page fetch_each_page
|
116
|
-
|
117
|
-
def fetch_each_page(*)
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
io = StringIO.new
|
122
|
-
Mangdown.configure_logger(file: io)
|
123
|
-
|
124
|
-
manga = Manga.new('uri', 'name')
|
125
|
-
manga.adapter = test_adapter
|
126
|
-
manga.load_chapters
|
127
|
-
|
128
|
-
result = manga.download_to(Dir.pwd)
|
129
|
-
|
130
|
-
assert_equal 5, manga.count
|
131
|
-
assert_equal 5, result.values.flatten.length
|
132
|
-
assert_equal 2, result[:succeeded].length
|
133
|
-
assert_equal 2, result[:skipped].length
|
134
|
-
assert_equal 1, result[:failed].length
|
135
|
-
|
136
|
-
assert_equal 1, io.string.scan(/ERROR/).length
|
137
|
-
|
138
|
-
Mangdown.configure_logger(file: STDOUT)
|
139
|
-
|
140
|
-
Chapter.class_eval do
|
141
|
-
alias download_to old_download_to
|
142
|
-
alias fetch_each_page old_fetch_each_page
|
143
|
-
end
|
40
|
+
skip
|
144
41
|
end
|
145
42
|
end
|
146
43
|
end
|
147
|
-
|
@@ -1,30 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'test_helper'
|
2
4
|
require 'fileutils'
|
3
5
|
|
4
6
|
module Mangdown
|
5
7
|
class CBZTest < Minitest::Test
|
8
|
+
# rubocop:disable Metrics/AbcSize
|
9
|
+
# rubocop:disable Metrics/MethodLength
|
6
10
|
def setup
|
7
11
|
@temp = Pathname.new(__dir__).join('tmp')
|
8
12
|
@root_dir = @temp.join('root 1')
|
9
|
-
@
|
10
|
-
@
|
13
|
+
@sub_dir1 = @root_dir.join('sub_dir 1')
|
14
|
+
@sub_dir2 = @root_dir.join('sub_dir 2')
|
11
15
|
|
12
16
|
Dir.mkdir(@temp)
|
13
17
|
Dir.mkdir(@root_dir)
|
14
|
-
Dir.mkdir(@
|
15
|
-
Dir.mkdir(@
|
18
|
+
Dir.mkdir(@sub_dir1)
|
19
|
+
Dir.mkdir(@sub_dir2)
|
16
20
|
|
17
21
|
(1..9).each do |i|
|
18
22
|
filename = "file #{i}.ext"
|
19
|
-
FileUtils.touch(@
|
20
|
-
FileUtils.touch(@
|
23
|
+
FileUtils.touch(@sub_dir1.join(filename))
|
24
|
+
FileUtils.touch(@sub_dir2.join(filename))
|
21
25
|
end
|
22
26
|
end
|
27
|
+
# rubocop:enable Metrics/AbcSize
|
28
|
+
# rubocop:enable Metrics/MethodLength
|
23
29
|
|
24
30
|
def teardown
|
25
31
|
FileUtils.rm_r(@temp)
|
26
32
|
end
|
27
33
|
|
34
|
+
# rubocop:disable Metrics/MethodLength
|
28
35
|
def test_all
|
29
36
|
CBZ.all(@root_dir)
|
30
37
|
|
@@ -45,13 +52,16 @@ module Mangdown
|
|
45
52
|
|
46
53
|
assert_equal 9, count
|
47
54
|
end
|
55
|
+
# rubocop:enable Metrics/MethodLength
|
48
56
|
|
57
|
+
# rubocop:disable Metrics/AbcSize
|
58
|
+
# rubocop:disable Metrics/MethodLength
|
49
59
|
def test_one
|
50
|
-
CBZ.one(@
|
60
|
+
CBZ.one(@sub_dir1)
|
51
61
|
|
52
62
|
list = Dir.new(@root_dir).entries
|
53
|
-
refute File.exist?(@
|
54
|
-
assert File.exist?(@
|
63
|
+
refute File.exist?(@sub_dir1), list.inspect
|
64
|
+
assert File.exist?(@sub_dir2), 'Should only validate the correct dir'
|
55
65
|
|
56
66
|
assert_includes list, 'sub_dir 00001'
|
57
67
|
assert_includes list, 'sub_dir 00001.cbz'
|
@@ -66,5 +76,7 @@ module Mangdown
|
|
66
76
|
|
67
77
|
assert_equal 9, count
|
68
78
|
end
|
79
|
+
# rubocop:enable Metrics/MethodLength
|
80
|
+
# rubocop:enable Metrics/AbcSize
|
69
81
|
end
|
70
82
|
end
|
@@ -1,7 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'test_helper'
|
2
4
|
|
3
5
|
module Mangdown
|
4
6
|
class EqualityTest < Minitest::Test
|
7
|
+
# rubocop:disable Metrics/MethodLength
|
8
|
+
# rubocop:disable Metrics/AbcSize
|
5
9
|
def test_spaceship_operator
|
6
10
|
name_struct = Struct.new(:name) do
|
7
11
|
include Equality
|
@@ -12,13 +16,16 @@ module Mangdown
|
|
12
16
|
assert name_struct.new('Name') < name_struct.new('Other Name')
|
13
17
|
assert name_struct.new('Other Name') > name_struct.new('Name')
|
14
18
|
|
19
|
+
# rubocop:disable Lint/UselessComparison
|
15
20
|
assert_equal 0, name_struct.new('Name') <=> name_struct.new('Name')
|
21
|
+
# rubocop:enable Lint/UselessComparison
|
16
22
|
|
17
|
-
names = %w
|
23
|
+
names = %w[a b c d e f].map { |name| name_struct.new(name) }
|
18
24
|
|
19
25
|
assert_equal names, names.shuffle.sort
|
20
26
|
assert_equal names.reverse, names.shuffle.sort.reverse
|
21
27
|
end
|
28
|
+
# rubocop:enable Metrics/MethodLength
|
29
|
+
# rubocop:enable Metrics/AbcSize
|
22
30
|
end
|
23
31
|
end
|
24
|
-
|