narou 3.2.5.1 → 3.3.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of narou might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.gitignore +2 -5
- data/.haml-lint.yml +7 -0
- data/.rubocop.yml +23 -5
- data/.scss-lint.yml +9 -0
- data/ChangeLog.md +86 -0
- data/Gemfile.lock +35 -35
- data/README.md +80 -64
- data/lib/backtracer.rb +2 -2
- data/lib/color.rb +5 -1
- data/lib/command.rb +7 -2
- data/lib/command/alias.rb +3 -5
- data/lib/command/backup.rb +3 -5
- data/lib/command/browser.rb +3 -5
- data/lib/command/clean.rb +5 -1
- data/lib/command/console.rb +33 -0
- data/lib/command/convert.rb +143 -117
- data/lib/command/csv.rb +2 -1
- data/lib/command/diff.rb +20 -18
- data/lib/command/download.rb +25 -14
- data/lib/command/folder.rb +3 -5
- data/lib/command/freeze.rb +3 -5
- data/lib/command/help.rb +20 -18
- data/lib/command/init.rb +4 -3
- data/lib/command/inspect.rb +2 -1
- data/lib/command/list.rb +10 -8
- data/lib/command/list/novel_decorator.rb +2 -1
- data/lib/command/log.rb +100 -0
- data/lib/command/log/tail.rb +76 -0
- data/lib/command/mail.rb +20 -17
- data/lib/command/remove.rb +7 -6
- data/lib/command/send.rb +23 -20
- data/lib/command/setting.rb +74 -40
- data/lib/command/tag.rb +16 -15
- data/lib/command/trace.rb +2 -2
- data/lib/command/update.rb +78 -128
- data/lib/command/update/general_lastup_updater.rb +3 -2
- data/lib/command/update/hotentry_manager.rb +2 -1
- data/lib/command/update/interval.rb +2 -1
- data/lib/command/version.rb +2 -1
- data/lib/command/web.rb +17 -3
- data/lib/commandbase.rb +34 -7
- data/lib/commandline.rb +54 -35
- data/lib/converterbase.rb +21 -15
- data/lib/database.rb +3 -2
- data/lib/device.rb +5 -4
- data/lib/device/epub.rb +2 -1
- data/lib/device/ibooks.rb +2 -1
- data/lib/device/ibunko.rb +2 -1
- data/lib/device/kindle.rb +2 -1
- data/lib/device/kobo.rb +2 -1
- data/lib/device/library/cygwin.rb +2 -1
- data/lib/device/library/linux.rb +2 -1
- data/lib/device/library/mac.rb +2 -1
- data/lib/device/library/windows.rb +2 -1
- data/lib/device/library/windows/eject.rb +2 -1
- data/lib/device/reader.rb +2 -1
- data/lib/diffviewer.rb +8 -11
- data/lib/downloader.rb +159 -151
- data/lib/eventable.rb +2 -1
- data/lib/extension.rb +16 -14
- data/lib/extensions/jruby.rb +2 -1
- data/lib/extensions/monkey_patches.rb +7 -0
- data/lib/extensions/monkey_patches/pathname.rb +22 -0
- data/lib/extensions/windows.rb +2 -1
- data/lib/extensions/windows_write_color.rb +2 -1
- data/lib/helper.rb +35 -20
- data/lib/html.rb +2 -1
- data/lib/illustration.rb +2 -1
- data/lib/ini.rb +2 -1
- data/lib/input.rb +2 -1
- data/lib/inspector.rb +3 -2
- data/lib/inventory.rb +3 -3
- data/lib/mailer.rb +3 -2
- data/lib/mixin/all.rb +8 -0
- data/lib/mixin/locker.rb +40 -0
- data/lib/mixin/output_error.rb +28 -0
- data/lib/narou.rb +69 -51
- data/lib/narou/api.rb +2 -4
- data/lib/narou_logger.rb +236 -108
- data/lib/novelconverter.rb +77 -69
- data/lib/novelinfo.rb +4 -2
- data/lib/novelsetting.rb +15 -12
- data/lib/progressbar.rb +13 -9
- data/lib/sitesetting.rb +39 -18
- data/lib/template.rb +5 -4
- data/lib/version.rb +3 -2
- data/lib/web/all.rb +2 -1
- data/lib/web/appserver.rb +83 -65
- data/lib/web/helper4web.rb +10 -5
- data/lib/web/progressbar4web.rb +8 -4
- data/lib/web/public/resources/default-style.css +2 -3
- data/lib/web/public/resources/narou.library.js +86 -60
- data/lib/web/public/resources/narou.queue.js +24 -30
- data/lib/web/public/resources/narou.ui.js +22 -3
- data/lib/web/public/theme/Cerulean/style.css +5 -5
- data/lib/web/public/theme/Darkly/style.css +5 -5
- data/lib/web/public/theme/Readable/style.css +5 -5
- data/lib/web/public/theme/Slate/style.css +2 -3
- data/lib/web/public/theme/Superhero/style.css +2 -3
- data/lib/web/public/theme/United/style.css +5 -5
- data/lib/web/pushserver.rb +10 -7
- data/lib/web/server_helpers.rb +16 -1
- data/lib/web/settingmessages.rb +10 -7
- data/lib/web/streaminginput.rb +2 -1
- data/lib/web/streaminglogger.rb +45 -32
- data/lib/web/views/_about.haml +6 -3
- data/lib/web/views/_header.haml +2 -3
- data/lib/web/views/_move_to_top.haml +2 -0
- data/lib/web/views/_queue.haml +7 -0
- data/lib/web/views/bookmarklet/insert_button.js.erb +1 -1
- data/lib/web/views/index.haml +30 -27
- data/lib/web/views/layout.haml +2 -0
- data/lib/web/views/novels/setting.haml +3 -4
- data/lib/web/views/settings.haml +22 -8
- data/lib/web/views/style.scss +54 -3
- data/lib/web/views/widget/download.haml +9 -3
- data/lib/web/views/widget/drag_and_drop.haml +10 -4
- data/lib/web/web_worker.rb +132 -0
- data/lib/worker.rb +142 -0
- data/narou.gemspec +80 -45
- data/narou.rb +6 -4
- data/template/novel.txt.erb +1 -0
- data/webnovel/kakuyomu.jp.yaml +9 -13
- data/webnovel/ncode.syosetu.com.yaml +3 -1
- data/webnovel/novel18.syosetu.com.yaml +8 -1
- data/webnovel/syosetu.org.yaml +3 -1
- data/webnovel/www.akatsuki-novels.com.yaml +4 -2
- data/webnovel/www.mai-net.net.yaml +3 -1
- metadata +109 -48
- data/lib/web/worker.rb +0 -126
data/lib/downloader.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
#
|
3
4
|
# Copyright 2013 whiteleaf. All rights reserved.
|
4
5
|
#
|
@@ -21,14 +22,15 @@ require_relative "input"
|
|
21
22
|
#
|
22
23
|
class Downloader
|
23
24
|
include Narou::Eventable
|
25
|
+
extend Memoist
|
24
26
|
|
25
27
|
SECTION_SAVE_DIR_NAME = "本文" # 本文を保存するディレクトリ名
|
26
28
|
CACHE_SAVE_DIR_NAME = "cache" # 差分用キャッシュ保存用ディレクトリ名
|
27
29
|
RAW_DATA_DIR_NAME = "raw" # 本文の生データを保存するディレクトリ名
|
28
30
|
TOC_FILE_NAME = "toc.yaml"
|
29
31
|
STEPS_WAIT_TIME = 5 # 数話ごとにかかるwaitの秒数
|
30
|
-
|
31
|
-
|
32
|
+
WAIT_TIME_TO_RETRY_NETWORK = 10 # タイムアウト等でリトライするまでの待機時間
|
33
|
+
LIMIT_TO_RETRY_NETWORK = 5 # タイムアウト等でリトライする回数上限
|
32
34
|
NOVEL_TYPE_SERIES = 1 # 連載
|
33
35
|
NOVEL_TYPE_SS = 2 # 短編
|
34
36
|
DISPLAY_LIMIT_DIGITS = 4 # indexの表示桁数限界
|
@@ -37,6 +39,7 @@ class Downloader
|
|
37
39
|
attr_reader :id, :setting
|
38
40
|
|
39
41
|
class InvalidTarget < StandardError; end
|
42
|
+
class SuspendDownload < StandardError; end
|
40
43
|
|
41
44
|
def initialize(target, options = {})
|
42
45
|
id = Downloader.get_id_by_target(target)
|
@@ -76,7 +79,7 @@ class Downloader
|
|
76
79
|
# 本文格納用ディレクトリを取得
|
77
80
|
#
|
78
81
|
def self.get_novel_section_save_dir(archive_path)
|
79
|
-
File.join(archive_path, SECTION_SAVE_DIR_NAME)
|
82
|
+
Pathname(File.join(archive_path, SECTION_SAVE_DIR_NAME))
|
80
83
|
end
|
81
84
|
|
82
85
|
#
|
@@ -107,16 +110,13 @@ class Downloader
|
|
107
110
|
file_title = data["file_title"] || data["title"] # 互換性維持のための処理
|
108
111
|
use_subdirectory = data["use_subdirectory"] || false
|
109
112
|
subdirectory = use_subdirectory ? create_subdirecotry_name(file_title) : ""
|
110
|
-
path =
|
111
|
-
if
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
"保存フォルダが消去されていたため、データベースのインデックスを削除しました。"
|
118
|
-
return nil
|
119
|
-
end
|
113
|
+
path = Database.archive_root_path.join(data["sitename"], subdirectory, file_title)
|
114
|
+
return path if path.exist?
|
115
|
+
@@database.delete(id)
|
116
|
+
@@database.save_database
|
117
|
+
error "#{path} が見つかりません。\n" \
|
118
|
+
"保存フォルダが消去されていたため、データベースのインデックスを削除しました。"
|
119
|
+
nil
|
120
120
|
end
|
121
121
|
|
122
122
|
#
|
@@ -206,7 +206,7 @@ class Downloader
|
|
206
206
|
puts "#{data_dir} を完全に削除しました"
|
207
207
|
else
|
208
208
|
# TOCは消しておかないと再DL時に古いデータがあると誤認する
|
209
|
-
|
209
|
+
data_dir.join(TOC_FILE_NAME).delete
|
210
210
|
end
|
211
211
|
@@database.delete(data["id"])
|
212
212
|
@@database.save_database
|
@@ -219,7 +219,7 @@ class Downloader
|
|
219
219
|
def self.get_cache_root_dir(target)
|
220
220
|
dir = get_novel_data_dir_by_target(target)
|
221
221
|
if dir
|
222
|
-
return
|
222
|
+
return dir.join(SECTION_SAVE_DIR_NAME, CACHE_SAVE_DIR_NAME)
|
223
223
|
end
|
224
224
|
nil
|
225
225
|
end
|
@@ -251,18 +251,15 @@ class Downloader
|
|
251
251
|
# 変数初期化
|
252
252
|
#
|
253
253
|
def initialize_variables(id, setting, options)
|
254
|
+
@id = id || database.create_new_id
|
254
255
|
@title = nil
|
255
|
-
@file_title = nil
|
256
256
|
@setting = setting
|
257
257
|
@force = options[:force]
|
258
|
-
@from_download = options[:from_download]
|
259
258
|
@stream = options[:stream]
|
260
259
|
@cache_dir = nil
|
261
260
|
@new_arrivals = false
|
262
|
-
@
|
263
|
-
@
|
264
|
-
@id = id || @@database.create_new_id
|
265
|
-
@new_novel = @@database[@id].!
|
261
|
+
@new_novel = record.!
|
262
|
+
@from_download = options[:from_download]
|
266
263
|
@section_download_cache = {}
|
267
264
|
@download_wait_steps = Inventory.load("local_setting")["download.wait-steps"] || 0
|
268
265
|
@download_use_subdirectory = use_subdirectory?
|
@@ -275,6 +272,14 @@ class Downloader
|
|
275
272
|
initialize_wait_counter
|
276
273
|
end
|
277
274
|
|
275
|
+
def database
|
276
|
+
@@database
|
277
|
+
end
|
278
|
+
|
279
|
+
def record
|
280
|
+
database[@id]
|
281
|
+
end
|
282
|
+
|
278
283
|
#
|
279
284
|
# ウェイト関係初期化
|
280
285
|
#
|
@@ -299,7 +304,7 @@ class Downloader
|
|
299
304
|
Inventory.load("local_setting")["download.use-subdirectory"] || false
|
300
305
|
else
|
301
306
|
# すでにDL済みの小説
|
302
|
-
|
307
|
+
record["use_subdirectory"] || false
|
303
308
|
end
|
304
309
|
end
|
305
310
|
|
@@ -382,14 +387,9 @@ class Downloader
|
|
382
387
|
case
|
383
388
|
when update_subtitles.size > 0
|
384
389
|
@cache_dir = create_cache_dir if old_toc.length > 0
|
385
|
-
|
386
|
-
|
387
|
-
if @cache_dir && Dir.glob(File.join(@cache_dir, "*")).count == 0
|
388
|
-
remove_cache_dir
|
389
|
-
end
|
390
|
-
rescue Interrupt
|
390
|
+
sections_download_and_save(update_subtitles)
|
391
|
+
if @cache_dir && @cache_dir.glob("*").count == 0
|
391
392
|
remove_cache_dir
|
392
|
-
raise
|
393
393
|
end
|
394
394
|
update_database
|
395
395
|
:ok
|
@@ -414,31 +414,34 @@ class Downloader
|
|
414
414
|
:none
|
415
415
|
end
|
416
416
|
|
417
|
-
|
417
|
+
record["general_all_no"] = latest_toc_subtitles.size
|
418
418
|
|
419
|
-
|
420
|
-
tags = @new_novel ? [] :
|
421
|
-
|
419
|
+
save_toc_once(latest_toc)
|
420
|
+
tags = @new_novel ? [] : record["tags"] || []
|
421
|
+
case novel_end?
|
422
|
+
when true
|
422
423
|
unless tags.include?("end")
|
423
424
|
update_database if update_subtitles.count == 0
|
424
|
-
|
425
|
-
Command::Tag.execute!(%W(#{id} --add end --color white --no-overwrite-color))
|
426
|
-
end
|
425
|
+
Command::Tag.execute!(%W(#{id} --add end --color white --no-overwrite-color), io: Narou::NullIO.new)
|
427
426
|
msg = old_toc.empty? ? "完結しているようです" : "完結したようです"
|
428
427
|
@stream.puts "<cyan>#{id_and_title.escape} は#{msg}</cyan>".termcolor
|
429
428
|
return_status = :ok
|
430
429
|
end
|
431
|
-
|
430
|
+
when false
|
432
431
|
if tags.include?("end")
|
433
432
|
update_database if update_subtitles.size == 0
|
434
|
-
|
435
|
-
Command::Tag.execute!(@id, "--delete", "end")
|
436
|
-
end
|
433
|
+
Command::Tag.execute!(@id, "--delete", "end", io: Narou::NullIO.new)
|
437
434
|
@stream.puts "<cyan>#{id_and_title.escape} は連載を再開したようです</cyan>".termcolor
|
438
435
|
return_status = :ok
|
439
436
|
end
|
440
437
|
end
|
441
438
|
return_status
|
439
|
+
rescue Interrupt, SuspendDownload
|
440
|
+
if latest_toc.present?
|
441
|
+
save_toc_once(latest_toc)
|
442
|
+
update_database(suspend: true)
|
443
|
+
end
|
444
|
+
raise Interrupt
|
442
445
|
ensure
|
443
446
|
@setting.clear
|
444
447
|
end
|
@@ -518,7 +521,7 @@ class Downloader
|
|
518
521
|
when "7"
|
519
522
|
Helper.open_directory(Downloader.get_novel_data_dir_by_target(latest_toc["toc_url"]))
|
520
523
|
when "8"
|
521
|
-
Command::Convert.execute!(latest_toc["toc_url"])
|
524
|
+
Command::Convert.execute!(latest_toc["toc_url"], sync: true)
|
522
525
|
end
|
523
526
|
unless Narou.web?
|
524
527
|
message = "" # 長いので二度は表示しない
|
@@ -536,7 +539,7 @@ class Downloader
|
|
536
539
|
return nil if @nosave_diff
|
537
540
|
now = Time.now
|
538
541
|
name = now.strftime("%Y.%m.%d@%H.%M.%S")
|
539
|
-
cache_dir =
|
542
|
+
cache_dir = get_novel_data_dir.join(SECTION_SAVE_DIR_NAME, CACHE_SAVE_DIR_NAME, name)
|
540
543
|
FileUtils.mkdir_p(cache_dir)
|
541
544
|
cache_dir
|
542
545
|
end
|
@@ -602,7 +605,7 @@ class Downloader
|
|
602
605
|
#
|
603
606
|
# データベース更新
|
604
607
|
#
|
605
|
-
def update_database
|
608
|
+
def update_database(suspend: false)
|
606
609
|
info = @setting["info"] || {}
|
607
610
|
data = {
|
608
611
|
"id" => @id,
|
@@ -610,50 +613,58 @@ class Downloader
|
|
610
613
|
"title" => get_title,
|
611
614
|
"file_title" => get_file_title,
|
612
615
|
"toc_url" => @setting["toc_url"],
|
613
|
-
"sitename" =>
|
616
|
+
"sitename" => sitename,
|
614
617
|
"novel_type" => get_novel_type,
|
615
618
|
"end" => novel_end?,
|
616
619
|
"last_update" => Time.now,
|
617
|
-
"new_arrivals_date" => (@new_arrivals ? Time.now :
|
620
|
+
"new_arrivals_date" => (@new_arrivals ? Time.now : record["new_arrivals_date"]),
|
618
621
|
"use_subdirectory" => @download_use_subdirectory,
|
619
622
|
"general_firstup" => info["general_firstup"],
|
620
623
|
"novelupdated_at" => get_novelupdated_at,
|
621
624
|
"general_lastup" => get_general_lastup,
|
622
625
|
"length" => novel_length,
|
626
|
+
"suspend" => suspend
|
623
627
|
}
|
624
|
-
if
|
625
|
-
|
628
|
+
if record
|
629
|
+
database[@id].merge!(data)
|
626
630
|
else
|
627
|
-
|
631
|
+
database[@id] = data
|
628
632
|
end
|
629
|
-
|
633
|
+
database.save_database
|
630
634
|
end
|
631
635
|
|
632
636
|
def get_novel_status
|
633
|
-
novel_status = NovelInfo.load(@setting, of: "nt-e")
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
end
|
637
|
+
novel_status = NovelInfo.load(@setting, of: "nt-e-sitename")
|
638
|
+
novel_status ||= {
|
639
|
+
"novel_type" => NOVEL_TYPE_SERIES,
|
640
|
+
"end" => nil, # nil で完結状態が定義されていなかったことを示す(扱いとしては未完結と同じ)
|
641
|
+
"sitename" => @setting["sitename"]
|
642
|
+
}
|
640
643
|
novel_status
|
641
644
|
end
|
645
|
+
memoize :get_novel_status
|
642
646
|
|
643
647
|
#
|
644
648
|
# 小説の種別を取得(連載か短編)
|
645
649
|
#
|
646
650
|
def get_novel_type
|
647
|
-
|
648
|
-
@novel_status["novel_type"]
|
651
|
+
get_novel_status["novel_type"]
|
649
652
|
end
|
650
653
|
|
651
654
|
#
|
652
655
|
# 小説が完結しているか調べる
|
653
656
|
#
|
654
657
|
def novel_end?
|
655
|
-
|
656
|
-
|
658
|
+
get_novel_status["end"]
|
659
|
+
end
|
660
|
+
|
661
|
+
#
|
662
|
+
# 掲載サイト名
|
663
|
+
#
|
664
|
+
# すでにレコードに登録されている場合はそちらを優先する
|
665
|
+
#
|
666
|
+
def sitename
|
667
|
+
record&.dig("sitename") || get_novel_status["sitename"]
|
657
668
|
end
|
658
669
|
|
659
670
|
#
|
@@ -667,25 +678,22 @@ class Downloader
|
|
667
678
|
# 小説を格納するためのディレクトリ名を取得する
|
668
679
|
#
|
669
680
|
def get_file_title
|
670
|
-
return @file_title if @file_title
|
671
681
|
# すでにデータベースに登録されているならそれを引き続き使うようにする
|
672
|
-
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
@file_title += " " + Helper.replace_filename_special_chars(get_title, true).strip
|
679
|
-
end
|
680
|
-
@file_title
|
682
|
+
file_title = record&.dig("file_title")
|
683
|
+
return file_title if file_title
|
684
|
+
ncode = @setting["ncode"]
|
685
|
+
return ncode unless @setting["append_title_to_folder_name"]
|
686
|
+
scrubbed_title = Helper.replace_filename_special_chars(get_title, true).strip
|
687
|
+
Helper.truncate_folder_title("#{ncode} #{scrubbed_title}")
|
681
688
|
end
|
689
|
+
memoize :get_file_title
|
682
690
|
|
683
691
|
#
|
684
692
|
# 小説のタイトルを取得する
|
685
693
|
#
|
686
694
|
def get_title
|
687
695
|
return @title if @title
|
688
|
-
@title = @setting["title"] ||
|
696
|
+
@title = @setting["title"] || record["title"]
|
689
697
|
if @setting["title_strip_pattern"]
|
690
698
|
@title = @title.gsub(/#{@setting["title_strip_pattern"]}/, "").gsub(/^[ \s]*(.+?)[ \s]*?$/, "\\1")
|
691
699
|
end
|
@@ -793,10 +801,8 @@ class Downloader
|
|
793
801
|
raise if through_error # エラー処理はしなくていいからそのまま例外を受け取りたい時用
|
794
802
|
if e.message.include?("404")
|
795
803
|
@stream.error "小説が削除されているか非公開な可能性があります"
|
796
|
-
if
|
797
|
-
|
798
|
-
Command::Tag.execute!(%W(#{@id} --add 404 --color white --no-overwrite-color))
|
799
|
-
end
|
804
|
+
if database.novel_exists?(@id)
|
805
|
+
Command::Tag.execute!(%W(#{@id} --add 404 --color white --no-overwrite-color), io: Narou::NullIO.new)
|
800
806
|
Command::Freeze.execute!(@id, "--on")
|
801
807
|
end
|
802
808
|
else
|
@@ -838,7 +844,7 @@ class Downloader
|
|
838
844
|
# 前回ダウンロードしたはずの本文ファイルが存在するか
|
839
845
|
section_file_name = "#{index} #{old["file_subtitle"]}.yaml"
|
840
846
|
section_file_relative_path = File.join(SECTION_SAVE_DIR_NAME, section_file_name)
|
841
|
-
unless
|
847
|
+
unless get_novel_data_dir.join(section_file_relative_path).exist?
|
842
848
|
# あるはずのファイルが存在しなかったので、再ダウンロードが必要
|
843
849
|
next true
|
844
850
|
end
|
@@ -862,8 +868,7 @@ class Downloader
|
|
862
868
|
unless deffer
|
863
869
|
# 差分がある場合はこのあと保存されて更新されるので、差分がない場合のみ
|
864
870
|
# タイムスタンプを更新しておく
|
865
|
-
|
866
|
-
File.utime(now, now, File.join(get_novel_data_dir, section_file_relative_path))
|
871
|
+
FileUtils.touch(get_novel_data_dir.join(section_file_relative_path))
|
867
872
|
end
|
868
873
|
deffer
|
869
874
|
end
|
@@ -898,15 +903,15 @@ class Downloader
|
|
898
903
|
def get_section_file_timestamp(old_subtitles_info, latest_subtitles_info)
|
899
904
|
download_time = old_subtitles_info["download_time"]
|
900
905
|
unless download_time
|
901
|
-
download_time = File.mtime(
|
906
|
+
download_time = File.mtime(section_file_path(old_subtitles_info))
|
902
907
|
end
|
903
908
|
latest_subtitles_info["download_time"] = download_time
|
904
909
|
download_time
|
905
910
|
end
|
906
911
|
|
907
912
|
def title_to_filename(title)
|
908
|
-
Helper.
|
909
|
-
Helper.
|
913
|
+
Helper.truncate_path(
|
914
|
+
Helper.replace_filename_special_chars(
|
910
915
|
HTML.new(title).delete_ruby_tag
|
911
916
|
)
|
912
917
|
)
|
@@ -967,7 +972,6 @@ class Downloader
|
|
967
972
|
end
|
968
973
|
|
969
974
|
def slim_subtitle(string)
|
970
|
-
# HTML.new(string).delete_ruby_tag.delete("\n")
|
971
975
|
HTML.new(string).delete_ruby_tag.delete("\n")
|
972
976
|
end
|
973
977
|
|
@@ -1005,8 +1009,8 @@ class Downloader
|
|
1005
1009
|
|
1006
1010
|
section_file_name = "#{index} #{file_subtitle}.yaml"
|
1007
1011
|
section_file_relative_path = File.join(SECTION_SAVE_DIR_NAME, section_file_name)
|
1008
|
-
section_file_full_path =
|
1009
|
-
if
|
1012
|
+
section_file_full_path = get_novel_data_dir.join(section_file_relative_path)
|
1013
|
+
if section_file_full_path.exist?
|
1010
1014
|
if @force
|
1011
1015
|
if different_section?(section_file_relative_path, info)
|
1012
1016
|
@stream.print " (更新あり)"
|
@@ -1036,12 +1040,9 @@ class Downloader
|
|
1036
1040
|
# すでに保存されている内容とDLした内容が違うかどうか
|
1037
1041
|
#
|
1038
1042
|
def different_section?(old_relative_path, new_subtitle_info)
|
1039
|
-
path =
|
1040
|
-
|
1041
|
-
|
1042
|
-
else
|
1043
|
-
return true
|
1044
|
-
end
|
1043
|
+
path = get_novel_data_dir.join(old_relative_path)
|
1044
|
+
return true unless path.exist?
|
1045
|
+
YAML.load_file(path)["element"] != new_subtitle_info["element"]
|
1045
1046
|
end
|
1046
1047
|
|
1047
1048
|
#
|
@@ -1049,8 +1050,8 @@ class Downloader
|
|
1049
1050
|
#
|
1050
1051
|
def move_to_cache_dir(relative_path)
|
1051
1052
|
return if @nosave_diff
|
1052
|
-
path =
|
1053
|
-
if
|
1053
|
+
path = get_novel_data_dir.join(relative_path)
|
1054
|
+
if path.exist? && @cache_dir
|
1054
1055
|
FileUtils.mv(path, @cache_dir)
|
1055
1056
|
end
|
1056
1057
|
end
|
@@ -1093,14 +1094,13 @@ class Downloader
|
|
1093
1094
|
raw = Helper.restore_entity(raw)
|
1094
1095
|
save_raw_data(raw, subtitle_info)
|
1095
1096
|
element = extract_elements_in_section(raw, subtitle_info["subtitle"])
|
1096
|
-
element["data_type"] = "text"
|
1097
1097
|
else
|
1098
1098
|
save_raw_data(raw, subtitle_info, ".html")
|
1099
|
-
%w(introduction body
|
1099
|
+
%w(introduction postscript body).each { |type| @setting[type] = nil }
|
1100
1100
|
@setting.multi_match(raw, "body_pattern", "introduction_pattern", "postscript_pattern")
|
1101
1101
|
element = { "data_type" => "html" }
|
1102
|
-
%w(introduction body
|
1103
|
-
element[type] = @setting[type]
|
1102
|
+
%w(introduction postscript body).each { |type|
|
1103
|
+
element[type] = @setting[type].to_s
|
1104
1104
|
}
|
1105
1105
|
end
|
1106
1106
|
subtitle_info["download_time"] = Time.now
|
@@ -1109,23 +1109,25 @@ class Downloader
|
|
1109
1109
|
end
|
1110
1110
|
|
1111
1111
|
def display_hint
|
1112
|
-
@stream.
|
1113
|
-
ヒント:
|
1114
|
-
503 がでた場合はしばらくアクセスが規制される場合があります。
|
1115
|
-
設定を変更してサーバーに対する負荷を軽減させましょう。
|
1116
|
-
|
1112
|
+
@stream.puts <<~HINT
|
1113
|
+
ヒント:
|
1114
|
+
503 がでた場合はしばらくアクセスが規制される場合があります。
|
1115
|
+
設定を変更してサーバーに対する負荷を軽減させましょう。(下記参照)
|
1116
|
+
小説家になろう系列の場合、10分程度時間を置く必要があります。
|
1117
|
+
(メンテナンス等でも503になる場合があります。公式サイトを確認してください)
|
1117
1118
|
|
1118
|
-
下記の設定のどれか、もしくは全てを変更することで調整できます。
|
1119
|
+
下記の設定のどれか、もしくは全てを変更することで調整できます。
|
1120
|
+
(download.interval が最重要設定。1話ごとの間隔が短すぎると規制されやすい)
|
1119
1121
|
|
1120
|
-
#
|
1121
|
-
narou s
|
1122
|
+
# 1話ごとに入るウェイトを変更する(単位:秒)
|
1123
|
+
narou s download.interval=1.0
|
1122
1124
|
|
1123
|
-
#
|
1124
|
-
narou s download.
|
1125
|
+
# 10話ごとに通常より長いウェイトを入れる
|
1126
|
+
narou s download.wait-steps=10
|
1125
1127
|
|
1126
|
-
#
|
1127
|
-
narou s
|
1128
|
-
|
1128
|
+
# Update時の作品間の待機時間を変更する(単位:秒)
|
1129
|
+
narou s update.interval=3.0
|
1130
|
+
HINT
|
1129
1131
|
end
|
1130
1132
|
|
1131
1133
|
#
|
@@ -1133,7 +1135,7 @@ narou s download.wait-steps=5
|
|
1133
1135
|
#
|
1134
1136
|
def download_raw_data(url)
|
1135
1137
|
raw = nil
|
1136
|
-
retry_count =
|
1138
|
+
retry_count = LIMIT_TO_RETRY_NETWORK
|
1137
1139
|
cookie = @setting["cookie"] || ""
|
1138
1140
|
begin
|
1139
1141
|
open_uri_options = make_open_uri_options("Cookie" => cookie, allow_redirections: :safe)
|
@@ -1141,40 +1143,41 @@ narou s download.wait-steps=5
|
|
1141
1143
|
raw = Helper.pretreatment_source(fp.read, @setting["encoding"])
|
1142
1144
|
end
|
1143
1145
|
rescue OpenURI::HTTPError, Errno::ECONNRESET, Errno::ETIMEDOUT, Net::OpenTimeout => e
|
1144
|
-
|
1146
|
+
case e.message
|
1147
|
+
when /^503/
|
1148
|
+
# 503 はアクセス規制やメンテ等でリトライしてもほぼ意味がないことが多いため一度で諦める
|
1149
|
+
@stream.error "server message: #{e.message}"
|
1150
|
+
display_hint
|
1151
|
+
raise SuspendDownload
|
1152
|
+
when /^404/
|
1153
|
+
@stream.error "server message: #{e.message}"
|
1154
|
+
@stream.puts "#{url} がダウンロード出来ませんでした。時間をおいて再度試してみてください"
|
1155
|
+
raise SuspendDownload
|
1156
|
+
else
|
1145
1157
|
if retry_count == 0
|
1146
1158
|
@stream.error "上限までリトライしましたがファイルがダウンロード出来ませんでした"
|
1147
|
-
|
1159
|
+
raise SuspendDownload
|
1148
1160
|
end
|
1149
1161
|
retry_count -= 1
|
1150
|
-
@stream.puts
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1154
|
-
|
1155
|
-
display_hint
|
1156
|
-
@@display_hint_once = true
|
1157
|
-
end
|
1158
|
-
sleep(WAITING_TIME_FOR_503)
|
1162
|
+
@stream.puts <<~MSG
|
1163
|
+
server message: #{e.message}
|
1164
|
+
リトライ待機中...
|
1165
|
+
MSG
|
1166
|
+
sleep(WAIT_TIME_TO_RETRY_NETWORK)
|
1159
1167
|
retry
|
1160
|
-
elsif e.message =~ /^404/
|
1161
|
-
@stream.error "#{url} がダウンロード出来ませんでした。時間をおいて再度試してみてください"
|
1162
|
-
exit Narou::EXIT_ERROR_CODE
|
1163
|
-
else
|
1164
|
-
raise
|
1165
1168
|
end
|
1166
1169
|
end
|
1167
1170
|
raw
|
1168
1171
|
end
|
1169
1172
|
|
1170
|
-
def
|
1171
|
-
@raw_dir ||=
|
1173
|
+
def raw_dir
|
1174
|
+
@raw_dir ||= get_novel_data_dir.join(RAW_DATA_DIR_NAME)
|
1172
1175
|
end
|
1173
1176
|
|
1174
1177
|
def init_raw_dir
|
1175
1178
|
return if @nosave_raw
|
1176
|
-
path =
|
1177
|
-
FileUtils.mkdir_p(path) unless
|
1179
|
+
path = raw_dir
|
1180
|
+
FileUtils.mkdir_p(path) unless path.exist?
|
1178
1181
|
end
|
1179
1182
|
|
1180
1183
|
#
|
@@ -1184,7 +1187,7 @@ narou s download.wait-steps=5
|
|
1184
1187
|
return if @nosave_raw
|
1185
1188
|
index = subtitle_info["index"]
|
1186
1189
|
file_subtitle = subtitle_info["file_subtitle"]
|
1187
|
-
path =
|
1190
|
+
path = raw_dir.join("#{index} #{file_subtitle}#{ext}")
|
1188
1191
|
File.write(path, raw_data)
|
1189
1192
|
end
|
1190
1193
|
|
@@ -1205,9 +1208,10 @@ narou s download.wait-steps=5
|
|
1205
1208
|
end
|
1206
1209
|
end
|
1207
1210
|
{
|
1211
|
+
"data_type" => "text",
|
1208
1212
|
"introduction" => introduction,
|
1209
|
-
"
|
1210
|
-
"
|
1213
|
+
"postscript" => postscript,
|
1214
|
+
"body" => lines.join("\n")
|
1211
1215
|
}
|
1212
1216
|
end
|
1213
1217
|
|
@@ -1235,28 +1239,33 @@ narou s download.wait-steps=5
|
|
1235
1239
|
# 小説データの格納ディレクトリパス
|
1236
1240
|
#
|
1237
1241
|
def get_novel_data_dir
|
1238
|
-
return @novel_data_dir if @novel_data_dir
|
1239
1242
|
raise "小説名がまだ設定されていません" unless get_file_title
|
1240
1243
|
subdirectory = @download_use_subdirectory ? Downloader.create_subdirecotry_name(get_file_title) : ""
|
1241
|
-
|
1242
|
-
@novel_data_dir
|
1244
|
+
Database.archive_root_path.join(sitename, subdirectory, get_file_title)
|
1243
1245
|
end
|
1246
|
+
memoize :get_novel_data_dir
|
1244
1247
|
|
1245
1248
|
#
|
1246
1249
|
# 小説本文の保存パスを生成
|
1247
1250
|
#
|
1248
|
-
def
|
1251
|
+
def section_file_path(subtitle_info)
|
1249
1252
|
filename = "#{subtitle_info["index"]} #{subtitle_info["file_subtitle"]}.yaml"
|
1250
|
-
|
1253
|
+
get_novel_data_dir.join(SECTION_SAVE_DIR_NAME, filename)
|
1254
|
+
end
|
1255
|
+
|
1256
|
+
def save_toc_once(toc)
|
1257
|
+
return if @save_toc_once
|
1258
|
+
save_novel_data(TOC_FILE_NAME, toc)
|
1259
|
+
@save_toc_once = true
|
1251
1260
|
end
|
1252
1261
|
|
1253
1262
|
#
|
1254
1263
|
# 小説データの格納ディレクトリに保存
|
1255
1264
|
#
|
1256
1265
|
def save_novel_data(filename, object)
|
1257
|
-
path =
|
1258
|
-
dir_path =
|
1259
|
-
unless
|
1266
|
+
path = get_novel_data_dir.join(filename)
|
1267
|
+
dir_path = path.dirname
|
1268
|
+
unless dir_path.exist?
|
1260
1269
|
FileUtils.mkdir_p(dir_path)
|
1261
1270
|
end
|
1262
1271
|
File.write(path, YAML.dump(object))
|
@@ -1265,8 +1274,7 @@ narou s download.wait-steps=5
|
|
1265
1274
|
#
|
1266
1275
|
# 小説データの格納ディレクトリから読み込む
|
1267
1276
|
def load_novel_data(filename)
|
1268
|
-
|
1269
|
-
YAML.load_file(File.join(dir_path, filename))
|
1277
|
+
YAML.load_file(get_novel_data_dir.join(filename))
|
1270
1278
|
rescue Errno::ENOENT
|
1271
1279
|
nil
|
1272
1280
|
end
|
@@ -1276,13 +1284,13 @@ narou s download.wait-steps=5
|
|
1276
1284
|
#
|
1277
1285
|
def init_novel_dir
|
1278
1286
|
novel_dir_path = get_novel_data_dir
|
1279
|
-
file_title =
|
1280
|
-
FileUtils.mkdir_p(novel_dir_path) unless
|
1287
|
+
file_title = novel_dir_path.basename.to_s
|
1288
|
+
FileUtils.mkdir_p(novel_dir_path) unless novel_dir_path.exist?
|
1281
1289
|
original_settings = NovelSetting.get_original_settings
|
1282
1290
|
default_settings = NovelSetting.load_default_settings
|
1283
1291
|
novel_setting = NovelSetting.new(@id, true, true)
|
1284
|
-
special_preset_dir =
|
1285
|
-
exists_special_preset_dir =
|
1292
|
+
special_preset_dir = Narou.preset_dir.join(@setting["domain"], @setting["ncode"])
|
1293
|
+
exists_special_preset_dir = special_preset_dir.exist?
|
1286
1294
|
templates = [
|
1287
1295
|
[NovelSetting::INI_NAME, NovelSetting::INI_ERB_BINARY_VERSION],
|
1288
1296
|
["converter.rb", 1.0],
|
@@ -1290,9 +1298,9 @@ narou s download.wait-steps=5
|
|
1290
1298
|
]
|
1291
1299
|
templates.each do |(filename, binary_version)|
|
1292
1300
|
if exists_special_preset_dir
|
1293
|
-
preset_file_path =
|
1294
|
-
if
|
1295
|
-
unless
|
1301
|
+
preset_file_path = special_preset_dir.join(filename)
|
1302
|
+
if preset_file_path.exist?
|
1303
|
+
unless novel_dir_path.join(filename).exist?
|
1296
1304
|
FileUtils.cp(preset_file_path, novel_dir_path)
|
1297
1305
|
end
|
1298
1306
|
next
|