manga-downloadr 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46c5a3170b1ad7a8bb7b31e656d35246441268e4
-  data.tar.gz: dcc2c96a192ae2afc9a58583fdae3990154d1699
+  metadata.gz: 0122407cb8382eb31f8b25dffe79e049b3ddef59
+  data.tar.gz: c9afad43701915f6b793049af828d27f7607014a
 SHA512:
-  metadata.gz: 1d5e7caa9b3f595d7aff8068116ff5ba6006deabf23f78eeb73eeafe1588a517eb8437d4c419b0fd7ec76cb15df499156ffeaa7bd8ad56c486afe4461ac2d5b8
-  data.tar.gz: 8ed2a881d0e49023524c66464fafc370f6d37c41a397df4ffc07473a569ff4f4c811ff4c7c41b91f50b72ce610a817842bcddc75a321835ba6de85ee1892bfcf
+  metadata.gz: 64a1017562b4b8fef387d5552d467bb7c542202baca577b1340f1d7f563a97c49b47af1e9de3039241d0258bded1eafbb35ae882aa84650c406ac86ea26315c4
+  data.tar.gz: 555996bb4fce50a1868b4e2660bd67f12e5d429ddca270d2d878849e0b810ab080deffb7fcdcb173bd1d5dfdb7c9c96db3489356b3178d143f098c7b8db8d887

data/bin/manga-downloadr

@@ -36,7 +36,7 @@ rescue OptionParser::ParseError
 end
 
 require 'manga-downloadr'
-generator = MangaDownloadr::Workflow.create(options[:url], options[:name], manga_root: options[:directory])
+generator = MangaDownloadr::Workflow.create(options[:url], options[:name], options[:directory])
 unless generator.state?(:chapter_urls)
   puts "Massive parallel scanning of all chapters "
   generator.fetch_chapter_urls!

data/lib/manga-downloadr.rb

@@ -10,6 +10,14 @@ require 'fastimage'
 require 'open-uri'
 require 'yaml'
 
+if ENV['RUBY_ENV'].nil?
+  require 'retryable_typhoeus'
+  Typhoeus::Request.send(:include, RetryableTyphoeus::RequestExtension)
+  Typhoeus::Hydra.send(:include, RetryableTyphoeus::HydraExtension)
+else
+  Typhoeus::Hydra.send(:alias_method, :queue_with_retry, :queue)
+end
+
 module MangaDownloadr
   ImageData = Struct.new(:folder, :filename, :url)
 
@@ -18,6 +26,7 @@ module MangaDownloadr
     attr_accessor :chapter_list, :chapter_pages, :chapter_images, :download_links, :chapter_pages_count
     attr_accessor :manga_title, :pages_per_volume, :page_size
     attr_accessor :processing_state
+    attr_accessor :fetch_page_urls_errors, :fetch_image_urls_errors, :fetch_images_errors
 
     def initialize(root_url = nil, manga_name = nil, manga_root = nil, options = {})
       root_url or raise ArgumentError.new("URL is required")
@@ -37,7 +46,10 @@ module MangaDownloadr
       self.pages_per_volume = options[:pages_per_volume] || 250
       self.page_size        = options[:page_size] || [600, 800]
 
-      self.processing_state = []
+      self.processing_state        = []
+      self.fetch_page_urls_errors  = []
+      self.fetch_image_urls_errors = []
+      self.fetch_images_errors     = []
     end
 
     def fetch_chapter_urls!
@@ -55,17 +67,26 @@ module MangaDownloadr
         begin
           request = Typhoeus::Request.new "http://www.mangareader.net#{chapter_link}"
           request.on_complete do |response|
-            chapter_doc = Nokogiri::HTML(response.body)
-            pages = chapter_doc.css('#selectpage #pageMenu option')
-            chapter_pages.merge!(chapter_link => pages.map { |p| p['value'] })
-            print '.'
+            begin
+              chapter_doc = Nokogiri::HTML(response.body)
+              pages = chapter_doc.css('#selectpage #pageMenu option')
+              chapter_pages.merge!(chapter_link => pages.map { |p| p['value'] })
+              print '.'
+            rescue => e
+              self.fetch_page_urls_errors << { url: chapter_link, error: e, body: response.body }
+              print 'x'
+            end
           end
-          hydra.queue request
+          hydra.queue_with_retry request
         rescue => e
           puts e
         end
       end
       hydra.run
+      unless fetch_page_urls_errors.empty?
+        puts "\n Errors fetching page urls:"
+        puts fetch_page_urls_errors
+      end
 
       self.chapter_pages_count = chapter_pages.values.inject(0) { |total, list| total += list.size }
       current_state :page_urls
@@ -78,22 +99,31 @@ module MangaDownloadr
           begin
             request = Typhoeus::Request.new "http://www.mangareader.net#{page_link}"
             request.on_complete do |response|
-              chapter_doc = Nokogiri::HTML(response.body)
-              image       = chapter_doc.css('#img').first
-              tokens      = image['alt'].match("^(.*?)\s\-\s(.*?)$")
-              extension   = File.extname(URI.parse(image['src']).path)
-
-              chapter_images.merge!(chapter_key => []) if chapter_images[chapter_key].nil?
-              chapter_images[chapter_key] << ImageData.new( tokens[1], "#{tokens[2]}#{extension}", image['src'] )
-              print '.'
+              begin
+                chapter_doc = Nokogiri::HTML(response.body)
+                image       = chapter_doc.css('#img').first
+                tokens      = image['alt'].match("^(.*?)\s\-\s(.*?)$")
+                extension   = File.extname(URI.parse(image['src']).path)
+
+                chapter_images.merge!(chapter_key => []) if chapter_images[chapter_key].nil?
+                chapter_images[chapter_key] << ImageData.new( tokens[1], "#{tokens[2]}#{extension}", image['src'] )
+                print '.'
+              rescue => e
+                self.fetch_image_urls_errors << { url: page_link, error: e }
+                print 'x'
+              end
             end
-            hydra.queue request
+            hydra.queue_with_retry request
           rescue => e
             puts e
           end
         end
       end
       hydra.run
+      unless fetch_image_urls_errors.empty?
+        puts "\nErrors fetching image urls:"
+        puts fetch_image_urls_errors
+      end
 
       current_state :image_urls
     end
@@ -102,30 +132,35 @@ module MangaDownloadr
       hydra = Typhoeus::Hydra.new(max_concurrency: hydra_concurrency)
       chapter_list.each_with_index do |chapter_key, chapter_index|
         chapter_images[chapter_key].each do |file|
-          begin
             downloaded_filename = File.join(manga_root_folder, file.folder, file.filename)
             next if File.exists?(downloaded_filename) # effectively resumes the download list without re-downloading everything
             request = Typhoeus::Request.new file.url
             request.on_complete do |response|
-              # download
-              FileUtils.mkdir_p(File.join(manga_root_folder, file.folder))
-              File.open(downloaded_filename, "wb+") { |f| f.write response.body }
-
-              # resize
-              image = Magick::Image.read( downloaded_filename ).first
-              resized = image.resize_to_fit(600, 800)
-              resized.write( downloaded_filename ) { self.quality = 50 }
-
-              print '.'
-              GC.start # to avoid a leak too big (ImageMagick is notorious for that, specially on resizes)
+              begin
+                # download
+                FileUtils.mkdir_p(File.join(manga_root_folder, file.folder))
+                File.open(downloaded_filename, "wb+") { |f| f.write response.body }
+
+                # resize
+                image = Magick::Image.read( downloaded_filename ).first
+                resized = image.resize_to_fit(600, 800)
+                resized.write( downloaded_filename ) { self.quality = 50 }
+
+                print '.'
+                GC.start # to avoid a leak too big (ImageMagick is notorious for that, specially on resizes)
+              rescue => e
+                self.fetch_images_errors << { url: file.url, error: e }
+                print '.'
+              end
             end
-            hydra.queue request
-          rescue => e
-            puts e
-          end
+          hydra.queue_with_retry request
         end
       end
       hydra.run
+      unless fetch_images_errors.empty?
+        puts "\nErrors downloading images:"
+        puts fetch_images_errors
+      end
 
       current_state :images
     end
@@ -171,10 +206,10 @@ module MangaDownloadr
         File.open("/tmp/#{obj.manga_name}.yaml", 'w') {|f| f.write(YAML::dump(obj)) }
       end
 
-      def create(root_url, manga_name, options = {})
+      def create(root_url, manga_name, manga_root, options = {})
         dump_file_name = "/tmp/#{manga_name}.yaml"
         return YAML::load(File.read(dump_file_name)) if File.exists?(dump_file_name)
-        MangaDownloadr::Workflow.new(root_url, manga_name, options)
+        MangaDownloadr::Workflow.new(root_url, manga_name, manga_root, options)
       end
     end
   end

data/lib/manga-downloadr/version.rb

@@ -1,3 +1,3 @@
 module MangaDownloadr
-  VERSION = "1.0.2"
+  VERSION = "1.0.3"
 end

data/lib/retryable_typhoeus.rb

@@ -0,0 +1,40 @@
+# original source: https://gist.github.com/kunalmodi/2939288
+module RetryableTyphoeus
+  DEFAULT_RETRIES = 1
+
+  module RequestExtension
+    def original_on_complete=(proc)
+      @original_on_complete = proc
+    end
+
+    def original_on_complete
+      @original_on_complete
+    end
+
+    def retries=(retries)
+      @retries = retries
+    end
+
+    def retries
+      @retries ||= 0
+    end
+  end
+
+  module HydraExtension
+    def queue_with_retry(request, opts = {})
+      request.retries = (opts[:retries] || RetryableTyphoeus::DEFAULT_RETRIES).to_i
+      request.original_on_complete ||= request.on_complete
+      request.on_complete do |response|
+        if response.success? || response.request.retries <= 0
+          request.original_on_complete.map do |callback|
+            response.handled_response = callback.call(response)
+          end
+        else
+          response.request.retries -= 1
+          queue response.request
+        end
+      end
+      queue request
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manga-downloadr
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - AkitaOnRails
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-19 00:00:00.000000000 Z
+date: 2014-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -133,6 +133,7 @@ files:
 - bin/manga-downloadr
 - lib/manga-downloadr/version.rb
 - lib/manga-downloadr.rb
+- lib/retryable_typhoeus.rb
 - README.md
 - LICENSE.txt
 homepage: http://github.com/akitaonrails/manga-downloadr