manga-downloadr 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b083f3caf90e18e3afa86b91fbbd6d54c1b14236
-  data.tar.gz: 14ba9e5c6f03356dd90446b966acf73caf201b28
+  metadata.gz: 26aa19f21806d820b467c181ab6a48552580e3a1
+  data.tar.gz: d51638cea0d79fd66c01520e117cc6e6a6b7b836
 SHA512:
-  metadata.gz: d15ce2ed73541b567ded076918b114b359443256749e5a12d488f0de9271d69ad0ba3be176557b33d41ec4aeb93952216a7398620b7d77a306ad75fb5a760959
-  data.tar.gz: 03a5734f26846feb483d645aaaf504adab7167cd73004a36e7b347e4c08d16927a33cfbf07289bc2b61a2ddf89a244b47dcdf0230c28a257feee2a6f10f41bdb
+  metadata.gz: 2d79dabfc879ccff72159929c7d6383d2f4f6fa8747b641e439caccbe9163f1335b99e2e5efd9740751198d902f28b51ace9f64001544518e286c0e3765189f3
+  data.tar.gz: 003812def9a76ab1bc988f7884bfb773aa439c1754c8c14c4d24270722c7a0dad97aee2084a9be5ed4e2dcedcdc372b9d01926cdc71eab824e4429224529a655

data/README.md

@@ -29,6 +29,10 @@ In this example, all the pages of the "One Punch Man" will be downloaded to the
 
     /tmp/onepunch-man/Onepunch-Man-Chap-00038-Pg-00011.jpg
 
+You can turn on HTTP cache to be able to resume an interrupted process later if you want:
+
+    $ manga-downloadr -u http://www.mangareader.net/onepunch-man -d /tmp/onepunch-man --cache
+
 ## Development
 
 Tests are in Rspec:
@@ -45,6 +49,16 @@ Crystal is also super fast (because its compiled to native code) and has very go
 
 This Ruby version uses native Threads. Because this is I/O intensive, we assume we can run several HTTP requests concurrently. But because Threads have significantly more overhead than Elixir or Crystal architectures, we will be limited by Ruby's MRI interpreter.
 
+There is not a test mode you can use for benchmark purposes:
+
+    time bin/manga-downloadr --test
+    # or in JRuby:
+    # time jruby --dev -S bin/manga-downloadr --test
+
+This will use One-Punch Man as a test sample and you can also turn on the cache to not have external I/O interference
+
+    time bin/manga-downloadr --test --cache
+
 ## Contributing
 
 1. Fork it ( https://github.com/akitaonrails/manga-downloadr/fork )

data/bin/manga-downloadr

@@ -2,12 +2,15 @@
 $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib")
 require "optparse"
 require "manga-downloadr"
+require "benchmark"
 
 opt_manga_directory  = "/tmp"
 opt_manga_root_uri   = ""
 opt_batch_size       = 50
 opt_resize_format    = "600x800"
 opt_pages_per_volume = 250
+opt_run_tests = false
+opt_cache_pages = false
 
 option_parser = OptionParser.new do |opts|
   # Set a banner, displayed at the top
@@ -34,6 +37,16 @@ option_parser = OptionParser.new do |opts|
     opt_pages_per_volume = volume.to_i
   end
 
+  opts.on( "-c", "--cache", "turn on the HTTP cache so you can resume a process if you needed to stop before ending") do |_|
+    opt_cache_pages = true
+  end
+
+  opts.on( "-t", "--test", "run a simulation to one-punch man manga") do |_|
+    opt_manga_root_uri = "http://www.mangareader.net/onepunch-man"
+    opt_manga_directory = "/tmp/cr-one-punch"
+    opt_run_tests = true
+  end
+
   # This displays the help screen, all programs are
   # assumed to have this option.
   opts.on( "-h", "--help", "Display this screen" ) do
@@ -52,6 +65,12 @@ end
 
 if opt_manga_root_uri.size > 0
   root_uri = URI.parse(opt_manga_root_uri)
-  config = MangaDownloadr::Config.new(root_uri.host, root_uri.path, opt_manga_directory, opt_batch_size, opt_resize_format, opt_pages_per_volume)
-  MangaDownloadr::Workflow.run(config)
+  config = MangaDownloadr::Config.new(root_uri.host, root_uri.path, opt_manga_directory, opt_batch_size, opt_resize_format, opt_pages_per_volume, opt_cache_pages)
+  if opt_run_tests
+    puts Benchmark.measure("One-Punch Man test - MRI Ruby version") {
+      MangaDownloadr::Workflow.run_tests(config)
+    }
+  else
+    MangaDownloadr::Workflow.run(config)
+  end
 end

data/lib/manga-downloadr/chapters.rb

@@ -1,8 +1,8 @@
 module MangaDownloadr
   class Chapters < DownloadrClient
-    def initialize(domain, root_uri)
+    def initialize(domain, root_uri, cache_http)
       @root_uri = root_uri
-      super(domain)
+      super(domain, cache_http)
     end
 
     def fetch

data/lib/manga-downloadr/concurrency.rb

@@ -1,26 +1,44 @@
+require "thread/pool"
+
 module MangaDownloadr
   class Concurrency
     def initialize(engine_klass = nil, config = Config.new, turn_on_engine = true)
-      @engine_klass = engine_klass
-      @config = config
+      @engine_klass   = engine_klass
+      @config         = config
       @turn_on_engine = turn_on_engine
     end
 
     def fetch(collection, &block)
+      pool    = Thread.pool(@config.download_batch_size)
+      mutex   = Mutex.new
+      results = []
+
+      collection.each do |item|
+        pool.process {
+          engine  = @turn_on_engine ? @engine_klass.new(@config.domain, @config.cache_http) : nil
+          reply = block.call(item, engine)&.flatten
+          mutex.synchronize do
+            results += ( reply || [] )
+          end
+        }
+      end
+      pool.shutdown
+
+      results
+    end
+
+    private
+
+    # this method is the same as the above but sequential, without Threads
+    # it's not to be used in the application, just to be used as a baseline for benchmark
+    def fetch_sequential(collection, &block)
       results = []
+      engine  = @turn_on_engine ? @engine_klass.new(@config.domain, @config.cache_http) : nil
       collection&.each_slice(@config.download_batch_size) do |batch|
-        mutex   = Mutex.new
-        threads = batch.map do |item|
-          Thread.new {
-            engine  = @turn_on_engine ? @engine_klass.new(@config.domain) : nil
-            Thread.current["results"] = block.call(item, engine)&.flatten
-            mutex.synchronize do
-              results += Thread.current["results"]
-            end
-          }
+        batch.each do |item|
+          batch_results = block.call(item, engine)&.flatten
+          results += ( batch_results || [])
         end
-        threads.each(&:join)
-        puts "Processed so far: #{results&.size}"
       end
       results
     end

data/lib/manga-downloadr/downloadr_client.rb

@@ -1,16 +1,31 @@
+require "digest"
+
 module MangaDownloadr
   class DownloadrClient
-    def initialize(domain)
+    def initialize(domain, cache_http)
       @domain = domain
+      @cache_http = cache_http
       @http_client = Net::HTTP.new(@domain)
     end
 
     def get(uri, &block)
-      response = @http_client.get(uri)
+      cache_path = "/tmp/manga-downloadr-cache/#{cache_filename(uri)}"
+      response = if @cache_http && File.exists?(cache_path)
+        body = File.read(cache_path)
+        MangaDownloadr::HTTPResponse.new("200", body)
+      else
+        @http_client.get(uri, { "User-Agent": USER_AGENT })
+      end
+
       case response.code
       when "301"
         get response.headers["Location"], &block
       when "200"
+        if @cache_http && !File.exists?(cache_path)
+          File.open(cache_path, "w") do |f|
+            f.write response.body
+          end
+        end
         parsed = Nokogiri::HTML(response.body)
         block.call(parsed)
       end
@@ -20,5 +35,9 @@ module MangaDownloadr
       sleep 1
       get(uri, &block)
     end
+
+    private def cache_filename(uri)
+      Digest::MD5.hexdigest(uri)
+    end
   end
 end

data/lib/manga-downloadr/image_downloader.rb

@@ -2,7 +2,7 @@ module MangaDownloadr
   class ImageDownloader < DownloadrClient
     def fetch(image_src, filename)
       File.delete(filename) if File.exists?(filename)
-      response = @http_client.get(image_src)
+      response = @http_client.get(image_src, { "User-Agent": USER_AGENT })
       case response.code
       when "301"
         fetch(response.headers["Location"], filename)

data/lib/manga-downloadr/records.rb

@@ -1,4 +1,8 @@
 module MangaDownloadr
   Image  = Struct.new *%i[host path filename]
-  Config = Struct.new *%i[domain root_uri download_directory download_batch_size image_dimensions pages_per_volume]
+  Config = Struct.new *%i[domain root_uri download_directory download_batch_size image_dimensions pages_per_volume cache_http]
+
+  HTTPResponse = Struct.new *%i[code body]
+
+  USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/45.0.2454.101 Chrome/45.0.2454.101 Safari/537.36"
 end

data/lib/manga-downloadr/version.rb

@@ -1,3 +1,3 @@
 module MangaDownloadr
-  VERSION = "2.0.0"
+  VERSION = "2.0.1"
 end

data/lib/manga-downloadr/workflow.rb

@@ -3,7 +3,7 @@ module MangaDownloadr
     def self.run(config = Config.new)
       FileUtils.mkdir_p config.download_directory
 
-      CM(Workflow, config)
+      CM(config, Workflow)
         .fetch_chapters
         .fetch_pages(config)
         .fetch_images(config)
@@ -15,9 +15,21 @@ module MangaDownloadr
       puts "Done!"
     end
 
+    def self.run_tests(config = Config.new)
+      FileUtils.mkdir_p "/tmp/manga-downloadr-cache"
+
+      CM(Workflow, config)
+        .fetch_chapters
+        .fetch_pages(config)
+        .fetch_images(config)
+        .unwrap
+
+      puts "Done!"
+    end
+
     def self.fetch_chapters(config)
       puts "Fetching chapters ..."
-      chapters = Chapters.new(config.domain, config.root_uri).fetch
+      chapters = Chapters.new(config.domain, config.root_uri, config.cache_http).fetch
       puts "Number of Chapters: #{chapters&.size}"
       chapters
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manga-downloadr
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - AkitaOnRails
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-06 00:00:00.000000000 Z
+date: 2016-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -30,70 +30,110 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.4.0
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.3
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.1.0
 description: downloads any manga from MangaReader.net
 email: boss@akitaonrails.com
 executables: