manga_fetch 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 870395ce4d4c86cedf154f07a78ba30ed2adc37c
-  data.tar.gz: f16ff576f1ae7e24645ebd86a5d9599c79d0bb7c
+  metadata.gz: b7548ec57ffa2092d910e848cf1a1f05949d9b57
+  data.tar.gz: 37c6122db7107bc6d4715482d72302dc0753e5b5
 SHA512:
-  metadata.gz: fa7be02326bc288750835ab04115464a6d5cf8fd5420dbda45281b79a149ae7b738e021d515f32b4bd0951c0e7883c30a25229c07ff3e3ca99ce5d7c8142eeb0
-  data.tar.gz: 3c6aff3e36cd2e026f29b9aafe61fb91470e0a24d01657eed18ccf9c9973f94cf9402c0d481dcba17aa4a93dbe4db9c804e5c507fe75dc585338af6c84faa416
+  metadata.gz: 812d5ad2a4dd2bca5f45e9cdafef83716b71f5dcea29ba9967872aada10523eb05dfb563ccdcb57a01f0767b152f5914adbbc6fc5c128d99b3a5e75894e3bd27
+  data.tar.gz: 0e78afcf03633e425b21e9b12a6b3a804608d7227fa5fcaea9ef1368c3b69da646b77778b217f6df9ac7afd3928521ac1a082ff98f1f028179d808d5abdf991e

data/bin/manga_fetch

@@ -1,14 +1,56 @@
 #!/usr/bin/env ruby
 
-require_relative "../lib/manga_fetch"
+require 'optparse'
 
-$verbose = true
+argv = OptionParser.new do |opts|
+  opts.banner = "Usage: manga_scrap <name> <min..max>"
+  $threads = 8
+  $verbose = true
+  $debug = false
+  $verbose = false if ENV["VERBOSE"] == "false"
+  $debug = true if ENV["DEBUG"] == "true"
+  $dir = "/tmp"
 
-m = ARGV[1].match(/\A(\d+)\.\.(\d+)\Z/)
-nums = m[1].to_i .. m[2].to_i
+  opts.on("--threads=N", "Number of threads to use (default: 1)") do |v|
+    $threads = Integer(v)
+  end
 
-list = MangaFetch.fetch(name: ARGV[0], num: nums).flatten.sort
+  opts.on("--dir=path", "--output=path", "Output directory (default: /tmp)") do |v|
+    $dir = v
+  end
 
-puts
-puts list
-list.each{|image| MangaFetch.download_image(src: image)}
+  opts.on("--verbose", "Display infos on stderr (default)") do
+    $verbose = true
+  end
+
+  opts.on("--no-verbose", "Display infos on stderr") do
+    $verbose = true
+  end
+
+  opts.on("--debug", "-d", "Developper mode") do
+    $debug = true
+  end
+end.parse!
+
+if $debug
+  require_relative "../lib/manga_fetch"
+else
+  require "manga_fetch"
+end
+
+m = argv[1].match(/\A(\d+)\.\.(\d+)\Z/)
+nums = (m[1].to_i .. m[2].to_i).to_a
+
+nb_elem_by_list = (nums.size.to_f / $threads).ceil
+lists = nums.each_slice(nb_elem_by_list).to_a
+
+STDERR.puts "Starting #{$threads} threads..." if $verbose
+lists.map.with_index do |current_nums, idx|
+  t = Thread.new do
+    STDERR.puts "Started #{idx+1} / #{$threads} threads" if $verbose
+    bot = MangaFetch::Fetcher.new
+    list = bot.fetch(name: argv[0], num: current_nums, download: true, dir: $dir).flatten.sort
+  end
+  sleep 0.5
+  t
+end.map(&:join)

data/lib/manga_fetch/downloader.rb

@@ -1,12 +1,12 @@
 module MangaFetch::Downloader
-  extend self
-
-  def download_image(src: nil, prefix: "/tmp")
-    image = @agent.get src
+  # Download an image defined with `src` in the directory `prefix`. It uses `@agent` by default
+  def download_image(src: nil, prefix: "/tmp", agent: nil)
+    prefix ||= "/tmp"
+    image = (agent || @agent).get src
     path = File.expand_path image.uri.to_s.split("/")[-3..-1].join("/"), prefix
     Dir.mkdir File.expand_path image.uri.to_s.split("/")[-3..-3].join("/"), prefix rescue nil
     Dir.mkdir File.expand_path image.uri.to_s.split("/")[-3..-2].join("/"), prefix rescue nil
     File.write path, image.body
-    STDERR.puts "File [".blue + path.yellow + "] saved"
+    STDERR.puts "File [".blue + path.yellow + "] saved (".blue + path.yellow + ")".blue
   end
 end

data/lib/manga_fetch/fetcher.rb

@@ -1,24 +1,27 @@
-module MangaFetch::Fetcher
-  extend self
+require_relative "downloader"
+
+class MangaFetch::Fetcher
+  include MangaFetch::Downloader
 
   # Fetch some pages form the manga named like `name`
-  def fetch(name: "berserk", num: nil)
-    init_agent("http://www.mangareader.net/#{name}")
+  def fetch(name: "berserk", num: nil, download: false, dir: nil)
+    init_agent(url: "http://www.mangareader.net/#{name}")
 
     images = if num.is_a? Fixnum
-      [ fetch_one(num) ]
+      [ fetch_one(num: num, download: download, dir: dir) ]
     elsif num.is_a? Enumerable
-      fetch_list(num)
+      fetch_list(nums: num, download: download, dir: dir)
     end
   end
 
-  def init_agent(url)
+  private def init_agent(url: nil)
     @agent = Mechanize.new
     @agent.get url
     @main = @agent.page
   end
 
-  def fetch_one(num: 1)
+  # fetch every images of the tome `num` loaded in `@main`
+  def fetch_one(num: 1, download: false, dir: nil)
     link = "#{@main.uri.to_s}/#{num}"
     STDERR.puts "Fetching Tome [".blue + "#{num}".yellow + "]".blue if $verbose
     first_page = @agent.get(link)
@@ -28,15 +31,16 @@ module MangaFetch::Fetcher
       STDERR.puts "Fetching Tome [".blue + "#{num}".yellow + "] Image [".blue + "#{i} / #{last_page_num}".yellow + "]".blue if $verbose
       current_page = @agent.get "#{first_page.uri.to_s}/#{i}"
       image = current_page.at("#img")[:src]
-      STDERR.puts "-> ".blue + image.red if $verbose
+      STDERR.puts "[#{download ? "D" : "S"}] -> ".blue + image.red if $verbose
+      download_image(src: image, prefix: dir) if download
       image
     end
-
   end
 
-  def fetch_list(nums)
+  # fetch every images of the tomes `nums` loaded in `@main`
+  def fetch_list(nums: nil, download: false, dir: nil)
     ones = []
-    nums.map{|num| ones << fetch_one(num: 1) } #  , threads: 4
+    nums.map{|num| ones << fetch_one(num: num, download: download, dir: dir) }
     ones
   end
 end

data/lib/manga_fetch.rb

@@ -4,11 +4,4 @@ require "colorize"
 module MangaFetch
 end
 
-require_relative "manga_fetch/downloader"
 require_relative "manga_fetch/fetcher"
-
-module MangaFetch
-  include Downloader
-  include Fetcher
-  extend self
-end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: manga_fetch
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Arthur Poulet
@@ -31,7 +31,7 @@ cert_chain:
   tcYkgfqUJPitIJx1RvWZpIyH5uJhRUYK3+vU9nMOxez5WbIlC1TtpByKAPMX+sht
   gib3AoIT8jh/2w==
   -----END CERTIFICATE-----
-date: 2016-08-26 00:00:00.000000000 Z
+date: 2016-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nomorebeer
@@ -93,7 +93,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.7.5
-description: add multi-thread and advanced features (download, opt parsing, ...)
+description: 
 email: arthur.poulet@mailoo.org
 executables:
 - manga_fetch