image-dumper 0.5.5 → 0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8bb0916cbaf373d4b57cf666146ba1d04db68835
-  data.tar.gz: d0dd26d401d89d3b5173653663d9f54fe6a0275b
+  metadata.gz: d620c7ba1dbc1bd22e170e520822d8711c1fd1f8
+  data.tar.gz: 61eb44fd3c2ed01b948acb3993081437bf65b8ca
 SHA512:
-  metadata.gz: 936f144bd3c4c50d566e45a145749920ff14f28be43f8b28b95baabdb3958bdeb2286c74133944703c6d37b6e6cffc1b8d07f4dcd30308e16db65a716ba02da9
-  data.tar.gz: 328ea9e69894a4529cb763ee3103b218de43bac5e3da1869f0cffe63188e678a503ad244e43ca65dc58eeb16bf6ddd8faf4ea3529b9770170e915dd37c72fa99
+  metadata.gz: 8e7a28e8d7fb7b2ee24043ae8c3cbdbbcfe5ec7b1267a1445cf20d54cfd641bd0e586b1da3db9f983e61b0b8f77ecff178df72be1a54cbe467965e620c410d33
+  data.tar.gz: b6381b814c485917103bb4bb15ed6bf6aa2092e47ad5e963f2480fd649445e8800aba1e7371e92bb183d459a7bbc2cfe84eaea5b7dd57606d7b340423ceff009

data/bin/dumper

@@ -17,95 +17,102 @@
 # You should have received a copy of the GNU General Public License
 # along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
 #++
-
 require 'dumper'
 
 options = {}
 
 OptionParser.new do |o|
-  options[:url]  = []
+  options[:url ] = []
   options[:path] = []
   
-  o.on '-l', '--list', 'Show available profiles' do
+  o.on '-l', '--list',            'Show available profiles' do
     abort 'Profiles available:'.tap { |s|
       Dumper::Profiles::list.sort { |a, b| a <=> b }.each { |p| s << "\n" + (' ' * 3) + p }
     } if ARGV.empty?
   end
 
-  o.on '-i', '--info PROFILE', 'Get info about profiles' do |profile|
+  o.on '-i', '--info PROFILE',    'Get info about profiles' do |profile|
     if Dumper::Profiles::list.include? profile
-      method = ("info_#{profile}").to_sym
+      method = ("info_#{profile}").to_sym      
       Dumper::Profiles::send(method).tap { |i|
-        puts "Option 'from' is #{i[:from] ? 'available' : 'not available'}"
-        puts "Option 'to'   is #{i[:to]   ? 'available' : 'not available'}"
+        puts "Option 'from' is #{i[:from] ? 'available' : 'not available'}."
+        puts "Option 'to'   is #{i[:to]   ? 'available' : 'not available'}."
+        abort "It dumps #{i[:type]}."
       }
     else
-      puts 'Profile not found.'
+      abort 'Profile not found.'
     end
-    abort
   end
   
-  o.on '-u', '--url URL', 'Target URL' do |url|
-    options[:url] << url
+  o.on '-u', '--url URL',         'Target URL'    do |url|
+    options[:url]    << url
   end
   
-  o.on '-f', '--file FILE', 'File containing a list of URLs, a double pipe (||) and the target folder, one per line' do |file|
-    file = File.open(file).read.gsub(/\r\n?/, "\n")
-    file.each_line { |line|
-      split             = line.split('||')
-      options[:url]     << split[0].strip
-      options[:path]    << split[1].strip
-    }
+  o.on '-o', '--from PAGE',       'Start to save from... (if allowed)' do |pages|
+    options[:from]    = pages.to_i
+  end
+  
+  o.on '-g', '--to PAGE',         'Finish to save at... (if allowed)'  do |pages|
+    options[:to]      = pages.to_i
+  end
+  
+  o.on '-p', '--path PATH',       'Target folder'      do |path|
+    options[:path]   << path
   end
   
-  o.on '-p', '--path PATH', 'Target folder' do |path|
-    options[:path] << path
+  o.on '-x', '--xpath XPATH',     'Use a custom xpath' do |xpath|
+    options[:xpath]   = xpath
   end
   
-  o.on '-r', '--profile PROFILE', 'Use the given profile' do |profile|
+  o.on '-r', '--profile PROFILE', 'Force the use of the given profile' do |profile|
     options[:profile] = profile
   end
   
-  o.on '-x', '--xpath XPATH', 'Custom xpath' do |xpath|
-    options[:xpath] = xpath
+  o.on '-t', '--threads MIN:MAX',  'Set the number of threads to use'   do |threads|
+    threads = threads.split ?:
+    Dumper::Profiles.pool_size = threads[0].to_i, threads[1].to_i
   end
   
-  o.on '-o', '--from PAGE', 'Start to save from... (if allowed)' do |pages|
-    options[:from] = pages.to_i
+  o.on '-s', '--silence',         'Print only important messages' do
+    Dumper::Profiles.verbose = false
   end
   
-  o.on '-g', '--to PAGE', 'Finish to save at... (if allowed)' do |pages|
-    options[:to] = pages.to_i
+  o.on '-f', '--file FILE',       'File containing in each line "URL||Folder"' do |file|
+    file = File.open(file).read.gsub(/\r\n?/, "\n")
+    file.each_line { |line|
+      split              = line.split '||'
+      options[:url]     << split[0].strip
+      options[:path]    << split[1].strip
+    }
   end
 end.parse!
 
 if options[:url].empty?
-  abort 'URL or list of URLs is required.'
+  abort 'An URL or a list of URLs is required.'
 elsif options[:path].empty?
   abort 'Path is required.'
 end
 
 options[:url].each_with_index { |url, i|
   begin
-
-    host = options.has_key?(:profile) ? options[:profile] : URI.parse(url).host.split(?.)[-2]
     Dir.mkdir(options[:path][i]) unless File.directory? options[:path][i]
-    
-    if Dumper::Profiles::list.include? host.gsub(?-, ?_)
-      method = ('get_' + host.gsub(?-, ?_)).to_sym
-      if options.include?(:from) && options.include?(:to)
-        Dumper::Profiles::send method, url, options[:path][i], options[:from], options[:to]
-      else
-        Dumper::Profiles::send method, url, options[:path][i]
+
+    host = options[:profile] || URI.parse(url).host.split(?.)[-2]
+
+    if options[:xpath].nil?  && Dumper::Profiles::list.include?(host.gsub(?-, ?_))
+      method = "get_#{host.gsub ?-, ?_}".to_sym
+      case
+        when options[:from] && options[:to] then Dumper::Profiles::send method, url, options[:path][i], options[:from], options[:to]
+        when options[:from]                 then Dumper::Profiles::send method, url, options[:path][i], options[:from]
+        else                                     Dumper::Profiles::send method, url, options[:path][i]
       end
     else
       Dumper::Profiles::get_generic  url, options[:path][i], options[:xpath]
     end
-
   rescue Nokogiri::XML::XPath::SyntaxError => e
-    puts e.to_s.gsub(/expression/, 'xpath')
+    puts e.to_s.gsub /expression/, 'xpath'
     puts 'Cannot dump.'
-  rescue OpenURI::HTTPError => e
+  rescue OpenURI::HTTPError   => e
     puts "Error opening #{url}: #{e}" 
   rescue URI::InvalidURIError => e
     puts "URL #{url} is not valid: #{e}"

data/lib/dumper.rb

@@ -1,36 +1,38 @@
-#--
-# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
-#
-# This file is part of Dumper.
-#
-# Smogon-API is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Smogon-API is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Smogon-API.  If not, see <http://www.gnu.org/licenses/>.
-#++
-
-require 'open-uri'
-require 'uri'
-require 'optparse'
-require 'net/http'
-require 'nokogiri'
-require 'openssl'
-require 'certified'
-require 'addressable/uri'
-require 'base64'
-
-require 'dumper/utils'
-Dir.glob(File.expand_path("../dumper/profiles/*.rb", __FILE__)).each { |f|
-  require "dumper/profiles/#{File.basename(f).split(?.)[0]}"
-}
-require 'dumper/dumper'
-
-require 'dumper/version'
+#--
+# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
+#
+# This file is part of Dumper.
+#
+# Smogon-API is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Smogon-API is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Smogon-API.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+require 'open-uri'
+require 'net/http'
+require 'uri'
+require 'optparse'
+require 'base64'
+require 'nokogiri'
+require 'openssl'
+require 'certified'
+require 'addressable/uri'
+require 'json'
+require 'thread/pool'
+
+require 'dumper/utils'
+require 'dumper/dumper'
+require 'dumper/version'
+
+Dir.glob(File.expand_path("../dumper/profiles/*.rb", __FILE__)).each { |f|
+  require "dumper/profiles/#{File.basename(f).split(?.)[0]}"
+}

data/lib/dumper/dumper.rb

@@ -17,60 +17,122 @@
 # along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
 #++
 
-module Dumper; module Profiles
-  
-  def self.list
-    Dir.glob(File.expand_path('../profiles/*.rb', __FILE__)).sort { |a, b| b <=> a }.map { |f|
-      f = File.basename(f).split(?.)[0]
-    }
-  end
-  
-  def self.get(path, p, ua = '', ref = '', filename = '')
-    p = p.to_s
-    
-    begin
-      if p.start_with? 'data:image/'
-        filename = File.join path, filename == '' ? rand(1000).to_s + '.' + p.split('data:image/')[1].split(?;)[0] : filename
-        filename.gsub!(File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR)
-        if File.exists? filename
-          puts "File #{filename} already exists."
-        else
-          puts "Downloading base64 image as #{filename}..."
-          p.gsub!(/data:image\/png;base64,/, '')
-          File.open(filename, 'wb') { |f| f.write Base64.decode64(p) }
-        end
-      else
-        filename = File.join path, filename == '' ? File.basename(p) : filename
-        filename.gsub!(File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR)
-        if File.exists? filename
-          puts "File #{filename} already exists."
-        else
-          filename = File.join path, rand(1000).to_s + '.jpg' unless filename[-4] == ?. || filename[-5] == ?.
-          puts "Downloading #{p} as #{filename}..."
-          File.open(filename, 'wb') { |f| f.write open(p, 'User-Agent' => ua, 'Referer' => ref).read }
-        end
+module Dumper
+  module Profiles
+    USER_AGENT = 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:16.0) Gecko/20100101 Firefox/16.0'
+
+    class Profile
+      include Dumper::Profiles
+
+      def initialize(&block)
+        min = pool_size[:min]
+        max = pool_size[:max]
+
+        @pool = Thread.pool min, max
+        puts "Using #{min}:#{max || min} threads..."
+
+        instance_eval &block
+      end
+
+      def dump(url, path, *args)
+        raise NotImplementedError
+      end
+
+      def shutdown
+        @pool.shutdown
       end
-    rescue Exception => e
-      p e
-      puts "Error downloading \#{p}."
-      return false
     end
-    return true
-  end
 
-  def self.get_generic(url, path, xpath)
-    uri = nil
-    Nokogiri::HTML(open(url)).xpath(xpath).each { |p|
-      if p.to_s.start_with? ?/
-        uri = URI(url) if uri.nil?
-        p   = "#{uri.scheme}://#{uri.host}#{p}"
+    def pool_size
+      {
+        min: @min || 4,
+        max: @max
+      }
+    end
+
+    def verbose?
+      @verbose == nil || @verbose == true
+    end
+
+    class << self
+      def pool_size=(min, max = nil)
+        @min = min
+        @max = max
       end
-      self.get path, p
-    }
-  end
-  
-  def method_missing(method, *args, &block)
-    "'#{method.split('get_')[1]}' profile not found."
+
+      def verbose=(verbose)
+        @verbose = verbose
+      end
+
+      def verbose?
+      @verbose == nil || @verbose == true
+      end
+
+      def list
+        Dir.glob(File.expand_path('../profiles/*.rb', __FILE__)).sort { |a, b| b <=> a }.map { |f|
+          f = File.basename(f).split(?.)[0]
+        }
+      end
+      
+      def get(path, url, options = {})
+        url     = url.to_s
+        
+        begin
+          if url.start_with? 'data:image/'
+            filename = File.join path, options[:filename] || rand(1000).to_s + '.' + url.split('data:image/')[1].split(?;)[0]
+            filename.gsub! File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR
+
+            url.gsub /data:image\/png;base64,/, ''
+
+            if File.exists? filename
+              puts "File #{filename} already exists."           if verbose?
+            else
+              puts "Downloading base64 image as #{filename}..." if verbose?
+              File.open(filename, 'wb') { |f|
+                f.write Base64.decode64(url)
+              }
+            end
+          else
+            filename = File.join path, options[:filename] || File.basename(url)
+            filename.gsub! File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR
+
+            if File.exists? filename
+              puts "File #{filename} already exists." if verbose?
+            else
+              filename = File.join(path, rand(1000).to_s + '.jpg') unless filename[-4] == ?. || filename[-5] == ?.
+              puts "Downloading #{url} as #{filename}..." if verbose?
+
+              File.open(filename, 'wb') { |f|
+                f.write open(url,
+                  'User-Agent' => options[:user_agent] || USER_AGENT,
+                  'Referer'    => options[:referer   ] || url
+                ).read
+              }
+            end
+          end
+        rescue Exception => e
+          p e
+          puts "Error downloading #{url}."
+          return false
+        end
+        
+        true
+      end
+
+      def get_generic(url, path, xpath)
+        uri = nil
+        Nokogiri::HTML(open(url)).xpath(xpath).each { |p|
+          if p.to_s.start_with? ?/
+            uri = URI(url) if uri.nil?
+            p   = "#{uri.scheme}://#{uri.host}#{p}"
+          end
+          get path, p
+        }
+      end
+    end
+    
+    def method_missing(method, *args, &block)
+      "'#{method.split('get_')[1]}' profile not found."
+    end
   end
-  
-end; end
+end

data/lib/dumper/profiles/4chan.rb

@@ -1,34 +1,50 @@
-#--
-# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
-#
-# This file is part of Dumper.
-#
-# Dumper is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Dumper is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
-#++
-
-module Dumper
-  module Profiles
-
-    def self.get_4chan(url, path, from = 1, to = 1)
-      Nokogiri::HTML(open(url)).xpath('//a[@class = "fileThumb"]/@href').each { |p|
-        self.get path, "http:#{p}"
-      }
-    end
-
-    def self.info_4chan
-      { :from => false, :to => false }
-    end
-
-  end
+#--
+# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
+#
+# This file is part of Dumper.
+#
+# Dumper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dumper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+module Dumper
+  module Profiles
+
+    class FourChan < Profile
+      def dump(url, path, from, to)
+        from -= 1
+        to   -= 1 if to >= 1
+
+        Nokogiri::HTML(open(url)).xpath('//a[@class = "fileThumb"]/@href')[from..to].each { |p|
+          @pool.process {
+            Dumper::Profiles.get path, "http:#{p}"
+          }
+        }
+      end
+    end
+
+    class << self
+      def get_4chan(url, path, from = 1, to = -1)
+        FourChan.new { |p|
+          p.dump     url, path, from, to
+          p.shutdown
+        }
+      end
+
+      def info_4chan
+        { from: :enabled, to: :enabled, type: :images }
+      end
+    end
+
+  end
 end