image-dumper 0.6.3 → 0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c4a215cb421e1431d04d5f6c326a9f5b83f6877
-  data.tar.gz: 348906e916a00fce766129e6f045bf146dc3f2cf
+  metadata.gz: 06a97439e109b4b00205dc78978c35e810ad4990
+  data.tar.gz: e564cf847b687c07c3e7496377e57f3a7d40265a
 SHA512:
-  metadata.gz: 7cb20d9ea19acfd8cf88a6b35777f1d58acd0175a8937ecc0b59de04889f6ba7a05a8020611905ce0832390e85724e5c4dcc5ba568d42c2f6c874eb392505d3a
-  data.tar.gz: d136a79f06cda4983f79488a35c060dfd55ffeb529942bdd4e090a9ef1e5734a3159640e9ef3be6f3d7d4638e9598153a0cf03e6aff818ba7bf2857a8d758aa4
+  metadata.gz: e213fdd13e3318788c99a60b4ad749a60927c561e03802f834b4d7d29371f08a07c04c68fa8e3fba16ded775ff211f388397400c989580522920f90ddef7b68f
+  data.tar.gz: c0e9703a9be61dfe13d948b5b45d0bf512725579d2f807e00e7e5da014a72d4def01992985f89a37747b80a7ac41c3b35a9854b5e01c34b031692f3cf82ca6ee

data/bin/dumper

@@ -27,14 +27,14 @@ OptionParser.new do |o|
   
   o.on '-l', '--list',            'Show available profiles' do
     abort 'Profiles available:'.tap { |s|
-      Dumper::Profiles::list.sort { |a, b| a <=> b }.each { |p| s << "\n" + (' ' * 3) + p }
+      Dumper.list.sort { |a, b| a <=> b }.each { |p| s << "\n" + (' ' * 3) + p }
     } if ARGV.empty?
   end
 
   o.on '-i', '--info PROFILE',    'Get info about profiles' do |profile|
-    if Dumper::Profiles::list.include? profile
+    if Dumper::Profiles.list.include? profile
       method = ("info_#{profile}").to_sym      
-      Dumper::Profiles::send(method).tap { |i|
+      Dumper::Profiles.send(method).tap { |i|
         puts "Option 'from' is #{i[:from] ? 'available' : 'not available'}."
         puts "Option 'to'   is #{i[:to]   ? 'available' : 'not available'}."
         abort "It dumps #{i[:type]}."
@@ -48,11 +48,11 @@ OptionParser.new do |o|
     options[:url]    << url
   end
   
-  o.on '-o', '--from PAGE',       'Start to save from... (if allowed)' do |pages|
+  o.on '-f', '--from PAGE',       'Start to save from... (if allowed)' do |pages|
     options[:from]    = pages.to_i
   end
   
-  o.on '-g', '--to PAGE',         'Finish to save at... (if allowed)'  do |pages|
+  o.on '-t', '--to PAGE',         'Finish to save at... (if allowed)'  do |pages|
     options[:to]      = pages.to_i
   end
   
@@ -70,14 +70,18 @@ OptionParser.new do |o|
   
   o.on '-t', '--threads MIN:MAX',  'Set the number of threads to use'   do |threads|
     threads = threads.split ?:
-    Dumper::Profiles.pool_size = threads[0].to_i, threads[1].to_i
+    Dumper.pool_size = threads[0].to_i, threads[1].to_i
   end
   
-  o.on '-s', '--silence',         'Print only important messages' do
-    Dumper::Profiles.verbose = false
+  o.on '-m', '--silence',          'Print only important messages'    do
+    Dumper.shut_up!
   end
   
-  o.on '-f', '--file FILE',       'File containing in each line "URL||Folder"' do |file|
+  o.on '-o', '--output ',          'Print the logs on the given file' do |file|
+    Dumper::Logger.redirect_on :file, file
+  end
+  
+  o.on '-s', '--file FILE',       'File containing in each line "URL||Folder"' do |file|
     file = File.open(file).read.gsub(/\r\n?/, "\n")
     file.each_line { |line|
       split              = line.split '||'
@@ -99,15 +103,15 @@ options[:url].each_with_index { |url, i|
 
     host = options[:profile] || URI.parse(url).host.split(?.)[-2]
 
-    if options[:xpath].nil?  && Dumper::Profiles::list.include?(host.gsub(?-, ?_))
+    if options[:xpath].nil?  && Dumper.list.include?(host.gsub(?-, ?_))
       method = "get_#{host.gsub ?-, ?_}".to_sym
       case
-        when options[:from] && options[:to] then Dumper::Profiles::send method, url, options[:path][i], options[:from], options[:to]
-        when options[:from]                 then Dumper::Profiles::send method, url, options[:path][i], options[:from]
-        else                                     Dumper::Profiles::send method, url, options[:path][i]
+        when options[:from] && options[:to] then Dumper::Profiles.send method, url, options[:path][i], options[:from], options[:to]
+        when options[:from]                 then Dumper::Profiles.send method, url, options[:path][i], options[:from]
+        else                                     Dumper::Profiles.send method, url, options[:path][i]
       end
     else
-      Dumper::Profiles::get_generic  url, options[:path][i], options[:xpath]
+      Dumper::Profiles.get_generic  url, options[:path][i], options[:xpath]
     end
   rescue Nokogiri::XML::XPath::SyntaxError => e
     puts e.to_s.gsub /expression/, 'xpath'

data/lib/dumper.rb

@@ -28,11 +28,17 @@ require 'certified'
 require 'addressable/uri'
 require 'json'
 require 'thread/pool'
+require 'observer'
 
 require 'dumper/utils'
+require 'dumper/logger'
+require 'dumper/profile'
+require 'dumper/profiles'
 require 'dumper/dumper'
 require 'dumper/version'
 
 Dir.glob(File.expand_path("../dumper/profiles/*.rb", __FILE__)).each { |f|
   require "dumper/profiles/#{File.basename(f).split(?.)[0]}"
 }
+
+Dumper.add_observer Dumper::Logger.new

data/lib/dumper/dumper.rb

@@ -18,128 +18,126 @@
 #++
 
 module Dumper
-  module Profiles
-    USER_AGENT = 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:16.0) Gecko/20100101 Firefox/16.0'
+  USER_AGENT = 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:16.0) Gecko/20100101 Firefox/16.0'
 
-    class Profile
-      include Dumper::Profiles
+  def pool_size
+    {
+      min: @min || 4,
+      max: @max
+    }
+  end
 
-      def initialize(&block)
-        min = pool_size[:min]
-        max = pool_size[:max]
+  def verbose?
+    @verbose == nil || @verbose == true
+  end
+    alias_method :mute?, :verbose?
 
-        @pool = Thread.pool min, max
-        puts "Using #{min}:#{max || min} threads..."
+  def mute?
+    @verbose == false
+  end
+    alias_method :muted?, :mute?
 
-        instance_eval &block
-      end
+  class << self
+    include Observable
 
-      def dump(url, path, *args)
-        raise NotImplementedError
-      end
+    def pool_size=(min, max = nil)
+      @min = min
+      @max = max
+    end
 
-      def shutdown
-        @pool.shutdown
-      end
+    def verbose=(verbose)
+      @verbose = verbose
     end
 
-    def pool_size
-      {
-        min: @min || 4,
-        max: @max
-      }
+    def shut_up!
+      @verbose == false
     end
+      alias_method :mute!,   :shut_up!
+
+    def verbose!
+      @verbose == true
+    end
+      alias_method :unmute!, :verbose!
 
     def verbose?
       @verbose == nil || @verbose == true
     end
 
-    class << self
-      def pool_size=(min, max = nil)
-        @min = min
-        @max = max
-      end
-
-      def verbose=(verbose)
-        @verbose = verbose
-      end
+    def list
+      Dir.glob(File.expand_path('../profiles/*.rb', __FILE__)).sort { |a, b| b <=> a }.map { |f|
+        f = File.basename(f).split(?.)[0]
+      }
+    end
+    
+    def get(path, url, options = {})
+      url    = url.to_s
+      errors = 0
+      
+      begin
+        if url.start_with? 'data:image/'
+          filename = File.join path, options[:filename] || rand(1000).to_s + '.' + url.split('data:image/')[1].split(?;)[0]
+          filename.gsub! File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR
 
-      def verbose?
-      @verbose == nil || @verbose == true
-      end
+          url.gsub /data:image\/png;base64,/, ''
 
-      def list
-        Dir.glob(File.expand_path('../profiles/*.rb', __FILE__)).sort { |a, b| b <=> a }.map { |f|
-          f = File.basename(f).split(?.)[0]
-        }
-      end
-      
-      def get(path, url, options = {})
-        url    = url.to_s
-        errors = 0
-        
-        begin
-          if url.start_with? 'data:image/'
-            filename = File.join path, options[:filename] || rand(1000).to_s + '.' + url.split('data:image/')[1].split(?;)[0]
-            filename.gsub! File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR
-
-            url.gsub /data:image\/png;base64,/, ''
-
-            if File.exists? filename
-              puts "File #{filename} already exists."           if verbose?
-            else
-              puts "Downloading base64 image as #{filename}..." if verbose?
-              File.open(filename, 'wb') { |f|
-                f.write Base64.decode64(url)
-              }
-            end
+          if File.exists? filename
+            changed
+            notify_observers error:  "File #{filename} already exists."
           else
-            filename = File.join path, options[:filename] || File.basename(url)
-            filename.gsub! File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR
-
-            if File.exists? filename
-              puts "File #{filename} already exists." if verbose?
-            else
-              filename = File.join(path, rand(1000).to_s + '.jpg') unless filename[-4] == ?. || filename[-5] == ?.
-              puts "Downloading #{url} as #{filename}..." if verbose?
-
-              File.open(filename, 'wb') { |f|
-                f.write open(url,
-                  'User-Agent' => options[:user_agent] || USER_AGENT,
-                  'Referer'    => options[:referer   ] || url
-                ).read
-              }
-            end
+            changed
+            notify_observers status: "Downloading base64 image as #{filename}..."
+            File.open(filename, 'wb') { |f|
+              f.write Base64.decode64(url)
+            }
           end
-        rescue Exception => e
-          if errors <= 3
-            errors += 1
-            sleep 3
-            retry
+        else
+          filename = File.join path, options[:filename] || File.basename(url)
+          filename.gsub! File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR
+
+          if File.exists? filename
+            changed
+            notify_observers error:  "File #{filename} already exists."
           else
-            p e
-            puts "Error downloading #{url}."
-            return false
+            filename = File.join(path, rand(1000).to_s + '.jpg') unless filename[-4] == ?. || filename[-5] == ?.
+            changed
+            notify_observers status: "Downloading #{url} as #{filename}..."
+
+            File.open(filename, 'wb') { |f|
+              f.write open(url,
+                'User-Agent' => options[:user_agent] || USER_AGENT,
+                'Referer'    => options[:referer   ] || url
+              ).read
+            }
           end
         end
-        
-        true
-      end
-
-      def get_generic(url, path, xpath)
-        uri = nil
-        Nokogiri::HTML(open(url)).xpath(xpath).each { |p|
-          if p.to_s.start_with? ?/
-            uri = URI(url) if uri.nil?
-            p   = "#{uri.scheme}://#{uri.host}#{p}"
-          end
-          get path, p
-        }
+      rescue Exception => e
+        if errors <= 3
+          errors += 1
+          sleep 3
+          retry
+        else
+          changed
+          notify_observers critical_error: "Error downloading #{url}.", critical_error_dump: e
+          return false
+        end
       end
+      
+      true
     end
-    
-    def method_missing(method, *args, &block)
-      "'#{method.split('get_')[1]}' profile not found."
+
+    def get_generic(url, path, xpath)
+      uri = nil
+      Nokogiri::HTML(open(url)).xpath(xpath).each { |p|
+        if p.to_s.start_with? ?/
+          uri = URI(url) if uri.nil?
+          p   = "#{uri.scheme}://#{uri.host}#{p}"
+        end
+        get path, p
+      }
     end
   end
+  
+  def method_missing(method, *args, &block)
+    "'#{method.split('get_')[1]}' profile not found."
+  end
 end

data/lib/dumper/logger.rb

@@ -0,0 +1,61 @@
+#--
+# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
+#
+# This file is part of Dumper.
+#
+# Dumper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dumper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+module Dumper
+  class Logger
+
+    class << self
+      def redirect_on(where, file)
+        @@where = where
+        @@file  = !file || file.empty? ? 'dumper.log' : file
+      end
+
+      def log_on_file(file, data)
+        File.open(file, ?a) { |file|
+          data.each { |status, message|
+            file.puts status == :critical_error_dump ? message.inspect : message
+          }
+        }
+      end
+
+      def log_on_screen(data)
+        data.each { |status, message|
+          if status == :critical_error_dump
+            p    message
+          else
+            puts message
+          end
+        }
+      end
+    end
+
+    def initialize
+      @@where = :screen
+    end
+
+    def update(data)
+      if @@where == :file
+        Logger.log_on_file @@file, data
+      else
+        Logger.log_on_screen(data) if Dumper.verbose?
+      end
+    end
+
+  end
+end

data/lib/dumper/profile.rb

@@ -0,0 +1,46 @@
+#--
+# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
+#
+# This file is part of Dumper.
+#
+# Dumper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dumper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+module Dumper
+  class Profile
+    include Dumper
+    include Observable
+
+    def initialize(&block)
+      add_observer Dumper::Logger.new
+      
+      min = pool_size[:min]
+      max = pool_size[:max]
+
+      @pool = Thread.pool min, max
+      changed
+      notify_observers error:  "Using #{min}:#{max || min} threads..."
+
+      instance_eval &block
+    end
+
+    def dump(url, path, *args)
+      raise NotImplementedError
+    end
+
+    def shutdown
+      @pool.shutdown
+    end
+  end
+end

data/lib/dumper/profiles.rb

@@ -0,0 +1,24 @@
+#--
+# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
+#
+# This file is part of Dumper.
+#
+# Dumper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dumper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+module Dumper
+  module Profiles
+    Profile = Dumper::Profile
+  end
+end

data/lib/dumper/profiles/4chan.rb

@@ -27,7 +27,7 @@ module Dumper
 
         Nokogiri::HTML(open(url)).xpath('//a[@class = "fileThumb"]/@href')[from..to].each { |p|
           @pool.process {
-            Dumper::Profiles.get path, "http:#{p}"
+            Dumper.get path, "http:#{p}"
           }
         }
       end

data/lib/dumper/profiles/behoimi.rb

@@ -23,9 +23,9 @@ module Dumper
     class Behoimi < Profile
       def dump(url, path, from, to)
         from.upto(to) { |i|
-          Nokogiri::HTML(open("#{url}&page=#{i}", 'User-Agent' => Dumper::Profiles::USER_AGENT, 'Referer' => url)).xpath('//img[@class="preview    "]/@src').each { |p|
+          Nokogiri::HTML(open("#{url}&page=#{i}", 'User-Agent' => Dumper::USER_AGENT, 'Referer' => url)).xpath('//img[@class="preview    "]/@src').each { |p|
             @pool.process {
-              Dumper::Profiles.get path, p.to_s.gsub('preview/', ''), { referer: url }
+              Dumper.get path, p.to_s.gsub('preview/', ''), { referer: url }
             }
           }
         }

data/lib/dumper/profiles/booru.rb

@@ -25,11 +25,12 @@ module Dumper
         page = 0
         
         from.upto(to) { |i|
-          puts "--- Page #{i} ---" if Dumper::Profiles.verbose?
+          changed
+          notify_observers status: "--- Page #{i} ---"
 
           Nokogiri::HTML(open("#{url}&pid=#{page}")).xpath('//span[@class="thumb"]').each { |u|
             @pool.process {
-              Dumper::Profiles.get path, u.child.child['src'].gsub(/thumbs/, 'img').gsub(/thumbnails\//, 'images/').gsub(/thumbnail_/, '')
+              Dumper.get path, u.child.child['src'].gsub(/thumbs/, 'img').gsub(/thumbnails\//, 'images/').gsub(/thumbnail_/, '')
             }
           }
           

data/lib/dumper/profiles/deviantart.rb

@@ -27,7 +27,7 @@ module Dumper
 
         Nokogiri::HTML(open(url)).xpath('//a[@class="thumb"]')[from..to].each { |u|
           @pool.process {
-            Dumper::Profiles.get path, u['data-super-img']
+            Dumper.get path, u['data-super-img']
           }
         }
       end

data/lib/dumper/profiles/fakku.rb

@@ -28,13 +28,14 @@ module Dumper
         cdn = open(url).read.split('window.params.thumbs')[1].split('\/thumbs\/')[0].gsub(/\\\//m, ?/)[5..-1] + '/images/'
 
         from.upto(to) { |i|
-          return if errors == 10
+          return if errors == 3
           
           file     = "%03d.jpg" % i
           filename =  "#{cdn}#{file}"
 
           @pool.process {
-            unless Dumper::Profiles.get path, URI.parse(URI.encode(filename, '[]')), { referer: url }
+            unless Dumper.get path, URI.parse(URI.encode(filename, '[]')), { referer: url }
+              sleep 3
               errors += 1
               
               file = File.join(path, file).gsub(File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR)

data/lib/dumper/profiles/fc2.rb

@@ -27,7 +27,7 @@ module Dumper
 
         Nokogiri::HTML(open(url)).xpath('//a[@target="_blank"]/@href')[from..to].each { |p|
           @pool.process {
-            Dumper::Profiles.get(path, p) if p.to_s.end_with?('jpg') || p.to_s.end_with?('png')
+            Dumper.get(path, p) if p.to_s.end_with?('jpg') || p.to_s.end_with?('png')
           }
         }
       end

data/lib/dumper/profiles/gelbooru.rb

@@ -25,11 +25,12 @@ module Dumper
         page = 0
         
         from.upto(to) { |i|
-          puts "--- Page #{i} now ---" if Dumper::Profiles.verbose?
+          changed
+          notify_observers status: "--- Page #{i} ---"
           
           Nokogiri::HTML(open("#{url}&pid=#{page}")).xpath('//span[@class="thumb"]').each { |u|
             @pool.process {
-              Dumper::Profiles.get path, u.child.child['src'].gsub(/thumbnails/, 'images').gsub(/thumbnail_/, '')
+              Dumper.get path, u.child.child['src'].gsub(/thumbnails/, 'images').gsub(/thumbnail_/, '')
             }
           }
           

data/lib/dumper/profiles/imagebam.rb

@@ -32,7 +32,7 @@ module Dumper
           }[from..to].each { |p|
             Nokogiri::HTML(open(p)).xpath('//img[@onclick="scale(this);"]/@src').each { |u|
               @pool.process {
-                Dumper::Profiles.get path, u
+                Dumper.get path, u
               }
             }
           }

data/lib/dumper/profiles/mangaeden.rb

@@ -33,7 +33,7 @@ module Dumper
           
           page = Nokogiri::HTML open("http://www.mangaeden.com#{p['href']}")
 
-          Dumper::Profiles.get dir, page.at_xpath('//img[@id="mainImg"]/@src'), { filename: '1.png' }
+          Dumper.get dir, page.at_xpath('//img[@id="mainImg"]/@src'), { filename: '1.png' }
           i += 1
           
           page.xpath('//a[@class="ui-state-default"]').each { |q|
@@ -41,7 +41,7 @@ module Dumper
 
             Nokogiri::HTML(open("http://www.mangaeden.com#{q['href']}")).xpath('//img[@id="mainImg"]/@src').each { |r|
               @pool.process {
-                Dumper::Profiles.get dir, r, { filename: "#{q.text.to_i}.png" }
+                Dumper.get dir, r, { filename: "#{q.text.to_i}.png" }
               }
             }
           }

data/lib/dumper/profiles/mangago.rb

@@ -31,7 +31,7 @@ module Dumper
             url  = page.at_xpath('//a[@id="pic_container"]/@href').to_s
             scan = page.at_xpath('//img[@id="page1"]/@src').to_s[0..-3]
 
-            Dumper::Profiles.get path, scan, { referer: url, filename: "#{i}.#{scan.split(?.).last}" }
+            Dumper.get path, scan, { referer: url, filename: "#{i}.#{scan.split(?.).last}" }
           }
         }
       end

data/lib/dumper/profiles/mangahere.rb

@@ -39,7 +39,7 @@ module Dumper
           Nokogiri::HTML(open(chapter)).xpath('//select[@class="wid60"]/option').each { |q|
             @pool.process {
               scan = Nokogiri::HTML(open(q['value'])).xpath('//section[@id="viewer"]/a/img/@src')[0].to_s
-              Dumper::Profiles.get dir, scan, { filename: "#{q.text}.png" }
+              Dumper.get dir, scan, { filename: "#{q.text}.png" }
             }
           }
         }

data/lib/dumper/profiles/multiplayer.rb

@@ -33,7 +33,7 @@ module Dumper
         
         gallery['objects'].reverse[0..to].each { |image|
           @pool.process {
-            Dumper::Profiles.get path, image['image']
+            Dumper.get path, image['image']
           }
         }
       end

data/lib/dumper/profiles/redblow.rb

@@ -27,7 +27,7 @@ module Dumper
       
       Nokogiri::HTML(open(url)).xpath('//img[@class="attachment-medium"]/@src')[from..to].each { |p|
         @pool.process {
-          Dumper::Profiles.get path, p, { referer: url }
+          Dumper.get path, p, { referer: url }
         }
       }
       end

data/lib/dumper/profiles/sankakucomplex.rb

@@ -27,7 +27,7 @@ module Dumper
           prefix = url.include?('idol.sankakucomplex') ? 'idol' : 'chan'
 
           from.upto(to) { |page|
-            u = url + "&page=#{page}"
+            u = "#{url}&page=#{page}"
             begin
               op = open u
             rescue Exception => e
@@ -37,13 +37,18 @@ module Dumper
 
             Nokogiri::HTML(op).xpath('//a/@href').each { |p|
               next unless p.to_s.start_with? '/post/show'
+              errors = 0
 
               @pool.process {
                 begin
                   img = Nokogiri::HTML(open("http://#{prefix}.sankakucomplex.com/#{p}")).at_xpath('//a[@itemprop="contentUrl"]/@href').to_s
-                  Dumper::Profiles.get path, img, { referer: u }
+                  Dumper.get path, img, { referer: u }
                 rescue Exception => e
-                  retry
+                  if errors <= 3
+                    sleep 3
+                    errors += 1
+                    retry
+                  end
                 end
               }
             }
@@ -58,7 +63,7 @@ module Dumper
             }
           }[from..to].each { |p|
             @pool.process {
-              Dumper::Profiles.get path, p, { referer: url }
+              Dumper.get path, p, { referer: url }
             }
           }
         end

data/lib/dumper/profiles/teca.rb

@@ -32,7 +32,7 @@ module Dumper
           }
         }.reverse[from..to].each { |p|
           @pool.process {
-            Dumper::Profiles.get path, "#{url}/#{p}"
+            Dumper.get path, "#{url}/#{p}"
           }
         }
       end

data/lib/dumper/profiles/yande.rb

@@ -23,12 +23,13 @@ module Dumper
     class YandeRe < Profile
       def dump(url, path, from, to)
         from.upto(to) { |i|
-          puts "--- Page #{i} ---" if Dumper::Profiles.verbose?
+          changed
+          notify_observers status: "--- Page #{i} ---"
 
-          Nokogiri::HTML(open("#{url}&page=#{i}", 'User-Agent' => Dumper::Profiles::USER_AGENT, 'Referer' => url)).xpath('//a[@class="thumb"]/@href').each { |p|
+          Nokogiri::HTML(open("#{url}&page=#{i}", 'User-Agent' => Dumper::USER_AGENT, 'Referer' => url)).xpath('//a[@class="thumb"]/@href').each { |p|
             @pool.process {
-              img = Nokogiri::HTML(open("https://yande.re#{p}", 'User-Agent' => Dumper::Profiles::USER_AGENT, 'Referer' => url)).at_xpath('//img[@id="image"]/@src').text
-              Dumper::Profiles.get path, img, { referer: url }
+              img = Nokogiri::HTML(open("https://yande.re#{p}", 'User-Agent' => Dumper::USER_AGENT, 'Referer' => url)).at_xpath('//img[@id="image"]/@src').text
+              Dumper.get path, img, { referer: url }
             }
           }
         }

data/lib/dumper/version.rb

@@ -19,6 +19,6 @@
 
 module Dumper
   def self.version
-    '0.6.3'
+    '0.7'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: image-dumper
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: '0.7'
 platform: ruby
 authors:
 - Giovanni Capuano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-30 00:00:00.000000000 Z
+date: 2014-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -145,6 +145,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/dumper/dumper.rb
+- lib/dumper/logger.rb
+- lib/dumper/profile.rb
 - lib/dumper/profiles/4chan.rb
 - lib/dumper/profiles/behoimi.rb
 - lib/dumper/profiles/booru.rb
@@ -161,6 +163,7 @@ files:
 - lib/dumper/profiles/sankakucomplex.rb
 - lib/dumper/profiles/teca.rb
 - lib/dumper/profiles/yande.rb
+- lib/dumper/profiles.rb
 - lib/dumper/utils.rb
 - lib/dumper/version.rb
 - lib/dumper.rb