image-dumper 0.5.5 → 0.6

data/lib/dumper/profiles/multiplayer.rb

@@ -1,42 +1,56 @@
-#--
-# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
-#
-# This file is part of Dumper.
-#
-# Dumper is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Dumper is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
-#++
-
-module Dumper
-  module Profiles
-
-    def self.get_multiplayer(url, path, from = 1, to = 1)
-      url += '?pagina=1' unless url.split(??).last.start_with? 'pagina'
-      
-      from.upto(to) { |i|
-        Nokogiri::HTML(open(url + i.to_s)).xpath('//div[@class="thumb"]/a/@href').each { |p|
-          next unless p.to_s.start_with? '/immagini/'
-          
-          Nokogiri::HTML(open("http://multiplayer.it#{p}")).xpath('//a[@class="button"]/@href').each { |u|
-            self.get path, u
-          }
-        }
-      }
-    end
-
-    def self.info_multiplayer
-      { :from => true, :to => true }
-    end
-
-  end
+#--
+# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
+#
+# This file is part of Dumper.
+#
+# Dumper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dumper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+module Dumper
+  module Profiles
+
+    class Multiplayer < Profile
+      def dump(url, path, from, to)
+        from -= 1
+        to   -= 1 if to >= 1
+
+        url.gsub! /\/giochi\//, '/immagini/galleria/'
+        url.gsub! '.html', ''
+        gallery = JSON.parse open("#{url}?from=#{from}").read
+
+        to -= from if to >= 1
+        
+        gallery['objects'].reverse[0..to].each { |image|
+          @pool.process {
+            Dumper::Profiles.get path, image['image']
+          }
+        }
+      end
+    end
+
+    class << self
+      def get_multiplayer(url, path, from = 1, to = -1)
+        Multiplayer.new { |p|
+          p.dump     url, path, from, to
+          p.shutdown
+        }
+      end
+
+      def info_multiplayer
+        { from: :enabled, to: :enabled, type: :images }
+      end
+    end
+
+  end
 end

data/lib/dumper/profiles/redblow.rb

@@ -1,37 +1,50 @@
-#--
-# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
-#
-# This file is part of Dumper.
-#
-# Dumper is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Dumper is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
-#++
-
-module Dumper
-  module Profiles
-
-    def self.get_redblow(url, path, from = 1, to = 1)
-      ua  = 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:16.0) Gecko/20100101 Firefox/16.0'
-      ref = url
-      
-      Nokogiri::HTML(open(url)).xpath('//img[@class="attachment-medium"]/@src').each { |p|
-        self.get path, p, ua, ref
-      }
-    end
-
-    def self.info_redblow
-      { :from => false, :to => false }
-    end
-
-  end
+#--
+# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
+#
+# This file is part of Dumper.
+#
+# Dumper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dumper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+module Dumper
+  module Profiles
+
+    class RedBlow < Profile
+      def dump(url, path, from, to)
+      from -= 1
+      to   -= 1 if to >= 1
+      
+      Nokogiri::HTML(open(url)).xpath('//img[@class="attachment-medium"]/@src')[from..to].each { |p|
+        @pool.process {
+          Dumper::Profiles.get path, p, { referer: url }
+        }
+      }
+      end
+    end
+
+    class << self
+      def get_redblow(url, path, from = 1, to = -1)
+        RedBlow.new { |p|
+          p.dump     url, path, from, to
+          p.shutdown
+        }
+      end
+
+      def info_redblow
+        { from: :enabled, to: :enabled, type: :images }
+      end
+    end
+
+  end
 end

data/lib/dumper/profiles/sankakucomplex.rb

@@ -1,59 +1,82 @@
-#--
-# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
-#
-# This file is part of Dumper.
-#
-# Dumper is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Dumper is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
-#++
-
-module Dumper
-  module Profiles
-
-    ### ALMOST BROKEN ###
-    def self.get_sankakucomplex(url, path, from = 1, to = 1)
-      if url.include?('idol.sankakucomplex') || url.include?('chan.sankakucomplex')
-        url.gsub!('index', 'index.content')
-        from.upto(to) { |page|
-          u = url + "&page=#{page}"
-          op = open(u)
-          begin
-            Nokogiri::HTML(open(u)).xpath('//a/@href').each { |p|
-              p = url.gsub(/post(.+)/, p.to_s)
-              begin
-                Nokogiri::HTML(open(p)).xpath('//a[@id="image-link"]/img/@src').each { |q|
-                  self.get path, q
-                }
-              rescue Exception => e
-                sleep 1
-                retry
-              end
-            }
-          rescue Exception => e
-            sleep 1
-            retry
-          end
-        }
-      else
-        Nokogiri::HTML(open(url)).xpath('//a[@class="highslide"]/@href').each { |p|
-          self.get path, p
-        }
-      end
-    end
-
-    def self.info_sankakucomplex
-      { :from => true, :to => true }
-    end
-
-  end
+#--
+# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
+#
+# This file is part of Dumper.
+#
+# Dumper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dumper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+module Dumper
+  module Profiles
+
+    class SankakuComplex < Profile
+      def dump(url, path, from, to)
+        if url.include?('idol.sankakucomplex') || url.include?('chan.sankakucomplex')
+          to     = 1 if to == -1
+          prefix = url.include?('idol.sankakucomplex') ? 'idol' : 'chan'
+
+          from.upto(to) { |page|
+            u = url + "&page=#{page}"
+            begin
+              op = open u
+            rescue Exception => e
+              sleep 1
+              retry
+            end
+
+            Nokogiri::HTML(op).xpath('//a/@href').each { |p|
+              next unless p.to_s.start_with? '/post/show'
+
+              @pool.process {
+                begin
+                  img = Nokogiri::HTML(open("http://#{prefix}.sankakucomplex.com/#{p}")).at_xpath('//a[@itemprop="contentUrl"]/@href').to_s
+                  Dumper::Profiles.get path, img, { referer: u }
+                rescue Exception => e
+                  retry
+                end
+              }
+            }
+          }
+        else
+          from -= 1
+          to   -= 1 if to >= 1
+
+          [].tap { |urls|
+            Nokogiri::HTML(open(url)).xpath('//a[@target="_blank"]/@href').each { |u|
+              urls << u if u.to_s.start_with? 'http://images.sankakucomplex.com/wp-content/'
+            }
+          }[from..to].each { |p|
+            @pool.process {
+              Dumper::Profiles.get path, p, { referer: url }
+            }
+          }
+        end
+      end
+    end
+
+    class << self
+      def get_sankakucomplex(url, path, from = 1, to = -1)
+        SankakuComplex.new { |p|
+          p.dump     url, path, from, to
+          p.shutdown
+        }
+      end
+
+      def info_sankakucomplex
+        { from: :enabled, to: :enabled, type: :images }
+      end
+    end
+
+  end
 end

data/lib/dumper/profiles/teca.rb

@@ -1,35 +1,55 @@
-#--
-# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
-#
-# This file is part of Dumper.
-#
-# Dumper is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Dumper is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
-#++
-
-module Dumper
-  module Profiles
-
-    def self.get_teca(url, path)
-      url = url.split('index').first
-      Nokogiri::HTML(open(url)).xpath('//a/@href').each { |p|
-        self.get(path, "#{url}/#{p}") if p.to_s =~ /.\.(png|bmp|jpeg|jpg|gif|tiff)$/i
-      }
-    end
-
-    def self.info_teca
-      { :from => false, :to => false }
-    end
-
-  end
+#--
+# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
+#
+# This file is part of Dumper.
+#
+# Dumper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dumper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+module Dumper
+  module Profiles
+
+    class Teca < Profile
+      def dump(url, path, from, to)
+        from -= 1
+        to   -= 1 if to >= 1
+        url   = url.split('index').first
+
+        [].tap { |urls|
+          Nokogiri::HTML(open(url)).xpath('//a/@href').each { |u|
+            urls << u if u.to_s =~ /.\.(png|bmp|jpeg|jpg|gif|tiff)$/i
+          }
+        }.reverse[from..to].each { |p|
+          @pool.process {
+            Dumper::Profiles.get path, "#{url}/#{p}"
+          }
+        }
+      end
+    end
+
+    class << self
+      def get_teca(url, path, from = 1, to = -1)
+        Teca.new { |p|
+          p.dump     url, path, from, to
+          p.shutdown
+        }
+      end
+
+      def info_teca
+        { from: :enabled, to: :enabled, type: :images }
+      end
+    end
+
+  end
 end

data/lib/dumper/profiles/yande.rb

@@ -1,43 +1,52 @@
-#--
-# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
-#
-# This file is part of Dumper.
-#
-# Dumper is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Dumper is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
-#++
-
-module Dumper
-  module Profiles
-
-    def self.get_yande(url, path, from = 1, to = 1)
-      ua  = 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:16.0) Gecko/20100101 Firefox/16.0'
-      ref = url
-
-      from.upto(to) { |i|
-        Nokogiri::HTML(open("#{url}&page=#{i}", 'User-Agent' => ua, 'Referer' => ref)).xpath('//a[@class="thumb"]/@href').each { |p|
-          img = Nokogiri::HTML(open("https://yande.re#{p}", 'User-Agent' => ua, 'Referer' => ref)).at_xpath('//img[@id="image"]/@src').text
-          self.get path, img, ua, ref
-        }
-        
-        puts "--- Page #{i} now... ---" # there are so much pages sometimes...
-        puts
-      }
-    end
-
-    def self.info_yande
-      { :from => true, :to => true }
-    end
-
-  end
+#--
+# Copyright(C) 2013 Giovanni Capuano <webmaster@giovannicapuano.net>
+#
+# This file is part of Dumper.
+#
+# Dumper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dumper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dumper.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+module Dumper
+  module Profiles
+
+    class YandeRe < Profile
+      def dump(url, path, from, to)
+        from.upto(to) { |i|
+          puts "--- Page #{i} ---" if Dumper::Profiles.verbose?
+
+          Nokogiri::HTML(open("#{url}&page=#{i}", 'User-Agent' => Dumper::Profiles::USER_AGENT, 'Referer' => url)).xpath('//a[@class="thumb"]/@href').each { |p|
+            @pool.process {
+              img = Nokogiri::HTML(open("https://yande.re#{p}", 'User-Agent' => Dumper::Profiles::USER_AGENT, 'Referer' => url)).at_xpath('//img[@id="image"]/@src').text
+              Dumper::Profiles.get path, img, { referer: url }
+            }
+          }
+        }
+      end
+    end
+
+    class << self
+      def get_yande(url, path, from = 1, to = 1)
+        YandeRe.new { |p|
+          p.dump     url, path, from, to
+          p.shutdown
+        }
+      end
+
+      def info_yande
+        { from: :enabled, to: :enabled, type: :pages }
+      end
+    end
+
+  end
 end