rypper 0.0.1

data/bin/rypper

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# -*- mode: ruby -*-
+
+require 'rypper'
+
+Rypper::CLI.main

data/lib/net/http_client.rb

@@ -0,0 +1,73 @@
+# -*- encoding: utf-8 -*-
+
+require 'net/http'
+require 'uri'
+
+module Net
+  class HTTPClient
+    class << self
+      def storage
+        @storage ||= {}
+      end
+
+      def from_storage(host, port=80, renew=false)
+        if self.storage.has_key?(host) and not renew
+          connection = self.storage[host]
+        else
+          connection = self.new(host, port)
+          self.storage[host] = connection
+        end
+        return connection
+      end
+    end
+
+    attr_reader :host, :port, :timeout
+    attr_writer :keep_alive
+    attr_accessor :user_agent
+
+    def initialize(host, port=80, timeout=15)
+      @host = host.to_s.strip
+      @port = port.to_i
+      @keep_alive = true
+      @timeout = timeout.to_i
+      @user_agent = "Net::HTTPClient/0.1 (Ruby #{RUBY_VERSION})"
+      @http = Net::HTTP.new(self.host, self.port)
+      @http.read_timeout = self.timeout
+      @http.open_timeout = self.timeout
+    end
+
+    def keep_alive?
+      @keep_alive ? true : false
+    end
+
+    def get(path, header={}, options={})
+      uri = path.is_a?(URI) ? path : self.to_uri(path)
+      header['Accept'] ||= '*/*'
+      header['Connection'] ||= (self.keep_alive? ? 'Keep-Alive' : 'Close')
+      header['Referer'] = uri.to_s if options[:referer_self]
+      header['User-Agent'] ||= self.user_agent
+      @http.start unless @http.started?
+      response = nil
+      begin
+        response = @http.request_get(uri.path, header)
+      rescue EOFError
+        @http = Net::HTTP.new(self.host, self.port)
+        @http.start
+        response = @http.request_get(uri.path, header)
+      end
+      if response.is_a?(Net::HTTPRedirection) && options[:follow_redirects]
+        uri_redirect = URI.parse(response['Location'])
+        header_redirect = options[:follow_with_header] ? header : options[:follow_header] || {}
+        options_redirect = options[:follow_with_options] ? options : options[:follow_options] || {}
+        options_redirect['Referer'] ||= uri.to_s 
+        connection = self.class.from_storage(uri_redirect.host, uri_redirect.port)
+        response = connection.get(uri_redirect, header_redirect, options_redirect)
+      end
+      return response
+    end
+
+    def to_uri(path=nil)
+      return URI::HTTP.build({:host => self.host, :port => self.port, :path => path})
+    end
+  end
+end

data/lib/rypper.rb

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+
+require 'net/http_client'
+
+require 'getoptlong'
+require 'uri'
+
+require 'rubygems'
+require 'nokogiri'
+
+require 'rypper/cli'
+require 'rypper/counter'
+require 'rypper/extractor'
+require 'rypper/loader'
+require 'rypper/uri'
+
+if File.basename($0) == __FILE__
+  Rypper::CLI.main()
+end

data/lib/rypper/cli.rb

@@ -0,0 +1,77 @@
+# -*- encoding: utf-8 -*-
+
+module Rypper
+  class CLI
+    OPTS = [
+      ['--help', '-h', GetoptLong::NO_ARGUMENT],
+      ['--output', '-o', GetoptLong::REQUIRED_ARGUMENT],
+      ['--overwrite', '-w', GetoptLong::NO_ARGUMENT],
+    ]
+
+    def self.getopt()
+      opts = {}
+      GetoptLong.new(*OPTS).each do |opt, arg|
+        opt_sym = opt.sub('--', '').to_sym
+        opt_type = OPTS.find {|e| e.first == opt}.last
+        if opt_type == GetoptLong::NO_ARGUMENT
+          opts[opt_sym] = true
+        else
+          opts[opt_sym] = arg
+        end
+      end
+      opts
+    end
+
+    def self.main()
+      opts = self.getopt()
+      argv = ARGV
+      if argv.count != 2
+        puts "USAGE: ruby #{$0} <uri> <selector>"
+        exit 1
+      end
+      uri = Rypper::URI.new(argv[0]) # 'http://www.mangafox.com/manga/history_s_strongest_disciple_kenichi/v[01-45:vol]/c[001-459:chap]/[1-99:pic].html'
+      uri.parse!
+      uri.first!
+    
+      extractor = nil
+      extractor = Rypper::Extractor.new(argv[1]) # '#image'
+      counter = uri.counter[uri.order.last]
+    
+      puts "Processing #{uri.uri} ..."
+      while true
+        html_uri = uri.to_uri
+        puts " * #{html_uri} ..."
+        html = Rypper::Loader.get(html_uri)
+        if html.is_a?(String)
+          extractor.extract!(html).each do |image_uri|
+            if image_uri.is_a?(String)
+              print "   * #{image_uri} ..."
+              image_path = uri.to_path(File.extname(image_uri))
+              if !File.exists?(image_path) || opts.has_key?(:overwrite)
+                Rypper::Loader.mkdir!(File.dirname(image_path))
+                image_file = File.open(image_path, 'w')
+                image_file.binmode
+                image_file.write(Rypper::Loader.get(image_uri))
+                image_file.close
+                puts ' OK'
+              else
+                puts ' Exists: Skipping'
+              end
+            else
+              puts ' * Imageless'
+            end
+          end
+        else
+          counter.last!
+          puts ' * Last'
+        end
+        uri.next!
+        break if uri.first?
+      end
+      
+      puts 'OK'
+      
+      exit 0
+    end
+  end
+end

data/lib/rypper/counter.rb

@@ -0,0 +1,60 @@
+# -*- encoding: utf-8 -*-
+
+module Rypper
+  class Counter
+    attr_reader :match
+    attr_reader :lower
+    attr_reader :upper
+    attr_reader :digits
+    attr_reader :state
+
+    def initialize(match, lower, upper, digits=1, state=nil)
+      @match = match
+      @lower = lower
+      @upper = upper
+      @digits = digits
+      @state = state || @lower
+    end
+
+    def first!
+      @state = @lower
+      self
+    end
+
+    def first?
+      @state == @lower
+    end
+
+    def prev!
+      if @state > @lower
+        @state -= 1
+      else
+        @state = @upper
+      end
+      self
+    end
+
+    def next!
+      if @state < @upper
+        @state += 1
+      else
+        @state = @lower
+      end
+      self
+    end
+
+    def last!
+      @state = @upper
+      self
+    end
+
+    def last?
+      @state == @upper
+    end
+
+    def to_s(digits=nil)
+      digits ||= self.digits
+      @state.to_s.rjust(digits, '0')
+    end
+  end
+end

data/lib/rypper/extractor.rb

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+
+module Rypper
+  class Extractor
+    attr_reader :selector
+
+    def initialize(selector)
+      @selector = selector
+    end
+
+    def extract!(html)
+      unless html.kind_of?(Nokogiri::HTML::Document)
+        html = Nokogiri::HTML(html)
+      end
+      res = []
+      elems = html.search(self.selector)
+      if elems.count == 1
+        elem = elems.first
+        if elem.name == 'img'
+          res << elem[:src]
+        elsif elem.name == 'a'
+          res << elem[:href]
+        else
+          res << elem
+        end
+      else
+        res = elems
+      end
+      res
+    end
+  end
+end

data/lib/rypper/loader.rb

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+
+module Rypper
+  class Loader
+    def self.mkdir!(path)
+      path = path.to_s
+      parts = []
+      path.split(File::Separator).each do |part|
+        parts << part
+        sub_path = File.join(*parts)
+        Dir.mkdir(sub_path) unless File.directory?(sub_path)
+      end
+      File.directory?(path)
+    end
+
+    def self.get(uri)
+      unless uri.kind_of?(URI)
+        uri = ::URI.parse(uri.to_s)
+      end
+      client = Net::HTTPClient.from_storage(uri.host)
+      response = client.get(uri)
+      if response.code.to_i == 200
+        response.body
+      else
+        response.code.to_i
+      end
+    end
+  end
+end

data/lib/rypper/uri.rb

@@ -0,0 +1,99 @@
+# -*- encoding: utf-8 -*-
+
+module Rypper
+  class URI
+    REGEXP_COUNTER = /\[(\d+)\-(\d+)\:(\w+)\]/
+    REGEXP_NAME = /\[\:(\w+)\]/
+
+    attr_accessor :uri
+    attr_reader :counter
+    attr_reader :order
+
+    def initialize(uri)
+      self.uri = uri.to_s
+    end
+
+    def parse!
+      @counter = {}
+      @order = []
+      self.uri.scan(REGEXP_COUNTER) do
+        match = $~
+        lower = match[1].to_i
+        upper = match[2].to_i
+        digits = match[1].start_with?('0') ? match[1].length : 1
+        name = match[3].intern
+        self.counter[name] = Counter.new(match.to_s, lower, upper, digits)
+        self.order << name
+      end
+      self
+    end
+
+    def first!
+      self.counter.each_value(&:first!)
+      self
+    end
+
+    def first?
+      self.counter.values.all?(&:first?)
+    end
+
+    def prev!
+      self.order.reverse.each do |name|
+        cntr = self.counter[name]
+        if cntr.prev!.state != cntr.upper
+          break
+        end
+      end
+      self
+    end
+
+    def next!
+      self.order.reverse.each do |name|
+        cntr = self.counter[name]
+        if cntr.next!.state != cntr.lower
+          break
+        end
+      end
+      self
+    end
+
+    def last!
+      self.counter.each_value(&:last!)
+      self
+    end
+
+    def last?
+      self.counter.values.all?(&:last?)
+    end
+
+    def to_s
+      s = self.uri.dup
+      self.counter.each do |name, counter|
+        value = counter.to_s
+        s.gsub!(counter.match, value)
+        s.gsub!(":[#{name}]", value)
+      end
+      s
+    end
+
+    def to_uri
+      ::URI.parse(self.to_s)
+    end
+
+    def to_path(extension=nil, cntr_sep=nil, path_sep=nil, cdigits=nil)
+      cntr_sep ||= '_'
+      path_sep ||= File::Separator
+      p = []
+      self.order.each do |name|
+        cnt = self.counter[name]
+        p << name.to_s
+        p << cntr_sep
+        p << cnt.to_s(cdigits)
+        p << path_sep
+      end
+      p.pop
+      p << extension unless extension.nil?
+      p.join()
+    end
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification 
+name: rypper
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Michael Nowak
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-01-15 00:00:00 +01:00
+default_executable: rypper
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 5
+        version: "1.5"
+  type: :runtime
+  version_requirements: *id001
+description: Rypper
+email: thexsystem@gmail.com
+executables: 
+- rypper
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/net/http_client.rb
+- lib/rypper/cli.rb
+- lib/rypper/counter.rb
+- lib/rypper/extractor.rb
+- lib/rypper/loader.rb
+- lib/rypper/uri.rb
+- lib/rypper.rb
+has_rdoc: true
+homepage: https://github.com/THExSYSTEM/rypper
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 8
+      - 7
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Rypper
+test_files: []
+