booru 0.0.1

data/lib/booru/client.rb

@@ -0,0 +1,33 @@
+# -*- encoding : utf-8 -*-
+module Booru
+  class Client
+    include Error
+    include Request
+
+    include API
+    include API::Artist
+    include API::Comment
+    include API::Favorite
+    include API::Note
+    include API::Pool
+    include API::Post
+    include API::Tag
+    include API::User
+    include API::Wiki
+
+    def initialize(user=nil, pass=nil, format='json')
+      @user = user
+      @pass = pass ? hash_password(pass) : nil
+      @format = format.to_sym
+    end
+
+    def hash_password(pass)
+      Digest::SHA1.hexdigest(pass)
+    end
+
+    def url_file_segment(url, md5)
+      CGI.unescape(url)
+    end
+
+  end
+end

data/lib/booru/client/behoimi.rb

@@ -0,0 +1,29 @@
+# -*- encoding : utf-8 -*-
+module Booru
+  class Behoimi < Client
+  attr_accessor :host, :referer
+
+    def initialize(user = nil, pass = nil)
+      @host    = "http://behoimi.org"
+      @referer = "http://behoimi.org/post/show"
+      super
+    end
+
+    def hash_password(pass)
+      salt = 'meganekko-heaven'
+      salted_pass = "#{salt}--#{pass}--"
+      Digest::SHA1.hexdigest(salted_pass)
+      super
+    end
+
+    def url_file_segment(url, md5)
+      strings = ["http://behoimi.org/"]
+      strings.each do |str|
+        url = url.gsub(str, "")
+      end
+      url = url.gsub(/data\/.{2}\/.{2}\//,"")
+      super
+    end
+
+  end
+end

data/lib/booru/client/danbooru.rb

@@ -0,0 +1,25 @@
+# -*- encoding : utf-8 -*-
+module Booru
+  class Danbooru < Client
+    attr_accessor :host, :referer
+
+    def initialize(user = nil, pass = nil)
+      @host    = "http://danbooru.donmai.us"
+      @referer = "http://danbooru.donmai.us"
+      super
+    end
+
+    def hash_password(pass)
+      salt = 'choujin-steiner'
+      salted_pass = "#{salt}--#{pass}--"
+      Digest::SHA1.hexdigest(salted_pass)
+      super
+    end
+
+    def url_file_segment(url, md5)
+      url = url.gsub(/http:\/\/.+\/.+\//, "")
+      super
+    end
+
+  end
+end

data/lib/booru/client/konachan.rb

@@ -0,0 +1,25 @@
+# -*- encoding : utf-8 -*-
+module Booru
+  class Konachan < Client
+    attr_accessor :host, :referer
+
+    def initialize(user = nil, pass = nil)
+      @host    = "http://konachan.com"
+      @referer = "http://konachan.com"
+      super
+    end
+
+    def hash_password(pass)
+      salt = 'So-I-Heard-You-Like-Mupkids-?'
+      salted_pass = "#{salt}--#{pass}--"
+      Digest::SHA1.hexdigest(salted_pass)
+      super
+    end
+
+    def url_file_segment(url, md5)
+      url = url.gsub(/http:\/\/.+\/image\/#{md5}\//, "")
+      super
+    end
+
+  end
+end

data/lib/booru/client/yandere.rb

@@ -0,0 +1,25 @@
+# -*- encoding : utf-8 -*-
+module Booru
+  class Yandere < Client
+    attr_accessor :host, :referer
+
+    def initialize(user = nil, pass = nil)
+      @host    = "https://yande.re"
+      @referer = "https://yande.re"
+      super
+    end
+
+    def hash_password(pass)
+      salt = 'choujin-steiner'
+      salted_pass = "#{salt}--#{pass}--"
+      Digest::SHA1.hexdigest(salted_pass)
+      super
+    end
+
+    def url_file_segment(url, md5)
+      url = url.gsub(/https:\/\/.+\/image\/#{md5}\//, "")
+      super
+    end
+
+  end
+end

data/lib/booru/downloader.rb

@@ -0,0 +1,95 @@
+# -*- encoding : utf-8 -*-
+module Booru
+  class Downloader
+    include Utils
+
+    def initialize(options = {})
+      board       = options[:board].downcase.to_sym
+      user        = options[:user]
+      pass        = options[:pass]
+      limit       = options[:max]
+      @tags       = sanitize(options[:tags])
+      @pool       = sanitize(options[:pool])
+      @store      = options[:output]
+      @user_agent = 'Mozilla/5.0'
+
+      @client = case board
+      when :konachan
+        Konachan.new(user,pass)
+      when :behoimi
+        Behoimi.new(user, pass)
+      when :yandere
+        Yandere.new(user, pass)
+      when :danbooru
+        Danbooru.new(user, pass)
+      else
+        puts "Image board unsupported. Exiting."
+        exit 1
+      end
+
+      @curr = 1
+      @page = 1
+      fetch_posts(@page)
+
+      # TODO Figure out a way to get total count
+      if limit
+        @total = limit
+      else
+        @total = 100
+      end
+      #@pages = @total.to_i/100 + 1
+      FileUtils.mkdir_p File.expand_path(@store)
+    end
+
+    def download
+      while @posts.length > 0 do
+        download_posts
+        paginate
+      end
+      puts "No more results for #{@tags}"
+    end
+
+    private
+
+    def paginate
+      @page += 1
+      puts "Fetching page #{@page} of #{@pages}"
+      fetch_posts(@page)
+    end
+
+    # Returns an array of posts
+    def fetch_posts(page)
+      params = {:page => page, :tags => @tags}
+      @posts = @client.list_posts(params)
+      # TODO Implement download by pool
+      #@posts = @client.list_pool_posts(params)
+    end
+
+    def download_posts
+      @posts.each do |post|
+        file_url = post['file_url']
+        md5 = post['md5']
+        tags = post['tags']
+
+        filename = @client.url_file_segment(file_url, md5)
+        filename = reduce_filename(filename, 100)
+        filename = clean_filename(filename)
+        filename = File.join(@store, filename)
+
+        if File.exist?(filename) && md5 == get_md5_hash(filename)
+          puts "(#{@curr}/#{@total}): File already exists, skipping."
+        else
+          puts "(#{@curr}/#{@total}): Saving..."
+          transfer(file_url, filename, @user_agent, @client.referer)
+          puts "Saved."
+        end
+
+        @curr += 1
+        if @curr > @total
+          puts "Maximum downloads reached."
+          exit
+        end
+      end
+    end
+  end
+end

data/lib/booru/error.rb

@@ -0,0 +1,32 @@
+# -*- encoding : utf-8 -*-
+module Booru
+  module Error
+    # Custom error class for rescuing all Moebooru errors
+    class BooruError < StandardError; end
+
+    # Raised when Booru returns the HTTP status code 420
+    # Reason: Record could not be saved
+    class InvalidRecordError     < BooruError; end
+    # Raised when Booru returns the HTTP status code 421
+    # Reason: User is throttled, try again later
+    class UserThrottled          < BooruError; end
+    # Raised when Booru returns the HTTP status code 422
+    # Reason: The resource is locked and cannot be modified
+    class LockedError            < BooruError; end
+    # Raised when Booru returns the HTTP status code 423
+    # Reason: Resource already exists
+    class AlreadyExistsError     < BooruError; end
+    # Raised when Booru returns the HTTP status code 424
+    # Reason: The given parameters were invalid
+    class InvalidParametersError < BooruError; end
+
+    # Raised when Booru returns the HTTP status code 403
+    class ForbiddenError         < StandardError; end
+    # Raised when Booru returns the HTTP status code 404
+    class NotFoundError          < StandardError; end
+    # Raised when Booru returns the HTTP status code 500
+    class InternalServerError    < StandardError; end
+    # Raised when Booru returns the HTTP status code 503
+    class UnavailableError       < StandardError; end
+  end
+end

data/lib/booru/request.rb

@@ -0,0 +1,55 @@
+# -*- encoding : utf-8 -*-
+module Booru
+  module Request
+
+    HEADERS = {
+      'User-Agent' => 'Mozilla/5.0',
+    }
+
+    def get(path, options={})
+      request(:get, path, HEADERS.merge(options))
+    end
+
+    def post(path, options = {})
+      request(:post, path, HEADERS.merge(options))
+    end
+
+    # Perform an HTTP request
+    def request(method, path, options)
+      case method
+      when :get
+        response = RestClient.get("#{@host}#{path}", options)
+      when :post
+        response = RestClient.post("#{@host}#{path}", options)
+      end
+      raise_errors(response)
+      response
+    end
+
+    private
+
+    def raise_errors(response)
+      case response.code.to_i
+      when 403
+        raise Booru::Error::AccessDeniedError
+      when 404
+        raise Booru::Error::NotFoundError
+      when 420
+        raise Booru::Error::InvalidRecordError
+      when 421
+        raise Booru::Error::UserThrottledError
+      when 422
+        raise Booru::Error::ResourceLockedError
+      when 423
+        raise Booru::Error::ResourceAlreadyExistsError
+      when 424
+        raise Booru::Error::InvalidParametersError
+      when 500
+        raise Booru::Error::InternalServerError
+      when 503
+        raise Booru::Error::UnavailableError
+      end
+    end
+
+  end
+end

data/lib/booru/utils.rb

@@ -0,0 +1,80 @@
+# -*- encoding : utf-8 -*-
+module Booru
+  module Utils
+
+    def clean_filename(filename)
+      filename.gsub(' ', '_').gsub(/[^a-zA-Z0-9_.]+/, '-')
+    end
+
+    # TODO Rather than just reduce filename size by removing
+    # characters, try to not to clip tags in the filename
+    def reduce_filename(filename, max = 255)
+      if filename.length > max
+        suffix   = File.extname(filename)
+        basename = filename.chomp(suffix)
+
+        size = max.to_i - suffix.length
+        shortname = basename[0..size]
+
+        [shortname, suffix].join('')
+      else
+        filename
+      end
+    end
+
+    def sanitize(values)
+        values = values.gsub(" ", "+") if values.is_a?(String)
+        if values.is_a?(Array)
+          values.each { |value| value.gsub(' ', '_') }
+          values = values.join("+")
+        end
+        values
+    end
+
+    def get_md5_hash(file)
+      string = File.read(file)
+      hash = Digest::MD5.hexdigest(string)
+    end
+
+    # Cross-platform 'which'
+    # http://stackoverflow.com/a/5471032
+    def which(cmd)
+      exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+      ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+        exts.each { |ext|
+          exe = "#{path}/#{cmd}#{ext}"
+          return exe if File.executable? exe
+        }
+      end
+      return nil
+    end
+
+    # File download wrapper
+    def transfer(url, output, user_agent, referer)
+      if which('wget')
+        wget(url, output, user_agent, referer)
+      elsif which('curl')
+        curl(url, output, user_agent, referer)
+      else
+        write(url, output, user_agent, referer)
+      end
+    end
+
+    def wget(url, output, user_agent, referer)
+      command = "wget -nv #{url} -O #{output} "\
+                "--user-agent=#{user_agent} --referer=#{referer}"
+      system(command)
+    end
+
+    def curl(url, output, user_agent, referer)
+      command = "curl -A #{user_agent} -e #{referer} --progress-bar "\
+                "-o #{output} #{url}"
+      system(command)
+    end
+
+    def write(url, output, user_agent, referer)
+      data = open(url, "User-Agent" => user_agent, "Referer" => referer).read
+      File.open(output, "wb") { |file| file.write(data) }
+    end
+  end
+end

data/lib/booru/version.rb

@@ -0,0 +1,4 @@
+# -*- encoding : utf-8 -*-
+module Booru
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: booru
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- David Palma
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nori
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A Moebooru API client and image downloader for Danbooru style image boards.
+email: requiem.der.seele@gmail.com
+executables:
+- booru.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/booru.rb
+- booru.gemspec
+- fixtures/konachan/post_index.json
+- fixtures/yandere/post_index.json
+- lib/booru.rb
+- lib/booru/api.rb
+- lib/booru/api/artist.rb
+- lib/booru/api/comment.rb
+- lib/booru/api/favorite.rb
+- lib/booru/api/forum.rb
+- lib/booru/api/note.rb
+- lib/booru/api/pool.rb
+- lib/booru/api/post.rb
+- lib/booru/api/tag.rb
+- lib/booru/api/user.rb
+- lib/booru/api/wiki.rb
+- lib/booru/client.rb
+- lib/booru/client/behoimi.rb
+- lib/booru/client/danbooru.rb
+- lib/booru/client/konachan.rb
+- lib/booru/client/yandere.rb
+- lib/booru/downloader.rb
+- lib/booru/error.rb
+- lib/booru/request.rb
+- lib/booru/utils.rb
+- lib/booru/version.rb
+homepage: https://github.com/mistofvongola/booru
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- - lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: A Ruby client for the Moebooru API.
+test_files: []