kindai 1.0.0

data/lib/kindai/publisher.rb

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+module Kindai
+  class Publisher
+    attr_accessor :root_path
+
+    def self.new_from_path(root_path)
+      me = self.new
+      me.root_path = root_path
+      me
+    end
+
+    def name(n)
+      config(:name, n)
+      self
+    end
+
+    def resize(width, height)
+      config(:resize, {:width => width, :height => height})
+      self
+    end
+
+    def trim(geometry = true)
+      config(:trim, geometry) unless config(:trim)
+      self
+    end
+
+    def zip
+      config(:zip, true)
+      self
+    end
+
+    def divide
+      config(:divide, true)
+      self
+    end
+
+    def empty(glob)
+      FileUtils.rm_r(Dir.glob(File.join(self.root_path, glob)))
+    end
+
+    def publish
+      Kindai::Util.logger.info("publish #{root_path}, #{config(:name)}")
+      raise "no name" unless config(:name)
+      if seems_finished?
+        Kindai::Util.logger.info("already published")
+        return
+      end
+      create_directory
+
+      path = original_path
+
+      path = trim!(path) if trim?
+      path = divide!(path) if divide?
+      path = resize!(path) if resize?
+      path = zip!(path) if zip?
+    end
+
+    def publish_auto
+      self.clone.trim.resize(1280, 960).trim.zip.name('iphone').publish
+      self.clone.trim.resize(600, 800).divide.zip.name('kindle').publish
+    end
+
+    # ------------------------------------
+    protected
+
+    def config(k, v = nil)
+      @config ||= {}
+      return @config[k] unless v
+      @config[k] = v
+      @config
+    end
+
+    def trim?
+      config(:trim)
+    end
+
+    def resize?
+      config(:resize)
+    end
+
+    def zip?
+      config(:zip)
+    end
+
+    def divide?
+      config(:divide)
+    end
+
+    # ---------- aciton --------------
+
+    def trim!(source_path)
+      return trim_path if files(source_path).length == files(trim_path).length
+      info = config(:trim).kind_of?(Hash) ? config(:trim) : Kindai::Util.trim_info_by_files(original_files)
+      files(source_path).each{|file|
+        dst = File.join(trim_path, File.basename(file))
+        Kindai::Util.trim_file_to(file, dst, info)
+        GC.start
+      }
+      return trim_path
+    end
+
+    def resize!(source_path)
+      files(source_path).each{|file|
+        dst = File.join(output_path, File.basename(file))
+        Kindai::Util.resize_file_to(file, dst, config(:resize))
+        GC.start
+      }
+      return output_path
+    end
+
+    def divide!(source_path)
+      files(source_path).each{|file|
+        Kindai::Util.divide_43(file, output_path)
+        GC.start
+      }
+      return output_path
+    end
+
+    def zip!(source_path)
+      Kindai::Util.generate_zip(source_path)
+      FileUtils.rm_r(self.output_path)
+      return source_path
+    end
+
+    # ---------util------------
+
+    def create_directory
+      Dir.mkdir(trim_path) unless File.directory?(trim_path)
+      Dir.mkdir(output_path) unless File.directory?(output_path)
+    end
+
+    def trim_path
+      File.join(root_path, 'trim')
+    end
+
+    def original_path
+      File.join(root_path, 'original')
+    end
+
+    def output_path
+      File.join(root_path, File.basename(root_path) + '_' + config(:name))
+    end
+
+    def original_files
+      files(original_path)
+    end
+
+    def files(path)
+      Dir.glob(File.join(path, '*jpg'))
+    end
+
+    def seems_finished?
+      zip? ? File.exists?(output_path + '.zip') : File.directory?(output_path)
+    end
+
+  end
+end

data/lib/kindai/searcher.rb

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+module Kindai
+  class Searcher
+    include Enumerable
+    attr_accessor :keyword
+    def self.search keyword
+      Kindai::Util.logger.debug "keyword: #{keyword}"
+      me = self.new
+      me.keyword = keyword
+      me
+    end
+
+    def length
+      @length ||= total_of(@keyword)
+    end
+
+    def each
+      (0..(1/0.0)).each{ |page|
+        Kindai::Util.logger.debug "page #{page}"
+        uris = result_for(@keyword, page)
+        return if uris.empty?
+        uris.each{ |uri|
+          yield Kindai::Book.new_from_permalink(uri)
+        }
+      }
+    end
+
+    protected
+    def total_of(keyword)
+      page = Nokogiri(Kindai::Util.fetch_uri(uri_for(keyword)))
+      total = page.at('.//opensearch:totalResults', {"opensearch"=>"http://a9.com/-/spec/opensearchrss/1.0/"} ).content.to_i
+
+      Kindai::Util.logger.debug "total: #{total}"
+      total
+    end
+
+    def result_for keyword, page = 0
+      page = Nokogiri Kindai::Util.fetch_uri(uri_for(keyword, page))
+      page.search('item').map{ |item|
+        item.at('link').content
+      }
+    end
+
+    def uri_for keyword, page = 0
+      count = 10
+      params = { :any => keyword, :dpid => 'kindai', :idx => page * count + 1, :cnt => count}
+      root = URI.parse("http://api.porta.ndl.go.jp/servicedp/opensearch")
+      path = '?' + Kindai::Util.expand_params(params)
+      root + path
+    end
+  end
+end

data/lib/kindai/spread.rb

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+module Kindai
+  class Spread
+    attr_accessor :book
+    attr_accessor :spread_number
+
+    def self.new_from_book_and_spread_number(book, spread_number)
+      raise TypeError, "#{book} is not Kindai::Book" unless book.is_a? Kindai::Book
+      me = new
+      me.book = book
+      me.spread_number = spread_number
+      me
+    end
+
+    def uri
+      book.base_uri.gsub(/koma=(\d+)/) { "koma=#{spread_number}" }
+    end
+
+    def image_uri
+      image = page.at("img#imMain")
+      raise "not exists" unless image
+      image['src']
+    end
+
+
+    def has_local_file?
+    end
+
+    def local_file_path
+    end
+
+    # protected
+    # XXX: book use this
+    def page
+      @page ||= Nokogiri Kindai::Util.fetch_uri self.uri
+    end
+
+  end
+end

data/lib/kindai/spread_downloader.rb

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+module Kindai
+  class SpreadDownloader
+    attr_accessor :spread
+    attr_accessor :retry_count
+    attr_accessor :book_path
+
+    def self.new_from_spread(spread)
+      raise TypeError, "#{spread} is not Kindai::Spread" unless spread.is_a? Kindai::Spread
+      me = self.new
+      me.spread = spread
+      me.retry_count = 30
+      me.book_path = Pathname.new(ENV["HOME"]).to_s
+      me
+    end
+
+    def download
+      return false if self.has_file?
+      self.create_directory
+      self.download_spread
+      return true
+    end
+
+    def create_directory
+      path = File.join self.book_path, "original"
+      Dir.mkdir(path) unless File.directory?(path)
+    end
+
+    def spread_path
+      path = File.join self.book_path, "original", "%03d.jpg" % self.spread.spread_number
+      File.expand_path path
+    end
+
+    def delete
+      return File.delete(self.spread_path) && true rescue false
+    end
+
+    def has_file?
+      File.size? self.spread_path
+    end
+
+    protected
+
+    def download_spread
+      failed_count = 0
+
+      begin
+        Kindai::Util.logger.info "downloading " + [self.spread.book.author, self.spread.book.title, "spread #{self.spread.spread_number} / #{self.spread.book.total_spread}"].join(' - ')
+        Kindai::Util.rich_download(spread.image_uri, self.spread_path)
+      rescue Interrupt => err
+        Kindai::Util.logger.error "#{err.class}: #{err.message}"
+        exit 1
+      rescue StandardError, TimeoutError => err
+        Kindai::Util.logger.warn "failed (#{failed_count+1}/#{self.retry_count}) #{err.class}: #{err.message}"
+        raise err if failed_count == self.retry_count
+
+        Kindai::Util.logger.info "sleep and retry"
+        failed_count += 1
+        sleep 10
+        retry
+      end
+    end
+
+  end
+end

data/lib/kindai/util.rb

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+require 'open3'
+require 'tempfile'
+require 'digest/sha1'
+require 'RMagick'
+require 'zipruby'
+
+module Kindai::Util
+  def self.logger
+    return @logger if @logger
+    @logger ||= Logger.new(STDOUT)
+    @logger.level = Logger::INFO
+    @logger
+  end
+
+  def self.debug_mode!
+    self.logger.level = Logger::DEBUG
+    Kindai::Util.logger.info "debug mode enabled"
+  end
+
+  def self.download(uri, file)
+    total = nil
+    uri = URI.parse(uri) unless uri.kind_of? URI
+
+    got = fetch_uri(uri)
+    open(file, 'w') {|local|
+      local.write(got)
+    }
+  rescue Exception, TimeoutError => error
+    if File.exists?(file)
+      logger.debug "delete cache"
+      File.delete(file)
+    end
+    raise error
+  end
+
+  def self.rich_download(uri, file)
+    total = nil
+    uri = URI.parse(uri) unless uri.kind_of? URI
+
+    got = fetch_uri(uri, true)
+    open(file, 'w') {|local|
+      local.write(got)
+    }
+  rescue Exception, TimeoutError => error
+    if File.exists?(file)
+      logger.debug "delete cache"
+      File.delete(file)
+    end
+    raise error
+  end
+
+  # input:  {:a => 'a', :b => 'bbb'}
+  # output: 'a=a&b=bbb
+  def self.expand_params(params)
+    params.each_pair.map{ |k, v| [URI.escape(k.to_s), URI.escape(v.to_s)].join('=')}.join('&')
+  end
+
+  def self.append_suffix(path, suffix)
+    path.gsub(/\.(\w+)$/, "-#{suffix}.\\1")
+  end
+
+  def self.execute_and_log(command)
+    logger.debug command
+    system command or raise "#{commands} failed"
+  end
+
+  def self.generate_zip(directory)
+    Kindai::Util.logger.info "zip #{directory}"
+    directory = File.expand_path(directory)
+    raise "#{directory} is not directory." unless File.directory? directory
+
+    filename = File.expand_path(File.join(directory, '..', "#{File.basename(directory)}.zip"))
+    files = Dir.glob(File.join(directory, '*jpg'))
+    begin
+      Zip::Archive.open(filename, Zip::CREATE) {|arc|
+        files.each{|f| arc.add_file(f) }
+      }
+    rescue => error
+      File.delete(filename) if File.exists?(filename)
+      logger.warn "#{error.class}: #{error.message}"
+      logger.warn "zipruby died. trying zip command"
+      generate_zip_system_command(directory)
+    end
+  end
+
+  def self.generate_zip_system_command(directory)
+    Kindai::Util.logger.info "zip(system) #{directory}"
+    from = Dir.pwd
+    Dir.chdir(directory)
+    execute_and_log "zip -q -r '../#{File.basename(directory)}.zip' *jpg"
+    Dir.chdir(from)
+  end
+
+  def self.fetch_uri(uri, rich = false)
+    uri = URI.parse(uri) unless uri.kind_of? URI
+    self.logger.debug "fetch_uri #{uri}"
+
+    return uri.read unless rich
+
+    total = nil
+    from = Time.now
+    got = uri.read(
+      :content_length_proc => proc{|_total|
+        total = _total
+      },
+      :progress_proc => proc{|now|
+        if Time.now - from > 0.2
+          from = Time.now
+          print "%3d%% #{now}/#{total}\r" % (now/total.to_f*100)
+          $stdout.flush
+        end
+      })
+    raise "received size unmatch(#{got.bytesize}, #{total})" if got.bytesize != total
+    return got
+  end
+
+  def self.trim_info_by_files(files)
+    Kindai::Util.logger.info "get trim info"
+    positions = {:x => [], :y => [], :width => [], :height => []}
+    files.each{|file|
+      pos = trim_info(file)
+
+      [:x, :y, :width, :height].each{|key|
+        positions[key] << pos[key]
+      }
+
+      GC.start
+    }
+
+    good_pos = {}
+    [:x, :y, :width, :height].each{|key|
+      good_pos[key] = average(positions[key])
+    }
+    Kindai::Util.logger.info "trim position #{good_pos}"
+    good_pos
+  end
+
+  # XXX: GC
+  def self.trim_info(img_path, erase_center_line = true)
+    debug = false
+    img = Magick::ImageList.new(img_path)
+
+    thumb = img.resize_to_fit(400, 400)
+
+    thumb.write('a1.jpg') if debug
+    # thumb = thumb.normalize
+    thumb = thumb.level(Magick::QuantumRange*0.4, Magick::QuantumRange*0.7)
+    thumb.write('a2.jpg') if debug
+
+    d = Magick::Draw.new
+    d.fill = 'white'
+    cut_x = 0.07
+    cut_y = 0.04
+    d.rectangle(thumb.columns * 0.4, 0, thumb.columns * 0.6, thumb.rows) if erase_center_line # center line
+    d.rectangle(0, 0, thumb.columns * cut_x, thumb.rows) # h
+    d.rectangle(0, thumb.rows * (1 - cut_y), thumb.columns, thumb.rows)    # j
+    d.rectangle(0, 0, thumb.columns, thumb.rows * cut_y)             # k
+    d.rectangle(thumb.columns * (1 - cut_x), 0, thumb.columns, thumb.rows) # l
+    d.draw(thumb)
+    thumb.write('a.jpg') if debug
+
+    # thumb = thumb.threshold(Magick::QuantumRange*0.8)
+    # thumb.write('b.jpg') if debug
+
+    thumb.fuzz = 50
+    thumb.trim!
+    thumb.write('c.jpg') if debug
+
+    scale = thumb.base_columns / thumb.page.width.to_f
+
+    info = {
+      :x => thumb.page.x * scale,
+      :y => thumb.page.y * scale,
+      :width => thumb.columns * scale,
+      :height => thumb.rows * scale
+    }
+
+    # erased by cente line?
+    if (thumb.page.x / thumb.page.width.to_f - 0.6).abs < 0.05 && erase_center_line
+      Kindai::Util.logger.info "retry trim(erased by center line?)"
+      new_info = trim_info(img_path, false)
+      Kindai::Util.logger.debug "x: #{info[:x]} -> #{new_info[:x]}"
+      Kindai::Util.logger.debug "width: #{info[:width]} -> #{new_info[:width]}"
+      info[:x] = new_info[:x]
+      info[:width] = new_info[:width]
+    end
+
+    img = nil
+    thumb = nil
+
+    return info
+  end
+
+  def self.trim_file_to(src_path, dst_path, info = nil)
+    info = trim_info(src_path) unless info
+    Kindai::Util.logger.info "trim #{src_path}"
+
+    img = Magick::ImageList.new(src_path)
+    img.crop! info[:x], info[:y], info[:width], info[:height]
+    img.write dst_path
+
+    img = nil
+  end
+
+
+  def self.resize_file_to(src_path, dst_path, info)
+    Kindai::Util.logger.info "resize #{src_path}"
+    img = Magick::ImageList.new(src_path)
+    img.resize_to_fit(info[:width], info[:height]).write dst_path
+
+    img = nil
+  end
+
+  def self.average(array)
+    array.inject{|a, b| a + b} / array.length.to_f
+  end
+
+  def self.divide_43(src_path, output_directory)
+    raise "#{src_path} not exist" unless File.exists? src_path
+    Kindai::Util.logger.info "divide #{src_path}"
+
+    output_base = File.join(output_directory, File.basename(src_path))
+
+    img = Magick::ImageList.new(src_path)
+
+    right = img.crop(img.columns - img.rows * 0.75, 0, img.columns * 0.75, img.rows)
+    right.write(append_suffix(output_base, '0'))
+    right = nil
+
+    left = img.crop(0, 0, img.rows * 0.75, img.rows)
+    left.write(append_suffix(output_base, '1'))
+    left = nil
+
+    File.delete(src_path) if File.basename(src_path) == output_directory
+  end
+
+
+end

data/lib/kindai.rb

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+require 'rubygems'
+require 'open-uri'
+require 'nokogiri'
+require 'nkf'
+require 'logger'
+require 'open-uri'
+require 'cgi'
+require 'pathname'
+require 'fileutils'
+
+module Kindai
+  VERSION = File.read(File.join(File.dirname(__FILE__), '../VERSION')).strip
+
+  require 'kindai/cli'
+  require 'kindai/util'
+  require 'kindai/book'
+  require 'kindai/spread'
+  require 'kindai/book_downloader'
+  require 'kindai/spread_downloader'
+  require 'kindai/searcher'
+  require 'kindai/interface'
+  require 'kindai/publisher'
+end

data/publish.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+self_file =
+  if File.symlink?(__FILE__)
+    require 'pathname'
+    Pathname.new(__FILE__).realpath
+  else
+    __FILE__
+  end
+$:.unshift(File.dirname(self_file) + "/lib")
+
+require 'kindai'
+
+warn 'WARNING: This script is deprecated. Use bin/kindai.rb'
+
+Kindai::CLI.execute(STDOUT, ['publish'].concat(ARGV))

data/spec/book_downloader_spec.rb

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Kindai::BookDownloader do
+  before do
+    @book = Kindai::Book.new_from_permalink('http://kindai.ndl.go.jp/info:ndljp/pid/922693')
+    @downloader   = Kindai::BookDownloader.new_from_book(@book)
+  end
+
+  it 'has book' do
+    @downloader.book.should == @book
+  end
+
+  it 'has retry_count' do
+    @downloader.retry_count.should == 30
+    @downloader.retry_count = 50
+    @downloader.retry_count.should == 50
+  end
+
+  it 'has base path' do
+    @downloader.base_path = "/path/to/library"
+    @downloader.book_path.should == "/path/to/library/正義熱血社 - 正義の叫"
+
+    @downloader.base_path = "/path/to/library/"
+    @downloader.book_path.should == "/path/to/library/正義熱血社 - 正義の叫"
+  end
+
+  it 'can download book' do
+    base_path = File.join(ENV['TMPDIR'] || ENV['TMP'] || ENV['TEMP'] || '/tmp', rand.to_s)
+    Dir.mkdir(base_path)
+    @downloader.base_path = base_path
+
+    @downloader.has_file?.should be_false
+    @downloader.download.should be_true
+    @downloader.has_file?.should be_true
+    @downloader.download.should be_false
+
+    @downloader.delete.should be_true
+    @downloader.has_file?.should be_false
+    @downloader.delete.should be_false
+
+    Dir.delete(base_path)
+  end
+
+
+end

data/spec/book_spec.rb

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Kindai::Book do
+  before do
+    @book = Kindai::Book.new_from_permalink('http://kindai.ndl.go.jp/info:ndljp/pid/922693')
+  end
+
+  it 'has title' do
+    @book.title.should == '正義の叫'
+  end
+
+  it 'has total spread' do
+    @book.total_spread.should == 20
+  end
+
+  it 'has author' do
+    @book.author.should == '正義熱血社'
+  end
+
+  it 'has spreads' do
+    @book.spreads.should have_exactly(@book.total_spread).spreads
+  end
+
+  it 'has base_uri' do
+    @book.base_uri.should == "http://kindai.da.ndl.go.jp/scrpt/ndlimageviewer-rgc.aspx?pid=info%3Andljp%2Fpid%2F922693&jp=42016454&vol=10010&koma=1&vs=10000,10000,0,0,0,0,0,0"
+  end
+
+end
+
+describe Kindai::Book, 'with series' do
+  before do
+    @book = Kindai::Book.new_from_permalink('http://kindai.da.ndl.go.jp/info:ndljp/pid/890078')
+  end
+
+  it 'has title' do
+    @book.title.should == '講談日露戦争記［第３冊］第3編'
+  end
+end