magicshelf 0.2.1

data/lib/magicshelf/filecleaner.rb

@@ -0,0 +1,19 @@
+require 'magicshelf/exception'
+
+module MagicShelf
+  class FileCleanerError < Error; end
+
+  class FileCleaner
+    attr_accessor :file
+
+    def enter()
+      #raise MagicShelf::FileCleanerError.new("workdir is not set") if @workdir == nil
+      yield
+    end
+
+    def process()
+      FileUtils.remove_file(@file)
+    end
+  end
+end
+

data/lib/magicshelf/fileextractor.rb

@@ -0,0 +1,48 @@
+require 'filemagic'
+require 'zip'
+require 'open3'
+require 'shellwords'
+require 'magicshelf/exception'
+
+module MagicShelf
+  class FileExtractorError < Error; end
+
+  class FileExtractor
+    attr_accessor :inputfile, :destdir
+
+    def enter()
+      raise MagicShelf::FileExtractorError.new("inputfile is not set") if @inputfile == nil
+      mimetype = FileMagic.new(FileMagic::MAGIC_MIME_TYPE).file(@inputfile)
+      raise MagicShelf::FileExtractorError.new("unsupported filetype: #{mimetype}") if not %w{application/x-rar application/zip}.include?(mimetype)
+      @mimetype = mimetype
+      
+      yield
+    end
+
+    def process()
+      case @mimetype
+      when "application/zip"
+        Zip::File.open(@inputfile) do |zip_file|
+          zip_file.each do |entry|
+            MagicShelf.logger.info("Extracting #{entry.name} ...")
+            entry.extract(File.join(@destdir,entry.name))
+          end
+        end
+      when "application/x-rar"
+        out, err, status = Open3.capture3("which unrar")
+        if status.exitstatus != 0
+          raise MagicShelf::FileExtractorError.new("cannot execute unrar, is it on your PATH?")
+        end
+        
+        out, err, status = Open3.capture3("unrar x #{Shellwords.escape(@inputfile)} #{Shellwords.escape(@destdir)}")
+        if status.exitstatus != 0
+          raise MagicShelf::FileExtractorError.new("unrar exits with status #{status.exitstatus}: \n #{out} \n #{err}")
+        end
+      else
+        raise MagicShelf::FileExtractorError.new("no way to extract file for the file with filetype: #{@mimetype}")
+      end
+
+    end
+
+  end
+end

data/lib/magicshelf/filemover.rb

@@ -0,0 +1,24 @@
+require 'magicshelf/exception'
+require 'gepub'
+require 'shellwords'
+
+module MagicShelf
+  class FileMoverError < Error; end
+
+  class FileMover
+    attr_accessor :inputfile, :outputfile
+
+    def enter()
+      MagicShelf.logger.debug('enter FileMover')
+      raise MagicShelf::FileMoverError.new("inputfile is not set") if @inputfile == nil
+      raise MagicShelf::FileMoverError.new("outputfile is not set") if @outputfile == nil
+
+      yield
+    end
+
+    def process()
+      FileUtils.mv(@inputfile, @outputfile)
+    end
+  end
+end
+

data/lib/magicshelf/filenamevalidator.rb

@@ -0,0 +1,31 @@
+require 'magicshelf/exception'
+
+module MagicShelf
+  class FileNameValidatorError < Error; end
+
+  class FileNameValidator
+    attr_accessor :workdir
+    def initialize()
+      @erase_original = true
+    end
+
+    def enter()
+      yield
+    end
+
+    def process()
+      @workdir ||= Dir.pwd
+      Dir.glob(File.join(@workdir,'**/*')).select{|f|File.file?(f)}.each do |f|
+        dirname = File.dirname(f)
+        basename = File.basename(f,'.*')
+        extname = File.extname(f)
+        newbasename = basename.gsub(/#/, '_').gsub(/\+/, '_')
+        if not (basename == newbasename)
+          newfilename = File.join(dirname, newbasename + extname)
+          FileUtils.mv(f, newfilename)
+          MagicShelf.logger.info("move #{f} to #{newfilename}.")
+        end
+      end
+    end
+  end
+end

data/lib/magicshelf/fileserver.rb

@@ -0,0 +1,103 @@
+require 'sinatra/base'
+require 'sinatra/config_file'
+require 'sinatra/reloader'
+require 'tilt/erb'
+require 'redis'
+require 'magicshelf/mobitask'
+
+module MagicShelf
+  class FileServer < Sinatra::Application
+    register Sinatra::ConfigFile
+    configure :development do
+      register Sinatra::Reloader
+    end
+
+    set :root, File.join(File.dirname(__FILE__), '../..') #set :views, File.join(File.dirname(__FILE__), '../..', 'views')
+    set :bind, '0.0.0.0'
+    config_file 'server_config.yml'
+
+    get '/' do
+      redirect "/files/", 303
+    end
+
+    get '/files/*' do |path|
+      sort_type = params['sort_by'] || "title"
+      files = []
+      Dir.chdir(settings.library_directory) {
+        files = Dir.glob(File.join('.',path, '*'))
+        case sort_type
+        when 'title'
+          files = files.sort
+        when 'title_reverse'
+          files = files.sort.reverse
+        when 'date'
+          files = files.sort_by{ |f| File.mtime(f) }
+        when 'date_reverse'
+          files = files.sort_by{ |f| File.mtime(f) }.reverse
+        else
+        end
+        files_withmtime = files.map do |f|
+          fname = (f.start_with?('./') ? f[2..-1] : f)
+          [fname, File.mtime(f).strftime("%Y/%m/%d %H:%M:%S")]
+        end
+        upperpath = nil
+        upperpath = path.split('/')[0...-1].join('/') if path != ""
+        erb :index, :locals => {:page_title => settings.page_title, :files_withmtime => files_withmtime, :path => path, :upperpath => upperpath, :sort_type => sort_type}
+      }
+    end
+
+    get '/get_file/*' do |path|
+      pass unless path # pass to a subsequent route
+      send_file(File.join(settings.library_directory, path))
+    end
+
+    get '/get_file*' do
+      'you come to this page without specifying the path to file. go back to the previous page!'
+    end
+
+    get '/generate_mobi/*' do |path|
+      #path = params['splat'].first
+      pass unless path # pass to a subsequent route
+
+      erb :generate_mobi, :locals => {:page_title => settings.page_title, :library_directory => settings.library_directory, :path => path}
+    end
+
+    post '/generate_mobi/*' do |path|
+      title      = params['title']
+      author     = params['author']
+      booktype   = params['booktype']
+      outputfile = params['outputfile']
+      outputfile = title + ".mobi" if outputfile.empty?
+
+      taskparams = {}
+      #taskparams.update(params)
+      taskparams['title']      = params['title']
+      taskparams['author']     = params['author']
+      taskparams['booktype']   = params['booktype']
+      taskparams['inputfile']  = File.join(settings.library_directory,path)
+      taskparams['outputfile'] = File.join(settings.library_directory,File.dirname(path),outputfile)
+
+      Resque.enqueue(MobiTask, taskparams)
+      
+      <<-EOF
+      <!DOCTYPE html>
+      <html lang="en">
+      <head>
+        <meta charset="UTF-8">
+        <title></title>
+      </head>
+      <body>
+      Now generating mobi file. wait for a while. <br><a href="/">Back to Top</a>
+      </body>
+      </html>
+      EOF
+    end
+
+    get '/generate_mobi*' do |f|
+      'you come to this page without specifying the path to file. go back to the previous page!'
+    end
+
+    run! if app_file == $0
+  end
+end
+

data/lib/magicshelf/kindlegenwrapper.rb

@@ -0,0 +1,33 @@
+require 'magicshelf/exception'
+require 'open3'
+
+module MagicShelf
+  class KindleGenWrapperError < Error; end
+
+  class KindleGenWrapper
+    attr_accessor :inputfile, :outputfile
+
+    def enter()
+      MagicShelf.logger.debug('enter KindleGenWrapper')
+
+      # check parameters
+      raise MagicShelf::KindleGenWrapperError.new("inputfile is not set") if @inputfile == nil
+      raise MagicShelf::KindleGenWrapperError.new("outputfile is not set") if @outputfile == nil
+      out, err, status = Open3.capture3("which kindlegen")
+      if status.exitstatus != 0
+        raise MagicShelf::KindleGenWrapperError.new("cannot execute kindlegen, is it on your PATH?")
+      end
+
+      yield
+    end
+
+    def process()
+      out, err, status = Open3.capture3("kindlegen #{@inputfile} -o #{@outputfile}")
+      if status.exitstatus != 0
+        raise MagicShelf::KindleGenWrapperError.new("kindlegen exited with #{status.exitstatus}: \n" + out + "\n" + err)
+      end
+    end
+
+  end
+end
+

data/lib/magicshelf/kindlestrip.rb

@@ -0,0 +1,250 @@
+#! ruby
+# -*- coding: utf-8 -*-
+#
+# It was translated into Ruby script by whiteleaf.
+# ( https://github.com/whiteleaf7/narou/blob/master/lib/kindlestrip.rb )
+#
+# original source code:
+# kindlestrip.py v.1.35 http://www.mobileread.com/forums/showthread.php?t=96903
+#
+# This script strips the penultimate record from a Mobipocket file.
+# This is useful because the current KindleGen add a compressed copy
+# of the source files used in this record, making the ebook produced
+# about twice as big as it needs to be.
+#
+#
+# This is free and unencumbered software released into the public domain.
+# 
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+# 
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+# 
+# For more information, please refer to <http://unlicense.org/>
+#
+# Written by Paul Durrant, 2010-2011, paul@durrant.co.uk, pdurrant on mobileread.com
+# With enhancements by Kevin Hendricks, KevinH on mobileread.com
+#
+# Changelog
+#  1.00 - Initial version
+#  1.10 - Added an option to output the stripped data
+#  1.20 - Added check for source files section (thanks Piquan)
+#  1.30 - Added prelim Support for K8 style mobis
+#  1.31 - removed the SRCS section but kept a 0 size entry for it
+#  1.32 - removes the SRCS section and its entry, now updates metadata 121 if needed
+#  1.33 - now uses and modifies mobiheader SRCS and CNT
+#  1.34 - added credit for Kevin Hendricks
+#  1.35 - fixed bug when more than one compilation (SRCS/CMET) records
+
+KINDLESTRIP_VERSION = '1.35'
+
+class StripException < StandardError; end
+
+class SectionStripper
+  def load_section(section)
+    if section + 1 == @num_sections
+      endoff = @data_file.length
+    else
+      endoff = @sections[section + 1][0]
+    end
+    off = @sections[section][0]
+    @data_file[off...endoff]
+  end
+
+  def patch(off, _new)
+    @data_file = @data_file[0, off] + new + @data_file[off + _new.length .. -1]
+  end
+
+  def strip(off, len)
+    @data_file = @data_file[0, off] + @data_file[off + len .. -1]
+  end
+
+  def patch_section(section, _new, in_off = 0)
+    if section + 1 == @num_sections
+      endoff = @data_file.length
+    else
+      endoff = @sections[section + 1][0]
+    end
+    raise unless off + in_off + _new.length <= endoff
+    patch(off + in_off, _new)
+  end
+
+  def updateEXTH121(srcs_secnum, srcs_cnt, mobiheader)
+    mobi_length, = mobiheader[0x14...0x18].unpack("N")
+    exth_flag, = mobiheader[0x80...0x84].unpack("N")
+    exth = "NONE"
+    begin
+      if exth_flag & 0x40 != 0
+        exth = mobiheader[16 + mobi_length .. -1]
+        if exth.length >= 4 && exth[0, 4] == "EXTH"
+          nitems, = exth[8...12].unpack("N")
+          pos = 12
+          nitems.times do
+            type, size = exth[pos ... pos + 8].unpack("NN")
+            #puts "#{type}, #{size}"
+            if type == 121
+              boundaryptr, = exth[pos + 8 ... pos + size].unpack("N")
+              if srcs_secnum <= boundaryptr
+                boundaryptr -= srcs_cnt
+                prefix = mobiheader[0, 16 + mobi_length + pos + 8]
+                suffix = mobiheader[16 + mobi_length + pos + 8 + 4 .. -1]
+                nval = [boundaryptr].pack("N")
+                mobiheader = prefix + nval + suffix
+              end
+            end
+            pos += size
+          end
+        end
+      end
+    rescue
+    end
+    mobiheader
+  end
+
+  def initialize(datain, verbose = true)
+    @verbose = verbose
+    if datain[0x3C...0x3C+8] != "BOOKMOBI"
+      raise StripException, "invalid file format"
+    end
+    @num_sections, = datain[76...78].unpack("n")
+
+    # get mobiheader and check SRCS section number and count
+    offset0, = datain.unpack("@78N")
+    offset1, = datain.unpack("@86N")
+    mobiheader = datain[offset0 ... offset1]
+    srcs_secnum, srcs_cnt = mobiheader.unpack("@224NN")
+    if srcs_secnum == 0xffffffff || srcs_cnt == 0
+      raise StripException, "File doesn't contain the sources section."
+    end
+
+    puts "Found SRCS section number %d, and count %d" % [srcs_secnum, srcs_cnt] if @verbose
+    # find its offset and length
+    _next = srcs_secnum + srcs_cnt
+    srcs_offset, = datain.unpack("@#{78+srcs_secnum*8}NN")
+    next_offset, = datain.unpack("@#{78+_next*8}NN")
+    srcs_length = next_offset - srcs_offset
+    if datain[srcs_offset ... srcs_offset+4] != "SRCS"
+      raise StripException, "SRCS section num does not point to SRCS."
+    end
+    puts "   beginning at offset %0x and ending at offset %0x" % [srcs_offset, srcs_length] if @verbose
+
+    # it appears bytes 68-71 always contain (2*num_sections) + 1
+    # this is not documented anyplace at all but it appears to be some sort of next 
+    # available unique_id used to identify specific sections in the palm db
+    @data_file = datain[0, 68] + [(@num_sections - srcs_cnt) * 2 + 1].pack("N")
+    @data_file += datain[72...76]
+
+    # write out the number of sections reduced by srtcs_cnt
+    @data_file = @data_file + [@num_sections - srcs_cnt].pack("n")
+
+    # we are going to remove srcs_cnt SRCS sections so the offset of every entry in the table
+    # up to the srcs secnum must begin 8 bytes earlier per section removed (each table entry is 8 )
+    delta = -8 * srcs_cnt
+    srcs_secnum.times do |i|
+      offset, flgval = datain.unpack("@#{78+i*8}NN")
+      offset += delta
+      @data_file += [offset].pack("N") + [flgval].pack("N")
+    end
+
+    # for every record after the srcs_cnt SRCS records we must start it
+    # earlier by 8*srcs_cnt + the length of the srcs sections themselves)
+    delta = delta - srcs_length
+    (srcs_secnum + srcs_cnt ... @num_sections).each do |i|
+      offset, = datain.unpack("@#{78+i*8}NN")
+      offset += delta
+      flgval = 2 * (i - srcs_cnt)
+      @data_file += [offset].pack("N") + [flgval].pack("N")
+    end
+
+    # now pad it out to begin right at the first offset
+    # typically this is 2 bytes of nulls
+    first_offset, = @data_file.unpack("@78NN")
+    @data_file += "\0" * (first_offset - @data_file.length)
+
+    # now finally add on every thing up to the original src_offset
+    @data_file += datain[offset0...srcs_offset]
+    
+    # and everything afterwards
+    @data_file += datain[srcs_offset + srcs_length .. -1]
+
+    #store away the SRCS section in case the user wants it output
+    @stripped_data_header = datain[srcs_offset ... srcs_offset + 16]
+    @stripped_data = datain[srcs_offset + 16 ... srcs_offset + srcs_length]
+
+    # update the number of sections count
+    @num_section = @num_sections - srcs_cnt
+
+    # update the srcs_secnum and srcs_cnt in the mobiheader
+    offset0, = @data_file.unpack("@78NN")
+    offset1, = @data_file.unpack("@86NN")
+    mobiheader = @data_file[offset0 ... offset1]
+    mobiheader = mobiheader[0, 0xe0] + [-1].pack("N") + [0].pack("N") + mobiheader[0xe8 .. -1]
+
+    # if K8 mobi, handle metadata 121 in old mobiheader
+    mobiheader = updateEXTH121(srcs_secnum, srcs_cnt, mobiheader)
+    @data_file = @data_file[0, offset0] + mobiheader + @data_file[offset1 .. -1]
+    puts "done" if @verbose
+  end
+
+  def get_result
+    @data_file
+  end
+
+  def get_stripped_data
+    @stripped_data
+  end
+
+  def get_header
+    @stripped_data_header
+  end
+
+  def self.strip(infile, outfile = nil, verbose = true)
+    outfile = infile unless outfile
+    data_file = File.binread(infile)
+    stripped_file = new(data_file, verbose)
+    File.binwrite(outfile, stripped_file.get_result)
+    stripped_file
+  end
+end
+
+if __FILE__ == $0
+  puts "KndleStrip v#{KINDLESTRIP_VERSION}. " +
+       "Written 2010-2012 by Paul Durrant and Kevin Hendricks."
+  if ARGV.length < 2 || ARGV.length > 3
+    puts "Strips the Sources record from Mobipocket ebooks"
+    puts "For ebooks generated using KindleGen 1.1 and later that add the source"
+    puts "Usage:"
+    puts "    %s <infile> <outfile> <strippeddatafile>" % File.basename(__FILE__)
+    puts "<strippeddatafile> is optional."
+    exit Narou::EXIT_ERROR_CODE
+  else
+    infile = ARGV[0]
+    outfile = ARGV[1]
+    begin
+      stripped_file = SectionStripper.strip(infile, outfile)
+      #print "Header Bytes: " + binascii.b2a_hex(strippedFile.getHeader())
+      if ARGV.length == 3
+        File.binwrite(ARGV[2], stripped_file.get_stripped_data)
+      end
+    rescue StripException => e
+      warn "Error: #{e.message}"
+      exit Narou::EXIT_ERROR_CODE
+    end
+  end
+end