repub 0.3.0

data/History.txt

@@ -0,0 +1,9 @@
+== 0.2.1 / 2009-06-26
+
+* Initial release
+
+== 0.3.0 / 2009-06-28
+
+* Switched to Nokogiri for HTML parsing
+* Better parsing for hierarchical TOCs
+* Many bug fixes

data/README.txt

@@ -0,0 +1,106 @@
+== DESCRIPTION:
+
+Simple HTML to ePub converter.
+
+== FEATURES/PROBLEMS:
+
+Few samples to get started:
+
+* Git User's Manual
+
+    repub -x 'title://h1' -x 'toc://div[@class="toc"]/dl' -x 'toc_item:dt' -x 'toc_section:following-sibling::*[1]/dl' \
+        http://www.kernel.org/pub/software/scm/git/docs/user-manual.html
+
+* Project Gutenberg's THE ADVENTURES OF SHERLOCK HOLMES
+
+    repub -x 'title:div[@class='book']//h1' -x 'toc://table' -x 'toc_item://tr' \
+        -X '//pre' -X '//hr' -X '//body/h1' -X '//body/h2' \
+	    http://www.gutenberg.org/dirs/etext99/advsh12h.htm
+
+* Project Gutenberg's ALICE'S ADVENTURES IN WONDERLAND
+
+    repub -x 'title:body/h1' -x 'toc://table' -x 'toc_item://tr' \
+	    -X '//pre' -X '//hr' -X '//body/h4' \
+	    http://www.gutenberg.org/files/11/11-h/11-h.htm
+
+* The Gelug-Kagyu Tradition of Mahamudra from Berzin Archives
+
+    repub http://www.berzinarchives.com/web/x/prn/p.html_680632258.html
+
+== SYNOPSIS:
+
+Usage: repub [options] url
+
+General options:
+  -D, --downloader NAME            Which downloader to use to get files (wget or httrack).
+                                   Default is wget.
+  -o, --output PATH                Output path for generated ePub file.
+                                   Default is /Users/dg/Projects/repub/<Parsed_Title>.epub
+  -w, --write-profile NAME         Save given options for later reuse as profile NAME.
+  -l, --load-profile NAME          Load options from saved profile NAME.
+  -W, --write-default              Save given options for later reuse as default profile.
+  -L, --list-profiles              List saved profiles.
+  -C, --cleanup                    Clean up download cache.
+  -v, --verbose                    Turn on verbose output.
+  -q, --quiet                      Turn off any output except errors.
+  -V, --version                    Show version.
+  -h, --help                       Show this help message.
+
+Parser options:
+  -x, --selector NAME:VALUE        Set parser XPath selector NAME to VALUE.
+                                   Recognized selectors are: [title toc toc_item toc_section]
+  -m, --meta NAME:VALUE            Set publication information metadata NAME to VALUE.
+                                   Valid metadata names are: [creator date description
+                                   language publisher relation rights subject title]
+  -F, --no-fixup                   Do not attempt to make document meet XHTML 1.0 Strict.
+                                   Default is to try and fix things that are broken. 
+  -e, --encoding NAME              Set source document encoding. Default is to autodetect.
+
+Post-processing options:
+  -s, --stylesheet PATH            Use custom stylesheet at PATH to add or override existing
+                                   CSS references in the source document.
+  -X, --remove SELECTOR            Remove source element using XPath selector.
+                                   Use -X- to ignore stored profile.
+  -R, --rx /PATTERN/REPLACEMENT/   Edit source HTML using regular expressions.
+                                   Use -R- to ignore stored profile.
+  -B, --browse                     After processing, open resulting HTML in default browser.
+
+== DEPENDENCIES:
+
+* Builder (https://rubyforge.org/projects/builder/)
+* Nokogiri (http://nokogiri.rubyforge.org/nokogiri/)
+* rchardet (https://rubyforge.org/projects/rchardet/)
+* launchy (http://copiousfreetime.rubyforge.org/launchy/)
+
+* wget or httrack
+* zip (Info-ZIP)
+
+== INSTALL:
+
+    gem install repub
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Invisible Llama <dg@invisiblellama.net>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+==

data/Rakefile

@@ -0,0 +1,30 @@
+begin
+  require 'bones'
+  Bones.setup
+rescue LoadError
+  begin
+    load 'tasks/setup.rb'
+  rescue LoadError
+    raise RuntimeError, '### please install the "bones" gem ###'
+  end
+end
+
+ensure_in_path 'lib'
+require 'repub'
+
+task :default => 'test:run'
+
+PROJ.name = 'repub'
+PROJ.authors = 'Dmitri Goutnik'
+PROJ.email = 'dg@invisiblellama.net'
+PROJ.url = 'http://github.com/invisiblellama/repub/tree/master'
+PROJ.version = Repub::VERSION
+PROJ.rubyforge.name = 'repub'
+PROJ.exclude = %w[tmp/ \.git \.DS_Store .*\.tmproj .*\.epub ^pkg/]
+
+PROJ.spec.opts << '--color'
+
+depend_on 'nokogiri'
+depend_on 'builder'
+depend_on 'chardet'
+depend_on 'launchy'

data/SAMPLES.txt

@@ -0,0 +1,23 @@
+* THE ADVENTURES OF SHERLOCK HOLMES
+
+repub -x 'title:div[@class='book']//h1' -x 'toc://table' -x 'toc_item://tr' -X '//pre' -X '//hr' -X '//body/h1' -X '//body/h2' http://www.gutenberg.org/dirs/etext99/advsh12h.htm
+
+* ALICE'S ADVENTURES IN WONDERLAND
+
+repub -x 'title:body/h1' -x 'toc://table' -x 'toc_item://tr' -X '//pre' -X '//hr' -X '//body/h4' http://www.gutenberg.org/files/11/11-h/11-h.htm
+
+* The Gelug-Kagyu Tradition of Mahamudra
+
+repub http://www.berzinarchives.com/web/x/prn/p.html_680632258.html
+
+* Брюс Стерлинг. Схизматрица
+
+repub -x 'title://h2' -x 'toc://table' -x 'toc_item://a' -X 'div' -X 'table' -X '//hr' http://lib.ru/STERLINGB/shizmatrica.txt_with-big-pictures.html
+
+* Айзек Азимов. Космические течения
+
+repub -x 'title://h2' -x 'toc://table' -x 'toc_item://a' -X 'div' -X 'table' -X '//hr' http://lib.ru/FOUNDATION/currspac.txt_with-big-pictures.html
+
+* Git User's Manual
+
+repub -x 'title://h1' -x 'toc://div[@class="toc"]/dl' -x 'toc_item:dt' -x 'toc_section:following-sibling::*[1]/dl' http://www.kernel.org/pub/software/scm/git/docs/user-manual.html

data/TODO

@@ -0,0 +1,3 @@
+√ add support for rx cleaning/modifying source doc
+√ make -q/-v actually do something
+  more parser tokens: author(s) etc

data/bin/repub

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby -w
+
+require File.expand_path(
+    File.join(File.dirname(__FILE__), %w[.. lib repub]))
+
+require 'repub/app'
+
+Repub::App.instance.run ARGV

data/lib/repub.rb

@@ -0,0 +1,46 @@
+module Repub
+
+  # :stopdoc:
+  VERSION = '0.3.0'
+  LIBPATH = File.expand_path(File.dirname(__FILE__)) + File::SEPARATOR
+  PATH = File.dirname(LIBPATH) + File::SEPARATOR
+  # :startdoc:
+
+  # Returns the version string for the library.
+  #
+  def self.version
+    VERSION
+  end
+
+  # Returns the library path for the module. If any arguments are given,
+  # they will be joined to the end of the libray path using
+  # <tt>File.join</tt>.
+  #
+  def self.libpath( *args )
+    args.empty? ? LIBPATH : File.join(LIBPATH, args.flatten)
+  end
+
+  # Returns the lpath for the module. If any arguments are given,
+  # they will be joined to the end of the path using
+  # <tt>File.join</tt>.
+  #
+  def self.path( *args )
+    args.empty? ? PATH : File.join(PATH, args.flatten)
+  end
+
+  # Utility method used to require all files ending in .rb that lie in the
+  # directory below this file that has the same name as the filename passed
+  # in. Optionally, a specific _directory_ name can be passed in such that
+  # the _filename_ does not have to be equivalent to the directory.
+  #
+  def self.require_all_libs_relative_to( fname, dir = nil )
+    dir ||= File.basename(fname, '.*')
+    search_me = File.expand_path(
+        File.join(File.dirname(fname), dir, '**', '*.rb'))
+
+    Dir.glob(search_me).each {|rb| p rb; require rb}
+  end
+  
+end
+
+$:.unshift Repub.libpath

data/lib/repub/app.rb

@@ -0,0 +1,42 @@
+require 'singleton'
+require 'rubygems'
+require 'launchy'
+require 'repub/app/utility'
+require 'repub/app/logger'
+require 'repub/app/options'
+require 'repub/app/profile'
+require 'repub/app/fetcher'
+require 'repub/app/parser'
+require 'repub/app/builder'
+
+module Repub
+  class App
+    include Singleton
+    
+    # Mix-in actual functionality
+    include Options, Profile, Fetcher, Parser, Builder, Logger
+
+    def self.name
+      File.basename($0)
+    end
+    
+    def self.data_path
+      File.join(File.expand_path('~'), '.repub')
+    end
+    
+    def run(args)
+      parse_options(args)
+      
+      log.level = options[:verbosity]
+      log.info "Making ePub from #{options[:url]}"
+      res = build(parse(fetch))
+      log.info "Saved #{res.output_path}"
+      
+      Launchy::Browser.run(res.asset_path) if options[:browser]
+    
+    rescue RuntimeError => ex
+      log.fatal "** ERROR: #{ex.to_s}"
+    end
+  
+  end
+end

data/lib/repub/app/builder.rb

@@ -0,0 +1,208 @@
+require 'fileutils'
+require 'tmpdir'
+require 'repub/epub'
+
+module Repub
+  class App
+    module Builder
+
+      class BuilderException < RuntimeError; end
+      
+      def build(parser)
+        Builder.new(options).build(parser)
+      end
+  
+      class Builder
+        include Epub, Logger
+        
+        attr_reader :output_path
+        attr_reader :asset_path
+        
+        def initialize(options)
+          @options = options
+        end
+        
+        def build(parser)
+          @parser = parser
+
+          # Initialize content.opf
+          @content = Content.new(@parser.uid)
+          # Default title is the parsed one
+          @content.metadata.title = @parser.title
+          # Override metadata values specified in options
+          if @options[:metadata]
+            @content.metadata.members.each do |m|
+              m = m.to_sym
+              next if m == :identifier   # do not allow to override uid
+              if @options[:metadata][m]
+                @content.metadata[m] = @options[:metadata][m]
+                log.debug "-- Setting metadata #{m} to \"#{@content.metadata[m]}\""
+              end
+            end
+          end
+          
+          # Initialize toc.ncx
+          @toc = Toc.new(@parser.uid)
+          # TOC title is the same as in content.opf
+          @toc.title = @content.metadata.title
+
+          # Setup output filename and path
+          @output_path = File.expand_path(@options[:output_path].if_blank('.'))
+          if File.exist?(@output_path) && File.directory?(@output_path)
+            @output_path = File.join(@output_path, @content.metadata.title.gsub(/\s/, '_'))
+          end
+          @output_path = @output_path +  '.epub'
+          log.debug "-- Setting output path to #{@output_path}"
+          
+          # Build EPUB
+          tmpdir = Dir.mktmpdir(App::name)
+          begin
+            FileUtils.chdir(tmpdir) do
+              copy_and_process_assets
+              write_meta_inf
+              write_mime_type
+              write_content
+              write_toc
+              write_epub
+            end
+          ensure
+            # Keep tmp folder if we're going open processed doc in browser
+            FileUtils.remove_entry_secure(tmpdir) unless @options[:browser]
+          end
+          self
+        end
+        
+        private
+        
+        MetaInf = 'META-INF'
+        
+        def postprocess_file(asset)
+          source = IO.read(asset)
+          # Do rx substitutions
+          if @options[:rx] && !@options[:rx].empty?
+            @options[:rx].each do |rx|
+              rx.strip!
+              delimiter = rx[0, 1]
+              rx = rx.gsub(/\\#{delimiter}/, "\n")
+              ra = rx.split(/#{delimiter}/).reject {|e| e.empty? }.each {|e| e.gsub!(/\n/, "#{delimiter}")}
+              raise ParserException, "Invalid regular expression" if ra.empty? || ra[0].nil? || ra.size > 2
+              pattern = ra[0]
+              replacement = ra[1] || ''
+              log.info "Replacing pattern /#{pattern.gsub(/#{delimiter}/, "\\#{delimiter}")}/ with \"#{replacement}\""
+              source.gsub!(Regexp.new(pattern), replacement)
+            end
+          end
+          # Add doctype if missing
+          if source !~ /\s*<!DOCTYPE/
+            log.debug "-- Adding missing doctype"
+            source = "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n" + source
+          end
+          # Save processed file
+          File.open(asset, 'w') do |f|
+            f.write(source)
+          end
+        end
+        
+        def postprocess_doc(asset)
+          doc = Nokogiri::HTML.parse(open(asset), nil, 'UTF-8')
+          # Substitute custom CSS
+          if (@options[:css] && !@options[:css].empty?)
+            doc.xpath('//link[@rel="stylesheet"]') do |link|
+              link[:href] = File.basename(@options[:css])
+              log.debug "-- Replacing CSS refs with #{link[:href]}"
+            end
+          end
+          # Remove elements
+          if @options[:remove] && !@options[:remove].empty?
+            @options[:remove].each do |selector|
+              log.info "Removing elements matching selector \"#{selector}\""
+              #p doc.search(selector).size
+              #p doc.search(selector)
+              doc.search(selector).remove
+            end
+          end
+          # Save processed doc
+          File.open(asset, 'w') do |f|
+            if @options[:fixup]
+              # HACK: Nokogiri seems to ignore the fact that xmlns and other attrs aleady present
+              # in html node and adds them anyway. Just remove them here to avoid duplicates.
+              doc.root.attributes.each {|name, value| doc.root.remove_attribute(name) }
+              doc.write_xhtml_to(f, :encoding => 'UTF-8')
+            else
+              doc.write_html_to(f, :encoding => 'UTF-8')
+            end
+          end
+        end
+        
+        def copy_and_process_assets
+          # Copy html
+          @parser.cache.assets[:documents].each do |asset|
+            log.debug "-- Processing document #{asset}"
+            # Copy asset from cache
+            FileUtils.cp(File.join(@parser.cache.path, asset), '.')
+            # Do post-processing
+            postprocess_file(asset)
+            postprocess_doc(asset)
+            @content.add_document(asset)
+            @asset_path = File.expand_path(asset)
+          end
+          # Copy css
+          if @options[:css].nil? || @options[:css].empty?
+            # No custom css, copy one from assets
+            @parser.cache.assets[:stylesheets].each do |css|
+              log.debug "-- Copying stylesheet #{css}"
+              FileUtils.cp(File.join(@parser.cache.path, css), '.')
+              @content.add_stylesheet(css)
+            end
+          else
+            # Copy custom css
+            log.debug "-- Using custom stylesheet #{@options[:css]}"
+            FileUtils.cp(@options[:css], '.')
+            @content.add_stylesheet(File.basename(@options[:css]))
+          end
+          # Copy images
+          @parser.cache.assets[:images].each do |image|
+            log.debug "-- Copying image #{image}"
+            FileUtils.cp(File.join(@parser.cache.path, image), '.')
+            @content.add_image(image)
+          end
+        end
+        
+        def write_meta_inf
+          FileUtils.mkdir_p(MetaInf)
+          FileUtils.chdir(MetaInf) do
+            Epub::Container.new.save
+          end
+        end
+        
+        def write_mime_type
+          File.open('mimetype', 'w') do |f|
+            f << 'application/epub+zip'
+          end
+        end
+    
+        def write_content
+          @content.save
+        end
+        
+        def write_toc
+          add_nav_points(@toc.nav_map, @parser.toc)
+          @toc.save
+        end
+        
+        def add_nav_points(nav_collection, toc)
+          toc.each do |t|
+            nav_point = nav_collection.add_nav_point(t.title, t.src)
+            add_nav_points(nav_point, t.subitems) if t.subitems
+          end
+        end
+        
+        def write_epub
+          %x(zip -X9 \"#{@output_path}\" mimetype)
+          %x(zip -Xr9D \"#{@output_path}\" * -xi mimetype)
+        end
+      end
+
+    end
+  end
+end