softcover-nonstop 1.0.beta2

data/lib/softcover/builders/html.rb

@@ -0,0 +1,239 @@
+require 'fileutils'
+
+module Softcover
+  module Builders
+    class Html < Builder
+      include Softcover::Utils
+
+      def setup
+        Dir.mkdir "html" unless File.directory?("html")
+        html_styles = path('html/stylesheets')
+        unless File.directory?(html_styles)
+          Dir.mkdir html_styles
+        end
+        template_dir = path("#{File.dirname(__FILE__)}/../template")
+        custom_css = path("#{template_dir}/html/stylesheets/custom.css")
+        target = path("#{html_styles}/custom.css")
+        FileUtils.cp(custom_css, target) unless File.exist?(target)
+        clean!
+      end
+
+      def build(options = {})
+        if Softcover::profiling?
+          require 'ruby-prof'
+          RubyProf.start
+        end
+
+        write_master_latex_file(manifest)
+
+        if manifest.markdown?
+          unless options[:'find-overfull']
+            remove_unneeded_polytex_files
+          end
+          manifest.chapters.each do |chapter|
+            write_latex_files(chapter, options)
+          end
+
+          # Reset the manifest to use PolyTeX.
+          self.manifest = Softcover::BookManifest.new(source: :polytex,
+                                                      verify_paths: false,
+                                                      origin: :markdown)
+        end
+
+        if manifest.polytex?
+          basename = File.basename(manifest.filename, '.tex')
+          @html  = converted_html(basename)
+          @title = basename
+          @mathjax = Softcover::Mathjax::config(chapter_number: false)
+          @src     = Softcover::Mathjax::AMS_SVG
+          erb_file = File.read(File.join(File.dirname(__FILE__),
+                                         '..', 'server', 'views',
+                                         'book.html.erb'))
+          file_content = ERB.new(erb_file).result(binding)
+          write_full_html_file(basename, file_content)
+          write_chapter_html_files(Nokogiri::HTML(file_content), erb_file)
+        end
+
+        if Softcover::profiling?
+          result = RubyProf.stop
+          printer = RubyProf::GraphPrinter.new(result)
+          printer.print(STDOUT, {})
+        end
+
+        true
+      end
+
+      # Removes any PolyTeX files not corresponding to current MD chapters.
+      def remove_unneeded_polytex_files
+        files_to_keep = manifest.chapters.map do |chapter|
+                          path("#{manifest.polytex_dir}/#{chapter.slug}.tex")
+                        end
+        all_files = Dir.glob(path("#{manifest.polytex_dir}/*.tex"))
+        files_to_remove = all_files - files_to_keep
+        FileUtils.rm(files_to_remove)
+      end
+
+      # Writes the LaTeX files for a given Markdown chapter.
+      def write_latex_files(chapter, options = {})
+        polytex_filename = path("#{manifest.polytex_dir}/#{chapter.slug}.tex")
+        if chapter.source == :polytex
+          FileUtils.cp path("chapters/#{chapter.full_name}"), polytex_filename
+        else
+          mkdir Softcover::Directories::TMP
+          markdown = File.read(path("chapters/#{chapter.full_name}"))
+          # Only write if the Markdown file hasn't changed since the last time
+          # it was converted, as then the current PolyTeX file is up-to-date.
+          # The call to File.exist?(filename) is just in case the PolyTeX file
+          # corresponding to the Markdown file was removed by hand in the
+          # interim.
+          unless (File.exist?(chapter.cache_filename) &&
+                  File.read(chapter.cache_filename) == digest(markdown) &&
+                  File.exist?(polytex_filename) &&
+                  !markdown.include?('\input'))
+            File.write(polytex_filename, polytex(chapter, markdown))
+          end
+        end
+      end
+
+      # Returns the PolyTeX for the chapter.
+      # As a side-effect, we cache a digest of the Markdown to prevent
+      # unnecessary conversions.
+      def polytex(chapter, markdown)
+        File.write(chapter.cache_filename, digest(markdown))
+        p = Polytexnic::Pipeline.new(markdown,
+                                     source: :markdown,
+                                     custom_commands: Softcover.custom_styles,
+                                     language_labels: language_labels)
+        p.polytex
+      end
+
+      # Returns the converted HTML.
+      def converted_html(basename)
+        polytex_filename = basename + '.tex'
+        polytex = File.read(polytex_filename)
+        # Replace the includes with the file contents, padding with a trailing
+        # newline for safety.
+        polytex.gsub!(/(^\s*\\include{(.*?)})/) do
+          File.read($2 + '.tex') + "\n"
+        end
+        Polytexnic::Pipeline.new(polytex,
+                                 custom_commands: Softcover.custom_styles,
+                                 language_labels: language_labels).to_html
+      end
+
+      # Writes the full HTML file for the book.
+      # The resulting file is a self-contained HTML document suitable
+      # for viewing in isolation.
+      def write_full_html_file(basename, file_content)
+        html_filename = File.join('html', basename + '.html')
+        File.open(html_filename, 'w') do |f|
+          f.write(file_content)
+        end
+        polytexnic_css = File.join('html', 'stylesheets', 'softcover.css')
+        source_css     = File.join(File.dirname(__FILE__),
+                                   "../template/#{polytexnic_css}")
+        FileUtils.cp source_css, polytexnic_css
+        write_pygments_file(:html, File.join('html', 'stylesheets'))
+        built_files.push html_filename
+      end
+
+      # Writes the full HTML file for each chapter.
+      # The resulting files are self-contained HTML documents suitable
+      # for viewing in isolation.
+      def write_chapter_html_files(html, erb_file)
+        reference_cache = split_into_chapters(html)
+        target_cache = build_target_cache(html)
+        manifest.chapters.each_with_index do |chapter, i|
+          update_cross_references(chapter, reference_cache, target_cache)
+          write_fragment_file(chapter)
+          write_complete_file(chapter, erb_file, i)
+        end
+      end
+
+      # Splits the full XML document into chapters.
+      def split_into_chapters(xml)
+        chapter_number = 0
+        current_chapter = manifest.chapters.first
+        reference_cache = {}
+        xml.css('#book>div').each do |node|
+          klass = node.attributes['class'].to_s
+          id = node.attributes['id'].to_s
+          if klass == 'chapter' || id == 'frontmatter'
+            current_chapter = manifest.chapters[chapter_number]
+            node['data-chapter'] = current_chapter.slug
+            chapter_number += 1
+          end
+
+          reference_cache[node['data-tralics-id']] = current_chapter
+          node.xpath('.//*[@data-tralics-id]').each do |labeled_node|
+            reference_cache[labeled_node['data-tralics-id']] = current_chapter
+          end
+
+          current_chapter.nodes.push node
+        end
+        reference_cache
+      end
+
+      # Builds a cache of targets for cross-references.
+      def build_target_cache(xml)
+        {}.tap do |target_cache|
+          xml.xpath("//*[@id]").each do |target|
+            target_cache[target['id']] = target
+          end
+        end
+      end
+
+      # Updates the book's cross-references.
+      def update_cross_references(chapter, ref_map, target_cache)
+        chapter.nodes.each do |node|
+          node.css('a.hyperref').each do |ref_node|
+            ref_id = ref_node['href'][1..-1]  # i.e., 'cha-foo_bar'
+            target = target_cache[ref_id]
+            unless target.nil?
+              id = target['id']
+              ref_chapter = if target['data-tralics-id'].nil?
+                              # This branch is true for chapter-star.
+                              chapter
+                            else
+                              ref_map[target['data-tralics-id']]
+                            end
+              ref_node['href'] = "#{ref_chapter.fragment_name}##{id}"
+            end
+          end
+        end
+      end
+
+      # Writes the chapter fragment HTML (omitting, e.g., <html> tags, etc.)
+      def write_fragment_file(chapter)
+        html_filename = File.join('html', "#{chapter.slug}_fragment.html")
+        File.open(html_filename, 'w') do |f|
+          chapter.nodes.each do |node|
+            f.write(node.to_xhtml)
+          end
+        end
+        built_files.push html_filename
+      end
+
+      # Writes the chapter as a complete, self-contained HTML document.
+      def write_complete_file(chapter, erb_file, n)
+        html_filename = File.join('html', chapter.slug + '.html')
+        File.open(html_filename, 'w') do |f|
+          @html = chapter.nodes.map(&:to_xhtml).join("\n")
+          @mathjax = Softcover::Mathjax::config(chapter_number: n)
+          @src     = Softcover::Mathjax::AMS_SVG
+          file_content = ERB.new(erb_file).result(binding)
+          f.write(file_content)
+        end
+        built_files.push html_filename
+      end
+
+      def clean!
+        # It's safe to remove HTML files in the html/ directory,
+        # as they are regenerated every time the book gets built.
+        # This also arranges to clear out unused HTML files, as happens when,
+        # e.g., the name of a LaTeX chapter file changes.
+        FileUtils.rm(Dir.glob(path('html/*.html')))
+      end
+    end
+  end
+end

data/lib/softcover/builders/mobi.rb

@@ -0,0 +1,59 @@
+module Softcover
+  module Builders
+    class Mobi < Builder
+      include Softcover::Utils
+      include Softcover::EpubUtils
+
+      def build!(options={})
+        Softcover::Builders::Epub.new.build!(options)
+        filename = mobi_filename(options)
+        command  = mobi_command(filename, options)
+        silent   = options[:silent] || Softcover.test?
+        if options[:quiet] || silent
+          silence { system(command) }
+        else
+          system(command)
+        end
+      end
+
+      # Returns the filename of the MOBI (preview if necessary).
+      def mobi_filename(options={})
+        options[:preview] ? manifest.filename + '-preview' : manifest.filename
+      end
+
+      # Returns the command for making a MOBI, based on the options.
+      def mobi_command(filename, options={})
+        if options[:kindlegen]
+          "#{kindlegen} ebooks/#{filename}.epub"
+        else
+          "#{calibre} ebooks/#{filename}.epub ebooks/#{filename}.mobi" +
+          " #{calibre_options}"
+        end
+      end
+
+      private
+
+        def calibre
+          @calibre ||= executable(dependency_filename(:calibre))
+        end
+
+        # Returns the options for the Calibre `ebook-convert` CLI.
+        def calibre_options
+          # Include both Mobipocket & KF8 formats.
+          # It took me forever to figure this out. It really should be
+          # the Calibre default.
+          opts = "--mobi-file-type both"
+          if cover?
+            opts += " --cover #{cover_img_path}"
+            # Get covers to work in Kindle desktop app.
+            opts += " --share-not-sync"
+          end
+          opts
+        end
+
+        def kindlegen
+          @kindlegen ||= executable(dependency_filename(:kindlegen))
+        end
+    end
+  end
+end

data/lib/softcover/builders/pdf.rb

@@ -0,0 +1,160 @@
+module Softcover
+  module Builders
+    class Pdf < Builder
+      include Softcover::Output
+      include Softcover::Utils
+
+      def build!(options={})
+        make_png_from_gif
+        if manifest.markdown?
+          # Build the HTML to produce PolyTeX as a side-effect,
+          # then update the manifest to reduce PDF generation
+          # to a previously solved problem.
+          Softcover::Builders::Html.new.build!
+          opts = options.merge({ source: :polytex, origin: :markdown})
+          self.manifest = Softcover::BookManifest.new(opts)
+        end
+
+        write_master_latex_file(manifest)
+
+        # Build the PolyTeX filename so it accepts both 'foo' and 'foo.tex'.
+        basename = File.basename(manifest.filename, '.tex')
+        book_filename = basename + '.tex'
+
+        # In debug mode, execute `xelatex` and exit.
+        if options[:debug]
+          execute "#{xelatex} #{book_filename}"
+          return    # only gets called in test env
+        end
+
+        polytex_filenames = manifest.pdf_chapter_filenames << book_filename
+        polytex_filenames.each do |filename|
+          polytex = File.read(filename)
+          latex   = Polytexnic::Pipeline.new(polytex,
+                                             language_labels: language_labels).
+                                            to_latex
+          if filename == book_filename
+            latex.gsub!(/\\include{(.*?)}/) do
+              "\\include{#{Softcover::Utils.tmpify(manifest, $1)}.tmp}"
+            end
+          end
+          File.write(Softcover::Utils.tmpify(manifest, filename), latex)
+        end
+        write_pygments_file(:latex, Softcover::Directories::STYLES)
+        copy_polytexnic_sty
+
+        # Renaming the PDF in the command is necessary because `execute`
+        # below uses `exec` (except in tests, where it breaks). Since `exec`
+        # causes the Ruby process to end, any Ruby code after `exec`
+        # is ignored.
+        # (The reason for using `exec` is so that LaTeX errors get emitted to
+        # the screen rather than just hanging the process.)
+        cmd = "#{pdf_cmd(book_filename, options)} " +
+              "; #{rename_pdf(basename, options)}"
+        # Here we use `system` when making a preview because the preview command
+        # needs to run after the main PDF build.
+        if options[:quiet] || options[:silent]
+          silence_stream(STDERR) do
+            silence { options[:preview] ? system(cmd) : execute(cmd) }
+          end
+        elsif options[:'find-overfull']
+          silence_stream(STDERR) { execute(cmd) }
+        else
+          options[:preview] ? system(cmd) : execute(cmd)
+        end
+      end
+
+      private
+
+        # Returns the `xelatex` executable.
+        # The `xelatex` program is roughly equivalent to the more standard
+        # `pdflatex`, but has better support for Unicode.
+        def xelatex
+          @xelatex ||= executable(dependency_filename(:latex))
+        end
+
+        # Make a PNG for each GIF in the images/ directory.
+        def make_png_from_gif
+          gif_pattern = File.join('images', '**', '*.gif')
+          gif_files = Dir.glob(gif_pattern).reject { |f| File.directory?(f) }
+          gif_files.each do |gif|
+            dir = File.dirname(gif)
+            ext = File.extname(gif)
+            img = File.basename(gif, ext)
+            png = File.join(dir, "#{img}.png")
+            system "#{convert} #{gif} #{png}"
+          end
+        end
+
+        # Returns the executable to ImageMagick's `convert` program.
+        def convert
+          @convert ||= executable(dependency_filename(:convert))
+        end
+
+        # Returns the command to build the PDF (once).
+        def build_pdf(book_filename, options={})
+          tmp_filename = Softcover::Utils.tmpify(manifest, book_filename)
+          if options[:silent] || options[:quite]
+              "#{xelatex} --interaction=nonstopmode #{tmp_filename} #{options[:filters]}"
+          elsif options[:nonstop]
+              "#{xelatex} --interaction=nonstopmode #{tmp_filename} #{options[:filters]}"
+	  else
+              "#{xelatex} #{tmp_filename} #{options[:filters]}"
+          end
+        end
+
+        # Returns the full command to build the PDF.
+        def pdf_cmd(book_filename, options={})
+          if options[:once]
+            build_pdf(book_filename, options)
+          elsif options[:'find-overfull']
+            # The way we do things, code listings show up as "Overfull", but
+            # they're actually fine, so filter them out.
+            filter_out_listings = "grep -v 3.22281pt"
+            # Because each chapter typically lives in a separate file, it's
+            # hard to correlate Overfull line numbers with lines in the source,
+            # so we use grep's -A flag to provide some context instead. Authors
+            # can then use their text editors' search function to find the
+            # corresponding place in the text.
+            show_context = 'grep -A 3 "Overfull \\\\\\\\hbox"'
+            build_pdf(book_filename,
+                      filters: "| #{filter_out_listings} | #{show_context}" )
+          else
+            # Run the command twice (to guarantee up-to-date cross-references).
+            cmd = build_pdf(book_filename, options)
+            "#{cmd} ; #{cmd}"
+          end
+        end
+
+        # Returns the command to rename the temp PDF.
+        # The purpose is to match the original filename.
+        # For example, foo_bar.tex should produce foo_bar.pdf.
+        # While we're at it, we move it to the standard ebooks/ directory.
+        def rename_pdf(basename, options={})
+          tmp_pdf = basename + '.tmp.pdf'
+          name    = options[:preview] ? basename + '-preview' : basename
+          pdf     = name + '.pdf'
+          mkdir('ebooks')
+          # Remove the intermediate tmp files unless only running once.
+          rm_tmp = keep_tmp_files?(options) ? "" : "&& rm -f *.tmp.*"
+          "mv -f #{tmp_pdf} #{File.join('ebooks', pdf)} #{rm_tmp}"
+        end
+
+        # Keeps tmp files when running once, including when finding overfull.
+        # The main purpose is to keep the *.aux files around for the ToC, xrefs,
+        # etc.
+        def keep_tmp_files?(options)
+          options[:once] || options[:'find-overfull'] || Softcover.test?
+        end
+
+        # Copies the style file to ensure it's always fresh.
+        def copy_polytexnic_sty
+          softcover_sty  = File.join(Softcover::Directories::STYLES,
+                                     'softcover.sty')
+          source_sty     = File.join(File.dirname(__FILE__),
+                                     '..', 'template', softcover_sty)
+          FileUtils.cp source_sty, softcover_sty
+        end
+    end
+  end
+end

data/lib/softcover/builders/preview.rb

@@ -0,0 +1,38 @@
+module Softcover
+  module Builders
+    class Preview < Builder
+
+      def build!
+        # Recall that MOBI generation makes an EPUB as a side-effect.
+        Softcover::Builders::Mobi.new.build!(preview: true)
+        Softcover::Builders::Pdf.new.build!(preview: true)
+        extract_pdf_pages
+      end
+
+      private
+
+        # Extracts pages from the PDF using GhostScript.
+        # The page range is set by the `pdf_preview_page_range` parameter
+        # in book.yml so that authors can override the default range.
+        def extract_pdf_pages
+          input  = File.join('ebooks', manifest.filename + '.pdf')
+          output = input.sub('.pdf', '-preview.pdf')
+          unless manifest.respond_to?(:pdf_preview_page_range)
+            $stderr.puts("Error: Preview not built")
+            $stderr.puts("Define pdf_preview_page_range in config/book.yml")
+            $stderr.puts("See http://manual.softcover.io/book/getting_started#sec-build_preview")
+            exit(1)
+          end
+          range = manifest.pdf_preview_page_range.split('..').map(&:to_i)
+          cmd  = %(yes | #{ghostscript} -dBATCH -sOutputFile="#{output}")
+          cmd += %( -dFirstPage=#{range.first} -dLastPage=#{range.last})
+          cmd += %( -sDEVICE=pdfwrite "#{input}" > /dev/null 2> /dev/null)
+          execute cmd
+        end
+
+        def ghostscript
+          @ghostscript ||= executable(dependency_filename(:ghostscript))
+        end
+    end
+  end
+end

data/lib/softcover/builders/utils/page.js

@@ -0,0 +1,62 @@
+// Processes an HTML page, including any MathJax math typesetting.
+// Output is written to the file `phantomjs_source.html`.
+
+var system = require('system');
+var fs = require('fs');
+var page = require('webpage').create();
+
+page.open(system.args[1], function (status) {
+    if (status !== "success") {
+      console.log("Unable to access network");
+    } else {
+      //
+      //  This gets called when MathJax is done
+      //
+      page.onAlert = function (msg) {
+        if (msg === "MathJax Done") {
+          var html = page.evaluate(function(){
+            var hiddenDiv = document.getElementById('MathJax_SVG_Hidden');
+            var svgs = document.getElementsByTagName('svg');
+            var origDefs = hiddenDiv.nextSibling.firstChild;
+            for (var i = 1; i < svgs.length; ++i)  {
+              var defs = origDefs.cloneNode(false);
+              var svg = svgs[i];
+              // append shallow defs and change xmlns.
+              svg.insertBefore(defs, svg.firstChild);
+              svg.setAttribute("xmlns", "http://www.w3.org/2000/svg");
+              // clone and copy all used paths into local defs.
+              var uses = svg.getElementsByTagName("use");
+              for (var k = 0; k < uses.length; ++k) {
+                var id = uses[k].getAttribute("href");
+                defs.appendChild(
+                  document.getElementById(id.substr(1)).cloneNode(true)
+                );
+                uses[k].setAttribute("xlink:href", id);
+              }
+              svg.style.position = "static";
+            }
+            svg_defs = document.getElementsByTagName('div')[0];
+            svg_defs.parentNode.removeChild(svg_defs);
+            return document.body.innerHTML;
+          });
+
+          utf8 = '<meta http-equiv="Content-type" content="text/html; charset=utf-8">'
+          fs.write('phantomjs_source.html', utf8 + html, 'w');
+          phantom.exit();
+        } else if (msg === "MathJax Timeout") {
+            console.log("Timed out waiting for MathJax");
+            phantom.exit();
+        } else { console.log(msg) }
+      }
+    }
+
+    page.evaluate(function () {
+      var script = document.createElement("script");
+      script.type = "text/x-mathjax-config";
+      script.text = "MathJax.Hub.Queue([alert,'MathJax Done'])";
+      document.head.appendChild(script);
+      // Time out after 60 seconds, which should be long enough for
+      // almost all documents.
+      setTimeout(function () { alert("MathJax Timeout") }, 60000);
+    });
+});