polytexnic 0.5.0

data/lib/polytexnic/postprocessors/latex.rb

@@ -0,0 +1,18 @@
+require 'polytexnic/literal'
+
+module Polytexnic
+  module Postprocessor
+    module Latex
+
+      # Restores literal environments (verbatim, code, math, etc.).
+      def replace_hashes(polytex)
+        puts polytex if debug?
+        polytex.tap do
+          literal_cache.each do |key, value|
+            polytex.gsub!(key, escape_backslashes(value))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/polytexnic/postprocessors/polytex.rb

@@ -0,0 +1,44 @@
+# encoding=utf-8
+module Polytexnic
+  module Postprocessor
+    module Polytex
+
+      # Removes references to the hypertarget package.
+      # TODO: Support hypertarget
+      # This isn't a priority, as you get most of what you need
+      # with hyperref.
+      def remove_hypertarget
+        @source.gsub!(/\\hypertarget.*$/, '')
+      end
+
+      # Fixes a kramdown verbatim bug.
+      # When converting code, kramdown outputs
+      # "\begin{verbatim}foo" instead of
+      # "\begin{verbatim}\nfoo".
+      def fix_verbatim_bug
+        @source.gsub!(/\\begin\{verbatim\}/) { |s| s + "\n" }
+      end
+
+      # Writes the PolyTeX code environments based on the code cache.
+      # I.e., code that looks like
+      # {lang="ruby"}
+      #     def foo
+      #       "bar"
+      #     end
+      # becomes
+      # %= lang:ruby
+      # \begin{code}
+      # def foo
+      #   "bar"
+      # end
+      # \end{code}
+      # which reduces syntax highlighting to a previously solved problem.
+      def write_polytex_code
+        code_cache.each do |key, (code, lang, in_codelisting)|
+          latex = "%= lang:#{lang}\n\\begin{code}\n#{code}\n\\end{code}"
+          @source.gsub!(key, latex)
+        end
+      end
+    end
+  end
+end

data/lib/polytexnic/preprocessor.rb

@@ -0,0 +1,23 @@
+# encoding=utf-8
+require 'polytexnic/literal'
+require 'polytexnic/preprocessors/html'
+require 'polytexnic/preprocessors/latex'
+require 'polytexnic/preprocessors/polytex'
+
+module Polytexnic
+  module Preprocessor
+    include Literal
+    include Html
+    include Latex
+    include Polytex
+
+    # Preprocesses the input based on output format.
+    def preprocess(format)
+      case format
+      when :html    then to_xml
+      when :latex   then to_processed_latex
+      when :polytex then to_polytex
+      end
+    end
+  end
+end

data/lib/polytexnic/preprocessors/html.rb

@@ -0,0 +1,349 @@
+# encoding=utf-8
+module Polytexnic
+  module Preprocessor
+    module Html
+
+      # Converts HTML to XML.
+      # The heart of the process is using Tralics to convert the input PolyTeX
+      # to XML. The raw PolyTeX needs to be processed first to make everything
+      # go smoothly, but after that the steps to producing the corresponding
+      # XML is straightforward.
+      def to_xml
+        polytex = process_for_tralics(@polytex)
+        doc = Nokogiri::XML(tralics_xml(polytex))
+        add_document_tag(doc)
+        @xml = doc.to_xml
+      end
+
+      private
+
+        # Processes the input PolyTeX for Tralics.
+        # The key steps are creating a clean document safe for making global
+        # substitutions (gsubs), and then making a bunch of gsubs.
+        def process_for_tralics(polytex)
+          clean_document(polytex).tap do |output|
+            process_spaces(output)
+            remove_commands(output)
+            hyperrefs(output)
+            title_fields(output)
+            maketitle(output)
+            label_names(output)
+            image_names(output)
+            restore_eq_labels(output)
+            convert_figure_centering(output)
+            convert_longtable(output)
+            mark_environments(output)
+            make_tabular_alignment_cache(output)
+          end
+        end
+
+        # Returns a clean document with cached literal environments.
+        # This is a key step: we cache literal environments that should be
+        # passed through the pipeline with no changes (verbatim, code, etc.).
+        # The result is a document that can safely be transformed using
+        # global substitutions.
+        def clean_document(polytex)
+          doc = cache_unicode(cache_literal(add_commands(polytex)))
+          inline_verbatim(doc)
+          cache_hrefs(doc)
+          remove_comments(doc)
+          double_backslashes(cache_display_inline_math(doc))
+        end
+
+        # Prepares spaces to be passed through the pipeline.
+        # Handles thin spaces ('\,') and normal spaces ('\ '), as well as
+        # end-of-sentence spaces.
+        def process_spaces(doc)
+          doc.gsub!(/\\,/, xmlelement('thinspace'))
+          # Match an end of sentence character, while also recognizing
+          # things like (Or otherwise.) and ``Yes, indeed!'' as being the
+          # ends of sentences.
+          end_of_sentence = '[.?!](?:\)|\'+)?'
+          # Handle a forced normal space '\ '.
+          doc.gsub!(/(#{end_of_sentence})\\ /) do
+            $1 + xmlelement('normalspace')
+          end
+          not_a_capital = '[^A-Z]'
+          # Case of "foo. A"
+          doc.gsub!(/(#{not_a_capital})(#{end_of_sentence})[ ]+([^\s])/) do
+            $1 + $2 + xmlelement('intersentencespace') + ' ' + $3
+          end
+          # Case of "foo.\n A"
+          doc.gsub!(/(#{not_a_capital})(#{end_of_sentence})\n[ ]+([^\s])/) do
+            $1 + $2 + xmlelement('intersentencespace') + ' ' + $3
+          end
+          # Case of "foo.\nA"
+          doc.gsub!(/(#{not_a_capital})(#{end_of_sentence})\n([^\n])/) do
+            $1 + $2 + xmlelement('intersentencespace') + ' ' + $3
+          end
+          # Handle the manual override to force an inter-sentence space, '\@',
+          # as in 'Superman II\@. A new sentence'.
+          doc.gsub!(/\\@\. /, '.' + xmlelement('intersentencespace') + ' ')
+        end
+
+        # Removes commands that might screw up Tralics.
+        def remove_commands(doc)
+          # Determine if we're using footnote symbols.
+          symbols_cmd = '\renewcommand{\thefootnote}{\fnsymbol{footnote}}'
+          @footnote_symbols = !!doc.match(/^\s*#{Regexp.escape(symbols_cmd)}/)
+
+          doc.gsub!(/^\s*\\renewcommand.*$/, '')
+        end
+
+        # Returns true if we should use footnote symbols in place of numbers.
+        def footnote_symbols?
+          @footnote_symbols
+        end
+
+        # Handles \verb environments.
+        # LaTeX supports an inline verbatim environment using
+        #   \verb+<stuff>+
+        # The + is arbitrary; any non-letter character is fine as long as it
+        # doesn't appear in <stuff>, so this code has exactly the same effect:
+        #   \verb!<stuff>!
+        #   \verb@<stuff>@
+        #   \verb8<stuff>8
+        # My preference is to use + or - if available.
+        def inline_verbatim(doc)
+          doc.gsub!(/\\verb([^A-Za-z])(.*?)\1/) do
+            key = digest($2)
+            literal_cache[key] = $2
+            xmlelement('inlineverbatim') { key }
+          end
+        end
+
+        # Removes commented-out lines.
+        def remove_comments(output)
+          output.gsub!(/[^\\]%.*$/, '')
+        end
+
+        # Converts LaTeX double backslashes to HTML breaks.
+        def double_backslashes(string)
+          lines = []
+          in_table = false
+          string.split("\n").each do |line|
+            in_table ||= (line =~ /^\s*\\begin{(?:tabular|longtable)}/)
+            line.gsub!('\\\\', xmlelement('backslashbreak')) unless in_table
+            lines << line
+            in_table = (in_table && line !~ /^\s*\\end{tabular}/)
+          end
+          lines.join("\n")
+        end
+
+        # Adds some default commands.
+        def add_commands(polytex)
+          line(custom_commands) + tralics_commands + polytex
+        end
+
+        # Pads a string with newlines.
+        def line(string)
+          "\n#{string}\n"
+        end
+
+        # Handles title fields.
+        def title_fields(string)
+          %w{title subtitle author date}.each do |field|
+            string.gsub! /\\#{field}\{(.*)\}/ do |s|
+              maketitle_elements[field] = $1
+              ''
+            end
+          end
+        end
+
+        # Replaces maketitle with an XML element.
+        def maketitle(string)
+          string.gsub! /\\maketitle/ do |s|
+            xmlelement('maketitle')
+          end
+        end
+
+        # Preserves label names.
+        # Tralics doesn't keep the names of labels, e.g., 'cha:foobar' in
+        # '\label{cha:foobar}'. But Tralics supplies a wide variety of
+        # pseudo-LaTeX commands to add arbitrary XML elements to the final
+        # document. In this case, the \xbox command does the trick. See
+        # http://www-sop.inria.fr/marelle/tralics/doc-x.html
+        # for more information.
+        def label_names(string)
+          string.gsub! /\\label\{(.*?)\}/ do |s|
+            label = $1.gsub(':', '-').gsub('_', underscore_digest)
+            "#{s}\n\\xbox{data-label}{#{label}}"
+          end
+        end
+
+        # Handles image names with underscores.
+        # This is a terrible kludge, and it's annoying that it's
+        # apparently necessary.
+        def image_names(string)
+          string.gsub! /\\image\{(.*?)\}/ do |s|
+            escaped_filename = $1.gsub('_', underscore_digest)
+            "\\image{#{escaped_filename}}"
+          end
+          string.gsub! /\\imagebox\{(.*?)\}/ do |s|
+            escaped_filename = $1.gsub('_', underscore_digest)
+            "\\imagebox{#{escaped_filename}}"
+          end
+        end
+
+        # Restores the equation labels.
+        def restore_eq_labels(output)
+          math_label_cache.each do |key, label|
+            output.gsub!(key, label)
+          end
+        end
+
+        # Handles centering in figures.
+        # The way we handle generic \begin{center}...\end{center} doesn't
+        # work in figures for some reason. Luckily, the preferred method
+        # is to use \centering anyway, so this kludge is actually better LaTeX.
+        def convert_figure_centering(output)
+          @in_figure = false
+          centered = output.split("\n").map do |line|
+            if line =~ /^\s*\\begin\{figure\}/
+              @in_figure = true
+              line
+            elsif @in_figure && line =~ /^\s*\\begin\{center\}/
+              '\centering'
+            elsif @in_figure && line =~ /^\s*\\end\{center\}/
+              ''
+            elsif @in_figure && line =~ /^\s*\\end\{figure\}/
+              @in_figure = false
+              line
+            else
+              line
+            end
+          end.join("\n")
+          output.replace(centered)
+        end
+
+        # Converts the longtable environment to simple tabular.
+        # This is mainly because kramdown outputs longtables by default,
+        # but as a side-effect you can also use longtables in PolyTeX
+        # input documents.
+        def convert_longtable(output)
+          output.gsub!('\begin{longtable}', '\begin{tabular}')
+          output.gsub!('\end{longtable}',   '\end{tabular}')
+        end
+
+        # Marks environments with their types.
+        # Tralics strips some information when processing LaTeX, such as
+        # whether a particular div defines a chapter. We remedy this by
+        # using the \AddAttToCurrent pseudo-LaTeX command to mark such
+        # environments with their types.
+        def mark_environments(string)
+
+          # Marks chapters with a 'chapter' type.
+          # Also handles \chapter*.
+          string.gsub! /^\s*\\chapter\*?\{(.*)\}/ do |s|
+            "#{s}\n\\AddAttToCurrent{type}{chapter}"
+          end
+
+          # Wrap codelistings in a 'codelisting' element.
+          string.gsub! /\\begin{codelisting}/ do |s|
+            "\\begin{xmlelement*}{codelisting}\n#{s}"
+          end
+          string.gsub! /\\end{codelisting}/ do |s|
+            "#{s}\n\\end{xmlelement*}"
+          end
+
+          # Wrap asides in an 'aside' element.
+          string.gsub! /\\begin{aside}/ do |s|
+            "\\begin{xmlelement*}{aside}\n#{s}"
+          end
+          string.gsub! /\\end{aside}/ do |s|
+            "#{s}\n\\end{xmlelement*}"
+          end
+
+          # Replace quotations and verse with corresponding XML elements.
+          string.gsub! /\\begin{quote}/ do |s|
+            quotation = '\AddAttToCurrent{class}{quotation}'
+            "\\begin{xmlelement*}{blockquote}\n#{quotation}"
+          end
+          string.gsub! /\\end{quote}/ do |s|
+            "\\end{xmlelement*}"
+          end
+          string.gsub! /\\begin{verse}/ do |s|
+            "\\begin{xmlelement*}{blockquote}\n\\AddAttToCurrent{class}{verse}"
+          end
+          string.gsub! /\\end{verse}/ do |s|
+            "\\end{xmlelement*}"
+          end
+
+          # Handle \begin{center}...\end{center}
+          string.gsub! /\\begin{center}/, '\begin{xmlelement*}{center}'
+          string.gsub! /\\end{center}/,   '\end{xmlelement*}'
+
+          # Handle \centering
+          string.gsub! /\\centering/, '\AddAttToCurrent{class}{center}'
+
+          # # Handle \image
+          # string.gsub! /\\image/, '\includegraphics'
+        end
+
+        # Collects alignment information for tabular environments.
+        # We suck out all the stuff like 'l|l|lr' in
+        # \begin{tabular}{l|l|lr}
+        # The reason is that we need to work around a couple of bugs in Tralics.
+        # I've tried in vain to figure out WTF is going on in the Tralics
+        # source, but it's easy enough in Ruby so I'm throwing it in here.
+        def make_tabular_alignment_cache(output)
+          alignment_regex = /^\s*\\begin{tabular}{((?:\|*[lcr]+\|*)+)}/
+          @tabular_alignment_cache = output.scan(alignment_regex).flatten
+        end
+
+        # Returns the XML produced by the Tralics program.
+        # There is a lot of ugly file manipulation here, but it's fundamentally
+        # straightforward. The heart of it is
+        #
+        #   system("#{tralics} -nomathml #{file.path} > log/tralics.log")
+        #
+        # which writes the converted PolyTeX file as XML, which then gets
+        # read in and lightly processed.
+        def tralics_xml(polytex)
+          file = Tempfile.new(['polytex', '.tex'])
+          puts polytex if debug?
+          file.write(polytex)
+          file.close
+          Dir.mkdir 'log' unless File.directory?('log')
+          system("#{tralics} -nomathml #{file.path} > log/tralics.log")
+          dirname = File.dirname(file.path)
+          xml_filename = File.basename(file.path, '.tex') + '.xml'
+          raw_xml = File.read(File.join(dirname, xml_filename))
+          xml = clean_xml(raw_xml)
+          puts xml if debug?
+          xml
+        ensure
+          xmlfile = file.path.sub('.tex', '.xml')
+          logfile = file.path.sub('.tex', '.log')
+          [xmlfile, logfile].each do |file|
+            File.delete(file) if File.exist?(file)
+          end
+          file.delete
+        end
+
+        # Wraps the whole document in <document></document>.
+        # Fragmentary documents come wrapped in 'unknown' tags.
+        # Full documents are wrapped in 'std' tags.
+        # Change either to 'document' for consistency.
+        def add_document_tag(doc)
+          %w[unknown std].each do |parent_tag|
+            node = doc.at_css(parent_tag)
+            node.name = 'document' unless node.nil?
+          end
+        end
+
+        def clean_xml(raw_xml)
+          nokogiri_ellipsis_workaround(raw_xml)
+        end
+
+        # Fixes a Nokogiri bug.
+        # As of this writing, the latest version of Nokogiri (1.5.6) doesn't
+        # handle the horizontal ellipsis character '&#133;' correctly in Ruby 2.
+        # The kludgy solution is to replace it with '…' in the raw XML,
+        # which does work.
+        def nokogiri_ellipsis_workaround(raw_xml)
+          raw_xml.gsub('&#133;', '…')
+        end
+    end
+  end
+end

data/lib/polytexnic/preprocessors/latex.rb

@@ -0,0 +1,43 @@
+module Polytexnic
+  module Preprocessor
+    module Latex
+
+      def to_processed_latex
+        @polytex = polish_tables(process_asides(clean_latex_document))
+      end
+
+      # Returns LaTeX with hashed versions of literal environments.
+      # Literal environments are hashed and passed through the pipeline
+      # so that we can process things like refs to hyperrefs using gsubs.
+      def clean_latex_document
+        cache_literal(@polytex, :latex)
+      end
+
+      def polish_tables(text)
+        text.tap do
+          text.gsub!(/^\s*(\\begin\{table\})/) do
+            "#{$1}\n\\begin{center}\n\\small\n"
+          end
+          text.gsub!(/^\s*(\\end\{table\})/) { "\\end{center}\n#{$1}" }
+        end
+      end
+
+      # Processes aside environments.
+      # In order to get nice framed & shaded aside boxes, we need to
+      # transform the default aside into a new environment.
+      def process_asides(text)
+        # Transform asides with headings and labels.
+        aside_regex = /\\begin{aside}\n\s*
+                       \\heading{(.*?)}\s*
+                       \\label{(.*?)}\n
+                       (.*?)
+                       \\end{aside}/mx
+        text.tap do
+          text.gsub!(aside_regex) do
+            %(\\begin{shaded_aside}{#{$1}}{#{$2}}\n#{$3}\n\\end{shaded_aside})
+          end
+        end
+      end
+    end
+  end
+end

data/lib/polytexnic/preprocessors/polytex.rb

@@ -0,0 +1,127 @@
+# encoding=utf-8
+module Polytexnic
+  module Preprocessor
+    module Polytex
+
+      # Converts Markdown to PolyTeX.
+      # We adopt a unified approach: rather than convert "Markdown" (I use
+      # the term loosely*) directly to HTML, we convert it to PolyTeX and
+      # then run everything through the PolyTeX pipeline. Happily, kramdown
+      # comes equipped with a `to_latex` method that does most of the heavy
+      # lifting. The ouput isn't as clean as that produced by Pandoc (our
+      # previous choice), but it comes with significant advantages: (1) It's
+      # written in Ruby, available as a gem, so its use eliminates an external
+      # dependency. (2) It's the foundation for the "Markdown" interpreter
+      # used by Leanpub, so by using it ourselves we ensure greater
+      # compatibility with Leanpub books.
+      #
+      # * <rant>The number of mutually incompatible markup languages going
+      # by the name "Markdown" is truly mind-boggling. Most of them add things
+      # to John Gruber's original Markdown language in an ever-expanding
+      # attempt to bolt on the functionality needed to write longer documents.
+      # At this point, I fear that "Markdown" has become little more than a
+      # marketing term.</rant>
+      def to_polytex
+        require 'Kramdown'
+        cleaned_markdown = cache_code_environments
+        cleaned_markdown.tap do |markdown|
+          convert_code_inclusion(markdown)
+        end
+        math_cache = cache_math(cleaned_markdown)
+        # Override the header ordering, which starts with 'section' by default.
+        lh = 'chapter,section,subsection,subsubsection,paragraph,subparagraph'
+        kramdown = Kramdown::Document.new(cleaned_markdown, latex_headers: lh)
+        @source = restore_inclusion(restore_math(kramdown.to_latex, math_cache))
+      end
+
+      def cache_code_environments
+        output = []
+        lines = @source.split("\n")
+        indentation = ' ' * 4
+        while (line = lines.shift)
+          if line =~ /\{lang="(.*?)"\}/
+            language = $1
+            code = []
+            while (line = lines.shift) && line.match(/^#{indentation}(.*)$/) do
+              code << $1
+            end
+            code = code.join("\n")
+            key = digest(code)
+            code_cache[key] = [code, language]
+            output << key
+            output << line
+          elsif line =~ /^```\s*$/        # basic code fences
+            while (line = lines.shift) && !line.match(/^```\s*$/)
+              output << indentation + line
+            end
+            output << "\n"
+          elsif line =~ /^```(\w+)\s*$/   # syntax-highlighted code fences
+            language = $1
+            code = []
+            while (line = lines.shift) && !line.match(/^```\s*$/) do
+              code << line
+            end
+            code = code.join("\n")
+            key = digest(code)
+            code_cache[key] = [code, language]
+            output << key
+          else
+            output << line
+          end
+        end
+        output.join("\n")
+      end
+
+      # Caches Leanpub-style math.
+      # Leanpub uses the notation {$$}...{/$$} for both inline and block math,
+      # with the only difference being the presences of newlines:
+      #     {$$} x^2 {/$$}  % inline
+      # and
+      #     {$$}
+      #     x^2             % block
+      #     {/$$}
+      # I personally hate this notation and convention, but anyone who really
+      # cares should just use PolyTeX instead of Markdown.
+      def cache_math(text)
+        cache = {}
+        text.gsub!(/\{\$\$\}\n(.*?)\n\{\/\$\$\}/) do
+          key = digest($1)
+          cache[[:block, key]] = $1
+          key
+        end
+        text.gsub!(/\{\$\$\}(.*?)\{\/\$\$\}/) do
+          key = digest($1)
+          cache[[:inline, key]] = $1
+          key
+        end
+        cache
+      end
+
+      # Restores the Markdown math.
+      # This is easy because we're running everything through our LaTeX
+      # pipeline.
+      def restore_math(text, cache)
+        cache.each do |(kind, key), value|
+          case kind
+          when :inline
+            open  = '\('
+            close =  '\)'
+          when :block
+            open  = '\[' + "\n"
+            close = "\n" + '\]'
+          end
+          text.gsub!(key, open + value + close)
+        end
+        text
+      end
+    end
+
+    # Adds support for <<(path/to/code) inclusion.
+    def convert_code_inclusion(text)
+      text.gsub!(/^\s*<<(\(.*?\))/) { "<!-- inclusion= <<#{$1}-->" }
+    end
+    def restore_inclusion(text)
+      text.gsub(/% <!-- inclusion= (.*?)-->/) { "%= #{$1}" }
+    end
+  end
+end