polytexnic 0.5.0

data/lib/polytexnic/utils.rb

@@ -0,0 +1,176 @@
+require 'securerandom'
+
+module Polytexnic
+  module Utils
+    extend self
+
+    # Returns the executable for the Tralics LaTeX-to-XML converter.
+    def tralics
+      File.join(File.dirname(__FILE__), '..', '..',
+                'precompiled_binaries', 'tralics')
+    end
+
+    # Returns a salted hash digest of the string.
+    def digest(string, options = {})
+      salt = options[:salt] || SecureRandom.base64
+      Digest::SHA1.hexdigest("#{salt}--#{string}")
+    end
+
+    # Returns a digest for passing things through the pipeline.
+    def pipeline_digest(element)
+      value = digest("#{Time.now.to_s}::#{element}")
+      @literal_cache[element.to_s] ||= value
+    end
+
+    # Returns a digest for use in labels.
+    # I like to use labels of the form cha:foo_bar, but for some reason
+    # Tralics removes the underscore in this case.
+    def underscore_digest
+      pipeline_digest('_')
+    end
+
+    # Escapes backslashes.
+    # Interpolated backslashes need extra escaping.
+    # We only escape '\\' by itself, i.e., a backslash followed by spaces
+    # or the end of line.
+    def escape_backslashes(string)
+      string.gsub(/\\(\s+|$)/) { '\\\\' + $1.to_s }
+    end
+
+    # Caches URLs for \href commands.
+    def cache_hrefs(doc, latex=false)
+      doc.tap do |text|
+        text.gsub!(/\\href{(.*?)}/) do
+          key = digest($1)
+          literal_cache[key] = $1
+          "\\href{#{key}}"
+        end
+      end
+    end
+
+    # Returns a Tralics pseudo-LaTeX XML element.
+    # The use of the 'skip' flag is a hack to be able to use xmlelement
+    # even when generating, e.g., LaTeX, where we simply want to yield the
+    # block.
+    def xmlelement(name, skip = false)
+      output = (skip ? "" : "\\begin{xmlelement}{#{name}}")
+      output << yield if block_given?
+      output << (skip ? "" : "\\end{xmlelement}")
+    end
+
+    # Returns some commands for Tralics.
+    # For various reasons, we don't actually want to include these in
+    # the style file that gets passed to LaTeX. For example,
+    # the commands with 'xmlelt' aren't even valid LaTeX; they're actually
+    # pseudo-LaTeX that has special meaning to the Tralics processor.
+    def tralics_commands
+      <<-'EOS'
+% Commands specific to Tralics
+\def\hyperref[#1]#2{\xmlelt{a}{\XMLaddatt{target}{#1}#2}}
+\newcommand{\heading}[1]{\xmlelt{heading}{#1}}
+\newcommand{\codecaption}[1]{\xmlelt{heading}{#1}}
+\newcommand{\sout}[1]{\xmlelt{sout}{#1}}
+\newcommand{\kode}[1]{\xmlelt{kode}{#1}}
+\newcommand{\filepath}[1]{\xmlelt{filepath}{#1}}
+\newcommand{\image}[1]{\xmlelt{image}{#1}}
+\newcommand{\imagebox}[1]{\xmlelt{imagebox}{#1}}
+
+% Code listings
+\usepackage{amsthm}
+\theoremstyle{definition}
+\newtheorem{codelisting}{Listing}[chapter]
+\newtheorem{aside}{Box}[chapter]
+      EOS
+    end
+
+    # Highlights source code.
+    def highlight_source_code(document)
+      if document.is_a?(String) # LaTeX
+        substitutions = {}
+        document.tap do
+          code_cache.each do |key, (content, language, in_codelisting)|
+            code   = highlight(key, content, language, 'latex')
+            output = code.split("\n")
+            horrible_backslash_kludge(add_font_info(output.first))
+            code = output.join("\n")
+            substitutions[key] = in_codelisting ? code : framed(code)
+          end
+          document.gsub!(Regexp.union(substitutions.keys), substitutions)
+        end
+      else # HTML
+        document.css('div.code').each do |code_block|
+          key = code_block.content
+          next unless (value = code_cache[key])
+          content, language = value
+          code_block.inner_html = highlight(key, content, language, 'html')
+        end
+      end
+    end
+
+    # Puts a frame around code.
+    def framed(code)
+      "\\begin{framed_shaded}\n#{code}\n\\end{framed_shaded}"
+    end
+
+    # Highlights a code sample.
+    def highlight(key, content, language, formatter)
+      highlight_cache[key] ||= Pygments.highlight(content,
+                                                  lexer: language,
+                                                  formatter: formatter)
+    end
+
+    # Adds some verbatim font info (including size).
+    # We prepend rather than replace the styles because the Pygments output
+    # includes a required override of the default commandchars.
+    # Since the substitution is only important in the context of a PDF book,
+    # it only gets made if there's a style in 'softcover.sty' in the
+    # current directory
+    def add_font_info(string)
+      if File.exist?('softcover.sty')
+        regex = '{code}{Verbatim}{(.*)}'
+        styles = File.read('softcover.sty').scan(/#{regex}/).flatten.first
+        string.gsub!("\\begin{Verbatim}[",
+                     "\\begin{Verbatim}[#{styles},")
+      end
+      string
+    end
+
+    # Does something horrible with backslashes.
+    # OK, so the deal is that code highlighted for LaTeX contains the line
+    # \begin{Verbatim}[commandchars=\\\{\}]
+    # Oh crap, there are backslashes in there. This means we have no chance
+    # of getting things to work after interpolating, gsubbing, and so on,
+    # because in Ruby '\\foo' is the same as '\\\\foo', '\}' is '}', etc.
+    # I thought I escaped (heh) this problem with the `escape_backslashes`
+    # method, but here the problem is extremely specific. In particular,
+    # \\\{\} is really \\ and \{ and \}, but Ruby doensn't know WTF to do
+    # with it, and thinks that it's "\\{}", which is the same as '\{}'.
+    # The solution is to replace '\\\\' with some number of backslashes.
+    # How many? I literally had to just keep adding backslashes until
+    # the output was correct when running `poly build:pdf`.
+    def horrible_backslash_kludge(string)
+      string.gsub!(/commandchars=\\\\/, 'commandchars=\\\\\\\\')
+    end
+
+    # Returns true if we are debugging, false otherwise.
+    # Manually change to `true` on an as-needed basis.
+    def debug?
+      false
+    end
+
+    # Returns true if we are profiling the code, false otherwise.
+    # Manually change to `true` on an as-needed basis.
+    def profiling?
+      return false if test?
+      false
+    end
+
+    def set_test_mode!
+      @@test_mode = true
+    end
+
+    def test?
+      defined?(@@test_mode) && @@test_mode
+    end
+  end
+end

data/lib/polytexnic/version.rb

@@ -0,0 +1,3 @@
+module Polytexnic
+  VERSION = "0.5.0"
+end

data/lib/polytexnic.rb

@@ -0,0 +1,92 @@
+# encoding=utf-8
+require "polytexnic/utils"
+require "polytexnic/version"
+require "polytexnic/postprocessor"
+require "polytexnic/preprocessor"
+require 'tempfile'
+require 'nokogiri'
+require 'digest/sha1'
+require 'pygments'
+require 'msgpack'
+
+module Polytexnic
+
+  def self.style_file
+    'polytexnic_commands.sty'
+  end
+
+  # Writes the contents of the custom polytexnic style file.
+  # This is used by the `generate` method in the `softcover` gem.
+  # We put it here because `custom.sty` lives inside `polytexnic`
+  # so that core can support, e.g., '\PolyTeXnic'.
+  def self.write_polytexnic_style_file(dir)
+    csf = File.join(File.dirname(__FILE__), '..', style_file)
+    File.write(File.join(dir, style_file), File.read(csf))
+  end
+
+  class Pipeline
+    include Polytexnic::Preprocessor
+    include Polytexnic::Postprocessor
+    include Polytexnic::Utils
+
+    attr_accessor :literal_cache, :code_cache, :polytex, :xml, :html,
+                  :math_label_cache, :highlight_cache, :maketitle_elements,
+                  :custom_commands
+
+    def initialize(source, options = {})
+      @literal_cache = {}
+      @code_cache = {}
+      @maketitle_elements = {}
+      @highlight_cache_filename = '.highlight_cache'
+      if File.exist?(@highlight_cache_filename)
+        content = File.read(@highlight_cache_filename)
+        @highlight_cache = MessagePack.unpack(content) unless content.empty?
+      end
+      @highlight_cache ||= {}
+      @math_label_cache = {}
+      @source_format = options[:source] || :polytex
+      @custom_commands = File.read(Polytexnic.style_file) rescue ''
+      @custom_commands += "\n" + (options[:custom_commands] || '')
+      @source = source
+      if markdown?
+        preprocess(:polytex)
+        postprocess(:polytex)
+      end
+      @polytex = @source
+    end
+
+    def to_html
+      if profiling?
+        require 'ruby-prof'
+        RubyProf.start
+      end
+
+      preprocess(:html)
+      postprocess(:html)
+      puts @html if debug?
+
+      if profiling?
+        result = RubyProf.stop
+        printer = RubyProf::GraphPrinter.new(result)
+        printer.print(STDOUT, {})
+      end
+      @html.strip
+    end
+
+    def to_latex
+      preprocess(:latex)
+      postprocess(:latex)
+      @latex
+    end
+
+    private
+
+      def markdown?
+        @source_format == :markdown || @source_format == :md
+      end
+
+      def polytex?
+        @source_format == :polytex
+      end
+    end
+end

data/notes/pandoc.md

@@ -0,0 +1,41 @@
+
+## Math
+
+### Inline math
+
+	pandoc -s spec/fixtures/inline_math.tex --mathjax -o tmp/inline_math.html
+
+### Math environments
+
+    pandoc -s spec/fixtures/math_environments --mathjax -o tmp/math_environments.html
+
+Pandoc handles equation environments wrong; it converts
+
+    \begin{equation}
+      \varphi^2 = \varphi + 1.
+    \end{equation}
+
+into the non-equivalent
+
+    \[
+      \varphi^2 = \varphi + 1.
+    \]
+
+
+## Verbatim environments
+
+Pandoc does well with inline verbatim text like `\verb+$x$+`, but it doesn't handle nested verbatim environments properly. In particular, it chokes on
+
+    \begin{verbatim}
+      \begin{verbatim}
+        This is verbatim text.
+      \end{verbatim}
+    \end{verbatim}
+    
+## Tables
+
+Pandoc doesn't handle tables at all.
+
+## Sections with cross-references
+
+Pandoc doesn't handle cross-references.

data/polytexnic.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'polytexnic/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "polytexnic"
+  gem.version       = Polytexnic::VERSION
+  gem.authors       = ["Michael Hartl", "Nick Merwin"]
+  gem.email         = ["michael@softcover.io"]
+  gem.description   = %q{Core translation engine for the polytexnic gem}
+  gem.summary       = %q{Convert from PolyTeX & Markdown to HTML & LaTeX}
+  gem.homepage      = "https://polytexnic.org/"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'nokogiri', '~> 1.5.0'
+  gem.add_dependency 'pygments.rb', '~> 0.4.2'
+  gem.add_dependency 'msgpack', '~> 0.4.2'
+  gem.add_dependency 'kramdown'
+
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'simplecov'
+end

data/polytexnic_commands.sty

@@ -0,0 +1,5 @@
+% Don't edit this file; it is overwritten every time the PDF gets built.
+
+% Add some custom commands needed by PolyTeXnic.
+\newcommand{\PolyTeX}{Poly\-\TeX}
+\newcommand{\PolyTeXnic}{Poly\-{\TeX}\-nic}

data/spec/fixtures/code_listing.tex

@@ -0,0 +1,14 @@
+We can include code listings, as seen in Listing~\ref{code:foo}.
+
+\begin{codelisting}
+\label{code:foo}
+Defining a foo function.
+%= ruby
+\begin{code}
+def foo
+  "bar"
+end
+\end{code}
+\end{codelisting}
+
+Including the type of source code (in this case, Ruby) should result in syntax highlighting. (The type is put in a comment so that the markup is still valid \LaTeX.)

data/spec/fixtures/figures.tex

@@ -0,0 +1,8 @@
+We include here a figure (Figure~\ref{fig:editor_shell}).
+
+\begin{figure}
+\begin{center}
+\includegraphics{images/figures/editor_shell.png}
+\end{center}
+\caption{The caption includes a label for use in cross-references.\label{fig:editor_shell}}
+\end{figure}

data/spec/fixtures/inline_math.html

@@ -0,0 +1,4 @@
+
+<p>This is some inline math: <span class="inline_math">\( \int_\Omega d\omega = \int_{\partial\Omega} \omega \)</span> (the general form of Stokes’s Theorem).<span class="intersentencespace"></span> Here’s some more: <span class="inline_math">\( \varphi^2 = \varphi + 1 \)</span>.</p>
+<p>Our main strategy for producing HTML is to pass it through as-is and let MathJax handle the rendering.
+</p>

data/spec/fixtures/inline_math.tex

@@ -0,0 +1,3 @@
+This is some inline math: $\int_\Omega d\omega = \int_{\partial\Omega} \omega$ (the general form of Stokes's Theorem). Here's some more: \( \varphi^2 = \varphi + 1 \).
+
+Our main strategy for producing HTML is to pass it through as-is and let MathJax handle the rendering. 

data/spec/fixtures/math_environments.html

@@ -0,0 +1,50 @@
+<p>This is an equation environment:</p>
+<div class="equation">\[
+\int_\Omega d\omega = \int_{\partial\Omega} \omega \qquad\mbox{Stokes's Theorem}
+\]
+</div><p class="noindent">This should be a centered equation with no numbering.</p>
+<p>We also want to support equation environments:</p>
+<div id="uid1" data-tralics-id="uid1" data-number="1" class="equation">\begin{equation}
+\int_\Omega d\omega = \int_{\partial\Omega} \omega \qquad\mbox{Stokes's Theorem}
+\end{equation}
+</div><p class="noindent">We’ll probably have to handle numbering and cross-references in a post-processing step.</p>
+<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+tempor incididunt ut labore et dolore magna aliqua.<span class="intersentencespace"></span> Ut enim ad minim veniam,
+quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+consequat.<span class="intersentencespace"></span> Duis aute irure dolor in reprehenderit in voluptate velit esse
+cillum dolore eu fugiat nulla pariatur.<span class="intersentencespace"></span> Excepteur sint occaecat cupidatat non
+proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<p>Of course, inline math like <span class="inline_math">\( \Omega &gt; 0 \)</span> and <span class="inline_math">\(  x^2 - 2 \equiv 0  \)</span> should also work fine.</p>
+<div id="uid2" data-tralics-id="uid2" data-number="2" class="equation">\begin{align}
+x^2 + y^2 &amp;= 1
+\\ y &amp;= \sqrt{1 - x^2}.
+\end{align}
+</div><p class="noindent">Maxwell’s Equations:</p>
+<div id="uid3" data-tralics-id="uid3" data-number="3" class="equation">\begin{equation}
+\begin{aligned}
+\nabla \times \vec{\mathbf{B}} -\, \frac1c\, \frac{\partial\vec{\mathbf{E}}}{\partial t} &amp; = \frac{4\pi}{c}\vec{\mathbf{j}} \\   \nabla \cdot \vec{\mathbf{E}} &amp; = 4 \pi \rho \\
+\nabla \times \vec{\mathbf{E}}\, +\, \frac1c\, \frac{\partial\vec{\mathbf{B}}}{\partial t} &amp; = \vec{\mathbf{0}} \\
+\nabla \cdot \vec{\mathbf{B}} &amp; = 0
+\end{aligned}
+\end{equation}
+</div><div id="uid4" data-tralics-id="uid4" data-number="4" class="equation">\begin{equation*}
+\left.\begin{aligned}
+dE  &amp;= \rho \\
+d*E &amp;= -\dot{B} \\
+dB &amp;= 0 \\
+d*B &amp;= J + \dot{E}
+\end{aligned}
+\right\}
+\qquad \text{Maxwell's equations}
+\end{equation*}
+</div><p class="noindent">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+tempor incididunt ut labore et dolore magna aliqua.<span class="intersentencespace"></span> Ut enim ad minim veniam,
+quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+consequat.<span class="intersentencespace"></span> Duis aute irure dolor in reprehenderit in voluptate velit esse
+cillum dolore eu fugiat nulla pariatur.<span class="intersentencespace"></span> Excepteur sint occaecat cupidatat non
+proident, sunt in culpa qui officia deserunt mollit anim id est laborum.<span class="intersentencespace"></span></p>
+<div id="uid5" data-tralics-id="uid5" data-number="5" class="equation">\begin{multline}
+a+b+c+d+e+f\\
++i+j+k+l+m+n
+\end{multline}
+</div>

data/spec/fixtures/math_environments.tex

@@ -0,0 +1,56 @@
+This is an equation environment:
+\[
+\int_\Omega d\omega = \int_{\partial\Omega} \omega \qquad\mbox{Stokes's Theorem}
+\]
+This should be a centered equation with no numbering.
+
+We also want to support equation environments:
+\begin{equation}
+\int_\Omega d\omega = \int_{\partial\Omega} \omega \qquad\mbox{Stokes's Theorem}
+\end{equation}
+We'll probably have to handle numbering and cross-references in a post-processing step.
+
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+
+Of course, inline math like $\Omega > 0$ and \( x^2 - 2 \equiv 0 \) should also work fine.
+
+\begin{align}
+x^2 + y^2 &= 1
+\\ y &= \sqrt{1 - x^2}.
+\end{align}
+
+Maxwell's Equations:
+\begin{equation}
+\begin{aligned}
+\nabla \times \vec{\mathbf{B}} -\, \frac1c\, \frac{\partial\vec{\mathbf{E}}}{\partial t} & = \frac{4\pi}{c}\vec{\mathbf{j}} \\   \nabla \cdot \vec{\mathbf{E}} & = 4 \pi \rho \\
+\nabla \times \vec{\mathbf{E}}\, +\, \frac1c\, \frac{\partial\vec{\mathbf{B}}}{\partial t} & = \vec{\mathbf{0}} \\
+\nabla \cdot \vec{\mathbf{B}} & = 0
+\end{aligned}
+\end{equation}
+
+\begin{equation*}
+\left.\begin{aligned}
+dE  &= \rho \\
+d*E &= -\dot{B} \\
+dB &= 0 \\
+d*B &= J + \dot{E}
+\end{aligned}
+\right\}
+\qquad \text{Maxwell's equations}
+\end{equation*}
+
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+\begin{multline}
+a+b+c+d+e+f\\
++i+j+k+l+m+n
+\end{multline}

data/spec/fixtures/section_xrefs.tex

@@ -0,0 +1,9 @@
+\section{Foo bar}
+\label{sec:foobar}
+
+This is a section. The next section is Section~\ref{sec:bazquux}
+
+\section{Baz quux}
+\label{sec:bazquux}
+
+This is another section. It comes immediately after Section~\ref{sec:foobar}.

data/spec/fixtures/sidebar.tex

@@ -0,0 +1,10 @@
+We also support numbered, cross-referenced boxes, as defined by a custom ``sidebar'' environment (since ``box'' is already a built-in \LaTeX command). An example is shown in Box~\ref{sidebar:example_box}.
+
+\begin{sidebar}{Example box}{sidebar:example_box}
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
+tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
+quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
+consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
+cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
+proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+\end{sidebar}

data/spec/fixtures/tables.tex

@@ -0,0 +1,8 @@
+This is a table:
+
+\begin{tabular}{ l c r }
+  1 & 2 & 3 \\
+  4 & 5 & 6 \\
+  7 & 8 & 9 \\
+\end{tabular}
+

data/spec/fixtures/verbatim_environments.html

@@ -0,0 +1,11 @@
+
+<p>This is inline verbatim text: <span class="inline_verbatim">\LaTeX and a literal $</span>.<span class="intersentencespace"></span> This is non-inline verbatim text:
+</p><pre class="verbatim">  % \begin{verbatim}
+  %   This is verbatim text.
+  % \end{verbatim}</pre>
+<p>It would also be nice to support the environment from the Verbatim package:
+</p><pre class="verbatim">  % \begin{Verbatim}
+  %   This is an even more powerful verbatim environment.
+  % \end{Verbatim}</pre>
+<p>We’ll see how it goes.
+</p>

data/spec/fixtures/verbatim_environments.tex

@@ -0,0 +1,13 @@
+This is inline verbatim text: \verb+\LaTeX and a literal $+. This is non-inline verbatim text:
+\begin{verbatim}
+  % \begin{verbatim}
+  %   This is verbatim text.
+  % \end{verbatim}
+\end{verbatim}
+It would also be nice to support the environment from the Verbatim package:
+\begin{Verbatim}
+  % \begin{Verbatim}
+  %   This is an even more powerful verbatim environment.
+  % \end{Verbatim}
+\end{Verbatim}
+We'll see how it goes.

data/spec/integration_spec.rb

@@ -0,0 +1,34 @@
+# encoding=utf-8
+require 'spec_helper'
+
+# Returns a list of fixture filenames.
+def filenames
+  %w[inline_math verbatim_environments math_environments]
+end
+
+# Returns the results of converting the TeX filename to HTML.
+def converted(filename)
+  Polytexnic::Pipeline.new(contents(filename, 'tex')).to_html
+end
+
+# Returns the contents of the HTML filename fixture.
+def html(filename)
+  contents(filename, 'html')
+end
+
+# Returns the contents of a filename using its extension.
+def contents(filename, extension)
+  File.open(File.join('spec', 'fixtures', "#{filename}.#{extension}")).read
+end
+
+describe Polytexnic::Pipeline do
+
+  filenames.each do |filename|
+    it "should correctly process #{filename}" do
+      tmp = 'tmp'
+      File.mkdir(tmp) unless File.directory?(tmp)
+      File.write(File.join(tmp, filename), converted(filename))
+      expect(converted(filename)).to resemble html(filename)
+    end
+  end
+end