coradoc 0.2.0 → 1.0.0

data/lib/coradoc/parser/asciidoc/content.rb

@@ -50,13 +50,13 @@ module Coradoc
           block_style("=")
         end
 
-        def block_style(delimiter="*", repeater = 4)
+        def block_style(delimiter = "*", repeater = 4)
           block_title.maybe >>
-          newline.maybe >>
-          block_type.maybe >>
-          str(delimiter).repeat(repeater).as(:delimiter) >> newline >>
-          text_line.repeat(1).as(:lines) >>
-          str(delimiter).repeat(repeater) >> newline
+            newline.maybe >>
+            block_type.maybe >>
+            str(delimiter).repeat(repeater).as(:delimiter) >> newline >>
+            text_line.repeat(1).as(:lines) >>
+            str(delimiter).repeat(repeater) >> newline
         end
 
         def block_type
@@ -65,7 +65,7 @@ module Coradoc
 
         def highlight
           text_id >> newline >>
-          underline >> highlight_text >> newline
+            underline >> highlight_text >> newline
         end
 
         def underline
@@ -79,14 +79,14 @@ module Coradoc
         # Table
         def table
           block_title >>
-          str("|===") >> line_ending >>
-          table_row.repeat(1).as(:rows) >>
-          str("|===") >> line_ending
+            str("|===") >> line_ending >>
+            table_row.repeat(1).as(:rows) >>
+            str("|===") >> line_ending
         end
 
         def table_row
-          (literal_space? >> str("|") >> (cell_content | empty_cell_content)).
-            repeat(1).as(:cols) >> line_ending
+          (literal_space? >> str("|") >> (cell_content | empty_cell_content))
+            .repeat(1).as(:cols) >> line_ending
         end
 
         def empty_cell_content
@@ -98,7 +98,7 @@ module Coradoc
         end
 
         def literal_space
-          (match[' '] | match[' \t']).repeat(1)
+          (match[" "] | match[' \t']).repeat(1)
         end
 
         # Override
@@ -113,7 +113,7 @@ module Coradoc
         # Text
         def text_line
           (asciidoc_char_with_id.absent? | text_id) >> literal_space? >>
-          text.as(:text) >> line_ending.as(:break)
+            text.as(:text) >> line_ending.as(:break)
         end
 
         def asciidoc_char
@@ -137,7 +137,7 @@ module Coradoc
 
         def glossary
           keyword.as(:key) >> str("::") >> space? >>
-          text.as(:value) >> line_ending.as(:break)
+            text.as(:value) >> line_ending.as(:break)
         end
 
         def ordered_list

data/lib/coradoc/parser/asciidoc/document_attributes.rb

@@ -11,7 +11,7 @@ module Coradoc
 
         def document_attributes
           str(":") >> attribute_name.as(:key) >> str(":") >>
-          space? >> attribute_value.as(:value) >> line_ending
+            space? >> attribute_value.as(:value) >> line_ending
         end
       end
     end

data/lib/coradoc/parser/asciidoc/header.rb

@@ -8,8 +8,8 @@ module Coradoc
 
         def header
           header_title >>
-          author.maybe.as(:author) >>
-          revision.maybe.as(:revision) >> newline.maybe
+            author.maybe.as(:author) >>
+            revision.maybe.as(:revision) >> newline.maybe
         end
 
         def header_title
@@ -18,14 +18,14 @@ module Coradoc
 
         def author
           words.as(:first_name) >> str(",") >>
-          space? >> words.as(:last_name) >>
-          space? >> str("<") >> email.as(:email) >> str(">") >> newline
+            space? >> words.as(:last_name) >>
+            space? >> str("<") >> email.as(:email) >> str(">") >> newline
         end
 
         def revision
           (word >> (str(".") >> word).maybe).as(:number) >>
-          str(",") >> space? >> date.as(:date ) >> str(":") >>
-          space? >> words.as(:remark) >> newline
+            str(",") >> space? >> date.as(:date) >> str(":") >>
+            space? >> words.as(:remark) >> newline
         end
       end
     end

data/lib/coradoc/parser/asciidoc/section.rb

@@ -23,7 +23,7 @@ module Coradoc
         # Heading
         def section_title(level = 2, max_level = 8)
           match("=").repeat(level, max_level).as(:level) >>
-          space? >> text.as(:text) >> endline.as(:break)
+            space? >> text.as(:text) >> endline.as(:break)
         end
 
         # section

data/lib/coradoc/reverse_adoc/LICENSE.txt

@@ -0,0 +1,25 @@
+BSD 2-Clause License
+
+Copyright (c) 2018, Ribose
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/lib/coradoc/reverse_adoc/README.adoc

@@ -0,0 +1,308 @@
+= AsciiDoc from HTML and Microsoft Word: reverse_adoc
+
+== Purpose
+
+Transforms HTML and Microsoft Word into AsciiDoc.
+
+Based on https://github.com/xijo/reverse_markdown
+
+reverse_adoc used to be a separate Gem, but now it's part of Coradoc.
+
+
+== Installation
+
+Install the gem:
+
+[source,console]
+----
+[sudo] gem install coradoc
+----
+
+or add it to your `Gemfile`:
+
+[source,ruby]
+----
+gem 'coradoc'
+----
+
+
+== Command-line usage
+
+=== HTML to AsciiDoc: `reverse_adoc`
+
+Convert HTML files to AsciiDoc:
+
+[source,console]
+----
+$ reverse_adoc file.html > file.adoc
+$ cat file.html | reverse_adoc > file.adoc
+----
+
+
+=== Microsoft Word to AsciiDoc: `w2a`
+
+Convert Word `.doc` or `.docx` files to AsciiDoc:
+
+[source,console]
+----
+$ w2a file.docx > file.adoc
+----
+
+[source,console]
+----
+$ w2a input.docx -o output.adoc
+----
+
+Help:
+
+[source,console]
+----
+$ w2a -h
+Usage: w2a [options] <file>
+    -a, --mathml2asciimath           Convert MathML to AsciiMath
+    -o, --output=FILENAME            Output file to write to
+    -e, --external-images            Export images if data URI
+    -v, --version                    Version information
+    -h, --help                       Prints this help
+----
+
+
+NOTE: `w2a` requires LibreOffice to be installed. It uses LibreOffice's
+export to XHTML. LibreOffice's export of XHTML is superior to the native Microsoft Word export
+to HTML: it exports lists (which Word keeps as paragraphs), and it exports OOMML into MathML.
+On the other hand, the LibreOffice export relies on default styling being used in the
+document, and it may not cope with ordered lists or headings with customised appearance.
+For best results, reset the styles in the document you're converting to those in
+the default `Normal.dot` template.
+
+NOTE: `w2a` requires the command-line version of LibreOffice, `soffice`. As it turns out,
+LibreOffice v6 appears to render formulae in HTML as images instead of MathML expressions;
+use LibreOffice v5. If you have both LibreOffice v5 and LibreOffice v6 installed, make sure
+that your OS path searches for the LibreOffice v5 version of `soffice` first; e.g. on Mac,
+include something like `/Applications/LibreOffice5.4.7.2.app/Contents/MacOS` in your PATH
+environment.
+
+NOTE: Some information in OOMML is not preserved in the export to MathML from LibreOffice;
+in particular, font shifts such as double-struck fonts.
+The LibreOffice exporter does seem to drop some text (possibly associated with
+MathML); use with caution.
+
+NOTE: Adapted from `w2m` of
+https://github.com/benbalter/word-to-markdown[Ben Balter's word-to-markdown]
+
+
+=== Common options
+
+
+==== MathML to AsciiMath conversion
+
+If you wish to convert the MathML in the document to AsciiMath, run the script with the
+`--mathml2asciimath` option:
+
+[source,console]
+----
+$ w2a --mathml2asciimath document.docx > document.adoc
+----
+
+
+==== Extracting images
+
+Images referred by the HTML can be extracted into the destination output folder by using:
+
+[source,console]
+----
+$ reverse_adoc input.docx -o output/file.adoc -e
+$ reverse_adoc input.docx --output output/file.adoc --external-images
+----
+
+
+Word embedded images can be extracted into the destination output folder by using:
+
+[source,console]
+----
+$ w2a input.docx -o output/file.adoc -e
+$ w2a input.docx --output output/file.adoc --external-images
+----
+
+
+==== Handling unknown HTML tags
+
+The `--unknown_tags` option allows you to specify how to handle unknown tags
+(default `pass_through`).
+
+Valid options are:
+
+* `pass_through` - Include the unknown tag completely into the result
+* `drop` - Drop the unknown tag and its content
+* `bypass` - Ignore the unknown tag but try to convert its content
+* `raise` - Raise an error to let you know
+
+
+==== Tagging of borders
+
+Specify how to handle tag borders with the option `--tag_border` (default `' '`).
+
+Valid options are:
+
+* `' '` - Add whitespace if there is none at tag borders.
+* `''` - Do not not add whitespace.
+
+
+== Features
+
+=== General
+
+`reverse_adoc` shares features as a port of `reverse_markdown`:
+
+* Module based -- if you miss a tag, just add it
+* Can deal with nested lists
+* Inline and block code is supported
+* Supports blockquote
+
+It supports the following HTML tags (these are supported by `reverse_markdown`):
+
+* `a`
+* `blockquote`
+* `br`
+* `code`, `tt` (added: `kbd`, `samp`, `var`)
+* `div`, `article`
+* `em`, `i` (added: `cite`)
+* `h1`, `h2`, `h3`, `h4`, `h5`, `h6`, `hr`
+* `img`
+* `li`, `ol`, `ul` (added: `dir`)
+* `p`, `pre`
+* `strong`, `b`
+* `table`, `td`, `th`, `tr`
+
+[NOTE]
+====
+* reverse_adoc does *not* support `del` or `strike`, because Asciidoctor does not out of the box.
+* As with reverse_markdown, `pre` is only treated as sourcecode if it is contained in a `div@class = highlight-` element, or has a `@brush` attribute naming the language (Confluence).
+* The gem does not support `p@align`, because Asciidoctor doesn't
+====
+
+In addition, it supports:
+
+* `aside`
+* `audio`, `video` (with `@src` attributes)
+* `figure`, `figcaption`
+* `mark`
+* `q`
+* `sub`, `sup`
+* `@id` anchors
+* `blockquote@cite`
+* `img/@width`, `img/@height`
+* `ol/@style`, `ol/@start`, `ol/@reversed`, `ul/@type`
+* `td/@colspan`, `td/@rowspan`, `td@/align`, `td@/valign`
+* `table/caption`, `table/@width`, `table/@frame` (partial), `table/@rules` (partial)
+* Lists and paragraphs within cells
+** Not tables within cells: Asciidoctor cannot deal with nested tables
+
+The gem does not support:
+
+* `col`, `colgroup`
+* `source`, `picture`
+* `bdi`, `bdo`, `ruby`, `rt`, `rp`, `wbr`
+* `frame`, `frameset`, `iframe`, `noframes`, `noscript`, `script`, `input`, `output`, `progress`
+* `map`, `canvas`, `dialog`, `embed`, `object`, `param`, `svg`, `track`
+* `fieldset`, `button`, `datalist`, `form`, `label`, `legend`, `menu`, `menulist`, `optgroup`, `option`, `select`, `textarea`
+* `big`, `dfn`, `font`, `s`, `small`, `span`, `strike`, `u`
+* `center`
+* `data`, `meter`
+* `del`, `ins`
+* `footer`, `header`, `main`, `nav`, `details`, `section`, `summary`, `template`
+
+
+=== MathML support
+
+If you are using this gem in the context of https://www.metanorma.com[Metanorma],
+Metanorma AsciiDoc accepts MathML as a native mathematical format. So you do not need
+to convert the MathML to AsciiMath.
+
+The gem will optionally invoke the https://github.com/metanorma/mathml2asciimath
+gem, to convert MathML to AsciiMath. The conversion is not perfect, and will need to be
+post-edited; but it's a lot better than nothing.
+
+NOTE: Asciidoctor does not support MathML input. HTML uses MathML.
+The gem will recognize MathML expressions in HTML, and will wrap them in Asciidoctor
+`stem:[ ]` macros. The result of this gem is not actually legal Asciidoctor for `stem`:
+Asciidoctor will presumably
+think this is AsciiMath in the `stem:[ ]` macro, try to pass it into MathJax as
+AsciiMath, and fail. But of course, MathJax has no problem with MathML, and some postprocessing
+on the Asciidoctor output can ensure that the MathML is treated by MathJax (or whatever else
+uses the output) as such; so this is still much better than nothing for stem processing.
+
+=== Word cleanup
+
+This gem is routinely used in the Metanorma project to export Word documents to AsciiDoc.
+The HTML export from Word that the gem uses, from LibreOffice, is much cleaner than the
+native HTML 4 export from Word; but it has some infelicities which this gem cleans up:
+
+* The HTML export has trouble with subscripts, and routinely exports them as headings; the `w2a`
+script tries to clean them up.
+* The `w2a` cleans up spaces, but it does not strip them.
+* Spaces are removed from anchors and cross-references.
+* Double underscores are removed from anchors and cross-references.
+* Cross-references to `_GoBack` and to `_Toc` followed by numbers (used to construct tables of contents) are ignored.
+
+== Ruby library usage
+
+=== General
+
+Simple to use.
+
+[source,ruby]
+----
+require 'coradoc/reverse_adoc'
+
+result = Coradoc::ReverseAdoc.convert input
+result.inspect # " *feelings* "
+----
+
+=== Configure with options
+
+Just pass your chosen configuration options in after the input. The given options will last for this operation only.
+
+[source,ruby]
+----
+require 'coradoc/reverse_adoc'
+
+Coradoc::ReverseAdoc.convert(input, unknown_tags: :raise, mathml2asciimath: true)
+----
+
+
+=== Preconfigure using an initializer
+
+Or configure it block style on a initializer level. These configurations will last for all conversions until they are set to something different.
+
+[source,ruby]
+----
+require 'coradoc/reverse_adoc'
+
+Coradoc::ReverseAdoc.config do |config|
+  config.unknown_tags      = :bypass
+  config.mathml2asciimath  = true
+  config.tag_border  = ''
+end
+----
+
+=== Convert HTML to a Coradoc AST
+
+[source,ruby]
+----
+require 'coradoc/reverse_adoc'
+
+# Options can be supplied as keyword arguments
+Coradoc::ReverseAdoc::HtmlConverter.to_coradoc("<b><i>Some input</i></b>")
+----
+
+
+== Related stuff
+
+* https://github.com/xijo/reverse_markdown[Xijo's original reverse_markdown gem]
+* https://github.com/xijo/reverse_markdown/wiki/Write-your-own-converter[Write custom converters] - Wiki entry about how to write your own converter
+* https://github.com/harlantwood/html_massage[html_massage] - A gem by Harlan T. Wood to convert regular sites into markdown using reverse_markdown
+* https://github.com/benbalter/word-to-markdown[word-to-markdown] - Convert word docs into markdown while using reverse_markdown, by Ben Balter
+* https://github.com/asciidocfx/HtmlToAsciidoc[HtmlToAsciidoc] - Javascript regexp-based converter of HTML to Asciidoctor
+* https://asciidoctor.org/docs/user-manual/[The Asciidoctor User Manual]
+

data/lib/coradoc/reverse_adoc/cleaner.rb

@@ -0,0 +1,125 @@
+module Coradoc::ReverseAdoc
+  class Cleaner
+    def tidy(string)
+      result = HtmlConverter.track_time "Removing inner whitespace" do
+        remove_inner_whitespaces(String.new(string))
+      end
+      result = HtmlConverter.track_time "Removing newlines" do
+        remove_newlines(result)
+      end
+      result = HtmlConverter.track_time "Removing leading newlines" do
+        remove_leading_newlines(result)
+      end
+      result = HtmlConverter.track_time "Cleaning tag borders" do
+        clean_tag_borders(result)
+      end
+      result = HtmlConverter.track_time "Cleaning punctuation characters" do
+        clean_punctuation_characters(result)
+      end
+    end
+
+    def remove_newlines(string)
+      string.gsub(/\n{3,}/, "\n\n")
+    end
+
+    def remove_leading_newlines(string)
+      string.gsub(/\A\n+/, "")
+    end
+
+    def remove_inner_whitespaces(string)
+      unless string.nil?
+        string.gsub!(/\n stem:\[/, "\nstem:[")
+        string.gsub!(/(stem:\[([^\]]|\\\])*\])\n(?=\S)/, "\\1 ")
+        string.gsub!(/(stem:\[([^\]]|\\\])*\])\s+(?=[\^-])/, "\\1")
+      end
+      result = +""
+      string.each_line do |line|
+        result << preserve_border_whitespaces(line) do
+          line.strip.gsub(/[ \t]{2,}/, " ")
+        end
+      end
+      result
+    end
+
+    # Find non-asterisk content that is enclosed by two or
+    # more asterisks. Ensure that only one whitespace occurs
+    # in the border area.
+    # Same for underscores and brackets.
+    def clean_tag_borders(string)
+      # result = string.gsub(/\s?\*{2,}.*?\*{2,}\s?/) do |match|
+      # preserve_border_whitespaces(match, default_border: Coradoc::ReverseAdoc.config.tag_border) do
+      #   match.strip.sub("** ", "**").sub(" **", "**")
+      # end
+      # end
+
+      # result = string.gsub(/\s?_{2,}.*?_{2,}\s?/) do |match|
+      #   preserve_border_whitespaces(match, default_border: Coradoc::ReverseAdoc.config.tag_border) do
+      #     match.strip.sub("__ ", "__").sub(" __", "__")
+      #   end
+      # end
+
+      result = string.gsub(/\s?~{2,}.*?~{2,}\s?/) do |match|
+        preserve_border_whitespaces(match,
+                                    default_border: Coradoc::ReverseAdoc.config.tag_border) do
+          match.strip.sub("~~ ", "~~").sub(" ~~", "~~")
+        end
+      end
+
+      result.gsub(/\s?\[.*?\]\s?/) do |match|
+        preserve_border_whitespaces(match) do
+          match.strip.sub("[ ", "[").sub(" ]", "]")
+        end
+      end
+    end
+
+    def clean_punctuation_characters(string)
+      string.gsub(/(\*\*|~~|__)\s([.!?'"])/, "\\1\\2")
+    end
+
+    # preprocesses HTML, rather than postprocessing it
+    def preprocess_word_html(string)
+      clean_headings(scrub_whitespace(string.dup))
+    end
+
+    def scrub_whitespace(string)
+      string.gsub!(/&nbsp;|&#xA0;|\u00a0/i, "&#xA0;") # HTML encoded spaces
+      string.sub!(/^\A[[:space:]]+/m, "") # document leading whitespace
+      string.sub!(/[[:space:]]+\z$/m, "") # document trailing whitespace
+      string.gsub!(/( +)$/, " ") # line trailing whitespace
+      string.gsub!(/\n\n\n\n/, "\n\n") # Quadruple line breaks
+      # string.delete!('?| ')               # Unicode non-breaking spaces, injected as tabs
+      string
+    end
+
+    # following added by me
+    def clean_headings(string)
+      string.gsub!(%r{<h([1-9])[^>]*></h\1>}, " ")
+      # I don't know why Libre Office is inserting them, but they need to go
+      string.gsub!(%r{<h([1-9])[^>]* style="vertical-align: super;[^>]*>(.+?)</h\1>},
+                   "<sup>\\2</sup>")
+      # I absolutely don't know why Libre Office is rendering superscripts as h1
+      string
+    end
+
+    private
+
+    def preserve_border_whitespaces(string, options = {})
+      return string if /\A\s*\Z/.match?(string)
+
+      default_border = options.fetch(:default_border, "")
+      # If the string contains part of a link so the characters [,],(,)
+      # then don't add any extra spaces
+      default_border = "" if /[\[(\])]/.match?(string)
+      string_start   = present_or_default(string[/\A\s*/], default_border)
+      string_end     = present_or_default(string[/\s*\Z/], default_border)
+      result         = yield
+      string_start + result + string_end
+    end
+
+    def present_or_default(string, default)
+      return default if string.nil? || string.empty?
+
+      string
+    end
+  end
+end

data/lib/coradoc/reverse_adoc/config.rb

@@ -0,0 +1,73 @@
+require "tmpdir"
+
+module Coradoc::ReverseAdoc
+  class Config
+    def initialize
+      @unknown_tags     = :pass_through
+      @input_format     = :html
+      @mathml2asciimath = false
+      @external_images  = false
+
+      # Destination to save file and images
+      @destination      = nil
+
+      # Source of HTML
+      # @sourcedir        = nil
+
+      # Image counter, assuming there are max 999 images
+      @image_counter = 1
+      # pad with 0s
+      @image_counter_pattern = "%03d"
+
+      @em_delimiter     = "_".freeze
+      @strong_delimiter = "*".freeze
+      @inline_options   = {}
+      @tag_border       = " ".freeze
+
+      @split_sections   = nil
+
+      # Document width - used to compute table sizes.
+      # This is an assumption for screen size in input document.
+      # If column widths are specified in absolute values, then we
+      # have to convert them to relative values, as AsciiDoc only
+      # supports those.
+      @doc_width        = 1000
+
+      # Plugin system
+      @plugins          = []
+
+      # Debugging options
+      @track_time       = false
+    end
+
+    def with(options = {})
+      old_options = @inline_options
+      @inline_options = options
+      result = yield
+      @inline_options = old_options
+      result
+    end
+
+    def self.declare_option(option)
+      define_method(option) do
+        @inline_options[option] || instance_variable_get(:"@#{option}")
+      end
+
+      attr_writer option
+    end
+
+    declare_option :unknown_tags
+    declare_option :tag_border
+    declare_option :mathml2asciimath
+    declare_option :external_images
+    declare_option :destination
+    declare_option :sourcedir
+    declare_option :image_counter
+    declare_option :image_counter_pattern
+    declare_option :input_format
+    declare_option :split_sections
+    declare_option :doc_width
+    declare_option :plugins
+    declare_option :track_time
+  end
+end