coradoc 1.0.0 → 1.1.1

data/lib/coradoc/{reverse_adoc → input/html}/plugins/plateau.rb

@@ -1,10 +1,10 @@
-module Coradoc::ReverseAdoc
+module Coradoc::Input::HTML
   class Plugin
     # This plugin enhances documents from the PLATEAU project
     # to extract more data.
     #
     # Usage:
-    # reverse_adoc -rcoradoc/reverse_adoc/plugins/plateau
+    # reverse_adoc -rcoradoc/input/html/plugins/plateau
     #   --external-images -u raise --output _out/index.adoc index.html
     class Plateau < Plugin
       def name
@@ -30,12 +30,20 @@ module Coradoc::ReverseAdoc
         html_tree_change_tag_name_by_css(".pitemdata", "h3")
         html_tree_change_tag_name_by_css(".sitemdata", "h4")
         html_tree_change_tag_name_by_css('td[bgcolor="#D0CECE"]', "th")
+        html_tree_change_tag_name_by_css('td[bgcolor="#d0cece"]', "th")
+        html_tree_change_tag_name_by_css('.framedata, .frame_container_box', 'aside')
+        html_tree_change_tag_name_by_css('.frame2data', 'pre')
+        # Assumption that all code snippets in those documents are XML...
+        html_tree_change_properties_by_css(".frame2data", class: "brush:xml;")
 
         # Remove some CSS ids that are not important to us
         html_tree_change_properties_by_css("#__nuxt", id: nil)
         html_tree_change_properties_by_css("#__layout", id: nil)
         html_tree_change_properties_by_css("#app", id: nil)
 
+        # Handle lists of document 02
+        html_tree_replace_with_children_by_css(".list_num-wrap")
+
         # Convert table/img caption to become a caption
         html_tree.css(".imagedata").each do |e|
           table = e.parent.next&.children&.first
@@ -75,66 +83,82 @@ module Coradoc::ReverseAdoc
           end
         end
 
-        html_tree_add_hook_pre_by_css ".text3data" do |node,|
-          text = html_tree_process_to_adoc(node).strip
-          next "" if text.empty? || text == "\u3000"
-
-          text = text.strip.gsub(/^/, "*** ")
-          "\n\n//-PT3D\n#{text}\n//-ENDPT3D\n\n"
-        end
-
-        html_tree_add_hook_pre_by_css ".text4data" do |node,|
-          text = html_tree_process_to_adoc(node).strip
-          next "" if text.empty? || text == "\u3000"
+        (3..4).each do |i|
+          html_tree_add_hook_pre_by_css ".text#{i}data" do |node,|
+            text = html_tree_process_to_adoc(node).strip
+            next "" if text.empty? || text == "\u3000"
 
-          text = text.strip.gsub(/^/, "**** ")
-          "\n\n//-PT4D\n#{text}\n//-ENDPT4D\n\n"
+            text = text.strip.gsub(/^/, "#{'*' * i} ")
+            "\n\n//-PT#{i}D\n#{text}\n//-ENDPT#{i}D\n\n"
+          end
         end
 
-        html_tree_add_hook_pre_by_css ".text2data_point ul" do |node,|
-          text = html_tree_process_to_adoc(node.children.first.children).strip
+        (2..3).each do |i|
+          html_tree_add_hook_pre_by_css ".text#{i}data_point ul" do |node,|
+            text = html_tree_process_to_adoc(node.children.first.children).strip
 
-          "** #{text}\n"
+            "#{'*' * i} #{text}\n"
+          end
         end
 
-        html_tree_add_hook_pre_by_css ".text3data_point ul" do |node,|
-          text = html_tree_process_to_adoc(node.children.first.children).strip
+        (1..20).each do |i|
+          html_tree_add_hook_pre_by_css ".numtextdata_num .list_num#{i}" do |node,|
+            text = html_tree_process_to_adoc(node).strip
 
-          "*** #{text}\n"
+            "[start=#{i}]\n. #{text}\n"
+          end
         end
 
         # html_tree_preview
       end
 
+      IM = /[A-Z0-9]{1,3}/
+
       def handle_headers(node, coradoc, state)
-        if coradoc.id.start_with?("toc0_")
-          content = coradoc.content.map(&:content).join
+        content = coradoc.content.map(&:content).join
+
+        if %w[toc0 toc_0].any? { |i| coradoc.id&.start_with?(i) }
           # Special content
           case content.strip
           when "はじめに" # Introduction
             coradoc.style = "abstract" # The older version document has ".preface"
+            coradoc.level_int = 1
           when "改定の概要" # Revision overview
             coradoc.style = "abstract" # The older version document has ".preface"
+            coradoc.level_int = 1
           when "参考文献" # Bibliography
             coradoc.style = "bibliography"
+            coradoc.level_int = 1
           when "改訂履歴" # Document history
             coradoc.style = "appendix"
+            coradoc.level_int = 1
+          when "0　概要" # Overview
+            coradoc.style = "abstract" # I'm not sure this is correct
+            coradoc.level_int = 1
+          when "索引" # Index
+            coradoc.style = "index" # I'm not sure this is correct
+            coradoc.level_int = 1
           else
-            warn "Unknown section #{coradoc.content.map(&:content).join.inspect}"
+            warn "Unknown section #{content.inspect}"
           end
+        end
 
-          # Ensure they are generated as level 1
-          coradoc.level_int = 1
+        if node.name == "h1"
+          if content.start_with?("Annex")
+            coradoc.style = "appendix"
+            coradoc.content.first.content.sub!(/\AAnnex [A-Z]/, "")
+          end
         end
 
         # Remove numbers
-        coradoc.content.first.content.sub!(/\A[\d\s.]+/, "")
+        coradoc.content.first.content.sub!(/\A(#{IM}\.)*#{IM}[[:space:]]/, "")
 
         coradoc
       end
 
       def handle_headers_h4(node, coradoc, state)
-        case coradoc.content.first.content
+        title = Coradoc.strip_unicode(coradoc.content.first.content)
+        case title
         when /\A\(\d+\)(.*)/
           coradoc.level_int = 4
           coradoc.content.first.content = $1.strip
@@ -143,8 +167,16 @@ module Coradoc::ReverseAdoc
           coradoc.level_int = 5
           coradoc.content.first.content = $1.strip
           coradoc
+        when /\A#{IM}\.#{IM}\.#{IM}\.#{IM}(.*)/
+          coradoc.level_int = 4
+          coradoc.content.first.content = $1.strip
         else
-          ["// FIXME\n", coradoc]
+          if title.empty?
+            # Strip instances of faulty empty paragraphs
+            nil
+          else
+            ["// FIXME\n", coradoc]
+          end
         end
       end
 
@@ -171,4 +203,4 @@ module Coradoc::ReverseAdoc
   end
 end
 
-Coradoc::ReverseAdoc.config.plugins << Coradoc::ReverseAdoc::Plugin::Plateau
+Coradoc::Input::HTML.config.plugins << Coradoc::Input::HTML::Plugin::Plateau

data/lib/coradoc/{reverse_adoc → input/html}/postprocessor.rb

@@ -1,4 +1,4 @@
-module Coradoc::ReverseAdoc
+module Coradoc::Input::HTML
   # Postprocessor's aim is to convert a Coradoc tree from
   # a mess that has been created from HTML into a tree that
   # is compatible with what we would get out of Coradoc, if
@@ -73,7 +73,7 @@ module Coradoc::ReverseAdoc
     end
 
     def split_sections
-      max_level = Coradoc::ReverseAdoc.config.split_sections
+      max_level = Coradoc::Input::HTML.config.split_sections
 
       return unless max_level
 
@@ -82,13 +82,18 @@ module Coradoc::ReverseAdoc
       previous_sections = {}
 
       determine_section_id = ->(elem) do
-        level = 0
-        section = elem
+        if elem.title.style == "appendix"
+          level = "A"
+        else
+          level = 1
+        end
+
+        section = previous_sections[elem]
         while section
-          level += 1 if elem.title.style == section.title.style
+          level = level.succ if elem.title.style == section.title.style
           section = previous_sections[section]
         end
-        level
+        level.is_a?(Integer) ? "%02d" % level : level
       end
 
       determine_style = ->(elem) do
@@ -106,7 +111,7 @@ module Coradoc::ReverseAdoc
             # we can compute numbers
             previous_sections[elem] = parent_sections[title.level_int]
             parent_sections[title.level_int] = elem
-            parent_sections[(title.level_int+1)..nil] = nil
+            parent_sections[(title.level_int + 1)..nil] = nil
 
             elem
           else
@@ -114,8 +119,7 @@ module Coradoc::ReverseAdoc
             # include tag.
             section_file = "sections/"
             section_file += parent_sections[1..title.level_int].map do |parent|
-              style = determine_style.(parent)
-              "%s%02d" % [style, determine_section_id.(parent)]
+              determine_style.(parent) + determine_section_id.(parent)
             end.join("/")
             section_file += ".adoc"
 

data/lib/coradoc/input/html.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "digest"
+require "nokogiri"
+require "coradoc/input"
+require_relative "html/errors"
+require_relative "html/cleaner"
+require_relative "html/config"
+require_relative "html/converters"
+require_relative "html/converters/base"
+require_relative "html/html_converter"
+require_relative "html/plugin"
+require_relative "html/postprocessor"
+
+module Coradoc
+  module Input::HTML
+    def self.convert(input, options = {})
+      Coradoc::Input::HTML::HtmlConverter.convert(input, options)
+    end
+
+    def self.to_coradoc(input, options = {})
+      Input::HTML::HtmlConverter.to_coradoc(input, options)
+    end
+
+    def self.config
+      @config ||= Config.new
+      yield @config if block_given?
+      @config
+    end
+
+    def self.cleaner
+      @cleaner ||= Cleaner.new
+    end
+
+    def self.processor_id
+      :html
+    end
+
+    def self.processor_match?(filename)
+      %w[.html .htm].any? { |i| filename.downcase.end_with?(i) }
+    end
+
+    def self.processor_execute(input, options = {})
+      to_coradoc(input, options)
+    end
+
+    def self.processor_postprocess(data, options)
+      if options[:output_processor] == :adoc
+        data.transform_values do |v|
+          Input::HTML::HtmlConverter.cleanup_result(v, options)
+        end
+      else
+        data
+      end
+    end
+
+    Coradoc::Input.define(self)
+  end
+end

data/lib/coradoc/input.rb

@@ -0,0 +1,12 @@
+require "coradoc"
+
+module Coradoc
+  module Input
+    @processors = {}
+    extend Converter::CommonInputOutputMethods
+  end
+end
+
+require "coradoc/input/adoc"
+require "coradoc/input/docx"
+require "coradoc/input/html"

data/lib/coradoc/output/adoc.rb

@@ -0,0 +1,17 @@
+module Coradoc
+  module Output::Adoc
+    def self.processor_id
+      :adoc
+    end
+
+    def self.processor_match?(filename)
+      %w[.adoc].any? { |i| filename.downcase.end_with?(i) }
+    end
+
+    def self.processor_execute(input, _options = {})
+      input.transform_values { |i| Coradoc::Generator.gen_adoc(i) }
+    end
+
+    Coradoc::Output.define(self)
+  end
+end

data/lib/coradoc/output/coradoc_tree_debug.rb

@@ -0,0 +1,19 @@
+module Coradoc
+  module Output::CoradocTreeDebug
+    def self.processor_id
+      :coradoc_tree_debug
+    end
+
+    def self.processor_match?(filename)
+      false
+    end
+
+    def self.processor_execute(input, _options = {})
+      out = StringIO.new
+      PP.pp(input, out)
+      { nil => out.string }
+    end
+
+    Coradoc::Output.define(self)
+  end
+end

data/lib/coradoc/output.rb

@@ -0,0 +1,11 @@
+require "coradoc"
+
+module Coradoc
+  module Output
+    @processors = {}
+    extend Converter::CommonInputOutputMethods
+  end
+end
+
+require "coradoc/output/adoc"
+require "coradoc/output/coradoc_tree_debug"

data/lib/coradoc/parser/asciidoc/admonition.rb

@@ -0,0 +1,24 @@
+module Coradoc
+  module Parser
+    module Asciidoc
+      module Admonition
+        def admonition_type
+          str('NOTE') | str('TIP') | str('EDITOR') |
+          str('IMPORTANT') | str('WARNING') | str('CAUTION') |
+          str('TODO') 
+          # requires atypical syntax for access?
+          # | str('DANGER')
+          # | str('SAFETY PRECAUTION')
+        end
+        def admonition_line
+          admonition_type.as(:admonition_type) >> str(': ') >>
+          (
+            (text.as(:text) >>
+            line_ending.as(:line_break)
+            ).repeat(1)
+            ).as(:content)
+        end
+      end
+    end
+  end
+end

data/lib/coradoc/parser/asciidoc/attribute_list.rb

@@ -0,0 +1,67 @@
+module Coradoc
+  module Parser
+    module Asciidoc
+      module AttributeList
+
+        def named_attribute_name
+          attribute_name
+        end
+
+        def named_attribute_value
+          match('[^\],]').repeat(1)
+        end
+
+        def named_attribute
+          (match('[a-zA-Z0-9_-]').repeat(1).as(:named_key) >>
+            str(' ').maybe >> str("=") >> str(' ').maybe >>
+            match['a-zA-Z0-9_\- \"'].repeat(1).as(:named_value) >>
+            str(' ').maybe
+            ).as(:named)
+        end
+
+        def positional_attribute
+          (match['a-zA-Z0-9_\-%'].repeat(1) >>
+            str("=").absent?
+            ).as(:positional)
+        end
+
+        def named_many
+          (named_attribute.repeat(1,1) >>
+              (str(",") >> space.maybe >> named_attribute).repeat(0))
+        end
+
+        def positional_one_named_many
+          (positional_attribute.repeat(1,1) >>
+            (str(",") >> space.maybe >> named_attribute).repeat(1))
+        end
+
+        def positional_many_named_many
+          (positional_attribute.repeat(1,1) >>
+            (str(",") >> space.maybe >> positional_attribute).repeat(1) >>
+            (str(",") >> space.maybe>> named_attribute).repeat(1))
+        end
+
+        def positional_many
+          (positional_attribute.repeat(1,1) >>
+            (str(",") >> space.maybe >> positional_attribute).repeat(0))
+        end
+
+        def positional_zero_or_one
+          positional_attribute.repeat(0,1)
+        end
+
+        def attribute_list(name = :attribute_list)
+          str('[') >> str("[").absent? >> 
+          ( named_many |
+            positional_one_named_many |
+            positional_many_named_many |
+            positional_many |
+            positional_zero_or_one
+          ).as(:attribute_array).as(name) >>
+          str("]")
+        end
+
+      end
+    end
+  end
+end

data/lib/coradoc/parser/asciidoc/base.rb

@@ -1,13 +1,43 @@
+require_relative "admonition"
+require_relative "attribute_list"
+require_relative "bibliography"
+require_relative "block"
+require_relative "citation"
+require_relative "content"
+require_relative "document_attributes"
+require_relative "header"
+require_relative "inline"
+require_relative "list"
+require_relative "paragraph"
+require_relative "section"
+require_relative "table"
+require_relative "term"
+
 module Coradoc
   module Parser
     module Asciidoc
       module Base
+        include Coradoc::Parser::Asciidoc::Admonition
+        include Coradoc::Parser::Asciidoc::AttributeList
+        include Coradoc::Parser::Asciidoc::Bibliography
+        include Coradoc::Parser::Asciidoc::Block
+        include Coradoc::Parser::Asciidoc::Citation
+        include Coradoc::Parser::Asciidoc::Content
+        include Coradoc::Parser::Asciidoc::DocumentAttributes
+        include Coradoc::Parser::Asciidoc::Header
+        include Coradoc::Parser::Asciidoc::Inline
+        include Coradoc::Parser::Asciidoc::List
+        include Coradoc::Parser::Asciidoc::Paragraph
+        include Coradoc::Parser::Asciidoc::Section
+        include Coradoc::Parser::Asciidoc::Table
+        include Coradoc::Parser::Asciidoc::Term
+
         def space?
           space.maybe
         end
 
         def space
-          match('\s').repeat(1)
+          str(' ').repeat(1)
         end
 
         def text
@@ -15,7 +45,7 @@ module Coradoc
         end
 
         def line_ending
-          match("[\n]")
+          str("\n")
         end
 
         def endline
@@ -23,11 +53,15 @@ module Coradoc
         end
 
         def newline
-          match["\r\n"].repeat(1)
+          (str("\n") | str("\r\n")).repeat(1)
+        end
+
+        def newline_single
+          (str("\n") | str("\r\n"))
         end
 
         def keyword
-          (match("[a-zA-Z0-9_-]") | str(".")).repeat(1)
+          (match('[a-zA-Z0-9_\-.,]') | str(".")).repeat(1)
         end
 
         def empty_line
@@ -62,22 +96,76 @@ module Coradoc
           word >> str("@") >> word >> str(".") >> word
         end
 
-        def attribute_name
-          match("[a-zA-Z0-9_-]").repeat(1)
-        end
-
-        def attribute_value
-          text | str("")
-        end
-
         def special_character
-          match("^[*_:=-]") | str("[#") | str("[[")
+          match("^[*:=-]") | str("[#") | str("[[")
         end
 
         def date
           digit.repeat(2, 4) >> str("-") >>
             digit.repeat(1, 2) >> str("-") >> digit.repeat(1, 2)
         end
+
+        def attr_name
+          match("[^\t\s]").repeat(1)
+        end
+
+        def file_path
+          match('[^\[]').repeat(1)
+        end
+
+        def include_directive
+          (str("include::") >> 
+            file_path.as(:path) >>
+            attribute_list >>
+          (newline | str("")).as(:line_break)
+          ).as(:include)
+        end
+
+        def inline_image
+          (str("image::") >> 
+            file_path.as(:path) >>
+            attribute_list >>
+          (line_ending)
+          ).as(:inline_image)
+        end
+
+        def block_image
+          (block_id.maybe >>
+            block_title.maybe >>
+            (attribute_list >> newline).maybe >>
+            match('^i') >> str("mage::") >>
+            file_path.as(:path) >>
+            attribute_list(:attribute_list_macro) >>
+            newline.as(:line_break)
+            ).as(:block_image)
+        end
+
+        def comment_line
+          tag.absent? >>
+          (str('//') >> str("/").absent? >>
+            space? >>
+            text.as(:comment_text)
+            ).as(:comment_line)
+        end
+
+        def tag
+          (str('//') >> str('/').absent? >>
+            space? >>
+            (str('tag') | str('end')).as(:prefix) >>
+            str('::') >> str(':').absent? >>
+            match('[^\[]').repeat(1).as(:name) >>
+            attribute_list >>
+            line_ending.maybe.as(:line_break)
+            ).as(:tag)
+        end
+
+        def comment_block
+          ( str('////') >> line_ending >>
+          ((line_ending >> str('////')).absent? >> any
+            ).repeat.as(:comment_text) >> 
+          line_ending >> str('////')
+          ).as(:comment_block)
+        end
       end
     end
   end

data/lib/coradoc/parser/asciidoc/bibliography.rb

@@ -0,0 +1,30 @@
+module Coradoc
+  module Parser
+    module Asciidoc
+      module Bibliography
+
+        def bibliography
+          (section_id.maybe >>
+          str("[bibliography]\n") >>
+          str("== ") >> match("[^\n]").repeat(1).as(:title) >> str("\n") >>
+          bib_entry.repeat(1).as(:entries)
+          ).as(:bibliography)
+        end
+
+        def bib_entry
+          (match("^*") >> str(' [[[') >>
+            match('[^,\[\]\n]').repeat(1).as(:anchor_name) >>
+          (  str(",") >>
+            match('[^\]\n]').repeat(1).as(:document_id)
+            ).maybe  >>
+          str("]]]") >>
+            (text_line.repeat(0,1) >>
+              text_line.repeat(0)
+            ).as(:ref_text).maybe >>
+            line_ending.repeat(1).as(:line_break).maybe
+          ).as(:bibliography_entry)
+        end
+      end
+    end
+  end
+end