coradoc 0.3.0 → 1.1.0

data/lib/coradoc/reverse_adoc/converters/text.rb

@@ -4,11 +4,7 @@ module Coradoc::ReverseAdoc
       def to_coradoc(node, state = {})
         return treat_empty(node, state) if node.text.strip.empty?
 
-        Coradoc::Element::TextElement.new(treat_text(node))
-      end
-
-      def convert(node, state = {})
-        Coradoc::Generator.gen_adoc(to_coradoc(node, state))
+        Coradoc::Element::TextElement.new(node.text)
       end
 
       private
@@ -25,39 +21,6 @@ module Coradoc::ReverseAdoc
           ""
         end
       end
-
-      def treat_text(node)
-        text = node.text
-        text = preserve_nbsp(text)
-        text = remove_border_newlines(text)
-        text = remove_inner_newlines(text)
-        text = escape_keychars(text)
-
-        text = preserve_keychars_within_backticks(text)
-        escape_links(text)
-      end
-
-      def preserve_nbsp(text)
-        text.gsub(/\u00A0/, " ")
-      end
-
-      def escape_links(text)
-        text.gsub(/<<([^>]*)>>/, "\\<<\\1>>")
-      end
-
-      def remove_border_newlines(text)
-        text.gsub(/\A\n+/, "").gsub(/\n+\z/, "")
-      end
-
-      def remove_inner_newlines(text)
-        text.tr("\n\t", " ").squeeze(" ")
-      end
-
-      def preserve_keychars_within_backticks(text)
-        text.gsub(/`.*?`/) do |match|
-          match.gsub('\_', "_").gsub('\*', "*")
-        end
-      end
     end
 
     register :text, Text.new

data/lib/coradoc/reverse_adoc/converters/tr.rb

@@ -7,10 +7,6 @@ module Coradoc::ReverseAdoc
         Coradoc::Element::Table::Row.new(content, header)
       end
 
-      def convert(node, state = {})
-        Coradoc::Generator.gen_adoc(to_coradoc(node, state))
-      end
-
       def table_header_row?(node)
         # node.element_children.all? {|child| child.name.to_sym == :th}
         node.previous_element.nil?

data/lib/coradoc/reverse_adoc/converters/video.rb

@@ -12,10 +12,6 @@ module Coradoc::ReverseAdoc
                                            attributes: attributes)
       end
 
-      def convert(node, state = {})
-        Coradoc::Generator.gen_adoc(to_coradoc(node, state))
-      end
-
       def options(node)
         autoplay = node["autoplay"]
         loop_attr = node["loop"]

data/lib/coradoc/reverse_adoc/converters.rb

@@ -10,7 +10,30 @@ module Coradoc::ReverseAdoc
     end
 
     def self.lookup(tag_name)
-      @@converters[tag_name.to_sym] or default_converter(tag_name)
+      converter = @@converters[tag_name.to_sym] || default_converter(tag_name)
+      converter = converter.new if converter.respond_to? :new
+      converter
+    end
+
+    # Note: process won't run plugin hooks
+    def self.process(node, state)
+      node = node.to_a if node.is_a? Nokogiri::XML::NodeSet
+      return node.map { |i| process(i, state) }.join if node.is_a? Array
+
+      lookup(node.name).convert(node, state)
+    end
+
+    def self.process_coradoc(node, state)
+      node = node.to_a if node.is_a? Nokogiri::XML::NodeSet
+      return node.map { |i| process_coradoc(i, state) } if node.is_a? Array
+
+      plugins = state[:plugin_instances] || {}
+      process = proc { lookup(node.name).to_coradoc(node, state) }
+      plugins.each do |i|
+        prev_process = process
+        process = proc { i.html_tree_run_hooks(node, state, &prev_process) }
+      end
+      process.(node, state)
     end
 
     def self.default_converter(tag_name)

data/lib/coradoc/reverse_adoc/html_converter.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "converters/markup"
 require_relative "converters/a"
 require_relative "converters/aside"
 require_relative "converters/audio"
@@ -8,6 +9,7 @@ require_relative "converters/br"
 require_relative "converters/bypass"
 require_relative "converters/code"
 require_relative "converters/div"
+require_relative "converters/dl"
 require_relative "converters/drop"
 require_relative "converters/em"
 require_relative "converters/figure"
@@ -34,28 +36,115 @@ require_relative "converters/tr"
 require_relative "converters/video"
 require_relative "converters/math"
 
-module Coradoc::ReverseAdoc
-  class HtmlConverter
-    def self.to_coradoc(input, options = {})
-      root = case input
-             when String
-               Nokogiri::HTML(input).root
-             when Nokogiri::XML::Document
-               input.root
-             when Nokogiri::XML::Node
-               input
-             end
-
-      return "" unless root
-
-      Coradoc::ReverseAdoc.config.with(options) do
-        Coradoc::ReverseAdoc::Converters.lookup(root.name).to_coradoc(root)
+module Coradoc
+  module ReverseAdoc
+    class HtmlConverter
+      def self.to_coradoc(input, options = {})
+        plugin_instances = options.delete(:plugin_instances)
+        ReverseAdoc.config.with(options) do
+          plugin_instances ||= Coradoc::ReverseAdoc.config.plugins.map(&:new)
+
+          root = track_time "Loading input HTML document" do
+            case input
+            when String
+              Nokogiri::HTML(input).root
+            when Nokogiri::XML::Document
+              input.root
+            when Nokogiri::XML::Node
+              input
+            end
+          end
+
+          return "" unless root
+
+          plugin_instances.each do |plugin|
+            plugin.html_tree = root
+            if plugin.respond_to?(:preprocess_html_tree)
+              track_time "Preprocessing document with #{plugin.name} plugin" do
+                plugin.preprocess_html_tree
+              end
+            end
+            root = plugin.html_tree
+          end
+
+          coradoc = track_time "Converting input document tree to Coradoc tree" do
+            Converters.process_coradoc(root, plugin_instances: plugin_instances)
+          end
+
+          coradoc = track_time "Post-process Coradoc tree" do
+            Postprocessor.process(coradoc)
+          end
+
+          plugin_instances.each do |plugin|
+            if plugin.respond_to?(:postprocess_coradoc_tree)
+              plugin.coradoc_tree = coradoc
+              track_time "Postprocessing Coradoc tree with #{plugin.name} plugin" do
+                plugin.postprocess_coradoc_tree
+              end
+              coradoc = plugin.coradoc_tree
+            end
+          end
+
+          coradoc
+        end
       end
-    end
 
-    def self.convert(input, options = {})
-      result = Coradoc::Generator.gen_adoc(to_coradoc(input, options))
-      Coradoc::ReverseAdoc.cleaner.tidy(result)
+      def self.convert(input, options = {})
+        ReverseAdoc.config.with(options) do
+          plugin_instances = Coradoc::ReverseAdoc.config.plugins.map(&:new)
+
+          options = options.merge(plugin_instances: plugin_instances)
+
+          coradoc = to_coradoc(input, options)
+
+          if coradoc.is_a?(Hash)
+            coradoc.to_h do |file, tree|
+              track_time "Converting file #{file || 'main'}" do
+                [file, convert_single_coradoc_to_adoc(file, tree, plugin_instances)]
+              end
+            end
+          else
+            convert_single_coradoc_to_adoc(nil, coradoc, plugin_instances)
+          end
+        end
+      end
+
+      def self.convert_single_coradoc_to_adoc(_file, coradoc, plugin_instances)
+        result = track_time "Converting Coradoc tree into Asciidoc" do
+          Coradoc::Generator.gen_adoc(coradoc)
+        end
+        result = track_time "Cleaning up the result" do
+          ReverseAdoc.cleaner.tidy(result)
+        end
+        plugin_instances.each do |plugin|
+          if plugin.respond_to?(:postprocess_asciidoc_string)
+            plugin.asciidoc_string = result
+            track_time "Postprocessing AsciiDoc string with #{plugin.name} plugin" do
+              plugin.postprocess_asciidoc_string
+            end
+            result = plugin.asciidoc_string
+          end
+        end
+        result
+      end
+
+      @track_time_indentation = 0
+      def self.track_time(task)
+        if ReverseAdoc.config.track_time
+          warn "  " * @track_time_indentation +
+            "* #{task} is starting..."
+          @track_time_indentation += 1
+          t0 = Time.now
+          ret = yield
+          time_elapsed = Time.now - t0
+          @track_time_indentation -= 1
+          warn "  " * @track_time_indentation +
+            "* #{task} took #{time_elapsed.round(3)} seconds"
+          ret
+        else
+          yield
+        end
+      end
     end
   end
 end

data/lib/coradoc/reverse_adoc/plugin.rb

@@ -0,0 +1,131 @@
+module Coradoc::ReverseAdoc
+  class Plugin
+    #### Plugin system general
+
+    # Allow building plugins with a shorthand syntax:
+    # plugin = Coradoc::ReverseAdoc::Plugin.new do
+    #   def name = "Test"
+    # end
+
+    def self.new(&block)
+      if self == Plugin
+        Class.new(Plugin, &block)
+      else
+        super
+      end
+    end
+
+    def initialize
+      @html_tree_hooks_pre = {}
+      @html_tree_hooks_post = {}
+    end
+
+    # define name to name a Plugin
+    def name
+      self.class.name
+    end
+
+    #### HTML Tree functionalities
+
+    attr_accessor :html_tree
+
+    def html_tree_change_tag_name_by_css(css, new_name)
+      html_tree.css(css).each do |e|
+        e.name = new_name
+      end
+    end
+
+    def html_tree_change_properties_by_css(css, properties)
+      html_tree.css(css).each do |e|
+        properties.each do |k,v|
+          e[k.to_s] = v
+        end
+      end
+    end
+
+    def html_tree_remove_by_css(css)
+      html_tree.css(css).each(&:remove)
+    end
+
+    def html_tree_replace_with_children_by_css(css)
+      html_tree.css(css).each do |e|
+        e.replace(e.children)
+      end
+    end
+
+    def html_tree_process_to_coradoc(tree, state={})
+      Coradoc::ReverseAdoc::Converters.process_coradoc(tree, state)
+    end
+
+    def html_tree_process_to_adoc(tree, state={})
+      Coradoc::ReverseAdoc::Converters.process(tree, state)
+    end
+
+    def html_tree_preview
+      Tempfile.open(%w"coradoc .html") do |i|
+        i << html_tree.to_html
+        system "chromium-browser", "--no-sandbox", i.path
+      end
+    end
+
+    # define preprocess_html_tree to process HTML trees
+
+    # Creates a hook to be called instead of converting an element
+    # to a Coradoc node.
+    #
+    # proc |html_node, state|
+    #   coradoc_node
+    # end
+    def html_tree_add_hook_pre(element, &block)
+      @html_tree_hooks_pre[element] = block
+    end
+
+    def html_tree_add_hook_pre_by_css(css, &block)
+      html_tree.css(css).each do |e|
+        html_tree_add_hook_pre(e, &block)
+      end
+    end
+
+    # Creates a hook to be called after converting an element
+    # to a Coradoc node.
+    #
+    # proc |html_node, coradoc_node, state|
+    #   coradoc_node
+    # end
+    def html_tree_add_hook_post(element, &block)
+      @html_tree_hooks_post[element] = block
+    end
+
+    def html_tree_add_hook_post_by_css(css, &block)
+      html_tree.css(css).each do |e|
+        html_tree_add_hook_post(e, &block)
+      end
+    end
+
+    def html_tree_run_hooks(node, state, &_block)
+      hook_pre = @html_tree_hooks_pre[node]
+      hook_post = @html_tree_hooks_post[node]
+
+      coradoc = hook_pre.(node, state) if hook_pre
+      coradoc ||= yield node, state
+
+      if hook_post
+        coradoc = hook_post.(node, coradoc, state)
+      end
+
+      coradoc
+    end
+
+    #### Coradoc tree functionalities
+
+    attr_accessor :coradoc_tree
+
+    # define postprocess_coradoc_tree to change coradoc tree
+
+    #### AsciiDoc string functionalities
+
+    attr_accessor :asciidoc_string
+
+    # define postprocess_asciidoc_string to change the coradoc string
+  end
+end

data/lib/coradoc/reverse_adoc/plugins/plateau.rb

@@ -0,0 +1,206 @@
+module Coradoc::ReverseAdoc
+  class Plugin
+    # This plugin enhances documents from the PLATEAU project
+    # to extract more data.
+    #
+    # Usage:
+    # reverse_adoc -rcoradoc/reverse_adoc/plugins/plateau
+    #   --external-images -u raise --output _out/index.adoc index.html
+    class Plateau < Plugin
+      def name
+        "PLATEAU"
+      end
+
+      def preprocess_html_tree
+        # Let's simplify the tree by removing what's extraneous
+        # html_tree_remove_by_css("script, style, img.container_imagebox:not([src])")
+        # html_tree_replace_with_children_by_css("div.container_box")
+        # html_tree_replace_with_children_by_css("div.col.col-12")
+        # html_tree_replace_with_children_by_css(".tabledatatext, .tabledatatextY")
+        # html_tree_replace_with_children_by_css("div.row")
+        #
+        # We can remove that, but it messes up the images and paragraphs.
+
+        # Remove side menu, so we can generate TOC ourselves
+        html_tree_remove_by_css(".sideMenu")
+
+        # Correct non-semantic classes into semantic HTML tags
+        html_tree_change_tag_name_by_css(".titledata", "h1")
+        html_tree_change_tag_name_by_css(".subtitledata", "h2")
+        html_tree_change_tag_name_by_css(".pitemdata", "h3")
+        html_tree_change_tag_name_by_css(".sitemdata", "h4")
+        html_tree_change_tag_name_by_css('td[bgcolor="#D0CECE"]', "th")
+        html_tree_change_tag_name_by_css('td[bgcolor="#d0cece"]', "th")
+        html_tree_change_tag_name_by_css('.framedata, .frame_container_box', 'aside')
+        html_tree_change_tag_name_by_css('.frame2data', 'pre')
+        # Assumption that all code snippets in those documents are XML...
+        html_tree_change_properties_by_css(".frame2data", class: "brush:xml;")
+
+        # Remove some CSS ids that are not important to us
+        html_tree_change_properties_by_css("#__nuxt", id: nil)
+        html_tree_change_properties_by_css("#__layout", id: nil)
+        html_tree_change_properties_by_css("#app", id: nil)
+
+        # Handle lists of document 02
+        html_tree_replace_with_children_by_css(".list_num-wrap")
+
+        # Convert table/img caption to become a caption
+        html_tree.css(".imagedata").each do |e|
+          table = e.parent.next&.children&.first
+          if table&.name == "table"
+            e.name = "caption"
+            table.prepend_child(e)
+            next
+          end
+
+          img = e.parent.previous&.children&.first
+          if img&.name == "img" && img["src"]
+            title = e.text.strip
+            img["title"] = title
+            e.remove
+            next
+          end
+        end
+
+        # Add hooks for H1, H2, H3, H4
+        html_tree_add_hook_post_by_css("h1, h2, h3", &method(:handle_headers))
+        html_tree_add_hook_post_by_css("h4", &method(:handle_headers_h4))
+
+        # Table cells aligned to center
+        html_tree_change_properties_by_css(".tableTopCenter", align: "center")
+
+        # Handle non-semantic lists and indentation
+        html_tree_add_hook_pre_by_css ".text2data" do |node,|
+          text = html_tree_process_to_adoc(node).strip
+          next "" if text.empty? || text == "\u3000"
+
+          if text.start_with?(/\d+\./)
+            text = text.sub(/\A\d+.\s*/, "")
+            ".. #{text}\n"
+          else
+            text = text.gsub(/^/, "** ")
+            "\n\n//-PT2D\n#{text}\n//-ENDPT2D\n\n"
+          end
+        end
+
+        (3..4).each do |i|
+          html_tree_add_hook_pre_by_css ".text#{i}data" do |node,|
+            text = html_tree_process_to_adoc(node).strip
+            next "" if text.empty? || text == "\u3000"
+
+            text = text.strip.gsub(/^/, "#{'*' * i} ")
+            "\n\n//-PT#{i}D\n#{text}\n//-ENDPT#{i}D\n\n"
+          end
+        end
+
+        (2..3).each do |i|
+          html_tree_add_hook_pre_by_css ".text#{i}data_point ul" do |node,|
+            text = html_tree_process_to_adoc(node.children.first.children).strip
+
+            "#{'*' * i} #{text}\n"
+          end
+        end
+
+        (1..20).each do |i|
+          html_tree_add_hook_pre_by_css ".numtextdata_num .list_num#{i}" do |node,|
+            text = html_tree_process_to_adoc(node).strip
+
+            "[start=#{i}]\n. #{text}\n"
+          end
+        end
+
+        # html_tree_preview
+      end
+
+      IM = /[A-Z0-9]{1,3}/
+
+      def handle_headers(node, coradoc, state)
+        content = coradoc.content.map(&:content).join
+
+        if %w[toc0 toc_0].any? { |i| coradoc.id&.start_with?(i) }
+          # Special content
+          case content.strip
+          when "はじめに" # Introduction
+            coradoc.style = "abstract" # The older version document has ".preface"
+            coradoc.level_int = 1
+          when "改定の概要" # Revision overview
+            coradoc.style = "abstract" # The older version document has ".preface"
+            coradoc.level_int = 1
+          when "参考文献" # Bibliography
+            coradoc.style = "bibliography"
+            coradoc.level_int = 1
+          when "改訂履歴" # Document history
+            coradoc.style = "appendix"
+            coradoc.level_int = 1
+          when "0　概要" # Overview
+            coradoc.style = "abstract" # I'm not sure this is correct
+            coradoc.level_int = 1
+          when "索引" # Index
+            coradoc.style = "index" # I'm not sure this is correct
+            coradoc.level_int = 1
+          else
+            warn "Unknown section #{content.inspect}"
+          end
+        end
+
+        if node.name == "h1"
+          if content.start_with?("Annex")
+            coradoc.style = "appendix"
+            coradoc.content.first.content.sub!(/\AAnnex [A-Z]/, "")
+          end
+        end
+
+        # Remove numbers
+        coradoc.content.first.content.sub!(/\A(#{IM}\.)*#{IM}[[:space:]]/, "")
+
+        coradoc
+      end
+
+      def handle_headers_h4(node, coradoc, state)
+        title = Coradoc.strip_unicode(coradoc.content.first.content)
+        case title
+        when /\A\(\d+\)(.*)/
+          coradoc.level_int = 4
+          coradoc.content.first.content = $1.strip
+          coradoc
+        when /\A\d+\)(.*)/
+          coradoc.level_int = 5
+          coradoc.content.first.content = $1.strip
+          coradoc
+        when /\A#{IM}\.#{IM}\.#{IM}\.#{IM}(.*)/
+          coradoc.level_int = 4
+          coradoc.content.first.content = $1.strip
+        else
+          if title.empty?
+            # Strip instances of faulty empty paragraphs
+            nil
+          else
+            ["// FIXME\n", coradoc]
+          end
+        end
+      end
+
+      def postprocess_asciidoc_string
+        str = self.asciidoc_string
+
+        ### Custom indentation handling
+        # If there's a step up, add [none]
+        str = str.gsub(%r{\s+//-ENDPT2D\s+//-PT3D\s+}, "\n[none]\n")
+        str = str.gsub(%r{\s+//-ENDPT2D\s+//-PT4D\s+}, "\n[none]\n")
+        str = str.gsub(%r{\s+//-ENDPT3D\s+//-PT4D\s+}, "\n[none]\n")
+        # Collapse blocks of text[2,3]data
+        str = str.gsub(%r{\s+//-ENDPT[234]D\s+//-PT[234]D\s+}, "\n\n")
+        # In the beginning, add [none]
+        str = str.gsub(%r{\s+//-PT[234]D\s+}, "\n\n[none]\n")
+        # If following with another list, ensure we readd styling
+        str = str.gsub(%r{\s+//-ENDPT[234]D\s+\*}, "\n\n[disc]\n*")
+        # Otherwise, clean up
+        str = str.gsub(%r{\s+//-ENDPT[234]D\s+}, "\n\n")
+
+        self.asciidoc_string = str
+      end
+    end
+  end
+end
+
+Coradoc::ReverseAdoc.config.plugins << Coradoc::ReverseAdoc::Plugin::Plateau