coradoc 0.3.0 → 1.0.0

data/lib/coradoc/reverse_adoc/html_converter.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "converters/markup"
 require_relative "converters/a"
 require_relative "converters/aside"
 require_relative "converters/audio"
@@ -8,6 +9,7 @@ require_relative "converters/br"
 require_relative "converters/bypass"
 require_relative "converters/code"
 require_relative "converters/div"
+require_relative "converters/dl"
 require_relative "converters/drop"
 require_relative "converters/em"
 require_relative "converters/figure"
@@ -34,28 +36,115 @@ require_relative "converters/tr"
 require_relative "converters/video"
 require_relative "converters/math"
 
-module Coradoc::ReverseAdoc
-  class HtmlConverter
-    def self.to_coradoc(input, options = {})
-      root = case input
-             when String
-               Nokogiri::HTML(input).root
-             when Nokogiri::XML::Document
-               input.root
-             when Nokogiri::XML::Node
-               input
-             end
-
-      return "" unless root
-
-      Coradoc::ReverseAdoc.config.with(options) do
-        Coradoc::ReverseAdoc::Converters.lookup(root.name).to_coradoc(root)
+module Coradoc
+  module ReverseAdoc
+    class HtmlConverter
+      def self.to_coradoc(input, options = {})
+        plugin_instances = options.delete(:plugin_instances)
+        ReverseAdoc.config.with(options) do
+          plugin_instances ||= Coradoc::ReverseAdoc.config.plugins.map(&:new)
+
+          root = track_time "Loading input HTML document" do
+            case input
+            when String
+              Nokogiri::HTML(input).root
+            when Nokogiri::XML::Document
+              input.root
+            when Nokogiri::XML::Node
+              input
+            end
+          end
+
+          return "" unless root
+
+          plugin_instances.each do |plugin|
+            plugin.html_tree = root
+            if plugin.respond_to?(:preprocess_html_tree)
+              track_time "Preprocessing document with #{plugin.name} plugin" do
+                plugin.preprocess_html_tree
+              end
+            end
+            root = plugin.html_tree
+          end
+
+          coradoc = track_time "Converting input document tree to Coradoc tree" do
+            Converters.process_coradoc(root, plugin_instances: plugin_instances)
+          end
+
+          coradoc = track_time "Post-process Coradoc tree" do
+            Postprocessor.process(coradoc)
+          end
+
+          plugin_instances.each do |plugin|
+            if plugin.respond_to?(:postprocess_coradoc_tree)
+              plugin.coradoc_tree = coradoc
+              track_time "Postprocessing Coradoc tree with #{plugin.name} plugin" do
+                plugin.postprocess_coradoc_tree
+              end
+              coradoc = plugin.coradoc_tree
+            end
+          end
+
+          coradoc
+        end
       end
-    end
 
-    def self.convert(input, options = {})
-      result = Coradoc::Generator.gen_adoc(to_coradoc(input, options))
-      Coradoc::ReverseAdoc.cleaner.tidy(result)
+      def self.convert(input, options = {})
+        ReverseAdoc.config.with(options) do
+          plugin_instances = Coradoc::ReverseAdoc.config.plugins.map(&:new)
+
+          options = options.merge(plugin_instances: plugin_instances)
+
+          coradoc = to_coradoc(input, options)
+
+          if coradoc.is_a?(Hash)
+            coradoc.to_h do |file, tree|
+              track_time "Converting file #{file || 'main'}" do
+                [file, convert_single_coradoc_to_adoc(file, tree, plugin_instances)]
+              end
+            end
+          else
+            convert_single_coradoc_to_adoc(nil, coradoc, plugin_instances)
+          end
+        end
+      end
+
+      def self.convert_single_coradoc_to_adoc(_file, coradoc, plugin_instances)
+        result = track_time "Converting Coradoc tree into Asciidoc" do
+          Coradoc::Generator.gen_adoc(coradoc)
+        end
+        result = track_time "Cleaning up the result" do
+          ReverseAdoc.cleaner.tidy(result)
+        end
+        plugin_instances.each do |plugin|
+          if plugin.respond_to?(:postprocess_asciidoc_string)
+            plugin.asciidoc_string = result
+            track_time "Postprocessing AsciiDoc string with #{plugin.name} plugin" do
+              plugin.postprocess_asciidoc_string
+            end
+            result = plugin.asciidoc_string
+          end
+        end
+        result
+      end
+
+      @track_time_indentation = 0
+      def self.track_time(task)
+        if ReverseAdoc.config.track_time
+          warn "  " * @track_time_indentation +
+            "* #{task} is starting..."
+          @track_time_indentation += 1
+          t0 = Time.now
+          ret = yield
+          time_elapsed = Time.now - t0
+          @track_time_indentation -= 1
+          warn "  " * @track_time_indentation +
+            "* #{task} took #{time_elapsed.round(3)} seconds"
+          ret
+        else
+          yield
+        end
+      end
     end
   end
 end

data/lib/coradoc/reverse_adoc/plugin.rb

@@ -0,0 +1,131 @@
+module Coradoc::ReverseAdoc
+  class Plugin
+    #### Plugin system general
+
+    # Allow building plugins with a shorthand syntax:
+    # plugin = Coradoc::ReverseAdoc::Plugin.new do
+    #   def name = "Test"
+    # end
+
+    def self.new(&block)
+      if self == Plugin
+        Class.new(Plugin, &block)
+      else
+        super
+      end
+    end
+
+    def initialize
+      @html_tree_hooks_pre = {}
+      @html_tree_hooks_post = {}
+    end
+
+    # define name to name a Plugin
+    def name
+      self.class.name
+    end
+
+    #### HTML Tree functionalities
+
+    attr_accessor :html_tree
+
+    def html_tree_change_tag_name_by_css(css, new_name)
+      html_tree.css(css).each do |e|
+        e.name = new_name
+      end
+    end
+
+    def html_tree_change_properties_by_css(css, properties)
+      html_tree.css(css).each do |e|
+        properties.each do |k,v|
+          e[k.to_s] = v
+        end
+      end
+    end
+
+    def html_tree_remove_by_css(css)
+      html_tree.css(css).each(&:remove)
+    end
+
+    def html_tree_replace_with_children_by_css(css)
+      html_tree.css(css).each do |e|
+        e.replace(e.children)
+      end
+    end
+
+    def html_tree_process_to_coradoc(tree, state={})
+      Coradoc::ReverseAdoc::Converters.process_coradoc(tree, state)
+    end
+
+    def html_tree_process_to_adoc(tree, state={})
+      Coradoc::ReverseAdoc::Converters.process(tree, state)
+    end
+
+    def html_tree_preview
+      Tempfile.open(%w"coradoc .html") do |i|
+        i << html_tree.to_html
+        system "chromium-browser", "--no-sandbox", i.path
+      end
+    end
+
+    # define preprocess_html_tree to process HTML trees
+
+    # Creates a hook to be called instead of converting an element
+    # to a Coradoc node.
+    #
+    # proc |html_node, state|
+    #   coradoc_node
+    # end
+    def html_tree_add_hook_pre(element, &block)
+      @html_tree_hooks_pre[element] = block
+    end
+
+    def html_tree_add_hook_pre_by_css(css, &block)
+      html_tree.css(css).each do |e|
+        html_tree_add_hook_pre(e, &block)
+      end
+    end
+
+    # Creates a hook to be called after converting an element
+    # to a Coradoc node.
+    #
+    # proc |html_node, coradoc_node, state|
+    #   coradoc_node
+    # end
+    def html_tree_add_hook_post(element, &block)
+      @html_tree_hooks_post[element] = block
+    end
+
+    def html_tree_add_hook_post_by_css(css, &block)
+      html_tree.css(css).each do |e|
+        html_tree_add_hook_post(e, &block)
+      end
+    end
+
+    def html_tree_run_hooks(node, state, &_block)
+      hook_pre = @html_tree_hooks_pre[node]
+      hook_post = @html_tree_hooks_post[node]
+
+      coradoc = hook_pre.(node, state) if hook_pre
+      coradoc ||= yield node, state
+
+      if hook_post
+        coradoc = hook_post.(node, coradoc, state)
+      end
+
+      coradoc
+    end
+
+    #### Coradoc tree functionalities
+
+    attr_accessor :coradoc_tree
+
+    # define postprocess_coradoc_tree to change coradoc tree
+
+    #### AsciiDoc string functionalities
+
+    attr_accessor :asciidoc_string
+
+    # define postprocess_asciidoc_string to change the coradoc string
+  end
+end

data/lib/coradoc/reverse_adoc/plugins/plateau.rb

@@ -0,0 +1,174 @@
+module Coradoc::ReverseAdoc
+  class Plugin
+    # This plugin enhances documents from the PLATEAU project
+    # to extract more data.
+    #
+    # Usage:
+    # reverse_adoc -rcoradoc/reverse_adoc/plugins/plateau
+    #   --external-images -u raise --output _out/index.adoc index.html
+    class Plateau < Plugin
+      def name
+        "PLATEAU"
+      end
+
+      def preprocess_html_tree
+        # Let's simplify the tree by removing what's extraneous
+        # html_tree_remove_by_css("script, style, img.container_imagebox:not([src])")
+        # html_tree_replace_with_children_by_css("div.container_box")
+        # html_tree_replace_with_children_by_css("div.col.col-12")
+        # html_tree_replace_with_children_by_css(".tabledatatext, .tabledatatextY")
+        # html_tree_replace_with_children_by_css("div.row")
+        #
+        # We can remove that, but it messes up the images and paragraphs.
+
+        # Remove side menu, so we can generate TOC ourselves
+        html_tree_remove_by_css(".sideMenu")
+
+        # Correct non-semantic classes into semantic HTML tags
+        html_tree_change_tag_name_by_css(".titledata", "h1")
+        html_tree_change_tag_name_by_css(".subtitledata", "h2")
+        html_tree_change_tag_name_by_css(".pitemdata", "h3")
+        html_tree_change_tag_name_by_css(".sitemdata", "h4")
+        html_tree_change_tag_name_by_css('td[bgcolor="#D0CECE"]', "th")
+
+        # Remove some CSS ids that are not important to us
+        html_tree_change_properties_by_css("#__nuxt", id: nil)
+        html_tree_change_properties_by_css("#__layout", id: nil)
+        html_tree_change_properties_by_css("#app", id: nil)
+
+        # Convert table/img caption to become a caption
+        html_tree.css(".imagedata").each do |e|
+          table = e.parent.next&.children&.first
+          if table&.name == "table"
+            e.name = "caption"
+            table.prepend_child(e)
+            next
+          end
+
+          img = e.parent.previous&.children&.first
+          if img&.name == "img" && img["src"]
+            title = e.text.strip
+            img["title"] = title
+            e.remove
+            next
+          end
+        end
+
+        # Add hooks for H1, H2, H3, H4
+        html_tree_add_hook_post_by_css("h1, h2, h3", &method(:handle_headers))
+        html_tree_add_hook_post_by_css("h4", &method(:handle_headers_h4))
+
+        # Table cells aligned to center
+        html_tree_change_properties_by_css(".tableTopCenter", align: "center")
+
+        # Handle non-semantic lists and indentation
+        html_tree_add_hook_pre_by_css ".text2data" do |node,|
+          text = html_tree_process_to_adoc(node).strip
+          next "" if text.empty? || text == "\u3000"
+
+          if text.start_with?(/\d+\./)
+            text = text.sub(/\A\d+.\s*/, "")
+            ".. #{text}\n"
+          else
+            text = text.gsub(/^/, "** ")
+            "\n\n//-PT2D\n#{text}\n//-ENDPT2D\n\n"
+          end
+        end
+
+        html_tree_add_hook_pre_by_css ".text3data" do |node,|
+          text = html_tree_process_to_adoc(node).strip
+          next "" if text.empty? || text == "\u3000"
+
+          text = text.strip.gsub(/^/, "*** ")
+          "\n\n//-PT3D\n#{text}\n//-ENDPT3D\n\n"
+        end
+
+        html_tree_add_hook_pre_by_css ".text4data" do |node,|
+          text = html_tree_process_to_adoc(node).strip
+          next "" if text.empty? || text == "\u3000"
+
+          text = text.strip.gsub(/^/, "**** ")
+          "\n\n//-PT4D\n#{text}\n//-ENDPT4D\n\n"
+        end
+
+        html_tree_add_hook_pre_by_css ".text2data_point ul" do |node,|
+          text = html_tree_process_to_adoc(node.children.first.children).strip
+
+          "** #{text}\n"
+        end
+
+        html_tree_add_hook_pre_by_css ".text3data_point ul" do |node,|
+          text = html_tree_process_to_adoc(node.children.first.children).strip
+
+          "*** #{text}\n"
+        end
+
+        # html_tree_preview
+      end
+
+      def handle_headers(node, coradoc, state)
+        if coradoc.id.start_with?("toc0_")
+          content = coradoc.content.map(&:content).join
+          # Special content
+          case content.strip
+          when "はじめに" # Introduction
+            coradoc.style = "abstract" # The older version document has ".preface"
+          when "改定の概要" # Revision overview
+            coradoc.style = "abstract" # The older version document has ".preface"
+          when "参考文献" # Bibliography
+            coradoc.style = "bibliography"
+          when "改訂履歴" # Document history
+            coradoc.style = "appendix"
+          else
+            warn "Unknown section #{coradoc.content.map(&:content).join.inspect}"
+          end
+
+          # Ensure they are generated as level 1
+          coradoc.level_int = 1
+        end
+
+        # Remove numbers
+        coradoc.content.first.content.sub!(/\A[\d\s.]+/, "")
+
+        coradoc
+      end
+
+      def handle_headers_h4(node, coradoc, state)
+        case coradoc.content.first.content
+        when /\A\(\d+\)(.*)/
+          coradoc.level_int = 4
+          coradoc.content.first.content = $1.strip
+          coradoc
+        when /\A\d+\)(.*)/
+          coradoc.level_int = 5
+          coradoc.content.first.content = $1.strip
+          coradoc
+        else
+          ["// FIXME\n", coradoc]
+        end
+      end
+
+      def postprocess_asciidoc_string
+        str = self.asciidoc_string
+
+        ### Custom indentation handling
+        # If there's a step up, add [none]
+        str = str.gsub(%r{\s+//-ENDPT2D\s+//-PT3D\s+}, "\n[none]\n")
+        str = str.gsub(%r{\s+//-ENDPT2D\s+//-PT4D\s+}, "\n[none]\n")
+        str = str.gsub(%r{\s+//-ENDPT3D\s+//-PT4D\s+}, "\n[none]\n")
+        # Collapse blocks of text[2,3]data
+        str = str.gsub(%r{\s+//-ENDPT[234]D\s+//-PT[234]D\s+}, "\n\n")
+        # In the beginning, add [none]
+        str = str.gsub(%r{\s+//-PT[234]D\s+}, "\n\n[none]\n")
+        # If following with another list, ensure we readd styling
+        str = str.gsub(%r{\s+//-ENDPT[234]D\s+\*}, "\n\n[disc]\n*")
+        # Otherwise, clean up
+        str = str.gsub(%r{\s+//-ENDPT[234]D\s+}, "\n\n")
+
+        self.asciidoc_string = str
+      end
+    end
+  end
+end
+
+Coradoc::ReverseAdoc.config.plugins << Coradoc::ReverseAdoc::Plugin::Plateau

data/lib/coradoc/reverse_adoc/postprocessor.rb

@@ -0,0 +1,148 @@
+module Coradoc::ReverseAdoc
+  # Postprocessor's aim is to convert a Coradoc tree from
+  # a mess that has been created from HTML into a tree that
+  # is compatible with what we would get out of Coradoc, if
+  # it parsed it directly.
+  class Postprocessor
+    def self.process(coradoc)
+      new(coradoc).process
+    end
+
+    def initialize(coradoc)
+      @tree = coradoc
+    end
+
+    # Collapse DIVs that only have a title, or nest another DIV.
+    def collapse_meaningless_sections
+      @tree = Coradoc::Element::Base.visit(@tree) do |elem, _dir|
+        if elem.is_a?(Coradoc::Element::Section) && elem.safe_to_collapse?
+          children_classes = Array(elem.contents).map(&:class)
+          count = children_classes.length
+          safe_classes = [Coradoc::Element::Section, Coradoc::Element::Title]
+
+          # Count > 0 because some documents use <div> as a <br>.
+          if count > 0 && children_classes.all? { |i| safe_classes.include?(i) }
+            next elem.contents
+          end
+        end
+        elem
+      end
+    end
+
+    # tree should now be more cleaned up, so we can progress with
+    # creating meaningful sections
+    def generate_meaningful_sections
+      @tree = Coradoc::Element::Base.visit(@tree) do |elem, dir|
+        # We are searching for an array, that has a title. This
+        # will be a candidate for our section array.
+        if dir == :post &&
+            elem.is_a?(Array) &&
+            !elem.grep(Coradoc::Element::Title).empty?
+
+          new_array = []
+          content_array = new_array
+          section_arrays_by_level = [new_array] * 8
+
+          # For each title element, we create a new section. Then we push
+          # all descendant sections into those sections. Otherwise, we push
+          # an element as content of current section.
+          elem.each do |e|
+            if e.is_a? Coradoc::Element::Title
+              title = e
+              content_array = []
+              section_array = []
+              level = title.level_int
+              section = Coradoc::Element::Section.new(
+                title, contents: content_array, sections: section_array
+              )
+              # Some documents may not be consistent and eg. follow H4 after
+              # H2. Let's ensure that proceeding sections will land in a
+              # correct place.
+              (8 - level).times do |j|
+                section_arrays_by_level[level + j] = section_array
+              end
+              section_arrays_by_level[level - 1] << section
+            else
+              content_array << e
+            end
+          end
+          next new_array
+        end
+        elem
+      end
+    end
+
+    def split_sections
+      max_level = Coradoc::ReverseAdoc.config.split_sections
+
+      return unless max_level
+
+      sections = {}
+      parent_sections = []
+      previous_sections = {}
+
+      determine_section_id = ->(elem) do
+        level = 0
+        section = elem
+        while section
+          level += 1 if elem.title.style == section.title.style
+          section = previous_sections[section]
+        end
+        level
+      end
+
+      determine_style = ->(elem) do
+        style = elem.title.style || "section"
+        style += "-"
+        style
+      end
+
+      @tree = Coradoc::Element::Base.visit(@tree) do |elem, dir|
+        title = elem.title if elem.is_a?(Coradoc::Element::Section)
+
+        if title && title.level_int <= max_level
+          if dir == :pre
+            # In the PRE pass, we build a tree of sections, so that
+            # we can compute numbers
+            previous_sections[elem] = parent_sections[title.level_int]
+            parent_sections[title.level_int] = elem
+            parent_sections[(title.level_int+1)..nil] = nil
+
+            elem
+          else
+            # In the POST pass, we replace the sections with their
+            # include tag.
+            section_file = "sections/"
+            section_file += parent_sections[1..title.level_int].map do |parent|
+              style = determine_style.(parent)
+              "%s%02d" % [style, determine_section_id.(parent)]
+            end.join("/")
+            section_file += ".adoc"
+
+            sections[section_file] = elem
+            up = "../" * (title.level_int - 1)
+            "\ninclude::#{up}#{section_file}[]\n"
+          end
+        else
+          elem
+        end
+      end
+
+      sections[nil] = @tree
+      @tree = sections
+    end
+
+    def process
+      collapse_meaningless_sections
+      generate_meaningful_sections
+      # Do it again to simplify the document further.
+      # Since the structure is changed, we may have new meaningful
+      # sections as only children of some meaningless sections.
+      collapse_meaningless_sections
+
+      split_sections
+
+      @tree
+    end
+  end
+end

data/lib/coradoc/reverse_adoc.rb

@@ -9,6 +9,9 @@ require_relative "reverse_adoc/config"
 require_relative "reverse_adoc/converters"
 require_relative "reverse_adoc/converters/base"
 require_relative "reverse_adoc/html_converter"
+require_relative "reverse_adoc/plugin"
+require_relative "reverse_adoc/postprocessor"
+
 
 module Coradoc::ReverseAdoc
   def self.convert(input, options = {})

data/lib/coradoc/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coradoc
-  VERSION = "0.3.0"
+  VERSION = "1.0.0"
 end

data/lib/reverse_adoc.rb

@@ -8,7 +8,7 @@ warn <<~END
   | You are referencing an old require here:
   |   #{caller.join("\n|   ")}
   |
-  | You should also replace 'coradoc' with 'reverse_adoc' in your gem dependencies.
+  | You should also replace 'reverse_adoc' with 'coradoc' in your gem dependencies.
   | reverse_adoc 2.0.0 will be kept with 'coradoc' as the only dependency.
   |
   | Please also ensure that you replace all references to ReverseAdoc in your code

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coradoc
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Ribose Inc.
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-05-21 00:00:00.000000000 Z
+date: 2024-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: marcel
@@ -278,7 +278,6 @@ files:
 - lib/coradoc/element/inline/cross_reference.rb
 - lib/coradoc/element/inline/hard_line_break.rb
 - lib/coradoc/element/inline/highlight.rb
-- lib/coradoc/element/inline/image.rb
 - lib/coradoc/element/inline/italic.rb
 - lib/coradoc/element/inline/link.rb
 - lib/coradoc/element/inline/monospace.rb
@@ -291,6 +290,7 @@ files:
 - lib/coradoc/element/list/ordered.rb
 - lib/coradoc/element/list/unordered.rb
 - lib/coradoc/element/list_item.rb
+- lib/coradoc/element/list_item_definition.rb
 - lib/coradoc/element/paragraph.rb
 - lib/coradoc/element/revision.rb
 - lib/coradoc/element/section.rb
@@ -323,6 +323,7 @@ files:
 - lib/coradoc/reverse_adoc/converters/bypass.rb
 - lib/coradoc/reverse_adoc/converters/code.rb
 - lib/coradoc/reverse_adoc/converters/div.rb
+- lib/coradoc/reverse_adoc/converters/dl.rb
 - lib/coradoc/reverse_adoc/converters/drop.rb
 - lib/coradoc/reverse_adoc/converters/em.rb
 - lib/coradoc/reverse_adoc/converters/figure.rb
@@ -333,6 +334,7 @@ files:
 - lib/coradoc/reverse_adoc/converters/img.rb
 - lib/coradoc/reverse_adoc/converters/li.rb
 - lib/coradoc/reverse_adoc/converters/mark.rb
+- lib/coradoc/reverse_adoc/converters/markup.rb
 - lib/coradoc/reverse_adoc/converters/math.rb
 - lib/coradoc/reverse_adoc/converters/ol.rb
 - lib/coradoc/reverse_adoc/converters/p.rb
@@ -350,6 +352,9 @@ files:
 - lib/coradoc/reverse_adoc/converters/video.rb
 - lib/coradoc/reverse_adoc/errors.rb
 - lib/coradoc/reverse_adoc/html_converter.rb
+- lib/coradoc/reverse_adoc/plugin.rb
+- lib/coradoc/reverse_adoc/plugins/plateau.rb
+- lib/coradoc/reverse_adoc/postprocessor.rb
 - lib/coradoc/transformer.rb
 - lib/coradoc/version.rb
 - lib/reverse_adoc.rb