coradoc 0.2.0 → 1.0.0

data/lib/coradoc/reverse_adoc/converters/video.rb

@@ -0,0 +1,25 @@
+module Coradoc::ReverseAdoc
+  module Converters
+    class Video < Base
+      def to_coradoc(node, _state = {})
+        src = node["src"]
+        id = node["id"]
+        title = extract_title(node)
+        attributes = Coradoc::Element::AttributeList.new
+        options = options(node)
+        attributes.add_named("options", options) if options.any?
+        Coradoc::Element::Video.new(title, id: id, src: src,
+                                           attributes: attributes)
+      end
+
+      def options(node)
+        autoplay = node["autoplay"]
+        loop_attr = node["loop"]
+        controls = node["controls"]
+        [autoplay, loop_attr, controls].compact
+      end
+    end
+
+    register :video, Video.new
+  end
+end

data/lib/coradoc/reverse_adoc/converters.rb

@@ -0,0 +1,53 @@
+module Coradoc::ReverseAdoc
+  module Converters
+    def self.register(tag_name, converter)
+      @@converters ||= {}
+      @@converters[tag_name.to_sym] = converter
+    end
+
+    def self.unregister(tag_name)
+      @@converters.delete(tag_name.to_sym)
+    end
+
+    def self.lookup(tag_name)
+      @@converters[tag_name.to_sym] or default_converter(tag_name)
+    end
+
+    # Note: process won't run plugin hooks
+    def self.process(node, state)
+      node = node.to_a if node.is_a? Nokogiri::XML::NodeSet
+      return node.map { |i| process(i, state) }.join if node.is_a? Array
+
+      lookup(node.name).convert(node, state)
+    end
+
+    def self.process_coradoc(node, state)
+      node = node.to_a if node.is_a? Nokogiri::XML::NodeSet
+      return node.map { |i| process_coradoc(i, state) } if node.is_a? Array
+
+      plugins = state[:plugin_instances] || {}
+      process = proc { lookup(node.name).to_coradoc(node, state) }
+      plugins.each do |i|
+        prev_process = process
+        process = proc { i.html_tree_run_hooks(node, state, &prev_process) }
+      end
+      process.(node, state)
+    end
+
+    def self.default_converter(tag_name)
+      case Coradoc::ReverseAdoc.config.unknown_tags.to_sym
+      when :pass_through
+        Coradoc::ReverseAdoc::Converters::PassThrough.new
+      when :drop
+        Coradoc::ReverseAdoc::Converters::Drop.new
+      when :bypass
+        Coradoc::ReverseAdoc::Converters::Bypass.new
+      when :raise
+        raise UnknownTagError, "unknown tag: #{tag_name}"
+      else
+        raise InvalidConfigurationError,
+              "unknown value #{Coradoc::ReverseAdoc.config.unknown_tags.inspect} for Coradoc::ReverseAdoc.config.unknown_tags"
+      end
+    end
+  end
+end

data/lib/coradoc/reverse_adoc/errors.rb

@@ -0,0 +1,10 @@
+module Coradoc::ReverseAdoc
+  class Error < StandardError
+  end
+
+  class UnknownTagError < Error
+  end
+
+  class InvalidConfigurationError < Error
+  end
+end

data/lib/coradoc/reverse_adoc/html_converter.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require_relative "converters/markup"
+require_relative "converters/a"
+require_relative "converters/aside"
+require_relative "converters/audio"
+require_relative "converters/blockquote"
+require_relative "converters/br"
+require_relative "converters/bypass"
+require_relative "converters/code"
+require_relative "converters/div"
+require_relative "converters/dl"
+require_relative "converters/drop"
+require_relative "converters/em"
+require_relative "converters/figure"
+require_relative "converters/h"
+require_relative "converters/head"
+require_relative "converters/hr"
+require_relative "converters/ignore"
+require_relative "converters/img"
+require_relative "converters/mark"
+require_relative "converters/li"
+require_relative "converters/ol"
+require_relative "converters/p"
+require_relative "converters/pass_through"
+require_relative "converters/pre"
+require_relative "converters/q"
+require_relative "converters/strong"
+require_relative "converters/sup"
+require_relative "converters/sub"
+require_relative "converters/table"
+require_relative "converters/td"
+require_relative "converters/th"
+require_relative "converters/text"
+require_relative "converters/tr"
+require_relative "converters/video"
+require_relative "converters/math"
+
+module Coradoc
+  module ReverseAdoc
+    class HtmlConverter
+      def self.to_coradoc(input, options = {})
+        plugin_instances = options.delete(:plugin_instances)
+        ReverseAdoc.config.with(options) do
+          plugin_instances ||= Coradoc::ReverseAdoc.config.plugins.map(&:new)
+
+          root = track_time "Loading input HTML document" do
+            case input
+            when String
+              Nokogiri::HTML(input).root
+            when Nokogiri::XML::Document
+              input.root
+            when Nokogiri::XML::Node
+              input
+            end
+          end
+
+          return "" unless root
+
+          plugin_instances.each do |plugin|
+            plugin.html_tree = root
+            if plugin.respond_to?(:preprocess_html_tree)
+              track_time "Preprocessing document with #{plugin.name} plugin" do
+                plugin.preprocess_html_tree
+              end
+            end
+            root = plugin.html_tree
+          end
+
+          coradoc = track_time "Converting input document tree to Coradoc tree" do
+            Converters.process_coradoc(root, plugin_instances: plugin_instances)
+          end
+
+          coradoc = track_time "Post-process Coradoc tree" do
+            Postprocessor.process(coradoc)
+          end
+
+          plugin_instances.each do |plugin|
+            if plugin.respond_to?(:postprocess_coradoc_tree)
+              plugin.coradoc_tree = coradoc
+              track_time "Postprocessing Coradoc tree with #{plugin.name} plugin" do
+                plugin.postprocess_coradoc_tree
+              end
+              coradoc = plugin.coradoc_tree
+            end
+          end
+
+          coradoc
+        end
+      end
+
+      def self.convert(input, options = {})
+        ReverseAdoc.config.with(options) do
+          plugin_instances = Coradoc::ReverseAdoc.config.plugins.map(&:new)
+
+          options = options.merge(plugin_instances: plugin_instances)
+
+          coradoc = to_coradoc(input, options)
+
+          if coradoc.is_a?(Hash)
+            coradoc.to_h do |file, tree|
+              track_time "Converting file #{file || 'main'}" do
+                [file, convert_single_coradoc_to_adoc(file, tree, plugin_instances)]
+              end
+            end
+          else
+            convert_single_coradoc_to_adoc(nil, coradoc, plugin_instances)
+          end
+        end
+      end
+
+      def self.convert_single_coradoc_to_adoc(_file, coradoc, plugin_instances)
+        result = track_time "Converting Coradoc tree into Asciidoc" do
+          Coradoc::Generator.gen_adoc(coradoc)
+        end
+        result = track_time "Cleaning up the result" do
+          ReverseAdoc.cleaner.tidy(result)
+        end
+        plugin_instances.each do |plugin|
+          if plugin.respond_to?(:postprocess_asciidoc_string)
+            plugin.asciidoc_string = result
+            track_time "Postprocessing AsciiDoc string with #{plugin.name} plugin" do
+              plugin.postprocess_asciidoc_string
+            end
+            result = plugin.asciidoc_string
+          end
+        end
+        result
+      end
+
+      @track_time_indentation = 0
+      def self.track_time(task)
+        if ReverseAdoc.config.track_time
+          warn "  " * @track_time_indentation +
+            "* #{task} is starting..."
+          @track_time_indentation += 1
+          t0 = Time.now
+          ret = yield
+          time_elapsed = Time.now - t0
+          @track_time_indentation -= 1
+          warn "  " * @track_time_indentation +
+            "* #{task} took #{time_elapsed.round(3)} seconds"
+          ret
+        else
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/coradoc/reverse_adoc/plugin.rb

@@ -0,0 +1,131 @@
+module Coradoc::ReverseAdoc
+  class Plugin
+    #### Plugin system general
+
+    # Allow building plugins with a shorthand syntax:
+    # plugin = Coradoc::ReverseAdoc::Plugin.new do
+    #   def name = "Test"
+    # end
+
+    def self.new(&block)
+      if self == Plugin
+        Class.new(Plugin, &block)
+      else
+        super
+      end
+    end
+
+    def initialize
+      @html_tree_hooks_pre = {}
+      @html_tree_hooks_post = {}
+    end
+
+    # define name to name a Plugin
+    def name
+      self.class.name
+    end
+
+    #### HTML Tree functionalities
+
+    attr_accessor :html_tree
+
+    def html_tree_change_tag_name_by_css(css, new_name)
+      html_tree.css(css).each do |e|
+        e.name = new_name
+      end
+    end
+
+    def html_tree_change_properties_by_css(css, properties)
+      html_tree.css(css).each do |e|
+        properties.each do |k,v|
+          e[k.to_s] = v
+        end
+      end
+    end
+
+    def html_tree_remove_by_css(css)
+      html_tree.css(css).each(&:remove)
+    end
+
+    def html_tree_replace_with_children_by_css(css)
+      html_tree.css(css).each do |e|
+        e.replace(e.children)
+      end
+    end
+
+    def html_tree_process_to_coradoc(tree, state={})
+      Coradoc::ReverseAdoc::Converters.process_coradoc(tree, state)
+    end
+
+    def html_tree_process_to_adoc(tree, state={})
+      Coradoc::ReverseAdoc::Converters.process(tree, state)
+    end
+
+    def html_tree_preview
+      Tempfile.open(%w"coradoc .html") do |i|
+        i << html_tree.to_html
+        system "chromium-browser", "--no-sandbox", i.path
+      end
+    end
+
+    # define preprocess_html_tree to process HTML trees
+
+    # Creates a hook to be called instead of converting an element
+    # to a Coradoc node.
+    #
+    # proc |html_node, state|
+    #   coradoc_node
+    # end
+    def html_tree_add_hook_pre(element, &block)
+      @html_tree_hooks_pre[element] = block
+    end
+
+    def html_tree_add_hook_pre_by_css(css, &block)
+      html_tree.css(css).each do |e|
+        html_tree_add_hook_pre(e, &block)
+      end
+    end
+
+    # Creates a hook to be called after converting an element
+    # to a Coradoc node.
+    #
+    # proc |html_node, coradoc_node, state|
+    #   coradoc_node
+    # end
+    def html_tree_add_hook_post(element, &block)
+      @html_tree_hooks_post[element] = block
+    end
+
+    def html_tree_add_hook_post_by_css(css, &block)
+      html_tree.css(css).each do |e|
+        html_tree_add_hook_post(e, &block)
+      end
+    end
+
+    def html_tree_run_hooks(node, state, &_block)
+      hook_pre = @html_tree_hooks_pre[node]
+      hook_post = @html_tree_hooks_post[node]
+
+      coradoc = hook_pre.(node, state) if hook_pre
+      coradoc ||= yield node, state
+
+      if hook_post
+        coradoc = hook_post.(node, coradoc, state)
+      end
+
+      coradoc
+    end
+
+    #### Coradoc tree functionalities
+
+    attr_accessor :coradoc_tree
+
+    # define postprocess_coradoc_tree to change coradoc tree
+
+    #### AsciiDoc string functionalities
+
+    attr_accessor :asciidoc_string
+
+    # define postprocess_asciidoc_string to change the coradoc string
+  end
+end

data/lib/coradoc/reverse_adoc/plugins/plateau.rb

@@ -0,0 +1,174 @@
+module Coradoc::ReverseAdoc
+  class Plugin
+    # This plugin enhances documents from the PLATEAU project
+    # to extract more data.
+    #
+    # Usage:
+    # reverse_adoc -rcoradoc/reverse_adoc/plugins/plateau
+    #   --external-images -u raise --output _out/index.adoc index.html
+    class Plateau < Plugin
+      def name
+        "PLATEAU"
+      end
+
+      def preprocess_html_tree
+        # Let's simplify the tree by removing what's extraneous
+        # html_tree_remove_by_css("script, style, img.container_imagebox:not([src])")
+        # html_tree_replace_with_children_by_css("div.container_box")
+        # html_tree_replace_with_children_by_css("div.col.col-12")
+        # html_tree_replace_with_children_by_css(".tabledatatext, .tabledatatextY")
+        # html_tree_replace_with_children_by_css("div.row")
+        #
+        # We can remove that, but it messes up the images and paragraphs.
+
+        # Remove side menu, so we can generate TOC ourselves
+        html_tree_remove_by_css(".sideMenu")
+
+        # Correct non-semantic classes into semantic HTML tags
+        html_tree_change_tag_name_by_css(".titledata", "h1")
+        html_tree_change_tag_name_by_css(".subtitledata", "h2")
+        html_tree_change_tag_name_by_css(".pitemdata", "h3")
+        html_tree_change_tag_name_by_css(".sitemdata", "h4")
+        html_tree_change_tag_name_by_css('td[bgcolor="#D0CECE"]', "th")
+
+        # Remove some CSS ids that are not important to us
+        html_tree_change_properties_by_css("#__nuxt", id: nil)
+        html_tree_change_properties_by_css("#__layout", id: nil)
+        html_tree_change_properties_by_css("#app", id: nil)
+
+        # Convert table/img caption to become a caption
+        html_tree.css(".imagedata").each do |e|
+          table = e.parent.next&.children&.first
+          if table&.name == "table"
+            e.name = "caption"
+            table.prepend_child(e)
+            next
+          end
+
+          img = e.parent.previous&.children&.first
+          if img&.name == "img" && img["src"]
+            title = e.text.strip
+            img["title"] = title
+            e.remove
+            next
+          end
+        end
+
+        # Add hooks for H1, H2, H3, H4
+        html_tree_add_hook_post_by_css("h1, h2, h3", &method(:handle_headers))
+        html_tree_add_hook_post_by_css("h4", &method(:handle_headers_h4))
+
+        # Table cells aligned to center
+        html_tree_change_properties_by_css(".tableTopCenter", align: "center")
+
+        # Handle non-semantic lists and indentation
+        html_tree_add_hook_pre_by_css ".text2data" do |node,|
+          text = html_tree_process_to_adoc(node).strip
+          next "" if text.empty? || text == "\u3000"
+
+          if text.start_with?(/\d+\./)
+            text = text.sub(/\A\d+.\s*/, "")
+            ".. #{text}\n"
+          else
+            text = text.gsub(/^/, "** ")
+            "\n\n//-PT2D\n#{text}\n//-ENDPT2D\n\n"
+          end
+        end
+
+        html_tree_add_hook_pre_by_css ".text3data" do |node,|
+          text = html_tree_process_to_adoc(node).strip
+          next "" if text.empty? || text == "\u3000"
+
+          text = text.strip.gsub(/^/, "*** ")
+          "\n\n//-PT3D\n#{text}\n//-ENDPT3D\n\n"
+        end
+
+        html_tree_add_hook_pre_by_css ".text4data" do |node,|
+          text = html_tree_process_to_adoc(node).strip
+          next "" if text.empty? || text == "\u3000"
+
+          text = text.strip.gsub(/^/, "**** ")
+          "\n\n//-PT4D\n#{text}\n//-ENDPT4D\n\n"
+        end
+
+        html_tree_add_hook_pre_by_css ".text2data_point ul" do |node,|
+          text = html_tree_process_to_adoc(node.children.first.children).strip
+
+          "** #{text}\n"
+        end
+
+        html_tree_add_hook_pre_by_css ".text3data_point ul" do |node,|
+          text = html_tree_process_to_adoc(node.children.first.children).strip
+
+          "*** #{text}\n"
+        end
+
+        # html_tree_preview
+      end
+
+      def handle_headers(node, coradoc, state)
+        if coradoc.id.start_with?("toc0_")
+          content = coradoc.content.map(&:content).join
+          # Special content
+          case content.strip
+          when "はじめに" # Introduction
+            coradoc.style = "abstract" # The older version document has ".preface"
+          when "改定の概要" # Revision overview
+            coradoc.style = "abstract" # The older version document has ".preface"
+          when "参考文献" # Bibliography
+            coradoc.style = "bibliography"
+          when "改訂履歴" # Document history
+            coradoc.style = "appendix"
+          else
+            warn "Unknown section #{coradoc.content.map(&:content).join.inspect}"
+          end
+
+          # Ensure they are generated as level 1
+          coradoc.level_int = 1
+        end
+
+        # Remove numbers
+        coradoc.content.first.content.sub!(/\A[\d\s.]+/, "")
+
+        coradoc
+      end
+
+      def handle_headers_h4(node, coradoc, state)
+        case coradoc.content.first.content
+        when /\A\(\d+\)(.*)/
+          coradoc.level_int = 4
+          coradoc.content.first.content = $1.strip
+          coradoc
+        when /\A\d+\)(.*)/
+          coradoc.level_int = 5
+          coradoc.content.first.content = $1.strip
+          coradoc
+        else
+          ["// FIXME\n", coradoc]
+        end
+      end
+
+      def postprocess_asciidoc_string
+        str = self.asciidoc_string
+
+        ### Custom indentation handling
+        # If there's a step up, add [none]
+        str = str.gsub(%r{\s+//-ENDPT2D\s+//-PT3D\s+}, "\n[none]\n")
+        str = str.gsub(%r{\s+//-ENDPT2D\s+//-PT4D\s+}, "\n[none]\n")
+        str = str.gsub(%r{\s+//-ENDPT3D\s+//-PT4D\s+}, "\n[none]\n")
+        # Collapse blocks of text[2,3]data
+        str = str.gsub(%r{\s+//-ENDPT[234]D\s+//-PT[234]D\s+}, "\n\n")
+        # In the beginning, add [none]
+        str = str.gsub(%r{\s+//-PT[234]D\s+}, "\n\n[none]\n")
+        # If following with another list, ensure we readd styling
+        str = str.gsub(%r{\s+//-ENDPT[234]D\s+\*}, "\n\n[disc]\n*")
+        # Otherwise, clean up
+        str = str.gsub(%r{\s+//-ENDPT[234]D\s+}, "\n\n")
+
+        self.asciidoc_string = str
+      end
+    end
+  end
+end
+
+Coradoc::ReverseAdoc.config.plugins << Coradoc::ReverseAdoc::Plugin::Plateau

data/lib/coradoc/reverse_adoc/postprocessor.rb

@@ -0,0 +1,148 @@
+module Coradoc::ReverseAdoc
+  # Postprocessor's aim is to convert a Coradoc tree from
+  # a mess that has been created from HTML into a tree that
+  # is compatible with what we would get out of Coradoc, if
+  # it parsed it directly.
+  class Postprocessor
+    def self.process(coradoc)
+      new(coradoc).process
+    end
+
+    def initialize(coradoc)
+      @tree = coradoc
+    end
+
+    # Collapse DIVs that only have a title, or nest another DIV.
+    def collapse_meaningless_sections
+      @tree = Coradoc::Element::Base.visit(@tree) do |elem, _dir|
+        if elem.is_a?(Coradoc::Element::Section) && elem.safe_to_collapse?
+          children_classes = Array(elem.contents).map(&:class)
+          count = children_classes.length
+          safe_classes = [Coradoc::Element::Section, Coradoc::Element::Title]
+
+          # Count > 0 because some documents use <div> as a <br>.
+          if count > 0 && children_classes.all? { |i| safe_classes.include?(i) }
+            next elem.contents
+          end
+        end
+        elem
+      end
+    end
+
+    # tree should now be more cleaned up, so we can progress with
+    # creating meaningful sections
+    def generate_meaningful_sections
+      @tree = Coradoc::Element::Base.visit(@tree) do |elem, dir|
+        # We are searching for an array, that has a title. This
+        # will be a candidate for our section array.
+        if dir == :post &&
+            elem.is_a?(Array) &&
+            !elem.grep(Coradoc::Element::Title).empty?
+
+          new_array = []
+          content_array = new_array
+          section_arrays_by_level = [new_array] * 8
+
+          # For each title element, we create a new section. Then we push
+          # all descendant sections into those sections. Otherwise, we push
+          # an element as content of current section.
+          elem.each do |e|
+            if e.is_a? Coradoc::Element::Title
+              title = e
+              content_array = []
+              section_array = []
+              level = title.level_int
+              section = Coradoc::Element::Section.new(
+                title, contents: content_array, sections: section_array
+              )
+              # Some documents may not be consistent and eg. follow H4 after
+              # H2. Let's ensure that proceeding sections will land in a
+              # correct place.
+              (8 - level).times do |j|
+                section_arrays_by_level[level + j] = section_array
+              end
+              section_arrays_by_level[level - 1] << section
+            else
+              content_array << e
+            end
+          end
+          next new_array
+        end
+        elem
+      end
+    end
+
+    def split_sections
+      max_level = Coradoc::ReverseAdoc.config.split_sections
+
+      return unless max_level
+
+      sections = {}
+      parent_sections = []
+      previous_sections = {}
+
+      determine_section_id = ->(elem) do
+        level = 0
+        section = elem
+        while section
+          level += 1 if elem.title.style == section.title.style
+          section = previous_sections[section]
+        end
+        level
+      end
+
+      determine_style = ->(elem) do
+        style = elem.title.style || "section"
+        style += "-"
+        style
+      end
+
+      @tree = Coradoc::Element::Base.visit(@tree) do |elem, dir|
+        title = elem.title if elem.is_a?(Coradoc::Element::Section)
+
+        if title && title.level_int <= max_level
+          if dir == :pre
+            # In the PRE pass, we build a tree of sections, so that
+            # we can compute numbers
+            previous_sections[elem] = parent_sections[title.level_int]
+            parent_sections[title.level_int] = elem
+            parent_sections[(title.level_int+1)..nil] = nil
+
+            elem
+          else
+            # In the POST pass, we replace the sections with their
+            # include tag.
+            section_file = "sections/"
+            section_file += parent_sections[1..title.level_int].map do |parent|
+              style = determine_style.(parent)
+              "%s%02d" % [style, determine_section_id.(parent)]
+            end.join("/")
+            section_file += ".adoc"
+
+            sections[section_file] = elem
+            up = "../" * (title.level_int - 1)
+            "\ninclude::#{up}#{section_file}[]\n"
+          end
+        else
+          elem
+        end
+      end
+
+      sections[nil] = @tree
+      @tree = sections
+    end
+
+    def process
+      collapse_meaningless_sections
+      generate_meaningful_sections
+      # Do it again to simplify the document further.
+      # Since the structure is changed, we may have new meaningful
+      # sections as only children of some meaningless sections.
+      collapse_meaningless_sections
+
+      split_sections
+
+      @tree
+    end
+  end
+end