coradoc 1.1.7 → 2.0.2

data/lib/coradoc/input/html/plugins/plateau.rb

@@ -1,213 +0,0 @@
-module Coradoc
-  module Input
-    module Html
-      class Plugin
-        # This plugin enhances documents from the PLATEAU project
-        # to extract more data.
-        #
-        # Usage:
-        # reverse_adoc -rcoradoc/input/html/plugins/plateau
-        #   --external-images -u raise --output _out/index.adoc index.html
-        class Plateau < Plugin
-          def name
-            "PLATEAU"
-          end
-
-          def preprocess_html_tree
-            # Let's simplify the tree by removing what's extraneous
-            # html_tree_remove_by_css("script, style, img.container_imagebox:not([src])")
-            # html_tree_replace_with_children_by_css("div.container_box")
-            # html_tree_replace_with_children_by_css("div.col.col-12")
-            # html_tree_replace_with_children_by_css(".tabledatatext, .tabledatatextY")
-            # html_tree_replace_with_children_by_css("div.row")
-            #
-            # We can remove that, but it messes up the images and paragraphs.
-
-            # Remove side menu, so we can generate TOC ourselves
-            html_tree_remove_by_css(".sideMenu")
-
-            # Correct non-semantic classes into semantic HTML tags
-            html_tree_change_tag_name_by_css(".titledata", "h1")
-            html_tree_change_tag_name_by_css(".subtitledata", "h2")
-            html_tree_change_tag_name_by_css(".pitemdata", "h3")
-            html_tree_change_tag_name_by_css(".sitemdata", "h4")
-            html_tree_change_tag_name_by_css('td[bgcolor="#D0CECE"]', "th")
-            html_tree_change_tag_name_by_css('td[bgcolor="#d0cece"]', "th")
-            html_tree_change_tag_name_by_css(".framedata, .frame_container_box",
-                                             "aside")
-            html_tree_change_tag_name_by_css(".frame2data", "pre")
-            # Assumption that all code snippets in those documents are XML...
-            html_tree_change_properties_by_css(".frame2data",
-                                               class: "brush:xml;")
-
-            # Remove some CSS ids that are not important to us
-            html_tree_change_properties_by_css("#__nuxt", id: nil)
-            html_tree_change_properties_by_css("#__layout", id: nil)
-            html_tree_change_properties_by_css("#app", id: nil)
-
-            # Handle lists of document 02
-            html_tree_replace_with_children_by_css(".list_num-wrap")
-
-            # Convert table/img caption to become a caption
-            html_tree.css(".imagedata").each do |e|
-              table = e.parent.next&.children&.first
-              if table&.name == "table"
-                e.name = "caption"
-                table.prepend_child(e)
-                next
-              end
-
-              img = e.parent.previous&.children&.first
-              if img&.name == "img" && img["src"]
-                title = e.text.strip
-                img["title"] = title
-                e.remove
-                next
-              end
-            end
-
-            # Add hooks for H1, H2, H3, H4
-            html_tree_add_hook_post_by_css("h1, h2, h3",
-                                           &method(:handle_headers))
-            html_tree_add_hook_post_by_css("h4", &method(:handle_headers_h4))
-
-            # Table cells aligned to center
-            html_tree_change_properties_by_css(".tableTopCenter",
-                                               align: "center")
-
-            # Handle non-semantic lists and indentation
-            html_tree_add_hook_pre_by_css ".text2data" do |node,|
-              text = html_tree_process_to_adoc(node).strip
-              next "" if text.empty? || text == "\u3000"
-
-              if text.start_with?(/\d+\./)
-                text = text.sub(/\A\d+.\s*/, "")
-                ".. #{text}\n"
-              else
-                text = text.gsub(/^/, "** ")
-                "\n\n//-PT2D\n#{text}\n//-ENDPT2D\n\n"
-              end
-            end
-
-            (3..4).each do |i|
-              html_tree_add_hook_pre_by_css ".text#{i}data" do |node,|
-                text = html_tree_process_to_adoc(node).strip
-                next "" if text.empty? || text == "\u3000"
-
-                text = text.strip.gsub(/^/, "#{'*' * i} ")
-                "\n\n//-PT#{i}D\n#{text}\n//-ENDPT#{i}D\n\n"
-              end
-            end
-
-            (2..3).each do |i|
-              html_tree_add_hook_pre_by_css ".text#{i}data_point ul" do |node,|
-                text = html_tree_process_to_adoc(node.children.first.children).strip
-
-                "#{'*' * i} #{text}\n"
-              end
-            end
-
-            (1..20).each do |i|
-              html_tree_add_hook_pre_by_css ".numtextdata_num .list_num#{i}" do |node,|
-                text = html_tree_process_to_adoc(node).strip
-
-                "[start=#{i}]\n. #{text}\n"
-              end
-            end
-
-            # html_tree_preview
-          end
-
-          IM = /[A-Z0-9]{1,3}/
-
-          def handle_headers(node, coradoc, _state)
-            content = coradoc.content.map(&:content).join
-
-            if %w[toc0 toc_0].any? { |i| coradoc.id&.start_with?(i) }
-              # Special content
-              case content.strip
-              when "はじめに" # Introduction
-                coradoc.style = "abstract" # The older version document has ".preface"
-                coradoc.level_int = 1
-              when "改定の概要" # Revision overview
-                coradoc.style = "abstract" # The older version document has ".preface"
-                coradoc.level_int = 1
-              when "参考文献" # Bibliography
-                coradoc.style = "bibliography"
-                coradoc.level_int = 1
-              when "改訂履歴" # Document history
-                coradoc.style = "appendix"
-                coradoc.level_int = 1
-              when "0　概要" # Overview
-                coradoc.style = "abstract" # I'm not sure this is correct
-                coradoc.level_int = 1
-              when "索引" # Index
-                coradoc.style = "index" # I'm not sure this is correct
-                coradoc.level_int = 1
-              else
-                warn "Unknown section #{content.inspect}"
-              end
-            end
-
-            if node.name == "h1" && content.start_with?("Annex")
-              coradoc.style = "appendix"
-              coradoc.content.first.content.sub!(/\AAnnex [A-Z]/, "")
-            end
-
-            # Remove numbers
-            coradoc.content.first.content.sub!(/\A(#{IM}\.)*#{IM}[[:space:]]/o,
-                                               "")
-
-            coradoc
-          end
-
-          def handle_headers_h4(_node, coradoc, _state)
-            title = Coradoc.strip_unicode(coradoc.content.first.content)
-            case title
-            when /\A\(\d+\)(.*)/
-              coradoc.level_int = 4
-              coradoc.content.first.content = $1.strip
-              coradoc
-            when /\A\d+\)(.*)/
-              coradoc.level_int = 5
-              coradoc.content.first.content = $1.strip
-              coradoc
-            when /\A#{IM}\.#{IM}\.#{IM}\.#{IM}(.*)/o
-              coradoc.level_int = 4
-              coradoc.content.first.content = $1.strip
-            else
-              if title.empty?
-                # Strip instances of faulty empty paragraphs
-                nil
-              else
-                ["// FIXME\n", coradoc]
-              end
-            end
-          end
-
-          def postprocess_asciidoc_string
-            str = asciidoc_string
-
-            ### Custom indentation handling
-            # If there's a step up, add [none]
-            str = str.gsub(%r{\s+//-ENDPT2D\s+//-PT3D\s+}, "\n[none]\n")
-            str = str.gsub(%r{\s+//-ENDPT2D\s+//-PT4D\s+}, "\n[none]\n")
-            str = str.gsub(%r{\s+//-ENDPT3D\s+//-PT4D\s+}, "\n[none]\n")
-            # Collapse blocks of text[2,3]data
-            str = str.gsub(%r{\s+//-ENDPT[234]D\s+//-PT[234]D\s+}, "\n\n")
-            # In the beginning, add [none]
-            str = str.gsub(%r{\s+//-PT[234]D\s+}, "\n\n[none]\n")
-            # If following with another list, ensure we readd styling
-            str = str.gsub(%r{\s+//-ENDPT[234]D\s+\*}, "\n\n[disc]\n*")
-            # Otherwise, clean up
-            str = str.gsub(%r{\s+//-ENDPT[234]D\s+}, "\n\n")
-
-            self.asciidoc_string = str
-          end
-        end
-      end
-    end
-  end
-end
-
-Coradoc::Input::Html.config.plugins << Coradoc::Input::Html::Plugin::Plateau

data/lib/coradoc/input/html/postprocessor.rb

@@ -1,220 +0,0 @@
-module Coradoc
-  module Input
-    module Html
-      # Postprocessor's aim is to convert a Coradoc tree from
-      # a mess that has been created from HTML into a tree that
-      # is compatible with what we would get out of Coradoc, if
-      # it parsed it directly.
-      class Postprocessor
-        Element = Coradoc::Element
-
-        def self.process(coradoc)
-          new(coradoc).process
-        end
-
-        def initialize(coradoc)
-          @tree = coradoc
-        end
-
-        # Extracts titles from lists. This happens in HTML files
-        # generated from DOCX documents by LibreOffice.
-        #
-        # We are interested in a particular tree:
-        # Element::List::Ordered items:
-        #   Element::List::Ordered items: (any depth)
-        #     Element::ListItem content:
-        #       Element::Title
-        #       (any number of other titles of the same scheme)
-        #
-        # This tree is flattened into:
-        # Element::Title
-        # Element::Title (any number of titles)
-        def extract_titles_from_lists
-          @tree = Element::Base.visit(@tree) do |elem, dir|
-            next elem unless dir == :pre
-            next elem unless elem.is_a?(Element::List::Ordered)
-            next elem if elem.items.length != 1
-
-            anchors = []
-            anchors << elem.anchor if elem.anchor
-
-            # Extract ListItem from any depth of List::Ordered
-            processed = elem
-            while processed.is_a?(Element::List::Ordered)
-              if processed.items.length != 1
-                backtrack = true
-                break
-              end
-              anchors << processed.anchor if processed.anchor
-              processed = processed.items.first
-            end
-
-            # Something went wrong? Anything not matching on the way?
-            next elem if backtrack
-            next elem unless processed.is_a?(Element::ListItem)
-
-            anchors << processed.anchor if processed.anchor
-
-            # Now we must have a title (or titles).
-            titles = processed.content.flatten
-
-            # Don't bother if there's no title in there.
-            next elem unless titles.any? { |i| i.is_a? Element::Title }
-
-            # Ordered is another iteration for our cleanup.
-            next elem unless titles.all? do |i|
-              i.is_a?(Element::Title) || i.is_a?(Element::List::Ordered)
-            end
-
-            # We are done now.
-            titles + anchors
-          end
-        end
-
-        # Collapse DIVs that only have a title, or nest another DIV.
-        def collapse_meaningless_sections
-          @tree = Element::Base.visit(@tree) do |elem, _dir|
-            if elem.is_a?(Element::Section) && elem.safe_to_collapse?
-              children_classes = Array(elem.contents).map(&:class)
-              count = children_classes.length
-              safe_classes = [Element::Section, Element::Title]
-
-              # Count > 0 because some documents use <div> as a <br>.
-              if count.positive? && children_classes.all? do |i|
-                safe_classes.include?(i)
-              end
-                contents = elem.contents.dup
-                contents.prepend(elem.anchor) if elem.anchor
-                next contents
-              end
-            end
-            elem
-          end
-        end
-
-        # tree should now be more cleaned up, so we can progress with
-        # creating meaningful sections
-        def generate_meaningful_sections
-          @tree = Element::Base.visit(@tree) do |elem, dir|
-            # We are searching for an array, that has a title. This
-            # will be a candidate for our section array.
-            if dir == :post &&
-                elem.is_a?(Array) &&
-                !elem.flatten.grep(Element::Title).empty?
-
-              elem = elem.flatten
-
-              new_array = []
-              content_array = new_array
-              section_arrays_by_level = [new_array] * 8
-
-              # For each title element, we create a new section. Then we push
-              # all descendant sections into those sections. Otherwise, we push
-              # an element as content of current section.
-              elem.each do |e|
-                if e.is_a? Element::Title
-                  title = e
-                  content_array = []
-                  section_array = []
-                  level = title.level_int
-                  section = Element::Section.new(
-                    title, contents: content_array, sections: section_array
-                  )
-                  # Some documents may not be consistent and eg. follow H4 after
-                  # H2. Let's ensure that proceeding sections will land in a
-                  # correct place.
-                  (8 - level).times do |j|
-                    section_arrays_by_level[level + j] = section_array
-                  end
-                  section_arrays_by_level[level - 1] << section
-                else
-                  content_array << e
-                end
-              end
-              next new_array
-            end
-            elem
-          end
-        end
-
-        def split_sections
-          max_level = Coradoc::Input::Html.config.split_sections
-
-          return unless max_level
-
-          sections = {}
-          parent_sections = []
-          previous_sections = {}
-
-          determine_section_id = ->(elem) do
-            level = if elem.title.style == "appendix"
-                      "A"
-                    else
-                      1
-                    end
-
-            section = previous_sections[elem]
-            while section
-              level = level.succ if elem.title.style == section.title.style
-              section = previous_sections[section]
-            end
-            level.is_a?(Integer) ? "%02d" % level : level
-          end
-
-          determine_style = ->(elem) do
-            style = elem.title.style || "section"
-            style += "-"
-            style
-          end
-
-          @tree = Element::Base.visit(@tree) do |elem, dir|
-            title = elem.title if elem.is_a?(Element::Section)
-
-            if title && title.level_int <= max_level
-              if dir == :pre
-                # In the PRE pass, we build a tree of sections, so that
-                # we can compute numbers
-                previous_sections[elem] = parent_sections[title.level_int]
-                parent_sections[title.level_int] = elem
-                parent_sections[(title.level_int + 1)..nil] = nil
-
-                elem
-              else
-                # In the POST pass, we replace the sections with their
-                # include tag.
-                section_file = "sections/"
-                section_file += parent_sections[1..title.level_int].map do |parent|
-                  determine_style.(parent) + determine_section_id.(parent)
-                end.join("/")
-                section_file += ".adoc"
-
-                sections[section_file] = elem
-                up = "../" * (title.level_int - 1)
-                "\ninclude::#{up}#{section_file}[]\n"
-              end
-            else
-              elem
-            end
-          end
-
-          sections[nil] = @tree
-          @tree = sections
-        end
-
-        def process
-          extract_titles_from_lists
-          collapse_meaningless_sections
-          generate_meaningful_sections
-          # Do it again to simplify the document further.
-          # Since the structure is changed, we may have new meaningful
-          # sections as only children of some meaningless sections.
-          collapse_meaningless_sections
-
-          split_sections
-
-          @tree
-        end
-      end
-    end
-  end
-end

data/lib/coradoc/input/html.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-require "digest"
-require "nokogiri"
-require "coradoc/input"
-require_relative "html/errors"
-require_relative "html/cleaner"
-require_relative "html/config"
-require_relative "html/converters"
-require_relative "html/converters/base"
-require_relative "html/html_converter"
-require_relative "html/plugin"
-require_relative "html/postprocessor"
-
-module Coradoc
-  module Input
-    module Html
-      def self.convert(input, options = {})
-        Coradoc::Input::Html::HtmlConverter.convert(input, options)
-      end
-
-      def self.to_coradoc(input, options = {})
-        Input::Html::HtmlConverter.to_coradoc(input, options)
-      end
-
-      def self.config
-        @config ||= Config.new
-        yield @config if block_given?
-        @config
-      end
-
-      def self.cleaner
-        @cleaner ||= Cleaner.new
-      end
-
-      def self.processor_id
-        :html
-      end
-
-      def self.processor_match?(filename)
-        %w[.html .htm].any? { |i| filename.downcase.end_with?(i) }
-      end
-
-      def self.processor_execute(input, options = {})
-        to_coradoc(input, options)
-      end
-
-      def self.processor_postprocess(data, options)
-        if options[:output_processor] == :adoc
-          data.transform_values do |v|
-            Input::Html::HtmlConverter.cleanup_result(v, options)
-          end
-        else
-          data
-        end
-      end
-
-      Coradoc::Input.define(self)
-    end
-  end
-end

data/lib/coradoc/legacy_parser.rb

@@ -1,200 +0,0 @@
-require "parslet"
-require "parslet/convenience"
-
-module Coradoc
-  class LegacyParser < Parslet::Parser
-    root :document
-
-    # Basic Elements
-    rule(:space) { match('\s') }
-    rule(:space?) { spaces.maybe }
-    rule(:spaces) { space.repeat(1) }
-    rule(:empty_line) { match("^\n") }
-
-    rule(:endline) { newline | any.absent? }
-    rule(:newline) { match["\r\n"].repeat(1) }
-    rule(:line_ending) { match("[\n]") }
-
-    rule(:inline_element) { text }
-    rule(:text) { match("[^\n]").repeat(1) }
-    rule(:digits) { match("[0-9]").repeat(1) }
-    rule(:word) { match("[a-zA-Z0-9_-]").repeat(1) }
-    rule(:special_character) { match("^[*_:=-]") | str("[#") }
-
-    rule(:text_line) do
-      special_character.absent? >>
-        match("[^\n]").repeat(1).as(:text) >>
-        line_ending.as(:break)
-    end
-
-    # Common Helpers
-    rule(:words) { word >> (space? >> word).repeat }
-    rule(:email) { word >> str("@") >> word >> str(".") >> word }
-
-    # Document
-    rule(:document) do
-      (
-        document_attributes.repeat(1).as(:document_attributes) |
-        section.as(:section) |
-        header.as(:header) |
-        block_with_title.as(:block) |
-        empty_line.repeat(1) |
-        any.as(:unparsed)
-      ).repeat(1).as(:document)
-    end
-
-    # Header
-    rule(:header) do
-      match("=") >> space? >> text.as(:title) >> newline >>
-        author.maybe.as(:author) >> revision.maybe.as(:revision)
-    end
-
-    rule(:author) do
-      words.as(:first_name) >> str(",") >> space? >> words.as(:last_name) >>
-        space? >> str("<") >> email.as(:email) >> str(">") >> endline
-    end
-
-    rule(:revision) do
-      (word >> (str(".") >> word).maybe).as(:number) >>
-        str(",") >> space? >> word.as(:date) >>
-        str(":") >> space? >> words.as(:remark) >> newline
-    end
-
-    # DocumentAttributes
-    rule(:document_attributes) do
-      str(":") >> attribute_name.as(:key) >> str(":") >>
-        space? >> attribute_value.as(:value) >> endline
-    end
-
-    # Section
-    rule(:section) do
-      heading.as(:title) >>
-        (list.as(:list) |
-         blocks.as(:blocks) |
-         paragraphs.as(:paragraphs)).maybe
-    end
-
-    # Heading
-    rule(:heading) do
-      (anchor_name >> newline).maybe >>
-        match("=").repeat(2, 8).as(:level) >>
-        space? >> text.as(:text) >> endline.as(:break)
-    end
-
-    rule(:anchor_name) { str("[#") >> keyword.as(:name) >> str("]") }
-
-    # List
-    rule(:list) do
-      unordered_list.as(:unordered) |
-        definition_list.as(:definition) | ordered_list.as(:ordered)
-    end
-
-    rule(:ordered_list) { olist_item.repeat(1) }
-    rule(:unordered_list) { ulist_item.repeat(1) }
-    rule(:definition_list) { dlist_item.repeat(1) }
-
-    rule(:olist_item) { match("\.") >> space >> text_line }
-    rule(:ulist_item) { match("\\*") >> space >> text_line }
-    rule(:dlist_item) do
-      str("term") >> space >> digits >> str("::") >> space >> text_line
-    end
-
-    # Block
-    rule(:block) { simple_block | open_block }
-    rule(:attribute_name) { keyword }
-    rule(:attribute_value) { text | str("") }
-    rule(:keyword) { match("[a-zA-Z0-9_-]").repeat(1) }
-    rule(:blocks) { block.repeat(1) >> (newline >> block.repeat(1)).maybe }
-
-    rule(:block_title) { str(".") >> text.as(:title) >> line_ending }
-    rule(:block_type) { str("[") >> keyword.as(:type) >> str("]") >> newline }
-
-    rule(:block_attribute) do
-      str("[") >> keyword.as(:key) >>
-        str("=") >> keyword.as(:value) >> str("]")
-    end
-
-    rule(:simple_block) do
-      block_attribute.as(:attributes) >> newline >>
-        text_line.repeat(1).as(:lines)
-    end
-
-    rule(:open_block) do
-      block_title >>
-        block_type >>
-        str("--").as(:delimiter) >> newline >>
-        text_line.repeat.as(:lines) >>
-        str("--") >> line_ending
-    end
-
-    rule(:example_block) do
-      block_title >>
-        block_type >>
-        str("====").as(:delimiter) >> newline >>
-        text_line.repeat(1).as(:lines) >>
-        str("====") >> newline
-    end
-
-    rule(:sidebar_block) do
-      block_title >>
-        block_type.maybe >>
-        str("****").as(:delimiter) >> newline >>
-        text_line.repeat(1).as(:lines) >>
-        str("****") >> newline
-    end
-
-    rule(:source_block) do
-      block_title >>
-        str("----").as(:delimiter) >> newline >>
-        text_line.repeat(1).as(:lines) >>
-        str("----") >> newline
-    end
-
-    rule(:quote_block) do
-      block_title >>
-        str("____").as(:delimiter) >> newline >>
-        text_line.repeat.as(:lines) >>
-        str("____") >> newline
-    end
-
-    rule(:block_with_title) do
-      example_block | quote_block |
-        sidebar_block | source_block | open_block |
-        (block_title >> text_line.repeat(1).as(:lines))
-    end
-
-    # Paragraph
-    rule(:paragraphs) do
-      paragraph >> (line_ending.repeat(1) >> paragraph).repeat.maybe
-    end
-
-    rule(:paragraph) { admonitions.repeat(1) | text_line.repeat(1) }
-
-    # Admonition
-    rule(:admonition_type) do
-      (str("NOTE") |
-       str("TIP") |
-       str("EDITOR") |
-       str("DANGER") |
-       str("CAUTION") |
-       str("WARNING") |
-       str("IMPORTANT")).as(:type)
-    end
-
-    rule(:admonitions) { admonition.as(:admonition).repeat(1) }
-    rule(:admonition) { inline_admonition | block_admonition }
-
-    rule(:inline_admonition) do
-      admonition_type >> str(":") >> space? >> text_line >> newline
-    end
-
-    rule(:block_admonition) do
-      str("[") >> admonition_type >> str("]") >> newline >> text_line >> newline
-    end
-
-    def self.parse(filename)
-      content = File.read(filename)
-      new.parse_with_debug(content)
-    end
-  end
-end