coradoc 1.1.7 → 2.0.1

data/coradoc-html/lib/coradoc/html/element_mapping.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Coradoc
+  module Html
+    # Element mapping between CoreModel and HTML elements
+    #
+    # This module provides bidirectional mapping between CoreModel types
+    # and HTML elements for conversion purposes.
+    module ElementMapping
+      class << self
+        # Map CoreModel class to HTML element
+        def model_to_html_element(model_class)
+          model_name = model_class.name.split('::').last.downcase.to_sym
+          MODEL_TO_HTML[model_name] || default_element_for(model_class)
+        end
+
+        # Map HTML element to CoreModel class
+        def html_element_to_model(tag_name, context = {})
+          tag = tag_name.to_s.downcase.to_sym
+          HTML_TO_MODEL[tag] || default_model_for(tag, context)
+        end
+
+        # Get default element for a model class
+        def default_element_for(model_class)
+          if model_class.ancestors.any? { |a| a.name&.include?('InlineElement') }
+            { tag: 'span', semantic: false }
+          else
+            { tag: 'div', semantic: false }
+          end
+        end
+
+        # Get default model for an HTML element
+        def default_model_for(tag, _context)
+          # Return CoreModel types
+          case tag
+          when :p, :div, :section, :article
+            'Coradoc::CoreModel::Block'
+          when :strong, :b, :em, :i, :code
+            'Coradoc::CoreModel::InlineElement'
+          else
+            'Coradoc::CoreModel::InlineElement'
+          end
+        end
+
+        # Mapping from CoreModel types to HTML elements
+        MODEL_TO_HTML = {
+          # Document structure
+          document: { tag: 'article', semantic: true },
+          section: { tag: 'section', semantic: true },
+          header: { tag: 'header', semantic: true },
+          title: { tag: 'h1', semantic: true },
+          structuralelement: { tag: 'section', semantic: true },
+
+          # Block elements
+          paragraph: { tag: 'p', semantic: true },
+          block: { tag: 'div', semantic: true },
+          example: { tag: 'div', class: 'example', semantic: false },
+          annotationblock: { tag: 'aside', semantic: true },
+          quote: { tag: 'blockquote', semantic: true },
+          verse: { tag: 'div', class: 'verse', semantic: false },
+          listing: { tag: 'pre', semantic: true },
+          literal: { tag: 'pre', semantic: true },
+          source: { tag: 'pre', semantic: true },
+          open: { tag: 'div', semantic: false },
+          pass: { tag: 'div', class: 'pass', semantic: false },
+
+          # Inline elements
+          inlineelement: { tag: 'span', semantic: false },
+          bold: { tag: 'strong', semantic: true },
+          italic: { tag: 'em', semantic: true },
+          monospace: { tag: 'code', semantic: true },
+          highlight: { tag: 'mark', semantic: true },
+          superscript: { tag: 'sup', semantic: true },
+          subscript: { tag: 'sub', semantic: true },
+          underline: { tag: 'u', semantic: false },
+          strikethrough: { tag: 'del', semantic: true },
+          smallcaps: { tag: 'span', class: 'small-caps', semantic: false },
+          link: { tag: 'a', semantic: true },
+          anchor: { tag: 'a', semantic: true },
+          xref: { tag: 'a', class: 'xref', semantic: true },
+          quotation: { tag: 'q', semantic: true },
+
+          # Lists
+          listblock: { tag: 'ul', semantic: true },
+          listitem: { tag: 'li', semantic: true },
+          orderedlist: { tag: 'ol', semantic: true },
+          unorderedlist: { tag: 'ul', semantic: true },
+
+          # Tables
+          table: { tag: 'table', semantic: true },
+          tablerow: { tag: 'tr', semantic: true },
+          tablecell: { tag: 'td', semantic: true },
+
+          # Media
+          image: { tag: 'img', semantic: true, self_closing: true },
+          video: { tag: 'video', semantic: true },
+          audio: { tag: 'audio', semantic: true },
+
+          # Other
+          break: { tag: 'hr', semantic: true, self_closing: true },
+          linebreak: { tag: 'br', semantic: true, self_closing: true },
+
+          # Text
+          textelement: { tag: 'text', semantic: false }
+        }.freeze
+
+        # Mapping from HTML elements to CoreModel types
+        HTML_TO_MODEL = {
+          # Block elements
+          p: 'Coradoc::CoreModel::Block',
+          div: 'Coradoc::CoreModel::Block',
+          section: 'Coradoc::CoreModel::StructuralElement',
+          article: 'Coradoc::CoreModel::StructuralElement',
+          header: 'Coradoc::CoreModel::StructuralElement',
+          aside: 'Coradoc::CoreModel::AnnotationBlock',
+          blockquote: 'Coradoc::CoreModel::Block',
+          pre: 'Coradoc::CoreModel::Block',
+
+          # Inline elements
+          strong: 'Coradoc::CoreModel::InlineElement',
+          b: 'Coradoc::CoreModel::InlineElement',
+          em: 'Coradoc::CoreModel::InlineElement',
+          i: 'Coradoc::CoreModel::InlineElement',
+          code: 'Coradoc::CoreModel::InlineElement',
+          mark: 'Coradoc::CoreModel::InlineElement',
+          sup: 'Coradoc::CoreModel::InlineElement',
+          sub: 'Coradoc::CoreModel::InlineElement',
+          u: 'Coradoc::CoreModel::InlineElement',
+          del: 'Coradoc::CoreModel::InlineElement',
+          s: 'Coradoc::CoreModel::InlineElement',
+          strike: 'Coradoc::CoreModel::InlineElement',
+
+          # Links
+          a: 'Coradoc::CoreModel::InlineElement',
+
+          # Lists
+          ul: 'Coradoc::CoreModel::ListBlock',
+          ol: 'Coradoc::CoreModel::ListBlock',
+          li: 'Coradoc::CoreModel::ListItem',
+          dl: 'Coradoc::CoreModel::ListBlock',
+          dt: 'Coradoc::CoreModel::ListItem',
+          dd: 'Coradoc::CoreModel::ListItem',
+
+          # Tables
+          table: 'Coradoc::CoreModel::Table',
+          tr: 'Coradoc::CoreModel::TableRow',
+          td: 'Coradoc::CoreModel::TableCell',
+          th: 'Coradoc::CoreModel::TableCell',
+
+          # Media
+          img: 'Coradoc::CoreModel::Image',
+          video: 'Coradoc::CoreModel::Block',
+          audio: 'Coradoc::CoreModel::Block',
+
+          # Other
+          hr: 'Coradoc::CoreModel::Block',
+          br: 'Coradoc::CoreModel::InlineElement',
+
+          # Headings
+          h1: 'Coradoc::CoreModel::StructuralElement',
+          h2: 'Coradoc::CoreModel::StructuralElement',
+          h3: 'Coradoc::CoreModel::StructuralElement',
+          h4: 'Coradoc::CoreModel::StructuralElement',
+          h5: 'Coradoc::CoreModel::StructuralElement',
+          h6: 'Coradoc::CoreModel::StructuralElement'
+        }.freeze
+
+        # Check if HTML element is block-level
+        def block_element?(tag)
+          BLOCK_ELEMENTS.include?(tag.to_sym)
+        end
+
+        # Check if HTML element is inline-level
+        def inline_element?(tag)
+          INLINE_ELEMENTS.include?(tag.to_sym)
+        end
+
+        # Check if HTML element is self-closing
+        def self_closing?(tag)
+          SELF_CLOSING_ELEMENTS.include?(tag.to_sym)
+        end
+
+        # Block-level HTML elements
+        BLOCK_ELEMENTS = %i[
+          div p section article aside header footer main nav
+          blockquote pre ul ol li dl dt dd
+          table tr td th thead tbody tfoot
+          h1 h2 h3 h4 h5 h6
+          hr
+          figure figcaption
+        ].freeze
+
+        # Inline-level HTML elements
+        INLINE_ELEMENTS = %i[
+          span strong em b i u s del ins mark small
+          code kbd samp var
+          a abbr cite dfn q
+          sub sup
+          time
+          br wbr
+        ].freeze
+
+        # Self-closing HTML elements
+        SELF_CLOSING_ELEMENTS = %i[
+          area base br col embed hr img input link meta param source track wbr
+        ].freeze
+      end
+    end
+  end
+end

data/coradoc-html/lib/coradoc/html/entity.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Coradoc
+  module Html
+    # HTML entity handling
+    module Entity
+      # Named HTML entities
+      NAMED_ENTITIES = {
+        'nbsp' => "\u00A0",
+        'lt' => '<',
+        'gt' => '>',
+        'amp' => '&',
+        'quot' => '"',
+        'apos' => "'",
+        'cent' => "\u00A2",
+        'pound' => "\u00A3",
+        'yen' => "\u00A5",
+        'euro' => "\u20AC",
+        'copy' => "\u00A9",
+        'reg' => "\u00AE",
+        'trade' => "\u2122",
+        'mdash' => "\u2014",
+        'ndash' => "\u2013",
+        'hellip' => "\u2026",
+        'laquo' => "\u00AB",
+        'raquo' => "\u00BB",
+        'ldquo' => "\u201C",
+        'rdquo' => "\u201D",
+        'lsquo' => "\u2018",
+        'rsquo' => "\u2019"
+      }.freeze
+
+      class << self
+        # Encode text to HTML entities
+        def encode(text, options = {})
+          return '' if text.nil?
+          return text unless text.is_a?(String)
+
+          encoded = text.dup
+
+          # Basic HTML entities
+          encoded = encoded
+                    .gsub('&', '&amp;')
+                    .gsub('<', '&lt;')
+                    .gsub('>', '&gt;')
+                    .gsub('"', '&quot;')
+
+          # Optionally encode additional characters
+          encoded = encoded.gsub("'", '&#39;') if options[:encode_quotes]
+
+          encoded = encoded.gsub("\u00A0", '&nbsp;') if options[:encode_nbsp]
+
+          encoded
+        end
+
+        # Decode HTML entities to text
+        def decode(text)
+          return '' if text.nil?
+          return text unless text.is_a?(String)
+
+          decoded = text.dup
+
+          # Decode named entities
+          NAMED_ENTITIES.each do |name, char|
+            decoded = decoded.gsub("&#{name};", char)
+          end
+
+          # Decode numeric entities (decimal)
+          decoded = decoded.gsub(/&#(\d+);/) do
+            [::Regexp.last_match(1).to_i].pack('U')
+          end
+
+          # Decode numeric entities (hexadecimal)
+          decoded = decoded.gsub(/&#x([0-9a-fA-F]+);/) do
+            [::Regexp.last_match(1).to_i(16)].pack('U')
+          end
+
+          # Decode basic entities last
+          decoded
+            .gsub('&quot;', '"')
+            .gsub('&#39;', "'")
+            .gsub('&#x27;', "'")
+            .gsub('&lt;', '<')
+            .gsub('&gt;', '>')
+            .gsub('&amp;', '&')
+        end
+
+        # Convert character to named entity if available
+        def to_named_entity(char)
+          entity_name = NAMED_ENTITIES.key(char)
+          entity_name ? "&#{entity_name};" : char
+        end
+
+        # Convert character to numeric entity
+        def to_numeric_entity(char, format: :decimal)
+          codepoint = char.ord
+
+          case format
+          when :decimal
+            "&##{codepoint};"
+          when :hex, :hexadecimal
+            "&#x#{codepoint.to_s(16)};"
+          else
+            char
+          end
+        end
+
+        # Check if text contains HTML entities
+        def has_entities?(text)
+          return false unless text.is_a?(String)
+
+          text.match?(/&[a-zA-Z]+;|&#\d+;|&#x[0-9a-fA-F]+;/)
+        end
+
+        # Normalize entities (convert all to named where possible, numeric otherwise)
+        def normalize(text)
+          return '' if text.nil?
+          return text unless text.is_a?(String)
+
+          # First decode to get actual characters
+          decoded = decode(text)
+
+          # Then encode back using named entities where possible
+          decoded.chars.map do |char|
+            case char
+            when '&', '<', '>', '"', "'"
+              encode(char)
+            else
+              named = to_named_entity(char)
+              named == char ? char : named
+            end
+          end.join
+        end
+      end
+    end
+  end
+end

data/coradoc-html/lib/coradoc/html/input/cleaner.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Coradoc
+  module Input
+    module Html
+      class Cleaner
+        # Pre-compiled regexes for performance
+        INNER_WHITESPACE_REGEX_1 = /\n stem:\[/
+        INNER_WHITESPACE_REGEX_2 = /(stem:\[([^\]]|\\\])*\])\n(?=\S)/
+        INNER_WHITESPACE_REGEX_3 = /(stem:\[([^\]]|\\\])*\])\s+(?=[\^-])/
+        NEWLINES_REGEX = /\n{3,}/
+        LEADING_NEWLINE_REGEX = /\A\n+/
+        WHITESPACE_REGEX = /[ \t\r\n]+/
+        TRAILING_WHITESPACE_REGEX = /[ \t\r\n]+\z/
+        MULTIPLE_WHITESPACE_REGEX = /[ \t]{2,}/
+        TAG_BORDER_REGEXES = {
+          asterisk: /\s?\*{2,}/,
+          underscore: /\s?_{2,}/,
+          tilde: /\s?~{2,}/,
+          bracket: /\s?\[.*?\]\s?/
+        }.freeze
+
+        def tidy(string)
+          return string.transform_values { |i| tidy(i) } if string.is_a? Hash
+
+          result = HtmlConverter.track_time 'Removing inner whitespace' do
+            remove_inner_whitespaces(String.new(string))
+          end
+          result = HtmlConverter.track_time 'Removing newlines' do
+            remove_newlines(result)
+          end
+          result = HtmlConverter.track_time 'Removing leading newlines' do
+            remove_leading_newlines(result)
+          end
+          result = HtmlConverter.track_time 'Cleaning tag borders' do
+            clean_tag_borders(result)
+          end
+          result = HtmlConverter.track_time 'Cleaning punctuation characters' do
+            clean_punctuation_characters(result)
+          end
+          result = remove_block_leading_newlines(result)
+          result = remove_section_attribute_newlines(result)
+        end
+
+        def remove_block_leading_newlines(string)
+          string.gsub("]\n****\n\n", "]\n****\n")
+        end
+
+        def remove_section_attribute_newlines(string)
+          string.gsub("]\n\n==", "]\n==")
+        end
+
+        def remove_newlines(string)
+          string.gsub(NEWLINES_REGEX, "\n\n")
+        end
+
+        def remove_leading_newlines(string)
+          string.gsub(LEADING_NEWLINE_REGEX, '')
+        end
+
+        def remove_inner_whitespaces(string)
+          unless string.nil?
+            string.gsub!("\n stem:[", "\nstem:[")
+            string.gsub!(INNER_WHITESPACE_REGEX_1, '\\1 ')
+            string.gsub!(INNER_WHITESPACE_REGEX_2, '\\1')
+          end
+          result = +''
+          string.each_line do |line|
+            result << preserve_border_whitespaces(line) do
+              # Use ASCII-only strip to preserve CJK fullwidth spaces
+              line.gsub(/\A[ \t\r\n]+/, '').gsub(/[ \t\r\n]+\z/, '').gsub(/[ \t]{2,}/, ' ')
+            end
+          end
+          result
+        end
+
+        # Find non-asterisk content that is enclosed by two or
+        # more asterisks. Ensure that only one whitespace occurs
+        # in the border area.
+        # Same for underscores and brackets.
+        def clean_tag_borders(string)
+          # result = string.gsub(/\s?\*{2,}.*?\*{2,}\s?/) do |match|
+          # preserve_border_whitespaces(match, default_border: Coradoc::Input::HTML.config.tag_border) do
+          #   match.strip.sub("** ", "**").sub(" **", "**")
+          # end
+          # end
+
+          # result = string.gsub(/\s?_{2,}.*?_{2,}\s?/) do |match|
+          #   preserve_border_whitespaces(match, default_border: Coradoc::Input::HTML.config.tag_border) do
+          #     match.strip.sub("__ ", "__").sub(" __", "__")
+          #   end
+          # end
+
+          result = string.gsub(/\s?~{2,}.*?~{2,}\s?/) do |match|
+            preserve_border_whitespaces(
+              match,
+              default_border: Coradoc::Html::Input.config.tag_border
+            ) do
+              match.strip.sub('~~ ', '~~').sub(' ~~', '~~')
+            end
+          end
+
+          result.gsub(/\s?\[.*?\]\s?/) do |match|
+            preserve_border_whitespaces(match) do
+              match.strip.sub('[ ', '[').sub(' ]', ']')
+            end
+          end
+        end
+
+        def clean_punctuation_characters(string)
+          string.gsub(/(\*\*|~~|__)\s([.!?'"])/, '\\1\\2')
+        end
+
+        # preprocesses HTML, rather than postprocessing it
+        def preprocess_word_html(string)
+          clean_headings(scrub_whitespace(string.dup))
+        end
+
+        def scrub_whitespace(string)
+          string.gsub!(/&nbsp;|&#xA0;|\u00a0/i, '&#xA0;') # HTML encoded spaces
+          string = Coradoc.strip_unicode(string) # Strip document-level leading and trailing whitespace
+          string.gsub!(/( +)$/, ' ') # line trailing whitespace
+          string.gsub!("\n\n\n\n", "\n\n") # Quadruple line breaks
+          # string.delete!('?| ')               # Unicode non-breaking spaces, injected as tabs
+          string
+        end
+
+        # following added by me
+        def clean_headings(string)
+          string.gsub!(%r{<h([1-9])[^>]*></h\1>}, ' ')
+          # I don't know why Libre Office is inserting them, but they need to go
+          string.gsub!(
+            %r{<h([1-9])[^>]* style="vertical-align: super;[^>]*>(.+?)</h\1>},
+            '<sup>\\2</sup>'
+          )
+          # I absolutely don't know why Libre Office is rendering superscripts as h1
+          string
+        end
+
+        private
+
+        def preserve_border_whitespaces(string, options = {})
+          return string if /\A\s*\Z/.match?(string)
+
+          default_border = options.fetch(:default_border, '')
+          # If the string contains part of a link so the characters [,],(,)
+          # then don't add any extra spaces
+          default_border = '' if /[\[(\])]/.match?(string)
+          string_start   = present_or_default(string[/\A\s*/], default_border)
+          string_end     = present_or_default(string[/\s*\Z/], default_border)
+          result         = yield
+          string_start + result + string_end
+        end
+
+        def present_or_default(string, default)
+          return default if string.nil? || string.empty?
+
+          string
+        end
+      end
+    end
+  end
+end

data/coradoc-html/lib/coradoc/html/input/config.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+
+module Coradoc
+  module Input
+    module Html
+      class Config
+        def initialize
+          @unknown_tags     = :pass_through
+          @input_format     = :html
+          @mathml2asciimath = false
+          @external_images  = false
+
+          # Destination to save file and images
+          @destination      = nil
+
+          # Source of HTML
+          # @sourcedir        = nil
+
+          # Image counter, assuming there are max 999 images
+          @image_counter = 1
+          # pad with 0s
+          @image_counter_pattern = '%03d'
+
+          @em_delimiter     = '_'
+          @strong_delimiter = '*'
+          @inline_options   = {}
+          @tag_border       = ' '
+
+          @split_sections   = nil
+
+          # Document width - used to compute table sizes.
+          # This is an assumption for screen size in input document.
+          # If column widths are specified in absolute values, then we
+          # have to convert them to relative values for better portability
+          # across output formats.
+          @doc_width        = 1000
+
+          # Plugin system
+          @plugins          = []
+
+          # Debugging options
+          @track_time       = false
+        end
+
+        def with(options = {})
+          old_options = @inline_options
+          @inline_options = options
+          result = yield
+          @inline_options = old_options
+          result
+        end
+
+        def self.declare_option(option)
+          define_method(option) do
+            @inline_options[option] || instance_variable_get(:"@#{option}")
+          end
+
+          attr_writer option
+        end
+
+        declare_option :unknown_tags
+        declare_option :tag_border
+        declare_option :mathml2asciimath
+        declare_option :external_images
+        declare_option :destination
+        declare_option :sourcedir
+        declare_option :image_counter
+        declare_option :image_counter_pattern
+        declare_option :input_format
+        declare_option :split_sections
+        declare_option :doc_width
+        declare_option :plugins
+        declare_option :track_time
+      end
+    end
+  end
+end

data/coradoc-html/lib/coradoc/html/input/converters/a.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'coradoc'
+
+module Coradoc
+  module Input
+    module Html
+      module Converters
+        class A < Base
+          def to_coradoc(node, state = {})
+            # Use treat_children_coradoc to get CoreModel elements
+            content = treat_children_coradoc(node, state)
+
+            href  = node['href']
+            title = extract_title(node)
+            id = node['id'] || node['name']
+
+            id = id&.gsub(/\s/, '')&.gsub(/__+/, '_')
+            id = nil if id&.empty?
+
+            return nil if /^_Toc\d+$|^_GoBack$/.match?(id)
+
+            # For inline anchors - return CoreModel InlineElement with format_type "anchor"
+            if id
+              return Coradoc::CoreModel::InlineElement.new(
+                format_type: 'anchor',
+                target: id
+              )
+            end
+
+            # For cross-references
+            if href.to_s.start_with?('#')
+              ref_id = href.sub(/^#/, '').gsub(/\s/, '').gsub(/__+/, '_')
+              # Convert content to string
+              content_str = if content.is_a?(Array)
+                              content.map { |c| c.respond_to?(:content) ? c.content : c.to_s }.join
+                            else
+                              content.to_s
+                            end
+              return Coradoc::CoreModel::InlineElement.new(
+                format_type: 'xref',
+                target: ref_id,
+                content: content_str.strip.empty? ? nil : content_str.strip
+              )
+            end
+
+            return nil if href.to_s.empty?
+
+            # For links
+            ambigous_characters = /[\w.?&#=%;\[\u{ff}-\u{10ffff}]/
+            right_constrain = textnode_after_start_with?(node, ambigous_characters)
+
+            # Convert content to string for the link
+            content_str = if content.is_a?(Array)
+                            content.map { |c| c.respond_to?(:content) ? c.content : c.to_s }.join
+                          else
+                            content.to_s
+                          end
+
+            out = []
+            # Add leading space if needed
+            if textnode_before_end_with?(node, ambigous_characters)
+              out << Coradoc::CoreModel::InlineElement.new(
+                format_type: 'text',
+                content: ' '
+              )
+            end
+
+            # Create link element
+            link = Coradoc::CoreModel::InlineElement.new(
+              format_type: 'link',
+              target: href,
+              content: content_str.strip,
+              metadata: {
+                title: (title.strip unless title.to_s.strip.empty?),
+                right_constrain: right_constrain
+              }.compact
+            )
+            out << link
+
+            # Return single element or array
+            out.length == 1 ? out.first : out
+          end
+        end
+
+        register :a, A.new
+      end
+    end
+  end
+end