metanorma-standoc 1.11.3 → 2.0.2

data/lib/metanorma/standoc/cleanup_symbols.rb

@@ -0,0 +1,48 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      # Indices sort after letter but before any following
+      # letter (x, x_m, x_1, xa); we use colon to force that sort order.
+      # Numbers sort *after* letters; we use thorn to force that sort order.
+      def symbol_key(sym)
+        key = sym.dup
+        key.traverse do |n|
+          n.name == "math" and
+            n.replace(grkletters(MathML2AsciiMath.m2a(n.to_xml)))
+        end
+        ret = Nokogiri::XML(key.to_xml)
+        HTMLEntities.new.decode(ret.text.downcase)
+          .gsub(/[\[\]{}<>()]/, "").gsub(/\s/m, "")
+          .gsub(/[[:punct:]]|[_^]/, ":\\0").gsub(/`/, "")
+          .gsub(/[0-9]+/, "þ\\0")
+      end
+
+      def grkletters(text)
+        text.gsub(/\b(alpha|beta|gamma|delta|epsilon|zeta|eta|theta|iota|kappa|
+                      lambda|mu|nu|xi|omicron|pi|rho|sigma|tau|upsilon|phi|chi|
+                      psi|omega)\b/xi, "&\\1;")
+      end
+
+      def extract_symbols_list(dlist)
+        dl_out = []
+        dlist.xpath("./dt | ./dd").each do |dtd|
+          if dtd.name == "dt"
+            dl_out << { dt: dtd.remove, key: symbol_key(dtd) }
+          else
+            dl_out.last[:dd] = dtd.remove
+          end
+        end
+        dl_out
+      end
+
+      def symbols_cleanup(docxml)
+        docxml.xpath("//definitions/dl").each do |dl|
+          dl_out = extract_symbols_list(dl)
+          dl_out.sort! { |a, b| a[:key] <=> b[:key] || a[:dt] <=> b[:dt] }
+          dl.children = dl_out.map { |d| d[:dt].to_s + d[:dd].to_s }.join("\n")
+        end
+        docxml
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_table.rb

@@ -0,0 +1,68 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def dl1_table_cleanup(xmldoc)
+        q = "//table/following-sibling::*[1][self::dl]"
+        xmldoc.xpath(q).each do |s|
+          s["key"] == "true" and s.previous_element << s.remove
+        end
+      end
+
+      # move Key dl after table footer
+      def dl2_table_cleanup(xmldoc)
+        q = "//table/following-sibling::*[1][self::p]"
+        xmldoc.xpath(q).each do |s|
+          if s.text =~ /^\s*key[^a-z]*$/i && s&.next_element&.name == "dl"
+            s.next_element["key"] = "true"
+            s.previous_element << s.next_element.remove
+            s.remove
+          end
+        end
+      end
+
+      def insert_thead(table)
+        thead = table.at("./thead")
+        return thead unless thead.nil?
+
+        if tname = table.at("./name")
+          thead = tname.add_next_sibling("<thead/>").first
+          return thead
+        end
+        table.children.first.add_previous_sibling("<thead/>").first
+      end
+
+      def header_rows_cleanup(xmldoc)
+        xmldoc.xpath("//table[@headerrows]").each do |s|
+          thead = insert_thead(s)
+          (thead.xpath("./tr").size...s["headerrows"].to_i).each do
+            row = s.at("./tbody/tr")
+            row.parent = thead
+          end
+          thead.xpath(".//td").each { |n| n.name = "th" }
+          s.delete("headerrows")
+        end
+      end
+
+      def table_cleanup(xmldoc)
+        dl1_table_cleanup(xmldoc)
+        dl2_table_cleanup(xmldoc)
+        notes_table_cleanup(xmldoc)
+        header_rows_cleanup(xmldoc)
+      end
+
+      # move notes into table
+      def notes_table_cleanup(xmldoc)
+        nomatches = false
+        until nomatches
+          nomatches = true
+          xmldoc.xpath("//table/following-sibling::*[1]"\
+                       "[self::note[not(@keep-separate = 'true')]]").each do |n|
+            n.delete("keep-separate")
+            n.previous_element << n.remove
+            nomatches = false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_terms.rb

@@ -0,0 +1,140 @@
+require_relative "term_lookup_cleanup"
+require_relative "cleanup_terms_designations"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def termdomain_cleanup(xmldoc)
+        xmldoc.xpath("//p/domain").each do |a|
+          parent = a.parent
+          prev = parent.previous
+          prev.next = a.remove
+          parent.text.strip.empty? and parent.remove
+        end
+      end
+
+      def termdomain1_cleanup(xmldoc)
+        xmldoc.xpath("//term").each do |t|
+          d = t.xpath("./domain | ./subject").last or next
+          defn = d.at("../definition") and defn.previous = d.remove
+        end
+      end
+
+      def termdefinition_cleanup(xmldoc)
+        generate_termdefinitions(xmldoc)
+        split_termdefinitions(xmldoc)
+        alternate_termdefinitions(xmldoc)
+      end
+
+      TERMDEF_BLOCKS =
+        "./p | ./ol | ./dl[not(@metadata = 'true')] | ./ul | ./figure | "\
+        "./formula | ./table".freeze
+
+      def generate_termdefinitions(xmldoc)
+        xmldoc.xpath("//term[not(definition)]").each do |d|
+          first_child = d.at(TERMDEF_BLOCKS) || next
+          t = Nokogiri::XML::Element.new("definition", xmldoc)
+          first_child.replace(t)
+          t << first_child.remove
+          d.xpath(TERMDEF_BLOCKS).each do |n|
+            t << n.remove
+          end
+        end
+      end
+
+      def split_termdefinitions(xmldoc)
+        xmldoc.xpath("//definition").each do |d|
+          if d.at("./p | ./ol | ./dl | ./ul")
+            d.children = "<verbal-definition>#{d.children}</verbal-definition>"
+          else
+            d.children = "<non-verbal-representation>"\
+                         "#{d.children}</non-verbal-representation>"
+          end
+        end
+      end
+
+      def alternate_termdefinitions(xmldoc)
+        xmldoc.xpath("//term").each do |t|
+          t.xpath("./definition").each do |d|
+            d1 = d.next_element or next
+            if (v = d.at("./verbal-definition")) &&
+                !d.at("./non-verbal-representation") &&
+                !d1.at("./verbal-definition") &&
+                nv = d1.at("./non-verbal-representation")
+              v.next = nv.remove
+              d1.remove
+            end
+          end
+        end
+      end
+
+      def termdocsource_cleanup(xmldoc)
+        f = xmldoc.at("//preface | //sections")
+        xmldoc.xpath("//termdocsource").each { |s| f.previous = s.remove }
+      end
+
+      def term_children_cleanup(xmldoc)
+        xmldoc.xpath("//terms[terms]").each { |t| t.name = "clause" }
+        xmldoc.xpath("//term").each do |t|
+          %w(termnote termexample termsource term).each do |w|
+            t.xpath("./#{w}").each { |n| t << n.remove }
+          end
+        end
+      end
+
+      def termdef_from_termbase(xmldoc)
+        xmldoc.xpath("//term").each do |x|
+          if (c = x.at("./origin/termref")) && !x.at("./definition")
+            x.at("./origin").previous = fetch_termbase(c["base"], c.text)
+          end
+        end
+      end
+
+      def termnote_example_cleanup(xmldoc)
+        %w(note example).each do |w|
+          xmldoc.xpath("//term#{w}[not(ancestor::term)]").each do |x|
+            x.name = w
+          end
+        end
+      end
+
+      def termdef_cleanup(xmldoc)
+        termdef_unnest_cleanup(xmldoc)
+        Metanorma::Standoc::TermLookupCleanup.new(xmldoc, @log).call
+        term_nonverbal_designations(xmldoc)
+        term_dl_to_metadata(xmldoc)
+        term_termsource_to_designation(xmldoc)
+        term_designation_reorder(xmldoc)
+        termdef_from_termbase(xmldoc)
+        termdomain_cleanup(xmldoc)
+        termdef_stem_cleanup(xmldoc)
+        termdefinition_cleanup(xmldoc)
+        termdomain1_cleanup(xmldoc)
+        termnote_example_cleanup(xmldoc)
+        term_children_cleanup(xmldoc)
+        termdocsource_cleanup(xmldoc)
+      end
+
+      def index_cleanup(xmldoc)
+        return unless @index_terms
+
+        xmldoc.xpath("//preferred").each do |p|
+          index_cleanup1(p.at("./expression/name | ./letter-symbol/name"),
+                         p.xpath("./field-of-application | ./usage-info")
+            &.map(&:text)&.join(", "))
+        end
+        xmldoc.xpath("//definitions/dl/dt").each do |p|
+          index_cleanup1(p, "")
+        end
+      end
+
+      def index_cleanup1(term, fieldofappl)
+        return unless term
+
+        idx = term.children.dup
+        fieldofappl.empty? or idx << ", &#x3c;#{fieldofappl}&#x3e;"
+        term << "<index><primary>#{idx.to_xml}</primary></index>"
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_terms_designations.rb

@@ -0,0 +1,199 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def termdef_stem_cleanup(xmldoc)
+        termdef_stem2admitted(xmldoc)
+        xmldoc.xpath("//term//expression/name[stem]").each do |n|
+          test = n.dup
+          test.at("./stem").remove
+          next unless test.text.strip.empty?
+
+          n.parent.name = "letter-symbol"
+        end
+      end
+
+      def termdef_stem2admitted(xmldoc)
+        xmldoc.xpath("//term/p/stem").each do |a|
+          if initial_formula(a.parent)
+            parent = a.parent
+            parent.replace("<admitted>#{term_expr(a.to_xml)}</admitted>")
+          end
+        end
+        xmldoc.xpath("//term/formula").each do |a|
+          initial_formula(a) and
+            a.replace("<admitted>#{term_expr(a.children.to_xml)}</admitted>")
+        end
+      end
+
+      def initial_formula(elem)
+        elem.elements.size == 1 && # para contains just stem expression
+          !elem.at("./preceding-sibling::p | ./preceding-sibling::dl | "\
+                   "./preceding-sibling::ol | ./preceding-sibling::ul")
+      end
+
+      # release termdef tags from surrounding paras
+      def termdef_unnest_cleanup(xmldoc)
+        desgn = "//p/admitted | //p/deprecates | //p/preferred | //p//related"
+        nodes = xmldoc.xpath(desgn)
+        while !nodes.empty?
+          nodes[0].parent.replace(nodes[0].parent.children)
+          nodes = xmldoc.xpath(desgn)
+        end
+      end
+
+      def term_dl_to_metadata(xmldoc)
+        xmldoc.xpath("//term[dl[@metadata = 'true']]").each do |t|
+          t.xpath("./dl[@metadata = 'true']").each do |dl|
+            prev = related2pref(dl_to_designation(dl)) or next
+            term_dl_to_designation_metadata(prev, dl)
+            term_dl_to_term_metadata(prev, dl)
+            term_dl_to_expression_metadata(prev, dl)
+            dl.remove
+          end
+        end
+      end
+
+      def term_dl_to_term_metadata(prev, dlist)
+        return unless prev.name == "preferred" &&
+          prev.at("./preceding-sibling::preferred").nil?
+
+        ins = term_element_insert_point(prev)
+        %w(domain subject).each do |a|
+          ins = dl_to_elems(ins, prev.parent, dlist, a)
+        end
+      end
+
+      def term_dl_to_designation_metadata(prev, dlist)
+        %w(absent geographic-area).each do |a|
+          dl_to_attrs(prev, dlist, a)
+        end
+        %w(field-of-application usage-info).reverse.each do |a|
+          dl_to_elems(prev.at("./expression"), prev, dlist, a)
+        end
+      end
+
+      def term_element_insert_point(prev)
+        ins = prev
+        while %w(preferred admitted deprecates related domain dl)
+            .include? ins&.next_element&.name
+          ins = ins.next_element
+        end
+        ins
+      end
+
+      def term_dl_to_expression_metadata(prev, dlist)
+        term_dl_to_expression_root_metadata(prev, dlist)
+        term_dl_to_expression_name_metadata(prev, dlist)
+        term_to_letter_symbol(prev, dlist)
+      end
+
+      def term_dl_to_expression_root_metadata(prev, dlist)
+        %w(isInternational).each do |a|
+          p = prev.at("./expression | ./letter-symbol | ./graphical-symbol")
+          dl_to_attrs(p, dlist, a)
+        end
+        %w(language script type).each do |a|
+          p = prev.at("./expression") or next
+          dl_to_attrs(p, dlist, a)
+        end
+      end
+
+      def term_dl_to_expression_name_metadata(prev, dlist)
+        %w(abbreviation-type pronunciation).reverse.each do |a|
+          dl_to_elems(prev.at("./expression/name"), prev, dlist, a)
+        end
+        g = dlist.at("./dt[text()='grammar']/following::dd//dl") and
+          term_dl_to_expression_grammar(prev, g)
+      end
+
+      def term_dl_to_expression_grammar(prev, dlist)
+        prev.at(".//expression") or return
+        prev.at(".//expression") << "<grammar><sentinel/></grammar>"
+        %w(gender number isPreposition isParticiple isAdjective isAdverb isNoun
+           grammar-value).reverse.each do |a|
+          dl_to_elems(prev.at(".//expression/grammar/*"), prev.elements.last,
+                      dlist, a)
+        end
+        term_dl_to_designation_category(prev, "gender")
+        term_dl_to_designation_category(prev, "number")
+        prev.at(".//expression/grammar/sentinel").remove
+      end
+
+      def term_dl_to_designation_category(prev, category)
+        cat = prev.at(".//expression/grammar/#{category}")
+        /,/.match?(cat&.text) and
+          cat.replace(cat.text.split(/,\s*/)
+            .map { |x| "<#{category}>#{x}</#{category}>" }.join)
+      end
+
+      def term_to_letter_symbol(prev, dlist)
+        ls = dlist.at("./dt[text()='letter-symbol']/following::dd/p")
+        return unless ls&.text == "true"
+
+        prev.at(".//expression").name = "letter-symbol"
+      end
+
+      def dl_to_designation(dlist)
+        prev = dlist.previous_element
+        unless %w(preferred admitted deprecates related).include? prev&.name
+          @log.add("AsciiDoc Input", dlist, "Metadata definition list does "\
+                                            "not follow a term designation")
+          return nil
+        end
+        prev
+      end
+
+      def term_nonverbal_designations(xmldoc)
+        xmldoc.xpath("//term/preferred | //term/admitted | //term/deprecates")
+          .each do |d|
+          d.text.strip.empty? or next
+          n = d.next_element
+          if %w(formula figure).include?(n&.name)
+            term_nonverbal_designations1(d, n)
+          else d.at("./expression/name") or
+            d.children = term_expr("")
+          end
+        end
+      end
+
+      def term_nonverbal_designations1(desgn, elem)
+        desgn = related2pref(desgn)
+        if elem.name == "figure"
+          elem.at("./name").remove
+          desgn.children =
+            "<graphical-symbol>#{elem.remove.to_xml}</graphical-symbol>"
+        else
+          desgn.children = term_expr(elem.at("./stem").to_xml)
+          elem.remove
+        end
+      end
+
+      def term_termsource_to_designation(xmldoc)
+        xmldoc.xpath("//term/termsource").each do |t|
+          p = t.previous_element
+          while %w(domain subject).include? p&.name
+            p = p.previous_element
+          end
+          %w(preferred admitted deprecates related).include?(p&.name) or
+            next
+          related2pref(p) << t.remove
+        end
+      end
+
+      def term_designation_reorder(xmldoc)
+        xmldoc.xpath("//term").each do |t|
+          %w(preferred admitted deprecates related)
+            .each_with_object([]) do |tag, m|
+            t.xpath("./#{tag}").each { |x| m << x.remove }
+          end.reverse.each do |x|
+            t.children.first.previous = x
+          end
+        end
+      end
+
+      def related2pref(elem)
+        elem&.name == "related" ? elem = elem.at("./preferred") : elem
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_text.rb

@@ -0,0 +1,74 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def textcleanup(result)
+        text = result.flatten.map { |l| l.sub(/\s*$/, "") } * "\n"
+        !@keepasciimath and text = asciimath2mathml(text)
+        text = text.gsub(/\s+<fn /, "<fn ")
+        text.gsub(%r{<passthrough\s+formats="metanorma">([^<]*)
+                  </passthrough>}mx) { HTMLEntities.new.decode($1) }
+      end
+
+      IGNORE_DUMBQUOTES =
+        "//pre | //pre//* | //tt | //tt//* | "\
+        "//sourcecode | //sourcecode//* | //bibdata//* | //stem | "\
+        "//stem//* | //figure[@class = 'pseudocode'] | "\
+        "//figure[@class = 'pseudocode']//*".freeze
+
+      def smartquotes_cleanup(xmldoc)
+        xmldoc.xpath("//date").each { |d| Metanorma::Utils::endash_date(d) }
+        if @smartquotes then smartquotes_cleanup1(xmldoc)
+        else dumbquote_cleanup(xmldoc)
+        end
+      end
+
+      def smartquotes_cleanup1(xmldoc)
+        uninterrupt_quotes_around_xml(xmldoc)
+        dumb2smart_quotes(xmldoc)
+      end
+
+      # "abc<tag/>", def => "abc",<tag/> def
+      def uninterrupt_quotes_around_xml(xmldoc)
+        xmldoc.traverse do |n|
+          next unless n.text? && n&.previous&.element?
+          next unless /^['"]/.match?(n.text)
+          next unless n.previous.ancestors("pre, tt, sourcecode, stem, figure")
+            .empty?
+
+          uninterrupt_quotes_around_xml1(n.previous)
+        end
+      end
+
+      def uninterrupt_quotes_around_xml1(elem)
+        prev = elem.at(".//preceding::text()[1]") or return
+        /\S$/.match?(prev.text) or return
+        foll = elem.at(".//following::text()[1]")
+        m = /^(["'][[:punct:]]*)(\s|$)/
+          .match(HTMLEntities.new.decode(foll&.text)) or return
+        foll.content = foll.text.sub(/^(["'][[:punct:]]*)/, "")
+        prev.content = "#{prev.text}#{m[1]}"
+      end
+
+      def dumb2smart_quotes(xmldoc)
+        (xmldoc.xpath("//*[child::text()]") - xmldoc.xpath(IGNORE_DUMBQUOTES))
+          .each do |x|
+          x.children.each do |n|
+            next unless n.text?
+
+            /[-'"(<>]|\.\.|\dx/.match(n) or next
+
+            n.replace(Metanorma::Utils::smartformat(n.text))
+          end
+        end
+      end
+
+      def dumbquote_cleanup(xmldoc)
+        xmldoc.traverse do |n|
+          next unless n.text?
+
+          n.replace(n.text.gsub(/(?<=\p{Alnum})\u2019(?=\p{Alpha})/, "'")) # .
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_toc.rb

@@ -0,0 +1,98 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def toc_cleanup(xmldoc)
+        toc_cleanup_para(xmldoc)
+        xmldoc.xpath("//toc").each { |t| toc_cleanup1(t, xmldoc) }
+        toc_cleanup_clause(xmldoc)
+        toc_metadata(xmldoc)
+      end
+
+      def toc_cleanup_para(xmldoc)
+        xmldoc.xpath("//p[toc]").each do |x|
+          x.xpath("./toc").reverse.each do |t|
+            x.next = t
+          end
+          x.remove if x.text.strip.empty?
+        end
+      end
+
+      def toc_index(toc, xmldoc)
+        depths = toc_index_depths(toc)
+        depths.keys.each_with_object([]) do |key, arr|
+          xmldoc.xpath(key).each do |x|
+            arr << toc_index1(key, x, depths)
+          end
+        end.sort_by { |a| a[:line] }
+      end
+
+      def toc_index1(key, entry, depths)
+        t = entry.at("./following-sibling::variant-title[@type = 'toc']") and
+          entry = t
+        { text: entry.children.to_xml, depth: depths[key].to_i,
+          target: entry.xpath("(./ancestor-or-self::*/@id)[last()]")[0].text,
+          line: entry.line }
+      end
+
+      def toc_index_depths(toc)
+        toc.xpath("./toc-xpath").each_with_object({}) do |x, m|
+          m[x.text] = x["depth"]
+        end
+      end
+
+      def toc_cleanup1(toc, xmldoc)
+        depth = 1
+        ret = ""
+        toc_index(toc, xmldoc).each do |x|
+          ret = toc_cleanup1_entry(x, depth, ret)
+          depth = x[:depth]
+        end
+        toc.children = "<ul>#{ret}</ul>"
+      end
+
+      def toc_cleanup1_entry(entry, depth, ret)
+        if depth > entry[:depth]
+          ret += "</ul></li>" * (depth - entry[:depth])
+        elsif depth < entry[:depth]
+          ret += "<li><ul>" * (entry[:depth] - depth)
+        end
+        ret + "<li><xref target='#{entry[:target]}'>#{entry[:text]}</xref></li>"
+      end
+
+      def toc_cleanup_clause(xmldoc)
+        xmldoc
+          .xpath("//clause[@type = 'toc'] | //annex[@type = 'toc']").each do |c|
+          c.xpath(".//ul[not(ancestor::ul)]").each do |ul|
+            toc_cleanup_clause_entry(xmldoc, ul)
+            ul.replace("<toc>#{ul.to_xml}</toc>")
+          end
+        end
+      end
+
+      def toc_cleanup_clause_entry(xmldoc, list)
+        list.xpath(".//xref[not(text())]").each do |x|
+          c1 = xmldoc.at("//*[@id = '#{x['target']}']")
+          t = c1.at("./variant-title[@type = 'toc']") || c1.at("./title")
+          x << t.dup.children
+        end
+      end
+
+      def toc_metadata(xmldoc)
+        return unless @htmltoclevels || @doctoclevels || @toclevels
+
+        ins = xmldoc.at("//misc-container") ||
+          xmldoc.at("//bibdata").after("<misc-container/>").next_element
+        toc_metadata1(ins)
+      end
+
+      def toc_metadata1(ins)
+        [[@toclevels, "TOC Heading Levels"],
+         [@toclevels, "TOC Heading Levels"],
+         [@toclevels, "TOC Heading Levels"]].each do |n|
+          n[0] and ins << "<presentation-metadata><name>#{n[1]}</name>"\
+                          "<value>#{n[0]}</value></presentation-metadata>"
+        end
+      end
+    end
+  end
+end