metanorma-standoc 1.11.3 → 2.0.2

data/lib/metanorma/standoc/cleanup_footnotes.rb

@@ -0,0 +1,109 @@
+require "date"
+require "htmlentities"
+require "json"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def footnote_content(fn)
+        c = fn.children.respond_to?(:to_xml) ? fn.children.to_xml : fn.children
+        c.gsub(/ id="[^"]+"/, "")
+      end
+
+      # include footnotes inside figure if they are the only content 
+      # of the paras following
+      def figure_footnote_cleanup(xmldoc)
+        nomatches = false
+        until nomatches
+          q = "//figure/following-sibling::*[1][self::p and *[1][self::fn]]"
+          nomatches = true
+          xmldoc.xpath(q).each do |s|
+            next if s.children.map { |c| c.text? && /[[:alpha:]]/.match(c.text) }.any?
+
+            s.previous_element << s.first_element_child.remove
+            s.remove
+            nomatches = false
+          end
+        end
+      end
+
+      def table_footnote_renumber1(fn, i, seen)
+        content = footnote_content(fn)
+        if seen[content] then outnum = seen[content]
+        else
+          i += 1
+          outnum = i
+          seen[content] = outnum
+        end
+        fn["reference"] = (outnum - 1 + "a".ord).chr
+        fn["table"] = true
+        [i, seen]
+      end
+
+      def table_footnote_renumber(xmldoc)
+        xmldoc.xpath("//table | //figure").each do |t|
+          seen = {}
+          i = 0
+          t.xpath(".//fn[not(ancestor::name)]").each do |fn|
+            i, seen = table_footnote_renumber1(fn, i, seen)
+          end
+        end
+      end
+
+      def other_footnote_renumber1(fn, i, seen)
+        unless fn["table"]
+          content = footnote_content(fn)
+          if seen[content] then outnum = seen[content]
+          else
+            i += 1
+            outnum = i
+            seen[content] = outnum
+          end
+          fn["reference"] = outnum.to_s
+        end
+        [i, seen]
+      end
+
+      def other_footnote_renumber(xmldoc)
+        seen = {}
+        i = 0
+        xmldoc.xpath("//fn").each do |fn|
+          i, seen = other_footnote_renumber1(fn, i, seen)
+        end
+      end
+
+      def title_footnote_move(xmldoc)
+        ins = xmldoc.at("//bibdata/language")
+        xmldoc.xpath("//bibdata/title//fn").each do |f|
+          f.name = "note"
+          f["type"] = "title-footnote"
+          f.delete("reference")
+          ins.previous = f.remove
+        end
+      end
+
+      def footnote_block_cleanup(xmldoc)
+        xmldoc.xpath("//footnoteblock").each do |f|
+          f.name = 'fn'
+          if id = xmldoc.at("//*[@id = '#{f.text}']")
+            f.children = id.remove.children
+          else
+            @log.add("Crossreferences", f,
+                     "Could not resolve footnoteblock:[#{f.text}]")
+            f.children = "[ERROR]"
+          end
+        end
+      end
+
+      def footnote_cleanup(xmldoc)
+        footnote_block_cleanup(xmldoc)
+        title_footnote_move(xmldoc)
+        table_footnote_renumber(xmldoc)
+        other_footnote_renumber(xmldoc)
+        xmldoc.xpath("//fn").each do |fn|
+          fn.delete("table")
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_image.rb

@@ -0,0 +1,70 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def svgmap_cleanup(xmldoc)
+        svgmap_moveattrs(xmldoc)
+        svgmap_populate(xmldoc)
+        Metanorma::Utils::svgmap_rewrite(xmldoc, @localdir)
+      end
+
+      def guid?(str)
+        /^_[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i
+          .match(str)
+      end
+
+      def svgmap_moveattrs(xmldoc)
+        xmldoc.xpath("//svgmap").each do |s|
+          f = s.at(".//figure") or next
+          (t = s.at("./name")) && !f.at("./name") and
+            f.children.first.previous = t.remove
+          if s["id"] && guid?(f["id"])
+            f["id"] = s["id"]
+            s.delete("id")
+          end
+          svgmap_moveattrs1(s, f)
+        end
+      end
+
+      def svgmap_moveattrs1(svgmap, figure)
+        %w(unnumbered number subsequence keep-with-next
+           keep-lines-together tag multilingual-rendering).each do |a|
+          next if figure[a] || !svgmap[a]
+
+          figure[a] = svgmap[a]
+          svgmap.delete(a)
+        end
+      end
+
+      def svgmap_populate(xmldoc)
+        xmldoc.xpath("//svgmap").each do |s|
+          s1 = s.dup
+          s.children.remove
+          f = s1.at(".//figure") and s << f
+          s1.xpath(".//li").each do |li|
+            t = li&.at(".//eref | .//link | .//xref") or next
+            href = t.xpath("./following-sibling::node()")
+            href.empty? or
+              s << %[<target href="#{svgmap_target(href)}">#{t.to_xml}</target>]
+          end
+        end
+      end
+
+      def svgmap_target(nodeset)
+        nodeset.each do |n|
+          next unless n.name == "link"
+
+          n.children = n["target"]
+        end
+        nodeset.text.sub(/^[,; ]/, "").strip
+      end
+
+      def img_cleanup(xmldoc)
+        return xmldoc unless @datauriimage
+
+        xmldoc.xpath("//image").each do |i|
+          i["src"] = Metanorma::Utils::datauri(i["src"], @localdir)
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_inline.rb

@@ -0,0 +1,190 @@
+require "metanorma-utils"
+require "digest"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def empty_text_before_first_element(elem)
+        elem.children.each do |c|
+          return false if c.text? && /\S/.match(c.text)
+          return true if c.element?
+        end
+        true
+      end
+
+      def strip_initial_space(elem)
+        return unless elem.children[0].text?
+
+        if /\S/.match?(elem.children[0].text)
+          elem.children[0].content = elem.children[0].text.gsub(/^ /, "")
+        else
+          elem.children[0].remove
+        end
+      end
+
+      def bookmark_cleanup(xmldoc)
+        li_bookmark_cleanup(xmldoc)
+        dt_bookmark_cleanup(xmldoc)
+      end
+
+      def bookmark_to_id(elem, bookmark)
+        parent = bookmark.parent
+        elem["id"] = bookmark.remove["id"]
+        strip_initial_space(parent)
+      end
+
+      def li_bookmark_cleanup(xmldoc)
+        xmldoc.xpath("//li[descendant::bookmark]").each do |x|
+          if x.at("./*[1][local-name() = 'p']/"\
+                  "*[1][local-name() = 'bookmark']") &&
+              empty_text_before_first_element(x.elements[0])
+            bookmark_to_id(x, x.elements[0].elements[0])
+          end
+        end
+      end
+
+      def dt_bookmark_cleanup(xmldoc)
+        xmldoc.xpath("//dt[descendant::bookmark]").each do |x|
+          if x.at("./*[1][local-name() = 'p']/"\
+                  "*[1][local-name() = 'bookmark']") &&
+              empty_text_before_first_element(x.elements[0])
+            bookmark_to_id(x, x.elements[0].elements[0])
+          elsif x.at("./*[1][local-name() = 'bookmark']") &&
+              empty_text_before_first_element(x)
+            bookmark_to_id(x, x.elements[0])
+          end
+        end
+      end
+
+      def concept_cleanup(xmldoc)
+        xmldoc.xpath("//concept[not(termxref)]").each do |x|
+          term = x.at("./refterm")
+          term&.remove if term&.text&.empty?
+          concept_cleanup1(x)
+        end
+      end
+
+      def concept_cleanup1(elem)
+        elem.children.remove if elem&.children&.text&.strip&.empty?
+        key_extract_locality(elem)
+        if /:/.match?(elem["key"]) then concept_termbase_cleanup(elem)
+        elsif refid? elem["key"] then concept_eref_cleanup(elem)
+        else concept_xref_cleanup(elem)
+        end
+        elem.delete("key")
+      end
+
+      def related_cleanup(xmldoc)
+        xmldoc.xpath("//related[not(termxref)]").each do |x|
+          term = x.at("./refterm")
+          term.replace("<preferred>#{term_expr(term.children.to_xml)}"\
+                       "</preferred>")
+          concept_cleanup1(x)
+        end
+      end
+
+      def key_extract_locality(elem)
+        return unless /,/.match?(elem["key"])
+
+        elem.add_child("<locality>#{elem['key'].sub(/^[^,]+,/, '')}</locality>")
+        elem["key"] = elem["key"].sub(/,.*$/, "")
+      end
+
+      def concept_termbase_cleanup(elem)
+        t = elem&.at("./xrefrender")&.remove&.children
+        termbase, key = elem["key"].split(/:/, 2)
+        elem.add_child(%(<termref base="#{termbase}" target="#{key}">) +
+                       "#{t&.to_xml}</termref>")
+      end
+
+      def concept_xref_cleanup(elem)
+        t = elem&.at("./xrefrender")&.remove&.children
+        elem.add_child(%(<xref target="#{elem['key']}">#{t&.to_xml}</xref>))
+      end
+
+      def concept_eref_cleanup(elem)
+        t = elem&.at("./xrefrender")&.remove&.children&.to_xml
+        l = elem&.at("./locality")&.remove&.children&.to_xml
+        elem.add_child "<eref bibitemid='#{elem['key']}'>#{l}</eref>"
+        extract_localities(elem.elements[-1])
+        elem.elements[-1].add_child(t) if t
+      end
+
+      def to_xreftarget(str)
+        return Metanorma::Utils::to_ncname(str) unless /^[^#]+#.+$/.match?(str)
+
+        /^(?<pref>[^#]+)#(?<suff>.+)$/ =~ str
+        pref = pref.gsub(%r([#{Metanorma::Utils::NAMECHAR}])o, "_")
+        suff = suff.gsub(%r([#{Metanorma::Utils::NAMECHAR}])o, "_")
+        "#{pref}##{suff}"
+      end
+
+      IDREF = "//*/@id | //review/@from | //review/@to | "\
+              "//callout/@target | //citation/@bibitemid | "\
+              "//eref/@bibitemid".freeze
+
+      def anchor_cleanup(elem)
+        anchor_cleanup1(elem)
+        xreftarget_cleanup(elem)
+        contenthash_id_cleanup(elem)
+      end
+
+      def anchor_cleanup1(elem)
+        elem.xpath(IDREF).each do |s|
+          if (ret = Metanorma::Utils::to_ncname(s.value)) != (orig = s.value)
+            s.value = ret
+            output = s.parent.dup
+            output.children.remove
+            @log.add("Anchors", s.parent,
+                     "normalised identifier in #{output} from #{orig}")
+          end
+        end
+      end
+
+      def xreftarget_cleanup(elem)
+        elem.xpath("//xref/@target").each do |s|
+          if (ret = to_xreftarget(s.value)) != (orig = s.value)
+            s.value = ret
+            output = s.parent.dup
+            output.children.remove
+            @log.add("Anchors", s.parent,
+                     "normalised identifier in #{output} from #{orig}")
+          end
+        end
+      end
+
+      def guid?(str)
+        /^_[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+          .match?(str)
+      end
+
+      def contenthash_id_cleanup(doc)
+        ids = contenthash_id_make(doc)
+        contenthash_id_update_refs(doc, ids)
+      end
+
+      def contenthash_id_make(doc)
+        doc.xpath("//*[@id]").each_with_object({}) do |x, m|
+          next unless guid?(x["id"])
+
+          m[x["id"]] = contenthash(x)
+          x["id"] = m[x["id"]]
+        end
+      end
+
+      def contenthash_id_update_refs(doc, ids)
+        [%w(review from), %w(review to), %w(callout target), %w(eref bibitemid),
+         %w(citation bibitemid), %w(xref target), %w(xref to)].each do |a|
+          doc.xpath("//#{a[0]}").each do |x|
+            ids[x[a[1]]] and x[a[1]] = ids[x[a[1]]]
+          end
+        end
+      end
+
+      def contenthash(elem)
+        Digest::MD5.hexdigest("#{elem.path}////#{elem.text}")
+          .sub(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, "_\\1-\\2-\\3-\\4-\\5")
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_maths.rb

@@ -0,0 +1,222 @@
+require "nokogiri"
+require "pathname"
+require "html2doc"
+require "asciimath2unitsml"
+require_relative "./cleanup_block"
+require_relative "./cleanup_footnotes"
+require_relative "./cleanup_ref"
+require_relative "./cleanup_ref_dl"
+require_relative "./cleanup_boilerplate"
+require_relative "./cleanup_section"
+require_relative "./cleanup_terms"
+require_relative "./cleanup_inline"
+require_relative "./cleanup_amend"
+require "relaton_iev"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def asciimath2mathml(text)
+        text = text.gsub(%r{<stem type="AsciiMath">(.+?)</stem>}m) do
+          "<amathstem>#{HTMLEntities.new.decode($1)}</amathstem>"
+        end
+        text = Html2Doc.asciimath_to_mathml(text,
+                                            ["<amathstem>", "</amathstem>"])
+        x =  Nokogiri::XML(text)
+        x.xpath("//*[local-name() = 'math'][not(parent::stem)]").each do |y|
+          y.wrap("<stem type='MathML'></stem>")
+        end
+        x.to_xml
+      end
+
+      def xml_unescape_mathml(xml)
+        return if xml.children.any? { |y| y.element? }
+
+        math = xml.text.gsub(/&lt;/, "<").gsub(/&gt;/, ">")
+          .gsub(/&quot;/, '"').gsub(/&apos;/, "'").gsub(/&amp;/, "&")
+          .gsub(/<[^: \r\n\t\/]+:/, "<").gsub(/<\/[^ \r\n\t:]+:/, "</")
+        xml.children = math
+      end
+
+      MATHML_NS = "http://www.w3.org/1998/Math/MathML".freeze
+
+      def mathml_preserve_space(math)
+        math.xpath(".//m:mtext", "m" => MATHML_NS).each do |x|
+          x.children = x.children.to_xml
+            .gsub(/^\s/, "&#xA0;").gsub(/\s$/, "&#xA0;")
+        end
+      end
+
+      def mathml_namespace(stem)
+        stem.xpath("./math").each { |x| x.default_namespace = MATHML_NS }
+      end
+
+      def mathml_mi_italics
+        { uppergreek: true, upperroman: true,
+          lowergreek: true, lowerroman: true }
+      end
+
+      # presuppose multichar mi upright, singlechar mi MathML default italic
+      def mathml_italicise(xml)
+        xml.xpath(".//m:mi[not(ancestor::*[@mathvariant])]",
+                  "m" => MATHML_NS).each do |i|
+          char = HTMLEntities.new.decode(i.text)
+          i["mathvariant"] = "normal" if mi_italicise?(char)
+        end
+      end
+
+      def mi_italicise?(char)
+        return false if char.length > 1
+
+        if /\p{Greek}/.match?(char)
+          (/\p{Lower}/.match(char) && !mathml_mi_italics[:lowergreek]) ||
+            (/\p{Upper}/.match(char) && !mathml_mi_italics[:uppergreek])
+        elsif /\p{Latin}/.match?(char)
+          (/\p{Lower}/.match(char) && !mathml_mi_italics[:lowerroman]) ||
+            (/\p{Upper}/.match(char) && !mathml_mi_italics[:upperroman])
+        else false
+        end
+      end
+
+      UNITSML_NS = "https://schema.unitsml.org/unitsml/1.0".freeze
+
+      def add_misc_container(xmldoc)
+        unless ins = xmldoc.at("//misc-container")
+          a = xmldoc.at("//termdocsource") || xmldoc.at("//bibdata")
+          a.next = "<misc-container/>"
+          ins = xmldoc.at("//misc-container")
+        end
+        ins
+      end
+
+      def mathml_unitsML(xmldoc)
+        return unless xmldoc.at(".//m:*", "m" => UNITSML_NS)
+
+        misc = add_misc_container(xmldoc)
+        unitsml = misc.add_child("<UnitsML xmlns='#{UNITSML_NS}'/>").first
+        %w(Unit CountedItem Quantity Dimension Prefix).each do |t|
+          gather_unitsml(unitsml, xmldoc, t)
+        end
+      end
+
+      def gather_unitsml(unitsml, xmldoc, tag)
+        tags = xmldoc.xpath(".//m:#{tag}", "m" => UNITSML_NS)
+          .each_with_object({}) do |x, m|
+          m[x["id"]] = x.remove
+        end
+        return if tags.empty?
+
+        set = unitsml.add_child("<#{tag}Set/>").first
+        tags.each_value { |v| set << v }
+      end
+
+      def asciimath2unitsml_options
+        { multiplier: :space }
+      end
+
+      def mathvariant_override(inner, outer)
+        case outer
+        when "bold"
+          case inner
+          when "normal" then "bold"
+          when "italic" then "bold-italic"
+          when "fraktur" then "bold-fraktur"
+          when "script" then "bold-script"
+          when "sans-serif" then "bold-sans-serif"
+          when "sans-serif-italic" then "sans-serif-bold-italic"
+          else inner
+          end
+        when "italic"
+          case inner
+          when "normal" then "italic"
+          when "bold" then "bold-italic"
+          when "sans-serif" then "sans-serif-italic"
+          when "bold-sans-serif" then "sans-serif-bold-italic"
+          else inner
+          end
+        when "bold-italic"
+          case inner
+          when "normal", "bold", "italic" then "bold-italic"
+          when "sans-serif", "bold-sans-serif", "sans-serif-italic"
+            "sans-serif-bold-italic"
+          else inner
+          end
+        when "fraktur"
+          case inner
+          when "normal" then "fraktur"
+          when "bold" then "bold-fraktur"
+          else inner
+          end
+        when "bold-fraktur"
+          case inner
+          when "normal", "fraktur" then "bold-fraktur"
+          else inner
+          end
+        when "script"
+          case inner
+          when "normal" then "script"
+          when "bold" then "bold-script"
+          else inner
+          end
+        when "bold-script"
+          case inner
+          when "normal", "script" then "bold-script"
+          else inner
+          end
+        when "sans-serif"
+          case inner
+          when "normal" then "sans-serif"
+          when "bold" then "bold-sans-serif"
+          when "italic" then "sans-serif-italic"
+          when "bold-italic" then "sans-serif-bold-italic"
+          else inner
+          end
+        when "bold-sans-serif"
+          case inner
+          when "normal", "bold", "sans-serif" then "bold-sans-serif"
+          when "italic", "bold-italic", "sans-serif-italic"
+            "sans-serif-bold-italic"
+          else inner
+          end
+        when "sans-serif-italic"
+          case inner
+          when "normal", "italic", "sans-serif" then "sans-serif-italic"
+          when "bold", "bold-italic", "sans-serif-bold"
+            "sans-serif-bold-italic"
+          else inner
+          end
+        when "sans-serif-bold-italic"
+          case inner
+          when "normal", "italic", "sans-serif", "sans-serif-italic",
+          "bold", "bold-italic", "sans-serif-bold"
+            "sans-serif-bold-italic"
+          else inner
+          end
+        else inner
+        end
+      end
+
+      def mathml_mathvariant(math)
+        math.xpath(".//*[@mathvariant]").each do |outer|
+          outer.xpath(".//*[@mathvariant]").each do |inner|
+            inner["mathvariant"] =
+              mathvariant_override(outer["mathvariant"], inner["mathvariant"])
+          end
+        end
+      end
+
+      def mathml_cleanup(xmldoc)
+        unitsml = Asciimath2UnitsML::Conv.new(asciimath2unitsml_options)
+        xmldoc.xpath("//stem[@type = 'MathML']").each do |x|
+          xml_unescape_mathml(x)
+          mathml_namespace(x)
+          mathml_preserve_space(x)
+          unitsml.MathML2UnitsML(x)
+          mathml_mathvariant(x)
+          mathml_italicise(x)
+        end
+        mathml_unitsML(xmldoc)
+      end
+    end
+  end
+end