metanorma-standoc 1.11.1 → 2.0.0

data/lib/metanorma/standoc/cleanup_inline.rb

@@ -0,0 +1,190 @@
+require "metanorma-utils"
+require "digest"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def empty_text_before_first_element(elem)
+        elem.children.each do |c|
+          return false if c.text? && /\S/.match(c.text)
+          return true if c.element?
+        end
+        true
+      end
+
+      def strip_initial_space(elem)
+        return unless elem.children[0].text?
+
+        if /\S/.match?(elem.children[0].text)
+          elem.children[0].content = elem.children[0].text.gsub(/^ /, "")
+        else
+          elem.children[0].remove
+        end
+      end
+
+      def bookmark_cleanup(xmldoc)
+        li_bookmark_cleanup(xmldoc)
+        dt_bookmark_cleanup(xmldoc)
+      end
+
+      def bookmark_to_id(elem, bookmark)
+        parent = bookmark.parent
+        elem["id"] = bookmark.remove["id"]
+        strip_initial_space(parent)
+      end
+
+      def li_bookmark_cleanup(xmldoc)
+        xmldoc.xpath("//li[descendant::bookmark]").each do |x|
+          if x.at("./*[1][local-name() = 'p']/"\
+                  "*[1][local-name() = 'bookmark']") &&
+              empty_text_before_first_element(x.elements[0])
+            bookmark_to_id(x, x.elements[0].elements[0])
+          end
+        end
+      end
+
+      def dt_bookmark_cleanup(xmldoc)
+        xmldoc.xpath("//dt[descendant::bookmark]").each do |x|
+          if x.at("./*[1][local-name() = 'p']/"\
+                  "*[1][local-name() = 'bookmark']") &&
+              empty_text_before_first_element(x.elements[0])
+            bookmark_to_id(x, x.elements[0].elements[0])
+          elsif x.at("./*[1][local-name() = 'bookmark']") &&
+              empty_text_before_first_element(x)
+            bookmark_to_id(x, x.elements[0])
+          end
+        end
+      end
+
+      def concept_cleanup(xmldoc)
+        xmldoc.xpath("//concept[not(termxref)]").each do |x|
+          term = x.at("./refterm")
+          term&.remove if term&.text&.empty?
+          concept_cleanup1(x)
+        end
+      end
+
+      def concept_cleanup1(elem)
+        elem.children.remove if elem&.children&.text&.strip&.empty?
+        key_extract_locality(elem)
+        if /:/.match?(elem["key"]) then concept_termbase_cleanup(elem)
+        elsif refid? elem["key"] then concept_eref_cleanup(elem)
+        else concept_xref_cleanup(elem)
+        end
+        elem.delete("key")
+      end
+
+      def related_cleanup(xmldoc)
+        xmldoc.xpath("//related[not(termxref)]").each do |x|
+          term = x.at("./refterm")
+          term.replace("<preferred>#{term_expr(term.children.to_xml)}"\
+                       "</preferred>")
+          concept_cleanup1(x)
+        end
+      end
+
+      def key_extract_locality(elem)
+        return unless /,/.match?(elem["key"])
+
+        elem.add_child("<locality>#{elem['key'].sub(/^[^,]+,/, '')}</locality>")
+        elem["key"] = elem["key"].sub(/,.*$/, "")
+      end
+
+      def concept_termbase_cleanup(elem)
+        t = elem&.at("./xrefrender")&.remove&.children
+        termbase, key = elem["key"].split(/:/, 2)
+        elem.add_child(%(<termref base="#{termbase}" target="#{key}">) +
+                       "#{t&.to_xml}</termref>")
+      end
+
+      def concept_xref_cleanup(elem)
+        t = elem&.at("./xrefrender")&.remove&.children
+        elem.add_child(%(<xref target="#{elem['key']}">#{t&.to_xml}</xref>))
+      end
+
+      def concept_eref_cleanup(elem)
+        t = elem&.at("./xrefrender")&.remove&.children&.to_xml
+        l = elem&.at("./locality")&.remove&.children&.to_xml
+        elem.add_child "<eref bibitemid='#{elem['key']}'>#{l}</eref>"
+        extract_localities(elem.elements[-1])
+        elem.elements[-1].add_child(t) if t
+      end
+
+      def to_xreftarget(str)
+        return Metanorma::Utils::to_ncname(str) unless /^[^#]+#.+$/.match?(str)
+
+        /^(?<pref>[^#]+)#(?<suff>.+)$/ =~ str
+        pref = pref.gsub(%r([#{Metanorma::Utils::NAMECHAR}])o, "_")
+        suff = suff.gsub(%r([#{Metanorma::Utils::NAMECHAR}])o, "_")
+        "#{pref}##{suff}"
+      end
+
+      IDREF = "//*/@id | //review/@from | //review/@to | "\
+              "//callout/@target | //citation/@bibitemid | "\
+              "//eref/@bibitemid".freeze
+
+      def anchor_cleanup(elem)
+        anchor_cleanup1(elem)
+        xreftarget_cleanup(elem)
+        contenthash_id_cleanup(elem)
+      end
+
+      def anchor_cleanup1(elem)
+        elem.xpath(IDREF).each do |s|
+          if (ret = Metanorma::Utils::to_ncname(s.value)) != (orig = s.value)
+            s.value = ret
+            output = s.parent.dup
+            output.children.remove
+            @log.add("Anchors", s.parent,
+                     "normalised identifier in #{output} from #{orig}")
+          end
+        end
+      end
+
+      def xreftarget_cleanup(elem)
+        elem.xpath("//xref/@target").each do |s|
+          if (ret = to_xreftarget(s.value)) != (orig = s.value)
+            s.value = ret
+            output = s.parent.dup
+            output.children.remove
+            @log.add("Anchors", s.parent,
+                     "normalised identifier in #{output} from #{orig}")
+          end
+        end
+      end
+
+      def guid?(str)
+        /^_[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+          .match?(str)
+      end
+
+      def contenthash_id_cleanup(doc)
+        ids = contenthash_id_make(doc)
+        contenthash_id_update_refs(doc, ids)
+      end
+
+      def contenthash_id_make(doc)
+        doc.xpath("//*[@id]").each_with_object({}) do |x, m|
+          next unless guid?(x["id"])
+
+          m[x["id"]] = contenthash(x)
+          x["id"] = m[x["id"]]
+        end
+      end
+
+      def contenthash_id_update_refs(doc, ids)
+        [%w(review from), %w(review to), %w(callout target), %w(eref bibitemid),
+         %w(citation bibitemid), %w(xref target), %w(xref to)].each do |a|
+          doc.xpath("//#{a[0]}").each do |x|
+            ids[x[a[1]]] and x[a[1]] = ids[x[a[1]]]
+          end
+        end
+      end
+
+      def contenthash(elem)
+        Digest::MD5.hexdigest("#{elem.path}////#{elem.text}")
+          .sub(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, "_\\1-\\2-\\3-\\4-\\5")
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_maths.rb

@@ -0,0 +1,222 @@
+require "nokogiri"
+require "pathname"
+require "html2doc"
+require "asciimath2unitsml"
+require_relative "./cleanup_block"
+require_relative "./cleanup_footnotes"
+require_relative "./cleanup_ref"
+require_relative "./cleanup_ref_dl"
+require_relative "./cleanup_boilerplate"
+require_relative "./cleanup_section"
+require_relative "./cleanup_terms"
+require_relative "./cleanup_inline"
+require_relative "./cleanup_amend"
+require "relaton_iev"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def asciimath2mathml(text)
+        text = text.gsub(%r{<stem type="AsciiMath">(.+?)</stem>}m) do
+          "<amathstem>#{HTMLEntities.new.decode($1)}</amathstem>"
+        end
+        text = Html2Doc.asciimath_to_mathml(text,
+                                            ["<amathstem>", "</amathstem>"])
+        x =  Nokogiri::XML(text)
+        x.xpath("//*[local-name() = 'math'][not(parent::stem)]").each do |y|
+          y.wrap("<stem type='MathML'></stem>")
+        end
+        x.to_xml
+      end
+
+      def xml_unescape_mathml(xml)
+        return if xml.children.any? { |y| y.element? }
+
+        math = xml.text.gsub(/&lt;/, "<").gsub(/&gt;/, ">")
+          .gsub(/&quot;/, '"').gsub(/&apos;/, "'").gsub(/&amp;/, "&")
+          .gsub(/<[^: \r\n\t\/]+:/, "<").gsub(/<\/[^ \r\n\t:]+:/, "</")
+        xml.children = math
+      end
+
+      MATHML_NS = "http://www.w3.org/1998/Math/MathML".freeze
+
+      def mathml_preserve_space(math)
+        math.xpath(".//m:mtext", "m" => MATHML_NS).each do |x|
+          x.children = x.children.to_xml
+            .gsub(/^\s/, "&#xA0;").gsub(/\s$/, "&#xA0;")
+        end
+      end
+
+      def mathml_namespace(stem)
+        stem.xpath("./math").each { |x| x.default_namespace = MATHML_NS }
+      end
+
+      def mathml_mi_italics
+        { uppergreek: true, upperroman: true,
+          lowergreek: true, lowerroman: true }
+      end
+
+      # presuppose multichar mi upright, singlechar mi MathML default italic
+      def mathml_italicise(xml)
+        xml.xpath(".//m:mi[not(ancestor::*[@mathvariant])]",
+                  "m" => MATHML_NS).each do |i|
+          char = HTMLEntities.new.decode(i.text)
+          i["mathvariant"] = "normal" if mi_italicise?(char)
+        end
+      end
+
+      def mi_italicise?(char)
+        return false if char.length > 1
+
+        if /\p{Greek}/.match?(char)
+          (/\p{Lower}/.match(char) && !mathml_mi_italics[:lowergreek]) ||
+            (/\p{Upper}/.match(char) && !mathml_mi_italics[:uppergreek])
+        elsif /\p{Latin}/.match?(char)
+          (/\p{Lower}/.match(char) && !mathml_mi_italics[:lowerroman]) ||
+            (/\p{Upper}/.match(char) && !mathml_mi_italics[:upperroman])
+        else false
+        end
+      end
+
+      UNITSML_NS = "https://schema.unitsml.org/unitsml/1.0".freeze
+
+      def add_misc_container(xmldoc)
+        unless ins = xmldoc.at("//misc-container")
+          a = xmldoc.at("//termdocsource") || xmldoc.at("//bibdata")
+          a.next = "<misc-container/>"
+          ins = xmldoc.at("//misc-container")
+        end
+        ins
+      end
+
+      def mathml_unitsML(xmldoc)
+        return unless xmldoc.at(".//m:*", "m" => UNITSML_NS)
+
+        misc = add_misc_container(xmldoc)
+        unitsml = misc.add_child("<UnitsML xmlns='#{UNITSML_NS}'/>").first
+        %w(Unit CountedItem Quantity Dimension Prefix).each do |t|
+          gather_unitsml(unitsml, xmldoc, t)
+        end
+      end
+
+      def gather_unitsml(unitsml, xmldoc, tag)
+        tags = xmldoc.xpath(".//m:#{tag}", "m" => UNITSML_NS)
+          .each_with_object({}) do |x, m|
+          m[x["id"]] = x.remove
+        end
+        return if tags.empty?
+
+        set = unitsml.add_child("<#{tag}Set/>").first
+        tags.each_value { |v| set << v }
+      end
+
+      def asciimath2unitsml_options
+        { multiplier: :space }
+      end
+
+      def mathvariant_override(inner, outer)
+        case outer
+        when "bold"
+          case inner
+          when "normal" then "bold"
+          when "italic" then "bold-italic"
+          when "fraktur" then "bold-fraktur"
+          when "script" then "bold-script"
+          when "sans-serif" then "bold-sans-serif"
+          when "sans-serif-italic" then "sans-serif-bold-italic"
+          else inner
+          end
+        when "italic"
+          case inner
+          when "normal" then "italic"
+          when "bold" then "bold-italic"
+          when "sans-serif" then "sans-serif-italic"
+          when "bold-sans-serif" then "sans-serif-bold-italic"
+          else inner
+          end
+        when "bold-italic"
+          case inner
+          when "normal", "bold", "italic" then "bold-italic"
+          when "sans-serif", "bold-sans-serif", "sans-serif-italic"
+            "sans-serif-bold-italic"
+          else inner
+          end
+        when "fraktur"
+          case inner
+          when "normal" then "fraktur"
+          when "bold" then "bold-fraktur"
+          else inner
+          end
+        when "bold-fraktur"
+          case inner
+          when "normal", "fraktur" then "bold-fraktur"
+          else inner
+          end
+        when "script"
+          case inner
+          when "normal" then "script"
+          when "bold" then "bold-script"
+          else inner
+          end
+        when "bold-script"
+          case inner
+          when "normal", "script" then "bold-script"
+          else inner
+          end
+        when "sans-serif"
+          case inner
+          when "normal" then "sans-serif"
+          when "bold" then "bold-sans-serif"
+          when "italic" then "sans-serif-italic"
+          when "bold-italic" then "sans-serif-bold-italic"
+          else inner
+          end
+        when "bold-sans-serif"
+          case inner
+          when "normal", "bold", "sans-serif" then "bold-sans-serif"
+          when "italic", "bold-italic", "sans-serif-italic"
+            "sans-serif-bold-italic"
+          else inner
+          end
+        when "sans-serif-italic"
+          case inner
+          when "normal", "italic", "sans-serif" then "sans-serif-italic"
+          when "bold", "bold-italic", "sans-serif-bold"
+            "sans-serif-bold-italic"
+          else inner
+          end
+        when "sans-serif-bold-italic"
+          case inner
+          when "normal", "italic", "sans-serif", "sans-serif-italic",
+          "bold", "bold-italic", "sans-serif-bold"
+            "sans-serif-bold-italic"
+          else inner
+          end
+        else inner
+        end
+      end
+
+      def mathml_mathvariant(math)
+        math.xpath(".//*[@mathvariant]").each do |outer|
+          outer.xpath(".//*[@mathvariant]").each do |inner|
+            inner["mathvariant"] =
+              mathvariant_override(outer["mathvariant"], inner["mathvariant"])
+          end
+        end
+      end
+
+      def mathml_cleanup(xmldoc)
+        unitsml = Asciimath2UnitsML::Conv.new(asciimath2unitsml_options)
+        xmldoc.xpath("//stem[@type = 'MathML']").each do |x|
+          xml_unescape_mathml(x)
+          mathml_namespace(x)
+          mathml_preserve_space(x)
+          unitsml.MathML2UnitsML(x)
+          mathml_mathvariant(x)
+          mathml_italicise(x)
+        end
+        mathml_unitsML(xmldoc)
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_ref.rb

@@ -0,0 +1,170 @@
+require "set"
+require "relaton_bib"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def biblio_reorder(xmldoc)
+        xmldoc.xpath("//references[@normative = 'false']").each do |r|
+          biblio_reorder1(r)
+        end
+      end
+
+      def biblio_reorder1(refs)
+        fold_notes_into_biblio(refs)
+        bib = sort_biblio(refs.xpath("./bibitem"))
+        insert = refs&.at("./bibitem")&.previous_element
+        refs.xpath("./bibitem").each(&:remove)
+        bib.reverse.each do |b|
+          insert and insert.next = b.to_xml or
+            refs.children.first.add_previous_sibling b.to_xml
+        end
+        extract_notes_from_biblio(refs)
+        refs.xpath("./references").each { |r| biblio_reorder1(r) }
+      end
+
+      def fold_notes_into_biblio(refs)
+        refs.xpath("./bibitem").each do |r|
+          while r&.next_element&.name == "note"
+            r.next_element["appended"] = true
+            r << r.next_element.remove
+          end
+        end
+      end
+
+      def extract_notes_from_biblio(refs)
+        refs.xpath("./bibitem").each do |r|
+          r.xpath("./note[@appended]").reverse.each do |n|
+            n.delete("appended")
+            r.next = n
+          end
+        end
+      end
+
+      def sort_biblio(bib)
+        bib
+      end
+
+      # default presuppose that all citations in biblio numbered
+      # consecutively, but that standards codes are preserved as is:
+      # only numeric references are renumbered
+      def biblio_renumber(xmldoc)
+        i = 0
+        xmldoc.xpath("//bibliography//references | //clause//references | "\
+                     "//annex//references").each do |r|
+          next if r["normative"] == "true"
+
+          r.xpath("./bibitem[not(@hidden = 'true')]").each do |b|
+            i += 1
+            next unless docid = b.at("./docidentifier[@type = 'metanorma']")
+            next unless /^\[\d+\]$/.match?(docid.text)
+
+            docid.children = "[#{i}]"
+          end
+        end
+      end
+
+      # move ref before p
+      def ref_cleanup(xmldoc)
+        xmldoc.xpath("//p/ref").each do |r|
+          parent = r.parent
+          parent.previous = r.remove
+        end
+      end
+
+      def normref_cleanup(xmldoc)
+        r = xmldoc.at(self.class::NORM_REF) || return
+        preface = ((r.xpath("./title/following-sibling::*") & # intersection
+                    r.xpath("./bibitem[1]/preceding-sibling::*")) -
+        r.xpath("./note[@type = 'boilerplate']/descendant-or-self::*"))
+        preface.each(&:remove)
+      end
+
+      def biblio_cleanup(xmldoc)
+        biblio_reorder(xmldoc)
+        biblio_nested(xmldoc)
+        biblio_renumber(xmldoc)
+        biblio_no_ext(xmldoc)
+      end
+
+      def biblio_no_ext(xmldoc)
+        xmldoc.xpath("//bibitem/ext").each(&:remove)
+      end
+
+      def biblio_nested(xmldoc)
+        xmldoc.xpath("//references[references]").each do |t|
+          t.name = "clause"
+          t.xpath("./references").each { |r| r["normative"] = t["normative"] }
+          t.delete("normative")
+        end
+      end
+
+      def format_ref(ref, type)
+        return @isodoc.docid_prefix(type, ref) if type != "metanorma"
+        return "[#{ref}]" if /^\d+$/.match(ref) && !/^\[.*\]$/.match(ref)
+
+        ref
+      end
+
+      ISO_PUBLISHER_XPATH =
+        "./contributor[role/@type = 'publisher']/"\
+        "organization[abbreviation = 'ISO' or abbreviation = 'IEC' or "\
+        "name = 'International Organization for Standardization' or "\
+        "name = 'International Electrotechnical Commission']".freeze
+
+      def reference_names(xmldoc)
+        xmldoc.xpath("//bibitem[not(ancestor::bibitem)]").each do |ref|
+          # isopub = ref.at(ISO_PUBLISHER_XPATH)
+          docid = ref.at("./docidentifier[@type = 'metanorma']") ||
+            ref.at("./docidentifier[not(@type = 'DOI')]") or next
+          reference = format_ref(docid.text, docid["type"])
+          @anchors[ref["id"]] = { xref: reference }
+        end
+      end
+
+      def fetch_termbase(_termbase, _id)
+        ""
+      end
+
+      def read_local_bibitem(uri)
+        return nil if %r{^https?://}.match?(uri)
+
+        file = "#{@localdir}#{uri}.rxl"
+        File.file?(file) or file = "#{@localdir}#{uri}.xml"
+        File.file?(file) or return nil
+        xml = Nokogiri::XML(File.read(file, encoding: "utf-8"))
+        ret = xml.at("//*[local-name() = 'bibdata']") or return nil
+        ret = Nokogiri::XML(ret.to_xml
+          .sub(%r{(<bibdata[^>]*?) xmlns=("[^"]+"|'[^']+')}, "\\1")).root
+        ret.name = "bibitem"
+        ins = ret.at("./*[local-name() = 'docidentifier']") or return nil
+        ins.previous = %{<uri type="citation">#{uri}</uri>}
+        ret&.at("./*[local-name() = 'ext']")&.remove
+        ret
+      end
+
+      # if citation uri points to local file, get bibitem from it
+      def fetch_local_bibitem(xmldoc)
+        xmldoc.xpath("//bibitem[formattedref][uri[@type = 'citation']]")
+          .each do |b|
+          uri = b&.at("./uri[@type = 'citation']")&.text
+          bibitem = read_local_bibitem(uri) or next
+          bibitem["id"] = b["id"]
+          b.replace(bibitem)
+        end
+      end
+
+      def bibitem_nested_id(xmldoc)
+        xmldoc.xpath("//bibitem//bibitem").each do |b|
+          b.delete("id")
+        end
+      end
+
+      def bibitem_cleanup(xmldoc)
+        bibitem_nested_id(xmldoc)
+        ref_dl_cleanup(xmldoc)
+        fetch_local_bibitem(xmldoc)
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_ref_dl.rb

@@ -0,0 +1,104 @@
+require "set"
+require "relaton_bib"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def ref_dl_cleanup(xmldoc)
+        xmldoc.xpath("//clause[@bibitem = 'true']").each do |c|
+          bib = dl_bib_extract(c) or next
+          validate_ref_dl(bib, c)
+          bibitemxml = RelatonBib::BibliographicItem.from_hash(bib).to_xml or next
+          bibitem = Nokogiri::XML(bibitemxml)
+          bibitem.root["id"] = c["id"] if c["id"] && !/^_/.match(c["id"])
+          c.replace(bibitem.root)
+        end
+      end
+
+      def validate_ref_dl(bib, c)
+        id = bib["id"]
+        id ||= c["id"] unless /^_/.match?(c["id"]) # do not accept implicit id
+        unless id
+          @log.add("Anchors", c,
+                   "The following reference is missing an anchor:\n" + c.to_xml)
+          return
+        end
+        @refids << id
+        bib["title"] or
+          @log.add("Bibliography", c, "Reference #{id} is missing a title")
+        bib["docid"] or
+          @log.add("Bibliography", c,
+                   "Reference #{id} is missing a document identifier (docid)")
+      end
+
+      def extract_from_p(tag, bib, key)
+        return unless bib[tag]
+
+        "<#{key}>#{bib[tag].at('p').children}</#{key}>"
+      end
+
+      # if the content is a single paragraph, replace it with its children
+      # single links replaced with uri
+      def p_unwrap(para)
+        elems = para.elements
+        if elems.size == 1 && elems[0].name == "p"
+          link_unwrap(elems[0]).children.to_xml.strip
+        else
+          para.to_xml.strip
+        end
+      end
+
+      def link_unwrap(para)
+        elems = para.elements
+        if elems.size == 1 && elems[0].name == "link"
+          para.at("./link").replace(elems[0]["target"].strip)
+        end
+        para
+      end
+
+      def dd_bib_extract(dtd)
+        return nil if dtd.children.empty?
+
+        dtd.at("./dl") and return dl_bib_extract(dtd)
+        elems = dtd.remove.elements
+        return p_unwrap(dtd) unless elems.size == 1 &&
+          %w(ol ul).include?(elems[0].name)
+
+        ret = []
+        elems[0].xpath("./li").each do |li|
+          ret << p_unwrap(li)
+        end
+        ret
+      end
+
+      def add_to_hash(bib, key, val)
+        Metanorma::Utils::set_nested_value(bib, key.split("."), val)
+      end
+
+      # definition list, with at most one level of unordered lists
+      def dl_bib_extract(c, nested = false)
+        dl = c.at("./dl") or return
+        bib = {}
+        key = ""
+        dl.xpath("./dt | ./dd").each do |dtd|
+          (dtd.name == "dt" and key = dtd.text.sub(/:+$/, "")) or
+            add_to_hash(bib, key, dd_bib_extract(dtd))
+        end
+        c.xpath("./clause").each do |c1|
+          key = c1&.at("./title")&.text&.downcase&.strip
+          next unless %w(contributor relation series).include? key
+
+          add_to_hash(bib, key, dl_bib_extract(c1, true))
+        end
+        if !nested && c.at("./title")
+          title = c.at("./title").remove.children.to_xml
+          bib["title"] = [bib["title"]] if bib["title"].is_a? Hash
+          bib["title"] = [bib["title"]] if bib["title"].is_a? String
+          bib["title"] = [] unless bib["title"]
+          bib["title"] << title if !title.empty?
+        end
+        bib
+      end
+    end
+  end
+end