metanorma-standoc 1.11.3 → 2.0.2

data/lib/metanorma/standoc/cleanup_ref.rb

@@ -0,0 +1,170 @@
+require "set"
+require "relaton_bib"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def biblio_reorder(xmldoc)
+        xmldoc.xpath("//references[@normative = 'false']").each do |r|
+          biblio_reorder1(r)
+        end
+      end
+
+      def biblio_reorder1(refs)
+        fold_notes_into_biblio(refs)
+        bib = sort_biblio(refs.xpath("./bibitem"))
+        insert = refs&.at("./bibitem")&.previous_element
+        refs.xpath("./bibitem").each(&:remove)
+        bib.reverse.each do |b|
+          insert and insert.next = b.to_xml or
+            refs.children.first.add_previous_sibling b.to_xml
+        end
+        extract_notes_from_biblio(refs)
+        refs.xpath("./references").each { |r| biblio_reorder1(r) }
+      end
+
+      def fold_notes_into_biblio(refs)
+        refs.xpath("./bibitem").each do |r|
+          while r&.next_element&.name == "note"
+            r.next_element["appended"] = true
+            r << r.next_element.remove
+          end
+        end
+      end
+
+      def extract_notes_from_biblio(refs)
+        refs.xpath("./bibitem").each do |r|
+          r.xpath("./note[@appended]").reverse.each do |n|
+            n.delete("appended")
+            r.next = n
+          end
+        end
+      end
+
+      def sort_biblio(bib)
+        bib
+      end
+
+      # default presuppose that all citations in biblio numbered
+      # consecutively, but that standards codes are preserved as is:
+      # only numeric references are renumbered
+      def biblio_renumber(xmldoc)
+        i = 0
+        xmldoc.xpath("//bibliography//references | //clause//references | "\
+                     "//annex//references").each do |r|
+          next if r["normative"] == "true"
+
+          r.xpath("./bibitem[not(@hidden = 'true')]").each do |b|
+            i += 1
+            next unless docid = b.at("./docidentifier[@type = 'metanorma']")
+            next unless /^\[\d+\]$/.match?(docid.text)
+
+            docid.children = "[#{i}]"
+          end
+        end
+      end
+
+      # move ref before p
+      def ref_cleanup(xmldoc)
+        xmldoc.xpath("//p/ref").each do |r|
+          parent = r.parent
+          parent.previous = r.remove
+        end
+      end
+
+      def normref_cleanup(xmldoc)
+        r = xmldoc.at(self.class::NORM_REF) || return
+        preface = ((r.xpath("./title/following-sibling::*") & # intersection
+                    r.xpath("./bibitem[1]/preceding-sibling::*")) -
+        r.xpath("./note[@type = 'boilerplate']/descendant-or-self::*"))
+        preface.each(&:remove)
+      end
+
+      def biblio_cleanup(xmldoc)
+        biblio_reorder(xmldoc)
+        biblio_nested(xmldoc)
+        biblio_renumber(xmldoc)
+        biblio_no_ext(xmldoc)
+      end
+
+      def biblio_no_ext(xmldoc)
+        xmldoc.xpath("//bibitem/ext").each(&:remove)
+      end
+
+      def biblio_nested(xmldoc)
+        xmldoc.xpath("//references[references]").each do |t|
+          t.name = "clause"
+          t.xpath("./references").each { |r| r["normative"] = t["normative"] }
+          t.delete("normative")
+        end
+      end
+
+      def format_ref(ref, type)
+        return @isodoc.docid_prefix(type, ref) if type != "metanorma"
+        return "[#{ref}]" if /^\d+$/.match(ref) && !/^\[.*\]$/.match(ref)
+
+        ref
+      end
+
+      ISO_PUBLISHER_XPATH =
+        "./contributor[role/@type = 'publisher']/"\
+        "organization[abbreviation = 'ISO' or abbreviation = 'IEC' or "\
+        "name = 'International Organization for Standardization' or "\
+        "name = 'International Electrotechnical Commission']".freeze
+
+      def reference_names(xmldoc)
+        xmldoc.xpath("//bibitem[not(ancestor::bibitem)]").each do |ref|
+          # isopub = ref.at(ISO_PUBLISHER_XPATH)
+          docid = ref.at("./docidentifier[@type = 'metanorma']") ||
+            ref.at("./docidentifier[not(@type = 'DOI')]") or next
+          reference = format_ref(docid.text, docid["type"])
+          @anchors[ref["id"]] = { xref: reference }
+        end
+      end
+
+      def fetch_termbase(_termbase, _id)
+        ""
+      end
+
+      def read_local_bibitem(uri)
+        return nil if %r{^https?://}.match?(uri)
+
+        file = "#{@localdir}#{uri}.rxl"
+        File.file?(file) or file = "#{@localdir}#{uri}.xml"
+        File.file?(file) or return nil
+        xml = Nokogiri::XML(File.read(file, encoding: "utf-8"))
+        ret = xml.at("//*[local-name() = 'bibdata']") or return nil
+        ret = Nokogiri::XML(ret.to_xml
+          .sub(%r{(<bibdata[^>]*?) xmlns=("[^"]+"|'[^']+')}, "\\1")).root
+        ret.name = "bibitem"
+        ins = ret.at("./*[local-name() = 'docidentifier']") or return nil
+        ins.previous = %{<uri type="citation">#{uri}</uri>}
+        ret&.at("./*[local-name() = 'ext']")&.remove
+        ret
+      end
+
+      # if citation uri points to local file, get bibitem from it
+      def fetch_local_bibitem(xmldoc)
+        xmldoc.xpath("//bibitem[formattedref][uri[@type = 'citation']]")
+          .each do |b|
+          uri = b&.at("./uri[@type = 'citation']")&.text
+          bibitem = read_local_bibitem(uri) or next
+          bibitem["id"] = b["id"]
+          b.replace(bibitem)
+        end
+      end
+
+      def bibitem_nested_id(xmldoc)
+        xmldoc.xpath("//bibitem//bibitem").each do |b|
+          b.delete("id")
+        end
+      end
+
+      def bibitem_cleanup(xmldoc)
+        bibitem_nested_id(xmldoc)
+        ref_dl_cleanup(xmldoc)
+        fetch_local_bibitem(xmldoc)
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_ref_dl.rb

@@ -0,0 +1,104 @@
+require "set"
+require "relaton_bib"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def ref_dl_cleanup(xmldoc)
+        xmldoc.xpath("//clause[@bibitem = 'true']").each do |c|
+          bib = dl_bib_extract(c) or next
+          validate_ref_dl(bib, c)
+          bibitemxml = RelatonBib::BibliographicItem.from_hash(bib).to_xml or next
+          bibitem = Nokogiri::XML(bibitemxml)
+          bibitem.root["id"] = c["id"] if c["id"] && !/^_/.match(c["id"])
+          c.replace(bibitem.root)
+        end
+      end
+
+      def validate_ref_dl(bib, c)
+        id = bib["id"]
+        id ||= c["id"] unless /^_/.match?(c["id"]) # do not accept implicit id
+        unless id
+          @log.add("Anchors", c,
+                   "The following reference is missing an anchor:\n" + c.to_xml)
+          return
+        end
+        @refids << id
+        bib["title"] or
+          @log.add("Bibliography", c, "Reference #{id} is missing a title")
+        bib["docid"] or
+          @log.add("Bibliography", c,
+                   "Reference #{id} is missing a document identifier (docid)")
+      end
+
+      def extract_from_p(tag, bib, key)
+        return unless bib[tag]
+
+        "<#{key}>#{bib[tag].at('p').children}</#{key}>"
+      end
+
+      # if the content is a single paragraph, replace it with its children
+      # single links replaced with uri
+      def p_unwrap(para)
+        elems = para.elements
+        if elems.size == 1 && elems[0].name == "p"
+          link_unwrap(elems[0]).children.to_xml.strip
+        else
+          para.to_xml.strip
+        end
+      end
+
+      def link_unwrap(para)
+        elems = para.elements
+        if elems.size == 1 && elems[0].name == "link"
+          para.at("./link").replace(elems[0]["target"].strip)
+        end
+        para
+      end
+
+      def dd_bib_extract(dtd)
+        return nil if dtd.children.empty?
+
+        dtd.at("./dl") and return dl_bib_extract(dtd)
+        elems = dtd.remove.elements
+        return p_unwrap(dtd) unless elems.size == 1 &&
+          %w(ol ul).include?(elems[0].name)
+
+        ret = []
+        elems[0].xpath("./li").each do |li|
+          ret << p_unwrap(li)
+        end
+        ret
+      end
+
+      def add_to_hash(bib, key, val)
+        Metanorma::Utils::set_nested_value(bib, key.split("."), val)
+      end
+
+      # definition list, with at most one level of unordered lists
+      def dl_bib_extract(c, nested = false)
+        dl = c.at("./dl") or return
+        bib = {}
+        key = ""
+        dl.xpath("./dt | ./dd").each do |dtd|
+          (dtd.name == "dt" and key = dtd.text.sub(/:+$/, "")) or
+            add_to_hash(bib, key, dd_bib_extract(dtd))
+        end
+        c.xpath("./clause").each do |c1|
+          key = c1&.at("./title")&.text&.downcase&.strip
+          next unless %w(contributor relation series).include? key
+
+          add_to_hash(bib, key, dl_bib_extract(c1, true))
+        end
+        if !nested && c.at("./title")
+          title = c.at("./title").remove.children.to_xml
+          bib["title"] = [bib["title"]] if bib["title"].is_a? Hash
+          bib["title"] = [bib["title"]] if bib["title"].is_a? String
+          bib["title"] = [] unless bib["title"]
+          bib["title"] << title if !title.empty?
+        end
+        bib
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_reqt.rb

@@ -0,0 +1,111 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def requirement_cleanup(xmldoc)
+        requirement_metadata(xmldoc)
+        requirement_inherit(xmldoc)
+        requirement_descriptions(xmldoc)
+      end
+
+      REQRECPER = "//requirement | //recommendation | //permission".freeze
+
+      def requirement_inherit(xmldoc)
+        xmldoc.xpath(REQRECPER).each do |r|
+          ins = requirement_inherit_insert(r)
+          r.xpath("./*//inherit").each { |i| ins.previous = i }
+        end
+      end
+
+      def requirement_inherit_insert(reqt)
+        ins = reqt.at("./classification") || reqt.at(
+          "./description | ./measurementtarget | ./specification | "\
+          "./verification | ./import | ./description | ./component | "\
+          "./requirement | ./recommendation | ./permission",
+        ) and return ins
+        requirement_inherit_insert1(reqt)
+      end
+
+      def requirement_inherit_insert1(reqt)
+        if t = reqt.at("./title")
+          t.next = " "
+          t.next
+        else
+          if reqt.children.empty? then reqt.add_child(" ")
+          else reqt.children.first.previous = " "
+          end
+          reqt.children.first
+        end
+      end
+
+      def requirement_descriptions(xmldoc)
+        xmldoc.xpath(REQRECPER).each do |r|
+          r.xpath(".//p[not(./*)][normalize-space(.)='']").each(&:remove)
+          r.children.each do |e|
+            requirement_description_wrap(r, e)
+          end
+          requirement_description_cleanup1(r)
+        end
+      end
+
+      def requirement_description_wrap(reqt, text)
+        return if (text.element? && (reqt_subpart(text.name) ||
+                %w(requirement recommendation
+                   permission).include?(text.name))) ||
+          (text.text.strip.empty? && !text.at(".//xref | .//eref | .//link"))
+
+        t = Nokogiri::XML::Element.new("description", reqt)
+        text.before(t)
+        t.children = text.remove
+      end
+
+      def requirement_description_cleanup1(reqt)
+        while d = reqt.at("./description[following-sibling::*[1]"\
+                          "[self::description]]")
+          n = d.next.remove
+          d << n.children
+        end
+        reqt.xpath("./description[normalize-space(.)='']").each do |r|
+          r.replace("\n")
+        end
+      end
+
+      def requirement_metadata(xmldoc)
+        xmldoc.xpath(REQRECPER).each do |r|
+          dl = r&.at("./dl[@metadata = 'true']")&.remove or next
+          requirement_metadata1(r, dl, r.at("./title"))
+        end
+      end
+
+      def requirement_metadata1_tags
+        %w(label subject inherit)
+      end
+
+      def requirement_metadata1(reqt, dlist, ins)
+        unless ins
+          reqt.children.first.previous = " "
+          ins = reqt.children.first
+        end
+        %w(obligation model type).each do |a|
+          dl_to_attrs(reqt, dlist, a)
+        end
+        requirement_metadata1_tags.each do |a|
+          ins = dl_to_elems(ins, reqt, dlist, a)
+        end
+        reqt_dl_to_classif(ins, reqt, dlist)
+      end
+
+      def reqt_dl_to_classif(ins, reqt, dlist)
+        if a = reqt.at("./classification[last()]") then ins = a end
+        dlist.xpath("./dt[text()='classification']").each do |e|
+          val = e.at("./following::dd/p") || e.at("./following::dd")
+          req_classif_parse(val.text).each do |r|
+            ins.next = "<classification><tag>#{r[0]}</tag>"\
+                       "<value>#{r[1]}</value></classification>"
+            ins = ins.next
+          end
+        end
+        ins
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_section.rb

@@ -0,0 +1,212 @@
+require "date"
+require "htmlentities"
+require "json"
+require "mathml2asciimath"
+require_relative "cleanup_section_names"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def make_preface(xml, sect)
+        if xml.at("//foreword | //introduction | //acknowledgements | "\
+                  "//*[@preface]")
+          preface = sect.add_previous_sibling("<preface/>").first
+          f = xml.at("//foreword") and preface.add_child f.remove
+          f = xml.at("//introduction") and preface.add_child f.remove
+          move_clauses_into_preface(xml, preface)
+          f = xml.at("//acknowledgements") and preface.add_child f.remove
+        end
+        make_abstract(xml, sect)
+      end
+
+      def move_clauses_into_preface(xml, preface)
+        xml.xpath("//*[@preface]").each do |c|
+          c.delete("preface")
+          preface.add_child c.remove
+        end
+      end
+
+      def make_abstract(xml, sect)
+        if xml.at("//abstract[not(ancestor::bibitem)]")
+          preface = sect.at("//preface") ||
+            sect.add_previous_sibling("<preface/>").first
+          abstract = xml.at("//abstract[not(ancestor::bibitem)]").remove
+          preface.prepend_child abstract.remove
+          bibabstract = bibabstract_location(xml)
+          dupabstract = abstract.dup
+          dupabstract.traverse { |n| n.remove_attribute("id") }
+          dupabstract.remove_attribute("language")
+          dupabstract.remove_attribute("script")
+          dupabstract&.at("./title")&.remove
+          bibabstract.next = dupabstract
+        end
+      end
+
+      def bibabstract_location(xml)
+        xml.at("//bibdata/script") || xml.at("//bibdata/language") ||
+          xml.at("//bibdata/contributor[not(following-sibling::contributor)]") ||
+          xml.at("//bibdata/date[not(following-sibling::date)]") ||
+          xml.at("//docnumber") ||
+          xml.at("//bibdata/docidentifier"\
+                 "[not(following-sibling::docidentifier)]") ||
+          xml.at("//bibdata/uri[not(following-sibling::uri)]") ||
+          xml.at("//bibdata/title[not(following-sibling::title)]")
+      end
+
+      def make_bibliography(xml, sect)
+        if xml.at("//sections/references")
+          biblio = sect.add_next_sibling("<bibliography/>").first
+          xml.xpath("//sections/references").each do |r|
+            biblio.add_child r.remove
+          end
+        end
+      end
+
+      def make_indexsect(xml, sect)
+        xml.xpath("//sections/indexsect").reverse_each do |r|
+          sect.next = r.remove
+        end
+      end
+
+      def sections_order_cleanup(xml)
+        s = xml.at("//sections")
+        make_preface(xml, s)
+        make_annexes(xml)
+        make_indexsect(xml, s)
+        make_bibliography(xml, s)
+        xml.xpath("//sections/annex").reverse_each { |r| s.next = r.remove }
+      end
+
+      def make_annexes(xml)
+        xml.xpath("//*[@annex]").each do |y|
+          y.delete("annex")
+          next if y.name == "annex" || !y.ancestors("annex").empty?
+
+          y.wrap("<annex/>")
+          y.parent["id"] = "_#{UUIDTools::UUID.random_create}"
+          y.parent["obligation"] = y["obligation"]
+          y.parent["language"] = y["language"]
+          y.parent["script"] = y["script"]
+        end
+      end
+
+      def maxlevel(xml)
+        max = 5
+        xml.xpath("//clause[@level]").each do |c|
+          max = c["level"].to_i if max < c["level"].to_i
+        end
+        max
+      end
+
+      def sections_level_cleanup(xml)
+        m = maxlevel(xml)
+        return if m < 6
+
+        m.downto(6).each do |l|
+          xml.xpath("//clause[@level = '#{l}']").each do |c|
+            c.delete("level")
+            c.previous_element << c.remove
+          end
+        end
+      end
+
+      def sections_cleanup(xml)
+        sections_order_cleanup(xml)
+        sections_level_cleanup(xml)
+        sections_names_cleanup(xml)
+        sections_variant_title_cleanup(xml)
+        change_clauses(xml)
+      end
+
+      def obligations_cleanup(xml)
+        obligations_cleanup_info(xml)
+        obligations_cleanup_norm(xml)
+        obligations_cleanup_inherit(xml)
+      end
+
+      def obligations_cleanup_info(xml)
+        s = xml.at("//foreword") and s["obligation"] = "informative"
+        s = xml.at("//introduction") and s["obligation"] = "informative"
+        s = xml.at("//acknowledgements") and s["obligation"] = "informative"
+        xml.xpath("//references").each { |r| r["obligation"] = "informative" }
+        xml.xpath("//preface//clause").each do |r|
+          r["obligation"] = "informative"
+        end
+      end
+
+      def obligations_cleanup_norm(xml)
+        s = xml.at("//clause[@type = 'scope']") and
+          s["obligation"] = "normative"
+        xml.xpath("//terms").each { |r| r["obligation"] = "normative" }
+        xml.xpath("//definitions").each { |r| r["obligation"] = "normative" }
+      end
+
+      def obligations_cleanup_inherit(xml)
+        xml.xpath("//annex | //clause[not(ancestor::boilerplate)]").each do |r|
+          r["obligation"] = "normative" unless r["obligation"]
+        end
+        xml.xpath(Utils::SUBCLAUSE_XPATH).each do |r|
+          o = r&.at("./ancestor::*/@obligation")&.text and r["obligation"] = o
+        end
+      end
+
+      def clausebefore_cleanup(xmldoc)
+        preface_clausebefore_cleanup(xmldoc)
+        sections_clausebefore_cleanup(xmldoc)
+      end
+
+      def preface_clausebefore_cleanup(xmldoc)
+        return unless xmldoc.at("//preface")
+
+        unless ins = xmldoc.at("//preface").children.first
+          xmldoc.at("//preface") << " "
+          ins = xmldoc.at("//preface").children.first
+        end
+        xmldoc.xpath("//preface//*[@beforeclauses = 'true']").each do |x|
+          x.delete("beforeclauses")
+          ins.previous = x.remove
+        end
+      end
+
+      def sections_clausebefore_cleanup(xmldoc)
+        return unless xmldoc.at("//sections")
+
+        unless ins = xmldoc.at("//sections").children.first
+          xmldoc.at("//sections") << " "
+          ins = xmldoc.at("//sections").children.first
+        end
+        xmldoc.xpath("//sections//*[@beforeclauses = 'true']").each do |x|
+          x.delete("beforeclauses")
+          ins.previous = x.remove
+        end
+      end
+
+      def floatingtitle_cleanup(xmldoc)
+        pop_floating_title(xmldoc)
+        floating_title_preface2sections(xmldoc)
+      end
+
+      def pop_floating_title(xmldoc)
+        loop do
+          found = false
+          xmldoc.xpath("//floating-title").each do |t|
+            next unless t.next_element.nil?
+            next if %w(sections annex preface).include? t.parent.name
+
+            t.parent.parent << t
+            found = true
+          end
+          break unless found
+        end
+      end
+
+      def floating_title_preface2sections(xmldoc)
+        t = xmldoc.at("//preface/floating-title") or return
+        s = xmldoc.at("//sections")
+        unless t.next_element
+          s.children.first.previous = t.remove
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_section_names.rb

@@ -0,0 +1,92 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def get_or_make_title(node)
+        unless node.at("./title")
+          if node.children.empty?
+            node << "<title/>"
+          else
+            node.children.first.previous = "<title/>"
+          end
+        end
+        node.at("./title")
+      end
+
+      def replace_title(doc, xpath, text, first = false)
+        return unless text
+
+        doc.xpath(xpath).each_with_index do |node, i|
+          next if first && !i.zero?
+
+          title = get_or_make_title(node)
+          fn = title.xpath("./fn")
+          fn.each(&:remove)
+          title.children = text
+          fn.each { |n| title << n }
+        end
+      end
+
+      def sections_names_cleanup(xml)
+        replace_title(xml, "//clause[@type = 'scope']", @i18n&.scope)
+        replace_title(xml, "//preface//abstract", @i18n&.abstract)
+        replace_title(xml, "//foreword", @i18n&.foreword)
+        replace_title(xml, "//introduction", @i18n&.introduction)
+        replace_title(xml, "//acknowledgements", @i18n&.acknowledgements)
+        section_names_refs_cleanup(xml)
+        section_names_terms_cleanup(xml)
+      end
+
+      def section_names_refs_cleanup(xml)
+        replace_title(xml, "//bibliography/references[@normative = 'true']",
+                      @i18n&.normref, true)
+        replace_title(xml, "//bibliography/references[@normative = 'false']",
+                      @i18n&.bibliography, true)
+      end
+
+      NO_SYMABBR = "[.//definitions[not(@type)]]".freeze
+      SYMABBR = "[.//definitions[@type = 'symbols']]"\
+                "[.//definitions[@type = 'abbreviated_terms']]".freeze
+      SYMnoABBR = "[.//definitions[@type = 'symbols']]"\
+                  "[not(.//definitions[@type = 'abbreviated_terms'])]".freeze
+      ABBRnoSYM = "[.//definitions[@type = 'abbreviated_terms']]"\
+                  "[not(.//definitions[@type = 'symbols'])]".freeze
+
+      def section_names_terms_cleanup(xml)
+        replace_title(xml, "//definitions[@type = 'symbols']", @i18n&.symbols)
+        replace_title(xml, "//definitions[@type = 'abbreviated_terms']",
+                      @i18n&.abbrev)
+        replace_title(xml, "//definitions[not(@type)]", @i18n&.symbolsabbrev)
+        replace_title(xml, "//terms#{SYMnoABBR} | //clause[.//terms]#{SYMnoABBR}",
+                      @i18n&.termsdefsymbols, true)
+        replace_title(xml, "//terms#{ABBRnoSYM} | //clause[.//terms]#{ABBRnoSYM}",
+                      @i18n&.termsdefabbrev, true)
+        replace_title(xml, "//terms#{SYMABBR} | //clause[.//terms]#{SYMABBR}",
+                      @i18n&.termsdefsymbolsabbrev, true)
+        replace_title(xml, "//terms#{NO_SYMABBR} | //clause[.//terms]#{NO_SYMABBR}",
+                      @i18n&.termsdefsymbolsabbrev, true)
+        replace_title(
+          xml,
+          "//terms[not(.//definitions)] | //clause[.//terms][not(.//definitions)]",
+          @i18n&.termsdef, true
+        )
+      end
+
+      SECTION_CONTAINERS = %w(foreword introduction acknowledgements abstract
+                              clause clause references terms definitions annex
+                              appendix).freeze
+
+      def sections_variant_title_cleanup(xml)
+        path = SECTION_CONTAINERS.map { |x| "./ancestor::#{x}" }.join(" | ")
+        xml.xpath("//p[@variant_title]").each do |p|
+          p.name = "variant-title"
+          p.delete("id")
+          p.delete("variant_title")
+          p.xpath("(#{path})[last()]").each do |sect|
+            (ins = sect.at("./title") and ins.next = p) or
+              sect.children.first.previous = p
+          end
+        end
+      end
+    end
+  end
+end