metanorma-standoc 1.11.4 → 2.0.0

data/lib/metanorma/standoc/cleanup_reqt.rb

@@ -0,0 +1,111 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def requirement_cleanup(xmldoc)
+        requirement_metadata(xmldoc)
+        requirement_inherit(xmldoc)
+        requirement_descriptions(xmldoc)
+      end
+
+      REQRECPER = "//requirement | //recommendation | //permission".freeze
+
+      def requirement_inherit(xmldoc)
+        xmldoc.xpath(REQRECPER).each do |r|
+          ins = requirement_inherit_insert(r)
+          r.xpath("./*//inherit").each { |i| ins.previous = i }
+        end
+      end
+
+      def requirement_inherit_insert(reqt)
+        ins = reqt.at("./classification") || reqt.at(
+          "./description | ./measurementtarget | ./specification | "\
+          "./verification | ./import | ./description | ./component | "\
+          "./requirement | ./recommendation | ./permission",
+        ) and return ins
+        requirement_inherit_insert1(reqt)
+      end
+
+      def requirement_inherit_insert1(reqt)
+        if t = reqt.at("./title")
+          t.next = " "
+          t.next
+        else
+          if reqt.children.empty? then reqt.add_child(" ")
+          else reqt.children.first.previous = " "
+          end
+          reqt.children.first
+        end
+      end
+
+      def requirement_descriptions(xmldoc)
+        xmldoc.xpath(REQRECPER).each do |r|
+          r.xpath(".//p[not(./*)][normalize-space(.)='']").each(&:remove)
+          r.children.each do |e|
+            requirement_description_wrap(r, e)
+          end
+          requirement_description_cleanup1(r)
+        end
+      end
+
+      def requirement_description_wrap(reqt, text)
+        return if (text.element? && (reqt_subpart(text.name) ||
+                %w(requirement recommendation
+                   permission).include?(text.name))) ||
+          (text.text.strip.empty? && !text.at(".//xref | .//eref | .//link"))
+
+        t = Nokogiri::XML::Element.new("description", reqt)
+        text.before(t)
+        t.children = text.remove
+      end
+
+      def requirement_description_cleanup1(reqt)
+        while d = reqt.at("./description[following-sibling::*[1]"\
+                          "[self::description]]")
+          n = d.next.remove
+          d << n.children
+        end
+        reqt.xpath("./description[normalize-space(.)='']").each do |r|
+          r.replace("\n")
+        end
+      end
+
+      def requirement_metadata(xmldoc)
+        xmldoc.xpath(REQRECPER).each do |r|
+          dl = r&.at("./dl[@metadata = 'true']")&.remove or next
+          requirement_metadata1(r, dl, r.at("./title"))
+        end
+      end
+
+      def requirement_metadata1_tags
+        %w(label subject inherit)
+      end
+
+      def requirement_metadata1(reqt, dlist, ins)
+        unless ins
+          reqt.children.first.previous = " "
+          ins = reqt.children.first
+        end
+        %w(obligation model type).each do |a|
+          dl_to_attrs(reqt, dlist, a)
+        end
+        requirement_metadata1_tags.each do |a|
+          ins = dl_to_elems(ins, reqt, dlist, a)
+        end
+        reqt_dl_to_classif(ins, reqt, dlist)
+      end
+
+      def reqt_dl_to_classif(ins, reqt, dlist)
+        if a = reqt.at("./classification[last()]") then ins = a end
+        dlist.xpath("./dt[text()='classification']").each do |e|
+          val = e.at("./following::dd/p") || e.at("./following::dd")
+          req_classif_parse(val.text).each do |r|
+            ins.next = "<classification><tag>#{r[0]}</tag>"\
+                       "<value>#{r[1]}</value></classification>"
+            ins = ins.next
+          end
+        end
+        ins
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_section.rb

@@ -0,0 +1,212 @@
+require "date"
+require "htmlentities"
+require "json"
+require "mathml2asciimath"
+require_relative "cleanup_section_names"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def make_preface(xml, sect)
+        if xml.at("//foreword | //introduction | //acknowledgements | "\
+                  "//*[@preface]")
+          preface = sect.add_previous_sibling("<preface/>").first
+          f = xml.at("//foreword") and preface.add_child f.remove
+          f = xml.at("//introduction") and preface.add_child f.remove
+          move_clauses_into_preface(xml, preface)
+          f = xml.at("//acknowledgements") and preface.add_child f.remove
+        end
+        make_abstract(xml, sect)
+      end
+
+      def move_clauses_into_preface(xml, preface)
+        xml.xpath("//*[@preface]").each do |c|
+          c.delete("preface")
+          preface.add_child c.remove
+        end
+      end
+
+      def make_abstract(xml, sect)
+        if xml.at("//abstract[not(ancestor::bibitem)]")
+          preface = sect.at("//preface") ||
+            sect.add_previous_sibling("<preface/>").first
+          abstract = xml.at("//abstract[not(ancestor::bibitem)]").remove
+          preface.prepend_child abstract.remove
+          bibabstract = bibabstract_location(xml)
+          dupabstract = abstract.dup
+          dupabstract.traverse { |n| n.remove_attribute("id") }
+          dupabstract.remove_attribute("language")
+          dupabstract.remove_attribute("script")
+          dupabstract&.at("./title")&.remove
+          bibabstract.next = dupabstract
+        end
+      end
+
+      def bibabstract_location(xml)
+        xml.at("//bibdata/script") || xml.at("//bibdata/language") ||
+          xml.at("//bibdata/contributor[not(following-sibling::contributor)]") ||
+          xml.at("//bibdata/date[not(following-sibling::date)]") ||
+          xml.at("//docnumber") ||
+          xml.at("//bibdata/docidentifier"\
+                 "[not(following-sibling::docidentifier)]") ||
+          xml.at("//bibdata/uri[not(following-sibling::uri)]") ||
+          xml.at("//bibdata/title[not(following-sibling::title)]")
+      end
+
+      def make_bibliography(xml, sect)
+        if xml.at("//sections/references")
+          biblio = sect.add_next_sibling("<bibliography/>").first
+          xml.xpath("//sections/references").each do |r|
+            biblio.add_child r.remove
+          end
+        end
+      end
+
+      def make_indexsect(xml, sect)
+        xml.xpath("//sections/indexsect").reverse_each do |r|
+          sect.next = r.remove
+        end
+      end
+
+      def sections_order_cleanup(xml)
+        s = xml.at("//sections")
+        make_preface(xml, s)
+        make_annexes(xml)
+        make_indexsect(xml, s)
+        make_bibliography(xml, s)
+        xml.xpath("//sections/annex").reverse_each { |r| s.next = r.remove }
+      end
+
+      def make_annexes(xml)
+        xml.xpath("//*[@annex]").each do |y|
+          y.delete("annex")
+          next if y.name == "annex" || !y.ancestors("annex").empty?
+
+          y.wrap("<annex/>")
+          y.parent["id"] = "_#{UUIDTools::UUID.random_create}"
+          y.parent["obligation"] = y["obligation"]
+          y.parent["language"] = y["language"]
+          y.parent["script"] = y["script"]
+        end
+      end
+
+      def maxlevel(xml)
+        max = 5
+        xml.xpath("//clause[@level]").each do |c|
+          max = c["level"].to_i if max < c["level"].to_i
+        end
+        max
+      end
+
+      def sections_level_cleanup(xml)
+        m = maxlevel(xml)
+        return if m < 6
+
+        m.downto(6).each do |l|
+          xml.xpath("//clause[@level = '#{l}']").each do |c|
+            c.delete("level")
+            c.previous_element << c.remove
+          end
+        end
+      end
+
+      def sections_cleanup(xml)
+        sections_order_cleanup(xml)
+        sections_level_cleanup(xml)
+        sections_names_cleanup(xml)
+        sections_variant_title_cleanup(xml)
+        change_clauses(xml)
+      end
+
+      def obligations_cleanup(xml)
+        obligations_cleanup_info(xml)
+        obligations_cleanup_norm(xml)
+        obligations_cleanup_inherit(xml)
+      end
+
+      def obligations_cleanup_info(xml)
+        s = xml.at("//foreword") and s["obligation"] = "informative"
+        s = xml.at("//introduction") and s["obligation"] = "informative"
+        s = xml.at("//acknowledgements") and s["obligation"] = "informative"
+        xml.xpath("//references").each { |r| r["obligation"] = "informative" }
+        xml.xpath("//preface//clause").each do |r|
+          r["obligation"] = "informative"
+        end
+      end
+
+      def obligations_cleanup_norm(xml)
+        s = xml.at("//clause[@type = 'scope']") and
+          s["obligation"] = "normative"
+        xml.xpath("//terms").each { |r| r["obligation"] = "normative" }
+        xml.xpath("//definitions").each { |r| r["obligation"] = "normative" }
+      end
+
+      def obligations_cleanup_inherit(xml)
+        xml.xpath("//annex | //clause[not(ancestor::boilerplate)]").each do |r|
+          r["obligation"] = "normative" unless r["obligation"]
+        end
+        xml.xpath(Utils::SUBCLAUSE_XPATH).each do |r|
+          o = r&.at("./ancestor::*/@obligation")&.text and r["obligation"] = o
+        end
+      end
+
+      def clausebefore_cleanup(xmldoc)
+        preface_clausebefore_cleanup(xmldoc)
+        sections_clausebefore_cleanup(xmldoc)
+      end
+
+      def preface_clausebefore_cleanup(xmldoc)
+        return unless xmldoc.at("//preface")
+
+        unless ins = xmldoc.at("//preface").children.first
+          xmldoc.at("//preface") << " "
+          ins = xmldoc.at("//preface").children.first
+        end
+        xmldoc.xpath("//preface//*[@beforeclauses = 'true']").each do |x|
+          x.delete("beforeclauses")
+          ins.previous = x.remove
+        end
+      end
+
+      def sections_clausebefore_cleanup(xmldoc)
+        return unless xmldoc.at("//sections")
+
+        unless ins = xmldoc.at("//sections").children.first
+          xmldoc.at("//sections") << " "
+          ins = xmldoc.at("//sections").children.first
+        end
+        xmldoc.xpath("//sections//*[@beforeclauses = 'true']").each do |x|
+          x.delete("beforeclauses")
+          ins.previous = x.remove
+        end
+      end
+
+      def floatingtitle_cleanup(xmldoc)
+        pop_floating_title(xmldoc)
+        floating_title_preface2sections(xmldoc)
+      end
+
+      def pop_floating_title(xmldoc)
+        loop do
+          found = false
+          xmldoc.xpath("//floating-title").each do |t|
+            next unless t.next_element.nil?
+            next if %w(sections annex preface).include? t.parent.name
+
+            t.parent.parent << t
+            found = true
+          end
+          break unless found
+        end
+      end
+
+      def floating_title_preface2sections(xmldoc)
+        t = xmldoc.at("//preface/floating-title") or return
+        s = xmldoc.at("//sections")
+        unless t.next_element
+          s.children.first.previous = t.remove
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_section_names.rb

@@ -0,0 +1,92 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def get_or_make_title(node)
+        unless node.at("./title")
+          if node.children.empty?
+            node << "<title/>"
+          else
+            node.children.first.previous = "<title/>"
+          end
+        end
+        node.at("./title")
+      end
+
+      def replace_title(doc, xpath, text, first = false)
+        return unless text
+
+        doc.xpath(xpath).each_with_index do |node, i|
+          next if first && !i.zero?
+
+          title = get_or_make_title(node)
+          fn = title.xpath("./fn")
+          fn.each(&:remove)
+          title.children = text
+          fn.each { |n| title << n }
+        end
+      end
+
+      def sections_names_cleanup(xml)
+        replace_title(xml, "//clause[@type = 'scope']", @i18n&.scope)
+        replace_title(xml, "//preface//abstract", @i18n&.abstract)
+        replace_title(xml, "//foreword", @i18n&.foreword)
+        replace_title(xml, "//introduction", @i18n&.introduction)
+        replace_title(xml, "//acknowledgements", @i18n&.acknowledgements)
+        section_names_refs_cleanup(xml)
+        section_names_terms_cleanup(xml)
+      end
+
+      def section_names_refs_cleanup(xml)
+        replace_title(xml, "//bibliography/references[@normative = 'true']",
+                      @i18n&.normref, true)
+        replace_title(xml, "//bibliography/references[@normative = 'false']",
+                      @i18n&.bibliography, true)
+      end
+
+      NO_SYMABBR = "[.//definitions[not(@type)]]".freeze
+      SYMABBR = "[.//definitions[@type = 'symbols']]"\
+                "[.//definitions[@type = 'abbreviated_terms']]".freeze
+      SYMnoABBR = "[.//definitions[@type = 'symbols']]"\
+                  "[not(.//definitions[@type = 'abbreviated_terms'])]".freeze
+      ABBRnoSYM = "[.//definitions[@type = 'abbreviated_terms']]"\
+                  "[not(.//definitions[@type = 'symbols'])]".freeze
+
+      def section_names_terms_cleanup(xml)
+        replace_title(xml, "//definitions[@type = 'symbols']", @i18n&.symbols)
+        replace_title(xml, "//definitions[@type = 'abbreviated_terms']",
+                      @i18n&.abbrev)
+        replace_title(xml, "//definitions[not(@type)]", @i18n&.symbolsabbrev)
+        replace_title(xml, "//terms#{SYMnoABBR} | //clause[.//terms]#{SYMnoABBR}",
+                      @i18n&.termsdefsymbols, true)
+        replace_title(xml, "//terms#{ABBRnoSYM} | //clause[.//terms]#{ABBRnoSYM}",
+                      @i18n&.termsdefabbrev, true)
+        replace_title(xml, "//terms#{SYMABBR} | //clause[.//terms]#{SYMABBR}",
+                      @i18n&.termsdefsymbolsabbrev, true)
+        replace_title(xml, "//terms#{NO_SYMABBR} | //clause[.//terms]#{NO_SYMABBR}",
+                      @i18n&.termsdefsymbolsabbrev, true)
+        replace_title(
+          xml,
+          "//terms[not(.//definitions)] | //clause[.//terms][not(.//definitions)]",
+          @i18n&.termsdef, true
+        )
+      end
+
+      SECTION_CONTAINERS = %w(foreword introduction acknowledgements abstract
+                              clause clause references terms definitions annex
+                              appendix).freeze
+
+      def sections_variant_title_cleanup(xml)
+        path = SECTION_CONTAINERS.map { |x| "./ancestor::#{x}" }.join(" | ")
+        xml.xpath("//p[@variant_title]").each do |p|
+          p.name = "variant-title"
+          p.delete("id")
+          p.delete("variant_title")
+          p.xpath("(#{path})[last()]").each do |sect|
+            (ins = sect.at("./title") and ins.next = p) or
+              sect.children.first.previous = p
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_symbols.rb

@@ -0,0 +1,48 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      # Indices sort after letter but before any following
+      # letter (x, x_m, x_1, xa); we use colon to force that sort order.
+      # Numbers sort *after* letters; we use thorn to force that sort order.
+      def symbol_key(sym)
+        key = sym.dup
+        key.traverse do |n|
+          n.name == "math" and
+            n.replace(grkletters(MathML2AsciiMath.m2a(n.to_xml)))
+        end
+        ret = Nokogiri::XML(key.to_xml)
+        HTMLEntities.new.decode(ret.text.downcase)
+          .gsub(/[\[\]{}<>()]/, "").gsub(/\s/m, "")
+          .gsub(/[[:punct:]]|[_^]/, ":\\0").gsub(/`/, "")
+          .gsub(/[0-9]+/, "þ\\0")
+      end
+
+      def grkletters(text)
+        text.gsub(/\b(alpha|beta|gamma|delta|epsilon|zeta|eta|theta|iota|kappa|
+                      lambda|mu|nu|xi|omicron|pi|rho|sigma|tau|upsilon|phi|chi|
+                      psi|omega)\b/xi, "&\\1;")
+      end
+
+      def extract_symbols_list(dlist)
+        dl_out = []
+        dlist.xpath("./dt | ./dd").each do |dtd|
+          if dtd.name == "dt"
+            dl_out << { dt: dtd.remove, key: symbol_key(dtd) }
+          else
+            dl_out.last[:dd] = dtd.remove
+          end
+        end
+        dl_out
+      end
+
+      def symbols_cleanup(docxml)
+        docxml.xpath("//definitions/dl").each do |dl|
+          dl_out = extract_symbols_list(dl)
+          dl_out.sort! { |a, b| a[:key] <=> b[:key] || a[:dt] <=> b[:dt] }
+          dl.children = dl_out.map { |d| d[:dt].to_s + d[:dd].to_s }.join("\n")
+        end
+        docxml
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_table.rb

@@ -0,0 +1,68 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def dl1_table_cleanup(xmldoc)
+        q = "//table/following-sibling::*[1][self::dl]"
+        xmldoc.xpath(q).each do |s|
+          s["key"] == "true" and s.previous_element << s.remove
+        end
+      end
+
+      # move Key dl after table footer
+      def dl2_table_cleanup(xmldoc)
+        q = "//table/following-sibling::*[1][self::p]"
+        xmldoc.xpath(q).each do |s|
+          if s.text =~ /^\s*key[^a-z]*$/i && s&.next_element&.name == "dl"
+            s.next_element["key"] = "true"
+            s.previous_element << s.next_element.remove
+            s.remove
+          end
+        end
+      end
+
+      def insert_thead(table)
+        thead = table.at("./thead")
+        return thead unless thead.nil?
+
+        if tname = table.at("./name")
+          thead = tname.add_next_sibling("<thead/>").first
+          return thead
+        end
+        table.children.first.add_previous_sibling("<thead/>").first
+      end
+
+      def header_rows_cleanup(xmldoc)
+        xmldoc.xpath("//table[@headerrows]").each do |s|
+          thead = insert_thead(s)
+          (thead.xpath("./tr").size...s["headerrows"].to_i).each do
+            row = s.at("./tbody/tr")
+            row.parent = thead
+          end
+          thead.xpath(".//td").each { |n| n.name = "th" }
+          s.delete("headerrows")
+        end
+      end
+
+      def table_cleanup(xmldoc)
+        dl1_table_cleanup(xmldoc)
+        dl2_table_cleanup(xmldoc)
+        notes_table_cleanup(xmldoc)
+        header_rows_cleanup(xmldoc)
+      end
+
+      # move notes into table
+      def notes_table_cleanup(xmldoc)
+        nomatches = false
+        until nomatches
+          nomatches = true
+          xmldoc.xpath("//table/following-sibling::*[1]"\
+                       "[self::note[not(@keep-separate = 'true')]]").each do |n|
+            n.delete("keep-separate")
+            n.previous_element << n.remove
+            nomatches = false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_terms.rb

@@ -0,0 +1,140 @@
+require_relative "term_lookup_cleanup"
+require_relative "cleanup_terms_designations"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def termdomain_cleanup(xmldoc)
+        xmldoc.xpath("//p/domain").each do |a|
+          parent = a.parent
+          prev = parent.previous
+          prev.next = a.remove
+          parent.text.strip.empty? and parent.remove
+        end
+      end
+
+      def termdomain1_cleanup(xmldoc)
+        xmldoc.xpath("//term").each do |t|
+          d = t.xpath("./domain | ./subject").last or next
+          defn = d.at("../definition") and defn.previous = d.remove
+        end
+      end
+
+      def termdefinition_cleanup(xmldoc)
+        generate_termdefinitions(xmldoc)
+        split_termdefinitions(xmldoc)
+        alternate_termdefinitions(xmldoc)
+      end
+
+      TERMDEF_BLOCKS =
+        "./p | ./ol | ./dl[not(@metadata = 'true')] | ./ul | ./figure | "\
+        "./formula | ./table".freeze
+
+      def generate_termdefinitions(xmldoc)
+        xmldoc.xpath("//term[not(definition)]").each do |d|
+          first_child = d.at(TERMDEF_BLOCKS) || next
+          t = Nokogiri::XML::Element.new("definition", xmldoc)
+          first_child.replace(t)
+          t << first_child.remove
+          d.xpath(TERMDEF_BLOCKS).each do |n|
+            t << n.remove
+          end
+        end
+      end
+
+      def split_termdefinitions(xmldoc)
+        xmldoc.xpath("//definition").each do |d|
+          if d.at("./p | ./ol | ./dl | ./ul")
+            d.children = "<verbal-definition>#{d.children}</verbal-definition>"
+          else
+            d.children = "<non-verbal-representation>"\
+                         "#{d.children}</non-verbal-representation>"
+          end
+        end
+      end
+
+      def alternate_termdefinitions(xmldoc)
+        xmldoc.xpath("//term").each do |t|
+          t.xpath("./definition").each do |d|
+            d1 = d.next_element or next
+            if (v = d.at("./verbal-definition")) &&
+                !d.at("./non-verbal-representation") &&
+                !d1.at("./verbal-definition") &&
+                nv = d1.at("./non-verbal-representation")
+              v.next = nv.remove
+              d1.remove
+            end
+          end
+        end
+      end
+
+      def termdocsource_cleanup(xmldoc)
+        f = xmldoc.at("//preface | //sections")
+        xmldoc.xpath("//termdocsource").each { |s| f.previous = s.remove }
+      end
+
+      def term_children_cleanup(xmldoc)
+        xmldoc.xpath("//terms[terms]").each { |t| t.name = "clause" }
+        xmldoc.xpath("//term").each do |t|
+          %w(termnote termexample termsource term).each do |w|
+            t.xpath("./#{w}").each { |n| t << n.remove }
+          end
+        end
+      end
+
+      def termdef_from_termbase(xmldoc)
+        xmldoc.xpath("//term").each do |x|
+          if (c = x.at("./origin/termref")) && !x.at("./definition")
+            x.at("./origin").previous = fetch_termbase(c["base"], c.text)
+          end
+        end
+      end
+
+      def termnote_example_cleanup(xmldoc)
+        %w(note example).each do |w|
+          xmldoc.xpath("//term#{w}[not(ancestor::term)]").each do |x|
+            x.name = w
+          end
+        end
+      end
+
+      def termdef_cleanup(xmldoc)
+        termdef_unnest_cleanup(xmldoc)
+        Metanorma::Standoc::TermLookupCleanup.new(xmldoc, @log).call
+        term_nonverbal_designations(xmldoc)
+        term_dl_to_metadata(xmldoc)
+        term_termsource_to_designation(xmldoc)
+        term_designation_reorder(xmldoc)
+        termdef_from_termbase(xmldoc)
+        termdomain_cleanup(xmldoc)
+        termdef_stem_cleanup(xmldoc)
+        termdefinition_cleanup(xmldoc)
+        termdomain1_cleanup(xmldoc)
+        termnote_example_cleanup(xmldoc)
+        term_children_cleanup(xmldoc)
+        termdocsource_cleanup(xmldoc)
+      end
+
+      def index_cleanup(xmldoc)
+        return unless @index_terms
+
+        xmldoc.xpath("//preferred").each do |p|
+          index_cleanup1(p.at("./expression/name | ./letter-symbol/name"),
+                         p.xpath("./field-of-application | ./usage-info")
+            &.map(&:text)&.join(", "))
+        end
+        xmldoc.xpath("//definitions/dl/dt").each do |p|
+          index_cleanup1(p, "")
+        end
+      end
+
+      def index_cleanup1(term, fieldofappl)
+        return unless term
+
+        idx = term.children.dup
+        fieldofappl.empty? or idx << ", &#x3c;#{fieldofappl}&#x3e;"
+        term << "<index><primary>#{idx.to_xml}</primary></index>"
+      end
+    end
+  end
+end