metanorma-standoc 1.8.6 → 1.9.2

data/lib/asciidoctor/standoc/cleanup.rb

@@ -2,27 +2,29 @@ require "nokogiri"
 require "pathname"
 require "open-uri"
 require "html2doc"
-require_relative "./cleanup_block.rb"
-require_relative "./cleanup_footnotes.rb"
-require_relative "./cleanup_ref.rb"
-require_relative "./cleanup_ref_dl.rb"
-require_relative "./cleanup_boilerplate.rb"
-require_relative "./cleanup_section.rb"
-require_relative "./cleanup_terms.rb"
-require_relative "./cleanup_inline.rb"
-require_relative "./cleanup_amend.rb"
-require_relative "./cleanup_maths.rb"
+require_relative "./cleanup_block"
+require_relative "./cleanup_footnotes"
+require_relative "./cleanup_ref"
+require_relative "./cleanup_ref_dl"
+require_relative "./cleanup_boilerplate"
+require_relative "./cleanup_section"
+require_relative "./cleanup_terms"
+require_relative "./cleanup_inline"
+require_relative "./cleanup_amend"
+require_relative "./cleanup_maths"
+require_relative "./cleanup_image"
+require_relative "./cleanup_reqt"
 require "relaton_iev"
 
 module Asciidoctor
   module Standoc
     module Cleanup
       def textcleanup(result)
-        text = result.flatten.map { |l| l.sub(/\s*$/, "") }  * "\n"
+        text = result.flatten.map { |l| l.sub(/\s*$/, "") } * "\n"
         !@keepasciimath and text = asciimath2mathml(text)
         text = text.gsub(/\s+<fn /, "<fn ")
         text.gsub(%r{<passthrough\s+formats="metanorma">([^<]*)
-                  </passthrough>}mx) { |m| HTMLEntities.new.decode($1) }
+                  </passthrough>}mx) { HTMLEntities.new.decode($1) }
       end
 
       def cleanup(xmldoc)
@@ -31,6 +33,7 @@ module Asciidoctor
         obligations_cleanup(xmldoc)
         table_cleanup(xmldoc)
         formula_cleanup(xmldoc)
+        form_cleanup(xmldoc)
         sourcecode_cleanup(xmldoc)
         figure_cleanup(xmldoc)
         element_name_cleanup(xmldoc)
@@ -73,22 +76,24 @@ module Asciidoctor
         xmldoc.xpath("//date").each { |d| Metanorma::Utils::endash_date(d) }
         xmldoc.traverse do |n|
           next unless n.text?
+
           if @smartquotes
             /[-'"(<>]|\.\.|\dx/.match(n) or next
-            n.ancestors("pre, tt, sourcecode, bibdata, on, stem, figure[@class = 'pseudocode']").empty? or next
+
+            n.ancestors("pre, tt, sourcecode, bibdata, on, "\
+                        "stem, figure[@class = 'pseudocode']").empty? or next
             n.replace(Metanorma::Utils::smartformat(n.text))
           else
-            n.replace(n.text.gsub(/(?<=\p{Alnum})\u2019(?=\p{Alpha})/, "'"))#.
-            #gsub(/</, "&lt;").gsub(/>/, "&gt;"))
+            n.replace(n.text.gsub(/(?<=\p{Alnum})\u2019(?=\p{Alpha})/, "'")) # .
+            # gsub(/</, "&lt;").gsub(/>/, "&gt;"))
           end
         end
       end
 
-      def docidentifier_cleanup(xmldoc)
-      end
+      def docidentifier_cleanup(xmldoc); end
 
       TEXT_ELEMS =
-        %w{status language script version author name callout phone email 
+        %w{status language script version author name callout phone email
            street city state country postcode identifier referenceFrom surname
            referenceTo docidentifier docnumber prefix initial addition forename
            title draft secretariat title-main title-intro title-part}.freeze
@@ -110,57 +115,6 @@ module Asciidoctor
         xmldoc.traverse { |n| n.name = n.name.gsub(/_/, "-") }
       end
 
-      def link_callouts_to_annotations(callouts, annotations)
-        callouts.each_with_index do |c, i|
-          c["target"] = "_" + UUIDTools::UUID.random_create
-          annotations[i]["id"] = c["target"]
-        end
-      end
-
-      def align_callouts_to_annotations(xmldoc)
-        xmldoc.xpath("//sourcecode").each do |x|
-          callouts = x.elements.select { |e| e.name == "callout" }
-          annotations = x.elements.select { |e| e.name == "annotation" }
-          callouts.size == annotations.size and
-            link_callouts_to_annotations(callouts, annotations)
-        end
-      end
-
-      def merge_annotations_into_sourcecode(xmldoc)
-        xmldoc.xpath("//sourcecode").each do |x|
-          while x&.next_element&.name == "annotation"
-            x.next_element.parent = x
-          end
-        end
-      end
-
-      def callout_cleanup(xmldoc)
-        merge_annotations_into_sourcecode(xmldoc)
-        align_callouts_to_annotations(xmldoc)
-      end
-
-      def sourcecode_cleanup(xmldoc)
-        xmldoc.xpath("//sourcecode").each do |x|
-          x.traverse do |n|
-            next unless n.text?
-            next unless /#{Regexp.escape(@sourcecode_markup_start)}/.match(n.text)
-            n.replace(sourcecode_markup(n))
-          end
-        end
-      end
-
-      def sourcecode_markup(n)
-        acc = []
-        n.text.split(/(#{Regexp.escape(@sourcecode_markup_start)}|#{Regexp.escape(@sourcecode_markup_end)})/).
-          each_slice(4).map do |a|
-          acc << Nokogiri::XML::Text.new(a[0], n.document).
-            to_xml(encoding: "US-ASCII", save_with: Nokogiri::XML::Node::SaveOptions::NO_DECLARATION)
-          next unless a.size == 4
-          acc << Asciidoctor.convert(a[2], backend: (self&.backend&.to_sym || :standoc), doctype: :inline)
-        end
-        acc.join
-      end
-
       # allows us to deal with doc relation localities,
       # temporarily stashed to "bpart"
       def bpart_cleanup(xmldoc)
@@ -170,23 +124,24 @@ module Asciidoctor
         end
       end
 
-      def img_cleanup(xmldoc)
-        return xmldoc unless @datauriimage
-        xmldoc.xpath("//image").each { |i| i["src"] = Metanorma::Utils::datauri(i["src"], @localdir) }
-      end
-
       def variant_cleanup(xmldoc)
         xmldoc.xpath("//*[variant]").each do |c|
-          c&.next&.text? && c&.next&.next&.name == "variant" && c.next.text.gsub(/\s/, "").empty? and
+          c&.next&.text? && c&.next&.next&.name == "variant" &&
+            c.next.text.gsub(/\s/, "").empty? and
             c.next.remove
         end
         xmldoc.xpath("//*[variant]").each do |c|
-          next unless c.children.any? { |n| n.name != "variant" && (!n.text? || !n.text.gsub(/\s/, "").empty?) }
+          next unless c.children.any? do |n|
+            n.name != "variant" && (!n.text? || !n.text.gsub(/\s/, "").empty?)
+          end
+
           c.xpath("./variant").each do |n|
-            if n.at_xpath('preceding-sibling::node()[not(self::text()[not(normalize-space())])][1][self::variantwrap]')
+            if n.at_xpath("preceding-sibling::node()"\
+                "[not(self::text()[not(normalize-space())])][1]"\
+                "[self::variantwrap]")
               n.previous_element << n
             else
-              n.replace('<variantwrap/>').first << n
+              n.replace("<variantwrap/>").first << n
             end
           end
         end

data/lib/asciidoctor/standoc/cleanup_block.rb

@@ -29,7 +29,7 @@ module Asciidoctor
       def dl2_table_cleanup(xmldoc)
         q = "//table/following-sibling::*[1][self::p]"
         xmldoc.xpath(q).each do |s|
-          if s.text =~ /^\s*key[^a-z]*$/i && !s.next_element.nil? && s.next_element.name == "dl"
+          if s.text =~ /^\s*key[^a-z]*$/i && s&.next_element&.name == "dl"
             s.next_element["key"] = "true"
             s.previous_element << s.next_element.remove
             s.remove
@@ -37,14 +37,15 @@ module Asciidoctor
         end
       end
 
-      def insert_thead(s)
-        thead = s.at("./thead")
+      def insert_thead(table)
+        thead = table.at("./thead")
         return thead unless thead.nil?
-        if tname = s.at("./name")
+
+        if tname = table.at("./name")
           thead = tname.add_next_sibling("<thead/>").first
           return thead
         end
-        s.children.first.add_previous_sibling("<thead/>").first
+        table.children.first.add_previous_sibling("<thead/>").first
       end
 
       def header_rows_cleanup(xmldoc)
@@ -80,22 +81,22 @@ module Asciidoctor
       end
 
       # include where definition list inside stem block
-      def formula_cleanup(x)
-        formula_cleanup_where1(x)
-        formula_cleanup_where2(x)
+      def formula_cleanup(formula)
+        formula_cleanup_where1(formula)
+        formula_cleanup_where2(formula)
       end
 
-      def formula_cleanup_where1(x)
+      def formula_cleanup_where1(formula)
         q = "//formula/following-sibling::*[1][self::dl]"
-        x.xpath(q).each do |s|
+        formula.xpath(q).each do |s|
           s["key"] == "true" and s.previous_element << s.remove
         end
       end
 
-      def formula_cleanup_where2(x)
+      def formula_cleanup_where2(formula)
         q = "//formula/following-sibling::*[1][self::p]"
-        x.xpath(q).each do |s|
-          if s.text =~ /^\s*where[^a-z]*$/i && !s.next_element.nil? && s.next_element.name == "dl"
+        formula.xpath(q).each do |s|
+          if s.text =~ /^\s*where[^a-z]*$/i && s&.next_element&.name == "dl"
             s.next_element["key"] = "true"
             s.previous_element << s.next_element.remove
             s.remove
@@ -114,7 +115,7 @@ module Asciidoctor
       def figure_dl_cleanup2(xmldoc)
         q = "//figure/following-sibling::*[self::p]"
         xmldoc.xpath(q).each do |s|
-          if s.text =~ /^\s*key[^a-z]*$/i && !s.next_element.nil? && s.next_element.name == "dl"
+          if s.text =~ /^\s*key[^a-z]*$/i && s&.next_element&.name == "dl"
             s.next_element["key"] = "true"
             s.previous_element << s.next_element.remove
             s.remove
@@ -125,7 +126,10 @@ module Asciidoctor
       # examples containing only figures become subfigures of figures
       def subfigure_cleanup(xmldoc)
         xmldoc.xpath("//example[figure]").each do |e|
-          next unless e.elements.map { |m| m.name }.reject { |m| %w(name figure).include? m }.empty?
+          next unless e.elements.map(&:name).reject do |m|
+            %w(name figure).include? m
+          end.empty?
+
           e.name = "figure"
         end
       end
@@ -140,82 +144,93 @@ module Asciidoctor
       ELEMS_ALLOW_NOTES = %w[p formula ul ol dl figure].freeze
 
       # if a note is at the end of a section, it is left alone
-      # if a note is followed by a non-note block, it is moved inside its preceding block if it is not delimited
+      # if a note is followed by a non-note block,
+      # it is moved inside its preceding block if it is not delimited
       # (so there was no way of making that block include the note)
       def note_cleanup(xmldoc)
         q = "//note[following-sibling::*[not(local-name() = 'note')]]"
         xmldoc.xpath(q).each do |n|
           next if n["keep-separate"] == "true"
           next unless n.ancestors("table").empty?
+
           prev = n.previous_element || next
           n.parent = prev if ELEMS_ALLOW_NOTES.include? prev.name
         end
-        xmldoc.xpath("//note[@keep-separate]").each { |n| n.delete("keep-separate") }
-        xmldoc.xpath("//termnote[@keep-separate]").each { |n| n.delete("keep-separate") }
+        xmldoc.xpath("//note[@keep-separate] | "\
+                     "//termnote[@keep-separate]").each do |n|
+          n.delete("keep-separate")
+        end
       end
 
-      def requirement_cleanup(x)
-        requirement_descriptions(x)
-        requirement_inherit(x)
+      def link_callouts_to_annotations(callouts, annotations)
+        callouts.each_with_index do |c, i|
+          c["target"] = "_#{UUIDTools::UUID.random_create}"
+          annotations[i]["id"] = c["target"]
+        end
       end
 
-      def requirement_inherit(x)
-        x.xpath("//requirement | //recommendation | //permission").each do |r|
-          ins = r.at("./classification") ||
-            r.at("./description | ./measurementtarget | ./specification | "\
-                 "./verification | ./import | ./description | ./requirement | "\
-                 "./recommendation | ./permission")
-          r.xpath("./*//inherit").each { |i| ins.previous = i }
+      def align_callouts_to_annotations(xmldoc)
+        xmldoc.xpath("//sourcecode").each do |x|
+          callouts = x.elements.select { |e| e.name == "callout" }
+          annotations = x.elements.select { |e| e.name == "annotation" }
+          callouts.size == annotations.size and
+            link_callouts_to_annotations(callouts, annotations)
         end
       end
 
-      def requirement_descriptions(x)
-        x.xpath("//requirement | //recommendation | //permission").each do |r|
-          r.children.each do |e|
-            unless e.element? && (reqt_subpart(e.name) ||
-                %w(requirement recommendation permission).include?(e.name))
-              t = Nokogiri::XML::Element.new("description", x)
-              e.before(t)
-              t.children = e.remove
-            end
+      def merge_annotations_into_sourcecode(xmldoc)
+        xmldoc.xpath("//sourcecode").each do |x|
+          while x&.next_element&.name == "annotation"
+            x.next_element.parent = x
           end
-          requirement_cleanup1(r)
         end
       end
 
-      def requirement_cleanup1(r)
-        while d = r.at("./description[following-sibling::*[1][self::description]]")
-          n = d.next.remove
-          d << n.children
+      def callout_cleanup(xmldoc)
+        merge_annotations_into_sourcecode(xmldoc)
+        align_callouts_to_annotations(xmldoc)
+      end
+
+      def sourcecode_cleanup(xmldoc)
+        xmldoc.xpath("//sourcecode").each do |x|
+          x.traverse do |n|
+            next unless n.text?
+            next unless /#{Regexp.escape(@sourcecode_markup_start)}/
+              .match?(n.text)
+
+            n.replace(sourcecode_markup(n))
+          end
         end
-        r.xpath("./description[normalize-space(.)='']").each { |d| d.replace("\n") }
       end
 
-      def svgmap_cleanup(xmldoc)
-        svgmap_populate(xmldoc)
-        Metanorma::Utils::svgmap_rewrite(xmldoc, @localdir)
+      def safe_noko(text, doc)
+        Nokogiri::XML::Text.new(text, doc).to_xml(
+          encoding: "US-ASCII",
+          save_with: Nokogiri::XML::Node::SaveOptions::NO_DECLARATION
+        )
       end
 
-      def svgmap_populate(xmldoc)
-        xmldoc.xpath("//svgmap").each do |s|
-          s1 = s.dup
-          s.children.remove
-          f = s1.at(".//figure") and s << f
-          s1.xpath(".//li").each do |li|
-            t = li&.at(".//eref | .//link | .//xref") or next
-            href = t.xpath("./following-sibling::node()")
-            next if href.empty?
-            s << %[<target href="#{svgmap_target(href)}">#{t.to_xml}</target>]
-          end
+      def sourcecode_markup(node)
+        acc = []
+        node.text.split(/(#{Regexp.escape(@sourcecode_markup_start)}|
+                          #{Regexp.escape(@sourcecode_markup_end)})/x)
+          .each_slice(4).map do |a|
+          acc << safe_noko(a[0], node.document)
+          next unless a.size == 4
+
+          acc << Asciidoctor.convert(
+            a[2], doctype: :inline, backend: (self&.backend&.to_sym || :standoc)
+          )
         end
+        acc.join
       end
 
-      def svgmap_target(nodeset)
-        nodeset.each do |n|
-          next unless n.name == "link"
-          n.children = n["target"]
+      def form_cleanup(xmldoc)
+        xmldoc.xpath("//select").each do |s|
+          while s&.next_element&.name == "option"
+            s << s.next_element
+          end
         end
-        nodeset.text.sub(/^[,; ]/, "").strip
       end
     end
   end

data/lib/asciidoctor/standoc/cleanup_boilerplate.rb

@@ -3,21 +3,26 @@ module Asciidoctor
     module Cleanup
       def external_terms_boilerplate(sources)
         @i18n.l10n(
-          @i18n.external_terms_boilerplate.gsub(/%/, sources || "???"), @lang, @script)
+          @i18n.external_terms_boilerplate.gsub(/%/, sources || "???"),
+          @lang, @script
+        )
       end
 
       def internal_external_terms_boilerplate(sources)
         @i18n.l10n(
-          @i18n.internal_external_terms_boilerplate.gsub(/%/, sources || "??"), @lang, @script)
+          @i18n.internal_external_terms_boilerplate.gsub(/%/, sources || "??"),
+          @lang, @script
+        )
       end
 
       def term_defs_boilerplate(div, source, term, preface, isodoc)
         a = @i18n.term_def_boilerplate and div.next = a
         source.each do |s|
           @anchors[s["bibitemid"]] or
-            @log.add("Crossreferences", nil, "term source #{s['bibitemid']} not referenced")
+            @log.add("Crossreferences", nil,
+                     "term source #{s['bibitemid']} not referenced")
         end
-        a = (source.empty? && term.nil?) ?  @i18n.no_terms_boilerplate :
+        a = source.empty? && term.nil? ?  @i18n.no_terms_boilerplate :
           term_defs_boilerplate_cont(source, term, isodoc)
         a and div.next = a
       end
@@ -33,17 +38,23 @@ module Asciidoctor
         end
       end
 
-      def norm_ref_preface(f)
-        refs = f.elements.select do |e|
-          ["reference", "bibitem"].include? e.name
+      def norm_ref_preface(ref)
+        if ref.at("./note[@type = 'boilerplate']")
+          unwrap_boilerplate_clauses(ref, ".")
+        else
+          refs = ref.elements.select do |e|
+            %w(references bibitem).include? e.name
+          end
+          pref = refs.empty? ? @i18n.norm_empty_pref : @i18n.norm_with_refs_pref
+          ref.at("./title").next = "<p>#{pref}</p>"
         end
-        f.at("./title").next =
-          "<p>#{(refs.empty? ? @i18n.norm_empty_pref : @i18n.norm_with_refs_pref)}</p>"
       end
 
-      TERM_CLAUSE = "//sections/terms | //sections/clause[descendant::terms]".freeze
+      TERM_CLAUSE = "//sections/terms | "\
+        "//sections/clause[descendant::terms]".freeze
 
-      NORM_REF = "//bibliography/references[@normative = 'true']".freeze
+      NORM_REF = "//bibliography/references[@normative = 'true'] | "\
+        "//bibliography/clause[.//references[@normative = 'true']]".freeze
 
       def boilerplate_isodoc(xmldoc)
         x = xmldoc.dup
@@ -55,44 +66,53 @@ module Asciidoctor
       end
 
       def termdef_boilerplate_cleanup(xmldoc)
-        #termdef_remove_initial_paras(xmldoc)
+        # termdef_remove_initial_paras(xmldoc)
       end
 
       def termdef_remove_initial_paras(xmldoc)
         xmldoc.xpath("//terms/p | //terms/ul").each(&:remove)
       end
 
-      def termdef_unwrap_boilerplate_clauses(xmldoc)
-        xmldoc.xpath(self.class::TERM_CLAUSE).each do |f|
-          f.xpath(".//clause[@type = 'boilerplate']").each do |c|
+      def unwrap_boilerplate_clauses(xmldoc, xpath)
+        xmldoc.xpath(xpath).each do |f|
+          f.xpath(".//clause[@type = 'boilerplate'] | "\
+                  ".//note[@type = 'boilerplate']").each do |c|
             c&.at("./title")&.remove
             c.replace(c.children)
           end
         end
       end
 
-      def boilerplate_cleanup(xmldoc)
-        isodoc = boilerplate_isodoc(xmldoc)
-        termdef_boilerplate_cleanup(xmldoc)
+      def termdef_boilerplate_insert(xmldoc, isodoc, once = false)
         xmldoc.xpath(self.class::TERM_CLAUSE).each do |f|
           next if f.at("./clause[@type = 'boilerplate']")
-          term_defs_boilerplate(f.at("./title"), xmldoc.xpath(".//termdocsource"),
+
+          term_defs_boilerplate(f.at("./title"),
+                                xmldoc.xpath(".//termdocsource"),
                                 f.at(".//term"), f.at(".//p"), isodoc)
+          break if once
         end
-        termdef_unwrap_boilerplate_clauses(xmldoc)
+      end
+
+      def boilerplate_cleanup(xmldoc)
+        isodoc = boilerplate_isodoc(xmldoc)
+        termdef_boilerplate_cleanup(xmldoc)
+        termdef_boilerplate_insert(xmldoc, isodoc)
+        unwrap_boilerplate_clauses(xmldoc, self.class::TERM_CLAUSE)
         f = xmldoc.at(self.class::NORM_REF) and norm_ref_preface(f)
         initial_boilerplate(xmldoc, isodoc)
       end
 
-      def initial_boilerplate(x, isodoc)
-        return if x.at("//boilerplate")
-        preface = x.at("//preface") || x.at("//sections") || x.at("//annex") ||
-          x.at("//references") || return
-        b = boilerplate(x, isodoc) or return
+      def initial_boilerplate(xml, isodoc)
+        return if xml.at("//boilerplate")
+
+        preface = xml.at("//preface") || xml.at("//sections") ||
+          xml.at("//annex") || xml.at("//references") || return
+        b = boilerplate(xml, isodoc) or return
         preface.previous = b
       end
 
-      def boilerplate_file(xmldoc)
+      def boilerplate_file(_xmldoc)
         File.join(@libdir, "boilerplate.xml")
       end
 
@@ -100,7 +120,7 @@ module Asciidoctor
         file = boilerplate_file(xml)
         file = File.join(@localdir, @boilerplateauthority) if @boilerplateauthority
         !file.nil? and File.exists?(file) or return
-        conv.populate_template((File.read(file, encoding: "UTF-8")), nil)
+        conv.populate_template(File.read(file, encoding: "UTF-8"), nil)
       end
 
       def bibdata_cleanup(xmldoc)
@@ -118,14 +138,16 @@ module Asciidoctor
       def bibdata_docidentifier_cleanup(xmldoc)
         ins = xmldoc.at("//bibdata/docidentifier")
         xmldoc.xpath("//bibdata/docidentifier").each_with_index do |b, i|
-          next if i == 0
+          next if i.zero?
+
           ins.next = b.remove
           ins = ins.next
         end
       end
 
       def gather_indirect_erefs(xmldoc, prefix)
-        xmldoc.xpath("//eref[@type = '#{prefix}']").each_with_object({}) do |e, m|
+        xmldoc.xpath("//eref[@type = '#{prefix}']")
+          .each_with_object({}) do |e, m|
           e.delete("type")
           m[e["bibitemid"]] = true
         end.keys