metanorma-standoc 1.11.3 → 2.0.2

data/lib/metanorma/standoc/cleanup.rb

@@ -0,0 +1,157 @@
+require "nokogiri"
+require "pathname"
+require "html2doc"
+require_relative "./cleanup_block"
+require_relative "./cleanup_table"
+require_relative "./cleanup_footnotes"
+require_relative "./cleanup_ref"
+require_relative "./cleanup_ref_dl"
+require_relative "./cleanup_boilerplate"
+require_relative "./cleanup_section"
+require_relative "./cleanup_terms"
+require_relative "./cleanup_symbols"
+require_relative "./cleanup_xref"
+require_relative "./cleanup_inline"
+require_relative "./cleanup_amend"
+require_relative "./cleanup_maths"
+require_relative "./cleanup_image"
+require_relative "./cleanup_reqt"
+require_relative "./cleanup_text"
+require_relative "./cleanup_toc"
+require "relaton_iev"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def cleanup(xmldoc)
+        element_name_cleanup(xmldoc)
+        sections_cleanup(xmldoc)
+        obligations_cleanup(xmldoc)
+        table_cleanup(xmldoc)
+        formula_cleanup(xmldoc)
+        form_cleanup(xmldoc)
+        sourcecode_cleanup(xmldoc)
+        figure_cleanup(xmldoc)
+        element_name_cleanup(xmldoc)
+        ref_cleanup(xmldoc)
+        note_cleanup(xmldoc)
+        clausebefore_cleanup(xmldoc)
+        floatingtitle_cleanup(xmldoc)
+        bibitem_cleanup(xmldoc)
+        normref_cleanup(xmldoc)
+        biblio_cleanup(xmldoc)
+        reference_names(xmldoc)
+        symbols_cleanup(xmldoc)
+        xref_cleanup(xmldoc)
+        concept_cleanup(xmldoc)
+        related_cleanup(xmldoc)
+        origin_cleanup(xmldoc)
+        bookmark_cleanup(xmldoc)
+        termdef_cleanup(xmldoc)
+        RelatonIev::iev_cleanup(xmldoc, @bibdb)
+        element_name_cleanup(xmldoc)
+        index_cleanup(xmldoc)
+        bpart_cleanup(xmldoc)
+        quotesource_cleanup(xmldoc)
+        callout_cleanup(xmldoc)
+        footnote_cleanup(xmldoc)
+        mathml_cleanup(xmldoc)
+        script_cleanup(xmldoc)
+        docidentifier_cleanup(xmldoc)
+        requirement_cleanup(xmldoc)
+        bibdata_cleanup(xmldoc)
+        svgmap_cleanup(xmldoc)
+        boilerplate_cleanup(xmldoc)
+        toc_cleanup(xmldoc)
+        metadata_cleanup(xmldoc)
+        smartquotes_cleanup(xmldoc)
+        variant_cleanup(xmldoc)
+        para_cleanup(xmldoc)
+        empty_element_cleanup(xmldoc)
+        img_cleanup(xmldoc)
+        anchor_cleanup(xmldoc)
+        xmldoc
+      end
+
+      def docidentifier_cleanup(xmldoc); end
+
+      TEXT_ELEMS =
+        %w{status language script version author name callout phone email
+           street city state country postcode identifier referenceFrom surname
+           referenceTo docidentifier docnumber prefix initial addition forename
+           title draft secretariat title-main title-intro title-part
+           verbal-definition non-verbal-representation}.freeze
+
+      # it seems Nokogiri::XML is treating the content of <script> as cdata,
+      # because of its use in HTML. Bad nokogiri. Undoing that, since we use
+      # script as a normal tag
+      def script_cleanup(xmldoc)
+        xmldoc.xpath("//script").each { |x| x.content = x.to_str }
+      end
+
+      def empty_element_cleanup(xmldoc)
+        xmldoc.xpath("//#{TEXT_ELEMS.join(' | //')}").each do |x|
+          next if x.name == "name" && x.parent.name == "expression"
+
+          x.remove if x.children.empty?
+        end
+      end
+
+      def element_name_cleanup(xmldoc)
+        xmldoc.traverse { |n| n.name = n.name.gsub(/_/, "-") }
+      end
+
+      # allows us to deal with doc relation localities,
+      # temporarily stashed to "bpart"
+      def bpart_cleanup(xmldoc)
+        xmldoc.xpath("//relation/bpart").each do |x|
+          extract_localities(x)
+          x.replace(x.children)
+        end
+      end
+
+      def variant_cleanup(xmldoc)
+        variant_space_cleanup(xmldoc)
+        xmldoc.xpath("//*[variant]").each do |c|
+          next unless c.children.any? do |n|
+            n.name != "variant" && (!n.text? || !n.text.gsub(/\s/, "").empty?)
+          end
+
+          variant_cleanup1(c)
+        end
+        xmldoc.xpath("//variantwrap").each { |n| n.name = "variant" }
+      end
+
+      def variant_cleanup1(elem)
+        elem.xpath("./variant").each do |n|
+          if n.at_xpath("preceding-sibling::node()"\
+                        "[not(self::text()[not(normalize-space())])][1]"\
+                        "[self::variantwrap]")
+            n.previous_element << n
+          else
+            n.replace("<variantwrap/>").first << n
+          end
+        end
+      end
+
+      def variant_space_cleanup(xmldoc)
+        xmldoc.xpath("//*[variant]").each do |c|
+          next if c.next.nil? || c.next.next.nil?
+
+          if c.next.text? && c.next.next.name == "variant"
+            c.next.text.gsub(/\s/, "").empty? and
+              c.next.remove
+          end
+        end
+      end
+
+      def metadata_cleanup(xmldoc)
+        return if @metadata_attrs.nil? || @metadata_attrs.empty?
+
+        ins = xmldoc.at("//misc-container") ||
+          xmldoc.at("//bibdata").after("<misc-container/>").next_element
+        ins << @metadata_attrs
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_amend.rb

@@ -0,0 +1,54 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def change_clauses(x)
+        x.xpath("//clause[@change]").each do |c|
+          a = create_amend(c)
+        end
+      end
+
+      def create_amend(c)
+        a = c.add_child("<amend id='_#{UUIDTools::UUID.random_create}'/>").first
+        c.elements.each do |e|
+          e.parent = a unless %w(amend title).include? e.name
+        end
+        create_amend1(c, a)
+      end
+
+      def create_amend1(c, a)
+        create_amend2(c, a)
+        d = a.at("./description")
+        d.xpath(".//autonumber").each { |e| d.previous = e }
+        d.xpath(".//p[normalize-space(.)='']").each { |e| e.remove }
+        move_attrs_to_amend(c, a)
+        a
+      end
+
+      def create_amend2(c, a)
+        q = a.at("./quote") and q.name = "newcontent"
+        if q.nil?
+          a.children = "<description>#{a.children.to_xml}</description>"
+        else
+          pre = q&.xpath("./preceding-sibling::*")&.remove
+          post = q&.xpath("./following-sibling::*")&.remove
+          pre.empty? or a << "<description>#{pre.to_xml}</description>"
+          a << q.remove
+          post.empty? or a << "<description>#{post.to_xml}</description>"
+        end
+      end
+
+      def move_attrs_to_amend(c, a)
+        %w(change path path_end title).each do |e|
+          next unless c[e]
+          a[e] = c[e]
+          c.delete(e)
+        end
+        return unless a["locality"]
+        loc = a.children.add_previous_sibling("<location/>")
+        extract_localities1(loc, a["locality"])
+        loc1 = loc.at("./localityStack") and loc.replace(loc1.elements)
+        a.delete("locality")
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_block.rb

@@ -0,0 +1,173 @@
+require "date"
+require "htmlentities"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def para_cleanup(xmldoc)
+        ["//p[not(ancestor::bibdata)]", "//ol[not(ancestor::bibdata)]",
+         "//ul[not(ancestor::bibdata)]", "//quote[not(ancestor::bibdata)]",
+         "//note[not(ancestor::bibitem or "\
+         "ancestor::table or ancestor::bibdata)]"].each do |w|
+          inject_id(xmldoc, w)
+        end
+      end
+
+      def inject_id(xmldoc, path)
+        xmldoc.xpath(path).each do |x|
+          x["id"] ||= Metanorma::Utils::anchor_or_uuid
+        end
+      end
+
+      # include where definition list inside stem block
+      def formula_cleanup(formula)
+        formula_cleanup_where1(formula)
+        formula_cleanup_where2(formula)
+      end
+
+      def formula_cleanup_where1(formula)
+        q = "//formula/following-sibling::*[1][self::dl]"
+        formula.xpath(q).each do |s|
+          s["key"] == "true" and s.previous_element << s.remove
+        end
+      end
+
+      def formula_cleanup_where2(formula)
+        q = "//formula/following-sibling::*[1][self::p]"
+        formula.xpath(q).each do |s|
+          if s.text =~ /^\s*where[^a-z]*$/i && s&.next_element&.name == "dl"
+            s.next_element["key"] = "true"
+            s.previous_element << s.next_element.remove
+            s.remove
+          end
+        end
+      end
+
+      def figure_dl_cleanup1(xmldoc)
+        q = "//figure/following-sibling::*[self::dl]"
+        xmldoc.xpath(q).each do |s|
+          s["key"] == "true" and s.previous_element << s.remove
+        end
+      end
+
+      # include key definition list inside figure
+      def figure_dl_cleanup2(xmldoc)
+        q = "//figure/following-sibling::*[self::p]"
+        xmldoc.xpath(q).each do |s|
+          if s.text =~ /^\s*key[^a-z]*$/i && s&.next_element&.name == "dl"
+            s.next_element["key"] = "true"
+            s.previous_element << s.next_element.remove
+            s.remove
+          end
+        end
+      end
+
+      # examples containing only figures become subfigures of figures
+      def subfigure_cleanup(xmldoc)
+        xmldoc.xpath("//example[figure]").each do |e|
+          next unless e.elements.map(&:name).reject do |m|
+            %w(name figure).include? m
+          end.empty?
+
+          e.name = "figure"
+        end
+      end
+
+      def figure_cleanup(xmldoc)
+        figure_footnote_cleanup(xmldoc)
+        figure_dl_cleanup1(xmldoc)
+        figure_dl_cleanup2(xmldoc)
+        subfigure_cleanup(xmldoc)
+      end
+
+      ELEMS_ALLOW_NOTES = %w[p formula ul ol dl figure].freeze
+
+      # if a note is at the end of a section, it is left alone
+      # if a note is followed by a non-note block,
+      # it is moved inside its preceding block if it is not delimited
+      # (so there was no way of making that block include the note)
+      def note_cleanup(xmldoc)
+        q = "//note[following-sibling::*[not(local-name() = 'note')]]"
+        xmldoc.xpath(q).each do |n|
+          next if n["keep-separate"] == "true" || !n.ancestors("table").empty?
+
+          prev = n.previous_element || next
+          n.parent = prev if ELEMS_ALLOW_NOTES.include? prev.name
+        end
+        xmldoc.xpath("//note[@keep-separate] | "\
+                     "//termnote[@keep-separate]").each do |n|
+          n.delete("keep-separate")
+        end
+      end
+
+      def link_callouts_to_annotations(callouts, annotations)
+        callouts.each_with_index do |c, i|
+          c["target"] = "_#{UUIDTools::UUID.random_create}"
+          annotations[i]["id"] = c["target"]
+        end
+      end
+
+      def align_callouts_to_annotations(xmldoc)
+        xmldoc.xpath("//sourcecode").each do |x|
+          callouts = x.elements.select { |e| e.name == "callout" }
+          annotations = x.elements.select { |e| e.name == "annotation" }
+          callouts.size == annotations.size and
+            link_callouts_to_annotations(callouts, annotations)
+        end
+      end
+
+      def merge_annotations_into_sourcecode(xmldoc)
+        xmldoc.xpath("//sourcecode").each do |x|
+          while x&.next_element&.name == "annotation"
+            x.next_element.parent = x
+          end
+        end
+      end
+
+      def callout_cleanup(xmldoc)
+        merge_annotations_into_sourcecode(xmldoc)
+        align_callouts_to_annotations(xmldoc)
+      end
+
+      def sourcecode_cleanup(xmldoc)
+        xmldoc.xpath("//sourcecode").each do |x|
+          x.traverse do |n|
+            next unless n.text?
+            next unless /#{Regexp.escape(@sourcecode_markup_start)}/
+              .match?(n.text)
+
+            n.replace(sourcecode_markup(n))
+          end
+        end
+      end
+
+      def safe_noko(text, doc)
+        Nokogiri::XML::Text.new(text, doc).to_xml(
+          encoding: "US-ASCII",
+          save_with: Nokogiri::XML::Node::SaveOptions::NO_DECLARATION,
+        )
+      end
+
+      def sourcecode_markup(node)
+        node.text.split(/(#{Regexp.escape(@sourcecode_markup_start)}|
+                          #{Regexp.escape(@sourcecode_markup_end)})/x)
+          .each_slice(4).map.with_object([]) do |a, acc|
+          acc << safe_noko(a[0], node.document)
+          next unless a.size == 4
+
+          acc << Asciidoctor.convert(
+            a[2], doctype: :inline, backend: (self&.backend&.to_sym || :standoc)
+          )
+        end.join
+      end
+
+      def form_cleanup(xmldoc)
+        xmldoc.xpath("//select").each do |s|
+          while s&.next_element&.name == "option"
+            s << s.next_element
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_boilerplate.rb

@@ -0,0 +1,213 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def external_terms_boilerplate(sources)
+        @i18n.l10n(
+          @i18n.external_terms_boilerplate.gsub(/%/, sources || "???"),
+          @lang, @script
+        )
+      end
+
+      def internal_external_terms_boilerplate(sources)
+        @i18n.l10n(
+          @i18n.internal_external_terms_boilerplate.gsub(/%/, sources || "??"),
+          @lang, @script
+        )
+      end
+
+      def term_defs_boilerplate(div, source, term, _preface, isodoc)
+        a = @i18n.term_def_boilerplate and div.next = a
+        source.each do |s|
+          @anchors[s["bibitemid"]] or
+            @log.add("Crossreferences", nil,
+                     "term source #{s['bibitemid']} not referenced")
+        end
+        a = if source.empty? && term.nil?
+              @i18n.no_terms_boilerplate
+            else
+              term_defs_boilerplate_cont(source, term, isodoc)
+            end
+        a and div.next = a
+      end
+
+      def term_defs_boilerplate_cont(src, term, isodoc)
+        sources = isodoc.sentence_join(src.map do |s|
+          %{<eref bibitemid="#{s['bibitemid']}"/>}
+        end)
+        if src.empty? then @i18n.internal_terms_boilerplate
+        elsif term.nil? then external_terms_boilerplate(sources)
+        else
+          internal_external_terms_boilerplate(sources)
+        end
+      end
+
+      def norm_ref_preface(ref)
+        if ref.at("./note[@type = 'boilerplate']")
+          unwrap_boilerplate_clauses(ref, ".")
+        else
+          refs = ref.elements.select do |e|
+            %w(references bibitem).include? e.name
+          end
+          pref = refs.empty? ? @i18n.norm_empty_pref : @i18n.norm_with_refs_pref
+          ref.at("./title").next = "<p>#{pref}</p>"
+        end
+      end
+
+      TERM_CLAUSE = "//sections/terms | "\
+                    "//sections/clause[descendant::terms]".freeze
+
+      NORM_REF = "//bibliography/references[@normative = 'true'] | "\
+                 "//bibliography/clause[.//references[@normative = 'true']]"
+        .freeze
+
+      def boilerplate_isodoc(xmldoc)
+        x = xmldoc.dup
+        x.root.add_namespace(nil, self.class::XML_NAMESPACE)
+        xml = Nokogiri::XML(x.to_xml)
+        @isodoc ||= isodoc(@lang, @script)
+        @isodoc.info(xml, nil)
+        @isodoc
+      end
+
+      def termdef_boilerplate_cleanup(xmldoc)
+        # termdef_remove_initial_paras(xmldoc)
+      end
+
+      def termdef_remove_initial_paras(xmldoc)
+        xmldoc.xpath("//terms/p | //terms/ul").each(&:remove)
+      end
+
+      def unwrap_boilerplate_clauses(xmldoc, xpath)
+        xmldoc.xpath(xpath).each do |f|
+          f.xpath(".//clause[@type = 'boilerplate'] | "\
+                  ".//note[@type = 'boilerplate']").each do |c|
+            c&.at("./title")&.remove
+            c.replace(c.children)
+          end
+        end
+      end
+
+      def termdef_boilerplate_insert(xmldoc, isodoc, once = false)
+        xmldoc.xpath(self.class::TERM_CLAUSE).each do |f|
+          next if f.at("./clause[@type = 'boilerplate']")
+
+          term_defs_boilerplate(f.at("./title"),
+                                xmldoc.xpath(".//termdocsource"),
+                                f.at(".//term"), f.at(".//p"), isodoc)
+          break if once
+        end
+      end
+
+      def boilerplate_cleanup(xmldoc)
+        isodoc = boilerplate_isodoc(xmldoc)
+        termdef_boilerplate_cleanup(xmldoc)
+        termdef_boilerplate_insert(xmldoc, isodoc)
+        unwrap_boilerplate_clauses(xmldoc, self.class::TERM_CLAUSE)
+        f = xmldoc.at(self.class::NORM_REF) and norm_ref_preface(f)
+        initial_boilerplate(xmldoc, isodoc)
+      end
+
+      def initial_boilerplate(xml, isodoc)
+        return if xml.at("//boilerplate")
+
+        preface = xml.at("//preface") || xml.at("//sections") ||
+          xml.at("//annex") || xml.at("//references") || return
+        b = boilerplate(xml, isodoc) or return
+        preface.previous = b
+      end
+
+      def boilerplate_file(_xmldoc)
+        File.join(@libdir, "boilerplate.xml")
+      end
+
+      def boilerplate(xml, conv)
+        file = boilerplate_file(xml)
+        if @boilerplateauthority
+          file = File.join(@localdir,
+                           @boilerplateauthority)
+        end
+        (!file.nil? and File.exists?(file)) or return
+        conv.populate_template(File.read(file, encoding: "UTF-8"), nil)
+      end
+
+      def bibdata_cleanup(xmldoc)
+        bibdata_anchor_cleanup(xmldoc)
+        bibdata_docidentifier_cleanup(xmldoc)
+        biblio_indirect_erefs(xmldoc, @internal_eref_namespaces&.uniq)
+      end
+
+      def bibdata_anchor_cleanup(xmldoc)
+        xmldoc.xpath("//bibdata//bibitem | //bibdata//note").each do |b|
+          b.delete("id")
+        end
+      end
+
+      def bibdata_docidentifier_cleanup(xmldoc)
+        ins = xmldoc.at("//bibdata/docidentifier")
+        xmldoc.xpath("//bibdata/docidentifier").each_with_index do |b, i|
+          next if i.zero?
+
+          ins.next = b.remove
+          ins = ins.next
+        end
+      end
+
+      def gather_indirect_erefs(xmldoc, prefix)
+        xmldoc.xpath("//eref[@type = '#{prefix}']")
+          .each_with_object({}) do |e, m|
+          e.delete("type")
+          m[e["bibitemid"]] = true
+        end.keys
+      end
+
+      def insert_indirect_biblio(xmldoc, refs, prefix)
+        ins = xmldoc.at("bibliography") or
+          xmldoc.root << "<bibliography/>" and ins = xmldoc.at("bibliography")
+        ins = ins.add_child("<references hidden='true' normative='false'/>").first
+        refs.each do |x|
+          ins << <<~BIB
+            <bibitem id="#{x}" type="internal">
+            <docidentifier type="repository">#{x.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
+            </bibitem>
+          BIB
+        end
+      end
+
+      def indirect_eref_to_xref(eref, ident)
+        loc = eref&.at("./localityStack[locality[@type = 'anchor']]")
+          &.remove&.text ||
+          eref&.at("./locality[@type = 'anchor']")&.remove&.text || ident
+        eref.name = "xref"
+        eref.delete("bibitemid")
+        eref.delete("citeas")
+        eref["target"] = loc
+        unless eref.document.at("//*[@id = '#{loc}']")
+          eref.children = %(** Missing target #{loc})
+          eref["target"] = ident
+        end
+      end
+
+      def resolve_local_indirect_erefs(xmldoc, refs, prefix)
+        refs.each_with_object([]) do |r, m|
+          id = r.sub(/^#{prefix}_/, "")
+          if n = xmldoc.at("//*[@id = '#{id}']") and
+              n.at("./ancestor-or-self::*[@type = '#{prefix}']")
+            xmldoc.xpath("//eref[@bibitemid = '#{r}']").each do |e|
+              indirect_eref_to_xref(e, id)
+            end
+          else m << r
+          end
+        end
+      end
+
+      def biblio_indirect_erefs(xmldoc, prefixes)
+        prefixes&.each do |prefix|
+          refs = gather_indirect_erefs(xmldoc, prefix)
+          refs = resolve_local_indirect_erefs(xmldoc, refs, prefix)
+          refs.empty? and next
+          insert_indirect_biblio(xmldoc, refs, prefix)
+        end
+      end
+    end
+  end
+end