metanorma-standoc 1.11.1 → 2.0.0

data/lib/metanorma/standoc/cleanup_block.rb

@@ -0,0 +1,173 @@
+require "date"
+require "htmlentities"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def para_cleanup(xmldoc)
+        ["//p[not(ancestor::bibdata)]", "//ol[not(ancestor::bibdata)]",
+         "//ul[not(ancestor::bibdata)]", "//quote[not(ancestor::bibdata)]",
+         "//note[not(ancestor::bibitem or "\
+         "ancestor::table or ancestor::bibdata)]"].each do |w|
+          inject_id(xmldoc, w)
+        end
+      end
+
+      def inject_id(xmldoc, path)
+        xmldoc.xpath(path).each do |x|
+          x["id"] ||= Metanorma::Utils::anchor_or_uuid
+        end
+      end
+
+      # include where definition list inside stem block
+      def formula_cleanup(formula)
+        formula_cleanup_where1(formula)
+        formula_cleanup_where2(formula)
+      end
+
+      def formula_cleanup_where1(formula)
+        q = "//formula/following-sibling::*[1][self::dl]"
+        formula.xpath(q).each do |s|
+          s["key"] == "true" and s.previous_element << s.remove
+        end
+      end
+
+      def formula_cleanup_where2(formula)
+        q = "//formula/following-sibling::*[1][self::p]"
+        formula.xpath(q).each do |s|
+          if s.text =~ /^\s*where[^a-z]*$/i && s&.next_element&.name == "dl"
+            s.next_element["key"] = "true"
+            s.previous_element << s.next_element.remove
+            s.remove
+          end
+        end
+      end
+
+      def figure_dl_cleanup1(xmldoc)
+        q = "//figure/following-sibling::*[self::dl]"
+        xmldoc.xpath(q).each do |s|
+          s["key"] == "true" and s.previous_element << s.remove
+        end
+      end
+
+      # include key definition list inside figure
+      def figure_dl_cleanup2(xmldoc)
+        q = "//figure/following-sibling::*[self::p]"
+        xmldoc.xpath(q).each do |s|
+          if s.text =~ /^\s*key[^a-z]*$/i && s&.next_element&.name == "dl"
+            s.next_element["key"] = "true"
+            s.previous_element << s.next_element.remove
+            s.remove
+          end
+        end
+      end
+
+      # examples containing only figures become subfigures of figures
+      def subfigure_cleanup(xmldoc)
+        xmldoc.xpath("//example[figure]").each do |e|
+          next unless e.elements.map(&:name).reject do |m|
+            %w(name figure).include? m
+          end.empty?
+
+          e.name = "figure"
+        end
+      end
+
+      def figure_cleanup(xmldoc)
+        figure_footnote_cleanup(xmldoc)
+        figure_dl_cleanup1(xmldoc)
+        figure_dl_cleanup2(xmldoc)
+        subfigure_cleanup(xmldoc)
+      end
+
+      ELEMS_ALLOW_NOTES = %w[p formula ul ol dl figure].freeze
+
+      # if a note is at the end of a section, it is left alone
+      # if a note is followed by a non-note block,
+      # it is moved inside its preceding block if it is not delimited
+      # (so there was no way of making that block include the note)
+      def note_cleanup(xmldoc)
+        q = "//note[following-sibling::*[not(local-name() = 'note')]]"
+        xmldoc.xpath(q).each do |n|
+          next if n["keep-separate"] == "true" || !n.ancestors("table").empty?
+
+          prev = n.previous_element || next
+          n.parent = prev if ELEMS_ALLOW_NOTES.include? prev.name
+        end
+        xmldoc.xpath("//note[@keep-separate] | "\
+                     "//termnote[@keep-separate]").each do |n|
+          n.delete("keep-separate")
+        end
+      end
+
+      def link_callouts_to_annotations(callouts, annotations)
+        callouts.each_with_index do |c, i|
+          c["target"] = "_#{UUIDTools::UUID.random_create}"
+          annotations[i]["id"] = c["target"]
+        end
+      end
+
+      def align_callouts_to_annotations(xmldoc)
+        xmldoc.xpath("//sourcecode").each do |x|
+          callouts = x.elements.select { |e| e.name == "callout" }
+          annotations = x.elements.select { |e| e.name == "annotation" }
+          callouts.size == annotations.size and
+            link_callouts_to_annotations(callouts, annotations)
+        end
+      end
+
+      def merge_annotations_into_sourcecode(xmldoc)
+        xmldoc.xpath("//sourcecode").each do |x|
+          while x&.next_element&.name == "annotation"
+            x.next_element.parent = x
+          end
+        end
+      end
+
+      def callout_cleanup(xmldoc)
+        merge_annotations_into_sourcecode(xmldoc)
+        align_callouts_to_annotations(xmldoc)
+      end
+
+      def sourcecode_cleanup(xmldoc)
+        xmldoc.xpath("//sourcecode").each do |x|
+          x.traverse do |n|
+            next unless n.text?
+            next unless /#{Regexp.escape(@sourcecode_markup_start)}/
+              .match?(n.text)
+
+            n.replace(sourcecode_markup(n))
+          end
+        end
+      end
+
+      def safe_noko(text, doc)
+        Nokogiri::XML::Text.new(text, doc).to_xml(
+          encoding: "US-ASCII",
+          save_with: Nokogiri::XML::Node::SaveOptions::NO_DECLARATION,
+        )
+      end
+
+      def sourcecode_markup(node)
+        node.text.split(/(#{Regexp.escape(@sourcecode_markup_start)}|
+                          #{Regexp.escape(@sourcecode_markup_end)})/x)
+          .each_slice(4).map.with_object([]) do |a, acc|
+          acc << safe_noko(a[0], node.document)
+          next unless a.size == 4
+
+          acc << Asciidoctor.convert(
+            a[2], doctype: :inline, backend: (self&.backend&.to_sym || :standoc)
+          )
+        end.join
+      end
+
+      def form_cleanup(xmldoc)
+        xmldoc.xpath("//select").each do |s|
+          while s&.next_element&.name == "option"
+            s << s.next_element
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_boilerplate.rb

@@ -0,0 +1,213 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def external_terms_boilerplate(sources)
+        @i18n.l10n(
+          @i18n.external_terms_boilerplate.gsub(/%/, sources || "???"),
+          @lang, @script
+        )
+      end
+
+      def internal_external_terms_boilerplate(sources)
+        @i18n.l10n(
+          @i18n.internal_external_terms_boilerplate.gsub(/%/, sources || "??"),
+          @lang, @script
+        )
+      end
+
+      def term_defs_boilerplate(div, source, term, _preface, isodoc)
+        a = @i18n.term_def_boilerplate and div.next = a
+        source.each do |s|
+          @anchors[s["bibitemid"]] or
+            @log.add("Crossreferences", nil,
+                     "term source #{s['bibitemid']} not referenced")
+        end
+        a = if source.empty? && term.nil?
+              @i18n.no_terms_boilerplate
+            else
+              term_defs_boilerplate_cont(source, term, isodoc)
+            end
+        a and div.next = a
+      end
+
+      def term_defs_boilerplate_cont(src, term, isodoc)
+        sources = isodoc.sentence_join(src.map do |s|
+          %{<eref bibitemid="#{s['bibitemid']}"/>}
+        end)
+        if src.empty? then @i18n.internal_terms_boilerplate
+        elsif term.nil? then external_terms_boilerplate(sources)
+        else
+          internal_external_terms_boilerplate(sources)
+        end
+      end
+
+      def norm_ref_preface(ref)
+        if ref.at("./note[@type = 'boilerplate']")
+          unwrap_boilerplate_clauses(ref, ".")
+        else
+          refs = ref.elements.select do |e|
+            %w(references bibitem).include? e.name
+          end
+          pref = refs.empty? ? @i18n.norm_empty_pref : @i18n.norm_with_refs_pref
+          ref.at("./title").next = "<p>#{pref}</p>"
+        end
+      end
+
+      TERM_CLAUSE = "//sections/terms | "\
+                    "//sections/clause[descendant::terms]".freeze
+
+      NORM_REF = "//bibliography/references[@normative = 'true'] | "\
+                 "//bibliography/clause[.//references[@normative = 'true']]"
+        .freeze
+
+      def boilerplate_isodoc(xmldoc)
+        x = xmldoc.dup
+        x.root.add_namespace(nil, self.class::XML_NAMESPACE)
+        xml = Nokogiri::XML(x.to_xml)
+        @isodoc ||= isodoc(@lang, @script)
+        @isodoc.info(xml, nil)
+        @isodoc
+      end
+
+      def termdef_boilerplate_cleanup(xmldoc)
+        # termdef_remove_initial_paras(xmldoc)
+      end
+
+      def termdef_remove_initial_paras(xmldoc)
+        xmldoc.xpath("//terms/p | //terms/ul").each(&:remove)
+      end
+
+      def unwrap_boilerplate_clauses(xmldoc, xpath)
+        xmldoc.xpath(xpath).each do |f|
+          f.xpath(".//clause[@type = 'boilerplate'] | "\
+                  ".//note[@type = 'boilerplate']").each do |c|
+            c&.at("./title")&.remove
+            c.replace(c.children)
+          end
+        end
+      end
+
+      def termdef_boilerplate_insert(xmldoc, isodoc, once = false)
+        xmldoc.xpath(self.class::TERM_CLAUSE).each do |f|
+          next if f.at("./clause[@type = 'boilerplate']")
+
+          term_defs_boilerplate(f.at("./title"),
+                                xmldoc.xpath(".//termdocsource"),
+                                f.at(".//term"), f.at(".//p"), isodoc)
+          break if once
+        end
+      end
+
+      def boilerplate_cleanup(xmldoc)
+        isodoc = boilerplate_isodoc(xmldoc)
+        termdef_boilerplate_cleanup(xmldoc)
+        termdef_boilerplate_insert(xmldoc, isodoc)
+        unwrap_boilerplate_clauses(xmldoc, self.class::TERM_CLAUSE)
+        f = xmldoc.at(self.class::NORM_REF) and norm_ref_preface(f)
+        initial_boilerplate(xmldoc, isodoc)
+      end
+
+      def initial_boilerplate(xml, isodoc)
+        return if xml.at("//boilerplate")
+
+        preface = xml.at("//preface") || xml.at("//sections") ||
+          xml.at("//annex") || xml.at("//references") || return
+        b = boilerplate(xml, isodoc) or return
+        preface.previous = b
+      end
+
+      def boilerplate_file(_xmldoc)
+        File.join(@libdir, "boilerplate.xml")
+      end
+
+      def boilerplate(xml, conv)
+        file = boilerplate_file(xml)
+        if @boilerplateauthority
+          file = File.join(@localdir,
+                           @boilerplateauthority)
+        end
+        (!file.nil? and File.exists?(file)) or return
+        conv.populate_template(File.read(file, encoding: "UTF-8"), nil)
+      end
+
+      def bibdata_cleanup(xmldoc)
+        bibdata_anchor_cleanup(xmldoc)
+        bibdata_docidentifier_cleanup(xmldoc)
+        biblio_indirect_erefs(xmldoc, @internal_eref_namespaces&.uniq)
+      end
+
+      def bibdata_anchor_cleanup(xmldoc)
+        xmldoc.xpath("//bibdata//bibitem | //bibdata//note").each do |b|
+          b.delete("id")
+        end
+      end
+
+      def bibdata_docidentifier_cleanup(xmldoc)
+        ins = xmldoc.at("//bibdata/docidentifier")
+        xmldoc.xpath("//bibdata/docidentifier").each_with_index do |b, i|
+          next if i.zero?
+
+          ins.next = b.remove
+          ins = ins.next
+        end
+      end
+
+      def gather_indirect_erefs(xmldoc, prefix)
+        xmldoc.xpath("//eref[@type = '#{prefix}']")
+          .each_with_object({}) do |e, m|
+          e.delete("type")
+          m[e["bibitemid"]] = true
+        end.keys
+      end
+
+      def insert_indirect_biblio(xmldoc, refs, prefix)
+        ins = xmldoc.at("bibliography") or
+          xmldoc.root << "<bibliography/>" and ins = xmldoc.at("bibliography")
+        ins = ins.add_child("<references hidden='true' normative='false'/>").first
+        refs.each do |x|
+          ins << <<~BIB
+            <bibitem id="#{x}" type="internal">
+            <docidentifier type="repository">#{x.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
+            </bibitem>
+          BIB
+        end
+      end
+
+      def indirect_eref_to_xref(eref, ident)
+        loc = eref&.at("./localityStack[locality[@type = 'anchor']]")
+          &.remove&.text ||
+          eref&.at("./locality[@type = 'anchor']")&.remove&.text || ident
+        eref.name = "xref"
+        eref.delete("bibitemid")
+        eref.delete("citeas")
+        eref["target"] = loc
+        unless eref.document.at("//*[@id = '#{loc}']")
+          eref.children = %(** Missing target #{loc})
+          eref["target"] = ident
+        end
+      end
+
+      def resolve_local_indirect_erefs(xmldoc, refs, prefix)
+        refs.each_with_object([]) do |r, m|
+          id = r.sub(/^#{prefix}_/, "")
+          if n = xmldoc.at("//*[@id = '#{id}']") and
+              n.at("./ancestor-or-self::*[@type = '#{prefix}']")
+            xmldoc.xpath("//eref[@bibitemid = '#{r}']").each do |e|
+              indirect_eref_to_xref(e, id)
+            end
+          else m << r
+          end
+        end
+      end
+
+      def biblio_indirect_erefs(xmldoc, prefixes)
+        prefixes&.each do |prefix|
+          refs = gather_indirect_erefs(xmldoc, prefix)
+          refs = resolve_local_indirect_erefs(xmldoc, refs, prefix)
+          refs.empty? and next
+          insert_indirect_biblio(xmldoc, refs, prefix)
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_footnotes.rb

@@ -0,0 +1,109 @@
+require "date"
+require "htmlentities"
+require "json"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def footnote_content(fn)
+        c = fn.children.respond_to?(:to_xml) ? fn.children.to_xml : fn.children
+        c.gsub(/ id="[^"]+"/, "")
+      end
+
+      # include footnotes inside figure if they are the only content 
+      # of the paras following
+      def figure_footnote_cleanup(xmldoc)
+        nomatches = false
+        until nomatches
+          q = "//figure/following-sibling::*[1][self::p and *[1][self::fn]]"
+          nomatches = true
+          xmldoc.xpath(q).each do |s|
+            next if s.children.map { |c| c.text? && /[[:alpha:]]/.match(c.text) }.any?
+
+            s.previous_element << s.first_element_child.remove
+            s.remove
+            nomatches = false
+          end
+        end
+      end
+
+      def table_footnote_renumber1(fn, i, seen)
+        content = footnote_content(fn)
+        if seen[content] then outnum = seen[content]
+        else
+          i += 1
+          outnum = i
+          seen[content] = outnum
+        end
+        fn["reference"] = (outnum - 1 + "a".ord).chr
+        fn["table"] = true
+        [i, seen]
+      end
+
+      def table_footnote_renumber(xmldoc)
+        xmldoc.xpath("//table | //figure").each do |t|
+          seen = {}
+          i = 0
+          t.xpath(".//fn[not(ancestor::name)]").each do |fn|
+            i, seen = table_footnote_renumber1(fn, i, seen)
+          end
+        end
+      end
+
+      def other_footnote_renumber1(fn, i, seen)
+        unless fn["table"]
+          content = footnote_content(fn)
+          if seen[content] then outnum = seen[content]
+          else
+            i += 1
+            outnum = i
+            seen[content] = outnum
+          end
+          fn["reference"] = outnum.to_s
+        end
+        [i, seen]
+      end
+
+      def other_footnote_renumber(xmldoc)
+        seen = {}
+        i = 0
+        xmldoc.xpath("//fn").each do |fn|
+          i, seen = other_footnote_renumber1(fn, i, seen)
+        end
+      end
+
+      def title_footnote_move(xmldoc)
+        ins = xmldoc.at("//bibdata/language")
+        xmldoc.xpath("//bibdata/title//fn").each do |f|
+          f.name = "note"
+          f["type"] = "title-footnote"
+          f.delete("reference")
+          ins.previous = f.remove
+        end
+      end
+
+      def footnote_block_cleanup(xmldoc)
+        xmldoc.xpath("//footnoteblock").each do |f|
+          f.name = 'fn'
+          if id = xmldoc.at("//*[@id = '#{f.text}']")
+            f.children = id.remove.children
+          else
+            @log.add("Crossreferences", f,
+                     "Could not resolve footnoteblock:[#{f.text}]")
+            f.children = "[ERROR]"
+          end
+        end
+      end
+
+      def footnote_cleanup(xmldoc)
+        footnote_block_cleanup(xmldoc)
+        title_footnote_move(xmldoc)
+        table_footnote_renumber(xmldoc)
+        other_footnote_renumber(xmldoc)
+        xmldoc.xpath("//fn").each do |fn|
+          fn.delete("table")
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_image.rb

@@ -0,0 +1,70 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def svgmap_cleanup(xmldoc)
+        svgmap_moveattrs(xmldoc)
+        svgmap_populate(xmldoc)
+        Metanorma::Utils::svgmap_rewrite(xmldoc, @localdir)
+      end
+
+      def guid?(str)
+        /^_[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i
+          .match(str)
+      end
+
+      def svgmap_moveattrs(xmldoc)
+        xmldoc.xpath("//svgmap").each do |s|
+          f = s.at(".//figure") or next
+          (t = s.at("./name")) && !f.at("./name") and
+            f.children.first.previous = t.remove
+          if s["id"] && guid?(f["id"])
+            f["id"] = s["id"]
+            s.delete("id")
+          end
+          svgmap_moveattrs1(s, f)
+        end
+      end
+
+      def svgmap_moveattrs1(svgmap, figure)
+        %w(unnumbered number subsequence keep-with-next
+           keep-lines-together tag multilingual-rendering).each do |a|
+          next if figure[a] || !svgmap[a]
+
+          figure[a] = svgmap[a]
+          svgmap.delete(a)
+        end
+      end
+
+      def svgmap_populate(xmldoc)
+        xmldoc.xpath("//svgmap").each do |s|
+          s1 = s.dup
+          s.children.remove
+          f = s1.at(".//figure") and s << f
+          s1.xpath(".//li").each do |li|
+            t = li&.at(".//eref | .//link | .//xref") or next
+            href = t.xpath("./following-sibling::node()")
+            href.empty? or
+              s << %[<target href="#{svgmap_target(href)}">#{t.to_xml}</target>]
+          end
+        end
+      end
+
+      def svgmap_target(nodeset)
+        nodeset.each do |n|
+          next unless n.name == "link"
+
+          n.children = n["target"]
+        end
+        nodeset.text.sub(/^[,; ]/, "").strip
+      end
+
+      def img_cleanup(xmldoc)
+        return xmldoc unless @datauriimage
+
+        xmldoc.xpath("//image").each do |i|
+          i["src"] = Metanorma::Utils::datauri(i["src"], @localdir)
+        end
+      end
+    end
+  end
+end