metanorma-standoc 1.7.0 → 1.8.2

data/lib/asciidoctor/standoc/cleanup_boilerplate.rb

@@ -106,6 +106,7 @@ module Asciidoctor
       def bibdata_cleanup(xmldoc)
         bibdata_anchor_cleanup(xmldoc)
         bibdata_docidentifier_cleanup(xmldoc)
+                biblio_indirect_erefs(xmldoc, @internal_eref_namespaces&.uniq)
       end
 
       def bibdata_anchor_cleanup(xmldoc)
@@ -122,6 +123,61 @@ module Asciidoctor
           ins = ins.next
         end
       end
+
+       def gather_indirect_erefs(xmldoc, prefix)
+        xmldoc.xpath("//eref[@type = '#{prefix}']").each_with_object({}) do |e, m|
+          e.delete("type")
+          m[e["bibitemid"]] = true
+        end.keys
+      end
+
+      def insert_indirect_biblio(xmldoc, refs, prefix)
+        ins = xmldoc.at("bibliography") or
+          xmldoc.root << "<bibliography/>" and ins = xmldoc.at("bibliography")
+        ins = ins.add_child("<references hidden='true' normative='false'/>").first
+        refs.each do |x|
+          ins << <<~END
+            <bibitem id="#{x}" type="internal">
+            <docidentifier type="repository">#{x.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
+            </bibitem>
+          END
+        end
+      end
+
+      def indirect_eref_to_xref(e, id)
+        loc = e&.at("./localityStack[locality[@type = 'anchor']]")&.remove&.text ||
+          e&.at("./locality[@type = 'anchor']")&.remove&.text || id
+        e.name = "xref"
+        e.delete("bibitemid")
+        e.delete("citeas")
+        e["target"] = loc
+        unless e.document.at("//*[@id = '#{loc}']")
+          e.children = %(** Missing target #{loc})
+          e["target"] = id
+        end
+      end
+
+      def resolve_local_indirect_erefs(xmldoc, refs, prefix)
+        refs.each_with_object([]) do |r, m|
+          id = r.sub(/^#{prefix}_/, "")
+          if xmldoc.at("//*[@id = '#{id}'][@type = '#{prefix}']")
+            xmldoc.xpath("//eref[@bibitemid = '#{r}']").each do |e|
+              indirect_eref_to_xref(e, id)
+            end
+          else
+            m << r
+          end
+        end
+      end
+
+      def biblio_indirect_erefs(xmldoc, prefixes)
+        prefixes&.each do |prefix|
+          refs = gather_indirect_erefs(xmldoc, prefix)
+          refs = resolve_local_indirect_erefs(xmldoc, refs, prefix)
+          refs.empty? and next
+          insert_indirect_biblio(xmldoc, refs, prefix)
+        end
+      end
     end
   end
 end

data/lib/asciidoctor/standoc/cleanup_inline.rb

@@ -1,3 +1,5 @@
+require "metanorma-utils"
+
 module Asciidoctor
   module Standoc
     module Cleanup
@@ -82,6 +84,8 @@ module Asciidoctor
 
       def xref_cleanup(xmldoc)
         xmldoc.xpath("//xref").each do |x|
+          /:/.match(x["target"]) and xref_to_internal_eref(x)
+          next unless x.name == "xref"
           if refid? x["target"]
             x.name = "eref"
             xref_to_eref(x)
@@ -91,6 +95,18 @@ module Asciidoctor
         end
       end
 
+      def xref_to_internal_eref(x)
+        a = x["target"].split(":", 3)
+        unless  a.size < 2 || a[0].empty? || a[1].empty?
+          x["target"] = "#{a[0]}_#{a[1]}"
+          a.size > 2 and x.children = %{anchor="#{a[2..-1].join("")}",#{x&.children&.text}}
+          x["type"] = a[0]
+          @internal_eref_namespaces << a[0]
+          x.name = "eref"
+          xref_to_eref(x)
+        end
+      end
+
       def quotesource_cleanup(xmldoc)
         xmldoc.xpath("//quote/source | //terms/source").each do |x|
           xref_to_eref(x)
@@ -139,29 +155,11 @@ module Asciidoctor
         extract_localities(x.first_element_child)
       end
 
-      NAMECHAR = "\u0000-\u0022\u0024\u002c\u002f\u003a-\u0040\\u005b-\u005e"\
-        "\u0060\u007b-\u00b6\u00b8-\u00bf\u00d7\u00f7\u037e\u2000-\u200b"\
-        "\u200e-\u203e\u2041-\u206f\u2190-\u2bff\u2ff0-\u3000".freeze
-      #"\ud800-\uf8ff\ufdd0-\ufdef\ufffe-\uffff".freeze
-      NAMESTARTCHAR = "\\u002d\u002e\u0030-\u0039\u00b7\u0300-\u036f"\
-        "\u203f-\u2040".freeze
-
-      def to_ncname(s)
-        start = s[0]
-        ret1 = %r([#{NAMECHAR}#]).match(start) ? "_" :
-          (%r([#{NAMESTARTCHAR}#]).match(start) ? "_#{start}" : start)
-        ret2 = s[1..-1] || ""
-        ret = (ret1 || "") + ret2.gsub(%r([#{NAMECHAR}#]), "_")
-        ret
-      end
-
-      module_function :to_ncname
-
       def to_xreftarget(s)
-        return to_ncname(s) unless /^[^#]+#.+$/.match(s)
+        return Metanorma::Utils::to_ncname(s) unless /^[^#]+#.+$/.match(s)
           /^(?<pref>[^#]+)#(?<suff>.+)$/ =~ s
-        pref = pref.gsub(%r([#{NAMECHAR}]), "_")
-        suff = suff.gsub(%r([#{NAMECHAR}]), "_")
+        pref = pref.gsub(%r([#{Metanorma::Utils::NAMECHAR}]), "_")
+        suff = suff.gsub(%r([#{Metanorma::Utils::NAMECHAR}]), "_")
         "#{pref}##{suff}"
       end
 
@@ -175,12 +173,11 @@ module Asciidoctor
 
       def anchor_cleanup1(x)
         x.xpath(IDREF).each do |s|
-          if (ret = to_ncname(s.value)) != (orig = s.value)
+          if (ret = Metanorma::Utils::to_ncname(s.value)) != (orig = s.value)
             s.value = ret
             output = s.parent.dup
             output.children.remove
-            @log.add("Anchors", s.parent, "normalised identifier in #{output} "\
-                     "from #{orig}")
+            @log.add("Anchors", s.parent, "normalised identifier in #{output} from #{orig}")
           end
         end
       end
@@ -191,8 +188,7 @@ module Asciidoctor
             s.value = ret
             output = s.parent.dup
             output.children.remove
-            @log.add("Anchors", s.parent, "normalised identifier in #{output} "\
-                     "from #{orig}")
+            @log.add("Anchors", s.parent, "normalised identifier in #{output} from #{orig}")
           end
         end
       end

data/lib/asciidoctor/standoc/cleanup_maths.rb

@@ -0,0 +1,86 @@
+require "nokogiri"
+require "pathname"
+require "open-uri"
+require "html2doc"
+require_relative "./cleanup_block.rb"
+require_relative "./cleanup_footnotes.rb"
+require_relative "./cleanup_ref.rb"
+require_relative "./cleanup_ref_dl.rb"
+require_relative "./cleanup_boilerplate.rb"
+require_relative "./cleanup_section.rb"
+require_relative "./cleanup_terms.rb"
+require_relative "./cleanup_inline.rb"
+require_relative "./cleanup_amend.rb"
+require "relaton_iev"
+
+module Asciidoctor
+  module Standoc
+    module Cleanup
+      def asciimath2mathml(text)
+        text = text.gsub(%r{<stem type="AsciiMath">(.+?)</stem>}m) do |m|
+            "<amathstem>#{HTMLEntities.new.decode($1)}</amathstem>"
+          end
+          text = Html2Doc.asciimath_to_mathml(text, ["<amathstem>", "</amathstem>"])
+          x =  Nokogiri::XML(text)
+          x.xpath("//*[local-name() = 'math'][not(parent::stem)]").each do |y|
+            y.wrap("<stem type='MathML'></stem>")
+          end
+          x.to_xml
+      end
+
+      def xml_unescape_mathml(x)
+        return if x.children.any? { |y| y.element? }
+        math = x.text.gsub(/&lt;/, "<").gsub(/&gt;/, ">").gsub(/&quot;/, '"').gsub(/&apos;/, "'").gsub(/&amp;/, "&").
+          gsub(/<[^: \r\n\t\/]+:/, "<").gsub(/<\/[^ \r\n\t:]+:/, "</")
+        x.children = math
+      end
+
+      MATHML_NS = "http://www.w3.org/1998/Math/MathML".freeze
+
+      def mathml_preserve_space(m)
+        m.xpath(".//m:mtext", "m" => MATHML_NS).each do |x|
+          x.children = x.children.to_xml.gsub(/^\s/, "&#xA0;").gsub(/\s$/, "&#xA0;")
+        end
+      end
+
+      def mathml_namespace(stem)
+        stem.xpath("./math", ).each { |x| x.default_namespace = MATHML_NS }
+      end
+
+      def mathml_mi_italics
+        { uppergreek: true, upperroman: true,
+          lowergreek: true, lowerroman: true }
+      end
+
+      # presuppose multichar mi upright, singlechar mi MathML default italic
+      def mathml_italicise(x)
+        x.xpath(".//m:mi[not(ancestor::*[@mathvariant])]", "m" => MATHML_NS).each do |i|
+          char = HTMLEntities.new.decode(i.text)
+          i["mathvariant"] = "normal" if mi_italicise?(char)
+        end
+      end
+
+      def mi_italicise?(c)
+        return false if c.length > 1
+        if /\p{Greek}/.match(c)
+          /\p{Lower}/.match(c) && !mathml_mi_italics[:lowergreek] ||
+            /\p{Upper}/.match(c) && !mathml_mi_italics[:uppergreek]
+        elsif /\p{Latin}/.match(c)
+          /\p{Lower}/.match(c) && !mathml_mi_italics[:lowerroman] ||
+            /\p{Upper}/.match(c) && !mathml_mi_italics[:upperroman]
+        else
+          false
+        end
+      end
+
+      def mathml_cleanup(xmldoc)
+        xmldoc.xpath("//stem[@type = 'MathML']").each do |x|
+          xml_unescape_mathml(x)
+          mathml_namespace(x)
+          mathml_preserve_space(x)
+          mathml_italicise(x)
+        end
+      end
+    end
+  end
+end

data/lib/asciidoctor/standoc/cleanup_ref.rb

@@ -120,91 +120,6 @@ module Asciidoctor
         end
       end
 
-      def ref_dl_cleanup(xmldoc)
-        xmldoc.xpath("//clause[@bibitem = 'true']").each do |c|
-          bib = dl_bib_extract(c) or next
-          validate_ref_dl(bib, c)
-          bibitemxml = RelatonBib::BibliographicItem.new(RelatonBib::HashConverter::hash_to_bib(bib)).to_xml or next
-          bibitem = Nokogiri::XML(bibitemxml)
-          bibitem.root["id"] = c["id"] if c["id"] && !/^_/.match(c["id"])
-          c.replace(bibitem.root)
-        end
-      end
-
-      def validate_ref_dl(bib, c)
-        id = bib["id"]
-        id ||= c["id"] unless /^_/.match(c["id"]) # do not accept implicit id
-        unless id
-          @log.add("Anchors", c, "The following reference is missing an anchor:\n" + c.to_xml)
-          return
-        end
-        bib["title"] or @log.add("Bibliography", c, "Reference #{id} is missing a title")
-        bib["docid"] or @log.add("Bibliography", c, "Reference #{id} is missing a document identifier (docid)")
-      end
-
-      def extract_from_p(tag, bib, key)
-        return unless bib[tag]
-        "<#{key}>#{bib[tag].at('p').children}</#{key}>"
-      end
-
-      # if the content is a single paragraph, replace it with its children
-      # single links replaced with uri
-      def p_unwrap(p)
-        elems = p.elements
-        if elems.size == 1 && elems[0].name == "p"
-          link_unwrap(elems[0]).children.to_xml.strip
-        else
-          p.to_xml.strip
-        end
-      end
-
-      def link_unwrap(p)
-        elems = p.elements
-        if elems.size == 1 && elems[0].name == "link"
-          p.at("./link").replace(elems[0]["target"].strip)
-        end
-        p
-      end
-
-      def dd_bib_extract(dtd)
-        return nil if dtd.children.empty?
-        dtd.at("./dl") and return dl_bib_extract(dtd)
-        elems = dtd.remove.elements
-        return p_unwrap(dtd) unless elems.size == 1 && %w(ol ul).include?(elems[0].name)
-        ret = []
-        elems[0].xpath("./li").each do |li|
-          ret << p_unwrap(li)
-        end
-        ret
-      end
-
-      def add_to_hash(bib, key, val)
-        Utils::set_nested_value(bib, key.split(/\./), val)
-      end
-
-      # definition list, with at most one level of unordered lists
-      def dl_bib_extract(c, nested = false)
-        dl = c.at("./dl") or return
-        bib = {}
-        key = ""
-        dl.xpath("./dt | ./dd").each do |dtd|
-          dtd.name == "dt" and key = dtd.text.sub(/:+$/, "") or add_to_hash(bib, key, dd_bib_extract(dtd))
-        end
-        c.xpath("./clause").each do |c1|
-          key = c1&.at("./title")&.text&.downcase&.strip
-          next unless %w(contributor relation series).include? key
-          add_to_hash(bib, key, dl_bib_extract(c1, true))
-        end
-        if !nested and c.at("./title")
-          title = c.at("./title").remove.children.to_xml
-          bib["title"] = [bib["title"]] if bib["title"].is_a? Hash
-          bib["title"] = [bib["title"]] if bib["title"].is_a? String
-          bib["title"] = [] unless bib["title"]
-          bib["title"] << title if !title.empty?
-        end
-        bib
-      end
-
       def fetch_termbase(termbase, id)
         ""
       end

data/lib/asciidoctor/standoc/cleanup_ref_dl.rb

@@ -0,0 +1,94 @@
+require "set"
+require "relaton_bib"
+
+module Asciidoctor
+  module Standoc
+    module Cleanup
+      def ref_dl_cleanup(xmldoc)
+        xmldoc.xpath("//clause[@bibitem = 'true']").each do |c|
+          bib = dl_bib_extract(c) or next
+          validate_ref_dl(bib, c)
+          bibitemxml = RelatonBib::BibliographicItem.new(RelatonBib::HashConverter::hash_to_bib(bib)).to_xml or next
+          bibitem = Nokogiri::XML(bibitemxml)
+          bibitem.root["id"] = c["id"] if c["id"] && !/^_/.match(c["id"])
+          c.replace(bibitem.root)
+        end
+      end
+
+      def validate_ref_dl(bib, c)
+        id = bib["id"]
+        id ||= c["id"] unless /^_/.match(c["id"]) # do not accept implicit id
+        unless id
+          @log.add("Anchors", c, "The following reference is missing an anchor:\n" + c.to_xml)
+          return
+        end
+        @refids << id
+        bib["title"] or @log.add("Bibliography", c, "Reference #{id} is missing a title")
+        bib["docid"] or @log.add("Bibliography", c, "Reference #{id} is missing a document identifier (docid)")
+      end
+
+      def extract_from_p(tag, bib, key)
+        return unless bib[tag]
+        "<#{key}>#{bib[tag].at('p').children}</#{key}>"
+      end
+
+      # if the content is a single paragraph, replace it with its children
+      # single links replaced with uri
+      def p_unwrap(p)
+        elems = p.elements
+        if elems.size == 1 && elems[0].name == "p"
+          link_unwrap(elems[0]).children.to_xml.strip
+        else
+          p.to_xml.strip
+        end
+      end
+
+      def link_unwrap(p)
+        elems = p.elements
+        if elems.size == 1 && elems[0].name == "link"
+          p.at("./link").replace(elems[0]["target"].strip)
+        end
+        p
+      end
+
+      def dd_bib_extract(dtd)
+        return nil if dtd.children.empty?
+        dtd.at("./dl") and return dl_bib_extract(dtd)
+        elems = dtd.remove.elements
+        return p_unwrap(dtd) unless elems.size == 1 && %w(ol ul).include?(elems[0].name)
+        ret = []
+        elems[0].xpath("./li").each do |li|
+          ret << p_unwrap(li)
+        end
+        ret
+      end
+
+      def add_to_hash(bib, key, val)
+        Metanorma::Utils::set_nested_value(bib, key.split(/\./), val)
+      end
+
+      # definition list, with at most one level of unordered lists
+      def dl_bib_extract(c, nested = false)
+        dl = c.at("./dl") or return
+        bib = {}
+        key = ""
+        dl.xpath("./dt | ./dd").each do |dtd|
+          dtd.name == "dt" and key = dtd.text.sub(/:+$/, "") or add_to_hash(bib, key, dd_bib_extract(dtd))
+        end
+        c.xpath("./clause").each do |c1|
+          key = c1&.at("./title")&.text&.downcase&.strip
+          next unless %w(contributor relation series).include? key
+          add_to_hash(bib, key, dl_bib_extract(c1, true))
+        end
+        if !nested and c.at("./title")
+          title = c.at("./title").remove.children.to_xml
+          bib["title"] = [bib["title"]] if bib["title"].is_a? Hash
+          bib["title"] = [bib["title"]] if bib["title"].is_a? String
+          bib["title"] = [] unless bib["title"]
+          bib["title"] << title if !title.empty?
+        end
+        bib
+      end
+    end
+  end
+end

data/lib/asciidoctor/standoc/converter.rb

@@ -1,6 +1,4 @@
 require "asciidoctor"
-require "fontist"
-require "fontist/manifest/install"
 require "metanorma/util"
 require "metanorma/standoc/version"
 require "asciidoctor/standoc/base"
@@ -16,7 +14,6 @@ require "asciidoctor/standoc/utils"
 require "asciidoctor/standoc/cleanup"
 require "asciidoctor/standoc/reqt"
 require_relative "./macros.rb"
-require_relative "./log.rb"
 
 module Asciidoctor
   module Standoc
@@ -73,8 +70,6 @@ module Asciidoctor
         basebackend "html"
         outfilesuffix ".xml"
         @libdir = File.dirname(self.class::_file || __FILE__)
-
-        install_fonts(opts)
       end
 
       class << self
@@ -90,62 +85,6 @@ module Asciidoctor
         File.join(@libdir, "../../isodoc/html", file)
       end
 
-      def flavor_name
-        self.class.name.split("::")&.[](-2).downcase.to_sym
-      end
-
-      def fonts_manifest
-        flavor = flavor_name
-        registry = Metanorma::Registry.instance
-        processor = registry.find_processor(flavor)
-
-        if processor.nil?
-          Metanorma::Util.log("[fontist] #{flavor} processor not found. " \
-            "Please go to github.com/metanorma/metanorma/issues to report " \
-            "this issue.", :warn)
-          return nil
-        elsif !defined? processor.fonts_manifest
-          Metanorma::Util.log("[fontist] #{flavor} processor don't require " \
-            "specific fonts", :debug)
-          return nil
-        end
-
-        processor.fonts_manifest
-      end
-
-      def install_fonts(options={})
-        if options[:no_install_fonts]
-          Metanorma::Util.log("[fontist] Skip font installation because" \
-            " --no-install-fonts argument passed", :debug)
-          return
-        end
-
-        manifest = fonts_manifest
-        return if manifest.nil?
-
-        begin
-          Fontist::Manifest::Install.from_hash(
-            fonts_manifest,
-            confirmation: options[:agree_to_terms] ? "yes" : "no"
-          )
-        rescue Fontist::Errors::LicensingError
-          if !options[:agree_to_terms]
-            Metanorma::Util.log("[fontist] --agree-to-terms option missing." \
-              " You must accept font licenses to install fonts.", :debug)
-          elsif options[:continue_without_fonts]
-            Metanorma::Util.log("[fontist] Processing will continue without" \
-              " fonts installed", :debug)
-          else
-            Metanorma::Util.log("[fontist] Aborting without proper fonts" \
-              " installed", :fatal)
-          end
-        rescue Fontist::Errors::NonSupportedFontError
-          Metanorma::Util.log("[fontist] '#{font}' font is not supported. " \
-            "Please go to github.com/metanorma/metanorma-#{flavor_name}/issues" \
-            " to report this issue.", :info)
-        end
-      end
-
       alias_method :embedded, :content
       alias_method :verse, :quote
       alias_method :audio, :skip