metanorma-standoc 2.4.9 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70483d0ae3e7369ea535a02fed125b9b073c2792cc9a589c22cd94ea98948d1e
-  data.tar.gz: 9fb3bcbfb78079b25cbdfb6f49fdefc4df8928a8d0782878396be6a6f286374c
+  metadata.gz: 1d6a56b88b9f58482764ff6aede297e389577151d0d112a215b2215d5cfb41cf
+  data.tar.gz: cb0a1a1d73930e89169a27185bcce3d19130f1150a22fd7345da61a73dbd759f
 SHA512:
-  metadata.gz: f2b70e60bd3f83f8218c16bf24d9ff8a164b4a0467cf6f47e4090da745034f048a5dd91d813e0242f365eccf52ab71e2af1b477b320d1e310412c8065b02742b
-  data.tar.gz: 2008df0fab61498838336fa8ddbf94efa3845583ed1740223a877b8ad27ccfc8d257553067f3a84226ff41ef0cc2bda91b2fd90e88a2d8be2aaf3032688bebef
+  metadata.gz: 158547c563e269b769f39123cbd03f749a4891f7e4c772e9462ea40a9221ee4b58839991b2580fc40d9a24007387866d2741867ee6a8f5f39770fa3d38a57138
+  data.tar.gz: ba7b26ed9ebfae0f92dedc50750aad4e31c53b5e5b16defe3e2a2ea6583049fab738f07449846ac62aa74af06944d725389fed9d54357bf9ff9c33b3196cac2f

data/lib/metanorma/standoc/blocks.rb

@@ -76,7 +76,7 @@ module Metanorma
       def stem(node)
         noko do |xml|
           xml.formula **formula_attrs(node) do |s|
-            stem_parse(node.lines.join("\n"), s, node.style.to_sym)
+            stem_parse(node.lines.join("\n"), s, node.style.to_sym, node.block?)
           end
         end
       end

data/lib/metanorma/standoc/cleanup.rb

@@ -1,24 +1,24 @@
 require "nokogiri"
 require "pathname"
 require "html2doc"
-require_relative "./cleanup_block"
-require_relative "./cleanup_table"
-require_relative "./cleanup_footnotes"
-require_relative "./cleanup_ref"
-require_relative "./cleanup_asciibib"
-require_relative "./cleanup_boilerplate"
-require_relative "./cleanup_bibdata"
-require_relative "./cleanup_section"
-require_relative "./cleanup_terms"
-require_relative "./cleanup_symbols"
-require_relative "./cleanup_xref"
-require_relative "./cleanup_inline"
-require_relative "./cleanup_amend"
-require_relative "./cleanup_maths"
-require_relative "./cleanup_image"
-require_relative "./cleanup_reqt"
-require_relative "./cleanup_text"
-require_relative "./cleanup_toc"
+require_relative "cleanup_block"
+require_relative "cleanup_table"
+require_relative "cleanup_footnotes"
+require_relative "cleanup_ref"
+require_relative "cleanup_asciibib"
+require_relative "cleanup_boilerplate"
+require_relative "cleanup_bibdata"
+require_relative "cleanup_section"
+require_relative "cleanup_terms"
+require_relative "cleanup_symbols"
+require_relative "cleanup_xref"
+require_relative "cleanup_inline"
+require_relative "cleanup_amend"
+require_relative "cleanup_maths"
+require_relative "cleanup_image"
+require_relative "cleanup_reqt"
+require_relative "cleanup_text"
+require_relative "cleanup_toc"
 require "relaton_iev"
 
 module Metanorma
@@ -51,6 +51,7 @@ module Metanorma
         normref_cleanup(xmldoc)
         biblio_cleanup(xmldoc)
         reference_names(xmldoc)
+        asciimath_cleanup(xmldoc) # feeds: mathml_cleanup, termdef_cleanup, symbols_cleanup
         symbols_cleanup(xmldoc) # feeds: termdef_cleanup
         xref_cleanup(xmldoc) # feeds: concept_cleanup, origin_cleanup
         concept_cleanup(xmldoc) # feeds: related_cleanup, termdef_cleanup
@@ -79,6 +80,7 @@ module Metanorma
         empty_element_cleanup(xmldoc)
         img_cleanup(xmldoc)
         anchor_cleanup(xmldoc)
+        link_cleanup(xmldoc)
         xmldoc
       end
 
@@ -107,7 +109,7 @@ module Metanorma
       end
 
       def element_name_cleanup(xmldoc)
-        xmldoc.traverse { |n| n.name = n.name.gsub(/_/, "-") }
+        xmldoc.traverse { |n| n.name = n.name.gsub("_", "-") }
       end
 
       # allows us to deal with doc relation localities,

data/lib/metanorma/standoc/cleanup_inline.rb

@@ -78,7 +78,7 @@ module Metanorma
       def concept_cleanup1(elem)
         elem.children.remove if elem&.children&.text&.strip&.empty?
         key_extract_locality(elem)
-        if /:/.match?(elem["key"]) then concept_termbase_cleanup(elem)
+        if elem["key"].include?(":") then concept_termbase_cleanup(elem)
         elsif refid? elem["key"] then concept_eref_cleanup(elem)
         else concept_xref_cleanup(elem)
         end
@@ -95,15 +95,14 @@ module Metanorma
       end
 
       def key_extract_locality(elem)
-        return unless /,/.match?(elem["key"])
-
+        elem["key"].include?(",") or return
         elem.add_child("<locality>#{elem['key'].sub(/^[^,]+,/, '')}</locality>")
         elem["key"] = elem["key"].sub(/,.*$/, "")
       end
 
       def concept_termbase_cleanup(elem)
         t = elem&.at("./xrefrender")&.remove&.children
-        termbase, key = elem["key"].split(/:/, 2)
+        termbase, key = elem["key"].split(":", 2)
         elem.add_child(%(<termref base="#{termbase}" target="#{key}">) +
                        "#{t&.to_xml}</termref>")
       end
@@ -207,6 +206,21 @@ module Metanorma
         end
       end
 
+      def link_cleanup(xmldoc)
+        xmldoc.xpath("//link[@target]").each do |l|
+          l["target"] = URI.parse(l["target"]).to_s
+        rescue StandardError
+          err = "Malformed URI: #{l['target']}"
+          @log.add("Anchors", l, err)
+          @fatalerror << err
+          warn err
+        end
+      end
+
+      def uri_component_encode(comp)
+        CGI.escape(comp).gsub("+", "%20")
+      end
+
       private
 
       # skip ZWNJ inserted to prevent regexes operating in asciidoctor

data/lib/metanorma/standoc/cleanup_maths.rb

@@ -3,17 +3,28 @@ require "asciimath2unitsml"
 module Metanorma
   module Standoc
     module Cleanup
-      def asciimath2mathml(text)
-        text = text.gsub(%r{<stem type="AsciiMath">(.+?)</stem>}m) do
-          "<amathstem>#{@c.decode($1)}</amathstem>"
+      def asciimath_cleanup(xml)
+        !@keepasciimath and asciimath2mathml(xml)
+      end
+
+      def asciimath2mathml(xml)
+        xpath = xml.xpath("//stem[@type = 'AsciiMath']")
+        xpath.each_with_index do |x, i|
+          progress_conv(i, 500, xpath.size, 1000, "AsciiMath")
+          asciimath2mathml_indiv(x)
         end
-        text = Html2Doc.new({})
-          .asciimath_to_mathml(text, ["<amathstem>", "</amathstem>"],
-                               retain_asciimath: true)
-        asciimath2mathml_wrap(text)
+        asciimath2mathml_wrap(xml)
+      end
+
+      def asciimath2mathml_indiv(elem)
+        elem["type"] = "MathML"
+        expr = @c.decode(elem.text)
+        ret = Plurimath::Math.parse(expr, "asciimath")
+          .to_mathml(display_style: elem["block"])
+        ret += "<asciimath>#{@c.encode(@c.decode(expr), :basic)}</asciimath>"
+        elem.children = ret
       rescue StandardError => e
-        asciimath2mathml_err(text, e)
-        text
+        asciimath2mathml_err(elem.to_xml, e)
       end
 
       def asciimath2mathml_err(text, expr)
@@ -23,25 +34,27 @@ module Metanorma
         warn err
       end
 
-      def asciimath2mathml_wrap(text)
-        x = Nokogiri::XML(text)
-        x.xpath("//*[local-name() = 'math'][@display]").each do |y|
+      def asciimath2mathml_wrap(xml)
+        xml.xpath("//*[local-name() = 'math'][@display]").each do |y|
           y.delete("display")
         end
-        x.xpath("//*[local-name() = 'math'][not(parent::stem)]").each do |y|
-          y.wrap("<stem type='MathML'></stem>")
-        end
-        x.xpath("//stem").each do |y|
-          y.next_element&.name == "asciimath" and y << y.next_element
-        end
-        to_xml(x)
+        # x.xpath("//stem").each do |y|
+        # y.next_element&.name == "asciimath" and y << y.next_element
+        # end
+        xml
+      end
+
+      def progress_conv(idx, step, total, threshold, msg)
+        return unless (idx % step).zero? && total > threshold && idx.positive?
+
+        warn "#{msg} #{idx} of #{total}"
       end
 
       def xml_unescape_mathml(xml)
         return if xml.children.any?(&:element?)
 
-        math = xml.text.gsub(/&lt;/, "<").gsub(/&gt;/, ">")
-          .gsub(/&quot;/, '"').gsub(/&apos;/, "'").gsub(/&amp;/, "&")
+        math = xml.text.gsub("&lt;", "<").gsub("&gt;", ">")
+          .gsub("&quot;", '"').gsub("&apos;", "'").gsub("&amp;", "&")
           .gsub(/<[^: \r\n\t\/]+:/, "<").gsub(/<\/[^ \r\n\t:]+:/, "</")
         xml.children = math
       end

data/lib/metanorma/standoc/cleanup_symbols.rb

@@ -1,3 +1,5 @@
+require "metanorma/standoc/utils"
+
 module Metanorma
   module Standoc
     module Cleanup
@@ -8,31 +10,12 @@ module Metanorma
       def symbol_key(sym)
         @c.decode(asciimath_key(sym).text)
           .gsub(/[\[\]{}<>()]/, "").gsub(/\s/m, "")
-          .gsub(/[[:punct:]]|[_^]/, ":\\0").gsub(/`/, "")
+          .gsub(/[[:punct:]]|[_^]/, ":\\0").gsub("`", "")
           .gsub(/[0-9]+/, "þ\\0")
           .tr("AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz",
               "ABCFEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
       end
 
-      def asciimath_key(sym)
-        key = sym.dup
-        key.traverse do |n|
-          if n.name == "stem" && a = n.at(".//asciimath")
-            n.children = @c.encode(
-              @c.decode(grkletters(a.text)), :basic
-            )
-          end
-        end
-        key.xpath(".//asciimath").each(&:remove)
-        Nokogiri::XML(key.to_xml)
-      end
-
-      def grkletters(text)
-        text.gsub(/\b(alpha|beta|gamma|delta|epsilon|zeta|eta|theta|iota|kappa|
-                      lambda|mu|nu|xi|omicron|pi|rho|sigma|tau|upsilon|phi|chi|
-                      psi|omega)\b/xi, "&\\1;")
-      end
-
       def extract_symbols_list(dlist)
         dl_out = []
         dlist.xpath("./dt | ./dd").each do |dtd|

data/lib/metanorma/standoc/cleanup_text.rb

@@ -3,7 +3,6 @@ module Metanorma
     module Cleanup
       def textcleanup(result)
         text = result.flatten.map { |l| l.sub(/\s*\Z/, "") } * "\n"
-        !@keepasciimath and text = asciimath2mathml(text)
         text = text.gsub(/\s+<fn /, "<fn ")
         %w(passthrough passthrough-inline).each do |v|
           text.gsub!(%r{<#{v}\s+formats="metanorma">([^<]*)

data/lib/metanorma/standoc/inline.rb

@@ -138,9 +138,10 @@ module Metanorma
         noko { |xml| xml.hr }.join
       end
 
-      def latex_parse1(text)
+      def latex_parse1(text, block)
         lxm_input = Unicode2LaTeX.unicode2latex(@c.decode(text))
-        results = Plurimath::Math.parse(lxm_input, "latex").to_mathml
+        results = Plurimath::Math.parse(lxm_input, "latex")
+          .to_mathml(display_style: block)
         if results.nil?
           @log.add("Math", nil,
                    "latexmlmath failed to process equation:\n#{lxm_input}")
@@ -149,22 +150,23 @@ module Metanorma
         results.sub(%r{<math ([^>]+ )?display="block"}, "<math \\1")
       end
 
-      def stem_parse(text, xml, style)
+      def stem_parse(text, xml, style, block)
         if /&lt;([^:>&]+:)?math(\s+[^>&]+)?&gt; |
           <([^:>&]+:)?math(\s+[^>&]+)?>/x.match? text
           math = xml_encode(text)
-          xml.stem type: "MathML" do |s|
+          xml.stem type: "MathML", block: block do |s|
             s << math
           end
-        elsif style == :latexmath then latex_parse(text, xml)
+        elsif style == :latexmath then latex_parse(text, xml, block)
         else
-          xml.stem text&.gsub(/&amp;#/, "&#"), type: "AsciiMath"
+          xml.stem text&.gsub("&amp;#", "&#"), type: "AsciiMath", block: block
         end
       end
 
-      def latex_parse(text, xml)
-        latex = latex_parse1(text) or return xml.stem type: "MathML"
-        xml.stem type: "MathML" do |s|
+      def latex_parse(text, xml, block)
+        latex = latex_parse1(text, block) or
+          return xml.stem type: "MathML", block: block
+        xml.stem type: "MathML", block: block do |s|
           math = Nokogiri::XML.fragment(latex.sub(/<\?[^>]+>/, ""))
             .elements[0]
           math.delete("alttext")
@@ -187,8 +189,8 @@ module Metanorma
           when :single then xml << "'#{node.text}'"
           when :superscript then xml.sup { |s| s << node.text }
           when :subscript then xml.sub { |s| s << node.text }
-          when :asciimath then stem_parse(node.text, xml, :asciimath)
-          when :latexmath then stem_parse(node.text, xml, :latexmath)
+          when :asciimath then stem_parse(node.text, xml, :asciimath, false)
+          when :latexmath then stem_parse(node.text, xml, :latexmath, false)
           when :mark then highlight_parse(node.text, xml)
           else
             case node.role

data/lib/metanorma/standoc/isodoc.rng

@@ -17,7 +17,7 @@
   these elements; we just want one namespace for any child grammars
   of this.
 -->
-<!-- VERSION v1.2.2 -->
+<!-- VERSION v1.2.3 -->
 <grammar xmlns:a="http://relaxng.org/ns/compatibility/annotations/1.0" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
   <include href="reqt.rng"/>
   <include href="basicdoc.rng">
@@ -192,9 +192,11 @@
         </attribute>
       </optional>
       <attribute name="citeas"/>
-      <attribute name="type">
-        <ref name="ReferenceFormat"/>
-      </attribute>
+      <optional>
+        <attribute name="type">
+          <ref name="ReferenceFormat"/>
+        </attribute>
+      </optional>
       <optional>
         <attribute name="alt"/>
       </optional>
@@ -836,6 +838,26 @@
         <ref name="paragraph"/>
       </element>
     </define>
+    <define name="stem">
+      <element name="stem">
+        <attribute name="type">
+          <choice>
+            <value>MathML</value>
+            <value>AsciiMath</value>
+            <value>LatexMath</value>
+          </choice>
+        </attribute>
+        <attribute name="block">
+          <data type="boolean"/>
+        </attribute>
+        <oneOrMore>
+          <choice>
+            <text/>
+            <ref name="AnyElement"/>
+          </choice>
+        </oneOrMore>
+      </element>
+    </define>
     <define name="em">
       <element name="em">
         <zeroOrMore>

data/lib/metanorma/standoc/term_lookup_cleanup.rb

@@ -49,7 +49,8 @@ module Metanorma
       def concept_cleanup
         xmldoc.xpath("//concept").each do |n|
           refterm = n.at("./refterm") or next
-          p = @termlookup[:secondary2primary][@c.encode(refterm.text)] and
+          lookup = normalize_ref_id_text(refterm.text.strip)
+          p = @termlookup[:secondary2primary][lookup] and
             refterm.children = @c.encode(p)
         end
       end
@@ -61,7 +62,7 @@ module Metanorma
       def related_cleanup
         xmldoc.xpath("//related").each do |n|
           refterm = n.at("./refterm") or next
-          lookup = @c.encode(refterm.text)
+          lookup = normalize_ref_id_text(refterm.text.strip)
           p = @termlookup[:secondary2primary][lookup] and
             refterm.children = @c.encode(p)
           p || @termlookup[:term][lookup] and
@@ -133,7 +134,7 @@ module Metanorma
       end
 
       def remove_missing_ref_msg1(_node, target, ret)
-        target2 = "_#{target.downcase.gsub(/-/, '_')}"
+        target2 = "_#{target.downcase.gsub('-', '_')}"
         if @terms_tags[target] || @terms_tags[target2]
           ret.strip!
           ret += ". Did you mean to point to a subterm?"
@@ -185,17 +186,20 @@ module Metanorma
 
       def pref_secondary2primary_preferred(term, res, primary)
         term.xpath("./preferred//name").each_with_index do |p, i|
-          i.positive? and res[domain_prefix(term, p.text)] = primary
-          @unique_designs[p.text] && term.at(".//domain") and
-            res[p.text] = primary
+          t = p.text.strip
+          i.positive? and
+            res[normalize_ref_id_text(domain_prefix(term, t))] = primary
+          @unique_designs[t] && term.at(".//domain") and
+            res[normalize_ref_id_text(t)] = primary
         end
       end
 
       def pref_secondary2primary_admitted(term, res, primary)
         term.xpath("./admitted//name").each do |p|
-          res[domain_prefix(term, p.text)] = primary
-          @unique_designs[p.text] && term.at(".//domain") and
-            res[p.text] = primary
+          t = p.text.strip
+          res[normalize_ref_id_text(domain_prefix(term, t))] = primary
+          @unique_designs[t] && term.at(".//domain") and
+            res[normalize_ref_id_text(t)] = primary
         end
       end
 
@@ -234,9 +238,13 @@ module Metanorma
       def normalize_ref_id1(term, node = nil)
         t = term.dup
         t.xpath(".//index").map(&:remove)
-        ret = t.text.strip
+        ret = asciimath_key(t).text.strip
         node and ret = domain_prefix(node, ret)
-        Metanorma::Utils::to_ncname(ret.gsub(/[[:space:]]+/, "-"))
+        normalize_ref_id_text(ret)
+      end
+
+      def normalize_ref_id_text(text)
+        Metanorma::Utils::to_ncname(text.gsub(/[[:space:]]+/, "-"))
       end
 
       def unique_text_id(text, prefix)
@@ -248,6 +256,8 @@ module Metanorma
           end
         end
       end
+
+      include ::Metanorma::Standoc::Utils
     end
   end
 end

data/lib/metanorma/standoc/utils.rb

@@ -70,10 +70,10 @@ module Metanorma
 
       def xml_encode(text)
         @c.encode(text, :basic, :hexadecimal)
-          .gsub(/&amp;gt;/, ">").gsub(/&amp;lt;/, "<").gsub(/&amp;amp;/, "&")
-          .gsub(/&gt;/, ">").gsub(/&lt;/, "<").gsub(/&amp;/, "&")
-          .gsub(/&quot;/, '"').gsub(/&#xa;/, "\n").gsub(/&amp;#/, "&#")
-          .gsub(/&apos;/, "'")
+          .gsub("&amp;gt;", ">").gsub("&amp;lt;", "<").gsub("&amp;amp;", "&")
+          .gsub("&gt;", ">").gsub("&lt;", "<").gsub("&amp;", "&")
+          .gsub("&quot;", '"').gsub("&#xa;", "\n").gsub("&amp;#", "&#")
+          .gsub("&apos;", "'")
       end
 
       # wrapped in <sections>
@@ -85,6 +85,25 @@ module Metanorma
         Nokogiri::XML(c).at("//xmlns:sections")
       end
 
+      def asciimath_key(sym)
+        key = sym.dup
+        key.traverse do |n|
+          if n.name == "stem" && a = n.at(".//asciimath")
+            n.children = @c.encode(
+              @c.decode(grkletters(a.text)), :basic
+            )
+          end
+        end
+        key.xpath(".//asciimath").each(&:remove)
+        Nokogiri::XML(key.to_xml)
+      end
+
+      def grkletters(text)
+        text.gsub(/\b(alpha|beta|gamma|delta|epsilon|zeta|eta|theta|iota|kappa|
+                      lambda|mu|nu|xi|omicron|pi|rho|sigma|tau|upsilon|phi|chi|
+                      psi|omega)\b/xi, "&\\1;")
+      end
+
       module_function :adoc2xml
 
       class EmptyAttr

data/lib/metanorma/standoc/validate.rb

@@ -1,7 +1,7 @@
 require "metanorma/standoc/utils"
-require_relative "./validate_section"
-require_relative "./validate_table"
-require_relative "./validate_term"
+require_relative "validate_section"
+require_relative "validate_table"
+require_relative "validate_term"
 require "nokogiri"
 require "jing"
 require "iev"

data/lib/metanorma/standoc/version.rb

@@ -19,6 +19,6 @@ module Metanorma
   end
 
   module Standoc
-    VERSION = "2.4.9".freeze
+    VERSION = "2.5.0".freeze
   end
 end

data/metanorma-standoc.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "asciidoctor", "~> 2.0.0"
   spec.add_dependency "iev", "~> 0.3.0"
-  spec.add_dependency "isodoc", "~> 2.5.4"
+  spec.add_dependency "isodoc", "~> 2.6.0"
   spec.add_dependency "metanorma", ">= 1.5.0"
   spec.add_dependency "metanorma-plugin-datastruct", "~> 0.2.0"
   spec.add_dependency "metanorma-plugin-glossarist", "~> 0.1.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-standoc
 version: !ruby/object:Gem::Version
-  version: 2.4.9
+  version: 2.5.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-24 00:00:00.000000000 Z
+date: 2023-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.5.4
+        version: 2.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.5.4
+        version: 2.6.0
 - !ruby/object:Gem::Dependency
   name: metanorma
   requirement: !ruby/object:Gem::Requirement