metanorma-standoc 2.4.9 → 2.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70483d0ae3e7369ea535a02fed125b9b073c2792cc9a589c22cd94ea98948d1e
-  data.tar.gz: 9fb3bcbfb78079b25cbdfb6f49fdefc4df8928a8d0782878396be6a6f286374c
+  metadata.gz: 5ccb10f13350a471175edadd3a93a922226bf6f383faa3b4bcceff74735ff5e8
+  data.tar.gz: c7e6f17307599cc0ae13f88868aba8c8168c1e9a638b57a8a8c325e33cf2f0be
 SHA512:
-  metadata.gz: f2b70e60bd3f83f8218c16bf24d9ff8a164b4a0467cf6f47e4090da745034f048a5dd91d813e0242f365eccf52ab71e2af1b477b320d1e310412c8065b02742b
-  data.tar.gz: 2008df0fab61498838336fa8ddbf94efa3845583ed1740223a877b8ad27ccfc8d257553067f3a84226ff41ef0cc2bda91b2fd90e88a2d8be2aaf3032688bebef
+  metadata.gz: ae8763f5ae8953bf858720cc6ab658281299794fb24a172957d144564d1ac55432a0f08611b9ab243f9f87ebf568edd508f199442ad9c4e675775d4f40cae4aa
+  data.tar.gz: 4e4fcee2168ab3218355a73075c50d3f42ba6d1068839dbe2260e60a8c65a25f113d55576db49e5a436ef6757a90915cc60b59150dfa30747469d898ac6fcf6d

data/lib/metanorma/standoc/blocks.rb

@@ -76,7 +76,7 @@ module Metanorma
       def stem(node)
         noko do |xml|
           xml.formula **formula_attrs(node) do |s|
-            stem_parse(node.lines.join("\n"), s, node.style.to_sym)
+            stem_parse(node.lines.join("\n"), s, node.style.to_sym, node.block?)
           end
         end
       end

data/lib/metanorma/standoc/cleanup.rb

@@ -1,24 +1,24 @@
 require "nokogiri"
 require "pathname"
 require "html2doc"
-require_relative "./cleanup_block"
-require_relative "./cleanup_table"
-require_relative "./cleanup_footnotes"
-require_relative "./cleanup_ref"
-require_relative "./cleanup_asciibib"
-require_relative "./cleanup_boilerplate"
-require_relative "./cleanup_bibdata"
-require_relative "./cleanup_section"
-require_relative "./cleanup_terms"
-require_relative "./cleanup_symbols"
-require_relative "./cleanup_xref"
-require_relative "./cleanup_inline"
-require_relative "./cleanup_amend"
-require_relative "./cleanup_maths"
-require_relative "./cleanup_image"
-require_relative "./cleanup_reqt"
-require_relative "./cleanup_text"
-require_relative "./cleanup_toc"
+require_relative "cleanup_block"
+require_relative "cleanup_table"
+require_relative "cleanup_footnotes"
+require_relative "cleanup_ref"
+require_relative "cleanup_asciibib"
+require_relative "cleanup_boilerplate"
+require_relative "cleanup_bibdata"
+require_relative "cleanup_section"
+require_relative "cleanup_terms"
+require_relative "cleanup_symbols"
+require_relative "cleanup_xref"
+require_relative "cleanup_inline"
+require_relative "cleanup_amend"
+require_relative "cleanup_maths"
+require_relative "cleanup_image"
+require_relative "cleanup_reqt"
+require_relative "cleanup_text"
+require_relative "cleanup_toc"
 require "relaton_iev"
 
 module Metanorma
@@ -51,6 +51,7 @@ module Metanorma
         normref_cleanup(xmldoc)
         biblio_cleanup(xmldoc)
         reference_names(xmldoc)
+        asciimath_cleanup(xmldoc) # feeds: mathml_cleanup, termdef_cleanup, symbols_cleanup
         symbols_cleanup(xmldoc) # feeds: termdef_cleanup
         xref_cleanup(xmldoc) # feeds: concept_cleanup, origin_cleanup
         concept_cleanup(xmldoc) # feeds: related_cleanup, termdef_cleanup
@@ -79,6 +80,7 @@ module Metanorma
         empty_element_cleanup(xmldoc)
         img_cleanup(xmldoc)
         anchor_cleanup(xmldoc)
+        link_cleanup(xmldoc)
         xmldoc
       end
 
@@ -107,7 +109,7 @@ module Metanorma
       end
 
       def element_name_cleanup(xmldoc)
-        xmldoc.traverse { |n| n.name = n.name.gsub(/_/, "-") }
+        xmldoc.traverse { |n| n.name = n.name.gsub("_", "-") }
       end
 
       # allows us to deal with doc relation localities,

data/lib/metanorma/standoc/cleanup_inline.rb

@@ -1,5 +1,6 @@
 require "metanorma-utils"
 require "digest"
+require "addressable/uri"
 
 module Metanorma
   module Standoc
@@ -78,7 +79,7 @@ module Metanorma
       def concept_cleanup1(elem)
         elem.children.remove if elem&.children&.text&.strip&.empty?
         key_extract_locality(elem)
-        if /:/.match?(elem["key"]) then concept_termbase_cleanup(elem)
+        if elem["key"].include?(":") then concept_termbase_cleanup(elem)
         elsif refid? elem["key"] then concept_eref_cleanup(elem)
         else concept_xref_cleanup(elem)
         end
@@ -95,15 +96,14 @@ module Metanorma
       end
 
       def key_extract_locality(elem)
-        return unless /,/.match?(elem["key"])
-
+        elem["key"].include?(",") or return
         elem.add_child("<locality>#{elem['key'].sub(/^[^,]+,/, '')}</locality>")
         elem["key"] = elem["key"].sub(/,.*$/, "")
       end
 
       def concept_termbase_cleanup(elem)
         t = elem&.at("./xrefrender")&.remove&.children
-        termbase, key = elem["key"].split(/:/, 2)
+        termbase, key = elem["key"].split(":", 2)
         elem.add_child(%(<termref base="#{termbase}" target="#{key}">) +
                        "#{t&.to_xml}</termref>")
       end
@@ -207,6 +207,21 @@ module Metanorma
         end
       end
 
+      def link_cleanup(xmldoc)
+        xmldoc.xpath("//link[@target]").each do |l|
+          l["target"] = Addressable::URI.parse(l["target"]).to_s
+        rescue Addressable::URI::InvalidURIError
+          err = "Malformed URI: #{l['target']}"
+          @log.add("Anchors", l, err)
+          @fatalerror << err
+          warn err
+        end
+      end
+
+      def uri_component_encode(comp)
+        CGI.escape(comp).gsub("+", "%20")
+      end
+
       private
 
       # skip ZWNJ inserted to prevent regexes operating in asciidoctor

data/lib/metanorma/standoc/cleanup_maths.rb

@@ -3,17 +3,28 @@ require "asciimath2unitsml"
 module Metanorma
   module Standoc
     module Cleanup
-      def asciimath2mathml(text)
-        text = text.gsub(%r{<stem type="AsciiMath">(.+?)</stem>}m) do
-          "<amathstem>#{@c.decode($1)}</amathstem>"
+      def asciimath_cleanup(xml)
+        !@keepasciimath and asciimath2mathml(xml)
+      end
+
+      def asciimath2mathml(xml)
+        xpath = xml.xpath("//stem[@type = 'AsciiMath']")
+        xpath.each_with_index do |x, i|
+          progress_conv(i, 500, xpath.size, 1000, "AsciiMath")
+          asciimath2mathml_indiv(x)
         end
-        text = Html2Doc.new({})
-          .asciimath_to_mathml(text, ["<amathstem>", "</amathstem>"],
-                               retain_asciimath: true)
-        asciimath2mathml_wrap(text)
+        asciimath2mathml_wrap(xml)
+      end
+
+      def asciimath2mathml_indiv(elem)
+        elem["type"] = "MathML"
+        expr = @c.decode(elem.text)
+        ret = Plurimath::Math.parse(expr, "asciimath")
+          .to_mathml(display_style: elem["block"])
+        ret += "<asciimath>#{@c.encode(@c.decode(expr), :basic)}</asciimath>"
+        elem.children = ret
       rescue StandardError => e
-        asciimath2mathml_err(text, e)
-        text
+        asciimath2mathml_err(elem.to_xml, e)
       end
 
       def asciimath2mathml_err(text, expr)
@@ -23,25 +34,27 @@ module Metanorma
         warn err
       end
 
-      def asciimath2mathml_wrap(text)
-        x = Nokogiri::XML(text)
-        x.xpath("//*[local-name() = 'math'][@display]").each do |y|
+      def asciimath2mathml_wrap(xml)
+        xml.xpath("//*[local-name() = 'math'][@display]").each do |y|
           y.delete("display")
         end
-        x.xpath("//*[local-name() = 'math'][not(parent::stem)]").each do |y|
-          y.wrap("<stem type='MathML'></stem>")
-        end
-        x.xpath("//stem").each do |y|
-          y.next_element&.name == "asciimath" and y << y.next_element
-        end
-        to_xml(x)
+        # x.xpath("//stem").each do |y|
+        # y.next_element&.name == "asciimath" and y << y.next_element
+        # end
+        xml
+      end
+
+      def progress_conv(idx, step, total, threshold, msg)
+        return unless (idx % step).zero? && total > threshold && idx.positive?
+
+        warn "#{msg} #{idx} of #{total}"
       end
 
       def xml_unescape_mathml(xml)
         return if xml.children.any?(&:element?)
 
-        math = xml.text.gsub(/&lt;/, "<").gsub(/&gt;/, ">")
-          .gsub(/&quot;/, '"').gsub(/&apos;/, "'").gsub(/&amp;/, "&")
+        math = xml.text.gsub("&lt;", "<").gsub("&gt;", ">")
+          .gsub("&quot;", '"').gsub("&apos;", "'").gsub("&amp;", "&")
           .gsub(/<[^: \r\n\t\/]+:/, "<").gsub(/<\/[^ \r\n\t:]+:/, "</")
         xml.children = math
       end

data/lib/metanorma/standoc/cleanup_symbols.rb

@@ -1,3 +1,5 @@
+require "metanorma/standoc/utils"
+
 module Metanorma
   module Standoc
     module Cleanup
@@ -8,31 +10,12 @@ module Metanorma
       def symbol_key(sym)
         @c.decode(asciimath_key(sym).text)
           .gsub(/[\[\]{}<>()]/, "").gsub(/\s/m, "")
-          .gsub(/[[:punct:]]|[_^]/, ":\\0").gsub(/`/, "")
+          .gsub(/[[:punct:]]|[_^]/, ":\\0").gsub("`", "")
           .gsub(/[0-9]+/, "þ\\0")
           .tr("AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz",
               "ABCFEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
       end
 
-      def asciimath_key(sym)
-        key = sym.dup
-        key.traverse do |n|
-          if n.name == "stem" && a = n.at(".//asciimath")
-            n.children = @c.encode(
-              @c.decode(grkletters(a.text)), :basic
-            )
-          end
-        end
-        key.xpath(".//asciimath").each(&:remove)
-        Nokogiri::XML(key.to_xml)
-      end
-
-      def grkletters(text)
-        text.gsub(/\b(alpha|beta|gamma|delta|epsilon|zeta|eta|theta|iota|kappa|
-                      lambda|mu|nu|xi|omicron|pi|rho|sigma|tau|upsilon|phi|chi|
-                      psi|omega)\b/xi, "&\\1;")
-      end
-
       def extract_symbols_list(dlist)
         dl_out = []
         dlist.xpath("./dt | ./dd").each do |dtd|

data/lib/metanorma/standoc/cleanup_text.rb

@@ -3,7 +3,6 @@ module Metanorma
     module Cleanup
       def textcleanup(result)
         text = result.flatten.map { |l| l.sub(/\s*\Z/, "") } * "\n"
-        !@keepasciimath and text = asciimath2mathml(text)
         text = text.gsub(/\s+<fn /, "<fn ")
         %w(passthrough passthrough-inline).each do |v|
           text.gsub!(%r{<#{v}\s+formats="metanorma">([^<]*)

data/lib/metanorma/standoc/converter.rb

@@ -12,7 +12,7 @@ require_relative "validate"
 require_relative "utils"
 require_relative "cleanup"
 require_relative "reqt"
-require_relative "./macros"
+require_relative "macros"
 
 module Metanorma
   module Standoc
@@ -22,6 +22,7 @@ module Metanorma
       Asciidoctor::Extensions.register do
         preprocessor Metanorma::Standoc::EmbedIncludeProcessor
         preprocessor Metanorma::Standoc::NamedEscapePreprocessor
+        preprocessor Metanorma::Standoc::LinkProtectPreprocessor
         preprocessor Metanorma::Standoc::Datamodel::AttributesTablePreprocessor
         preprocessor Metanorma::Standoc::Datamodel::DiagramPreprocessor
         preprocessor Metanorma::Plugin::Datastruct::Json2TextPreprocessor

data/lib/metanorma/standoc/inline.rb

@@ -138,9 +138,10 @@ module Metanorma
         noko { |xml| xml.hr }.join
       end
 
-      def latex_parse1(text)
+      def latex_parse1(text, block)
         lxm_input = Unicode2LaTeX.unicode2latex(@c.decode(text))
-        results = Plurimath::Math.parse(lxm_input, "latex").to_mathml
+        results = Plurimath::Math.parse(lxm_input, "latex")
+          .to_mathml(display_style: block)
         if results.nil?
           @log.add("Math", nil,
                    "latexmlmath failed to process equation:\n#{lxm_input}")
@@ -149,22 +150,23 @@ module Metanorma
         results.sub(%r{<math ([^>]+ )?display="block"}, "<math \\1")
       end
 
-      def stem_parse(text, xml, style)
+      def stem_parse(text, xml, style, block)
         if /&lt;([^:>&]+:)?math(\s+[^>&]+)?&gt; |
           <([^:>&]+:)?math(\s+[^>&]+)?>/x.match? text
           math = xml_encode(text)
-          xml.stem type: "MathML" do |s|
+          xml.stem type: "MathML", block: block do |s|
             s << math
           end
-        elsif style == :latexmath then latex_parse(text, xml)
+        elsif style == :latexmath then latex_parse(text, xml, block)
         else
-          xml.stem text&.gsub(/&amp;#/, "&#"), type: "AsciiMath"
+          xml.stem text&.gsub("&amp;#", "&#"), type: "AsciiMath", block: block
         end
       end
 
-      def latex_parse(text, xml)
-        latex = latex_parse1(text) or return xml.stem type: "MathML"
-        xml.stem type: "MathML" do |s|
+      def latex_parse(text, xml, block)
+        latex = latex_parse1(text, block) or
+          return xml.stem type: "MathML", block: block
+        xml.stem type: "MathML", block: block do |s|
           math = Nokogiri::XML.fragment(latex.sub(/<\?[^>]+>/, ""))
             .elements[0]
           math.delete("alttext")
@@ -187,8 +189,8 @@ module Metanorma
           when :single then xml << "'#{node.text}'"
           when :superscript then xml.sup { |s| s << node.text }
           when :subscript then xml.sub { |s| s << node.text }
-          when :asciimath then stem_parse(node.text, xml, :asciimath)
-          when :latexmath then stem_parse(node.text, xml, :latexmath)
+          when :asciimath then stem_parse(node.text, xml, :asciimath, false)
+          when :latexmath then stem_parse(node.text, xml, :latexmath, false)
           when :mark then highlight_parse(node.text, xml)
           else
             case node.role

data/lib/metanorma/standoc/isodoc.rng

@@ -17,7 +17,7 @@
   these elements; we just want one namespace for any child grammars
   of this.
 -->
-<!-- VERSION v1.2.2 -->
+<!-- VERSION v1.2.3 -->
 <grammar xmlns:a="http://relaxng.org/ns/compatibility/annotations/1.0" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
   <include href="reqt.rng"/>
   <include href="basicdoc.rng">
@@ -192,9 +192,11 @@
         </attribute>
       </optional>
       <attribute name="citeas"/>
-      <attribute name="type">
-        <ref name="ReferenceFormat"/>
-      </attribute>
+      <optional>
+        <attribute name="type">
+          <ref name="ReferenceFormat"/>
+        </attribute>
+      </optional>
       <optional>
         <attribute name="alt"/>
       </optional>
@@ -836,6 +838,26 @@
         <ref name="paragraph"/>
       </element>
     </define>
+    <define name="stem">
+      <element name="stem">
+        <attribute name="type">
+          <choice>
+            <value>MathML</value>
+            <value>AsciiMath</value>
+            <value>LatexMath</value>
+          </choice>
+        </attribute>
+        <attribute name="block">
+          <data type="boolean"/>
+        </attribute>
+        <oneOrMore>
+          <choice>
+            <text/>
+            <ref name="AnyElement"/>
+          </choice>
+        </oneOrMore>
+      </element>
+    </define>
     <define name="em">
       <element name="em">
         <zeroOrMore>

data/lib/metanorma/standoc/macros.rb

@@ -1,14 +1,14 @@
 require "uuidtools"
 require "yaml"
 require "csv"
-require_relative "./macros_inline"
-require_relative "./macros_plantuml"
-require_relative "./macros_terms"
-require_relative "./macros_form"
-require_relative "./macros_note"
-require_relative "./macros_embed"
-require_relative "./datamodel/attributes_table_preprocessor"
-require_relative "./datamodel/diagram_preprocessor"
+require_relative "macros_inline"
+require_relative "macros_plantuml"
+require_relative "macros_terms"
+require_relative "macros_form"
+require_relative "macros_note"
+require_relative "macros_embed"
+require_relative "datamodel/attributes_table_preprocessor"
+require_relative "datamodel/diagram_preprocessor"
 require "metanorma-plugin-datastruct"
 require "metanorma-plugin-glossarist"
 require "metanorma-plugin-lutaml"
@@ -22,7 +22,7 @@ module Metanorma
 
       def init_indent(line)
         /^(?<prefix>[ \t]*)(?<suffix>.*)$/ =~ line
-        prefix = prefix.gsub(/\t/, "\u00a0\u00a0\u00a0\u00a0")
+        prefix = prefix.gsub("\t", "\u00a0\u00a0\u00a0\u00a0")
           .gsub(/ /, "\u00a0")
         prefix + suffix
       end
@@ -31,9 +31,9 @@ module Metanorma
         ignore = false
         lines.each_with_index do |l, i|
           /^(--+|====+|\|===|\.\.\.\.+|\*\*\*\*+|\+\+\+\++|````+|____\+)$/
-            .match(l) && (ignore = !ignore)
-          next if l.empty? || l.match(/ \+$/) || /^\[.*\]$/.match?(l) || ignore
-          next if i == lines.size - 1 ||
+            .match(l) and (ignore = !ignore)
+          next if l.empty? || l.match(/ \+$/) || /^\[.*\]$/.match?(l) ||
+            ignore || i == lines.size - 1 ||
             (i < lines.size - 1 && lines[i + 1].empty?)
 
           lines[i] += " +"
@@ -60,5 +60,111 @@ module Metanorma
         ::Asciidoctor::Reader.new lines
       end
     end
+
+    # refer https://github.com/asciidoctor/asciidoctor/blob/main/lib/asciidoctor/substitutors.rb
+    # Not using TreeProcessor because that is still too close to
+    # inline expressions being processed on access (e.g. titles)
+    class LinkProtectPreprocessor < Asciidoctor::Extensions::Preprocessor
+      def init
+        pass = true # process as passthrough: init = true until
+        # hit end of doc header
+        is_delim = false # current line is a no-substititon block delimiter
+        pass_delim = false # current line is a passthrough delimiter
+        delimln = "" # delimiter line of current block(s);
+        # init value looks for end of doc header
+        { pass: pass, is_delim: is_delim, pass_delim: pass_delim,
+          delimln: delimln }
+      end
+
+      def process(_document, reader)
+        p = init
+        lines = reader.readlines.map do |t|
+          p = pass_status(p, t.rstrip)
+          !p[:pass] && t.include?(":") and t = inlinelinkmacro(inlinelink(t))
+          t
+        end
+        ::Asciidoctor::Reader.new lines
+      end
+
+      def pass_status(status, text)
+        text == "++++" && !status[:delimln] and status[:pass] = !status[:pass]
+        if status[:is_delim] && /^(-+|\*+|=+|_+)$/.match?(text)
+          status[:delimln] = text
+          status[:pass] = true
+        elsif status[:pass_delim]
+          status[:delimln] = "" # end of paragraph for paragraph with [pass]
+        elsif status[:delimln] && text == status[:delimln]
+          status[:pass] = false
+          status[:delimln] = nil
+        end
+        status[:is_delim] = /^\[(source|listing|literal|pass)\b/.match?(text)
+        status[:pass_delim] = /^\[(pass)\b/.match?(text)
+        status
+      end
+
+      PASS_INLINE_MACROS = %w(pass pass-format identifier std-link stem)
+        .join("|").freeze
+
+      PASS_INLINE_MACRO_STR = <<~REGEX.freeze
+        (
+          \\b(?<!-)                              # word-separator, no hyphen
+          (?:                                    # don't capture these!
+            (?:#{PASS_INLINE_MACROS}):[^\\s\\[]* | # macro name, :, second key. OR:
+            span:uri \\b [^\\s\\[]*              # span:uri, third key
+          )
+          \\[.*?(?<!\\\\)\\]                     # [ ... ] not preceded by \\
+        )
+      REGEX
+      PASS_INLINE_MACRO_RX = /#{PASS_INLINE_MACRO_STR}/xo.freeze
+
+      def pass_inline_split(text)
+        text.split(PASS_INLINE_MACRO_RX).each.map do |x|
+          PASS_INLINE_MACRO_RX.match?(x) ? x : yield(x)
+        end
+      end
+
+      # InlineLinkRx = %r((^|link:|#{CG_BLANK}|&lt;|[>\(\)\[\];"'])(\\?(?:https?|file|ftp|irc)://)(?:([^\s\[\]]+)\[(|#{CC_ALL}*?[^\\])\]|([^\s\[\]<]*([^\s,.?!\[\]<\)]))))m
+      #
+      InlineLinkRx = %r((^|(?<!-)\blink:(?!\+)|\p{Blank}|&lt;|[<>\(\)\[\];"'])(\\?(?:https?|file|ftp|irc)://)(?:([^\s\[\]]+)(?:(\[(|.*?[^\\])\])|([^\s\[\]<]*([^\s,.?!\[\]<\)])))))m.freeze
+
+      def inlinelink(text)
+        text.include?("://") or return text
+        pass_inline_split(text) do |x|
+          inlinelink_escape(x)
+        end.join
+      end
+
+      def inlinelink_escape(text)
+        text.gsub(InlineLinkRx) do
+          body, suffix = $4.nil? ? [$3 + $6, "[]"] : [$3, ""]
+          p = $1 and s = $2 and b = $4
+          if p == "link:" then "#{p}++#{s}#{body}++#{b}#{suffix}"
+          elsif p == "<"
+            "#{p}link:++#{s}#{body.sub(/>$/, '')}++#{b}#{suffix}>"
+          else "#{p}link:++#{s}#{body}++#{b}#{suffix}"
+          end
+        end
+      end
+
+      # InlineLinkMacroRx = /\\?(?:link|(mailto)):(|[^:\s\[][^\s\[]*)\[(|#{CC_ALL}*?[^\\])\]/m
+      InlineLinkMacroRx1 = <<~REGEX.freeze
+        (\\\\?\\b(?<!-)                  # optional backslash, no hyphen, word boundary
+          (?:link|mailto):)              # link: or mailto:
+        (?!\\+)                          # no link:+ passthrough
+        (|[^:\\s\\[][^\\s\\[]*)          # link: ... up to [
+        (\\[(|.*?[^\\\\])\\])            # [ ... ], no ]
+      REGEX
+      InlineLinkMacroRx = /#{InlineLinkMacroRx1}/x.freeze
+
+      def inlinelinkmacro(text)
+        (text.include?("[") &&
+          ((text.include? "link:") || (text.include? "ilto:"))) or return text
+        pass_inline_split(text) do |x|
+          x.gsub(InlineLinkMacroRx) do
+            "#{$1}++#{$2}++#{$3}"
+          end
+        end.join
+      end
+    end
   end
 end

data/lib/metanorma/standoc/macros_inline.rb

@@ -44,7 +44,7 @@ module Metanorma
         text = attr["text"]
         text = "((#{text}))" unless /^\(\(.+\)\)$/.match?(text)
         out = parent.sub_macros(text)
-        out.sub(/<index>/, "<index to='#{target}'>")
+        out.sub("<index>", "<index to='#{target}'>")
       end
     end
 
@@ -148,7 +148,7 @@ module Metanorma
         content = CSV.parse_line(out).map do |x|
           x.sub!(/^(["'])(.+)\1/, "\\2")
           m = /^(.*?)(:\d+)?$/.match(x)
-          %{<toc-xpath depth='#{m[2]&.sub(/:/, '') || 1}'>#{m[1]}</toc-xpath>}
+          %{<toc-xpath depth='#{m[2]&.sub(':', '') || 1}'>#{m[1]}</toc-xpath>}
         end.join
         "<toc>#{content}</toc>"
       end

data/lib/metanorma/standoc/term_lookup_cleanup.rb

@@ -49,7 +49,8 @@ module Metanorma
       def concept_cleanup
         xmldoc.xpath("//concept").each do |n|
           refterm = n.at("./refterm") or next
-          p = @termlookup[:secondary2primary][@c.encode(refterm.text)] and
+          lookup = normalize_ref_id_text(refterm.text.strip)
+          p = @termlookup[:secondary2primary][lookup] and
             refterm.children = @c.encode(p)
         end
       end
@@ -61,7 +62,7 @@ module Metanorma
       def related_cleanup
         xmldoc.xpath("//related").each do |n|
           refterm = n.at("./refterm") or next
-          lookup = @c.encode(refterm.text)
+          lookup = normalize_ref_id_text(refterm.text.strip)
           p = @termlookup[:secondary2primary][lookup] and
             refterm.children = @c.encode(p)
           p || @termlookup[:term][lookup] and
@@ -133,7 +134,7 @@ module Metanorma
       end
 
       def remove_missing_ref_msg1(_node, target, ret)
-        target2 = "_#{target.downcase.gsub(/-/, '_')}"
+        target2 = "_#{target.downcase.gsub('-', '_')}"
         if @terms_tags[target] || @terms_tags[target2]
           ret.strip!
           ret += ". Did you mean to point to a subterm?"
@@ -185,17 +186,20 @@ module Metanorma
 
       def pref_secondary2primary_preferred(term, res, primary)
         term.xpath("./preferred//name").each_with_index do |p, i|
-          i.positive? and res[domain_prefix(term, p.text)] = primary
-          @unique_designs[p.text] && term.at(".//domain") and
-            res[p.text] = primary
+          t = p.text.strip
+          i.positive? and
+            res[normalize_ref_id_text(domain_prefix(term, t))] = primary
+          @unique_designs[t] && term.at(".//domain") and
+            res[normalize_ref_id_text(t)] = primary
         end
       end
 
       def pref_secondary2primary_admitted(term, res, primary)
         term.xpath("./admitted//name").each do |p|
-          res[domain_prefix(term, p.text)] = primary
-          @unique_designs[p.text] && term.at(".//domain") and
-            res[p.text] = primary
+          t = p.text.strip
+          res[normalize_ref_id_text(domain_prefix(term, t))] = primary
+          @unique_designs[t] && term.at(".//domain") and
+            res[normalize_ref_id_text(t)] = primary
         end
       end
 
@@ -234,9 +238,13 @@ module Metanorma
       def normalize_ref_id1(term, node = nil)
         t = term.dup
         t.xpath(".//index").map(&:remove)
-        ret = t.text.strip
+        ret = asciimath_key(t).text.strip
         node and ret = domain_prefix(node, ret)
-        Metanorma::Utils::to_ncname(ret.gsub(/[[:space:]]+/, "-"))
+        normalize_ref_id_text(ret)
+      end
+
+      def normalize_ref_id_text(text)
+        Metanorma::Utils::to_ncname(text.gsub(/[[:space:]]+/, "-"))
       end
 
       def unique_text_id(text, prefix)
@@ -248,6 +256,8 @@ module Metanorma
           end
         end
       end
+
+      include ::Metanorma::Standoc::Utils
     end
   end
 end

data/lib/metanorma/standoc/utils.rb

@@ -70,10 +70,10 @@ module Metanorma
 
       def xml_encode(text)
         @c.encode(text, :basic, :hexadecimal)
-          .gsub(/&amp;gt;/, ">").gsub(/&amp;lt;/, "<").gsub(/&amp;amp;/, "&")
-          .gsub(/&gt;/, ">").gsub(/&lt;/, "<").gsub(/&amp;/, "&")
-          .gsub(/&quot;/, '"').gsub(/&#xa;/, "\n").gsub(/&amp;#/, "&#")
-          .gsub(/&apos;/, "'")
+          .gsub("&amp;gt;", ">").gsub("&amp;lt;", "<").gsub("&amp;amp;", "&")
+          .gsub("&gt;", ">").gsub("&lt;", "<").gsub("&amp;", "&")
+          .gsub("&quot;", '"').gsub("&#xa;", "\n").gsub("&amp;#", "&#")
+          .gsub("&apos;", "'")
       end
 
       # wrapped in <sections>
@@ -85,6 +85,25 @@ module Metanorma
         Nokogiri::XML(c).at("//xmlns:sections")
       end
 
+      def asciimath_key(sym)
+        key = sym.dup
+        key.traverse do |n|
+          if n.name == "stem" && a = n.at(".//asciimath")
+            n.children = @c.encode(
+              @c.decode(grkletters(a.text)), :basic
+            )
+          end
+        end
+        key.xpath(".//asciimath").each(&:remove)
+        Nokogiri::XML(key.to_xml)
+      end
+
+      def grkletters(text)
+        text.gsub(/\b(alpha|beta|gamma|delta|epsilon|zeta|eta|theta|iota|kappa|
+                      lambda|mu|nu|xi|omicron|pi|rho|sigma|tau|upsilon|phi|chi|
+                      psi|omega)\b/xi, "&\\1;")
+      end
+
       module_function :adoc2xml
 
       class EmptyAttr

data/lib/metanorma/standoc/validate.rb

@@ -1,7 +1,7 @@
 require "metanorma/standoc/utils"
-require_relative "./validate_section"
-require_relative "./validate_table"
-require_relative "./validate_term"
+require_relative "validate_section"
+require_relative "validate_table"
+require_relative "validate_term"
 require "nokogiri"
 require "jing"
 require "iev"

data/lib/metanorma/standoc/version.rb

@@ -19,6 +19,6 @@ module Metanorma
   end
 
   module Standoc
-    VERSION = "2.4.9".freeze
+    VERSION = "2.5.1".freeze
   end
 end

data/metanorma-standoc.gemspec

@@ -28,10 +28,11 @@ Gem::Specification.new do |spec|
   end
   spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
+  spec.add_dependency "addressable", "~> 2.8.0"
   spec.add_dependency "asciidoctor", "~> 2.0.0"
   spec.add_dependency "iev", "~> 0.3.0"
-  spec.add_dependency "isodoc", "~> 2.5.4"
-  spec.add_dependency "metanorma", ">= 1.5.0"
+  spec.add_dependency "isodoc", "~> 2.6.0"
+  spec.add_dependency "metanorma", ">= 1.6.0"
   spec.add_dependency "metanorma-plugin-datastruct", "~> 0.2.0"
   spec.add_dependency "metanorma-plugin-glossarist", "~> 0.1.1"
   spec.add_dependency "metanorma-plugin-lutaml"

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-standoc
 version: !ruby/object:Gem::Version
-  version: 2.4.9
+  version: 2.5.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-24 00:00:00.000000000 Z
+date: 2023-08-09 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
 - !ruby/object:Gem::Dependency
   name: asciidoctor
   requirement: !ruby/object:Gem::Requirement
@@ -44,28 +58,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.5.4
+        version: 2.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.5.4
+        version: 2.6.0
 - !ruby/object:Gem::Dependency
   name: metanorma
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: 1.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: metanorma-plugin-datastruct
   requirement: !ruby/object:Gem::Requirement