metanorma-standoc 2.0.3 → 2.0.5.1

data/lib/metanorma/standoc/cleanup_xref.rb

@@ -3,11 +3,12 @@ module Metanorma
     module Cleanup
       # extending localities to cover ISO referencing
       LOCALITY_REGEX_STR = <<~REGEXP.freeze
-        ^((?<locality>section|clause|part|paragraph|chapter|page|
+        ^(((?<conn>and|or|from|to)!)?
+            (?<locality>section|clause|part|paragraph|chapter|page|
                       table|annex|figure|example|note|formula|list|time|anchor|
                       locality:[^ \\t\\n\\r:,;=]+)(\\s+|=)
-               (?<ref>[^"][^ \\t\\n,:-]*|"[^"]+")
-                 (-(?<to>[^"][^ \\t\\n,:-]*|"[^"]"))?|
+               (?<ref>[^"][^ \\t\\n,:;-]*|"[^"]+")
+                 (-(?<to>[^"][^ \\t\\n,:;-]*|"[^"]"))?|
           (?<locality2>whole|locality:[^ \\t\\n\\r:,;=]+))(?<punct>[,:;]?)\\s*
          (?<text>.*)$
       REGEXP
@@ -30,16 +31,37 @@ module Metanorma
       def extract_localities1(elem, text)
         b = elem.add_child("<localityStack/>").first if LOCALITY_RE.match text
         while (m = LOCALITY_RE.match text)
-          ref = m[:ref] ? "<referenceFrom>#{tq m[:ref]}</referenceFrom>" : ""
-          refto = m[:to] ? "<referenceTo>#{tq m[:to]}</referenceTo>" : ""
-          b.add_child("<locality type='#{locality_label(m)}'>#{ref}#{refto}"\
-                      "</locality>")
+          add_locality(b, m)
           text = m[:text]
           b = elem.add_child("<localityStack/>").first if m[:punct] == ";"
         end
+        fill_in_eref_connectives(elem)
         elem.add_child(text) if text
       end
 
+      def add_locality(stack, match)
+        stack.children.empty? && match[:conn] and
+          stack["connective"] = match[:conn]
+        ref =
+          match[:ref] ? "<referenceFrom>#{tq match[:ref]}</referenceFrom>" : ""
+        refto = match[:to] ? "<referenceTo>#{tq match[:to]}</referenceTo>" : ""
+        stack.add_child("<locality type='#{locality_label(match)}'>#{ref}"\
+                        "#{refto}</locality>")
+      end
+
+      def fill_in_eref_connectives(elem)
+        return if elem.xpath("./localityStack").size < 2
+
+        elem.xpath("./localityStack[not(@connective)]").each do |l|
+          n = l.next_element
+          l["connective"] = if n && n.name == "localityStack" &&
+              n["connective"] == "to"
+                              "from"
+                            else "and"
+                            end
+        end
+      end
+
       def locality_label(match)
         loc = match[:locality] || match[:locality2]
         /^locality:/.match?(loc) ? loc : loc&.downcase
@@ -65,8 +87,48 @@ module Metanorma
       end
 
       def xref_cleanup(xmldoc)
+        xref_compound_cleanup(xmldoc)
+        xref_cleanup1(xmldoc)
+        xref_compound_wrapup(xmldoc)
+      end
+
+      def xref_compound_cleanup(xmldoc)
+        xmldoc.xpath("//xref").each do |x|
+          /;/.match?(x["target"]) or next
+          locations = x["target"].split(";")
+          x["target"] = locations.first.sub(/^[^!]*!/, "")
+          xref_compound_cleanup1(x, locations)
+        end
+      end
+
+      def xref_compound_cleanup1(xref, locations)
+        xref.children.empty? and xref.children = "<sentinel/>"
+        xref_parse_compound_locations(locations).reverse.each do |y|
+          xref.children.first.previous =
+            "<xref target='#{y[1]}' connective='#{y[0]}'/>"
+        end
+        xref&.at("./sentinel")&.remove
+      end
+
+      def xref_parse_compound_locations(locations)
+        l = locations.map { |y| y.split("!", 2) }
+        l.map.with_index do |y, i|
+          if y.size == 1
+            y.unshift(l.dig(i + 1, 0) == "to" ? "from" : "and")
+          end
+          y
+        end
+      end
+
+      def xref_compound_wrapup(xmldoc)
+        xmldoc.xpath("//xref//xref").each do |x|
+          x.name = "location"
+        end
+      end
+
+      def xref_cleanup1(xmldoc)
         xmldoc.xpath("//xref").each do |x|
-          /:/.match(x["target"]) and xref_to_internal_eref(x)
+          /:/.match?(x["target"]) and xref_to_internal_eref(x)
           next unless x.name == "xref"
 
           if refid? x["target"]

data/lib/metanorma/standoc/converter.rb

@@ -53,6 +53,7 @@ module Metanorma
         inline_macro Metanorma::Standoc::FormOptionMacro
         inline_macro Metanorma::Standoc::ToCInlineMacro
         inline_macro Metanorma::Standoc::PassInlineMacro
+        inline_macro Metanorma::Standoc::StdLinkInlineMacro
         inline_macro Metanorma::Plugin::Lutaml::LutamlFigureInlineMacro
         inline_macro Metanorma::Plugin::Lutaml::LutamlTableInlineMacro
         block_macro Metanorma::Plugin::Lutaml::LutamlDiagramBlockMacro
@@ -91,7 +92,7 @@ module Metanorma
       end
 
       class << self
-        attr_accessor :_file
+        attr_accessor :_file, :embed_hdr
       end
 
       def self.inherited(konv) # rubocop:disable Lint/MissingSuper

data/lib/metanorma/standoc/inline.rb

@@ -40,20 +40,29 @@ module Metanorma
       end
 
       def inline_anchor_xref_attrs(node)
-        m = /^(?<drop>droploc%)?(?<case>capital%|lowercase%)?(?<drop2>droploc%)?
-          (?<fn>fn:?\s*)?(?<text>.*)$/x.match node.text
+        m = inline_anchor_xref_match(node)
         t = node.target.gsub(/^#/, "").gsub(%r{(\.xml|\.adoc)(#.*$)}, "\\2")
         m.nil? and return { target: t, type: "inline", text: node.text }
-        droploc = m[:drop].nil? && m[:drop2].nil? ? nil : true
-        f = m[:fn].nil? ? "inline" : "footnote"
-        c = if %i[case fn drop drop2].any? do |x|
-                 !m[x].nil?
-               end
-              m[:text]
-            else node.text
-            end
-        { target: t, type: f, case: m[:case]&.sub(/%$/, ""), droploc: droploc,
-          text: c }
+        { target: t, type: m[:fn].nil? ? "inline" : "footnote",
+          case: m[:case]&.sub(/%$/, ""),
+          droploc: m[:drop].nil? && m[:drop2].nil? ? nil : true,
+          text: inline_anchor_xref_text(m, node),
+          hidden: m[:hidden] }
+      end
+
+      def inline_anchor_xref_match(node)
+        /^(hidden%(?<hidden>[^,]+),?)?
+          (?<drop>droploc%)?(?<case>capital%|lowercase%)?(?<drop2>droploc%)?
+          (?<fn>fn:?\s*)?(?<text>.*)$/x.match node.text
+      end
+
+      def inline_anchor_xref_text(match, node)
+        if %i[case fn drop drop2 hidden].any? do |x|
+             !match[x].nil?
+           end
+          match[:text]
+        else node.text
+        end
       end
 
       def inline_anchor_link(node)

data/lib/metanorma/standoc/isodoc.rng

@@ -152,9 +152,7 @@
             <data type="boolean"/>
           </attribute>
         </optional>
-        <oneOrMore>
-          <ref name="PureTextElement"/>
-        </oneOrMore>
+        <ref name="XrefBody"/>
       </element>
     </define>
     <define name="erefType">
@@ -188,6 +186,42 @@
         <ref name="PureTextElement"/>
       </oneOrMore>
     </define>
+    <define name="localityStack">
+      <element name="localityStack">
+        <optional>
+          <attribute name="connective">
+            <choice>
+              <value>and</value>
+              <value>or</value>
+              <value>from</value>
+              <value>to</value>
+              <value/>
+            </choice>
+          </attribute>
+        </optional>
+        <zeroOrMore>
+          <ref name="locality"/>
+        </zeroOrMore>
+      </element>
+    </define>
+    <define name="sourceLocalityStack">
+      <element name="sourceLocalityStack">
+        <optional>
+          <attribute name="connective">
+            <choice>
+              <value>and</value>
+              <value>or</value>
+              <value>from</value>
+              <value>to</value>
+              <value/>
+            </choice>
+          </attribute>
+        </optional>
+        <zeroOrMore>
+          <ref name="sourceLocality"/>
+        </zeroOrMore>
+      </element>
+    </define>
     <define name="ul">
       <element name="ul">
         <attribute name="id">
@@ -2641,4 +2675,30 @@
       </zeroOrMore>
     </element>
   </define>
+  <define name="XrefBody">
+    <zeroOrMore>
+      <ref name="XrefTarget"/>
+    </zeroOrMore>
+    <oneOrMore>
+      <ref name="PureTextElement"/>
+    </oneOrMore>
+  </define>
+  <define name="XrefTarget">
+    <element name="location">
+      <attribute name="target">
+        <data type="string">
+          <param name="pattern">\i\c*|\c+#\c+</param>
+        </data>
+      </attribute>
+      <attribute name="connective">
+        <choice>
+          <value>and</value>
+          <value>or</value>
+          <value>from</value>
+          <value>to</value>
+          <value/>
+        </choice>
+      </attribute>
+    </element>
+  </define>
 </grammar>

data/lib/metanorma/standoc/macros.rb

@@ -82,7 +82,7 @@ module Metanorma
             .match(l) && (ignore = !ignore)
           next if l.empty? || l.match(/ \+$/) || /^\[.*\]$/.match?(l) || ignore
           next if i == lines.size - 1 ||
-            i < lines.size - 1 && lines[i + 1].empty?
+            (i < lines.size - 1 && lines[i + 1].empty?)
 
           lines[i] += " +"
         end
@@ -140,7 +140,7 @@ module Metanorma
       parse_content_as :text
 
       def process(parent, target, attrs)
-        /^(?<lang>[^-]*)(-(?<script>.*))?$/ =~ target
+        /^(?<lang>[^-]*)(?:-(?<script>.*))?$/ =~ target
         out = Asciidoctor::Inline.new(parent, :quoted, attrs["text"]).convert
         if script
           %{<variant lang=#{lang} script=#{script}>#{out}</variant>}
@@ -201,5 +201,17 @@ module Metanorma
         %{<passthrough formats="#{format}">#{out}</passthrough>}
       end
     end
+
+    class StdLinkInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :"std-link"
+      parse_content_as :text
+      using_format :short
+
+      def process(parent, _target, attrs)
+        create_anchor(parent, "hidden%#{attrs['text']}",
+                      type: :xref, target: "_#{UUIDTools::UUID.random_create}")
+      end
+    end
   end
 end

data/lib/metanorma/standoc/macros_embed.rb

@@ -5,13 +5,24 @@ module Metanorma
         return reader if reader.eof?
 
         lines = reader.readlines
-        while !lines.grep(/^embed::/).empty?
-          headings = lines.grep(/^== /).map(&:strip)
-          lines = lines.map do |line|
-            /^embed::/.match?(line) ? embed(line, doc, reader, headings) : line
-          end.flatten
+        headings = lines.grep(/^== /).map(&:strip)
+        ret = lines.each_with_object({ lines: [], hdr: [] }) do |line, m|
+          process1(line, m, doc, reader, headings)
         end
-        ::Asciidoctor::Reader.new lines
+        #doc.converter.embed_hdr = ret[:hdr]
+        doc.attributes["embed_hdr"] = ret[:hdr]
+        ::Asciidoctor::Reader.new ret[:lines].flatten
+      end
+
+      def process1(line, acc, doc, reader, headings)
+        if /^embed::/.match?(line)
+          e = embed(line, doc, reader, headings)
+          acc[:lines] << e[:lines]
+          acc[:hdr] << e[:hdr]
+        else
+          acc[:lines] << line
+        end
+        acc
       end
 
       def filename(line, doc, reader)
@@ -27,26 +38,36 @@ module Metanorma
         end
       end
 
+      def read(inc_path)
+        ::File.open inc_path, "r" do |fd|
+          readlines_safe(fd).map(&:chomp)
+        end
+      end
+
       def embed(line, doc, reader, headings)
         inc_path = filename(line, doc, reader) or return line
         lines = filter_sections(read(inc_path), headings)
         doc = Asciidoctor::Document.new [], { safe: :safe }
         reader = ::Asciidoctor::PreprocessorReader.new doc, lines
-        strip_header(reader.read_lines)
+        ret = strip_header(reader.read_lines)
+        embed_recurse(ret, doc, reader, headings)
       end
 
-      def read(inc_path)
-        ::File.open inc_path, "r" do |fd|
-          readlines_safe(fd).map(&:chomp)
+      def embed_recurse(ret, doc, reader, headings)
+        ret1 = ret[:lines].each_with_object({ lines: [], hdr: [] }) do |line, m|
+          process1(line, m, doc, reader, headings)
         end
+        { lines: ret1[:lines],
+          hdr: { text: ret[:hdr].join("\n"), child: ret1[:hdr] } }
       end
 
       def strip_header(lines)
-        return lines unless !lines.empty? && lines.first.start_with?("= ")
+        return { lines: lines, hdr: nil } unless !lines.empty? &&
+          lines.first.start_with?("= ")
 
         skip = true
-        lines.each_with_object([]) do |l, m|
-          m << l unless skip
+        lines.each_with_object({ hdr: [], lines: [] }) do |l, m|
+          m[skip ? :hdr : :lines] << l
           skip = false if !/\S/.match?(l)
         end
       end
@@ -56,7 +77,7 @@ module Metanorma
         lines.each_with_index.with_object([]) do |(l, i), m|
           if headings.include?(l.strip)
             skip = true
-            m.unshift while !m.empty? && /^\S/.match?(m[-1])
+            m.pop while !m.empty? && /^\S/.match?(m[-1])
           elsif skip && /^== |^embed::|^include::/.match?(l)
             skip = false
             j = i

data/lib/metanorma/standoc/macros_note.rb

@@ -22,9 +22,10 @@ module Metanorma
           para.set_attr("name", "todo")
           para.set_attr("caption", "TODO")
           para.lines[0].sub!(/^TODO: /, "")
-          todo = Asciidoctor::Block.new(parent, :admonition, attributes: para.attributes,
-                                                             source: para.lines,
-                                                             content_model: :compound)
+          todo = Asciidoctor::Block
+            .new(parent, :admonition, attributes: para.attributes,
+                                      source: para.lines,
+                                      content_model: :compound)
           parent.blocks[parent.blocks.index(para)] = todo
         end
       end

data/lib/metanorma/standoc/processor.rb

@@ -3,6 +3,10 @@ require "metanorma/processor"
 module Metanorma
   module Standoc
     class Processor < Metanorma::Processor
+      class << self
+        attr_reader :asciidoctor_backend
+            end
+
       def initialize # rubocop:disable Lint/MissingSuper
         @short = :standoc
         @input_format = :asciidoc
@@ -22,7 +26,8 @@ module Metanorma
       end
 
       def html_path(file)
-        File.join(File.dirname(__FILE__), "..", "..", "isodoc", "html", file)
+        File.join(File.dirname(__FILE__), "..", "..", "isodoc", "html",
+                  file)
       end
 
       def output(isodoc_node, inname, outname, format, options = {})

data/lib/metanorma/standoc/ref.rb

@@ -121,7 +121,7 @@ module Metanorma
           t.formattedref **{ format: "application/x-isodoc+xml" } do |i|
             i << ref_normalise_no_format(match[:text])
           end
-          yr_match = /[:-](?<year>(19|20)[0-9][0-9])\b/.match(code[:id])
+          yr_match = /[:-](?<year>(?:19|20)[0-9][0-9])\b/.match(code[:id])
           refitem_render1(match, code, t)
           docnumber(t, code[:id]) unless /^\d+$|^\(.+\)$/.match?(code[:id])
           conditional_date(t, yr_match || match, false)
@@ -149,7 +149,7 @@ module Metanorma
       end
 
       def refitem1yr(code)
-        yr_match = /[:-](?<year>(19|20)[0-9][0-9])\b/.match(code)
+        yr_match = /[:-](?<year>(?:19|20)[0-9][0-9])\b/.match(code)
         yr_match ? yr_match[:year] : nil
       end
 
@@ -163,20 +163,20 @@ module Metanorma
 
       ISO_REF =
         %r{^<ref\sid="(?<anchor>[^"]+)">
-      \[(?<usrlbl>\([^)]+\))?(?<code>(ISO|IEC)[^0-9]*\s[0-9-]+|IEV)
-      (:(?<year>[0-9][0-9-]+))?\]</ref>,?\s*(?<text>.*)$}xm.freeze
+      \[(?<usrlbl>\([^)]+\))?(?<code>(?:ISO|IEC)[^0-9]*\s[0-9-]+|IEV)
+      (?::(?<year>[0-9][0-9-]+))?\]</ref>,?\s*(?<text>.*)$}xm.freeze
 
       ISO_REF_NO_YEAR =
         %r{^<ref\sid="(?<anchor>[^"]+)">
-      \[(?<usrlbl>\([^)]+\))?(?<code>(ISO|IEC)[^0-9]*\s[0-9-]+):
-      (--|&\#821[12];)\]</ref>,?\s*
-        (<fn[^>]*>\s*<p>(?<fn>[^\]]+)</p>\s*</fn>)?,?\s?(?<text>.*)$}xm
+      \[(?<usrlbl>\([^)]+\))?(?<code>(?:ISO|IEC)[^0-9]*\s[0-9-]+):
+      (?:--|&\#821[12];)\]</ref>,?\s*
+        (?:<fn[^>]*>\s*<p>(?<fn>[^\]]+)</p>\s*</fn>)?,?\s?(?<text>.*)$}xm
           .freeze
 
       ISO_REF_ALL_PARTS =
         %r{^<ref\sid="(?<anchor>[^"]+)">
-      \[(?<usrlbl>\([^)]+\))?(?<code>(ISO|IEC)[^0-9]*\s[0-9]+)
-      (:(?<year>--|&\#821[12];|[0-9][0-9-]+))?\s
+      \[(?<usrlbl>\([^)]+\))?(?<code>(?:ISO|IEC)[^0-9]*\s[0-9]+)
+      (?::(?<year>--|&\#821[12];|[0-9][0-9-]+))?\s
       \(all\sparts\)\]</ref>,?\s*
         (<fn[^>]*>\s*<p>(?<fn>[^\]]+)</p>\s*</fn>,?\s?)?(?<text>.*)$}xm.freeze
 
@@ -214,18 +214,17 @@ module Metanorma
         end
       end
 
-      def reference_preproc(node)
+      def reference(node)
         refs = node.items.each_with_object([]) do |b, m|
           m << reference1code(b.text, node)
         end
+        reference_populate(refs)
+      end
+
+      def reference_populate(refs)
         results = refs.each_with_index.with_object(Queue.new) do |(ref, i), res|
           fetch_ref_async(ref.merge(ord: i), i, res)
         end
-        [refs, results]
-      end
-
-      def reference(node)
-        refs, results = reference_preproc(node)
         ret = reference_queue(refs, results)
         noko do |xml|
           ret.each { |b| reference1out(b, xml) }

data/lib/metanorma/standoc/ref_utility.rb

@@ -2,7 +2,7 @@ module Metanorma
   module Standoc
     module Refs
       def set_date_range(date, text)
-        matched = /^(?<from>[0-9]+)(-+(?<to>[0-9]+))?$/.match text
+        matched = /^(?<from>[0-9]+)(?:-+(?<to>[0-9]+))?$/.match text
         return unless matched[:from]
 
         if matched[:to]
@@ -113,10 +113,11 @@ module Metanorma
         { id: match[:anchor], type: "standard" }
       end
 
-      MALFORMED_REF =
-        "no anchor on reference, markup may be malformed: see "\
-        "https://www.metanorma.com/author/topics/document-format/bibliography/ , "\
-        "https://www.metanorma.com/author/iso/topics/markup/#bibliographies".freeze
+      MALFORMED_REF = <<~REF.freeze
+        no anchor on reference, markup may be malformed: see
+        https://www.metanorma.com/author/topics/document-format/bibliography/ ,
+        https://www.metanorma.com/author/iso/topics/markup/#bibliographies
+      REF
 
       def ref_normalise(ref)
         ref.gsub(/&amp;amp;/, "&amp;").gsub(%r{^<em>(.*)</em>}, "\\1")

data/lib/metanorma/standoc/render.rb

@@ -25,6 +25,9 @@ module Metanorma
           sectionsplit: node.attr("sectionsplit"),
           baseassetpath: node.attr("base-asset-path"),
           aligncrosselements: node.attr("align-cross-elements"),
+          tocfigures: @tocfigures,
+          toctables: @toctables,
+          tocrecommendations: @tocrecommendations,
         }
       end
 
@@ -61,12 +64,13 @@ module Metanorma
           bare: node.attr("bare"),
           baseassetpath: node.attr("base-asset-path"),
           aligncrosselements: node.attr("align-cross-elements"),
+          tocfigures: @tocfigures,
+          toctables: @toctables,
+          tocrecommendations: @tocrecommendations,
         }
 
         if fonts_manifest = node.attr(FONTS_MANIFEST)
-          attrs[IsoDoc::XslfoPdfConvert::MN2PDF_OPTIONS] = {
-            IsoDoc::XslfoPdfConvert::MN2PDF_FONT_MANIFEST => fonts_manifest,
-          }
+          attrs[IsoDoc::XslfoPdfConvert::MN2PDF_FONT_MANIFEST] = fonts_manifest
         end
 
         attrs

data/lib/metanorma/standoc/table.rb

@@ -19,9 +19,7 @@ module Metanorma
           xml.table **attr_code(table_attrs(node)) do |xml_table|
             colgroup(node, xml_table)
             table_name(node, xml_table)
-            %i(head body foot).reject do |tblsec|
-              node.rows[tblsec].empty?
-            end
+            %i(head body foot).reject { |tblsec| node.rows[tblsec].empty? }
             table_head_body_and_foot node, xml_table
           end
         end
@@ -50,12 +48,12 @@ module Metanorma
         end
       end
 
-      def table_cell1(cell, thd)
-        thd << if cell.style == :asciidoc
-                 cell.content
-               else
-                 cell.text
-               end
+      def table_cell1(cell)
+        if cell.style == :asciidoc
+          cell.content
+        else
+          cell.text
+        end
       end
 
       def table_cell(node, xml_tr, tblsec)
@@ -65,7 +63,7 @@ module Metanorma
         cell_tag = "td"
         cell_tag = "th" if tblsec == :head || node.style == :header
         xml_tr.send cell_tag, **attr_code(cell_attributes) do |thd|
-          table_cell1(node, thd)
+          thd << table_cell1(node)
         end
       end
 

data/lib/metanorma/standoc/term_lookup_cleanup.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true.
 require "metanorma/standoc/utils"
 
-
 module Metanorma
   module Standoc
     # Intelligent term lookup xml modifier
@@ -110,15 +109,15 @@ module Metanorma
         if type == "term" || ((!type || node.parent.name == "related") && t)
           node["target"] = t
         elsif type == "symbol" ||
-          ((!type || node.parent.name == "related") && s)
+            ((!type || node.parent.name == "related") && s)
           node["target"] = s
         end
       end
 
       def replace_automatic_generated_ids_terms
         r = xmldoc.xpath("//term").each.with_object({}) do |n, res|
-          normalize_id_and_memorize(n, res, "./preferred//name",
-                                    "term")
+          normalize_id_and_memorize(n, res, "./preferred//name", "term")
+          normalize_id_and_memorize(n, res, "./admitted//name", "term")
         end
         s = xmldoc.xpath("//definitions//dt").each.with_object({}) do |n, res|
           normalize_id_and_memorize(n, res, ".", "symbol")
@@ -127,11 +126,13 @@ module Metanorma
       end
 
       def pref_secondary2primary
+        term = ""
         xmldoc.xpath("//term").each.with_object({}) do |n, res|
           n.xpath("./preferred//name").each_with_index do |p, i|
             i.zero? and term = p.text
             i.positive? and res[p.text] = term
           end
+          n.xpath("./admitted//name").each { |p| res[p.text] = term }
         end
       end
 
@@ -141,7 +142,7 @@ module Metanorma
       end
 
       def normalize_id_and_memorize_init(node, res_table, text_selector, prefix)
-        term_text = normalize_ref_id(node.at(text_selector))
+        term_text = normalize_ref_id(node.at(text_selector)) or return
         unless AUTOMATIC_GENERATED_ID_REGEXP.match(node["id"]).nil? &&
             !node["id"].nil?
           id = unique_text_id(term_text, prefix)
@@ -160,9 +161,12 @@ module Metanorma
       end
 
       def normalize_ref_id(term)
+        return nil if term.nil?
+
         t = term.dup
         t.xpath(".//index").map(&:remove)
-        Metanorma::Utils::to_ncname(t.text.downcase.gsub(/[[:space:]]/, "-"))
+        Metanorma::Utils::to_ncname(t.text.strip.downcase
+          .gsub(/[[:space:]]+/, "-"))
       end
 
       def unique_text_id(text, prefix)

data/lib/metanorma/standoc/utils.rb

@@ -32,7 +32,9 @@ module Metanorma
         doc = ::Nokogiri::XML.parse(NOKOHEAD)
         fragment = doc.fragment("")
         ::Nokogiri::XML::Builder.with fragment, &block
-        fragment.to_xml(encoding: "US-ASCII", indent: 0).lines.map do |l|
+        fragment.to_xml(encoding: "US-ASCII", indent: 0,
+                        save_with: Nokogiri::XML::Node::SaveOptions::AS_XML)
+          .lines.map do |l|
           l.gsub(/>\n$/, ">").gsub(/\s*\n$/m, " ").gsub("–", "\u0096")
             .gsub("—", "\u0097").gsub("–", "\u0096")
             .gsub("—", "\u0097")