metanorma-standoc 2.0.4 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48ca241528c217b9b563a2d3f6f2e13b52f2fa24e679ca2de431ebde8d10fb98
-  data.tar.gz: f6b70cd60a959376474a35ef4d364f69f9e15fd6582a53571a71cec995d90a01
+  metadata.gz: 219579e1dcc116cb03bfddaa5d11819180d7e5b8b306f499335a39a61c2e8075
+  data.tar.gz: 7d8c958d3ae1e834d5e8a93c8ee3c8ebb4040e4dcee899f1bc66c87e538cdad7
 SHA512:
-  metadata.gz: f175a4b49b2aa24d8c8b566a6929401f1a61fb217cf2ee03b506d93482a086afb7b09c85e7c6d88c93d1ef57d41e5a79dc6122bf41624842118dfa75759823b4
-  data.tar.gz: c395e93a7f1d3c03c63f1063f1689beb3e1725b81d26b2b38769038d491e8fade8462ab1b2f3b6324eede90c7e03b1473ce40873b903fd3859852436d69e7a9a
+  metadata.gz: a4925a6bbee7f29186d5a98cd90a51c647b4443b383f5b09e714554c1bc12da5f88c7c7143236077d96ea3685f2885c4057f245c687a5404d9f6c77489ea8063
+  data.tar.gz: 3644084092782115188b9240f0e8dc68e4ef4b2bb663e5e4270214ab861018ec3a3c4bfd082fbb49f9134af62e714acb309fa774d7ebeaac85a4b20444ef5ef9

data/lib/metanorma/standoc/base.rb

@@ -26,29 +26,40 @@ module Metanorma
       end
 
       def init(node)
+        init_vars
+        init_misc(node)
+        init_processing(node)
+        init_toc(node)
+        init_output(node)
+        init_i18n(node)
+        init_biblio(node)
+        @metadata_attrs = metadata_attrs(node)
+      end
+
+      def init_vars
         @fn_number ||= 0
-        @draft = false
         @refids = Set.new
         @anchors = {}
         @internal_eref_namespaces = []
         @seen_headers = []
+        @embed_hdr = []
+      end
+
+      def init_misc(node)
         @draft = node.attributes.has_key?("draft")
+        @index_terms = node.attr("index-terms")
+        @boilerplateauthority = node.attr("boilerplate-authority")
+        @embed_hdr = node.attr("embed_hdr")
+      end
+
+      def init_processing(node)
         @novalid = node.attr("novalid")
         @smartquotes = node.attr("smartquotes") != "false"
         @keepasciimath = node.attr("mn-keep-asciimath") &&
           node.attr("mn-keep-asciimath") != "false"
-        @index_terms = node.attr("index-terms")
         @sourcecode_markup_start = node.attr("sourcecode-markup-start") || "{{{"
         @sourcecode_markup_end = node.attr("sourcecode-markup-end") || "}}}"
         @datauriimage = node.attr("data-uri-image") != "false"
-        @boilerplateauthority = node.attr("boilerplate-authority")
-        @sourcecode_markup_start = node.attr("sourcecode-markup-start") || "{{{"
-        @sourcecode_markup_end = node.attr("sourcecode-markup-end") || "}}}"
-        init_toc(node)
-        init_output(node)
-        init_i18n(node)
-        init_biblio(node)
-        @metadata_attrs = metadata_attrs(node)
       end
 
       def init_toc(node)

data/lib/metanorma/standoc/blocks.rb

@@ -188,7 +188,7 @@ module Metanorma
 
       def quote_attribution(node, out)
         if node.attr("citetitle")
-          m = /^(?<cite>[^,]+)(,(?<text>.*$))?$/m.match node.attr("citetitle")
+          m = /^(?<cite>[^,]+)(?:,(?<text>.*$))?$/m.match node.attr("citetitle")
           out.source **attr_code(target: m[:cite], type: "inline") do |s|
             s <<  m[:text]
           end

data/lib/metanorma/standoc/cleanup_amend.rb

@@ -1,53 +1,56 @@
 module Metanorma
   module Standoc
     module Cleanup
-      def change_clauses(x)
-        x.xpath("//clause[@change]").each do |c|
-          a = create_amend(c)
+      def change_clauses(docxml)
+        docxml.xpath("//clause[@change]").each do |c|
+          create_amend(c)
         end
       end
 
-      def create_amend(c)
-        a = c.add_child("<amend id='_#{UUIDTools::UUID.random_create}'/>").first
-        c.elements.each do |e|
+      def create_amend(clause)
+        a = clause.add_child("<amend id='_#{UUIDTools::UUID.random_create}'/>")
+          .first
+        clause.elements.each do |e|
           e.parent = a unless %w(amend title).include? e.name
         end
-        create_amend1(c, a)
+        create_amend1(clause, a)
       end
 
-      def create_amend1(c, a)
-        create_amend2(c, a)
-        d = a.at("./description")
+      def create_amend1(clause, amend)
+        create_amend2(clause, amend)
+        d = amend.at("./description")
         d.xpath(".//autonumber").each { |e| d.previous = e }
-        d.xpath(".//p[normalize-space(.)='']").each { |e| e.remove }
-        move_attrs_to_amend(c, a)
-        a
+        d.xpath(".//p[normalize-space(.)='']").each(&:remove)
+        move_attrs_to_amend(clause, amend)
+        amend
       end
 
-      def create_amend2(c, a)
-        q = a.at("./quote") and q.name = "newcontent"
+      def create_amend2(_clause, amend)
+        q = amend.at("./quote") and q.name = "newcontent"
         if q.nil?
-          a.children = "<description>#{a.children.to_xml}</description>"
-        else
-          pre = q&.xpath("./preceding-sibling::*")&.remove
-          post = q&.xpath("./following-sibling::*")&.remove
-          pre.empty? or a << "<description>#{pre.to_xml}</description>"
-          a << q.remove
-          post.empty? or a << "<description>#{post.to_xml}</description>"
+          amend.children = "<description>#{amend.children.to_xml}</description>"
+          return
         end
+        pre = q&.xpath("./preceding-sibling::*")&.remove
+        post = q&.xpath("./following-sibling::*")&.remove
+        pre.empty? or amend << "<description>#{pre.to_xml}</description>"
+        amend << q.remove
+        post.empty? or amend << "<description>#{post.to_xml}</description>"
       end
 
-      def move_attrs_to_amend(c, a)
+      def move_attrs_to_amend(clause, amend)
         %w(change path path_end title).each do |e|
-          next unless c[e]
-          a[e] = c[e]
-          c.delete(e)
+          next unless clause[e]
+
+          amend[e] = clause[e]
+          clause.delete(e)
         end
-        return unless a["locality"]
-        loc = a.children.add_previous_sibling("<location/>")
-        extract_localities1(loc, a["locality"])
+        return unless amend["locality"]
+
+        loc = amend.children.add_previous_sibling("<location/>")
+        extract_localities1(loc, amend["locality"])
         loc1 = loc.at("./localityStack") and loc.replace(loc1.elements)
-        a.delete("locality")
+        amend.delete("locality")
       end
     end
   end

data/lib/metanorma/standoc/cleanup_boilerplate.rb

@@ -133,6 +133,7 @@ module Metanorma
       def bibdata_cleanup(xmldoc)
         bibdata_anchor_cleanup(xmldoc)
         bibdata_docidentifier_cleanup(xmldoc)
+        bibdata_embed_hdr_cleanup(xmldoc)
         biblio_indirect_erefs(xmldoc, @internal_eref_namespaces&.uniq)
       end
 
@@ -190,8 +191,8 @@ module Metanorma
       def resolve_local_indirect_erefs(xmldoc, refs, prefix)
         refs.each_with_object([]) do |r, m|
           id = r.sub(/^#{prefix}_/, "")
-          if n = xmldoc.at("//*[@id = '#{id}']") and
-              n.at("./ancestor-or-self::*[@type = '#{prefix}']")
+          n = xmldoc.at("//*[@id = '#{id}']")
+          if n&.at("./ancestor-or-self::*[@type = '#{prefix}']")
             xmldoc.xpath("//eref[@bibitemid = '#{r}']").each do |e|
               indirect_eref_to_xref(e, id)
             end
@@ -208,6 +209,29 @@ module Metanorma
           insert_indirect_biblio(xmldoc, refs, prefix)
         end
       end
+
+      def bibdata_embed_hdr_cleanup(xmldoc)
+        return if @embed_hdr.nil? || @embed_hdr.empty?
+
+        embed_recurse(xmldoc.at("//bibdata"), @embed_hdr.first)
+      end
+
+      def hdr2bibitem(hdr)
+        xml = Asciidoctor
+          .convert(hdr[:text], backend: Processor.new.asciidoctor_backend,
+                               header_footer: true)
+        b = Nokogiri::XML(xml).at("//xmlns:bibdata")
+        b.name = "bibitem"
+        b.delete("type")
+        embed_recurse(b, hdr)
+        b.to_xml
+      end
+
+      def embed_recurse(bibitem, node)
+        node[:child].map { |x| hdr2bibitem(x) }.each do |x|
+          bibitem << "<relation type='derivedFrom'>#{x}</relation>"
+        end
+      end
     end
   end
 end

data/lib/metanorma/standoc/cleanup_ref_dl.rb

@@ -15,16 +15,21 @@ module Metanorma
         end
       end
 
+      # do not accept implicit id
       def validate_ref_dl(bib, clause)
         id = bib["id"]
-        # do not accept implicit id
         id ||= clause["id"] unless /^_/.match?(clause["id"])
         unless id
           @log.add("Anchors", clause,
-                   "The following reference is missing an anchor:\n#{clause.to_xml}")
+                   "The following reference is missing an anchor:\n"\
+                   "#{clause.to_xml}")
           return
         end
         @refids << id
+        validate_ref_dl1(bib, id, clause)
+      end
+
+      def validate_ref_dl1(bib, id, clause)
         bib["title"] or
           @log.add("Bibliography", clause, "Reference #{id} is missing a title")
         bib["docid"] or

data/lib/metanorma/standoc/cleanup_text.rb

@@ -9,12 +9,6 @@ module Metanorma
                   </passthrough>}mx) { HTMLEntities.new.decode($1) }
       end
 
-      IGNORE_DUMBQUOTES =
-        "//pre | //pre//* | //tt | //tt//* | "\
-        "//sourcecode | //sourcecode//* | //bibdata//* | //stem | "\
-        "//stem//* | //figure[@class = 'pseudocode'] | "\
-        "//figure[@class = 'pseudocode']//*".freeze
-
       def smartquotes_cleanup(xmldoc)
         xmldoc.xpath("//date").each { |d| Metanorma::Utils::endash_date(d) }
         if @smartquotes then smartquotes_cleanup1(xmldoc)
@@ -31,14 +25,21 @@ module Metanorma
       def uninterrupt_quotes_around_xml(xmldoc)
         xmldoc.traverse do |n|
           next unless n.text? && n&.previous&.element?
-          next unless /^['"]/.match?(n.text)
-          next unless n.previous.ancestors("pre, tt, sourcecode, stem, figure")
-            .empty?
+          next if uninterrupt_quotes_around_xml_skip(n)
 
           uninterrupt_quotes_around_xml1(n.previous)
         end
       end
 
+      def uninterrupt_quotes_around_xml_skip(elem)
+        !(/^['"]/.match?(elem.text) &&
+          elem.previous.ancestors("pre, tt, sourcecode, stem, figure, bibdata")
+          .empty? &&
+          ((elem.previous.text.strip.empty? &&
+            !empty_tag_with_text_content?(elem.previous)) ||
+           elem.previous.name == "index"))
+      end
+
       def uninterrupt_quotes_around_xml1(elem)
         prev = elem.at(".//preceding::text()[1]") or return
         /\S$/.match?(prev.text) or return
@@ -49,19 +50,40 @@ module Metanorma
         prev.content = "#{prev.text}#{m[1]}"
       end
 
-      def dumb2smart_quotes(xmldoc)
-        (xmldoc.xpath("//*[child::text()]") - xmldoc.xpath(IGNORE_DUMBQUOTES))
-          .each do |x|
-          x.children.each do |n|
-            next unless n.text?
+      def block?(elem)
+        %w(title name variant-title clause figure annex example introduction
+           foreword acknowledgements note li th td dt dd p quote label
+           abstract preferred admitted related deprecates field-of-application
+           usage-info expression pronunciation grammar-value domain
+           definition termnote termexample modification description
+           newcontent floating-title).include? elem.name
+      end
 
-            /[-'"(<>]|\.\.|\dx/.match(n) or next
+      def empty_tag_with_text_content?(elem)
+        %w(eref xref termref link).include? elem.name
+      end
 
-            n.replace(Metanorma::Utils::smartformat(n.text))
-          end
+      def dumb2smart_quotes(xmldoc)
+        prev = ""
+        xmldoc.traverse do |x|
+          block?(x) and prev = ""
+          empty_tag_with_text_content?(x) and prev = "dummy"
+          next unless x.text?
+
+          x.ancestors("pre, tt, sourcecode, stem, figure, bibdata").empty? and
+            dumb2smart_quotes1(x, prev)
+          prev = x.text if x.ancestors("index").empty?
         end
       end
 
+      def dumb2smart_quotes1(curr, prev)
+        /[-'"(<>]|\.\.|\dx/.match?(curr.text) or return
+
+        /^["']/.match?(curr.text) && prev.match?(/\S$/) and
+          curr.content = curr.text.sub(/^"/, "”").sub(/"’"/, "‘")
+        curr.replace(Metanorma::Utils::smartformat(curr.text))
+      end
+
       def dumbquote_cleanup(xmldoc)
         xmldoc.traverse do |n|
           next unless n.text?

data/lib/metanorma/standoc/converter.rb

@@ -92,7 +92,7 @@ module Metanorma
       end
 
       class << self
-        attr_accessor :_file
+        attr_accessor :_file, :embed_hdr
       end
 
       def self.inherited(konv) # rubocop:disable Lint/MissingSuper

data/lib/metanorma/standoc/macros.rb

@@ -140,7 +140,7 @@ module Metanorma
       parse_content_as :text
 
       def process(parent, target, attrs)
-        /^(?<lang>[^-]*)(-(?<script>.*))?$/ =~ target
+        /^(?<lang>[^-]*)(?:-(?<script>.*))?$/ =~ target
         out = Asciidoctor::Inline.new(parent, :quoted, attrs["text"]).convert
         if script
           %{<variant lang=#{lang} script=#{script}>#{out}</variant>}

data/lib/metanorma/standoc/macros_embed.rb

@@ -5,13 +5,24 @@ module Metanorma
         return reader if reader.eof?
 
         lines = reader.readlines
-        while !lines.grep(/^embed::/).empty?
-          headings = lines.grep(/^== /).map(&:strip)
-          lines = lines.map do |line|
-            /^embed::/.match?(line) ? embed(line, doc, reader, headings) : line
-          end.flatten
+        headings = lines.grep(/^== /).map(&:strip)
+        ret = lines.each_with_object({ lines: [], hdr: [] }) do |line, m|
+          process1(line, m, doc, reader, headings)
         end
-        ::Asciidoctor::Reader.new lines
+        #doc.converter.embed_hdr = ret[:hdr]
+        doc.attributes["embed_hdr"] = ret[:hdr]
+        ::Asciidoctor::Reader.new ret[:lines].flatten
+      end
+
+      def process1(line, acc, doc, reader, headings)
+        if /^embed::/.match?(line)
+          e = embed(line, doc, reader, headings)
+          acc[:lines] << e[:lines]
+          acc[:hdr] << e[:hdr]
+        else
+          acc[:lines] << line
+        end
+        acc
       end
 
       def filename(line, doc, reader)
@@ -27,26 +38,36 @@ module Metanorma
         end
       end
 
+      def read(inc_path)
+        ::File.open inc_path, "r" do |fd|
+          readlines_safe(fd).map(&:chomp)
+        end
+      end
+
       def embed(line, doc, reader, headings)
         inc_path = filename(line, doc, reader) or return line
         lines = filter_sections(read(inc_path), headings)
         doc = Asciidoctor::Document.new [], { safe: :safe }
         reader = ::Asciidoctor::PreprocessorReader.new doc, lines
-        strip_header(reader.read_lines)
+        ret = strip_header(reader.read_lines)
+        embed_recurse(ret, doc, reader, headings)
       end
 
-      def read(inc_path)
-        ::File.open inc_path, "r" do |fd|
-          readlines_safe(fd).map(&:chomp)
+      def embed_recurse(ret, doc, reader, headings)
+        ret1 = ret[:lines].each_with_object({ lines: [], hdr: [] }) do |line, m|
+          process1(line, m, doc, reader, headings)
         end
+        { lines: ret1[:lines],
+          hdr: { text: ret[:hdr].join("\n"), child: ret1[:hdr] } }
       end
 
       def strip_header(lines)
-        return lines unless !lines.empty? && lines.first.start_with?("= ")
+        return { lines: lines, hdr: nil } unless !lines.empty? &&
+          lines.first.start_with?("= ")
 
         skip = true
-        lines.each_with_object([]) do |l, m|
-          m << l unless skip
+        lines.each_with_object({ hdr: [], lines: [] }) do |l, m|
+          m[skip ? :hdr : :lines] << l
           skip = false if !/\S/.match?(l)
         end
       end

data/lib/metanorma/standoc/macros_note.rb

@@ -22,9 +22,10 @@ module Metanorma
           para.set_attr("name", "todo")
           para.set_attr("caption", "TODO")
           para.lines[0].sub!(/^TODO: /, "")
-          todo = Asciidoctor::Block.new(parent, :admonition, attributes: para.attributes,
-                                                             source: para.lines,
-                                                             content_model: :compound)
+          todo = Asciidoctor::Block
+            .new(parent, :admonition, attributes: para.attributes,
+                                      source: para.lines,
+                                      content_model: :compound)
           parent.blocks[parent.blocks.index(para)] = todo
         end
       end

data/lib/metanorma/standoc/processor.rb

@@ -3,6 +3,10 @@ require "metanorma/processor"
 module Metanorma
   module Standoc
     class Processor < Metanorma::Processor
+      class << self
+        attr_reader :asciidoctor_backend
+            end
+
       def initialize # rubocop:disable Lint/MissingSuper
         @short = :standoc
         @input_format = :asciidoc
@@ -22,7 +26,8 @@ module Metanorma
       end
 
       def html_path(file)
-        File.join(File.dirname(__FILE__), "..", "..", "isodoc", "html", file)
+        File.join(File.dirname(__FILE__), "..", "..", "isodoc", "html",
+                  file)
       end
 
       def output(isodoc_node, inname, outname, format, options = {})

data/lib/metanorma/standoc/ref.rb

@@ -121,7 +121,7 @@ module Metanorma
           t.formattedref **{ format: "application/x-isodoc+xml" } do |i|
             i << ref_normalise_no_format(match[:text])
           end
-          yr_match = /[:-](?<year>(19|20)[0-9][0-9])\b/.match(code[:id])
+          yr_match = /[:-](?<year>(?:19|20)[0-9][0-9])\b/.match(code[:id])
           refitem_render1(match, code, t)
           docnumber(t, code[:id]) unless /^\d+$|^\(.+\)$/.match?(code[:id])
           conditional_date(t, yr_match || match, false)
@@ -149,7 +149,7 @@ module Metanorma
       end
 
       def refitem1yr(code)
-        yr_match = /[:-](?<year>(19|20)[0-9][0-9])\b/.match(code)
+        yr_match = /[:-](?<year>(?:19|20)[0-9][0-9])\b/.match(code)
         yr_match ? yr_match[:year] : nil
       end
 
@@ -163,20 +163,20 @@ module Metanorma
 
       ISO_REF =
         %r{^<ref\sid="(?<anchor>[^"]+)">
-      \[(?<usrlbl>\([^)]+\))?(?<code>(ISO|IEC)[^0-9]*\s[0-9-]+|IEV)
-      (:(?<year>[0-9][0-9-]+))?\]</ref>,?\s*(?<text>.*)$}xm.freeze
+      \[(?<usrlbl>\([^)]+\))?(?<code>(?:ISO|IEC)[^0-9]*\s[0-9-]+|IEV)
+      (?::(?<year>[0-9][0-9-]+))?\]</ref>,?\s*(?<text>.*)$}xm.freeze
 
       ISO_REF_NO_YEAR =
         %r{^<ref\sid="(?<anchor>[^"]+)">
-      \[(?<usrlbl>\([^)]+\))?(?<code>(ISO|IEC)[^0-9]*\s[0-9-]+):
-      (--|&\#821[12];)\]</ref>,?\s*
-        (<fn[^>]*>\s*<p>(?<fn>[^\]]+)</p>\s*</fn>)?,?\s?(?<text>.*)$}xm
+      \[(?<usrlbl>\([^)]+\))?(?<code>(?:ISO|IEC)[^0-9]*\s[0-9-]+):
+      (?:--|&\#821[12];)\]</ref>,?\s*
+        (?:<fn[^>]*>\s*<p>(?<fn>[^\]]+)</p>\s*</fn>)?,?\s?(?<text>.*)$}xm
           .freeze
 
       ISO_REF_ALL_PARTS =
         %r{^<ref\sid="(?<anchor>[^"]+)">
-      \[(?<usrlbl>\([^)]+\))?(?<code>(ISO|IEC)[^0-9]*\s[0-9]+)
-      (:(?<year>--|&\#821[12];|[0-9][0-9-]+))?\s
+      \[(?<usrlbl>\([^)]+\))?(?<code>(?:ISO|IEC)[^0-9]*\s[0-9]+)
+      (?::(?<year>--|&\#821[12];|[0-9][0-9-]+))?\s
       \(all\sparts\)\]</ref>,?\s*
         (<fn[^>]*>\s*<p>(?<fn>[^\]]+)</p>\s*</fn>,?\s?)?(?<text>.*)$}xm.freeze
 

data/lib/metanorma/standoc/ref_utility.rb

@@ -2,7 +2,7 @@ module Metanorma
   module Standoc
     module Refs
       def set_date_range(date, text)
-        matched = /^(?<from>[0-9]+)(-+(?<to>[0-9]+))?$/.match text
+        matched = /^(?<from>[0-9]+)(?:-+(?<to>[0-9]+))?$/.match text
         return unless matched[:from]
 
         if matched[:to]
@@ -113,10 +113,11 @@ module Metanorma
         { id: match[:anchor], type: "standard" }
       end
 
-      MALFORMED_REF =
-        "no anchor on reference, markup may be malformed: see "\
-        "https://www.metanorma.com/author/topics/document-format/bibliography/ , "\
-        "https://www.metanorma.com/author/iso/topics/markup/#bibliographies".freeze
+      MALFORMED_REF = <<~REF.freeze
+        no anchor on reference, markup may be malformed: see
+        https://www.metanorma.com/author/topics/document-format/bibliography/ ,
+        https://www.metanorma.com/author/iso/topics/markup/#bibliographies
+      REF
 
       def ref_normalise(ref)
         ref.gsub(/&amp;amp;/, "&amp;").gsub(%r{^<em>(.*)</em>}, "\\1")

data/lib/metanorma/standoc/term_lookup_cleanup.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true.
 require "metanorma/standoc/utils"
 
-
 module Metanorma
   module Standoc
     # Intelligent term lookup xml modifier
@@ -110,15 +109,15 @@ module Metanorma
         if type == "term" || ((!type || node.parent.name == "related") && t)
           node["target"] = t
         elsif type == "symbol" ||
-          ((!type || node.parent.name == "related") && s)
+            ((!type || node.parent.name == "related") && s)
           node["target"] = s
         end
       end
 
       def replace_automatic_generated_ids_terms
         r = xmldoc.xpath("//term").each.with_object({}) do |n, res|
-          normalize_id_and_memorize(n, res, "./preferred//name",
-                                    "term")
+          normalize_id_and_memorize(n, res, "./preferred//name", "term")
+          normalize_id_and_memorize(n, res, "./admitted//name", "term")
         end
         s = xmldoc.xpath("//definitions//dt").each.with_object({}) do |n, res|
           normalize_id_and_memorize(n, res, ".", "symbol")
@@ -127,11 +126,13 @@ module Metanorma
       end
 
       def pref_secondary2primary
+        term = ""
         xmldoc.xpath("//term").each.with_object({}) do |n, res|
           n.xpath("./preferred//name").each_with_index do |p, i|
             i.zero? and term = p.text
             i.positive? and res[p.text] = term
           end
+          n.xpath("./admitted//name").each { |p| res[p.text] = term }
         end
       end
 
@@ -141,7 +142,7 @@ module Metanorma
       end
 
       def normalize_id_and_memorize_init(node, res_table, text_selector, prefix)
-        term_text = normalize_ref_id(node.at(text_selector))
+        term_text = normalize_ref_id(node.at(text_selector)) or return
         unless AUTOMATIC_GENERATED_ID_REGEXP.match(node["id"]).nil? &&
             !node["id"].nil?
           id = unique_text_id(term_text, prefix)
@@ -160,6 +161,8 @@ module Metanorma
       end
 
       def normalize_ref_id(term)
+        return nil if term.nil?
+
         t = term.dup
         t.xpath(".//index").map(&:remove)
         Metanorma::Utils::to_ncname(t.text.strip.downcase

data/lib/metanorma/standoc/validate.rb

@@ -49,9 +49,79 @@ module Metanorma
         iev_validate(doc.root)
         concept_validate(doc, "concept", "refterm")
         concept_validate(doc, "related", "preferred//name")
+        table_validate(doc)
         @fatalerror.empty? or clean_abort(@fatalerror.join("\n"), doc.to_xml)
       end
 
+      def table_validate(doc)
+        doc.xpath("//table[colgroup]").each do |t|
+          maxrowcols_validate(t, t.xpath("./colgroup/col").size)
+        end
+        doc.xpath("//table[.//*[@colspan] | .//*[@rowspan]]").each do |t|
+          maxrowcols_validate(t, max_td_count(t))
+        end
+      end
+
+      def max_td_count(table)
+        max = 0
+        table.xpath(".//tr").each do |tr|
+          n = tr.xpath("./td | ./th").size
+          max < n and max = n
+        end
+        max
+      end
+
+      def maxrowcols_validate(table, maxcols)
+        cells2d = table.xpath(".//tr").each_with_object([]) { |_r, m| m << {} }
+        table.xpath(".//tr").each_with_index do |tr, r|
+          curr = 0
+          tr.xpath("./td | ./th").each do |td|
+            curr = maxcols_validate1(td, r, curr, cells2d, maxcols)
+          end
+        end
+        maxrows_validate(table, cells2d)
+      end
+
+      # code doesn't actually do anything, since Asciidoctor refuses to generate
+      # table with inconsistent column count
+      def maxcols_validate1(tcell, row, curr, cells2d, maxcols)
+        rs = tcell&.attr("rowspan")&.to_i || 1
+        cs = tcell&.attr("colspan")&.to_i || 1
+        curr = table_tracker_update(cells2d, row, curr, rs, cs)
+        maxcols_check(curr + cs - 1, maxcols, tcell)
+        curr + cs
+      end
+
+      def table_tracker_update(cells2d, row, curr, rowspan, colspan)
+        cells2d[row] ||= {}
+        while cells2d[row][curr]
+          curr += 1
+        end
+        (row..(row + rowspan - 1)).each do |y2|
+          cells2d[y2] ||= {}
+          (curr..(curr + colspan - 1)).each { |x2| cells2d[y2][x2] = 1 }
+        end
+        curr
+      end
+
+      def maxrows_validate(table, cells2d)
+        if cells2d.any? { |x| x.size != cells2d.first.size }
+          @log.add("Table", table,
+                   "Table rows in table are inconsistent: check rowspan")
+          @fatalerror << "Table rows in table are inconsistent: check rowspan"
+        end
+      end
+
+      # if maxcols or maxrows negative, do not check them
+      def maxcols_check(col, maxcols, tcell)
+        if maxcols.positive? && col > maxcols
+          @log.add("Table", tcell, "Table exceeds maximum number of columns "\
+                                   "defined (#{maxcols})")
+          @fatalerror << "Table exceeds maximum number of columns defined "\
+                         "(#{maxcols})"
+        end
+      end
+
       def norm_ref_validate(doc)
         found = false
         doc.xpath("//references[@normative = 'true']/bibitem").each do |b|
@@ -71,10 +141,9 @@ module Metanorma
           next if doc.at("//term[@id = '#{x['target']}']")
           next if doc.at("//definitions//dt[@id = '#{x['target']}']")
 
-          ref = x&.at("../#{refterm}")&.text
           @log.add("Anchors", x,
-                   "#{tag.capitalize} #{ref} is pointing to "\
-                   "#{x['target']}, which is not a term or symbol")
+                   "#{tag.capitalize} #{x&.at("../#{refterm}")&.text} is "\
+                   "pointing to #{x['target']}, which is not a term or symbol")
           found = true
         end
         found and
@@ -122,23 +191,23 @@ module Metanorma
 
       SVG_NS = "http://www.w3.org/2000/svg".freeze
 
+      WILDCARD_ATTRS =
+        "//*[@format] | //stem | //bibdata//description | "\
+        "//formattedref | //bibdata//note | //bibdata/abstract | "\
+        "//bibitem/abstract | //bibitem/note | //misc-container".freeze
+
       # RelaxNG cannot cope well with wildcard attributes. So we strip
       # any attributes from FormattedString instances (which can contain
       # xs:any markup, and are signalled with @format) before validation.
       def formattedstr_strip(doc)
-        doc.xpath("//*[@format] | //stem | //bibdata//description | "\
-                  "//formattedref | //bibdata//note | //bibdata/abstract | "\
-                  "//bibitem/abstract | //bibitem/note | //misc-container",
-                  "m" => SVG_NS).each do |n|
+        doc.xpath(WILDCARD_ATTRS, "m" => SVG_NS).each do |n|
           n.elements.each do |e|
             e.traverse do |e1|
               e1.element? and e1.each { |k, _v| e1.delete(k) }
             end
           end
         end
-        doc.xpath("//m:svg", "m" => SVG_NS).each do |n|
-          n.replace("<svg/>")
-        end
+        doc.xpath("//m:svg", "m" => SVG_NS).each { |n| n.replace("<svg/>") }
         doc
       end
 

data/lib/metanorma/standoc/version.rb

@@ -19,6 +19,6 @@ module Metanorma
   end
 
   module Standoc
-    VERSION = "2.0.4".freeze
+    VERSION = "2.0.5".freeze
   end
 end

data/metanorma-standoc.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "asciidoctor", "~> 2.0.0"
   spec.add_dependency "iev", "~> 0.3.0"
   spec.add_dependency "isodoc", "~> 2.0.0"
-  spec.add_dependency "metanorma-plugin-datastruct"
+  spec.add_dependency "metanorma-plugin-datastruct", "~> 0.2.0"
   spec.add_dependency "metanorma-plugin-lutaml"
   spec.add_dependency "ruby-jing"
   # relaton-cli not just relaton, to avoid circular reference in metanorma
@@ -52,6 +52,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "sassc", "2.4.0"
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"
-  spec.add_development_dependency "vcr", "~> 5.0.0"
+  spec.add_development_dependency "vcr", "~> 6.1.0"
   spec.add_development_dependency "webmock"
 end

data/spec/assets/a2.adoc

@@ -1,8 +1,10 @@
-= X
-A
+= A2
+A2
 
 == Clause 2
 
 X
 
 embed::spec/assets/a3.adoc[]
+
+embed::spec/assets/a3a.adoc[]

data/spec/assets/a3.adoc

@@ -1,5 +1,5 @@
-= X
-A
+= A3
+A3
 
 == Clause 3
 

data/spec/assets/a3a.adoc

@@ -0,0 +1,7 @@
+= A3a
+A3a
+
+== Clause 3a
+
+X
+

data/spec/metanorma/cleanup_spec.rb

@@ -9,13 +9,21 @@ RSpec.describe Metanorma::Standoc do
       == "Quotation" A's
 
       '24:00:00'.
+
+      _emphasis_ *strong* `monospace` "double quote" 'single quote'
     INPUT
     output = <<~OUTPUT
-             #{BLANK_HDR}
+      #{BLANK_HDR}
                     <sections>
         <clause id="_" inline-header="false" obligation="normative">
         <title>“Quotation” A’s</title>
         <p id='_'>‘24:00:00’.</p>
+        <p id='_'>
+       <em>emphasis</em>
+       <strong>strong</strong>
+       <tt>monospace</tt>
+        “double quote” ‘single quote’
+       </p>
       </clause>
              </sections>
              </standard-document>
@@ -149,6 +157,10 @@ RSpec.describe Metanorma::Standoc do
     input = <<~INPUT
       #{ASCIIDOC_BLANK_HDR}
 
+      "*word*",
+
+      "link:http://example.com[]",
+
       "((ppt))",
 
       "((ppm))", "((ppt))"
@@ -160,16 +172,16 @@ RSpec.describe Metanorma::Standoc do
       ....
     INPUT
     output = <<~OUTPUT
-            #{BLANK_HDR}
-                   <sections>
-                              <p id='_'>
-                 &#8220;ppt&#8221;,
+      #{BLANK_HDR}
+        <sections>
+          <p id='_'>“<strong>word</strong>”,</p>
+           <p id='_'>“<link target='http://example.com'/>”,</p>
+          <p id='_'>&#8220;ppt&#8221;,
                  <index>
                    <primary>ppt</primary>
                  </index>
                </p>
-               <p id='_'>
-                 &#8220;ppm&#8221;,
+          <p id='_'>&#8220;ppm&#8221;,
                  <index>
                    <primary>ppm</primary>
                  </index>
@@ -178,12 +190,11 @@ RSpec.describe Metanorma::Standoc do
                    <primary>ppt</primary>
                  </index>
                </p>
-               <p id='_'>
-        &#8220;ppm
+          <p id='_'>&#8220;ppm
         <index>
           <primary>ppm</primary>
         </index>
-        &#8220;&#160;
+        &#8221;&#160;
       </p>
       <figure id='_'>
         <pre id='_'>((ppm))",</pre>

data/spec/metanorma/cleanup_terms_spec.rb

@@ -3,6 +3,7 @@ require "relaton_iec"
 require "fileutils"
 
 RSpec.describe Metanorma::Standoc do
+=begin
   it "processes term and designation metadata and term sources" do
     input = <<~INPUT
       #{ASCIIDOC_BLANK_HDR}
@@ -180,8 +181,9 @@ RSpec.describe Metanorma::Standoc do
     expect(xmlpp(strip_guid(Asciidoctor.convert(input, *OPTIONS))))
       .to be_equivalent_to xmlpp(output)
   end
-
-  it "permits multiple preferred terms, and treats them as synonyms in concepts" do
+=end
+  it "permits multiple preferred terms and admitted terms, "\
+     "and treats them as synonyms in concepts" do
     input = <<~INPUT
       #{ASCIIDOC_BLANK_HDR}
       == Terms and Definitions
@@ -209,6 +211,9 @@ RSpec.describe Metanorma::Standoc do
       {{First Designation}}
 
       {{Second Designation}}
+
+      {{Third Designation}}
+
     INPUT
     output = <<~OUTPUT
          #{BLANK_HDR}
@@ -279,11 +284,18 @@ RSpec.describe Metanorma::Standoc do
         </p>
         <p id='_'>
           <concept>
-            <refterm>Second Designation</refterm>
+            <refterm>First Designation</refterm>
             <renderterm>Second Designation</renderterm>
             <xref target='term-first-designation'/>
           </concept>
         </p>
+        <p id='_'>
+          <concept>
+            <refterm>First Designation</refterm>
+            <renderterm>Third Designation</renderterm>
+            <xref target='term-first-designation'/>
+          </concept>
+        </p>
       </clause>
          </sections>
          </standard-document>

data/spec/metanorma/macros_spec.rb

@@ -1364,8 +1364,70 @@ RSpec.describe Metanorma::Standoc do
       embed::spec/assets/a1.adoc[]
     INPUT
     output = <<~OUTPUT
-      #{BLANK_HDR}
-               <sections>
+      <standard-document xmlns='https://www.metanorma.org/ns/standoc' type='semantic' version='#{Metanorma::Standoc::VERSION}'>
+         <bibdata type='standard'>
+           <title language='en' format='text/plain'>Document title</title>
+           <language>en</language>
+           <script>Latn</script>
+           <status>
+             <stage>published</stage>
+           </status>
+           <copyright>
+             <from>2022</from>
+           </copyright>
+           <ext>
+             <doctype>article</doctype>
+           </ext>
+           <relation type='derivedFrom'>
+             <bibitem>
+               <title language='en' format='text/plain'>A2</title>
+               <language>en</language>
+               <script>Latn</script>
+               <status>
+                 <stage>published</stage>
+               </status>
+               <copyright>
+                 <from>2022</from>
+               </copyright>
+               <ext>
+                 <doctype>article</doctype>
+               </ext>
+               <relation type='derivedFrom'>
+                 <bibitem>
+                   <title language='en' format='text/plain'>A3</title>
+                   <language>en</language>
+                   <script>Latn</script>
+                   <status>
+                     <stage>published</stage>
+                   </status>
+                   <copyright>
+                     <from>2022</from>
+                   </copyright>
+                   <ext>
+                     <doctype>article</doctype>
+                   </ext>
+                 </bibitem>
+               </relation>
+               <relation type='derivedFrom'>
+                 <bibitem>
+                   <title language='en' format='text/plain'>A3a</title>
+                   <language>en</language>
+                   <script>Latn</script>
+                   <status>
+                     <stage>published</stage>
+                   </status>
+                   <copyright>
+                     <from>2022</from>
+                   </copyright>
+                   <ext>
+                     <doctype>article</doctype>
+                   </ext>
+                 </bibitem>
+               </relation>
+             </bibitem>
+           </relation>
+         </bibdata>
+         <sections>
            <clause id='clause1' inline-header='false' obligation='normative'>
              <title>Clause</title>
            </clause>
@@ -1381,6 +1443,10 @@ RSpec.describe Metanorma::Standoc do
              <title>Clause 3</title>
              <p id='_'>X</p>
            </clause>
+           <clause id='_' inline-header='false' obligation='normative'>
+              <title>Clause 3a</title>
+              <p id='_'>X</p>
+            </clause>
            <clause id='_' inline-header='false' obligation='normative'>
              <title>Clause 4</title>
              <p id='_'>X</p>

data/spec/metanorma/validate_spec.rb

@@ -478,6 +478,114 @@ RSpec.describe Metanorma::Standoc do
     expect(File.exist?("test.xml")).to be false
   end
 
+  it "does not warn and abort if columns and rows not out of bounds" do
+    FileUtils.rm_f "test.xml"
+    FileUtils.rm_f "test.err"
+    begin
+      input = <<~INPUT
+        = Document title
+        Author
+        :docfile: test.adoc
+        :nodoc:
+
+        == Clause
+
+        [cols="1,1,1,1"]
+        |===
+        3.2+| a | a
+
+        | a
+        | a | a | a | a
+        |===
+      INPUT
+      expect { Asciidoctor.convert(input, *OPTIONS) }.not_to raise_error(SystemExit)
+    rescue SystemExit
+    end
+    expect(File.read("test.err"))
+      .not_to include "Table exceeds maximum number of columns defined"
+    expect(File.read("test.err"))
+      .not_to include "Table rows in table are inconsistent: check rowspan"
+  end
+
+  xit "warns and aborts if columns out of bounds against colgroup" do
+    FileUtils.rm_f "test.xml"
+    FileUtils.rm_f "test.err"
+    begin
+      input = <<~INPUT
+        = Document title
+        Author
+        :docfile: test.adoc
+        :nodoc:
+
+        [cols="1,1,1,1"]
+        |===
+        5+| a
+
+        | a 4+| a
+        | a | a |a |a
+        |===
+      INPUT
+      expect { Asciidoctor.convert(input, *OPTIONS) }.to raise_error(SystemExit)
+    rescue SystemExit
+    end
+    expect(File.read("test.err"))
+      .to include "Table exceeds maximum number of columns defined (4)"
+    expect(File.read("test.err"))
+      .not_to include "Table rows in table are inconsistent: check rowspan"
+  end
+
+  xit "warns and aborts if columns out of bounds against cell count per row" do
+    FileUtils.rm_f "test.xml"
+    FileUtils.rm_f "test.err"
+    begin
+      input = <<~INPUT
+        = Document title
+        Author
+        :docfile: test.adoc
+        :nodoc:
+
+        |===
+        2.3+| a | a
+
+        | a | a | a
+        | a | a | a
+        |===
+      INPUT
+      expect { Asciidoctor.convert(input, *OPTIONS) }.to raise_error(SystemExit)
+    rescue SystemExit
+    end
+    expect(File.read("test.err"))
+      .to include "Table exceeds maximum number of columns defined (3)"
+    expect(File.read("test.err"))
+      .not_to include "Table rows in table are inconsistent: check rowspan"
+  end
+
+  it "warns and aborts if rows out of bounds" do
+    FileUtils.rm_f "test.xml"
+    FileUtils.rm_f "test.err"
+    begin
+      input = <<~INPUT
+        = Document title
+        Author
+        :docfile: test.adoc
+        :nodoc:
+
+        |===
+        .4+| a | a | a | a
+
+        | a | a | a
+        | a | a | a
+        |===
+      INPUT
+      expect { Asciidoctor.convert(input, *OPTIONS) }.to raise_error(SystemExit)
+    rescue SystemExit
+    end
+    expect(File.read("test.err"))
+      .not_to include "Table exceeds maximum number of columns defined"
+    expect(File.read("test.err"))
+      .to include "Table rows in table are inconsistent: check rowspan"
+  end
+
   it "err file succesfully created for docfile path" do
     FileUtils.rm_rf "test"
     FileUtils.mkdir_p "test"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-standoc
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-07 00:00:00.000000000 Z
+date: 2022-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -56,16 +56,16 @@ dependencies:
   name: metanorma-plugin-datastruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: metanorma-plugin-lutaml
   requirement: !ruby/object:Gem::Requirement
@@ -352,14 +352,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 6.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 6.1.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -518,6 +518,7 @@ files:
 - spec/assets/a1.adoc
 - spec/assets/a2.adoc
 - spec/assets/a3.adoc
+- spec/assets/a3a.adoc
 - spec/assets/a4.adoc
 - spec/assets/boilerplate.xml
 - spec/assets/codes.yml
@@ -643,7 +644,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.32
+rubygems_version: 3.3.9
 signing_key:
 specification_version: 4
 summary: metanorma-standoc realises standards following the Metanorma standoc model