metanorma-standoc 2.0.2 → 2.0.5

data/lib/metanorma/standoc/ref_utility.rb

@@ -2,7 +2,7 @@ module Metanorma
   module Standoc
     module Refs
       def set_date_range(date, text)
-        matched = /^(?<from>[0-9]+)(-+(?<to>[0-9]+))?$/.match text
+        matched = /^(?<from>[0-9]+)(?:-+(?<to>[0-9]+))?$/.match text
         return unless matched[:from]
 
         if matched[:to]
@@ -75,7 +75,8 @@ module Metanorma
 
       def analyse_ref_repo_path(ret)
         return ret unless m =
-                            /^(?<type>repo|path):\((?<key>[^,]+),?(?<id>.*)\)$/.match(ret[:id])
+                            /^(?<type>repo|path):\((?<key>[^,]+),?(?<id>.*)\)$/
+                              .match(ret[:id])
 
         id = m[:id].empty? ? m[:key].sub(%r{^[^/]+/}, "") : m[:id]
         ret.merge(id: id, type: m[:type], key: m[:key], nofetch: true)
@@ -112,10 +113,11 @@ module Metanorma
         { id: match[:anchor], type: "standard" }
       end
 
-      MALFORMED_REF =
-        "no anchor on reference, markup may be malformed: see "\
-        "https://www.metanorma.com/author/topics/document-format/bibliography/ , "\
-        "https://www.metanorma.com/author/iso/topics/markup/#bibliographies".freeze
+      MALFORMED_REF = <<~REF.freeze
+        no anchor on reference, markup may be malformed: see
+        https://www.metanorma.com/author/topics/document-format/bibliography/ ,
+        https://www.metanorma.com/author/iso/topics/markup/#bibliographies
+      REF
 
       def ref_normalise(ref)
         ref.gsub(/&amp;amp;/, "&amp;").gsub(%r{^<em>(.*)</em>}, "\\1")

data/lib/metanorma/standoc/render.rb

@@ -25,6 +25,9 @@ module Metanorma
           sectionsplit: node.attr("sectionsplit"),
           baseassetpath: node.attr("base-asset-path"),
           aligncrosselements: node.attr("align-cross-elements"),
+          tocfigures: @tocfigures,
+          toctables: @toctables,
+          tocrecommendations: @tocrecommendations,
         }
       end
 
@@ -61,12 +64,13 @@ module Metanorma
           bare: node.attr("bare"),
           baseassetpath: node.attr("base-asset-path"),
           aligncrosselements: node.attr("align-cross-elements"),
+          tocfigures: @tocfigures,
+          toctables: @toctables,
+          tocrecommendations: @tocrecommendations,
         }
 
         if fonts_manifest = node.attr(FONTS_MANIFEST)
-          attrs[IsoDoc::XslfoPdfConvert::MN2PDF_OPTIONS] = {
-            IsoDoc::XslfoPdfConvert::MN2PDF_FONT_MANIFEST => fonts_manifest,
-          }
+          attrs[IsoDoc::XslfoPdfConvert::MN2PDF_FONT_MANIFEST] = fonts_manifest
         end
 
         attrs

data/lib/metanorma/standoc/table.rb

@@ -19,9 +19,7 @@ module Metanorma
           xml.table **attr_code(table_attrs(node)) do |xml_table|
             colgroup(node, xml_table)
             table_name(node, xml_table)
-            %i(head body foot).reject do |tblsec|
-              node.rows[tblsec].empty?
-            end
+            %i(head body foot).reject { |tblsec| node.rows[tblsec].empty? }
             table_head_body_and_foot node, xml_table
           end
         end
@@ -50,12 +48,12 @@ module Metanorma
         end
       end
 
-      def table_cell1(cell, thd)
-        thd << if cell.style == :asciidoc
-                 cell.content
-               else
-                 cell.text
-               end
+      def table_cell1(cell)
+        if cell.style == :asciidoc
+          cell.content
+        else
+          cell.text
+        end
       end
 
       def table_cell(node, xml_tr, tblsec)
@@ -65,7 +63,7 @@ module Metanorma
         cell_tag = "td"
         cell_tag = "th" if tblsec == :head || node.style == :header
         xml_tr.send cell_tag, **attr_code(cell_attributes) do |thd|
-          table_cell1(node, thd)
+          thd << table_cell1(node)
         end
       end
 

data/lib/metanorma/standoc/term_lookup_cleanup.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true.
 require "metanorma/standoc/utils"
 
-
 module Metanorma
   module Standoc
     # Intelligent term lookup xml modifier
@@ -58,7 +57,7 @@ module Metanorma
 
       def set_termxref_tags_target
         xmldoc.xpath("//termxref").each do |node|
-          target = normalize_ref_id(node.text)
+          target = normalize_ref_id(node)
           if termlookup[:term][target].nil? && termlookup[:symbol][target].nil?
             remove_missing_ref(node, target)
             next
@@ -78,7 +77,7 @@ module Metanorma
 
       def remove_missing_ref_term(node, target)
         log.add("AsciiDoc Input", node,
-                %(Error: Term reference in `term[#{target}]` missing: \
+                %(Error: Term reference to `#{target}` missing: \
                 "#{target}" is not defined in document))
         node.name = "strong"
         node&.at("../xrefrender")&.remove
@@ -110,15 +109,15 @@ module Metanorma
         if type == "term" || ((!type || node.parent.name == "related") && t)
           node["target"] = t
         elsif type == "symbol" ||
-          ((!type || node.parent.name == "related") && s)
+            ((!type || node.parent.name == "related") && s)
           node["target"] = s
         end
       end
 
       def replace_automatic_generated_ids_terms
         r = xmldoc.xpath("//term").each.with_object({}) do |n, res|
-          normalize_id_and_memorize(n, res, "./preferred//name",
-                                    "term")
+          normalize_id_and_memorize(n, res, "./preferred//name", "term")
+          normalize_id_and_memorize(n, res, "./admitted//name", "term")
         end
         s = xmldoc.xpath("//definitions//dt").each.with_object({}) do |n, res|
           normalize_id_and_memorize(n, res, ".", "symbol")
@@ -127,11 +126,13 @@ module Metanorma
       end
 
       def pref_secondary2primary
+        term = ""
         xmldoc.xpath("//term").each.with_object({}) do |n, res|
           n.xpath("./preferred//name").each_with_index do |p, i|
             i.zero? and term = p.text
             i.positive? and res[p.text] = term
           end
+          n.xpath("./admitted//name").each { |p| res[p.text] = term }
         end
       end
 
@@ -141,7 +142,7 @@ module Metanorma
       end
 
       def normalize_id_and_memorize_init(node, res_table, text_selector, prefix)
-        term_text = normalize_ref_id(node.at(text_selector).text)
+        term_text = normalize_ref_id(node.at(text_selector)) or return
         unless AUTOMATIC_GENERATED_ID_REGEXP.match(node["id"]).nil? &&
             !node["id"].nil?
           id = unique_text_id(term_text, prefix)
@@ -155,12 +156,17 @@ module Metanorma
         node.xpath(text_selector).each_with_index do |p, i|
           next unless i.positive?
 
-          res_table[normalize_ref_id(p.text)] = node["id"]
+          res_table[normalize_ref_id(p)] = node["id"]
         end
       end
 
-      def normalize_ref_id(text)
-        Metanorma::Utils::to_ncname(text.downcase.gsub(/[[:space:]]/, "-"))
+      def normalize_ref_id(term)
+        return nil if term.nil?
+
+        t = term.dup
+        t.xpath(".//index").map(&:remove)
+        Metanorma::Utils::to_ncname(t.text.strip.downcase
+          .gsub(/[[:space:]]+/, "-"))
       end
 
       def unique_text_id(text, prefix)

data/lib/metanorma/standoc/terms.rb

@@ -92,7 +92,7 @@ module Metanorma
         end
       end
 
-      def term_source_attrs(_node, seen_xref)
+      def termsource_origin_attrs(_node, seen_xref)
         { case: seen_xref.children[0]["case"],
           droploc: seen_xref.children[0]["droploc"],
           bibitemid: seen_xref.children[0]["target"],
@@ -103,7 +103,7 @@ module Metanorma
         if seen_xref.children[0].name == "concept"
           xml_t.origin { |o| o << seen_xref.children[0].to_xml }
         else
-          attrs = term_source_attrs(node, seen_xref)
+          attrs = termsource_origin_attrs(node, seen_xref)
           attrs.delete(:text)
           xml_t.origin **attr_code(attrs) do |o|
             o << seen_xref.children[0].children.to_xml
@@ -135,13 +135,16 @@ module Metanorma
         matched
       end
 
+      def termsource_attrs(node, matched)
+        status = node.attr("status") ||
+          (matched[:text] ? "modified" : "identical")
+        { status: status, type: node.attr("type") || "authoritative" }
+      end
+
       def termsource(node)
-        matched = extract_termsource_refs(node.content, node) || return
+        matched = extract_termsource_refs(node.content, node) or return
         noko do |xml|
-          status = node.attr("status") ||
-            (matched[:text] ? "modified" : "identical")
-          attrs = { status: status, type: node.attr("type") || "authoritative" }
-          xml.termsource **attrs do |xml_t|
+          xml.termsource **termsource_attrs(node, matched) do |xml_t|
             seen_xref = Nokogiri::XML.fragment(matched[:xref])
             add_term_source(node, xml_t, seen_xref, matched)
           end

data/lib/metanorma/standoc/utils.rb

@@ -32,7 +32,9 @@ module Metanorma
         doc = ::Nokogiri::XML.parse(NOKOHEAD)
         fragment = doc.fragment("")
         ::Nokogiri::XML::Builder.with fragment, &block
-        fragment.to_xml(encoding: "US-ASCII", indent: 0).lines.map do |l|
+        fragment.to_xml(encoding: "US-ASCII", indent: 0,
+                        save_with: Nokogiri::XML::Node::SaveOptions::AS_XML)
+          .lines.map do |l|
           l.gsub(/>\n$/, ">").gsub(/\s*\n$/m, " ").gsub("–", "\u0096")
             .gsub("—", "\u0097").gsub("–", "\u0096")
             .gsub("—", "\u0097")

data/lib/metanorma/standoc/validate.rb

@@ -49,9 +49,79 @@ module Metanorma
         iev_validate(doc.root)
         concept_validate(doc, "concept", "refterm")
         concept_validate(doc, "related", "preferred//name")
+        table_validate(doc)
         @fatalerror.empty? or clean_abort(@fatalerror.join("\n"), doc.to_xml)
       end
 
+      def table_validate(doc)
+        doc.xpath("//table[colgroup]").each do |t|
+          maxrowcols_validate(t, t.xpath("./colgroup/col").size)
+        end
+        doc.xpath("//table[.//*[@colspan] | .//*[@rowspan]]").each do |t|
+          maxrowcols_validate(t, max_td_count(t))
+        end
+      end
+
+      def max_td_count(table)
+        max = 0
+        table.xpath(".//tr").each do |tr|
+          n = tr.xpath("./td | ./th").size
+          max < n and max = n
+        end
+        max
+      end
+
+      def maxrowcols_validate(table, maxcols)
+        cells2d = table.xpath(".//tr").each_with_object([]) { |_r, m| m << {} }
+        table.xpath(".//tr").each_with_index do |tr, r|
+          curr = 0
+          tr.xpath("./td | ./th").each do |td|
+            curr = maxcols_validate1(td, r, curr, cells2d, maxcols)
+          end
+        end
+        maxrows_validate(table, cells2d)
+      end
+
+      # code doesn't actually do anything, since Asciidoctor refuses to generate
+      # table with inconsistent column count
+      def maxcols_validate1(tcell, row, curr, cells2d, maxcols)
+        rs = tcell&.attr("rowspan")&.to_i || 1
+        cs = tcell&.attr("colspan")&.to_i || 1
+        curr = table_tracker_update(cells2d, row, curr, rs, cs)
+        maxcols_check(curr + cs - 1, maxcols, tcell)
+        curr + cs
+      end
+
+      def table_tracker_update(cells2d, row, curr, rowspan, colspan)
+        cells2d[row] ||= {}
+        while cells2d[row][curr]
+          curr += 1
+        end
+        (row..(row + rowspan - 1)).each do |y2|
+          cells2d[y2] ||= {}
+          (curr..(curr + colspan - 1)).each { |x2| cells2d[y2][x2] = 1 }
+        end
+        curr
+      end
+
+      def maxrows_validate(table, cells2d)
+        if cells2d.any? { |x| x.size != cells2d.first.size }
+          @log.add("Table", table,
+                   "Table rows in table are inconsistent: check rowspan")
+          @fatalerror << "Table rows in table are inconsistent: check rowspan"
+        end
+      end
+
+      # if maxcols or maxrows negative, do not check them
+      def maxcols_check(col, maxcols, tcell)
+        if maxcols.positive? && col > maxcols
+          @log.add("Table", tcell, "Table exceeds maximum number of columns "\
+                                   "defined (#{maxcols})")
+          @fatalerror << "Table exceeds maximum number of columns defined "\
+                         "(#{maxcols})"
+        end
+      end
+
       def norm_ref_validate(doc)
         found = false
         doc.xpath("//references[@normative = 'true']/bibitem").each do |b|
@@ -71,10 +141,9 @@ module Metanorma
           next if doc.at("//term[@id = '#{x['target']}']")
           next if doc.at("//definitions//dt[@id = '#{x['target']}']")
 
-          ref = x&.at("../#{refterm}")&.text
           @log.add("Anchors", x,
-                   "#{tag.capitalize} #{ref} is pointing to "\
-                   "#{x['target']}, which is not a term or symbol")
+                   "#{tag.capitalize} #{x&.at("../#{refterm}")&.text} is "\
+                   "pointing to #{x['target']}, which is not a term or symbol")
           found = true
         end
         found and
@@ -120,20 +189,25 @@ module Metanorma
         end
       end
 
+      SVG_NS = "http://www.w3.org/2000/svg".freeze
+
+      WILDCARD_ATTRS =
+        "//*[@format] | //stem | //bibdata//description | "\
+        "//formattedref | //bibdata//note | //bibdata/abstract | "\
+        "//bibitem/abstract | //bibitem/note | //misc-container".freeze
+
       # RelaxNG cannot cope well with wildcard attributes. So we strip
       # any attributes from FormattedString instances (which can contain
       # xs:any markup, and are signalled with @format) before validation.
       def formattedstr_strip(doc)
-        doc.xpath("//*[@format] | //stem | //bibdata//description | "\
-                  "//formattedref | //bibdata//note | //bibdata/abstract | "\
-                  "//bibitem/abstract | //bibitem/note | //misc-container")
-          .each do |n|
+        doc.xpath(WILDCARD_ATTRS, "m" => SVG_NS).each do |n|
           n.elements.each do |e|
             e.traverse do |e1|
               e1.element? and e1.each { |k, _v| e1.delete(k) }
             end
           end
         end
+        doc.xpath("//m:svg", "m" => SVG_NS).each { |n| n.replace("<svg/>") }
         doc
       end
 

data/lib/metanorma/standoc/version.rb

@@ -19,6 +19,6 @@ module Metanorma
   end
 
   module Standoc
-    VERSION = "2.0.2".freeze
+    VERSION = "2.0.5".freeze
   end
 end

data/metanorma-standoc.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "asciidoctor", "~> 2.0.0"
   spec.add_dependency "iev", "~> 0.3.0"
   spec.add_dependency "isodoc", "~> 2.0.0"
-  spec.add_dependency "metanorma-plugin-datastruct"
+  spec.add_dependency "metanorma-plugin-datastruct", "~> 0.2.0"
   spec.add_dependency "metanorma-plugin-lutaml"
   spec.add_dependency "ruby-jing"
   # relaton-cli not just relaton, to avoid circular reference in metanorma
@@ -52,6 +52,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "sassc", "2.4.0"
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"
-  spec.add_development_dependency "vcr", "~> 5.0.0"
+  spec.add_development_dependency "vcr", "~> 6.1.0"
   spec.add_development_dependency "webmock"
 end

data/spec/assets/a2.adoc

@@ -1,8 +1,10 @@
-= X
-A
+= A2
+A2
 
 == Clause 2
 
 X
 
 embed::spec/assets/a3.adoc[]
+
+embed::spec/assets/a3a.adoc[]

data/spec/assets/a3.adoc

@@ -1,5 +1,5 @@
-= X
-A
+= A3
+A3
 
 == Clause 3
 

data/spec/assets/a3a.adoc

@@ -0,0 +1,7 @@
+= A3a
+A3a
+
+== Clause 3a
+
+X
+

data/spec/metanorma/base_spec.rb

@@ -1051,7 +1051,7 @@ QU1FOiB0ZXN0Cgo=
       .new(:standoc, header_footer: true)
       .doc_extract_attributes(node)
 
-    expect(options.dig(:mn2pdf, :font_manifest))
+    expect(options[:font_manifest])
       .to eq(node[Metanorma::Standoc::Base::FONTS_MANIFEST])
   end
 

data/spec/metanorma/cleanup_blocks_spec.rb

@@ -1039,4 +1039,140 @@ RSpec.describe Metanorma::Standoc do
     annotation_id = output.at("//xmlns:annotation/@id").text
     expect(callout_id).to eq(annotation_id)
   end
+  it "deduplicates identifiers in inline SVGs" do
+    input = <<~INPUT
+      #{BLANK_HDR}
+        <sections>
+        <svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 256 256">
+         <defs>
+           <linearGradient id="gradient1">
+             <stop class="stop1" offset="0%" xlink:href="#gradient1"/>
+             <stop class="stop2" offset="100%"/>
+             <style>url(#gradient1)</style>
+           </linearGradient>
+         </defs>
+         <circle fill="url(#gradient1)" cx="128" cy="128" r="100" />
+       </svg>
+               <svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 256 256">
+         <defs>
+           <linearGradient id="gradient2">
+             <stop class="stop1" offset="0%" xlink:href="#gradient2"/>
+             <stop class="stop2" offset="100%"/>
+             <style>url(#gradient2)</style>
+           </linearGradient>
+         </defs>
+         <circle fill="url(#gradient2)" cx="128" cy="128" r="100" />
+       </svg>
+             <svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 256 256">
+         <defs>
+           <linearGradient id="gradient1">
+             <stop class="stop1" offset="0%" xlink:href="#gradient1"/>
+             <stop class="stop2" offset="100%"/>
+             <style>url(#gradient1)</style>
+           </linearGradient>
+         </defs>
+         <circle fill="url(#gradient1)" cx="128" cy="128" r="100" />
+       </svg>
+             </sections>
+      </standard-document>
+    INPUT
+    output = <<~OUTPUT
+      #{BLANK_HDR}
+                 <sections>
+                   <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 256 256">
+         <defs>
+           <linearGradient id="gradient1_inject_0">
+             <stop class="stop1" offset="0%" xlink:href="#gradient1_inject_0"/>
+             <stop class="stop2" offset="100%"/>
+             <style>url(#gradient1_inject_0)</style>
+           </linearGradient>
+         </defs>
+         <circle fill="url(#gradient1_inject_0)" cx="128" cy="128" r="100"/>
+       </svg>
+               <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 256 256">
+         <defs>
+           <linearGradient id="gradient2">
+             <stop class="stop1" offset="0%" xlink:href="#gradient2"/>
+             <stop class="stop2" offset="100%"/>
+             <style>url(#gradient2)</style>
+           </linearGradient>
+         </defs>
+         <circle fill="url(#gradient2)" cx="128" cy="128" r="100"/>
+       </svg>
+             <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" viewBox="0 0 256 256">
+         <defs>
+           <linearGradient id="gradient1_inject_2">
+             <stop class="stop1" offset="0%" xlink:href="#gradient1_inject_2"/>
+             <stop class="stop2" offset="100%"/>
+             <style>url(#gradient1_inject_2)</style>
+           </linearGradient>
+         </defs>
+         <circle fill="url(#gradient1_inject_2)" cx="128" cy="128" r="100"/>
+       </svg>
+             </sections>
+      </standard-document>
+    OUTPUT
+    expect(Metanorma::Standoc::Converter.new(nil, *OPTIONS)
+      .cleanup(Nokogiri::XML(input)).to_xml)
+      .to be_equivalent_to xmlpp(output)
+  end
+
+  it "deduplicates identifiers in embedded SVGs" do
+    input = <<~INPUT
+      #{ASCIIDOC_BLANK_HDR.sub(/:data-uri-image: false/, ':data-uri-image: true')}
+
+      image::spec/fixtures/action_schemaexpg1.svg[]
+
+      image::spec/examples/rice_images/rice_image1.png[]
+
+      image::spec/fixtures/action_schemaexpg1.svg[]
+    INPUT
+
+    output = <<~OUTPUT
+       #{BLANK_HDR}
+                <sections>
+          <figure id='_'>
+            <svg xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' version='1.1' id='Layer_1' x='0px' y='0px' viewBox='0 0 595.28 841.89' style='enable-background:new 0 0 595.28 841.89;' xml:space='preserve'>
+              <style/>
+              <image/>
+              <a xlink:href='mn://action_schema'>
+                <rect x='123.28' y='273.93' class='st0' width='88.05' height='41.84'/>
+              </a>
+              <a xlink:href='mn://basic_attribute_schema'>
+                <rect x='324.69' y='450.52' class='st0' width='132.62' height='40.75'/>
+              </a>
+              <a xlink:href='mn://support_resource_schema'>
+                <rect x='324.69' y='528.36' class='st0' width='148.16' height='40.75'/>
+              </a>
+            </svg>
+          </figure>
+          <figure id='_'>
+              <image/>
+          </figure>
+          <figure id='_'>
+            <svg xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' version='1.1' id='Layer_1_inject_1' x='0px' y='0px' viewBox='0 0 595.28 841.89' style='enable-background:new 0 0 595.28 841.89;' xml:space='preserve'>
+              <style/>
+              <image/>
+              <a xlink:href='mn://action_schema'>
+                <rect x='123.28' y='273.93' class='st0' width='88.05' height='41.84'/>
+              </a>
+              <a xlink:href='mn://basic_attribute_schema'>
+                <rect x='324.69' y='450.52' class='st0' width='132.62' height='40.75'/>
+              </a>
+              <a xlink:href='mn://support_resource_schema'>
+                <rect x='324.69' y='528.36' class='st0' width='148.16' height='40.75'/>
+              </a>
+            </svg>
+          </figure>
+        </sections>
+      </standard-document>
+    OUTPUT
+    xml = Nokogiri::XML(Asciidoctor.convert(input, *OPTIONS))
+    xml.xpath("//*[local-name() = 'image']").each do |x|
+      x.replace("<image/>")
+    end
+    expect(xmlpp(strip_guid(xml.to_xml)
+      .gsub(%r{<style.*?</style>}m, "<style/>")))
+      .to be_equivalent_to xmlpp(output)
+  end
 end