metanorma-standoc 2.0.1 → 2.0.4

data/lib/metanorma/standoc/cleanup_xref.rb

@@ -3,11 +3,12 @@ module Metanorma
     module Cleanup
       # extending localities to cover ISO referencing
       LOCALITY_REGEX_STR = <<~REGEXP.freeze
-        ^((?<locality>section|clause|part|paragraph|chapter|page|
+        ^(((?<conn>and|or|from|to)!)?
+            (?<locality>section|clause|part|paragraph|chapter|page|
                       table|annex|figure|example|note|formula|list|time|anchor|
                       locality:[^ \\t\\n\\r:,;=]+)(\\s+|=)
-               (?<ref>[^"][^ \\t\\n,:-]*|"[^"]+")
-                 (-(?<to>[^"][^ \\t\\n,:-]*|"[^"]"))?|
+               (?<ref>[^"][^ \\t\\n,:;-]*|"[^"]+")
+                 (-(?<to>[^"][^ \\t\\n,:;-]*|"[^"]"))?|
           (?<locality2>whole|locality:[^ \\t\\n\\r:,;=]+))(?<punct>[,:;]?)\\s*
          (?<text>.*)$
       REGEXP
@@ -30,16 +31,37 @@ module Metanorma
       def extract_localities1(elem, text)
         b = elem.add_child("<localityStack/>").first if LOCALITY_RE.match text
         while (m = LOCALITY_RE.match text)
-          ref = m[:ref] ? "<referenceFrom>#{tq m[:ref]}</referenceFrom>" : ""
-          refto = m[:to] ? "<referenceTo>#{tq m[:to]}</referenceTo>" : ""
-          b.add_child("<locality type='#{locality_label(m)}'>#{ref}#{refto}"\
-                      "</locality>")
+          add_locality(b, m)
           text = m[:text]
           b = elem.add_child("<localityStack/>").first if m[:punct] == ";"
         end
+        fill_in_eref_connectives(elem)
         elem.add_child(text) if text
       end
 
+      def add_locality(stack, match)
+        stack.children.empty? && match[:conn] and
+          stack["connective"] = match[:conn]
+        ref =
+          match[:ref] ? "<referenceFrom>#{tq match[:ref]}</referenceFrom>" : ""
+        refto = match[:to] ? "<referenceTo>#{tq match[:to]}</referenceTo>" : ""
+        stack.add_child("<locality type='#{locality_label(match)}'>#{ref}"\
+                        "#{refto}</locality>")
+      end
+
+      def fill_in_eref_connectives(elem)
+        return if elem.xpath("./localityStack").size < 2
+
+        elem.xpath("./localityStack[not(@connective)]").each do |l|
+          n = l.next_element
+          l["connective"] = if n && n.name == "localityStack" &&
+              n["connective"] == "to"
+                              "from"
+                            else "and"
+                            end
+        end
+      end
+
       def locality_label(match)
         loc = match[:locality] || match[:locality2]
         /^locality:/.match?(loc) ? loc : loc&.downcase
@@ -47,19 +69,66 @@ module Metanorma
 
       def xref_to_eref(elem)
         elem["bibitemid"] = elem["target"]
-        unless elem["citeas"] = @anchors&.dig(elem["target"], :xref)
-          @internal_eref_namespaces.include?(elem["type"]) or
-            @log.add("Crossreferences", elem,
-                     "#{elem['target']} does not have a corresponding "\
-                     "anchor ID in the bibliography!")
+        if ref = @anchors&.dig(elem["target"], :xref)
+          elem["citeas"] = HTMLEntities.new.encode(ref, :hexadecimal)
+        else
+          elem["citeas"] = ""
+          xref_to_eref1(elem)
         end
         elem.delete("target")
         extract_localities(elem) unless elem.children.empty?
       end
 
+      def xref_to_eref1(elem)
+        @internal_eref_namespaces.include?(elem["type"]) or
+          @log.add("Crossreferences", elem,
+                   "#{elem['target']} does not have a corresponding "\
+                   "anchor ID in the bibliography!")
+      end
+
       def xref_cleanup(xmldoc)
+        xref_compound_cleanup(xmldoc)
+        xref_cleanup1(xmldoc)
+        xref_compound_wrapup(xmldoc)
+      end
+
+      def xref_compound_cleanup(xmldoc)
+        xmldoc.xpath("//xref").each do |x|
+          /;/.match?(x["target"]) or next
+          locations = x["target"].split(";")
+          x["target"] = locations.first.sub(/^[^!]*!/, "")
+          xref_compound_cleanup1(x, locations)
+        end
+      end
+
+      def xref_compound_cleanup1(xref, locations)
+        xref.children.empty? and xref.children = "<sentinel/>"
+        xref_parse_compound_locations(locations).reverse.each do |y|
+          xref.children.first.previous =
+            "<xref target='#{y[1]}' connective='#{y[0]}'/>"
+        end
+        xref&.at("./sentinel")&.remove
+      end
+
+      def xref_parse_compound_locations(locations)
+        l = locations.map { |y| y.split("!", 2) }
+        l.map.with_index do |y, i|
+          if y.size == 1
+            y.unshift(l.dig(i + 1, 0) == "to" ? "from" : "and")
+          end
+          y
+        end
+      end
+
+      def xref_compound_wrapup(xmldoc)
+        xmldoc.xpath("//xref//xref").each do |x|
+          x.name = "location"
+        end
+      end
+
+      def xref_cleanup1(xmldoc)
         xmldoc.xpath("//xref").each do |x|
-          /:/.match(x["target"]) and xref_to_internal_eref(x)
+          /:/.match?(x["target"]) and xref_to_internal_eref(x)
           next unless x.name == "xref"
 
           if refid? x["target"]

data/lib/metanorma/standoc/converter.rb

@@ -53,6 +53,7 @@ module Metanorma
         inline_macro Metanorma::Standoc::FormOptionMacro
         inline_macro Metanorma::Standoc::ToCInlineMacro
         inline_macro Metanorma::Standoc::PassInlineMacro
+        inline_macro Metanorma::Standoc::StdLinkInlineMacro
         inline_macro Metanorma::Plugin::Lutaml::LutamlFigureInlineMacro
         inline_macro Metanorma::Plugin::Lutaml::LutamlTableInlineMacro
         block_macro Metanorma::Plugin::Lutaml::LutamlDiagramBlockMacro
@@ -61,6 +62,7 @@ module Metanorma
         block Metanorma::Standoc::PlantUMLBlockMacro
         block Metanorma::Plugin::Lutaml::LutamlDiagramBlock
         block Metanorma::Standoc::PseudocodeBlockMacro
+        preprocessor Metanorma::Standoc::EmbedIncludeProcessor
       end
 
       include ::Asciidoctor::Converter

data/lib/metanorma/standoc/inline.rb

@@ -40,20 +40,29 @@ module Metanorma
       end
 
       def inline_anchor_xref_attrs(node)
-        m = /^(?<drop>droploc%)?(?<case>capital%|lowercase%)?(?<drop2>droploc%)?
-          (?<fn>fn:?\s*)?(?<text>.*)$/x.match node.text
+        m = inline_anchor_xref_match(node)
         t = node.target.gsub(/^#/, "").gsub(%r{(\.xml|\.adoc)(#.*$)}, "\\2")
         m.nil? and return { target: t, type: "inline", text: node.text }
-        droploc = m[:drop].nil? && m[:drop2].nil? ? nil : true
-        f = m[:fn].nil? ? "inline" : "footnote"
-        c = if %i[case fn drop drop2].any? do |x|
-                 !m[x].nil?
-               end
-              m[:text]
-            else node.text
-            end
-        { target: t, type: f, case: m[:case]&.sub(/%$/, ""), droploc: droploc,
-          text: c }
+        { target: t, type: m[:fn].nil? ? "inline" : "footnote",
+          case: m[:case]&.sub(/%$/, ""),
+          droploc: m[:drop].nil? && m[:drop2].nil? ? nil : true,
+          text: inline_anchor_xref_text(m, node),
+          hidden: m[:hidden] }
+      end
+
+      def inline_anchor_xref_match(node)
+        /^(hidden%(?<hidden>[^,]+),?)?
+          (?<drop>droploc%)?(?<case>capital%|lowercase%)?(?<drop2>droploc%)?
+          (?<fn>fn:?\s*)?(?<text>.*)$/x.match node.text
+      end
+
+      def inline_anchor_xref_text(match, node)
+        if %i[case fn drop drop2 hidden].any? do |x|
+             !match[x].nil?
+           end
+          match[:text]
+        else node.text
+        end
       end
 
       def inline_anchor_link(node)

data/lib/metanorma/standoc/isodoc.rng

@@ -32,6 +32,43 @@
         <ref name="DocumentType"/>
       </element>
     </define>
+    <define name="index">
+      <element name="index">
+        <optional>
+          <attribute name="to">
+            <data type="IDREF"/>
+          </attribute>
+        </optional>
+        <element name="primary">
+          <oneOrMore>
+            <choice>
+              <ref name="PureTextElement"/>
+              <ref name="stem"/>
+            </choice>
+          </oneOrMore>
+        </element>
+        <optional>
+          <element name="secondary">
+            <oneOrMore>
+              <choice>
+                <ref name="PureTextElement"/>
+                <ref name="stem"/>
+              </choice>
+            </oneOrMore>
+          </element>
+        </optional>
+        <optional>
+          <element name="tertiary">
+            <oneOrMore>
+              <choice>
+                <ref name="PureTextElement"/>
+                <ref name="stem"/>
+              </choice>
+            </oneOrMore>
+          </element>
+        </optional>
+      </element>
+    </define>
     <define name="bibitem">
       <element name="bibitem">
         <attribute name="id">
@@ -115,9 +152,7 @@
             <data type="boolean"/>
           </attribute>
         </optional>
-        <oneOrMore>
-          <ref name="PureTextElement"/>
-        </oneOrMore>
+        <ref name="XrefBody"/>
       </element>
     </define>
     <define name="erefType">
@@ -151,6 +186,42 @@
         <ref name="PureTextElement"/>
       </oneOrMore>
     </define>
+    <define name="localityStack">
+      <element name="localityStack">
+        <optional>
+          <attribute name="connective">
+            <choice>
+              <value>and</value>
+              <value>or</value>
+              <value>from</value>
+              <value>to</value>
+              <value/>
+            </choice>
+          </attribute>
+        </optional>
+        <zeroOrMore>
+          <ref name="locality"/>
+        </zeroOrMore>
+      </element>
+    </define>
+    <define name="sourceLocalityStack">
+      <element name="sourceLocalityStack">
+        <optional>
+          <attribute name="connective">
+            <choice>
+              <value>and</value>
+              <value>or</value>
+              <value>from</value>
+              <value>to</value>
+              <value/>
+            </choice>
+          </attribute>
+        </optional>
+        <zeroOrMore>
+          <ref name="sourceLocality"/>
+        </zeroOrMore>
+      </element>
+    </define>
     <define name="ul">
       <element name="ul">
         <attribute name="id">
@@ -1027,6 +1098,26 @@
         </zeroOrMore>
       </element>
     </define>
+    <define name="sub">
+      <element name="sub">
+        <zeroOrMore>
+          <choice>
+            <ref name="PureTextElement"/>
+            <ref name="stem"/>
+          </choice>
+        </zeroOrMore>
+      </element>
+    </define>
+    <define name="sup">
+      <element name="sup">
+        <zeroOrMore>
+          <choice>
+            <ref name="PureTextElement"/>
+            <ref name="stem"/>
+          </choice>
+        </zeroOrMore>
+      </element>
+    </define>
     <define name="pagebreak">
       <element name="pagebreak">
         <optional>
@@ -1041,6 +1132,16 @@
     </define>
   </include>
   <!-- end overrides -->
+  <define name="image" combine="choice">
+    <element name="svg">
+      <oneOrMore>
+        <choice>
+          <text/>
+          <ref name="AnyElement"/>
+        </choice>
+      </oneOrMore>
+    </element>
+  </define>
   <define name="MultilingualRenderingType">
     <choice>
       <value>common</value>
@@ -2216,7 +2317,18 @@
           <ref name="MultilingualRenderingType"/>
         </attribute>
       </optional>
-      <ref name="paragraph"/>
+      <oneOrMore>
+        <choice>
+          <ref name="formula"/>
+          <ref name="ul"/>
+          <ref name="ol"/>
+          <ref name="dl"/>
+          <ref name="quote"/>
+          <ref name="sourcecode"/>
+          <ref name="paragraph"/>
+          <ref name="figure"/>
+        </choice>
+      </oneOrMore>
     </element>
   </define>
   <define name="termsource">
@@ -2563,4 +2675,30 @@
       </zeroOrMore>
     </element>
   </define>
+  <define name="XrefBody">
+    <zeroOrMore>
+      <ref name="XrefTarget"/>
+    </zeroOrMore>
+    <oneOrMore>
+      <ref name="PureTextElement"/>
+    </oneOrMore>
+  </define>
+  <define name="XrefTarget">
+    <element name="location">
+      <attribute name="target">
+        <data type="string">
+          <param name="pattern">\i\c*|\c+#\c+</param>
+        </data>
+      </attribute>
+      <attribute name="connective">
+        <choice>
+          <value>and</value>
+          <value>or</value>
+          <value>from</value>
+          <value>to</value>
+          <value/>
+        </choice>
+      </attribute>
+    </element>
+  </define>
 </grammar>

data/lib/metanorma/standoc/macros.rb

@@ -7,6 +7,7 @@ require_relative "./macros_plantuml"
 require_relative "./macros_terms"
 require_relative "./macros_form"
 require_relative "./macros_note"
+require_relative "./macros_embed"
 require_relative "./datamodel/attributes_table_preprocessor"
 require_relative "./datamodel/diagram_preprocessor"
 require "metanorma-plugin-datastruct"
@@ -81,7 +82,7 @@ module Metanorma
             .match(l) && (ignore = !ignore)
           next if l.empty? || l.match(/ \+$/) || /^\[.*\]$/.match?(l) || ignore
           next if i == lines.size - 1 ||
-            i < lines.size - 1 && lines[i + 1].empty?
+            (i < lines.size - 1 && lines[i + 1].empty?)
 
           lines[i] += " +"
         end
@@ -200,5 +201,17 @@ module Metanorma
         %{<passthrough formats="#{format}">#{out}</passthrough>}
       end
     end
+
+    class StdLinkInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :"std-link"
+      parse_content_as :text
+      using_format :short
+
+      def process(parent, _target, attrs)
+        create_anchor(parent, "hidden%#{attrs['text']}",
+                      type: :xref, target: "_#{UUIDTools::UUID.random_create}")
+      end
+    end
   end
 end

data/lib/metanorma/standoc/macros_embed.rb

@@ -0,0 +1,72 @@
+module Metanorma
+  module Standoc
+    class EmbedIncludeProcessor < Asciidoctor::Extensions::Preprocessor
+      def process(doc, reader)
+        return reader if reader.eof?
+
+        lines = reader.readlines
+        while !lines.grep(/^embed::/).empty?
+          headings = lines.grep(/^== /).map(&:strip)
+          lines = lines.map do |line|
+            /^embed::/.match?(line) ? embed(line, doc, reader, headings) : line
+          end.flatten
+        end
+        ::Asciidoctor::Reader.new lines
+      end
+
+      def filename(line, doc, reader)
+        m = /^embed::([^\[]+)\[/.match(line)
+        f = doc.normalize_system_path m[1], reader.dir, nil,
+                                      target_name: "include file"
+        File.exist?(f) ? f : nil
+      end
+
+      def readlines_safe(file)
+        if file.eof? then []
+        else file.readlines
+        end
+      end
+
+      def embed(line, doc, reader, headings)
+        inc_path = filename(line, doc, reader) or return line
+        lines = filter_sections(read(inc_path), headings)
+        doc = Asciidoctor::Document.new [], { safe: :safe }
+        reader = ::Asciidoctor::PreprocessorReader.new doc, lines
+        strip_header(reader.read_lines)
+      end
+
+      def read(inc_path)
+        ::File.open inc_path, "r" do |fd|
+          readlines_safe(fd).map(&:chomp)
+        end
+      end
+
+      def strip_header(lines)
+        return lines unless !lines.empty? && lines.first.start_with?("= ")
+
+        skip = true
+        lines.each_with_object([]) do |l, m|
+          m << l unless skip
+          skip = false if !/\S/.match?(l)
+        end
+      end
+
+      def filter_sections(lines, headings)
+        skip = false
+        lines.each_with_index.with_object([]) do |(l, i), m|
+          if headings.include?(l.strip)
+            skip = true
+            m.pop while !m.empty? && /^\S/.match?(m[-1])
+          elsif skip && /^== |^embed::|^include::/.match?(l)
+            skip = false
+            j = i
+            j -= 1 while j >= 0 && /^\S/.match?(m[j])
+            lines[j..i].each { |n| m << n }
+          else
+            skip or m << l
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/ref.rb

@@ -131,6 +131,7 @@ module Metanorma
       # TODO: alternative where only title is available
       def refitemcode(item, node)
         m = NON_ISO_REF.match(item) and return refitem1code(item, m).compact
+        m = NON_ISO_REF1.match(item) and return refitem1code(item, m).compact
         @log.add("AsciiDoc Input", node, "#{MALFORMED_REF}: #{item}")
         {}
       end
@@ -180,7 +181,11 @@ module Metanorma
         (<fn[^>]*>\s*<p>(?<fn>[^\]]+)</p>\s*</fn>,?\s?)?(?<text>.*)$}xm.freeze
 
       NON_ISO_REF = %r{^<ref\sid="(?<anchor>[^"]+)">
-      \[(?<usrlbl>\([^)]+\))?(?<code>[^\]]+?)\]</ref>,?\s*(?<text>.*)$}xm
+      \[(?<usrlbl>\([^)]+\))?(?<code>.+?)\]</ref>,?\s*(?<text>.*)$}xm
+        .freeze
+
+      NON_ISO_REF1 = %r{^<ref\sid="(?<anchor>[^"]+)">
+      (?<usrlbl>\([^)]+\))?(?<code>.+?)</ref>,?\s*(?<text>.*)$}xm
         .freeze
 
       def reference1_matches(item)
@@ -209,18 +214,17 @@ module Metanorma
         end
       end
 
-      def reference_preproc(node)
+      def reference(node)
         refs = node.items.each_with_object([]) do |b, m|
           m << reference1code(b.text, node)
         end
+        reference_populate(refs)
+      end
+
+      def reference_populate(refs)
         results = refs.each_with_index.with_object(Queue.new) do |(ref, i), res|
           fetch_ref_async(ref.merge(ord: i), i, res)
         end
-        [refs, results]
-      end
-
-      def reference(node)
-        refs, results = reference_preproc(node)
         ret = reference_queue(refs, results)
         noko do |xml|
           ret.each { |b| reference1out(b, xml) }

data/lib/metanorma/standoc/ref_utility.rb

@@ -75,7 +75,8 @@ module Metanorma
 
       def analyse_ref_repo_path(ret)
         return ret unless m =
-                            /^(?<type>repo|path):\((?<key>[^,]+),?(?<id>.*)\)$/.match(ret[:id])
+                            /^(?<type>repo|path):\((?<key>[^,]+),?(?<id>.*)\)$/
+                              .match(ret[:id])
 
         id = m[:id].empty? ? m[:key].sub(%r{^[^/]+/}, "") : m[:id]
         ret.merge(id: id, type: m[:type], key: m[:key], nofetch: true)

data/lib/metanorma/standoc/render.rb

@@ -25,6 +25,9 @@ module Metanorma
           sectionsplit: node.attr("sectionsplit"),
           baseassetpath: node.attr("base-asset-path"),
           aligncrosselements: node.attr("align-cross-elements"),
+          tocfigures: @tocfigures,
+          toctables: @toctables,
+          tocrecommendations: @tocrecommendations,
         }
       end
 
@@ -61,12 +64,13 @@ module Metanorma
           bare: node.attr("bare"),
           baseassetpath: node.attr("base-asset-path"),
           aligncrosselements: node.attr("align-cross-elements"),
+          tocfigures: @tocfigures,
+          toctables: @toctables,
+          tocrecommendations: @tocrecommendations,
         }
 
         if fonts_manifest = node.attr(FONTS_MANIFEST)
-          attrs[IsoDoc::XslfoPdfConvert::MN2PDF_OPTIONS] = {
-            IsoDoc::XslfoPdfConvert::MN2PDF_FONT_MANIFEST => fonts_manifest,
-          }
+          attrs[IsoDoc::XslfoPdfConvert::MN2PDF_FONT_MANIFEST] = fonts_manifest
         end
 
         attrs

data/lib/metanorma/standoc/table.rb

@@ -19,9 +19,7 @@ module Metanorma
           xml.table **attr_code(table_attrs(node)) do |xml_table|
             colgroup(node, xml_table)
             table_name(node, xml_table)
-            %i(head body foot).reject do |tblsec|
-              node.rows[tblsec].empty?
-            end
+            %i(head body foot).reject { |tblsec| node.rows[tblsec].empty? }
             table_head_body_and_foot node, xml_table
           end
         end
@@ -50,12 +48,12 @@ module Metanorma
         end
       end
 
-      def table_cell1(cell, thd)
-        thd << if cell.style == :asciidoc
-                 cell.content
-               else
-                 cell.text
-               end
+      def table_cell1(cell)
+        if cell.style == :asciidoc
+          cell.content
+        else
+          cell.text
+        end
       end
 
       def table_cell(node, xml_tr, tblsec)
@@ -65,7 +63,7 @@ module Metanorma
         cell_tag = "td"
         cell_tag = "th" if tblsec == :head || node.style == :header
         xml_tr.send cell_tag, **attr_code(cell_attributes) do |thd|
-          table_cell1(node, thd)
+          thd << table_cell1(node)
         end
       end
 

data/lib/metanorma/standoc/term_lookup_cleanup.rb

@@ -58,7 +58,7 @@ module Metanorma
 
       def set_termxref_tags_target
         xmldoc.xpath("//termxref").each do |node|
-          target = normalize_ref_id(node.text)
+          target = normalize_ref_id(node)
           if termlookup[:term][target].nil? && termlookup[:symbol][target].nil?
             remove_missing_ref(node, target)
             next
@@ -78,7 +78,7 @@ module Metanorma
 
       def remove_missing_ref_term(node, target)
         log.add("AsciiDoc Input", node,
-                %(Error: Term reference in `term[#{target}]` missing: \
+                %(Error: Term reference to `#{target}` missing: \
                 "#{target}" is not defined in document))
         node.name = "strong"
         node&.at("../xrefrender")&.remove
@@ -141,7 +141,7 @@ module Metanorma
       end
 
       def normalize_id_and_memorize_init(node, res_table, text_selector, prefix)
-        term_text = normalize_ref_id(node.at(text_selector).text)
+        term_text = normalize_ref_id(node.at(text_selector))
         unless AUTOMATIC_GENERATED_ID_REGEXP.match(node["id"]).nil? &&
             !node["id"].nil?
           id = unique_text_id(term_text, prefix)
@@ -155,12 +155,15 @@ module Metanorma
         node.xpath(text_selector).each_with_index do |p, i|
           next unless i.positive?
 
-          res_table[normalize_ref_id(p.text)] = node["id"]
+          res_table[normalize_ref_id(p)] = node["id"]
         end
       end
 
-      def normalize_ref_id(text)
-        Metanorma::Utils::to_ncname(text.downcase.gsub(/[[:space:]]/, "-"))
+      def normalize_ref_id(term)
+        t = term.dup
+        t.xpath(".//index").map(&:remove)
+        Metanorma::Utils::to_ncname(t.text.strip.downcase
+          .gsub(/[[:space:]]+/, "-"))
       end
 
       def unique_text_id(text, prefix)