metanorma-standoc 2.1.1 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2c3089664934dc34b5f41b7e03a7bc2d2b1060e17decafb29dfc1f36fb7bed0
-  data.tar.gz: 7ecc333b2847252e7965be128c701f4cb2241d148179cd43b66a9f5cd2a2e108
+  metadata.gz: 56e2b9105572c28858ffc12d075b16615ec8195836ad4bf7d7d2122fbf81dc05
+  data.tar.gz: 2594f703aeba2f78a5bf3df715709f78b3b6da1a751cb52fedd3ebce299eaf2d
 SHA512:
-  metadata.gz: 5308c4354dc5fd14e1d5d18dc32e7b52d81ba492e53381771e7dfecea4c83af0bd73387af2bfb6c913687352eaad85c44ded592e1c95179d5d41b3974bc923ff
-  data.tar.gz: ba8122c376d231eb6f1f4f4ea235b608b4dd1769394a899fe9fcd6f5bee011f2384cf1455fae0d86d32648e37aee080c0db18755d4a761e793ad5164b500ce61
+  metadata.gz: eeef127fe3ea160b49aa62cbb1aa665fb4e0cdb6d1712f646667394a41633c89864a294900c444abc1fceb7fbe2649bdfb2f539d1ccc66d3df89cd247ea31655
+  data.tar.gz: 2d1ded28e889db67a42275a5890a8e942b2b966e342e7bb7bc2b21129875394dff03ce1c1cd634b409ce72a68de0e4fd4eff518731cb3b8809c70d5c0c5f0449

data/lib/metanorma/standoc/base.rb

@@ -171,7 +171,9 @@ module Metanorma
       end
 
       def doctype(node)
-        node.attr("doctype")&.gsub(/\s+/, "-")&.downcase
+        ret = node.attr("doctype")&.gsub(/\s+/, "-")&.downcase || "standard"
+        ret = "standard" if ret == "article"
+        ret
       end
 
       def front(node, xml)

data/lib/metanorma/standoc/biblio.rng

@@ -614,12 +614,12 @@
     <optional>
       <ref name="fetched"/>
     </optional>
-    <choice>
-      <oneOrMore>
-        <ref name="btitle"/>
-      </oneOrMore>
+    <optional>
       <ref name="formattedref"/>
-    </choice>
+    </optional>
+    <oneOrMore>
+      <ref name="btitle"/>
+    </oneOrMore>
     <zeroOrMore>
       <ref name="bsource"/>
     </zeroOrMore>
@@ -986,36 +986,34 @@
           <ref name="SeriesType"/>
         </attribute>
       </optional>
-      <choice>
+      <optional>
         <ref name="formattedref"/>
-        <group>
-          <ref name="btitle"/>
-          <optional>
-            <ref name="bplace"/>
-          </optional>
-          <optional>
-            <ref name="seriesorganization"/>
-          </optional>
-          <optional>
-            <ref name="abbreviation"/>
-          </optional>
-          <optional>
-            <ref name="seriesfrom"/>
-          </optional>
-          <optional>
-            <ref name="seriesto"/>
-          </optional>
-          <optional>
-            <ref name="seriesnumber"/>
-          </optional>
-          <optional>
-            <ref name="seriespartnumber"/>
-          </optional>
-          <optional>
-            <ref name="seriesrun"/>
-          </optional>
-        </group>
-      </choice>
+      </optional>
+      <ref name="btitle"/>
+      <optional>
+        <ref name="bplace"/>
+      </optional>
+      <optional>
+        <ref name="seriesorganization"/>
+      </optional>
+      <optional>
+        <ref name="abbreviation"/>
+      </optional>
+      <optional>
+        <ref name="seriesfrom"/>
+      </optional>
+      <optional>
+        <ref name="seriesto"/>
+      </optional>
+      <optional>
+        <ref name="seriesnumber"/>
+      </optional>
+      <optional>
+        <ref name="seriespartnumber"/>
+      </optional>
+      <optional>
+        <ref name="seriesrun"/>
+      </optional>
     </element>
   </define>
   <define name="SeriesType">

data/lib/metanorma/standoc/blocks.rb

@@ -80,14 +80,17 @@ module Metanorma
 
       def term_example(node)
         noko do |xml|
-          xml.termexample **attr_code(id_attr(node)) do |ex|
+          xml.termexample **attr_code(id_attr(node)
+            .merge(
+              keepasterm: node.option?("termexample") ? "true" : nil,
+            )) do |ex|
             wrap_in_para(node, ex)
           end
         end.join("\n")
       end
 
       def example(node)
-        return term_example(node) if in_terms?
+        return term_example(node) if in_terms? || node.option?("termexample")
 
         role = node.role || node.attr("style")
         %w(recommendation requirement permission).include?(role) and
@@ -237,10 +240,12 @@ module Metanorma
       end
 
       def pass(node)
+        c = HTMLEntities.new
         noko do |xml|
           xml.passthrough **attr_code(formats:
                                       node.attr("format") || "metanorma") do |p|
-            p << HTMLEntities.new.encode(node.content, :basic, :hexadecimal)
+            p << c.encode(c.decode(node.content),
+                          :basic, :hexadecimal)
           end
         end
       end

data/lib/metanorma/standoc/blocks_notes.rb

@@ -8,6 +8,7 @@ module Metanorma
             number: node.attr("number"),
             subsequence: node.attr("subsequence"),
             "keep-separate": node.attr("keep-separate"),
+            keepasterm: node.option?("termnote") ? "true" : nil,
           )))
       end
 
@@ -61,6 +62,8 @@ module Metanorma
       end
 
       def note(node)
+        return termnote(node) if node.option?("termnote")
+
         noko do |xml|
           xml.note **note_attrs(node) do |c|
             wrap_in_para(node, c)

data/lib/metanorma/standoc/cleanup.rb

@@ -25,6 +25,7 @@ module Metanorma
     module Cleanup
       def cleanup(xmldoc)
         element_name_cleanup(xmldoc)
+        passthrough_cleanup(xmldoc)
         sections_cleanup(xmldoc)
         obligations_cleanup(xmldoc)
         table_cleanup(xmldoc)

data/lib/metanorma/standoc/cleanup_footnotes.rb

@@ -5,12 +5,15 @@ require "json"
 module Metanorma
   module Standoc
     module Cleanup
-      def footnote_content(fn)
-        c = fn.children.respond_to?(:to_xml) ? fn.children.to_xml : fn.children
+      def footnote_content(fnote)
+        c = if fnote.children.respond_to?(:to_xml)
+              fnote.children.to_xml
+            else fn.children
+            end
         c.gsub(/ id="[^"]+"/, "")
       end
 
-      # include footnotes inside figure if they are the only content 
+      # include footnotes inside figure if they are the only content
       # of the paras following
       def figure_footnote_cleanup(xmldoc)
         nomatches = false
@@ -18,7 +21,9 @@ module Metanorma
           q = "//figure/following-sibling::*[1][self::p and *[1][self::fn]]"
           nomatches = true
           xmldoc.xpath(q).each do |s|
-            next if s.children.map { |c| c.text? && /[[:alpha:]]/.match(c.text) }.any?
+            next if s.children.map do |c|
+                      c.text? && /[[:alpha:]]/.match(c.text)
+                    end.any?
 
             s.previous_element << s.first_element_child.remove
             s.remove
@@ -27,16 +32,16 @@ module Metanorma
         end
       end
 
-      def table_footnote_renumber1(fn, i, seen)
-        content = footnote_content(fn)
+      def table_footnote_renumber1(fnote, i, seen)
+        content = footnote_content(fnote)
         if seen[content] then outnum = seen[content]
         else
           i += 1
           outnum = i
           seen[content] = outnum
         end
-        fn["reference"] = (outnum - 1 + "a".ord).chr
-        fn["table"] = true
+        fnote["reference"] = (outnum - 1 + "a".ord).chr
+        fnote["table"] = true
         [i, seen]
       end
 
@@ -84,7 +89,7 @@ module Metanorma
 
       def footnote_block_cleanup(xmldoc)
         xmldoc.xpath("//footnoteblock").each do |f|
-          f.name = 'fn'
+          f.name = "fn"
           if id = xmldoc.at("//*[@id = '#{f.text}']")
             f.children = id.remove.children
           else

data/lib/metanorma/standoc/cleanup_inline.rb

@@ -185,6 +185,23 @@ module Metanorma
         Digest::MD5.hexdigest("#{elem.path}////#{elem.text}")
           .sub(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, "_\\1-\\2-\\3-\\4-\\5")
       end
+
+      def passthrough_cleanup(doc)
+        doc.xpath("//passthrough-inline").each do |p|
+          p.name = "passthrough"
+          p.children = select_odd_chars(p.children.to_xml)
+        end
+        doc.xpath("//identifier").each do |p|
+          p.children = select_odd_chars(p.children.to_xml)
+        end
+      end
+
+      private
+
+      # skip ZWNJ inserted to prevent regexes operating in asciidoctor
+      def select_odd_chars(text)
+        text.gsub(/(?!&)([[:punct:]])\u200c/, "\\1")
+      end
     end
   end
 end

data/lib/metanorma/standoc/cleanup_terms.rb

@@ -93,7 +93,9 @@ module Metanorma
       def termnote_example_cleanup(xmldoc)
         %w(note example).each do |w|
           xmldoc.xpath("//term#{w}[not(ancestor::term)]").each do |x|
-            x.name = w
+            if x["keepasterm"] then x.delete("keepasterm")
+            else x.name = w
+            end
           end
         end
       end

data/lib/metanorma/standoc/cleanup_text.rb

@@ -5,8 +5,11 @@ module Metanorma
         text = result.flatten.map { |l| l.sub(/\s*\Z/, "") } * "\n"
         !@keepasciimath and text = asciimath2mathml(text)
         text = text.gsub(/\s+<fn /, "<fn ")
-        text.gsub(%r{<passthrough\s+formats="metanorma">([^<]*)
-                  </passthrough>}mx) { HTMLEntities.new.decode($1) }
+        %w(passthrough passthrough-inline).each do |v|
+          text.gsub!(%r{<#{v}\s+formats="metanorma">([^<]*)
+                    </#{v}>}mx) { HTMLEntities.new.decode($1) }
+        end
+        text
       end
 
       def smartquotes_cleanup(xmldoc)
@@ -33,7 +36,8 @@ module Metanorma
 
       def uninterrupt_quotes_around_xml_skip(elem)
         !(/\A['"]/.match?(elem.text) &&
-          elem.previous.ancestors("pre, tt, sourcecode, stem, figure, bibdata")
+          elem.previous.ancestors("pre, tt, sourcecode, stem, figure, bibdata,
+                                  passthrough, identifer")
           .empty? &&
           ((elem.previous.text.strip.empty? &&
             !empty_tag_with_text_content?(elem.previous)) ||
@@ -70,7 +74,8 @@ module Metanorma
           empty_tag_with_text_content?(x) and prev = "dummy"
           next unless x.text?
 
-          x.ancestors("pre, tt, sourcecode, stem, figure, bibdata").empty? and
+          x.ancestors("pre, tt, sourcecode, stem, figure, bibdata, passthrough,
+                      identifier").empty? and
             dumb2smart_quotes1(x, prev)
           prev = x.text if x.ancestors("index").empty?
         end

data/lib/metanorma/standoc/converter.rb

@@ -36,6 +36,7 @@ module Metanorma
         inline_macro Metanorma::Standoc::DomainTermInlineMacro
         inline_macro Metanorma::Standoc::InheritInlineMacro
         inline_macro Metanorma::Standoc::HTML5RubyMacro
+        inline_macro Metanorma::Standoc::IdentifierInlineMacro
         inline_macro Metanorma::Standoc::ConceptInlineMacro
         inline_macro Metanorma::Standoc::AutonumberInlineMacro
         inline_macro Metanorma::Standoc::VariantInlineMacro
@@ -63,6 +64,7 @@ module Metanorma
         block Metanorma::Plugin::Lutaml::LutamlDiagramBlock
         block Metanorma::Standoc::PseudocodeBlockMacro
         preprocessor Metanorma::Standoc::EmbedIncludeProcessor
+        preprocessor Metanorma::Standoc::NamedEscapePreprocessor
       end
 
       include ::Asciidoctor::Converter

data/lib/metanorma/standoc/isodoc.rng

@@ -282,6 +282,9 @@
             <ref name="MultilingualRenderingType"/>
           </attribute>
         </optional>
+        <optional>
+          <ref name="tname"/>
+        </optional>
         <oneOrMore>
           <ref name="ul_li"/>
         </oneOrMore>
@@ -324,6 +327,9 @@
             </choice>
           </attribute>
         </optional>
+        <optional>
+          <ref name="tname"/>
+        </optional>
         <oneOrMore>
           <ref name="li"/>
         </oneOrMore>
@@ -360,6 +366,9 @@
             <ref name="MultilingualRenderingType"/>
           </attribute>
         </optional>
+        <optional>
+          <ref name="tname"/>
+        </optional>
         <oneOrMore>
           <ref name="dt"/>
           <ref name="dd"/>

data/lib/metanorma/standoc/lists.rb

@@ -38,9 +38,8 @@ module Metanorma
 
         noko do |xml|
           xml.ul **ul_attrs(node) do |xml_ul|
-            node.items.each do |item|
-              ul_li(xml_ul, item)
-            end
+            list_caption(node, xml_ul)
+            node.items.each { |item| ul_li(xml_ul, item) }
           end
         end.join("\n")
       end
@@ -62,6 +61,7 @@ module Metanorma
       def olist(node)
         noko do |xml|
           xml.ol **ol_attrs(node) do |xml_ol|
+            list_caption(node, xml_ol)
             node.items.each { |item| li(xml_ol, item) }
           end
         end.join("\n")
@@ -98,6 +98,7 @@ module Metanorma
       def dlist(node)
         noko do |xml|
           xml.dl **dl_attrs(node) do |xml_dl|
+            list_caption(node, xml_dl)
             node.items.each do |terms, dd|
               dt(terms, xml_dl)
               dd(dd, xml_dl)
@@ -115,6 +116,10 @@ module Metanorma
           end
         end.join("\n")
       end
+
+      def list_caption(node, out)
+        a = node.title and out.name { |n| n << a }
+      end
     end
   end
 end

data/lib/metanorma/standoc/macros.rb

@@ -191,6 +191,7 @@ module Metanorma
       end
     end
 
+    # inject ZWNJ to prevent Asciidoctor from attempting regex substitutions
     class PassInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
       use_dsl
       named :"pass-format"
@@ -198,7 +199,21 @@ module Metanorma
       def process(parent, target, attrs)
         format = target || "metanorma"
         out = Asciidoctor::Inline.new(parent, :quoted, attrs[1]).convert
-        %{<passthrough formats="#{format}">#{out}</passthrough>}
+          .gsub(/((?![<>&])[[:punct:]])/, "\\1&#x200c;")
+        %{<passthrough-inline formats="#{format}">#{out}</passthrough-inline>}
+      end
+    end
+
+    class IdentifierInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :identifier
+      parse_content_as :raw
+      using_format :short
+
+      def process(parent, _target, attrs)
+        out = Asciidoctor::Inline.new(parent, :quoted, attrs["text"]).convert
+          .gsub(/((?![<>&])[[:punct:]])/, "\\1&#x200c;")
+        %{<identifier>#{out}</identifier>}
       end
     end
 
@@ -213,5 +228,17 @@ module Metanorma
                       type: :xref, target: "_#{UUIDTools::UUID.random_create}")
       end
     end
+
+    class NamedEscapePreprocessor < Asciidoctor::Extensions::Preprocessor
+      def process(_document, reader)
+        c = HTMLEntities.new
+        lines = reader.readlines.map do |l|
+          l.split(/(&[A-Za-z][^;]*;)/).map do |s|
+            /^&[A-Za-z]/.match?(s) ? c.encode(c.decode(s), :hexadecimal) : s
+          end.join
+        end
+        ::Asciidoctor::Reader.new lines
+      end
+    end
   end
 end

data/lib/metanorma/standoc/macros_note.rb

@@ -18,15 +18,10 @@ module Metanorma
         (document.find_by context: :paragraph).each do |para|
           next unless /^TODO: /.match? para.lines[0]
 
-          parent = para.parent
           para.set_attr("name", "todo")
           para.set_attr("caption", "TODO")
           para.lines[0].sub!(/^TODO: /, "")
-          todo = Asciidoctor::Block
-            .new(parent, :admonition, attributes: para.attributes,
-                                      source: para.lines,
-                                      content_model: :compound)
-          parent.blocks[parent.blocks.index(para)] = todo
+          para.context = :admonition
         end
       end
     end

data/lib/metanorma/standoc/version.rb

@@ -19,6 +19,6 @@ module Metanorma
   end
 
   module Standoc
-    VERSION = "2.1.1".freeze
+    VERSION = "2.1.2".freeze
   end
 end

data/metanorma-standoc.gemspec

@@ -46,7 +46,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
   spec.add_development_dependency "guard", "~> 2.14"
   spec.add_development_dependency "guard-rspec", "~> 4.7"
-  spec.add_development_dependency "metanorma-iso"
+  spec.add_development_dependency "metanorma-iso", "~>2.1.2"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.6"
   spec.add_development_dependency "rubocop", "~> 1.5.2"

data/spec/metanorma/base_spec.rb

@@ -6,6 +6,42 @@ RSpec.describe Metanorma::Standoc do
     expect(Metanorma::Standoc::VERSION).not_to be nil
   end
 
+  it "processes named entities" do
+    FileUtils.rm_f "test.doc"
+    input = <<~INPUT
+      = Document title
+      Author
+      :docfile: test.adoc
+      :novalid:
+      :no-pdf:
+
+      Text &times; text
+    INPUT
+    output = <<~OUTPUT
+      <standard-document xmlns='https://www.metanorma.org/ns/standoc' type='semantic' version='#{Metanorma::Standoc::VERSION}'>
+        <bibdata type='standard'>
+          <title language='en' format='text/plain'>Document title</title>
+          <language>en</language>
+          <script>Latn</script>
+          <status>
+            <stage>published</stage>
+          </status>
+          <copyright>
+            <from>#{Time.now.year}</from>
+          </copyright>
+          <ext>
+            <doctype>standard</doctype>
+          </ext>
+        </bibdata>
+        <sections>
+        <p id='_'>Text × text</p>
+        </sections>
+      </standard-document>
+    OUTPUT
+    expect(xmlpp(strip_guid(Asciidoctor.convert(input, *OPTIONS))))
+      .to be_equivalent_to xmlpp(output)
+  end
+
   it "assigns default scripts to major languages" do
     FileUtils.rm_f "test.doc"
     input = <<~INPUT
@@ -29,7 +65,7 @@ RSpec.describe Metanorma::Standoc do
                <from>#{Time.now.year}</from>
              </copyright>
              <ext>
-               <doctype>article</doctype>
+               <doctype>standard</doctype>
              </ext>
            </bibdata>
            <sections> </sections>
@@ -125,7 +161,7 @@ RSpec.describe Metanorma::Standoc do
             </owner>
           </copyright>
           <ext>
-            <doctype>article</doctype>
+            <doctype>standard</doctype>
           </ext>
         </bibdata>
         <sections> </sections>
@@ -468,7 +504,7 @@ RSpec.describe Metanorma::Standoc do
            <keyword>b</keyword>
            <keyword>c</keyword>
            <ext>
-           <doctype>article</doctype>
+           <doctype>standard</doctype>
              <editorialgroup>
                <technical-committee number="1" type="A">TC</technical-committee>
                <technical-committee number="11" type="A1">TC1</technical-committee>
@@ -789,7 +825,7 @@ RSpec.describe Metanorma::Standoc do
             <from>#{Time.now.year}</from>
           </copyright>
           <ext>
-            <doctype>article</doctype>
+            <doctype>standard</doctype>
           </ext>
         </bibdata>
         <sections> </sections>
@@ -924,7 +960,7 @@ RSpec.describe Metanorma::Standoc do
           </bibitem>
         </relation>
         <ext>
-          <doctype>article</doctype>
+          <doctype>standard</doctype>
         </ext>
       </bibdata>
       <sections> </sections>

data/spec/metanorma/blocks_spec.rb

@@ -304,7 +304,7 @@ RSpec.describe Metanorma::Standoc do
            <from>#{Date.today.year}</from>
          </copyright>
          <ext>
-         <doctype>article</doctype>
+         <doctype>standard</doctype>
          </ext>
        </bibdata>
        <sections><p id="foreword">Foreword</p>
@@ -440,6 +440,32 @@ RSpec.describe Metanorma::Standoc do
       .to be_equivalent_to xmlpp(output)
   end
 
+  it "processes term notes outside of terms sections" do
+    input = <<~INPUT
+      #{ASCIIDOC_BLANK_HDR}
+      == Clause
+
+      [NOTE%termnote]
+      ====
+      XYZ
+      ====
+    INPUT
+    output = <<~OUTPUT
+                   #{BLANK_HDR}
+            <sections>
+                <clause id='_' inline-header='false' obligation='normative'>
+      <title>Clause</title>
+      <termnote id='_'>
+        <p id='_'>XYZ</p>
+      </termnote>
+    </clause>
+            </sections>
+            </standard-document>
+    OUTPUT
+    expect(xmlpp(strip_guid(Asciidoctor.convert(input, *OPTIONS))))
+      .to be_equivalent_to xmlpp(output)
+  end
+
   it "processes term notes as plain notes in nonterm clauses" do
     input = <<~INPUT
       #{ASCIIDOC_BLANK_HDR}
@@ -722,6 +748,32 @@ RSpec.describe Metanorma::Standoc do
       .to be_equivalent_to xmlpp(output)
   end
 
+  it "processes term examples outside of terms sections" do
+    input = <<~INPUT
+      #{ASCIIDOC_BLANK_HDR}
+      == Clause
+
+      [%termexample]
+      ====
+      XYZ
+      ====
+    INPUT
+    output = <<~OUTPUT
+                   #{BLANK_HDR}
+            <sections>
+                <clause id='_' inline-header='false' obligation='normative'>
+      <title>Clause</title>
+      <termexample id='_'>
+        <p id='_'>XYZ</p>
+      </termexample>
+    </clause>
+            </sections>
+            </standard-document>
+    OUTPUT
+    expect(xmlpp(strip_guid(Asciidoctor.convert(input, *OPTIONS))))
+      .to be_equivalent_to xmlpp(output)
+  end
+
   it "processes term examples as plain examples in nonterm clauses" do
     input = <<~INPUT
       #{ASCIIDOC_BLANK_HDR}