metanorma-standoc 1.3.24 → 1.3.29

data/lib/asciidoctor/standoc/cleanup_block.rb

@@ -131,10 +131,13 @@ module Asciidoctor
       def note_cleanup(xmldoc)
         q = "//note[following-sibling::*[not(local-name() = 'note')]]"
         xmldoc.xpath(q).each do |n|
+          next if n["keep-separate"] == "true"
           next unless n.ancestors("table").empty?
           prev = n.previous_element || next
           n.parent = prev if ELEMS_ALLOW_NOTES.include? prev.name
         end
+        xmldoc.xpath("//note[@keep-separate]").each { |n| n.delete("keep-separate") }
+        xmldoc.xpath("//termnote[@keep-separate]").each { |n| n.delete("keep-separate") }
       end
 
       def requirement_cleanup(x)

data/lib/asciidoctor/standoc/cleanup_boilerplate.rb

@@ -18,7 +18,6 @@ module Asciidoctor
         source.each do |s|
           @anchors[s["bibitemid"]] or
             @log.add("Crossreferences", nil, "term source #{s['bibitemid']} not referenced")
-                      #warn "term source #{s['bibitemid']} not referenced" 
         end
         if source.empty? && term.nil?
           div.next = @no_terms_boilerplate
@@ -49,8 +48,7 @@ module Asciidoctor
       TERM_CLAUSE = "//sections/terms | "\
         "//sections/clause[descendant::terms]".freeze
 
-      NORM_REF = "//bibliography/references[title = 'Normative References' or "\
-        "title = 'Normative references']".freeze
+      NORM_REF = "//bibliography/references[@normative = 'true']".freeze
 
       def boilerplate_isodoc(xmldoc)
         x = xmldoc.dup

data/lib/asciidoctor/standoc/cleanup_inline.rb

@@ -90,6 +90,9 @@ module Asciidoctor
       end
 
       def origin_cleanup(xmldoc)
+        xmldoc.xpath("//origin/concept[termref]").each do |x|
+          x.replace(x.children)
+        end
         xmldoc.xpath("//origin").each do |x|
           x["citeas"] = @anchors&.dig(x["bibitemid"], :xref) ||
             @log.add("Crossreferences", x,

data/lib/asciidoctor/standoc/cleanup_ref.rb

@@ -5,7 +5,7 @@ module Asciidoctor
   module Standoc
     module Cleanup
       def biblio_reorder(xmldoc)
-        xmldoc.xpath("//references[title = 'Bibliography']").each do |r|
+        xmldoc.xpath("//references[@normative = 'false']").each do |r|
           biblio_reorder1(r)
         end
       end
@@ -49,8 +49,8 @@ module Asciidoctor
       # consecutively, but that standards codes are preserved as is:
       # only numeric references are renumbered
       def biblio_renumber(xmldoc)
-        r = xmldoc.at("//references[title = 'Bibliography'] | "\
-                      "//clause[title = 'Bibliography'][.//bibitem]") or return
+        r = xmldoc.at("//references[@normative = 'false'] | "\
+                      "//clause[.//references[@normative = 'false']]") or return
         r.xpath(".//bibitem[not(ancestor::bibitem)]").each_with_index do |b, i|
           next unless docid = b.at("./docidentifier[@type = 'metanorma']")
           next unless  /^\[\d+\]$/.match(docid.text)
@@ -67,18 +67,23 @@ module Asciidoctor
       end
 
       def normref_cleanup(xmldoc)
-        r = xmldoc.at(NORM_REF) || return
-        #return if r.at("./bibitem[1]/preceding-sibling::*[1][local-name()='title']")
-        preface = r.xpath("./title/following-sibling::*") &
+        r = xmldoc.at(self.class::NORM_REF) || return
+        preface = r.xpath("./title/following-sibling::*") & # intersection
           r.xpath("./bibitem[1]/preceding-sibling::*")
         preface.each { |n| n.remove }
       end
 
       def biblio_cleanup(xmldoc)
         biblio_reorder(xmldoc)
+        biblio_nested(xmldoc)
         biblio_renumber(xmldoc)
+      end
+
+      def biblio_nested(xmldoc)
         xmldoc.xpath("//references[references]").each do |t|
           t.name = "clause"
+          t.xpath("./references").each { |r| r["normative"] = t["normative"] }
+          t.delete("normative")
         end
       end
 
@@ -187,6 +192,10 @@ module Asciidoctor
         end
         bib
       end
+
+      def fetch_termbase(termbase, id)
+        ""
+      end
     end
   end
 end

data/lib/asciidoctor/standoc/cleanup_section.rb

@@ -11,7 +11,7 @@ module Asciidoctor
     module Cleanup
       def make_preface(x, s)
         if x.at("//foreword | //introduction | //acknowledgements | "\
-            "//clause[@preface]")
+            "//*[@preface]")
           preface = s.add_previous_sibling("<preface/>").first
           f = x.at("//foreword") and preface.add_child f.remove
           f = x.at("//introduction") and preface.add_child f.remove
@@ -22,7 +22,7 @@ module Asciidoctor
       end
 
       def move_clauses_into_preface(x, preface)
-        x.xpath("//clause[@preface]").each do |c|
+        x.xpath("//*[@preface]").each do |c|
           c.delete("preface")
           preface.add_child c.remove
         end
@@ -66,9 +66,22 @@ module Asciidoctor
       def sections_order_cleanup(x)
         s = x.at("//sections")
         make_preface(x, s)
+        make_annexes(x)
         make_bibliography(x, s)
         x.xpath("//sections/annex").reverse_each { |r| s.next = r.remove }
       end
+      
+      def make_annexes(x)
+        x.xpath("//*[@annex]").each do |y|
+          y.delete("annex")
+          next if y.name == "annex" || !y.ancestors("annex").empty?
+          y.wrap("<annex/>")
+          y.parent["id"] = "_#{UUIDTools::UUID.random_create}"
+          y.parent["obligation"] = y["obligation"]
+          y.parent["language"] = y["language"]
+          y.parent["script"] = y["script"]
+        end
+      end
 
       def maxlevel(x)
         max = 5
@@ -117,7 +130,7 @@ module Asciidoctor
       end
 
       def obligations_cleanup_inherit(x)
-        x.xpath("//annex | //clause").each do |r|
+        x.xpath("//annex | //clause[not(ancestor::boilerplate)]").each do |r|
           r["obligation"] = "normative" unless r["obligation"]
         end
         x.xpath(Utils::SUBCLAUSE_XPATH).each do |r|
@@ -192,7 +205,16 @@ module Asciidoctor
         end
       end
 
+      def termdef_from_termbase(xmldoc)
+        xmldoc.xpath("//term").each do |x|
+          if c = x.at("./origin/termref") and !x.at("./definition")
+            x.at("./origin").previous = fetch_termbase(c["base"], c.text)
+          end
+        end
+      end
+
       def termdef_cleanup(xmldoc)
+        termdef_from_termbase(xmldoc)
         termdef_unnest_cleanup(xmldoc)
         termdef_stem_cleanup(xmldoc)
         termdomain_cleanup(xmldoc)
@@ -209,15 +231,21 @@ module Asciidoctor
       # Numbers sort *after* letters; we use thorn to force that sort order.
       def symbol_key(x)
         key = x.dup
-        key.xpath("//*[local-name() = 'math']").each do |m|
-          m.replace(MathML2AsciiMath.m2a(m.to_xml))
+        key.traverse do |n|
+          next unless n.name == "math"
+          n.replace(grkletters(MathML2AsciiMath.m2a(n.to_xml)))
         end
-        ret = Nokogiri::XML(MathML2AsciiMath.m2a(key.to_xml))
-        HTMLEntities.new.decode(ret.text).strip.
+        ret = Nokogiri::XML(key.to_xml)
+        HTMLEntities.new.decode(ret.text).
+          gsub(/[\[\]\{\}<>\(\)]/, "").strip.
           gsub(/[[:punct]]|[_^]/, ":\\0").gsub(/`/, "").
           gsub(/[0-9]+/, "þ\\0")
       end
 
+      def grkletters(x)
+        x.gsub(/\b(alpha|beta|gamma|delta|epsilon|zeta|eta|theta|iota|kappa|lambda|mu|nu|xi|omicron|pi|rho|sigma|tau|upsilon|phi|chi|psi|omega)\b/i, "&\\1;")
+      end
+
       def extract_symbols_list(dl)
         dl_out = []
         dl.xpath("./dt | ./dd").each do |dtd|
@@ -233,7 +261,7 @@ module Asciidoctor
       def symbols_cleanup(docxml)
         docxml.xpath("//definitions/dl").each do |dl|
           dl_out = extract_symbols_list(dl)
-          dl_out.sort! { |a, b| a[:key] <=> b[:key] }
+          dl_out.sort! { |a, b| a[:key] <=> b[:key] || a[:dt] <=> b[:dt] }
           dl.children = dl_out.map { |d| d[:dt].to_s + d[:dd].to_s }.join("\n")
         end
         docxml

data/lib/asciidoctor/standoc/front.rb

@@ -78,7 +78,9 @@ module Asciidoctor
 
       def datetypes
         %w{ published accessed created implemented obsoleted
-            confirmed updated issued circulated unchanged received }
+            confirmed updated issued circulated unchanged received 
+            vote-started vote-ended
+        }
       end
 
       def metadata_date(node, xml)
@@ -190,8 +192,8 @@ module Asciidoctor
         ["en"].each do |lang|
           at = { language: lang, format: "text/plain" }
           xml.title **attr_code(at) do |t|
-            t << Utils::asciidoc_sub(node.attr("title") || node.attr("title-en") ||
-                                     node.title)
+            t << (Utils::asciidoc_sub(node.attr("title") || node.attr("title-en")) ||
+                  node.title)
           end
         end
       end

data/lib/asciidoctor/standoc/inline.rb

@@ -3,6 +3,7 @@ require "htmlentities"
 require "unicode2latex"
 require "mime/types"
 require "base64"
+require 'English'
 
 module Asciidoctor
   module Standoc
@@ -106,23 +107,47 @@ module Asciidoctor
           gsub(/"/, '"').gsub(/
/, "\n")
       end
 
+      def latex_run1(lxm_input, cmd)
+        IO.popen(cmd, "r+", external_encoding: "UTF-8") do |io|
+          io.write(lxm_input)
+          io.close_write
+          io.read
+        end
+      end
+
+      def latex_run(lxm_input)
+        results = nil
+        Metanorma::Standoc::Requirements[:latexml].cmd.each_with_index do |cmd, i|
+          warn "Retrying with #{cmd}" if i > 0
+          results = latex_run1(lxm_input, cmd)
+          if $CHILD_STATUS.to_i.zero?
+            warn "Success!" if i > 0
+            break
+          end
+        end
+        $CHILD_STATUS.to_i.zero? ? results : nil
+      end
+
+      def latex_parse(text)
+        lxm_input = Unicode2LaTeX.unicode2latex(HTMLEntities.new.decode(text))
+        results = latex_run(lxm_input)
+        results.nil? and
+          @log.add('Math', nil,
+                   "latexmlmath failed to process equation:\n#{lxm_input}")
+        results
+      end
+
       def stem_parse(text, xml, style)
         if /<([^:>&]+:)?math(\s+[^>&]+)?> |
           <([^:>&]+:)?math(\s+[^>&]+)?>/x.match text
           math = xml_encode(text)
           xml.stem math, **{ type: "MathML" }
         elsif style == :latexmath
-          latex_cmd = Metanorma::Standoc::Requirements[:latexml].cmd
-          latexmlmath_input =
-            Unicode2LaTeX::unicode2latex(HTMLEntities.new.decode(text)).
-            gsub(/'/, '\\').gsub(/\n/, " ")
-          latex = IO.popen(latex_cmd, "r+", external_encoding: "UTF-8") do |io|
-            io.write(latexmlmath_input)
-            io.close_write
-            io.read
-          end
+          latex = latex_parse(text) or return xml.stem **{ type: "MathML" }
           xml.stem **{ type: "MathML" } do |s|
-            s << latex.sub(/<\?[^>]+>/, "")
+            math = Nokogiri::XML.fragment(latex.sub(/<\?[^>]+>/, "")).elements[0]
+            math.delete("alttext")
+            s.parent.children = math
           end
         else
           xml.stem text, **{ type: "AsciiMath" }
@@ -172,14 +197,14 @@ module Asciidoctor
         types = /^data:/.match(uri) ? datauri2mime(uri) : MIME::Types.type_for(uri)
         type = types.first.to_s
         uri = uri.sub(%r{^data:image/\*;}, "data:#{type};")
-        attr_code(src: @datauriimage ? datauri(uri) : uri,
-          id: Utils::anchor_or_uuid,
-          mimetype: type,
-          height: node.attr("height") || "auto",
-          width: node.attr("width") || "auto" ,
-          filename: node.attr("filename"),
-          title: node.attr("titleattr"),
-          alt: node.alt == node.attr("default-alt") ? nil : node.alt)
+        attr_code(src: uri, #@datauriimage ? datauri(uri) : uri,
+                  id: Utils::anchor_or_uuid,
+                  mimetype: type,
+                  height: node.attr("height") || "auto",
+                  width: node.attr("width") || "auto" ,
+                  filename: node.attr("filename"),
+                  title: node.attr("titleattr"),
+                  alt: node.alt == node.attr("default-alt") ? nil : node.alt)
       end
 
       def inline_image(node)

data/lib/asciidoctor/standoc/isodoc.rng

@@ -129,6 +129,9 @@
             </choice>
           </attribute>
         </optional>
+        <attribute name="normative">
+          <data type="boolean"/>
+        </attribute>
         <optional>
           <ref name="section-title"/>
         </optional>
@@ -305,6 +308,21 @@
     </define>
   </include>
   <!-- end overrides -->
+  <define name="TextElement" combine="choice">
+    <ref name="concept"/>
+  </define>
+  <define name="concept">
+    <element name="concept">
+      <optional>
+        <attribute name="term"/>
+      </optional>
+      <choice>
+        <ref name="eref"/>
+        <ref name="xref"/>
+        <ref name="termref"/>
+      </choice>
+    </element>
+  </define>
   <define name="BasicBlock" combine="choice">
     <choice>
       <ref name="requirement"/>
@@ -911,7 +929,10 @@
   </define>
   <define name="origin">
     <element name="origin">
-      <ref name="erefType"/>
+      <choice>
+        <ref name="erefType"/>
+        <ref name="termref"/>
+      </choice>
     </element>
   </define>
   <define name="modification">
@@ -919,6 +940,15 @@
       <ref name="paragraph"/>
     </element>
   </define>
+  <define name="termref">
+    <element name="termref">
+      <attribute name="base"/>
+      <attribute name="target"/>
+      <optional>
+        <text/>
+      </optional>
+    </element>
+  </define>
   <define name="structuredidentifier">
     <element name="structuredidentifier">
       <optional>

data/lib/asciidoctor/standoc/macros.rb

@@ -2,6 +2,7 @@ require "asciidoctor/extensions"
 require "fileutils"
 require "uuidtools"
 require_relative "./macros_plantuml.rb"
+require_relative "./macros_yaml2text.rb"
 
 module Asciidoctor
   module Standoc
@@ -58,7 +59,7 @@ module Asciidoctor
       named :concept
       name_positional_attributes "id", "word", "term"
       #match %r{concept:(?<target>[^\[]*)\[(?<content>|.*?[^\\])\]$}
-      match /\{\{(?<content>|.*?[^\\])\}\}$/
+      match /\{\{(?<content>|.*?[^\\])\}\}/
       using_format :short
 
       # deal with locality attrs and their disruption of positional attrs

data/lib/asciidoctor/standoc/macros_yaml2text.rb

@@ -0,0 +1,142 @@
+require 'ostruct'
+
+module Asciidoctor
+  module Standoc
+    class YamlBlockStruct < OpenStruct
+      def to_a
+        @table.to_h.keys
+      end
+
+      def values
+        @table.to_h.values
+      end
+
+      def each
+        return to_a.each unless block_given?
+
+        to_a.each do |key|
+          yield(key)
+        end
+      end
+    end
+
+    class YamlContextRenderer
+      attr_reader :context_object, :context_name
+
+      def initialize(context_object:, context_name:)
+        @context_object = context_object
+        @context_name = context_name
+      end
+
+      def respond_to_missing?(name)
+        respond_to?(name)
+      end
+
+      def method_missing(name, *_args)
+        return context_object if name.to_s == context_name
+
+        super
+      end
+
+      def render(template)
+        ERB.new(template).result(binding)
+      end
+    end
+
+    class Yaml2TextPreprocessor < Asciidoctor::Extensions::Preprocessor
+      BLOCK_START_REGEXP = /\{(.+?)\.\*,(.+),(.+)\}/.freeze
+      BLOCK_END_REGEXP = /\A\{[A-Z]+\}\z/.freeze
+      # search document for block `yaml2text`
+      #   after that take template from block and read file into this template
+      #   example:
+      #     [yaml2text,foobar.yaml]
+      #     ----
+      #     === {item.name}
+      #     {item.desc}
+      #
+      #     {item.symbol}:: {item.symbol_def}
+      #     ----
+      #
+      #   with content of `foobar.yaml` file equal to:
+      #     - name: spaghetti
+      #       desc: wheat noodles of 9mm diameter
+      #       symbol: SPAG
+      #       symbol_def: the situation is message like spaghetti at a kid's meal
+      #
+      #   will produce:
+      #     === spaghetti
+      #     wheat noodles of 9mm diameter
+      #
+      #     SPAG:: the situation is message like spaghetti at a kid's meal
+      def process(document, reader)
+        input_lines = reader.readlines.to_enum
+        Reader.new(processed_lines(document, input_lines))
+      end
+
+      private
+
+      def processed_lines(document, input_lines)
+        result = []
+        loop do
+          line = input_lines.next
+          if yaml_block_match = line.match(/^\[yaml2text,(.+?),(.+?)\]/)
+            mark = input_lines.next
+            current_yaml_block = []
+            while (yaml_block_line = input_lines.next) != mark
+              current_yaml_block.push(yaml_block_line)
+            end
+            content = nested_open_struct_from_yaml(yaml_block_match[1], document)
+            result.push(*
+              parse_blocks_recursively(lines: current_yaml_block,
+                                       attributes: content,
+                                       context_name: yaml_block_match[2]))
+          else
+            result.push(line)
+          end
+        end
+        result
+      end
+
+      def nested_open_struct_from_yaml(file_path, document)
+        docfile_directory = File.dirname(document.attributes['docfile'] || '.')
+        yaml_file_path = document.path_resolver.system_path(file_path, docfile_directory)
+        content = YAML.safe_load(File.read(yaml_file_path))
+        # Load content as json, then parse with JSON as nested open_struct
+        JSON.parse(content.to_json, object_class: YamlBlockStruct)
+      end
+
+      def parse_blocks_recursively(lines:,
+                                   attributes:,
+                                   context_name:,
+                                   parent_context: nil)
+        lines = lines.to_enum
+        result = []
+        loop do
+          line = lines.next
+          if line.match(BLOCK_START_REGEXP)
+            line.gsub!(BLOCK_START_REGEXP, '<% \1.each.with_index do |\2,index| %>')
+          end
+
+          if line.match(BLOCK_END_REGEXP)
+            line.gsub!(BLOCK_END_REGEXP, '<% end %>')
+          end
+          line = line.gsub(/{(.+?[^}]*)}/, '<%= \1 %>').gsub(/[a-z\.]+\#/, 'index')
+          result.push(line)
+        end
+        result = parse_context_block(context_lines: result,
+                                     context_items: attributes,
+                                     context_name: context_name,
+                                     parent_context: parent_context)
+        result
+      end
+
+      def parse_context_block(context_lines:,
+                              context_items:,
+                              context_name:,
+                              parent_context: nil)
+        renderer = YamlContextRenderer.new(context_object: context_items, context_name: context_name)
+        renderer.render(context_lines.join('\n')).split('\n')
+      end
+    end
+  end
+end