metanorma-standoc 2.4.0 → 2.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11179b51c3de0cefbd6cd55d2cd3e6945e0d72cb379ec9a32ed792b1239444fd
-  data.tar.gz: e3c6231ecd99e4f631da9173b1c89f10213428c331147b54efa1eccf8833a017
+  metadata.gz: 04c2c98697be18adacdad75abadd4f5d323d6082091a17de0a70f3ffa2488317
+  data.tar.gz: 8824f124552d8e6ac6d38c5330dbd194bd6171bf24f0b870b9b7dff281e6df52
 SHA512:
-  metadata.gz: d000c768270a37dacc417c3d29cab591a708bf7e7b47c51bbded56120c711c6bc82257335a44f34401cfb06d96dd8c64cb410d1664064534147fee3bf7525d85
-  data.tar.gz: 11799198a90c8b67a358365115cebf4594bfd1be0d58cbc193db94fa95096f98cbbd5974fb65f8b6dbc07349d6a76080591ed6703979d448ee768cd8e5f1ceae
+  metadata.gz: cfe8279a23b6c7323927907f5063e2b1dc98b64ad83d742ced62f4bdd6b098bd520cbcaa051319ad0d247efab17d2d989554bce9ba70ba8251fa01e28f384fe5
+  data.tar.gz: 261168bf0efc23682c9e9a1fb8294a0e6ecf3c541a3ed4fcc6fb31a8116c211c967b8b8f4c26c021d8eceb84a284996bcf0952443bef289705229d669c86e819

data/Gemfile

@@ -10,3 +10,4 @@ if File.exist? 'Gemfile.devel'
   eval File.read('Gemfile.devel'), nil, 'Gemfile.devel' # rubocop:disable Security/Eval
 end
 
+

data/lib/metanorma/standoc/cleanup.rb

@@ -31,11 +31,12 @@ module Metanorma
         obligations_cleanup(xmldoc)
         para_index_cleanup(xmldoc)
         block_index_cleanup(xmldoc)
-        table_cleanup(xmldoc)
+        table_cleanup(xmldoc) # feeds: blocksource_cleanup
         formula_cleanup(xmldoc)
         form_cleanup(xmldoc)
         sourcecode_cleanup(xmldoc) # feeds: callout_cleanup
         figure_cleanup(xmldoc)
+        blocksource_cleanup(xmldoc)
         requirement_cleanup(xmldoc) # feeds: xref_cleanup
         element_name_cleanup(xmldoc)
         ref_cleanup(xmldoc) # feeds: bibitem_cleanup

data/lib/metanorma/standoc/cleanup_block.rb

@@ -228,6 +228,13 @@ module Metanorma
                    "Style override set for ordered list")
         end
       end
+
+      def blocksource_cleanup(xmldoc)
+        xmldoc.xpath("//figure//termsource | //table//termsource").each do |s|
+          s.name = "source"
+          s.delete("type")
+        end
+      end
     end
   end
 end

data/lib/metanorma/standoc/cleanup_footnotes.rb

@@ -90,16 +90,28 @@ module Metanorma
       end
 
       def footnote_block_cleanup(xmldoc)
-        xmldoc.xpath("//footnoteblock").each do |f|
+        ids = xmldoc.xpath("//footnoteblock").each_with_object([]) do |f, m|
           f.name = "fn"
+          m << f.text
           if id = xmldoc.at("//*[@id = '#{f.text}']")
-            f.children = id.remove.children
-          else
-            @log.add("Crossreferences", f,
-                     "Could not resolve footnoteblock:[#{f.text}]")
-            f.children = "[ERROR]"
+            f.children = id.dup.children
+          else footnote_block_error(f)
           end
         end
+        footnote_block_remove(xmldoc, ids)
+      end
+
+      def footnote_block_remove(xmldoc, ids)
+        ids.each do |id|
+          n = xmldoc.at("//*[@id = '#{id}']") and
+            n.remove
+        end
+      end
+
+      def footnote_block_error(fnote)
+        @log.add("Crossreferences", fnote,
+                 "Could not resolve footnoteblock:[#{fnote.text}]")
+        fnote.children = "[ERROR]"
       end
 
       def footnote_cleanup(xmldoc)

data/lib/metanorma/standoc/cleanup_maths.rb

@@ -11,10 +11,23 @@ module Metanorma
           .asciimath_to_mathml(text, ["<amathstem>", "</amathstem>"],
                                retain_asciimath: true)
         asciimath2mathml_wrap(text)
+      rescue StandardError => e
+        asciimath2mathml_err(text, e)
+        text
+      end
+
+      def asciimath2mathml_err(text, expr)
+        err = "Malformed MathML: #{expr}\n#{text}"
+        @log.add("Maths", nil, err)
+        @fatalerror << err
+        warn err
       end
 
       def asciimath2mathml_wrap(text)
         x = Nokogiri::XML(text)
+        x.xpath("//*[local-name() = 'math'][@display]").each do |y|
+          y.delete("display")
+        end
         x.xpath("//*[local-name() = 'math'][not(parent::stem)]").each do |y|
           y.wrap("<stem type='MathML'></stem>")
         end
@@ -43,7 +56,9 @@ module Metanorma
       end
 
       def mathml_namespace(stem)
-        stem.xpath("./math").each { |x| x.default_namespace = MATHML_NS }
+        stem.xpath("./*[local-name() = 'math']").each do |x|
+          x.default_namespace = MATHML_NS
+        end
       end
 
       def mathml_mi_italics

data/lib/metanorma/standoc/cleanup_section.rb

@@ -1,7 +1,6 @@
 require "date"
 require "htmlentities"
 require "json"
-require "mathml2asciimath"
 require_relative "cleanup_section_names"
 
 module Metanorma

data/lib/metanorma/standoc/cleanup_symbols.rb

@@ -17,9 +17,9 @@ module Metanorma
       def asciimath_key(sym)
         key = sym.dup
         key.traverse do |n|
-          if n.name == "math"
+          if n.name == "stem" && a = n.at(".//asciimath")
             n.children = @c.encode(
-              @c.decode(grkletters(MathML2AsciiMath.m2a(n.to_xml))), :basic
+              @c.decode(grkletters(a.text)), :basic
             )
           end
         end

data/lib/metanorma/standoc/cleanup_table.rb

@@ -46,16 +46,30 @@ module Metanorma
       def table_cleanup(xmldoc)
         dl1_table_cleanup(xmldoc)
         dl2_table_cleanup(xmldoc)
+        sources_table_cleanup(xmldoc)
         notes_table_cleanup(xmldoc)
         header_rows_cleanup(xmldoc)
       end
 
+      def sources_table_cleanup(xmldoc)
+        nomatches = false
+        until nomatches
+          nomatches = true
+          xmldoc.xpath("//table/following-sibling::*[1]" \
+                       "[self::termsource]").each do |n|
+            n.previous_element << n.remove
+            nomatches = false
+            # will be renamed source from termsource later
+          end
+        end
+      end
+
       # move notes into table
       def notes_table_cleanup(xmldoc)
         nomatches = false
         until nomatches
           nomatches = true
-          xmldoc.xpath("//table/following-sibling::*[1]"\
+          xmldoc.xpath("//table/following-sibling::*[1]" \
                        "[self::note[not(@keep-separate = 'true')]]").each do |n|
             n.delete("keep-separate")
             n.previous_element << n.remove

data/lib/metanorma/standoc/cleanup_terms.rb

@@ -27,7 +27,7 @@ module Metanorma
       end
 
       TERMDEF_BLOCKS =
-        "./p | ./ol | ./dl[not(@metadata = 'true')] | ./ul | ./figure | "\
+        "./p | ./ol | ./dl[not(@metadata = 'true')] | ./ul | ./figure | " \
         "./formula | ./table".freeze
 
       def generate_termdefinitions(xmldoc)
@@ -47,7 +47,7 @@ module Metanorma
           if d.at("./p | ./ol | ./dl | ./ul")
             d.children = "<verbal-definition>#{d.children}</verbal-definition>"
           else
-            d.children = "<non-verbal-representation>"\
+            d.children = "<non-verbal-representation>" \
                          "#{d.children}</non-verbal-representation>"
           end
         end

data/lib/metanorma/standoc/cleanup_terms_designations.rb

@@ -2,7 +2,6 @@ module Metanorma
   module Standoc
     module Cleanup
       def termdef_stem_cleanup(xmldoc)
-        termdef_stem2admitted(xmldoc)
         xmldoc.xpath("//term//expression/name[stem]").each do |n|
           test = n.dup
           test.at("./stem").remove
@@ -12,25 +11,6 @@ module Metanorma
         end
       end
 
-      def termdef_stem2admitted(xmldoc)
-        xmldoc.xpath("//term/p/stem").each do |a|
-          if initial_formula(a.parent)
-            parent = a.parent
-            parent.replace("<admitted>#{term_expr(a.to_xml)}</admitted>")
-          end
-        end
-        xmldoc.xpath("//term/formula").each do |a|
-          initial_formula(a) and
-            a.replace("<admitted>#{term_expr(a.children.to_xml)}</admitted>")
-        end
-      end
-
-      def initial_formula(elem)
-        elem.elements.size == 1 && # para contains just stem expression
-          !elem.at("./preceding-sibling::p | ./preceding-sibling::dl | "\
-                   "./preceding-sibling::ol | ./preceding-sibling::ul")
-      end
-
       # release termdef tags from surrounding paras
       def termdef_unnest_cleanup(xmldoc)
         desgn = "//p/admitted | //p/deprecates | //p/preferred | //p//related"
@@ -136,7 +116,7 @@ module Metanorma
       def dl_to_designation(dlist)
         prev = dlist.previous_element
         unless %w(preferred admitted deprecates related).include? prev&.name
-          @log.add("AsciiDoc Input", dlist, "Metadata definition list does "\
+          @log.add("AsciiDoc Input", dlist, "Metadata definition list does " \
                                             "not follow a term designation")
           return nil
         end

data/lib/metanorma/standoc/inline.rb

@@ -3,7 +3,7 @@ require "unicode2latex"
 require "mime/types"
 require "base64"
 require "English"
-require "latexmath"
+require "plurimath"
 
 module Metanorma
   module Standoc
@@ -140,18 +140,22 @@ module Metanorma
 
       def latex_parse1(text)
         lxm_input = Unicode2LaTeX.unicode2latex(@c.decode(text))
-        results = Latexmath.parse(lxm_input).to_mathml
-        results.nil? and
+        results = Plurimath::Math.parse(lxm_input, "latex").to_mathml
+        if results.nil?
           @log.add("Math", nil,
                    "latexmlmath failed to process equation:\n#{lxm_input}")
-        results&.sub(%r{<math ([^>]+ )?display="block"}, "<math \\1")
+          return
+        end
+        results.sub(%r{<math ([^>]+ )?display="block"}, "<math \\1")
       end
 
       def stem_parse(text, xml, style)
         if /&lt;([^:>&]+:)?math(\s+[^>&]+)?&gt; |
           <([^:>&]+:)?math(\s+[^>&]+)?>/x.match? text
           math = xml_encode(text)
-          xml.stem math, type: "MathML"
+          xml.stem type: "MathML" do |s|
+            s << math
+          end
         elsif style == :latexmath then latex_parse(text, xml)
         else
           xml.stem text&.gsub(/&amp;#/, "&#"), type: "AsciiMath"

data/lib/metanorma/standoc/isodoc.rng

@@ -382,6 +382,9 @@
         <optional>
           <ref name="dl"/>
         </optional>
+        <optional>
+          <ref name="source"/>
+        </optional>
       </element>
     </define>
     <define name="figure">
@@ -404,9 +407,6 @@
           <attribute name="class"/>
         </optional>
         <ref name="BlockAttributes"/>
-        <optional>
-          <ref name="source"/>
-        </optional>
         <optional>
           <ref name="tname"/>
         </optional>
@@ -431,6 +431,20 @@
         <zeroOrMore>
           <ref name="note"/>
         </zeroOrMore>
+        <optional>
+          <ref name="source"/>
+        </optional>
+      </element>
+    </define>
+    <define name="source">
+      <element name="source">
+        <attribute name="status">
+          <ref name="SourceStatusType"/>
+        </attribute>
+        <ref name="origin"/>
+        <optional>
+          <ref name="modification"/>
+        </optional>
       </element>
     </define>
     <define name="sourcecode">
@@ -2099,10 +2113,7 @@
   <define name="termsource">
     <element name="termsource">
       <attribute name="status">
-        <choice>
-          <value>identical</value>
-          <value>modified</value>
-        </choice>
+        <ref name="SourceStatusType"/>
       </attribute>
       <attribute name="type">
         <choice>
@@ -2116,6 +2127,17 @@
       </optional>
     </element>
   </define>
+  <define name="SourceStatusType">
+    <choice>
+      <value>identical</value>
+      <value>modified</value>
+      <value>restyled</value>
+      <value>context-added</value>
+      <value>generalisation</value>
+      <value>specialisation</value>
+      <value>unspecified</value>
+    </choice>
+  </define>
   <define name="origin">
     <element name="origin">
       <choice>

data/lib/metanorma/standoc/utils.rb

@@ -18,7 +18,7 @@ module Metanorma
       end
 
       def noko(&block)
-        Metanorma::Utils::noko(&block)
+        Metanorma::Utils::noko(@script, &block)
       end
 
       def attr_code(attributes)
@@ -68,11 +68,12 @@ module Metanorma
         para
       end
 
-       def xml_encode(text)
+      def xml_encode(text)
         @c.encode(text, :basic, :hexadecimal)
           .gsub(/&amp;gt;/, ">").gsub(/&amp;lt;/, "<").gsub(/&amp;amp;/, "&")
           .gsub(/&gt;/, ">").gsub(/&lt;/, "<").gsub(/&amp;/, "&")
           .gsub(/&quot;/, '"').gsub(/&#xa;/, "\n").gsub(/&amp;#/, "&#")
+          .gsub(/&apos;/, "'")
       end
 
       class EmptyAttr

data/lib/metanorma/standoc/validate.rb

@@ -53,10 +53,39 @@ module Metanorma
         table_validate(doc)
         @fatalerror += requirement_validate(doc)
         image_validate(doc)
+        math_validate(doc)
         @fatalerror.empty? or
           clean_abort(@fatalerror.join("\n"), doc)
       end
 
+      MATHML_NS = "http://www.w3.org/1998/Math/MathML".freeze
+
+      def math_validate(doc)
+        doc.xpath("//m:math", "m" => MATHML_NS).each do |m|
+          math = mathml_sanitise(m.dup)
+          Plurimath::Math.parse(math, "mathml").to_mathml
+        rescue StandardError => e
+          math_validate_error(math, m, e)
+        end
+      end
+
+      def mathml_sanitise(math)
+        math.to_xml(encoding: "US-ASCII").gsub(/ xmlns=["'][^"']+["']/, "")
+          .gsub(%r{<[^:/>]+:}, "<").gsub(%r{</[^:/>]+:}, "</")
+          #.gsub(/&#([^;]+);/) { |x| "&#x#{$1.to_i.to_s(16)};" }
+      end
+
+      def math_validate_error(math, elem, error)
+        a = elem.parent.at("./asciimath")
+        l = elem.parent.at("./latexmath")
+        orig = ""
+        a and orig += "\n\tAsciimath original: #{@c.decode(a.children.to_xml)}"
+        l and orig += "\n\tLatexmath original: #{@c.decode(l.children.to_xml)}"
+        @log.add("Mathematics", elem,
+                 "Invalid MathML: #{math}\n #{error}#{orig}")
+        @fatalerror << "Invalid MathML: #{math}"
+      end
+
       def nested_asset_validate(doc)
         nested_asset_validate_basic(doc)
         nested_note_validate(doc)

data/lib/metanorma/standoc/validate_section.rb

@@ -13,10 +13,16 @@ module Metanorma
         root.xpath("//sourcecode").each do |x|
           callouts = x.elements.select { |e| e.name == "callout" }
           annotations = x.elements.select { |e| e.name == "annotation" }
-          if callouts.size != annotations.size
-            @log.add("AsciiDoc Input", x,
-                     "mismatch of callouts and annotations")
-          end
+          callouts_error(x, callouts, annotations)
+        end
+      end
+
+      def callouts_error(elem, callouts, annotations)
+        if callouts.size != annotations.size && !annotations.empty?
+          err = "mismatch of callouts (#{callouts.size}) and annotations " \
+                "(#{annotations.size})"
+          @log.add("AsciiDoc Input", elem, err)
+          @fatalerror << err
         end
       end
 

data/lib/metanorma/standoc/version.rb

@@ -19,6 +19,6 @@ module Metanorma
   end
 
   module Standoc
-    VERSION = "2.4.0".freeze
+    VERSION = "2.4.2".freeze
   end
 end

data/metanorma-standoc.gemspec

@@ -31,18 +31,17 @@ Gem::Specification.new do |spec|
   spec.add_dependency "asciidoctor", "~> 2.0.0"
   spec.add_dependency "iev", "~> 0.3.0"
   spec.add_dependency "isodoc", "~> 2.5.0"
-  spec.add_dependency "metanorma"
+  spec.add_dependency "metanorma", ">= 1.5.0"
   spec.add_dependency "metanorma-plugin-datastruct", "~> 0.2.0"
   spec.add_dependency "metanorma-plugin-lutaml"
   spec.add_dependency "ruby-jing"
   # relaton-cli not just relaton, to avoid circular reference in metanorma
   spec.add_dependency "asciimath2unitsml", "~> 0.4.0"
   spec.add_dependency "concurrent-ruby"
-  spec.add_dependency "latexmath"
-  spec.add_dependency "mathml2asciimath"
+  spec.add_dependency "plurimath"
   spec.add_dependency "pngcheck"
   spec.add_dependency "relaton-cli", "~> 1.15.0"
-  spec.add_dependency "relaton-iev", "~> 1.1.0"
+  spec.add_dependency "relaton-iev", "~> 1.1.5"
   spec.add_dependency "unicode2latex", "~> 0.0.1"
 
   spec.add_development_dependency "debug"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-standoc
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.4.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-13 00:00:00.000000000 Z
+date: 2023-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.0
 - !ruby/object:Gem::Dependency
   name: metanorma-plugin-datastruct
   requirement: !ruby/object:Gem::Requirement
@@ -137,21 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: latexmath
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: mathml2asciimath
+  name: plurimath
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -198,14 +184,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.1.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 1.1.5
 - !ruby/object:Gem::Dependency
   name: unicode2latex
   requirement: !ruby/object:Gem::Requirement