metanorma-standoc 2.3.9 → 2.4.1

data/lib/metanorma/standoc/spans_to_bibitem.rb

@@ -1,3 +1,5 @@
+require_relative "spans_to_bibitem_preprocessing"
+
 module Metanorma
   module Standoc
     module Cleanup
@@ -23,146 +25,6 @@ module Metanorma
           ret
         end
 
-        def extract_spans(bib)
-          bib.xpath("./formattedref//span").each_with_object([]) do |s, m|
-            s.at("./ancestor::span") and next
-            extract_spans1(s, m)
-          end
-        end
-
-        def extract_spans1(span, acc)
-          keys = span["class"].split(".", 2)
-          acc << { key: keys[0], type: keys[1],
-                   val: span.children.to_xml }
-          (span["class"] == "type" and span.remove) or
-            span.replace(span.children)
-        end
-
-        def extract_docid(bib)
-          bib.xpath("./docidentifier").each_with_object([]) do |d, m|
-            m << { key: "docid", type: d["type"], val: d.text }
-            d.remove unless bib.at("./title")
-          end
-        end
-
-        def empty_span_hash
-          { contrib: [], docid: [], uri: [], date: [], extent: {}, in: {} }
-        end
-
-        def spans_preprocess(spans)
-          ret = empty_span_hash
-          spans.each { |s| span_preprocess1(s, ret) }
-          host_rearrange(ret)
-        end
-
-        def span_preprocess1(span, ret)
-          case span[:key]
-          when "uri", "docid"
-            val = link_unwrap(Nokogiri::XML.fragment(span[:val])).to_xml
-            ret[span[:key].to_sym] << { type: span[:type], val: val }
-          when "date"
-            ret[span[:key].to_sym] << { type: span[:type] || "published",
-                                        val: span[:val] }
-          when "pages", "volume", "issue"
-            ret[:extent][span[:key].to_sym] ||= []
-            ret[:extent][span[:key].to_sym] << span[:val]
-          when "pubplace", "title", "type", "series"
-            ret[span[:key].to_sym] = span[:val]
-          when "in_title"
-            ret[:in][:title] = span[:val]
-          when "publisher"
-            ret[:contrib] << { role: "publisher", entity: "organization",
-                               name: span[:val] }
-          when "surname", "initials", "givenname", "formatted-initials"
-            ret[:contrib] = spans_preprocess_contrib(span, ret[:contrib])
-          when "fullname"
-            ret[:contrib] = spans_preprocess_fullname(span, ret[:contrib])
-          when "organization"
-            ret[:contrib] = spans_preprocess_org(span, ret[:contrib])
-          when "in_surname", "in_initials", "in_givenname",
-            "in_formatted-initials"
-            ret[:in][:contrib] ||= []
-            span[:key].sub!(/^in_/, "")
-            ret[:in][:contrib] =
-              spans_preprocess_contrib(span, ret[:in][:contrib])
-          when "in_fullname"
-            ret[:in][:contrib] ||= []
-            span[:key].sub!(/^in_/, "")
-            ret[:in][:contrib] =
-              spans_preprocess_fullname(span, ret[:in][:contrib])
-          when "in_organization"
-            ret[:in][:contrib] ||= []
-            span[:key].sub!(/^in_/, "")
-            ret[:in][:contrib] =
-              spans_preprocess_org(span, ret[:in][:contrib])
-          else
-            msg = "unrecognised key '#{span[:key]}' in " \
-                  "`span:#{span[:key]}[#{span[:val]}]`"
-            @err << { msg: msg }
-          end
-        end
-
-        def host_rearrange(ret)
-          ret[:in][:title] or return ret
-          ret[:in].merge!(empty_span_hash, { type: "misc" }) do |_, old, _|
-            old
-          end
-          %i(series).each do |k|
-            ret[:in][k] = ret[k]
-            ret.delete(k)
-          end
-          /^in/.match?(ret[:type]) and ret[:in][:type] =
-                                         ret[:type].sub(/^in/, "")
-          ret
-        end
-
-        def spans_preprocess_contrib(span, contrib)
-          span[:key] == "initials" and span[:key] = "formatted-initials"
-          spans_preprocess_new_contrib?(span, contrib) and
-            contrib << { role: span[:type] || "author", entity: "person" }
-          if multiple_givennames?(span, contrib)
-            contrib[-1][:givenname] = [contrib[-1][:givenname],
-                                       span[:val]].flatten
-          else contrib[-1][span[:key].to_sym] = span[:val]
-          end
-          contrib
-        end
-
-        def spans_preprocess_new_contrib?(span, contrib)
-          contrib.empty? ||
-            (span[:key] == "surname" && contrib[-1][:surname]) ||
-            contrib[-1][:role] != (span[:type] || "author")
-        end
-
-        def multiple_givennames?(span, contrib)
-          (%w(formatted-initials givenname).include?(span[:key]) &&
-            (contrib[-1][:"formatted-initials"] || contrib[-1][:givenname])) or
-            return false
-          if contrib[-1][:"formatted-initials"]
-            contrib[-1][:givenname] = contrib[-1][:"formatted-initials"]
-            contrib[-1].delete(:"formatted-initials")
-          end
-          true
-        end
-
-        def spans_preprocess_fullname(span, contrib)
-          name = span[:val].gsub(/\.(?=\p{Alpha})/, ". ").split(/ /)
-          out = { role: span[:type] || "author", entity: "person",
-                  surname: name[-1] }
-          if name.size > 1 && name[0..-2].all? { |x| /\.$/.match?(x) }
-            out[:"formatted-initials"] = name[0..-2].join(" ")
-          else out[:givenname] = name[0..-2]
-          end
-          contrib << out
-          contrib
-        end
-
-        def spans_preprocess_org(span, contrib)
-          contrib << { role: span[:type] || "author", entity: "organization",
-                       name: span[:val] }
-          contrib
-        end
-
         def convert
           ret = spans_to_bibitem(@spans)
           @out = Nokogiri::XML("<bibitem>#{ret}</bibitem>").root
@@ -174,17 +36,20 @@ module Metanorma
           ret = ""
           spans[:title] and ret += "<title>#{spans[:title]}</title>"
           ret += spans_to_bibitem_docid(spans)
-          spans[:contrib].each do |s|
-            ret += span_to_contrib(s, spans[:title])
-          end
-          spans[:series] and
-            ret += "<series><title>#{spans[:series]}</title></series>"
+          ret += spans_to_contribs(spans)
+          ret += spans_to_bibitem_edn(spans)
+          ret += spans_to_series(spans)
           spans[:pubplace] and ret += "<place>#{spans[:pubplace]}</place>"
           ret += spans_to_bibitem_host(spans)
           ret += spans_to_bibitem_extent(spans[:extent])
           ret
         end
 
+        def spans_to_series(spans)
+          spans[:series] or return ""
+          "<series><title>#{spans[:series]}</title></series>"
+        end
+
         def spans_to_bibitem_host(spans)
           spans[:in].empty? and return ""
           ret =
@@ -201,6 +66,14 @@ module Metanorma
           ret
         end
 
+        def spans_to_bibitem_edn(spans)
+          ret = ""
+          spans[:edition] and ret += "<edition>#{spans[:edition]}</edition>"
+          spans[:version] and ret += "<version>#{spans[:version]}</version>"
+          spans[:note] and ret += "<note>#{spans[:note]}</note>"
+          ret
+        end
+
         def spans_to_bibitem_extent(spans)
           ret = ""
           { volume: "volume", issue: "issue", pages: "page" }.each do |k, v|
@@ -236,6 +109,14 @@ module Metanorma
           "<date#{type}>#{val}</date>"
         end
 
+        def spans_to_contribs(spans)
+          ret = ""
+          spans[:contrib].each do |s|
+            ret += span_to_contrib(s, spans[:title])
+          end
+          ret
+        end
+
         def span_to_contrib(span, title)
           e = if span[:entity] == "organization"
                 "<organization><name>#{span[:name]}</name></organization>"

data/lib/metanorma/standoc/spans_to_bibitem_preprocessing.rb

@@ -0,0 +1,148 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      class SpansToBibitem
+        def extract_spans(bib)
+          bib.xpath("./formattedref//span").each_with_object([]) do |s, m|
+            s.at("./ancestor::span") and next
+            extract_spans1(s, m)
+          end
+        end
+
+        def extract_spans1(span, acc)
+          keys = span["class"].split(".", 2)
+          acc << { key: keys[0], type: keys[1],
+                   val: span.children.to_xml }
+          (span["class"] == "type" and span.remove) or
+            span.replace(span.children)
+        end
+
+        def extract_docid(bib)
+          bib.xpath("./docidentifier").each_with_object([]) do |d, m|
+            m << { key: "docid", type: d["type"], val: d.text }
+            d.remove unless bib.at("./title")
+          end
+        end
+
+        def empty_span_hash
+          { contrib: [], docid: [], uri: [], date: [], extent: {}, in: {} }
+        end
+
+        def spans_preprocess(spans)
+          ret = empty_span_hash
+          spans.each { |s| span_preprocess1(s, ret) }
+          host_rearrange(ret)
+        end
+
+        def span_preprocess1(span, ret)
+          case span[:key]
+          when "uri", "docid"
+            val = link_unwrap(Nokogiri::XML.fragment(span[:val])).to_xml
+            ret[span[:key].to_sym] << { type: span[:type], val: val }
+          when "date"
+            ret[span[:key].to_sym] << { type: span[:type] || "published",
+                                        val: span[:val] }
+          when "pages", "volume", "issue"
+            ret[:extent][span[:key].to_sym] ||= []
+            ret[:extent][span[:key].to_sym] << span[:val]
+          when "pubplace", "title", "type", "series", "edition", "version",
+            "note"
+            ret[span[:key].to_sym] = span[:val]
+          when "in_title"
+            ret[:in][:title] = span[:val]
+          when "publisher"
+            ret[:contrib] << { role: "publisher", entity: "organization",
+                               name: span[:val] }
+          when "surname", "initials", "givenname", "formatted-initials"
+            ret[:contrib] = spans_preprocess_contrib(span, ret[:contrib])
+          when "fullname"
+            ret[:contrib] = spans_preprocess_fullname(span, ret[:contrib])
+          when "organization"
+            ret[:contrib] = spans_preprocess_org(span, ret[:contrib])
+          when "in_surname", "in_initials", "in_givenname",
+            "in_formatted-initials"
+            ret[:in][:contrib] ||= []
+            span[:key].sub!(/^in_/, "")
+            ret[:in][:contrib] =
+              spans_preprocess_contrib(span, ret[:in][:contrib])
+          when "in_fullname"
+            ret[:in][:contrib] ||= []
+            span[:key].sub!(/^in_/, "")
+            ret[:in][:contrib] =
+              spans_preprocess_fullname(span, ret[:in][:contrib])
+          when "in_organization"
+            ret[:in][:contrib] ||= []
+            span[:key].sub!(/^in_/, "")
+            ret[:in][:contrib] =
+              spans_preprocess_org(span, ret[:in][:contrib])
+          else
+            msg = "unrecognised key '#{span[:key]}' in " \
+                  "`span:#{span[:key]}[#{span[:val]}]`"
+            @err << { msg: msg }
+          end
+        end
+
+        def host_rearrange(ret)
+          ret[:in][:title] or return ret
+          ret[:in].merge!(empty_span_hash, { type: "misc" }) do |_, old, _|
+            old
+          end
+          %i(series).each do |k|
+            ret[:in][k] = ret[k]
+            ret.delete(k)
+          end
+          /^in/.match?(ret[:type]) and ret[:in][:type] =
+                                         ret[:type].sub(/^in/, "")
+          ret
+        end
+
+        def spans_preprocess_contrib(span, contrib)
+          span[:key] == "initials" and span[:key] = "formatted-initials"
+          spans_preprocess_new_contrib?(span, contrib) and
+            contrib << { role: span[:type] || "author", entity: "person" }
+          if multiple_givennames?(span, contrib)
+            contrib[-1][:givenname] = [contrib[-1][:givenname],
+                                       span[:val]].flatten
+          else contrib[-1][span[:key].to_sym] = span[:val]
+          end
+          contrib
+        end
+
+        def spans_preprocess_new_contrib?(span, contrib)
+          contrib.empty? ||
+            (span[:key] == "surname" && contrib[-1][:surname]) ||
+            contrib[-1][:role] != (span[:type] || "author")
+        end
+
+        def multiple_givennames?(span, contrib)
+          (%w(formatted-initials givenname).include?(span[:key]) &&
+            (contrib[-1][:"formatted-initials"] || contrib[-1][:givenname])) or
+            return false
+          if contrib[-1][:"formatted-initials"]
+            contrib[-1][:givenname] = contrib[-1][:"formatted-initials"]
+            contrib[-1].delete(:"formatted-initials")
+          end
+          true
+        end
+
+        def spans_preprocess_fullname(span, contrib)
+          name = span[:val].gsub(/\.(?=\p{Alpha})/, ". ").split(/ /)
+          out = { role: span[:type] || "author", entity: "person",
+                  surname: name[-1] }
+          if name.size > 1 && name[0..-2].all? { |x| /\.$/.match?(x) }
+            out[:"formatted-initials"] = name[0..-2].join(" ")
+          else out[:givenname] = name[0..-2]
+          end
+          contrib << out
+          contrib
+        end
+
+        def spans_preprocess_org(span, contrib)
+          contrib << { role: span[:type] || "author", entity: "organization",
+                       name: span[:val] }
+          contrib
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/utils.rb

@@ -18,7 +18,7 @@ module Metanorma
       end
 
       def noko(&block)
-        Metanorma::Utils::noko(&block)
+        Metanorma::Utils::noko(@script, &block)
       end
 
       def attr_code(attributes)
@@ -68,6 +68,14 @@ module Metanorma
         para
       end
 
+      def xml_encode(text)
+        @c.encode(text, :basic, :hexadecimal)
+          .gsub(/&amp;gt;/, ">").gsub(/&amp;lt;/, "<").gsub(/&amp;amp;/, "&")
+          .gsub(/&gt;/, ">").gsub(/&lt;/, "<").gsub(/&amp;/, "&")
+          .gsub(/&quot;/, '"').gsub(/&#xa;/, "\n").gsub(/&amp;#/, "&#")
+          .gsub(/&apos;/, "'")
+      end
+
       class EmptyAttr
         def attr(_any_attribute)
           nil

data/lib/metanorma/standoc/validate.rb

@@ -53,18 +53,46 @@ module Metanorma
         table_validate(doc)
         @fatalerror += requirement_validate(doc)
         image_validate(doc)
+        math_validate(doc)
         @fatalerror.empty? or
           clean_abort(@fatalerror.join("\n"), doc)
       end
 
+      MATHML_NS = "http://www.w3.org/1998/Math/MathML".freeze
+
+      def math_validate(doc)
+        doc.xpath("//m:math", "m" => MATHML_NS).each do |m|
+          math = mathml_sanitise(m.dup)
+          Plurimath::Math.parse(math, "mathml").to_mathml
+        rescue StandardError => e
+          math_validate_error(math, m, e)
+        end
+      end
+
+      def mathml_sanitise(math)
+        math.to_xml(encoding: "US-ASCII").gsub(/ xmlns=["'][^"']+["']/, "")
+          .gsub(%r{<[^:/>]+:}, "<").gsub(%r{</[^:/>]+:}, "</")
+          #.gsub(/&#([^;]+);/) { |x| "&#x#{$1.to_i.to_s(16)};" }
+      end
+
+      def math_validate_error(math, elem, error)
+        a = elem.parent.at("./asciimath")
+        l = elem.parent.at("./latexmath")
+        orig = ""
+        a and orig += "\n\tAsciimath original: #{@c.decode(a.children.to_xml)}"
+        l and orig += "\n\tLatexmath original: #{@c.decode(l.children.to_xml)}"
+        @log.add("Mathematics", elem,
+                 "Invalid MathML: #{math}\n #{error}#{orig}")
+        @fatalerror << "Invalid MathML: #{math}"
+      end
+
       def nested_asset_validate(doc)
         nested_asset_validate_basic(doc)
         nested_note_validate(doc)
       end
 
       def nested_asset_validate_basic(doc)
-        a = "//formula | //example | //figure | //termnote | //termexample | " \
-            "//table"
+        a = "//example | //figure | //termnote | //termexample | //table"
         doc.xpath("#{a} | //note").each do |m|
           m.xpath(a.gsub(%r{//}, ".//")).each do |n|
             nested_asset_report(m, n, doc)
@@ -82,7 +110,6 @@ module Metanorma
 
       def nested_asset_report(outer, inner, doc)
         outer.name == "figure" && inner.name == "figure" and return
-        outer.name != "formula" && inner.name == "formula" and return
         err =
           "There is an instance of #{inner.name} nested within #{outer.name}"
         @log.add("Syntax", inner, err)

data/lib/metanorma/standoc/version.rb

@@ -19,6 +19,6 @@ module Metanorma
   end
 
   module Standoc
-    VERSION = "2.3.9".freeze
+    VERSION = "2.4.1".freeze
   end
 end

data/metanorma-standoc.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "asciidoctor", "~> 2.0.0"
   spec.add_dependency "iev", "~> 0.3.0"
-  spec.add_dependency "isodoc", "~> 2.4.0"
+  spec.add_dependency "isodoc", "~> 2.5.0"
   spec.add_dependency "metanorma"
   spec.add_dependency "metanorma-plugin-datastruct", "~> 0.2.0"
   spec.add_dependency "metanorma-plugin-lutaml"
@@ -38,8 +38,7 @@ Gem::Specification.new do |spec|
   # relaton-cli not just relaton, to avoid circular reference in metanorma
   spec.add_dependency "asciimath2unitsml", "~> 0.4.0"
   spec.add_dependency "concurrent-ruby"
-  spec.add_dependency "latexmath"
-  spec.add_dependency "mathml2asciimath"
+  spec.add_dependency "plurimath"
   spec.add_dependency "pngcheck"
   spec.add_dependency "relaton-cli", "~> 1.15.0"
   spec.add_dependency "relaton-iev", "~> 1.1.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-standoc
 version: !ruby/object:Gem::Version
-  version: 2.3.9
+  version: 2.4.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-02-27 00:00:00.000000000 Z
+date: 2023-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.5.0
 - !ruby/object:Gem::Dependency
   name: metanorma
   requirement: !ruby/object:Gem::Requirement
@@ -137,21 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: latexmath
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: mathml2asciimath
+  name: plurimath
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -520,6 +506,7 @@ files:
 - lib/metanorma/standoc/reqt.rng
 - lib/metanorma/standoc/section.rb
 - lib/metanorma/standoc/spans_to_bibitem.rb
+- lib/metanorma/standoc/spans_to_bibitem_preprocessing.rb
 - lib/metanorma/standoc/table.rb
 - lib/metanorma/standoc/term_lookup_cleanup.rb
 - lib/metanorma/standoc/terms.rb