isodoc 2.10.0 → 3.0.0

data/lib/isodoc/presentation_function/inline.rb

@@ -13,7 +13,7 @@ module IsoDoc
       node["style"] == "id" and anchor_id_postprocess(node)
       xref_empty?(node) or return
       target = docid_l10n(node["target"]) ||
-        expand_citeas(docid_l10n(node["citeas"]))
+        docid_l10n(expand_citeas(node["citeas"]))
       link = anchor_linkend(node, target)
       link += eref_localities(node.xpath(ns("./locality | ./localityStack")),
                               link, node)
@@ -25,8 +25,8 @@ module IsoDoc
     # <locality type="section"><reference>3.1</reference></locality></origin>
 
     def unnest_linkend(node)
-      node.at(ns("./xref[@nested]")) or return
-      node.xpath(ns("./xref[@nested]")).each { |x| x.delete("nested") }
+      node.at(ns("./fmt-xref[@nested]")) or return
+      node.xpath(ns("./fmt-xref[@nested]")).each { |x| x.delete("nested") }
       node.xpath(ns("./location | ./locationStack")).each(&:remove)
       node.replace(node.children)
     end
@@ -39,31 +39,56 @@ module IsoDoc
     def anchor_id_postprocess(node); end
 
     def xref(docxml)
-      docxml.xpath(ns("//xref")).each { |f| xref1(f) }
-      docxml.xpath(ns("//xref//xref")).each { |f| f.replace(f.children) }
+      #docxml.xpath(ns("//display-text")).each { |f| f.replace(f.children) }
+      docxml.xpath(ns("//fmt-xref")).each { |f| xref1(f) }
+      docxml.xpath(ns("//fmt-xref//fmt-xref")).each { |f| f.replace(f.children) }
+      docxml.xpath(ns("//fmt-xref//xref")).each { |f| f.replace(f.children) }
     end
 
     def eref(docxml)
-      docxml.xpath(ns("//eref")).each { |f| xref1(f) }
-      docxml.xpath(ns("//eref//xref")).each { |f| f.replace(f.children) }
+      docxml.xpath(ns("//eref[@deleteme]")).each { |f| redundant_eref(f) }
+      docxml.xpath(ns("//fmt-eref")).each { |f| xref1(f) }
+      docxml.xpath(ns("//fmt-eref//fmt-xref")).each { |f| f.replace(f.children) }
       docxml.xpath(ns("//erefstack")).each { |f| erefstack1(f) }
     end
 
+    # redundant eref copied from quote/source
+    def redundant_eref(elem)
+      if elem.next.name == "semx"
+        elem.next.elements.first.delete("deleteme")
+        elem.next.replace(elem.next.children)
+      end
+      elem.remove
+    end
+
     def origin(docxml)
-      docxml.xpath(ns("//origin[not(termref)]")).each { |f| xref1(f) }
+      docxml.xpath(ns("//fmt-origin[not(.//termref)]")).each { |f| xref1(f) }
     end
 
-    def quotesource(docxml)
-      docxml.xpath(ns("//quote/source")).each { |f| xref1(f) }
-      docxml.xpath(ns("//quote/source//xref")).each do |f|
+    # KILL
+    def quotesourcex(docxml)
+      docxml.xpath(ns("//quote//source")).each { |f| xref1(f) }
+      docxml.xpath(ns("//quote//source//xref")).each do |f|
         f.replace(f.children)
       end
     end
 
+    # do not change to Presentation XML rendering
+    def sem_xml_descendant?(node)
+      !node.ancestors("preferred, admitted, deprecated, related, " \
+        "definition, termsource").empty? and return true
+      !node.ancestors("xref, eref, origin, link").empty? and return true
+      !node.ancestors("requirement, recommendation, permission").empty? &&
+        node.ancestors("fmt-provision").empty? and return true
+      false
+    end
+
     def xref1(node)
+      sem_xml_descendant?(node) and return
       get_linkend(node)
     end
 
+    # there should be no //variant in bibdata now
     def variant(xml)
       b = xml.xpath(ns("//bibdata//variant"))
       (xml.xpath(ns("//variant")) - b).each { |f| variant1(f) }
@@ -85,7 +110,10 @@ module IsoDoc
 
     def identifier(docxml)
       docxml.xpath(ns("//identifier")).each do |n|
-        n.name = "tt"
+        %w(bibdata bibitem requirement recommendation permission)
+          .include?(n.parent.name) and next
+          s = semx_fmt_dup(n)
+          n.next = "<fmt-identifier><tt>#{to_xml(s)}</tt></fmt-identifier>"
       end
     end
 
@@ -98,7 +126,10 @@ module IsoDoc
 
     def date1(elem)
       elem["value"] && elem["format"] or return
-      elem.replace(@i18n.date(elem["value"], elem["format"].strip))
+      val = @i18n.date(elem["value"], elem["format"].strip)
+      d = semx_fmt_dup(elem)
+      d << val
+      elem.next = "<fmt-date>#{to_xml(d)}</fmt-date>"
     end
 
     def inline_format(docxml)
@@ -160,9 +191,9 @@ module IsoDoc
     def passthrough1(elem, formats)
       (elem["formats"] && !elem["formats"].empty?) or elem["formats"] = "all"
       f = elem["formats"].split(",")
-      (f - formats).size == f.size or elem["formats"] = "all"
-      elem["formats"] == "all" and elem["formats"] = formats.join(",")
-      elem["formats"] = ",#{elem['formats']},"
+      (f - formats).size == f.size or f = "all"
+      f == ["all"] and f = formats.dup
+      elem["formats"] = " #{f.join(' ')} "
     end
 
     def extract_custom_charsets(docxml)

data/lib/isodoc/presentation_function/math.rb

@@ -7,9 +7,13 @@ module IsoDoc
     MATHML = { "m" => "http://www.w3.org/1998/Math/MathML" }.freeze
 
     def mathml(docxml)
-      docxml.xpath("//m:math", MATHML).each { |f| mathml_linebreak(f) }
-      locale = twitter_cldr_localiser
+      docxml.xpath(ns("//stem")).each { |s| stem_dup(s) }
+      locale = @lang.to_sym
+      @numfmt = Plurimath::NumberFormatter
+        .new(locale, localize_number: @localizenumber,
+                     localizer_symbols: twitter_cldr_localiser_symbols)
       docxml.xpath("//m:math", MATHML).each do |f| # rubocop:disable Style/CombinableLoops
+        f.parent&.parent&.name == "fmt-stem" or next
         mathml1(f, locale)
       end
     end
@@ -18,81 +22,124 @@ module IsoDoc
     # TwitterCldr::DataReaders::NumberDataReader.new(locale).symbols
     def localize_maths(node, locale)
       node.xpath(".//m:mn", MATHML).each do |x|
-        num = BigDecimal(x.text)
-        precision = /\./.match?(x.text) ? x.text.sub(/^.*\./, "").size : 0
-        x.children = localized_number(num, locale, precision)
+        fmt = x["data-metanorma-numberformat"]
+        x.delete("data-metanorma-numberformat")
+        x.children =
+          if !fmt.nil? && !fmt.empty?
+            explicit_number_formatter(x, locale, fmt)
+          else implicit_number_formatter(x, locale)
+          end
       rescue ArgumentError
+      rescue StandardError, RuntimeError => e
+        warn "Failure to localise MathML/mn\n#{node.parent.to_xml}\n#{e}"
       end
     end
 
-    # By itself twitter-cldr does not support fraction part digits grouping
-    # and custom delimeter, will decorate fraction part manually
-    def localized_number(num, locale, precision)
-      localized = localized_number1(num, locale, precision)
-      twitter_cldr_reader_symbols = twitter_cldr_reader(locale)
-      return localized unless twitter_cldr_reader_symbols[:decimal]
-
-      integer, fraction = localized.split(twitter_cldr_reader_symbols[:decimal])
-      return localized if fraction.nil? || fraction.empty?
+    def normalise_number(num)
+      n = BigDecimal(num).to_s("F")
+      /\.\d/.match?(num) or n.sub!(/\.\d+$/, "")
+      n
+    end
 
-      [integer, decorate_fraction_part(fraction, locale)]
-        .join(twitter_cldr_reader_symbols[:decimal])
+    def implicit_number_formatter(num, locale)
+      num.ancestors("formula").empty? or return
+      ## by default, no formatting in formulas
+      fmt = { significant: num_totaldigits(num.text) }.compact
+      n = normalise_number(num.text)
+      @numfmt.localized_number(n, locale:, format: fmt,
+                                  precision: num_precision(num.text))
     end
 
-    def localized_number1(num, locale, precision)
-      if precision.zero?
-        num.localize(locale).to_s
-      else
-        num.localize(locale).to_decimal.to_s(precision: precision)
+    def numberformat_extract(options)
+      options.gsub!(/([a-z_]+)='/, %('\\1=))
+      CSV.parse_line(options, quote_char: "'").each_with_object({}) do |x, acc|
+        m = /^(.+?)=(.+)?$/.match(x) or next
+        acc[m[1].to_sym] = m[2].sub(/^(["'])(.+)\1$/, "\\2")
       end
     end
 
-    def decorate_fraction_part(fract, locale)
-      result = []
-      twitter_cldr_reader_symbols = twitter_cldr_reader(locale)
-      fract = fract.slice(0..(twitter_cldr_reader_symbols[:precision] || -1))
-      fr_group_digits = twitter_cldr_reader_symbols[:fraction_group_digits] || 1
-      until fract.empty?
-        result.push(fract.slice!(0, fr_group_digits))
+    def numberformat_type(ret)
+      %i(precision significant digit_count group_digits fraction_group_digits)
+        .each do |i|
+        ret[i] &&= ret[i].to_i
+      end
+      %i(notation exponent_sign number_sign locale).each do |i|
+        ret[i] &&= ret[i].to_sym
       end
-      result.join(twitter_cldr_reader_symbols[:fraction_group].to_s)
+      ret
     end
 
-    def twitter_cldr_localiser_symbols
-      {}
+    def explicit_number_formatter(num, locale, options)
+      ret = numberformat_type(numberformat_extract(options))
+      l = ret[:locale] || locale
+      precision, symbols, significant = explicit_number_formatter_cfg(num, ret)
+      n = normalise_number(num.text)
+      Plurimath::NumberFormatter.new(l)
+        .localized_number(n, precision:,
+                             format: symbols.merge(significant:))
     end
 
-    def twitter_cldr_reader(locale)
-      return @twitter_cldr_reader if @twitter_cldr_reader
+    def explicit_number_formatter_cfg(num, fmt)
+      symbols = twitter_cldr_localiser_symbols.dup.transform_values do |v|
+        v.is_a?(String) ? HTMLEntities.new.decode(v) : v
+      end.merge(fmt)
+      symbols = large_notation_fmt(symbols, num.text)
+      [symbols[:precision] || num_precision(num.text), symbols,
+       explicit_number_formatter_signif(num, symbols)]
+    end
 
-      num = TwitterCldr::DataReaders::NumberDataReader.new(locale)
-      @twitter_cldr_reader = num.symbols.merge!(twitter_cldr_localiser_symbols)
-        .merge!(parse_localize_number)
-      @twitter_cldr_reader
+    def explicit_number_formatter_signif(num, symbols)
+      signif = symbols[:significant]
+      (symbols.keys & %i(precision digit_count)).empty? and
+        signif ||= num_totaldigits(num.text)
+      signif
     end
 
-    def twitter_cldr_localiser
-      locale = TwitterCldr.supported_locale?(@lang.to_sym) ? @lang.to_sym : :en
-      twitter_cldr_reader(locale)
-      locale
+    def large_notation_fmt(symbols, num)
+      n = symbols[:large_notation]
+      min = BigDecimal(symbols[:large_notation_min] || "1e-6")
+      max = BigDecimal(symbols[:large_notation_max] || "1e6")
+      n1 = large_notation_fmt1(num, n, min, max) and symbols[:notation] = n1
+      symbols.delete(:large_notation)
+      symbols.delete(:large_notation_min)
+      symbols.delete(:large_notation_max)
+      symbols
     end
 
-    def parse_localize_number
-      @localizenumber or return {}
-      m = %r{(?<group>[^#])?(?<groupdigits>#+0)(?<decimal>.)(?<fractdigits>#+)(?<fractgroup>[^#])?}
-        .match(@localizenumber) or return {}
-      ret = { decimal: m[:decimal], group_digits: m[:groupdigits].size,
-              fraction_group_digits: m[:fractdigits].size,
-              group: m[:group] || "",
-              fraction_group: m[:fractgroup] || "" }.compact
-      %i(group fraction_group).each { |x| ret[x] == " " and ret[x] = "\u00A0" }
-      ret
+    def large_notation_fmt1(num, notation, min, max)
+      notation.nil? || notation == "nil" and return nil
+      val = BigDecimal(num).abs
+      val.zero? and return nil
+      val < min and return notation
+      val > max and return notation
+      nil
+    end
+
+    def num_precision(num)
+      precision = nil
+      /\.(?!\d+e)/.match?(num) and
+        precision = twitter_cldr_localiser_symbols[:precision] ||
+          num.sub(/^.*\./, "").size
+      precision
+    end
+
+    def num_totaldigits(num)
+      totaldigits = nil
+      /\.(?=\d+e)/.match?(num) and
+        totaldigits = twitter_cldr_localiser_symbols[:significant] ||
+          num.sub(/^0\./, ".").sub(/^.*\./, "").sub(/e.*$/, "").size
+      totaldigits
+    end
+
+    def twitter_cldr_localiser_symbols
+      {}
     end
 
     def asciimath_dup(node)
       @suppressasciimathdup || node.parent.at(ns("./asciimath")) and return
       math = node.to_xml.gsub(/ xmlns=["'][^"']+["']/, "")
         .gsub(%r{<[^:/>]+:}, "<").gsub(%r{</[^:/>]+:}, "</")
+        .gsub(%r{ data-metanorma-numberformat="[^"]+"}, "")
       ret = Plurimath::Math.parse(math, "mathml").to_asciimath
       node.next = "<asciimath>#{@c.encode(ret, :basic)}</asciimath>"
     rescue StandardError => e
@@ -100,12 +147,13 @@ module IsoDoc
     end
 
     def maths_just_numeral(node)
-      mn = node.at(".//m:mn", MATHML).children
-      if node.parent.name == "stem"
-        node.parent.replace(mn)
-      else
+      mn = node.at(".//m:mn", MATHML).children.text
+        .sub(/\^([0-9+-]+)$/, "<sup>\\1</sup>")
+      #if node.parent.parent.name == "stem"
+        #node.replace(mn)
+      #else
         node.replace(mn)
-      end
+      #end
     end
 
     def mathml1(node, locale)
@@ -113,24 +161,33 @@ module IsoDoc
       mathml_number(node, locale)
     end
 
-    def mathml_linebreak(node)
-      node.at(".//*/@linebreak") or return
-      m = Plurimath::Math.parse(node.to_xml, :mathml)
-        .to_mathml(split_on_linebreak: true)
-      ret = Nokogiri::XML("<m>#{m}</m>").root
-      ret.elements.each_with_index do |e, i|
-        i.zero? or e.previous = "<br/>"
+    def stem_dup(node)
+      sem_xml_descendant?(node) and return
+      ret = semx_fmt_dup(node)
+      f = Nokogiri::XML::Node.new("fmt-stem", node.document)
+      t = node["type"] and f["type"] = t
+      f << ret
+      node.next = f
+    end
+
+    # convert any Ascii superscripts to correct(ish) MathML
+    # Not bothering to match times, base of 1.0 x 10^-20, just ^-20
+    def mn_to_msup(node)
+      node.xpath(".//m:mn", MATHML).each do |n|
+        m = %r{^(.+)\^([0-9+-]+)$}.match(n.text) or next
+        n.replace("<msup><mn>#{m[1]}</mn><mn>#{m[2]}</mn></msup>")
       end
-      node.replace(<<~OUTPUT)
-        <math-with-linebreak>#{ret.children}</math-with-linebreak><math-no-linebreak>#{node.to_xml}</math-no-linebreak>
-      OUTPUT
     end
 
     def mathml_number(node, locale)
       justnumeral = numeric_mathml?(node)
       justnumeral or asciimath_dup(node)
       localize_maths(node, locale)
-      justnumeral and maths_just_numeral(node)
+      if justnumeral
+        maths_just_numeral(node)
+      else
+        mn_to_msup(node)
+      end
     end
 
     def numeric_mathml?(node)
@@ -152,11 +209,11 @@ module IsoDoc
     def mathml_number_to_number(node)
       (node.elements.size == 1 && node.elements.first.name == "mn") or return
       repl = node.at("./m:mn", MATHML).children
-      if node.parent.name == "stem"
-        node.parent.replace(repl)
-      else
+      #if node.parent.name == "stem"
+        #node.parent.replace(repl)
+      #else
         node.replace(repl)
-      end
+      #end
     end
   end
 end

data/lib/isodoc/presentation_function/metadata.rb

@@ -3,13 +3,35 @@ module IsoDoc
     def metadata(docxml)
       toc_metadata(docxml)
       fonts_metadata(docxml)
-      preprocess_xslt_insert(docxml)
+      attachments_extract(docxml)
+      localized_strings(docxml)
+      a = docxml.at(ns("//metanorma-extension")) or return
+      a.elements.empty? and a.remove
+    end
+
+    def localized_strings(docxml)
       a = docxml.at(ns("//bibdata")) or return
       a.next =
         "<localized-strings>#{i8n_name(trim_hash(@i18n.get), '').join}" \
         "</localized-strings>"
     end
 
+    def attachments_extract(docxml)
+      docxml.at(ns("//metanorma-extension/attachment")) or return
+      dir = File.join(@localdir, "_#{@outputfile}_attachments")
+      FileUtils.rm_rf(dir)
+      FileUtils.mkdir_p(dir)
+      docxml.xpath(ns("//metanorma-extension/attachment")).each do |a|
+        save_attachment(a, dir)
+      end
+    end
+
+    def save_attachment(attachment, dir)
+      n = File.join(dir, File.basename(attachment["name"]))
+      c = attachment.text.sub(%r{^data:[^;]+;(?:charset=[^;]+;)?base64,}, "")
+      File.open(n, "wb") { |f| f.write(Base64.decode64(c)) }
+    end
+
     def extension_insert(xml, path = [])
       ins = extension_insert_pt(xml)
       path.each do |n|
@@ -20,9 +42,11 @@ module IsoDoc
 
     def extension_insert_pt(xml)
       xml.at(ns("//metanorma-extension")) ||
-        xml.at(ns("//bibdata"))&.after("<metanorma-extension/>")
+        xml.at(ns("//bibdata"))
+          &.after("<metanorma-extension> </metanorma-extension>")
           &.next_element ||
-        xml.root.elements.first.before("<metanorma-extension/>")
+        xml.root.elements.first
+          .before("<metanorma-extension> </metanorma-extension>")
           .previous_element
     end
 
@@ -41,7 +65,7 @@ module IsoDoc
     def fonts_metadata(xmldoc)
       ins = presmeta_insert_pt(xmldoc)
       @fontist_fonts and CSV.parse_line(@fontist_fonts, col_sep: ";")
-        .map(&:strip).reverse.each do |f|
+        .map(&:strip).reverse_each do |f|
         ins.next = presmeta("fonts", f)
       end
       @fontlicenseagreement and
@@ -50,7 +74,7 @@ module IsoDoc
 
     def presmeta_insert_pt(xmldoc)
       xmldoc.at(ns("//presentation-metadata")) ||
-        xmldoc.at(ns("//metanorma-extension")) || xmldoc.at(ns("//bibdata"))
+        extension_insert_pt(xmldoc).children.last
     end
 
     def presmeta(name, value)
@@ -58,67 +82,6 @@ module IsoDoc
         "</presentation-metadata>"
     end
 
-    def preprocess_xslt_insert(docxml)
-      content = ""
-      p = passthrough_xslt and content += p
-      p = preprocess_xslt_read and content += File.read(p)
-      content.empty? and return
-      ins = extension_insert(docxml, %w(render))
-      ins << content
-    end
-
-    COPY_XSLT =
-      '<xsl:copy><xsl:apply-templates select="@* | node()"/></xsl:copy>'.freeze
-    COPY_CHILDREN_XSLT =
-      '<xsl:apply-templates select="node()"/>'.freeze
-
-    def xslt_template(content)
-      <<~XSLT
-        <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/1999/xhtml" version="1.0">
-          <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="no"/>
-          <xsl:template match="@* | node()">#{COPY_XSLT}</xsl:template>
-        #{content}
-        </xsl:stylesheet>
-      XSLT
-    end
-
-    def passthrough_xslt
-      @output_formats.nil? and return nil
-      @output_formats.empty? and return nil
-      @output_formats.each_key.with_object([]) do |k, m|
-        m << <<~XSLT
-          <preprocess-xslt format="#{k}">
-            #{xslt_template(<<~XSLT1)
-              <xsl:template match="*[local-name() = 'passthrough']">
-                <xsl:if test="contains(@formats,',#{k},')"> <!-- delimited -->
-                #{COPY_XSLT}
-                </xsl:if>
-              </xsl:template>
-            XSLT1
-            }
-          </preprocess-xslt>
-        XSLT
-        m << <<~XSLT
-          <preprocess-xslt format="#{k}">
-            #{xslt_template(<<~XSLT1)
-              <xsl:template match="*[local-name() = 'math-with-linebreak']">
-              #{k == 'pdf' ? COPY_CHILDREN_XSLT : ''}
-              </xsl:template>
-              <xsl:template match="*[local-name() = 'math-no-linebreak']">
-              #{k == 'pdf' ? '' : COPY_CHILDREN_XSLT}
-              </xsl:template>
-            XSLT1
-            }
-          </preprocess-xslt>
-        XSLT
-      end.join("\n")
-    end
-
-    # read in from file, but with `<preprocess-xslt @format="">` wrapper
-    def preprocess_xslt_read
-      html_doc_path("preprocess.xslt")
-    end
-
     def i18n_tag(key, value)
       "<localized-string key='#{key}' language='#{@lang}'>#{value}" \
         "</localized-string>"