metanorma 1.7.7 → 2.0.1

data/lib/metanorma/collection/xrefprocess/xrefprocess.rb

@@ -0,0 +1,222 @@
+module Metanorma
+  class Collection
+    module XrefProcess
+      class << self
+        # xml.root["type"] = key # to force recognition of internal refs
+        # bookmarks etc as new id elements introduced in Presentation XML:
+        # add doc suffix
+        def xref_preprocess(xml, isodoc)
+          @isodoc = isodoc
+          key = (0...8).map { rand(65..90).chr }.join # random string
+          xml.root["type"] = key
+          Metanorma::Utils::anchor_attributes
+            .each do |(tag_name, attribute_name)|
+            ::Metanorma::Collection::Util::add_suffix_to_attributes(
+              xml, xml.root["document_suffix"], tag_name, attribute_name, isodoc
+            )
+          end
+          key
+        end
+
+        def ns(xpath)
+          @isodoc.ns(xpath)
+        end
+
+        def xref_process(section, xml, key, ident, isodoc)
+          @isodoc ||= isodoc
+          svg_preprocess(section, Metanorma::Utils::to_ncname(ident))
+          refs = eref_to_internal_eref(section, xml, key)
+          refs += xref_to_internal_eref(section, xml, key)
+          ins = new_hidden_ref(section)
+          copied_refs = copy_repo_items_biblio(ins, section, xml)
+          insert_indirect_biblio(ins, refs - copied_refs, key, xml)
+        end
+
+        def svg_preprocess(xml, doc_suffix)
+          suffix = doc_suffix.nil? || doc_suffix.blank? ? "" : "_#{doc_suffix}"
+          xml.xpath("//m:svg", "m" => "http://www.w3.org/2000/svg").each do |s|
+            m = svgmap_wrap(s)
+            svg_xrefs(s, m, suffix)
+          end
+          xml
+        end
+
+        def svgmap_wrap(svg)
+          ret = svg.at("./ancestor::xmlns:svgmap") and return ret
+          ret = svg.at("./ancestor::xmlns:figure")
+          ret.wrap("<svgmap/>")
+          svg.at("./ancestor::xmlns:svgmap")
+        end
+
+        def svg_xrefs(svg, svgmap, suffix)
+          svg.xpath(".//m:a", "m" => "http://www.w3.org/2000/svg").each do |a|
+            /^#/.match? a["href"] or next
+            a["href"] = a["href"].sub(/^#/, "")
+            svgmap << "<target href='#{a['href']}'>" \
+              "<xref target='#{a['href']}#{suffix}'/></target>"
+          end
+        end
+
+        def make_anchor(elem, anchor)
+          elem.at(ns("./localityStack | ./locality")) and return
+          elem.text.strip.empty? && elem.elements.empty? and elem << anchor
+          elem <<
+            "<localityStack><locality type='anchor'><referenceFrom>" \
+              "#{anchor}</referenceFrom></locality></localityStack>"
+        end
+
+        def xref_to_internal_eref(section, xml, key)
+          key or return [] # no sectionsplit, no playing with xrefs
+          bibitems, indirect = xref_to_internal_eref_prep(section, xml)
+          section.xpath(ns("//xref")).each_with_object({}) do |x, m|
+            xref_prefix_key(x, key, indirect)
+            x["bibitemid"] = x["target"]
+            m[x["bibitemid"]] = true
+            xref_to_internal_eref_anchor(x, key, bibitems,
+                                         xml.root["document_suffix"])
+          end.keys
+        end
+
+        def xref_to_internal_eref_prep(section, xml)
+          bibitems = ::Metanorma::Collection::Util::gather_bibitems(section)
+          indirect_bibitems = ::Metanorma::Collection::Util::gather_bibitems(xml)
+            .select { |_, v| indirect_bib?(v) }
+          [bibitems, indirect_bibitems]
+        end
+
+        def xref_to_internal_eref_anchor(xref, key, bibitems, document_suffix)
+          t = xref["target"]
+          if d = bibitems[t]&.at(ns("./docidentifier[@type = 'repository']"))
+            m = %r{^([^/]+)}.match(d.text) and
+              t.sub!(%r(#{m[0]}_), "")
+          end
+          key and t.sub!(%r{^#{key}_}, "")
+          make_anchor(xref, t.sub(%r(_#{document_suffix}$), ""))
+          xref.delete("target")
+          xref.name = "eref"
+        end
+
+        def xref_prefix_key(xref, key, indirect)
+          if b = indirect[xref["target"]]
+            t = b.at(ns("./docidentifier[@type = 'repository']"))
+            xref["type"] = t.text.sub(%r{/.*$}, "")
+          elsif key
+            xref["target"] = "#{key}_#{xref['target']}"
+            xref["type"] = key
+          end
+        end
+
+        def eref_to_internal_eref(section, xml, key)
+          bibitems, indirect, bibids = eref_to_internal_eref_prep(section, xml)
+          eref_to_internal_eref_select(section, xml, bibitems)
+            .each_with_object([]) do |x, m|
+              url = bibitems[x]&.at(ns("./uri[@type = 'citation']"))&.text
+              bibids[x]&.each do |e|
+                e.at(ns("./localityStack | ./locality")) and next
+                id = eref_to_internal_eref1(e, key, url, indirect) and m << id
+              end
+            end
+        end
+
+        def eref_to_internal_eref_prep(section, xml)
+          bibitems = ::Metanorma::Collection::Util::gather_bibitems(xml)
+            .delete_if { |_, v| internal_bib?(v) }
+          indirect = ::Metanorma::Collection::Util::gather_bibitems(xml)
+            .select { |_, v| indirect_bib?(v) }
+          bibitemids = ::Metanorma::Collection::Util::gather_bibitemids(section)
+          [bibitems, indirect, bibitemids]
+        end
+
+        def eref_to_internal_eref1(elem, key, url, indirect)
+          if url
+            elem.name = "link"
+            elem["target"] = url
+            nil
+          elsif !indirect[elem["bibitemid"]]
+            nil
+          else
+            eref_to_internal_eref1_internal(elem, key, indirect)
+          end
+        end
+
+        def eref_to_internal_eref1_internal(elem, key, indirect)
+          t = elem["bibitemid"]
+          if key
+            t = "#{key}_#{t}"
+            elem["type"] = key
+          elsif d = indirect[t]&.at(ns("./docidentifier[@type = 'repository']"))
+            m = %r{^([^/]+)}.match(d.text) and
+              t.sub!(%r(#{m[0]}_), "")
+          end
+          make_anchor(elem, t)
+          elem["bibitemid"]
+        end
+
+        def eref_to_internal_eref_select(section, _xml, bibitems)
+          refs = ::Metanorma::Collection::Util::gather_bibitemids(section).keys
+          refs.uniq.reject do |x|
+            b = bibitems[x] and (indirect_bib?(b) || internal_bib?(b))
+          end
+        end
+
+        def internal_bib?(bibitem)
+          bibitem["type"] == "internal" ||
+            bibitem.at(ns("./docidentifier[@type = 'repository']"))
+        end
+
+        def indirect_bib?(bibitem)
+          a = bibitem.at(ns("./docidentifier[@type = 'repository']")) or
+            return false
+          %r{^current-metanorma-collection/}.match?(a.text) and return false
+          a.text.include?("/")
+        end
+
+        # from standoc
+        def new_hidden_ref(xmldoc)
+          ins = xmldoc.at("bibliography") or
+            xmldoc.root << "<bibliography/>" and ins = xmldoc.at("bibliography")
+          ins.add_child("<references hidden='true' normative='false'/>").first
+        end
+
+        def copy_repo_items_biblio(ins, section, xml)
+          bibitems = ::Metanorma::Collection::Util::gather_bibitems(section)
+          xml.xpath(ns("//references/bibitem[docidentifier/@type = 'repository']"))
+            .each_with_object([]) do |b, m|
+              bibitems[b["id"]] or next
+              # section.at("//*[@bibitemid = '#{b['id']}']") or next
+              ins << b.dup
+              m << b["id"]
+            end
+        end
+
+        def insert_indirect_biblio(ins, refs, key, xml)
+          refs.empty? and return
+          internal_bibitems, = insert_indirect_biblio_prep(xml)
+          refs.compact.reject do |x|
+            # external_bibitems[x.sub(/^#{key}_/, "")]
+          end.each do |x|
+            ins << if b = internal_bibitems[x.sub(/^#{key}_/, "")]
+                     b.dup.tap { |m| m["id"] = x }
+                   else new_indirect_bibitem(x, key)
+                   end
+          end
+        end
+
+        def insert_indirect_biblio_prep(xml)
+          bibitems = ::Metanorma::Collection::Util::gather_bibitems(xml)
+          internal_bibitems = bibitems.select { |_, v| internal_bib?(v) }
+          external_bibitems = bibitems.reject { |_, v| internal_bib?(v) }
+          [internal_bibitems, external_bibitems]
+        end
+
+        def new_indirect_bibitem(ident, prefix)
+          <<~BIBENTRY
+            <bibitem id="#{ident}" type="internal">
+            <docidentifier type="repository">#{ident.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
+            </bibitem>
+          BIBENTRY
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/{compile.rb → compile/compile.rb}

@@ -6,11 +6,11 @@ require "fontist"
 require "fontist/manifest/install"
 require_relative "compile_validate"
 require_relative "compile_options"
-require_relative "fontist_utils"
-require_relative "util"
-require_relative "sectionsplit"
+require_relative "../util/fontist_helper"
+require_relative "../util/util"
 require_relative "extract"
-require_relative "worker_pool"
+require_relative "../collection/sectionsplit/sectionsplit"
+require_relative "../util/worker_pool"
 
 module Metanorma
   class Compile
@@ -86,7 +86,7 @@ module Metanorma
     def relaton_export(isodoc, options)
       return unless options[:relaton]
 
-      xml = Nokogiri::XML(isodoc) { |config| config.huge }
+      xml = Nokogiri::XML(isodoc, &:huge)
       bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
       # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
       # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
@@ -113,7 +113,7 @@ module Metanorma
       fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
                  orig_filename: File.expand_path(filename),
                  presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
-      @queue = ::Metanorma::WorkersPool
+      @queue = ::Metanorma::Util::WorkersPool
         .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
       gather_and_install_fonts(file, options.dup, extensions)
       process_exts_run(fnames, file, isodoc, extensions, options)
@@ -149,7 +149,8 @@ module Metanorma
       if ext == :rxl
         relaton_export(isodoc, options.merge(relaton: fnames[:out]))
       elsif options[:passthrough_presentation_xml] && ext == :presentation
-        f = File.exist?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
+        #f = File.exist?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
+        f = File.exist?(fnames[:orig_filename]) ? fnames[:orig_filename] : fnames[:f]
         FileUtils.cp f, fnames[:presentationxml]
       elsif ext == :html && options[:sectionsplit]
         sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
@@ -189,7 +190,7 @@ module Metanorma
       ext != :presentation
     end
 
-     # assume we pass in Presentation XML, but we want to recover Semantic XML
+    # assume we pass in Presentation XML, but we want to recover Semantic XML
     def sectionsplit_convert(input_filename, file, output_filename = nil,
                              opts = {})
       @isodoc ||= IsoDoc::PresentationXMLConvert.new({})
@@ -198,10 +199,16 @@ module Metanorma
         File.open(input_filename, "w:UTF-8") { |f| f.write(file) }
       presxml = File.read(input_filename, encoding: "utf-8")
       _xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)
-      Sectionsplit.new(input: input_filename, isodoc: @isodoc, xml: presxml,
-                       base: File.basename(output_filename || filename),
-                       output: output_filename || filename,
-                       dir: dir, compile_opts: opts).build_collection
+
+      ::Metanorma::Collection::Sectionsplit.new(
+        input: input_filename,
+        isodoc: @isodoc,
+        xml: presxml,
+        base: File.basename(output_filename || filename),
+        output: output_filename || filename,
+        dir: dir,
+        compile_opts: opts,
+      ).build_collection
     end
 
     private

data/lib/metanorma/{compile_options.rb → compile/compile_options.rb}

@@ -57,7 +57,8 @@ module Metanorma
     end
 
     def font_install(opt)
-      FontistUtils.install_fonts(@processor, opt) unless @fontist_installed
+      @fontist_installed or
+        Util::FontistHelper.install_fonts(@processor, opt)
       @fontist_installed = true
     end
 
@@ -83,16 +84,19 @@ module Metanorma
     def copy_isodoc_options_attrs(options, ret)
       ret[:datauriimage] = true if options[:datauriimage]
       ret[:sourcefilename] = options[:filename]
-      %i(bare sectionsplit no_install_fonts baseassetpath aligncrosselements
+      %i(bare sectionsplit install_fonts baseassetpath aligncrosselements
          tocfigures toctables tocrecommendations strict)
         .each { |x| ret[x] ||= options[x] }
     end
 
     def font_manifest_mn2pdf(options, ret, ext)
-      custom_fonts = FontistUtils.has_custom_fonts?(@processor, options, ret)
+      custom_fonts = Util::FontistHelper
+        .has_custom_fonts?(@processor, options, ret)
+
       ext == :pdf && custom_fonts and
-        ret[:mn2pdf] = { font_manifest: FontistUtils
-          .location_manifest(@processor, ret) }
+        ret[:mn2pdf] = {
+          font_manifest: Util::FontistHelper.location_manifest(@processor, ret),
+        }
     end
   end
 end

data/lib/metanorma/{compile_validate.rb → compile/compile_validate.rb}

@@ -35,7 +35,7 @@ module Metanorma
 
     private
 
-    STDTYPE2FLAVOR = { }.freeze
+    STDTYPE2FLAVOR = {}.freeze
 
     def stdtype2flavor(stdtype)
       flavor = STDTYPE2FLAVOR[stdtype] || stdtype

data/lib/metanorma/{extract.rb → compile/extract.rb}

@@ -3,7 +3,7 @@ module Metanorma
     def relaton_export(isodoc, options)
       return unless options[:relaton]
 
-      xml = Nokogiri::XML(isodoc) { |config| config.huge }
+      xml = Nokogiri::XML(isodoc, &:huge)
       bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
       # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
       # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
@@ -24,7 +24,7 @@ module Metanorma
         extract_types = %i[sourcecode image requirement]
       FileUtils.rm_rf dirname
       FileUtils.mkdir_p dirname
-      xml = Nokogiri::XML(isodoc) { |config| config.huge }
+      xml = Nokogiri::XML(isodoc, &:huge)
       sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
       image_export(xml, dirname) if extract_types.include? :image
       extract_types.include?(:requirement) and

data/lib/metanorma/{config.rb → config/config.rb}

@@ -15,7 +15,7 @@ module Metanorma
     attr_accessor :logs
 
     def initialize
-      @logs ||= %i[warning error fatal]
+      @logs = %i[warning error fatal]
     end
   end
 

data/lib/metanorma/input/asciidoc.rb

@@ -35,9 +35,9 @@ module Metanorma
 
       def extract_metanorma_options(file)
         headerextract = file.sub(/\n\n.*$/m, "\n")
-        /\n:mn-document-class:\s+(?<type>[^\n]+)\n/ =~ headerextract
-        /\n:mn-output-extensions:\s+(?<extensions>[^\n]+)\n/ =~ headerextract
-        /\n:mn-relaton-output-file:\s+(?<relaton>[^\n]+)\n/ =~ headerextract
+        /\n:mn-document-class:\s+(?<type>\S[^\n]*)\n/ =~ headerextract
+        /\n:mn-output-extensions:\s+(?<extensions>\S[^\n]*)\n/ =~ headerextract
+        /\n:mn-relaton-output-file:\s+(?<relaton>\S[^\n]*)\n/ =~ headerextract
         /\n(?<asciimath>:mn-keep-asciimath:[^\n]*)\n/ =~ headerextract
         /\n(?<novalid>:novalid:[^\n]*)\n/ =~ headerextract
         asciimath = if defined?(asciimath)

data/lib/metanorma/input/base.rb

@@ -1,13 +1,9 @@
-
 module Metanorma
   module Input
-
     class Base
-
-      def process(file, filename, type)
+      def process(_file, _filename, _type)
         raise "This is an abstract class"
       end
-
     end
   end
 end

data/lib/metanorma/processor/processor.rb

@@ -0,0 +1,54 @@
+# Registry of all Metanorma types and entry points
+#
+
+module Metanorma
+  class Processor
+    attr_reader :short, :input_format, :asciidoctor_backend
+
+    def initialize
+      raise "This is an abstract class!"
+    end
+
+    def output_formats
+      {
+        xml: "xml",
+        presentation: "presentation.xml",
+        rxl: "rxl",
+      }
+    end
+
+    def input_to_isodoc(file, filename, options = {})
+      Metanorma::Input::Asciidoc.new.process(file, filename,
+                                             @asciidoctor_backend, options)
+    end
+
+    # def input_to_isodoc(file, filename)
+    #   raise "This is an abstract class!"
+    # end
+
+    def use_presentation_xml(ext)
+      case ext
+      when :html, :doc, :pdf then true
+      else
+        false
+      end
+    end
+
+    def options_preprocess(options)
+      options[:output_formats] ||= output_formats
+    end
+
+    def output(isodoc_node, _inname, outname, _format, _options = {})
+      File.open(outname, "w:UTF-8") { |f| f.write(isodoc_node) }
+    end
+
+    def extract_options(file)
+      Metanorma::Input::Asciidoc.new.extract_options(file)
+        .merge(output_formats: output_formats)
+    end
+
+    def extract_metanorma_options(file)
+      Metanorma::Input::Asciidoc.new.extract_metanorma_options(file)
+    end
+  end
+end

data/lib/metanorma/processor.rb

@@ -1,54 +1,6 @@
-# Registry of all Metanorma types and entry points
-#
+require_relative "./processor/processor"
 
 module Metanorma
   class Processor
-    attr_reader :short, :input_format, :asciidoctor_backend
-
-    def initialize
-      raise "This is an abstract class!"
-    end
-
-    def output_formats
-      {
-        xml: "xml",
-        presentation: "presentation.xml",
-        rxl: "rxl",
-      }
-    end
-
-    def input_to_isodoc(file, filename, options = {})
-      Metanorma::Input::Asciidoc.new.process(file, filename,
-                                             @asciidoctor_backend, options)
-    end
-
-    # def input_to_isodoc(file, filename)
-    #   raise "This is an abstract class!"
-    # end
-
-    def use_presentation_xml(ext)
-      case ext
-      when :html, :doc, :pdf then true
-      else
-        false
-      end
-    end
-
-    def options_preprocess(options)
-      options[:output_formats] ||= output_formats
-    end
-
-    def output(isodoc_node, _inname, outname, _format, _options = {})
-      File.open(outname, "w:UTF-8") { |f| f.write(isodoc_node) }
-    end
-
-    def extract_options(file)
-      Metanorma::Input::Asciidoc.new.extract_options(file)
-        .merge(output_formats: output_formats)
-    end
-
-    def extract_metanorma_options(file)
-      Metanorma::Input::Asciidoc.new.extract_metanorma_options(file)
-    end
   end
 end

data/lib/metanorma/{registry.rb → registry/registry.rb}

@@ -1,5 +1,4 @@
 # Registry of all Metanorma types and entry points
-#
 
 require "singleton"
 

data/lib/metanorma/shale_monkeypatch.rb

@@ -0,0 +1,15 @@
+module Shale
+  module Adapter
+    module Nokogiri
+      class Node
+        def content
+          @node
+        end
+
+        def to_xml
+          @node.to_xml
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/util/fontist_helper.rb

@@ -0,0 +1,130 @@
+require "fontist"
+module Metanorma
+  module Util
+    class FontistHelper
+      class << self
+        private
+
+        def validate_options(options)
+          agree_to_terms = options[:agree_to_terms] || false
+          continue_without_fonts = options[:continue_without_fonts] || false
+          no_progress = !options[:progress]
+
+          [agree_to_terms, continue_without_fonts, no_progress]
+        end
+
+        def validate_install_fonts(processor, options)
+          unless install_fonts?(options)
+            Util.log("[fontist] Skip font installation because" \
+                     " --no-install-fonts argument passed", :debug)
+            return false
+          end
+          unless has_custom_fonts?(processor, options, options)
+            Util.log("[fontist] Skip font installation because "\
+                     "fonts_manifest is missing", :debug)
+            return false
+          end
+          true
+        end
+
+        def install_fonts_safe(manifest, agree, continue, no_progress)
+          fontist_install(manifest, agree, no_progress)
+        rescue Fontist::Errors::LicensingError
+          license_error_log(continue)
+        rescue Fontist::Errors::FontError => e
+          log_level = continue ? :warning : :fatal
+          Util.log("[fontist] '#{e.font}' font is not supported. " \
+                   "Please report this issue at github.com/metanorma/metanorma" \
+                   "/issues to report this issue.", log_level)
+        rescue Fontist::Errors::FormulaIndexNotFoundError
+          fontist_update_repo(manifest, agree, continue, no_progress)
+        end
+
+        def fontist_install(manifest, agree, no_progress)
+          if agree
+            no_license_log
+          else
+            Fontist.log_level = :info
+          end
+
+          Fontist::Manifest::Install.from_hash(
+            manifest,
+            confirmation: agree ? "yes" : "no",
+            no_progress: no_progress,
+          )
+        end
+
+        def license_error_log(continue)
+          if continue
+            Util.log(
+              "[fontist] Processing will continue without fonts installed",
+              :debug,
+            )
+          else
+            Util.log("[fontist] Aborting without proper fonts installed," \
+                     " make sure that you have set option --agree-to-terms",
+                     :fatal)
+          end
+        end
+
+        def no_license_log
+          Util.log(
+            "[fontist] Font licenses are not shown with --agree-to-terms option.",
+            :info,
+          )
+        end
+
+        def fontist_update_repo(manifest, agree, continue, no_progress)
+          if @@updated_formulas_repo
+            Util.log(
+              "[fontist] Bug: formula index not found after 'fontist update'",
+              :fatal,
+            )
+          end
+          Util.log("[fontist] Missing formula index. Fetching it...", :debug)
+          Fontist::Formula.update_formulas_repo
+          @@updated_formulas_repo = true
+          install_fonts_safe(manifest, agree, continue, no_progress)
+        end
+      end
+
+      def self.install_fonts(processor, options)
+        return unless validate_install_fonts(processor, options)
+
+        @@updated_formulas_repo = false
+        manifest = processor.fonts_manifest.dup
+        append_source_fonts(manifest, options)
+        agree_to_terms, can_without_fonts, no_progress = validate_options(options)
+
+        install_fonts_safe(
+          manifest,
+          agree_to_terms,
+          can_without_fonts,
+          no_progress,
+        )
+      end
+
+      def self.has_custom_fonts?(processor, options, source_attributes)
+        install_fonts?(options) \
+          && processor.respond_to?(:fonts_manifest) \
+          && !processor.fonts_manifest.nil? \
+          || source_attributes[:fonts]
+      end
+
+      def self.location_manifest(processor, source_attributes)
+        Fontist::Manifest::Locations.from_hash(
+          append_source_fonts(processor.fonts_manifest.dup, source_attributes),
+        )
+      end
+
+      def self.append_source_fonts(manifest, source_attributes)
+        source_attributes[:fonts]&.split(";")&.each { |f| manifest[f] = nil }
+        manifest
+      end
+
+      def self.install_fonts?(options)
+        options[:install_fonts].nil? || options[:install_fonts]
+      end
+    end
+  end
+end