metanorma 2.0.0 → 2.5.5

data/lib/metanorma/collection/renderer/renderer.rb

@@ -1,21 +1,51 @@
 require "isodoc"
 require "htmlentities"
+require "mime/types"
 require_relative "fileprocess"
 require_relative "../../util/fontist_helper"
-require_relative "../../util/util"
+require "metanorma-core"
 require_relative "../filelookup/filelookup"
+require_relative "../multilingual/multilingual"
 require_relative "utils"
 require_relative "render_word"
 require_relative "navigation"
+require_relative "svg"
+require_relative "../log"
 
 module Metanorma
   class Collection
     class Renderer
-      FORMATS = %i[html xml doc pdf].freeze
+      FORMATS = %i[html xml doc pdf pdf-portfolio].freeze
 
-      attr_accessor :isodoc, :nested
+      attr_accessor :isodoc, :isodoc_presxml
       attr_reader :xml, :compile, :compile_options, :documents, :outdir,
-                  :manifest
+                  :manifest, :fatal_errors
+
+      # Run the block with the renderer in nested mode, restoring the previous
+      # mode afterwards. Nested mode (used by sectionsplit, which re-enters this
+      # renderer's update_xrefs on the pre-split document) preserves unresolved
+      # erefs and skips the finalising reference passes.
+      def with_nested
+        saved = @nested
+        @nested = true
+        yield
+      ensure
+        @nested = saved
+      end
+
+      # Run the block with the renderer preserving unresolved cross-document
+      # references as stubs instead of stripping them -- for an isolated build of
+      # a collection member (compiled without the rest of its collection
+      # present). Unlike +with_nested+, the ordinary intra-document passes still
+      # run (xref_process, svgmap); only the cross-document resolve/strip is
+      # turned into preserve, so a later reinflation pass can relink the stubs.
+      def with_preserve_unresolved
+        saved = @preserve_unresolved
+        @preserve_unresolved = true
+        yield
+      ensure
+        @preserve_unresolved = saved
+      end
 
       # This is only going to render the HTML collection
       # @param xml [Metanorma::Collection] input XML collection
@@ -32,21 +62,26 @@ module Metanorma
       def initialize(collection, folder, options = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         check_options options
         @xml = Nokogiri::XML collection.to_xml # @xml is the collection manifest
-        @xml.root.default_namespace = "http://metanorma.org"
-        @lang = collection.bibdata.language.first || "en"
-        @script = collection.bibdata.script.first || "Latn"
+        @lang = Array(collection.bibdata.language).first || "en"
+        @script = Array(collection.bibdata.script).first || "Latn"
         @locale = @xml.at("//xmlns:bibdata/xmlns:locale")&.text
-        @doctype = doctype
+        @registry = Metanorma::Registry.instance
+        @flavor = options[:flavor] || flavor
         @compile = Compile.new
-        @compile.load_flavor(@doctype)
+        @compile.load_flavor(@flavor)
 
-        @isodoc = isodoc_create # output processor for flavour
+        # output processor for flavour
+        @isodoc = Util::isodoc_create(Util::taste2flavor(@flavor), @lang,
+                                      @script, @xml)
+        @isodoc_presxml = Util::isodoc_create(Util::taste2flavor(@flavor),
+                                              @lang, @script, @xml,
+                                              presxml: true)
         @outdir = dir_name_cleanse(options[:output_folder])
-        @coverpage = options[:coverpage] || collection.coverpage
         @format = ::Metanorma::Util.sort_extensions_execution(options[:format])
         @compile_options = options[:compile] || {}
         @compile_options[:install_fonts] = true if options[:install_fonts]
         @log = options[:log]
+        @log&.add_msg(METANORMA_LOG_MESSAGES)
         @bibdata = collection.bibdata
         @documents = collection.documents
         @bibdatas = collection.documents
@@ -58,10 +93,45 @@ module Metanorma
         @final = collection.final
         @c = HTMLEntities.new
         @files_to_delete = []
-        @nested = options[:nested] # if false, this is the root instance of Renderer
+        # Per-member rendering failures (missing outputs, pooled compile
+        # errors), accumulated by file_compile_verify; Collection#render
+        # aborts on them once every member has rendered (#586).
+        @fatal_errors = []
+        @nested = options[:nested]
+        # if false, this is the root instance of Renderer
         # if true, then this is not the last time Renderer will be run
         # (e.g. this is sectionsplit)
+        # Isolated/incremental build: preserve unresolved cross-document
+        # references as stubs instead of resolving or stripping them, so each
+        # member compiles independently and a later reinflation pass relinks.
+        @preserve_unresolved = options[:preserve_unresolved]
+        # Durable content-addressed store for staged artefacts. Opt-in: a plain
+        # build sets neither :preserve_unresolved nor :artifact_store_dir, so it
+        # gets the no-op Null store and is unaffected. Staging (:preserve_unresolved)
+        # defaults the store directory to ArtifactStore::DEFAULT_DIRNAME when no
+        # :artifact_store_dir is given, so callers need not name it. (Reinflation
+        # reads stored stubs via the manifest, not this object, so it does not
+        # trigger the default -- that would only create an empty directory.)
+        @artifact_store =
+          if (dir = options[:artifact_store_dir]) || options[:preserve_unresolved]
+            ArtifactStore.new(dir || ArtifactStore::DEFAULT_DIRNAME)
+          else
+            NullArtifactStore.new
+          end
+        # Reinflation pass: the input is a stored stub-bearing Semantic XML that
+        # already has intra-doc xrefs and the document suffix applied, so run
+        # only the cross-document resolution, not the passes already done.
+        @reinflate = options[:reinflate]
+        # Cache retention: by default keep only each document's latest artefacts,
+        # pruning superseded content-hash versions on write; keep_cache retains
+        # every version (history / multiple states side by side).
+        @keep_cache = options[:keep_cache]
 
+        @coverpage = options[:coverpage] || collection.coverpage
+        @coverpage_pdf_portflio = options[:coverpage_pdf_portfolio] ||
+          collection.coverpage_pdf_portfolio || Util::taste2coverpage_pdf_portfolio(@flavor)
+        collection.directives = directives_normalise(collection.directives)
+        @directives = collection.directives
         # list of files in the collection
         @files = Metanorma::Collection::FileLookup.new(folder, self)
         @files.add_section_split
@@ -69,8 +139,57 @@ module Metanorma
         create_non_existing_directory(@outdir)
       end
 
+      def directives_normalise(directives)
+        @coverpage_pdf_portflio and
+          directives = directives_normalise_coverpage_pdf_portfolio(directives)
+        directives_normalise_keystore_pdf_portfolio(directives)
+      end
+
+      def directives_normalise_coverpage_pdf_portfolio(directives)
+        directives_resolve_filepath(directives, "coverpage-pdf-portfolio",
+                                    @coverpage_pdf_portflio)
+      end
+
+      def directives_normalise_keystore_pdf_portfolio(directives)
+        f = directives.find { |d| d.key == "keystore-pdf-portfolio" }
+        f.nil? || f.value.nil? and return directives
+        directives_resolve_filepath(directives, "keystore-pdf-portfolio",
+                                    f.value)
+      end
+
+      def directives_resolve_filepath(directives, name, val)
+        abs = Pathname.new(val).absolute?
+        directives.reject! { |d| d.key == name }
+        val = Util::rel_path_resolve(@dirname, val)
+        val =
+          if abs then directives_host_materialise(val)
+          else Pathname.new(val).relative_path_from(Pathname.new(@outdir)).to_s
+          end
+        directives << ::Metanorma::Collection::Config::Directive
+          .new(key: name, value: val)
+        directives
+      end
+
+      # An absolute directive value can point inside a gem payload (e.g. a
+      # taste's data/ file from Util.taste2coverpage_pdf_portfolio). A
+      # spawned child process that is not this Ruby (mn2pdf/java) may be
+      # unable to read such a path (a packaged virtual filesystem mount,
+      # e.g. tebako on Windows). Copy the file into the output folder —
+      # the child already reads the collection XML from there — and point
+      # the directive at the copy. A missing file keeps its value: the
+      # child reports that as it always has.
+      def directives_host_materialise(val)
+        File.exist?(val) or return val
+        dst = File.expand_path(File.join(@outdir, File.basename(val)))
+        dst == File.expand_path(val) and return val
+        FileUtils.mkdir_p(@outdir)
+        FileUtils.cp(val, dst)
+        dst
+      end
+
       def flush_files
         warn "\n\n\n\n\nDone: #{DateTime.now.strftime('%H:%M:%S')}"
+        warn "\nFiles to delete:\n"
         warn @files.files_to_delete
         @files.files_to_delete.each { |f| FileUtils.rm_f(f) }
         @files_to_delete.each { |f| FileUtils.rm_f(f) }
@@ -85,61 +204,145 @@ module Metanorma
         warn "\n\n\n\n\nRender Init: #{DateTime.now.strftime('%H:%M:%S')}"
         cr = new(col, File.dirname(col.file), options)
         cr.files
+        cr.rxl(options)
         cr.concatenate(col, options)
         options[:format]&.include?(:html) and cr.coverpage
         cr.flush_files
         cr
       end
 
+      def rxl(options)
+        @bibdata or return
+        options[:site_generate] and options[:format] << :xml
+        options[:format].include?(:rxl) or return
+        File.open(File.join(@outdir, "collection.rxl"), "w:UTF-8") do |f|
+          f.write(@bibdata.to_xml(bibdata: true))
+        end
+      end
+
       def concatenate(col, options)
         warn "\n\n\n\n\nConcatenate: #{DateTime.now.strftime('%H:%M:%S')}"
-        (options[:format] & %i(pdf doc)).empty? or
+        concatenate_presentation?(options) and
           options[:format] << :presentation
         concatenate_prep(col, options)
         concatenate_outputs(options)
       end
 
+      def concatenate_presentation?(options)
+        !(options[:format] & %i(pdf doc)).empty? ||
+          (@directives.detect { |d| d.key == "bilingual" } &&
+          options[:format].include?(:html))
+      end
+
       def concatenate_prep(col, options)
+        warn options[:format]
         %i(xml presentation).each do |e|
           options[:format].include?(e) or next
           ext = e == :presentation ? "presentation.xml" : e.to_s
           File.open(File.join(@outdir, "collection.#{ext}"), "w:UTF-8") do |f|
             b = concatenate1(col.clone, e).to_xml
-            e == :presentation and
-              b.sub!("<metanorma-collection>", "<metanorma-collection xmlns='http://metanorma.org'>")
-            # TODO BEING FORCED TO DO THAT BECAUSE SHALE IS NOT DEALING WITH DEFAULT NAMESPACES
+            e == :presentation and b = concatenate_presentation(b)
             f.write(b)
           end
         end
       end
 
+      def concatenate_presentation(xml)
+        @directives.detect { |d| d.key == "bilingual" } and
+          xml = Metanorma::Collection::Multilingual
+            .new({ align_cross_elements: %w(p note) }).to_bilingual(xml)
+        xml
+      end
+
       def concatenate_outputs(options)
-        pres = File.join(@outdir, "collection.presentation.xml")
-        options[:format].include?(:pdf) and pdfconv.convert(pres)
+        pres, compile_opts = concatenate_outputs_prep(options)
+        warn pp compile_opts
+        options[:format].include?(:pdf) and pdfconv(compile_opts).convert(pres)
+        options[:format].include?(:"pdf-portfolio") and
+          pdfconv(pdf_portfolio_mn2pdf_options)
+            .convert(pres, nil, nil,
+                     File.join(@outdir, "collection.portfolio.pdf"))
         options[:format].include?(:doc) and docconv_convert(pres)
+        bilingual_output(options, pres)
+      end
+
+      def concatenate_outputs_prep(_options)
+        pres = File.join(@outdir, "collection.presentation.xml")
+        fonts = extract_added_fonts(pres)
+        if fonts
+          # Install fonts before trying to locate them
+          font_options = @compile_options.merge({ fonts: fonts })
+          ::Metanorma::Util::FontistHelper.install_fonts(@compile.processor,
+                                                         font_options)
+
+          mn2pdf = {
+            font_manifest: ::Metanorma::Util::FontistHelper
+              .location_manifest(@compile.processor, { fonts: fonts }),
+          }
+        end
+        [pres, { fonts: fonts, mn2pdf: mn2pdf }.compact]
+      end
+
+      def extract_added_fonts(pres)
+        File.exist?(pres) or return
+        xml = Nokogiri::XML(File.read(pres, encoding: "UTF-8"), &:huge)
+        x = xml.xpath("//*[local-name() = 'presentation-metadata']/" \
+                      "*[local-name() = 'fonts']")
+        x.empty? and return
+        x.map(&:text).join(";")
+      end
+
+      def pdf_portfolio_mn2pdf_options
+        f1 = @directives.find { |d| d.key == "keystore-pdf-portfolio" }&.value
+        f2 = @directives.find do |d|
+          d.key == "keystore-password-pdf-portfolio"
+        end&.value
+        {  "pdf-portfolio": "true", pdfkeystore: f1,
+           pdfkeystorepassword: f2 }.compact
+      end
+
+      def bilingual_output(options, pres)
+        @directives.detect { |d| d.key == "bilingual" } &&
+          options[:format].include?(:html) and
+          Metanorma::Collection::Multilingual.new(
+            { flavor: Util::taste2flavor(flavor).to_sym,
+              converter_options: PdfOptionsNode.new(Util::taste2flavor(flavor),
+                                                    @compile_options),
+              outdir: @outdir },
+          ).to_html(pres)
       end
 
       def concatenate1(out, ext)
-        out.directives << ::Metanorma::Collection::Config::Directive.new(key: "documents-inline")
+        out.directives << ::Metanorma::Collection::Config::Directive
+          .new(key: "documents-inline")
         out.bibdatas.each_key do |ident|
           id = @isodoc.docid_prefix(nil, ident.dup)
-          @files.get(id, :attachment) || @files.get(id, :outputs).nil? and next
+          @files.get(id, :attachment) and next
+          # A non-attachment document with no compiled output for THIS format
+          # cannot be inlined: leaving it in place serialises as a bibdata-only,
+          # namespace-less <metanorma> that mn2pdf silently drops. Skip loudly
+          # rather than crash on a nil path or corrupt the collection PDF.
+          # (Sectionsplit parents get a :presentation output upstream, so they
+          # inline whole for the PDF/presentation path.)
+          outputs = @files.get(id, :outputs)
+          # The registered path may exist without the file: a member whose
+          # rendering failed registers its intended outputs but writes
+          # nothing (#586), so check the disk, not just the registration.
+          if outputs.nil? || outputs[ext].nil? || !File.exist?(outputs[ext])
+            warn "[metanorma] collection document '#{id}' has no compiled " \
+                 "#{ext} output to inline; skipping (its doc-container would " \
+                 "otherwise be bibdata-only)."
+            next
+          end
           out.documents[Util::key id] =
-            Metanorma::Collection::Document.raw_file(@files.get(id,
-                                                                :outputs)[ext])
+            Metanorma::Collection::Document.raw_file(outputs[ext])
         end
         out
       end
 
-      # infer the flavour from the first document identifier; relaton does that
-      def doctype
-        if (docid = @xml.at("//bibdata/docidentifier/@type")&.text)
-          dt = docid.downcase
-        elsif (docid = @xml.at("//bibdata/docidentifier")&.text)
-          dt = docid.sub(/\s.*$/, "").lowercase
-        else return "standoc"
-        end
-        @registry = Metanorma::Registry.instance
+      # TODO: infer flavor from publisher when available
+      def flavor
+        dt = @xml.at("//bibdata/ext/flavor")&.text or return "standoc"
         @registry.alias(dt.to_sym)&.to_s || dt
       end
 
@@ -147,9 +350,10 @@ module Metanorma
       # collection manifest
       def coverpage
         @coverpage or return
+        @coverpage_path = Util::rel_path_resolve(@dirname, @coverpage)
         warn "\n\n\n\n\nCoverpage: #{DateTime.now.strftime('%H:%M:%S')}"
         File.open(File.join(@outdir, "index.html"), "w:UTF-8") do |f|
-          f.write @isodoc.populate_template(File.read(@coverpage))
+          f.write @isodoc.populate_template(File.read(@coverpage_path))
         end
       end
     end

data/lib/metanorma/collection/renderer/svg.rb

@@ -0,0 +1,107 @@
+module Metanorma
+  class Collection
+    class Renderer
+      # Converts SVG images to data URIs for inline embedding
+      def svg_datauri(docxml, docid)
+        rel = @files.get(docid, :rel_path)
+        parent = @files.get(docid, :parentid) and
+          rel = @files.get(parent, :rel_path)
+        # if sectionsplit, use orig file dir
+        dir = File.join(@dirname, File.dirname(rel))
+        datauri_encode(docxml, dir)
+      end
+
+      # Resolves SVG map references and processes SVG ID disambiguation.
+      #
+      # Delegates SVG manipulation to Vectory gem, which handles:
+      # 1. Document-level suffixing for cross-document ID disambiguation
+      # 2. Index-based suffixing for multiple svgmaps within one document
+      # 3. Link remapping and SVG extraction
+      #
+      # Metanorma's role: State requirements and coordinate the workflow.
+      # Vectory's role: Handle all SVG ID manipulation internally.
+      #
+      # @param docxml [Nokogiri::XML::Document] The document XML to process
+      # @param docid [String] Document identifier for retrieving suffix
+      # @param presxml [Boolean, nil] Whether this is presentation XML
+      def svgmap_resolve(docxml, docid, presxml)
+        ids, docxml, isodoc, tag = svgmap_resolve_prep(docxml, docid, presxml)
+
+        # Stage 1: Resolve EREF references to their targets
+        resolve_svgmap_erefs(docxml, tag, isodoc, ids, presxml)
+
+        # Stage 2: Normalize prefixes (Vectory expects eref, not fmt-eref)
+        normalize_svgmap_prefixes(docxml)
+
+        # Stage 3: Process with Vectory
+        # Pass document suffix to Vectory as id_suffix for proper SVG ID disambiguation.
+        # Vectory handles both id_suffix (document-level) and index suffix internally.
+        doc_suffix = @files.get(docid, :document_suffix)
+        Vectory::SvgMapping.new(docxml, "", id_suffix: doc_suffix).call
+
+        # Stage 4: Extract processed SVG content
+        extract_svgmap_content(docxml, isodoc)
+      end
+
+      # Prepares document and context for svgmap resolution
+      def svgmap_resolve_prep(docxml, docid, presxml)
+        ids = @files.get(docid, :ids)
+        docxml = svg_unnest(svg_datauri(docxml, docid))
+        isodoc = IsoDoc::PresentationXMLConvert.new({})
+        isodoc.bibitem_lookup(docxml)
+        tag = presxml ? "fmt-eref" : "eref"
+        [ids, docxml, isodoc, tag]
+      end
+
+      private
+
+      # Resolves EREF elements within svgmap to their actual link targets
+      def resolve_svgmap_erefs(docxml, tag, isodoc, ids, presxml)
+        docxml.xpath(ns("//svgmap//#{tag}")).each do |e|
+          svgmap_resolve_eref(e, isodoc, docxml, ids, presxml)
+        end
+      end
+
+      # Converts fmt-eref/fmt-link/fmt-xref back to eref/link/xref
+      # Vectory expects non-prefixed element names for proper processing
+      def normalize_svgmap_prefixes(docxml)
+        svgmap_fmt_prefix_remove(docxml)
+      end
+
+      # Extracts processed SVG content from svgmap elements
+      def extract_svgmap_content(docxml, isodoc)
+        docxml.xpath(ns("//svgmap")).each { |s| isodoc.svgmap_extract(s) }
+      end
+
+      # Converts Presentation XML fmt- prefix back to standard element names
+      # Vectory expects eref/link/xref, not fmt-eref/fmt-link/fmt-xref
+      def svgmap_fmt_prefix_remove(docxml)
+        docxml.xpath(ns("//svgmap/target")).each do |t|
+          n = t.at(ns(".//fmt-link | .//fmt-xref | .//fmt-eref")) or next
+          n.name = n.name.sub(/^fmt-/, "")
+          t.children = n
+        end
+      end
+
+      # Removes nested image elements within svgmap, flattening the structure
+      def svg_unnest(docxml)
+        docxml.xpath(ns("//svgmap//image[.//*[name() = 'image']]")).each do |i|
+          s = i.elements.detect { |e| e.name == "svg" } and
+            i.replace(s)
+        end
+        docxml
+      end
+
+      # Resolves a single EREF element to its target link
+      def svgmap_resolve_eref(eref, isodoc, _docxml, ids, presxml)
+        href = isodoc.eref_target(eref) or return
+        href = href[:link]
+        href == "##{eref['bibitemid']}" ||
+          (href =~ /^#/ && !ids[href.sub(/^#/, "")]) and return
+        eref["target"] = href.strip
+        eref.name = presxml ? "fmt-link" : "link"
+        eref.elements&.remove
+      end
+    end
+  end
+end